diff --git a/Cargo.lock b/Cargo.lock index faecee73..36205337 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1808,6 +1808,7 @@ name = "bigname-worker" version = "0.1.0" dependencies = [ "anyhow", + "bigname-adapters", "bigname-domain", "bigname-execution", "bigname-manifests", @@ -1815,6 +1816,7 @@ dependencies = [ "bigname-test-support", "clap", "futures-util", + "serde", "serde_json", "sqlx", "tokio", diff --git a/apps/indexer/src/main/backfill/reservation_execution.rs b/apps/indexer/src/main/backfill/reservation_execution.rs index 44bb6ec7..e5aa3789 100644 --- a/apps/indexer/src/main/backfill/reservation_execution.rs +++ b/apps/indexer/src/main/backfill/reservation_execution.rs @@ -56,7 +56,8 @@ use scan_all::{ const HASH_PINNED_BACKFILL_SCAN_MODE: &str = "hash_pinned_block"; pub(crate) const COINBASE_SQL_BACKFILL_SCAN_MODE: &str = "coinbase_sql_hash_pinned_logs_v1"; -pub(crate) const DEFAULT_HASH_PINNED_BACKFILL_CHUNK_BLOCKS: i64 = 1_024; +pub(crate) const DEFAULT_HASH_PINNED_BACKFILL_CHUNK_BLOCKS: i64 = + bigname_storage::MAX_LIVE_CONTIGUOUS_GAP_FILL_BLOCKS; pub(crate) const COMPACT_SOURCE_IDENTITY_SELECTED_TARGET_THRESHOLD: usize = 10_000; pub(crate) async fn create_hash_pinned_backfill_job( diff --git a/apps/indexer/src/main/normalized_replay_catchup.rs b/apps/indexer/src/main/normalized_replay_catchup.rs index f2da6baf..82652577 100644 --- a/apps/indexer/src/main/normalized_replay_catchup.rs +++ b/apps/indexer/src/main/normalized_replay_catchup.rs @@ -25,7 +25,7 @@ mod indexes; mod sources; use cursors::{ - advance_cursor, ensure_cursor, record_cursor_failure, + advance_cursor, clear_cursor_failure, ensure_cursor, record_cursor_failure, rewind_cursor_for_newly_observed_older_logs, }; use indexes::{ @@ -276,6 +276,7 @@ pub(crate) async fn run_normalized_replay_catchup_iteration( ) .await?; } + clear_cursor_failure(pool, &config.deployment_profile, chain).await?; return Ok(CatchupIterationStatus::Idle); }; if pending_base_rederive_replay_target.is_some() { @@ -334,6 +335,7 @@ pub(crate) async fn run_normalized_replay_catchup_iteration( ) .await?; } + clear_cursor_failure(pool, &config.deployment_profile, chain).await?; return Ok(CatchupIterationStatus::Idle); } diff --git a/apps/indexer/src/main/normalized_replay_catchup/cursors.rs b/apps/indexer/src/main/normalized_replay_catchup/cursors.rs index 5a05d06d..5d1e1abb 100644 --- a/apps/indexer/src/main/normalized_replay_catchup/cursors.rs +++ b/apps/indexer/src/main/normalized_replay_catchup/cursors.rs @@ -273,6 +273,39 @@ pub(super) async fn record_cursor_failure( Ok(()) } +/// A successful idle iteration is a successful health signal even though there is no cursor +/// advancement to clear a previous transient failure. The predicate avoids rewriting a healthy +/// cursor on every poll. +pub(super) async fn clear_cursor_failure( + pool: &PgPool, + deployment_profile: &str, + chain: &str, +) -> Result<()> { + sqlx::query( + r#" + UPDATE normalized_replay_cursors + SET + last_failure_reason = NULL, + last_failure_at = NULL, + updated_at = now() + WHERE deployment_profile = $1 + AND chain_id = $2 + AND cursor_kind = $3 + AND (last_failure_reason IS NOT NULL OR last_failure_at IS NOT NULL) + "#, + ) + .bind(deployment_profile) + .bind(chain) + .bind(CURSOR_KIND_RAW_FACT_NORMALIZED_EVENTS) + .execute(pool) + .await + .with_context(|| { + format!("failed to clear normalized replay cursor failure for {deployment_profile}/{chain}") + })?; + + Ok(()) +} + fn postgres_text_safe(text: &str) -> String { text.replace('\0', "\\u0000") } diff --git a/apps/indexer/src/main/reconciliation/canonical.rs b/apps/indexer/src/main/reconciliation/canonical.rs index 1b23aa7b..e26fa4f1 100644 --- a/apps/indexer/src/main/reconciliation/canonical.rs +++ b/apps/indexer/src/main/reconciliation/canonical.rs @@ -1,7 +1,8 @@ use anyhow::{Result, bail}; use bigname_storage::{ - CanonicalityState, ChainCheckpoint, ChainCheckpointUpdate, advance_chain_checkpoints, - chain_lineage_contains_ancestor, load_chain_lineage_block, mark_chain_lineage_range_orphaned, + CanonicalityState, ChainCheckpoint, ChainCheckpointUpdate, MAX_LIVE_CONTIGUOUS_GAP_FILL_BLOCKS, + advance_chain_checkpoints, chain_lineage_contains_ancestor, load_chain_lineage_block, + mark_chain_lineage_range_orphaned, upsert_chain_lineage_blocks_recanonicalizing_orphaned as upsert_recanonicalized_lineage_blocks, upsert_chain_lineage_blocks_without_snapshots, upsert_chain_lineage_blocks_without_snapshots_recanonicalizing_orphaned as upsert_recanonicalized_lineage_blocks_without_snapshots, @@ -9,7 +10,6 @@ use bigname_storage::{ use tracing::{info, warn}; use crate::{ - backfill::DEFAULT_HASH_PINNED_BACKFILL_CHUNK_BLOCKS, provider::{ChainProviderOps, ProviderBlock, ProviderHeadSnapshot, ProviderRegistry}, runtime::{IntakeChainTask, checkpoint_mode}, }; @@ -48,8 +48,6 @@ use stored_lineage::{ const MAX_PARENT_FETCH_DEPTH: usize = 131_072; // Live polling fails closed before it tries to ingest a large catch-up range. // Hash-pinned backfill owns larger bounded gaps. -const MAX_LIVE_CONTIGUOUS_GAP_FILL_BLOCKS: i64 = DEFAULT_HASH_PINNED_BACKFILL_CHUNK_BLOCKS; - #[allow(dead_code)] pub(crate) async fn poll_provider_heads( pool: &sqlx::PgPool, diff --git a/apps/indexer/src/main/reconciliation/canonical/stored_lineage/coverage.rs b/apps/indexer/src/main/reconciliation/canonical/stored_lineage/coverage.rs index b7fa922a..0b551898 100644 --- a/apps/indexer/src/main/reconciliation/canonical/stored_lineage/coverage.rs +++ b/apps/indexer/src/main/reconciliation/canonical/stored_lineage/coverage.rs @@ -3,22 +3,14 @@ use std::sync::Mutex; use anyhow::Result; use bigname_manifests::{ - UncoveredWatchedTuple, find_uncovered_watched_tuples, - load_active_manifest_abi_events_by_chain_and_source_families, load_discovery_admission_epoch, - load_log_producing_source_families, load_watched_contracts_by_addresses, + UncoveredWatchedTuple, WATCHED_COVERAGE_VERIFICATION_CHUNK_BLOCKS, + find_uncovered_watched_tuples, load_active_manifest_topic0s_by_chain_and_source_families, + load_discovery_admission_epoch, load_log_producing_source_families, + load_watched_contracts_by_addresses, }; -use bigname_storage::ChainLineageBlock; +use bigname_storage::{ChainLineageBlock, ensure_backfill_family_topic_sets_undrifted}; use sqlx::Row; -#[path = "coverage/topic_drift.rs"] -mod topic_drift; - -use topic_drift::ensure_family_topic_sets_undrifted; - -/// Frontier extensions verify coverage in chunks of this many blocks, so a -/// deep gap costs a handful of anti-join queries once and every promotion -/// cycle afterwards is an O(1) in-memory comparison. -pub(crate) const COVERAGE_FRONTIER_VERIFICATION_CHUNK_BLOCKS: i64 = 131_072; const MAX_REPORTED_UNCOVERED_TUPLES: i64 = 20; /// Process-lifetime, per-chain memo of the block interval whose watched-tuple @@ -143,7 +135,7 @@ pub(super) async fn stored_path_has_required_raw_fact_coverage( /// Extend the chain's verified coverage frontier until it contains /// `[required_from, required_through]`, verifying in -/// [`COVERAGE_FRONTIER_VERIFICATION_CHUNK_BLOCKS`] chunks that opportunistically +/// [`WATCHED_COVERAGE_VERIFICATION_CHUNK_BLOCKS`] chunks that opportunistically /// look ahead as far as `verify_ahead_through_block` (the stored promotion /// anchor) so subsequent cycles are O(1). A violation in the look-ahead beyond /// the required target falls back to verifying exactly up to the target, so an @@ -159,8 +151,13 @@ async fn ensure_verified_coverage_frontier( let log_producing_source_families = load_log_producing_source_families(pool, chain) .await .map_err(|error| error.to_string())?; - let current_topic0s_by_family = - load_current_topic0s_by_family(pool, chain, &log_producing_source_families).await?; + let current_topic0s_by_family = load_active_manifest_topic0s_by_chain_and_source_families( + pool, + chain, + &log_producing_source_families, + ) + .await + .map_err(|error| error.to_string())?; let topic_set_fingerprint = topic_set_fingerprint(¤t_topic0s_by_family); let discovery_admission_epoch = load_discovery_admission_epoch(pool, chain) .await @@ -189,7 +186,7 @@ async fn ensure_verified_coverage_frontier( let extension_from = interval .as_ref() .map_or(required_from, |existing| existing.through_block + 1); - if let Err(_look_ahead_drift) = ensure_family_topic_sets_undrifted( + if let Err(_look_ahead_drift) = ensure_backfill_family_topic_sets_undrifted( pool, chain, ¤t_topic0s_by_family, @@ -202,7 +199,7 @@ async fn ensure_verified_coverage_frontier( // must not block promoting the covered prefix: recheck scoped to the // target, and if that passes, cap the look-ahead so the memo never // covers a span the drift guard did not clear. - ensure_family_topic_sets_undrifted( + ensure_backfill_family_topic_sets_undrifted( pool, chain, ¤t_topic0s_by_family, @@ -222,7 +219,7 @@ async fn ensure_verified_coverage_frontier( while through_block < required_through { let chunk_from = through_block + 1; let chunk_through = chunk_from - .saturating_add(COVERAGE_FRONTIER_VERIFICATION_CHUNK_BLOCKS - 1) + .saturating_add(WATCHED_COVERAGE_VERIFICATION_CHUNK_BLOCKS - 1) .min(verify_ahead_through_block); let violations = find_uncovered_watched_tuples( pool, @@ -316,36 +313,6 @@ fn uncovered_tuples_refusal( ) } -/// Current manifest topic0 sets per log-producing family; also the input to -/// the frontier memo's ABI fingerprint. -async fn load_current_topic0s_by_family( - pool: &sqlx::PgPool, - chain: &str, - log_producing_source_families: &[String], -) -> std::result::Result>, String> { - if log_producing_source_families.is_empty() { - return Ok(BTreeMap::new()); - } - let events = load_active_manifest_abi_events_by_chain_and_source_families( - pool, - chain, - log_producing_source_families, - ) - .await - .map_err(|error| error.to_string())?; - let mut current_topic0s_by_family = BTreeMap::>::new(); - for event in events { - let Some(topic0) = event.topic0 else { - continue; - }; - current_topic0s_by_family - .entry(event.source_family) - .or_default() - .insert(topic0.to_ascii_lowercase()); - } - Ok(current_topic0s_by_family) -} - fn topic_set_fingerprint(current_topic0s_by_family: &BTreeMap>) -> String { let mut fingerprint = String::new(); for (source_family, topic0s) in current_topic0s_by_family { diff --git a/apps/indexer/src/main/reconciliation/replay/classification.rs b/apps/indexer/src/main/reconciliation/replay/classification.rs index 35377cf3..5033f651 100644 --- a/apps/indexer/src/main/reconciliation/replay/classification.rs +++ b/apps/indexer/src/main/reconciliation/replay/classification.rs @@ -470,14 +470,9 @@ fn closure_source_families_for_contracts( } fn closure_or_dependency_source_families() -> Vec { - NORMALIZED_EVENT_REPLAY_CONTRACTS + bigname_adapters::CLOSURE_OR_DEPENDENCY_REPLAY_SOURCE_FAMILIES .iter() - .filter(|contract| contract.raw_fact_replay_participant) - .filter(|contract| !contract.model.restricted_replay_supported()) - .flat_map(|contract| contract.source_families.iter().copied()) - .collect::>() - .into_iter() - .map(str::to_owned) + .map(|source_family| (*source_family).to_owned()) .collect() } diff --git a/apps/indexer/src/main/reconciliation/replay/classification/tests.rs b/apps/indexer/src/main/reconciliation/replay/classification/tests.rs index c1250817..28c843a1 100644 --- a/apps/indexer/src/main/reconciliation/replay/classification/tests.rs +++ b/apps/indexer/src/main/reconciliation/replay/classification/tests.rs @@ -47,6 +47,21 @@ fn closure_or_dependency_contracts_do_not_claim_restricted_replay_proofs() { } } +#[test] +fn shared_latched_target_families_match_replay_contracts() { + let contract_families = NORMALIZED_EVENT_REPLAY_CONTRACTS + .iter() + .filter(|contract| contract.raw_fact_replay_participant) + .filter(|contract| contract.model != ReplayDependencyModel::StatelessRawFact) + .flat_map(|contract| contract.source_families.iter().copied()) + .collect::>(); + let shared_families = bigname_adapters::CLOSURE_OR_DEPENDENCY_REPLAY_SOURCE_FAMILIES + .iter() + .copied() + .collect::>(); + assert_eq!(shared_families, contract_families); +} + #[test] fn implemented_full_closure_contracts_are_enumerated() { let actual = NORMALIZED_EVENT_REPLAY_CONTRACTS diff --git a/apps/indexer/src/main/reconciliation/replay/profile_scope.rs b/apps/indexer/src/main/reconciliation/replay/profile_scope.rs index cbba68ce..aab3dc29 100644 --- a/apps/indexer/src/main/reconciliation/replay/profile_scope.rs +++ b/apps/indexer/src/main/reconciliation/replay/profile_scope.rs @@ -3,6 +3,7 @@ use bigname_manifests::{ WatchedSourceSelector, load_manifest_declared_watched_source_selector_plan, load_watched_chain_plan, load_watched_contracts_by_addresses, }; +use bigname_storage::load_active_manifest_deployment_profile; use super::scoped::replay_source_scope_from_requested_scope; use crate::{ @@ -19,7 +20,11 @@ pub(super) async fn ensure_replay_matches_deployment_profile_scope( request: &RawFactNormalizedEventReplayRequest, range: Option<(i64, i64)>, ) -> Result<()> { - let active_profile = infer_active_manifest_deployment_profile(pool).await?; + let Some(active_profile) = load_active_manifest_deployment_profile(pool).await? else { + bail!( + "deployment_profile cannot be enforced because the active manifest/discovery corpus does not match a supported deployment profile" + ); + }; if request.deployment_profile != active_profile { bail!( "deployment_profile {} does not match active manifest/discovery corpus profile {active_profile}", @@ -276,39 +281,3 @@ async fn ensure_active_watched_chain_for_replay_profile( Ok(()) } - -async fn infer_active_manifest_deployment_profile(pool: &sqlx::PgPool) -> Result { - let rows = sqlx::query_as::<_, (String, String)>( - r#" - SELECT DISTINCT chain, deployment_epoch - FROM manifest_versions - WHERE rollout_status = 'active' - ORDER BY chain, deployment_epoch - "#, - ) - .fetch_all(pool) - .await - .context( - "failed to load active manifest/discovery corpus for replay deployment_profile enforcement", - )?; - - if rows.is_empty() { - bail!("deployment_profile cannot be enforced because no active manifests are loaded"); - } - - let all_mainnet = rows.iter().all(|(chain, _)| chain.ends_with("-mainnet")); - if all_mainnet { - return Ok("mainnet".to_owned()); - } - - let all_sepolia_dev = rows.iter().all(|(chain, deployment_epoch)| { - chain.ends_with("-sepolia") && deployment_epoch.ends_with("_sepolia_dev") - }); - if all_sepolia_dev { - return Ok("sepolia-dev".to_owned()); - } - - bail!( - "deployment_profile cannot be enforced because the active manifest/discovery corpus does not match a supported deployment profile" - ); -} diff --git a/apps/indexer/src/main/tests/normalized_replay_catchup.rs b/apps/indexer/src/main/tests/normalized_replay_catchup.rs index d0b5e433..a890fc6f 100644 --- a/apps/indexer/src/main/tests/normalized_replay_catchup.rs +++ b/apps/indexer/src/main/tests/normalized_replay_catchup.rs @@ -832,6 +832,66 @@ async fn normalized_replay_catchup_rebuilds_deferred_indexes_when_configured_cha Ok(()) } +#[tokio::test] +async fn successful_idle_iteration_clears_stale_cursor_failure() -> Result<()> { + let database = TestDatabase::new().await?; + create_normalized_replay_cursor_table(database.pool()).await?; + sqlx::query( + r#" + INSERT INTO normalized_replay_cursors ( + deployment_profile, + chain_id, + cursor_kind, + range_start_block_number, + next_block_number, + target_block_number, + last_completed_block_number, + last_failure_reason, + last_failure_at + ) + VALUES ( + 'mainnet', 'ethereum-mainnet', 'raw_fact_normalized_events', 1, 2, 1, 1, + 'transient provider failure', now() + ) + "#, + ) + .execute(database.pool()) + .await?; + + let config = normalized_replay_catchup::NormalizedReplayCatchupConfig::new( + "mainnet".to_owned(), + vec!["ethereum-mainnet".to_owned()], + 1_000, + 1_000, + 1, + )? + .with_defer_projection_indexes(false); + assert_eq!( + normalized_replay_catchup::run_normalized_replay_catchup_iteration( + database.pool(), + &config, + "ethereum-mainnet", + ) + .await?, + normalized_replay_catchup::CatchupIterationStatus::Idle + ); + + let failure = sqlx::query_as::<_, (Option, Option)>( + r#" + SELECT last_failure_reason, last_failure_at + FROM normalized_replay_cursors + WHERE deployment_profile = 'mainnet' + AND chain_id = 'ethereum-mainnet' + AND cursor_kind = 'raw_fact_normalized_events' + "#, + ) + .fetch_one(database.pool()) + .await?; + assert_eq!(failure, (None, None)); + + database.cleanup().await +} + async fn create_normalized_replay_cursor_table(pool: &PgPool) -> Result<()> { sqlx::query( r#" diff --git a/apps/indexer/src/main/tests/replay.rs b/apps/indexer/src/main/tests/replay.rs index 916fa1e0..97267d29 100644 --- a/apps/indexer/src/main/tests/replay.rs +++ b/apps/indexer/src/main/tests/replay.rs @@ -1052,6 +1052,73 @@ async fn replay_normalized_events_rejects_deployment_profile_outside_active_mani database.cleanup().await } +#[tokio::test] +async fn replay_normalized_events_accepts_sepolia_manifest_profile() -> Result<()> { + let database = TestDatabase::new().await?; + let chain = "ethereum-sepolia"; + let reverse_address = "0x00000000000000000000000000000000000000d5"; + let claimed_address = "0x5555555555555555555555555555555555555555"; + let block = provider_block( + "0xd5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5", + Some("0x1111111111111111111111111111111111111111111111111111111111111111"), + 105, + ); + + insert_active_replay_watched_contract_with_source_family( + database.pool(), + 40, + chain, + "ens_v1_reverse_l1", + Uuid::from_u128(0x940), + reverse_address, + "reverse_registrar", + ) + .await?; + sqlx::query( + "UPDATE manifest_versions SET deployment_epoch = 'ens_v1_sepolia_dev' WHERE manifest_id = 40", + ) + .execute(database.pool()) + .await?; + insert_active_replay_manifest( + database.pool(), + 41, + "ens", + "ens_v1_resolver_l1", + chain, + "ens_v1_sepolia_dev", + ) + .await?; + insert_chain_lineage_for_block(database.pool(), chain, &block, CanonicalityState::Canonical) + .await?; + insert_raw_reverse_claimed_log( + database.pool(), + chain, + &block, + reverse_address, + claimed_address, + CanonicalityState::Canonical, + ) + .await?; + + let outcome = replay_raw_fact_normalized_events( + database.pool(), + RawFactNormalizedEventReplayRequest { + deployment_profile: "sepolia".to_owned(), + chain: chain.to_owned(), + selection: RawFactNormalizedEventReplaySelection::BlockRange { + from_block: block.block_number, + to_block: block.block_number, + }, + }, + ) + .await?; + + assert_eq!(outcome.selected_block_count, 1); + assert_eq!(outcome.normalized_event_inserted_count, 1); + + database.cleanup().await +} + #[tokio::test] async fn replay_normalized_events_skips_noncanonical_raw_logs_in_selected_block_hashes() -> Result<()> { diff --git a/apps/worker/Cargo.toml b/apps/worker/Cargo.toml index 5cf5cd9c..8df76bf2 100644 --- a/apps/worker/Cargo.toml +++ b/apps/worker/Cargo.toml @@ -9,12 +9,14 @@ license.workspace = true anyhow.workspace = true clap.workspace = true futures-util.workspace = true +serde.workspace = true sqlx.workspace = true tokio.workspace = true tracing.workspace = true tracing-subscriber.workspace = true uuid.workspace = true +bigname-adapters.workspace = true bigname-domain.workspace = true bigname-execution.workspace = true bigname-manifests.workspace = true diff --git a/apps/worker/src/inspect.rs b/apps/worker/src/inspect.rs index 4c295b9c..06fe7954 100644 --- a/apps/worker/src/inspect.rs +++ b/apps/worker/src/inspect.rs @@ -1,5 +1,6 @@ mod backfill; mod canonicality; +mod data_completeness; mod execution_trace; mod formatting; mod manifest_drift; @@ -11,9 +12,9 @@ mod tests; use anyhow::{Context, Result}; use bigname_storage::DatabaseConfig; -use clap::{Args, Subcommand}; +use clap::{Args, Subcommand, ValueEnum}; use sqlx::postgres::{PgConnectOptions, PgPoolOptions}; -use std::str::FromStr; +use std::{path::PathBuf, str::FromStr}; use uuid::Uuid; pub(crate) use manifest_drift::render_manifest_drift_alert_observations; @@ -32,6 +33,10 @@ pub(crate) enum InspectCommand { about = "Inspect canonicality, durable raw fact counts, and retained payload-cache metadata for one block hash" )] Canonicality(InspectCanonicalityArgs), + #[command( + about = "Check whether this database is data-complete enough to serve: reconciliation frontier, watch-set code-observation coverage, replay and projection cursors, and projection content" + )] + DataCompleteness(InspectDataCompletenessArgs), #[command(about = "Inspect one persisted execution trace and its ordered steps")] ExecutionTrace(InspectExecutionTraceArgs), #[command(about = "Inspect stored manifest drift and proxy implementation alert observations")] @@ -60,6 +65,39 @@ pub(crate) struct InspectCanonicalityArgs { pub(crate) block_hash: String, } +#[derive(Args, Debug)] +pub(crate) struct InspectDataCompletenessArgs { + #[command(flatten)] + pub(crate) database: DatabaseConfig, + #[arg(long)] + pub(crate) json: bool, + #[arg(long)] + pub(crate) fail_on_incomplete: bool, + #[arg(long)] + pub(crate) max_head_lag_blocks: Option, + /// Optional on-disk manifest profile root used as the external active-corpus authority. + #[arg(long)] + pub(crate) manifests_root: Option, + /// Raw-log retention contract to verify. + #[arg(long, value_enum, default_value_t = RetentionMode::Minimal)] + pub(crate) retention_mode: RetentionMode, +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq, ValueEnum)] +pub(crate) enum RetentionMode { + Minimal, + LogAudit, +} + +impl RetentionMode { + pub(crate) const fn as_str(self) -> &'static str { + match self { + Self::Minimal => "minimal", + Self::LogAudit => "log-audit", + } + } +} + #[derive(Args, Debug)] pub(crate) struct InspectExecutionTraceArgs { #[command(flatten)] @@ -102,6 +140,9 @@ pub(crate) async fn inspect_command(args: InspectArgs) -> Result<()> { match args.command { InspectCommand::BackfillJob(args) => backfill::inspect_backfill_job(args).await, InspectCommand::Canonicality(args) => canonicality::inspect_canonicality(args).await, + InspectCommand::DataCompleteness(args) => { + data_completeness::inspect_data_completeness(args).await + } InspectCommand::ExecutionTrace(args) => { execution_trace::inspect_execution_trace(args).await } diff --git a/apps/worker/src/inspect/data_completeness.rs b/apps/worker/src/inspect/data_completeness.rs new file mode 100644 index 00000000..302e601d --- /dev/null +++ b/apps/worker/src/inspect/data_completeness.rs @@ -0,0 +1,282 @@ +mod backfill_coverage; +mod evaluate; +mod manifest_corpus; +mod projection_content; + +#[cfg(test)] +mod tests; + +use anyhow::{Result, bail}; +use bigname_manifests::load_watched_contracts; +use serde_json::{Value, json}; + +use super::{InspectDataCompletenessArgs, connect_read_only}; +use backfill_coverage::load_backfill_coverage; +use evaluate::{ + CheckStatus, DEFAULT_MAX_HEAD_LAG_BLOCKS, DataCompletenessReport, evaluate_data_completeness, +}; +use manifest_corpus::inspect_manifest_corpus; +use projection_content::load_projection_content; + +pub(in crate::inspect) async fn inspect_data_completeness( + args: InspectDataCompletenessArgs, +) -> Result<()> { + let pool = connect_read_only(&args.database).await?; + let adapter_event_kinds = bigname_adapters::adapter_normalized_event_kind_declarations(); + let read = bigname_storage::load_data_completeness_with_adapter_event_kinds( + &pool, + &adapter_event_kinds, + ) + .await?; + let watched_contracts = load_watched_contracts(&pool).await?; + let backfill_coverage = load_backfill_coverage(&pool, &read).await?; + let manifest_corpus = inspect_manifest_corpus(&pool, args.manifests_root.as_deref()).await?; + let projection_content = + load_projection_content(&pool, &read.active_manifest_event_sources).await?; + let max_head_lag_blocks = args + .max_head_lag_blocks + .unwrap_or(DEFAULT_MAX_HEAD_LAG_BLOCKS); + let report = evaluate_data_completeness( + &read, + &watched_contracts, + &backfill_coverage, + &manifest_corpus, + &projection_content, + max_head_lag_blocks, + args.retention_mode, + ); + + println!("{}", render_data_completeness(&report)); + + if args.fail_on_incomplete && !report.data_complete() { + bail!("database is not data-complete"); + } + Ok(()) +} + +fn render_data_completeness(report: &DataCompletenessReport) -> Value { + json!({ + "command": "inspect data-completeness", + "read_only": true, + "data_complete": report.data_complete(), + "max_head_lag_blocks": report.max_head_lag_blocks, + "max_lineage_ahead_blocks": report.max_lineage_ahead_blocks, + "retention_mode": report.retention_mode.as_str(), + "checks": [ + check("manifest_corpus_matches_repository", report.manifest_corpus_matches_repository(), json!({ + "repository_supplied": report.manifest_corpus.repository_supplied, + "verified": report.manifest_corpus.verified, + "repository_root": report.manifest_corpus.repository_root, + "repository_status": report.manifest_corpus.repository_status, + "repository_error": report.manifest_corpus.repository_error, + "expected_active_manifest_count": report.manifest_corpus.expected_active_manifest_count, + "database_active_manifest_count": report.manifest_corpus.database_active_manifest_count, + "missing_active_manifests": report.manifest_corpus.missing_active_manifests, + "unexpected_active_manifests": report.manifest_corpus.unexpected_active_manifests, + "mismatched_manifest_payloads": report.manifest_corpus.mismatched_manifest_payloads, + })), + check("reconciliation_frontier_at_head", report.frontier_at_head(), json!({ + "chains": report.frontiers.iter().map(|frontier| json!({ + "chain": frontier.chain_id.as_str(), + "canonical_block_number": frontier.canonical_block_number, + "checkpoint_canonical_lineage_match": frontier.checkpoint_canonical_lineage_match, + "lineage_head_block_number": frontier.lineage_head_block_number, + "head_lag_blocks": frontier.head_lag_blocks, + "missing_from_storage": frontier.missing_from_storage, + })).collect::>(), + })), + check("reconciliation_lineage_contiguous", report.lineage_contiguous(), json!({ + "chains": report.frontiers.iter().map(|frontier| json!({ + "chain": frontier.chain_id.as_str(), + "contiguous": frontier.contiguous, + "missing_block_count": frontier.missing_block_count, + "duplicate_canonical_height_count": frontier.duplicate_canonical_height_count, + "disconnected_canonical_parent_count": frontier.disconnected_canonical_parent_count, + })).collect::>(), + })), + check("reconciliation_history_from_declared_start", report.history_from_declared_start(), json!({ + "truncated_chains": report.chains_history_truncated.iter().map(|gap| json!({ + "chain": gap.chain.as_str(), + "declared_start_block": gap.declared_start_block, + "lineage_floor_block": gap.lineage_floor_block, + })).collect::>(), + "chains_without_finite_start": report.chains_without_finite_start.iter().map(|chain| json!({ + "chain": chain.chain.as_str(), + "open_ended_target_count": chain.open_ended_target_count, + })).collect::>(), + })), + check("stored_lineage_backfill_coverage", report.stored_lineage_backfill_coverage(), json!({ + "uncovered_tuple_count": report.backfill_coverage_gaps.len(), + "uncovered_tuples": report.backfill_coverage_gaps.iter().map(|gap| json!({ + "chain": gap.chain.as_str(), + "source_family": gap.source_family.as_str(), + "address": gap.address.as_str(), + "required_from_block": gap.required_from_block, + "required_to_block": gap.required_to_block, + })).collect::>(), + "topic_drift_count": report.backfill_coverage_topic_drifts.len(), + "topic_drifts": report.backfill_coverage_topic_drifts.iter().map(|drift| json!({ + "chain": drift.chain.as_str(), + "required_from_block": drift.required_from_block, + "required_to_block": drift.required_to_block, + "reason": drift.reason.as_str(), + })).collect::>(), + })), + check("watch_set_code_observation_coverage", report.watch_set_observed(), json!({ + "active_watched_target_count": report.active_watched_target_count, + "unobserved_target_count": report.unobserved_targets.len(), + "unobserved_targets": report.unobserved_targets.iter().take(20).map(|target| json!({ + "chain": target.chain.as_str(), + "address": target.address.as_str(), + "source_family": target.source_family.as_str(), + "active_from_block_number": target.active_from_block_number, + "max_observed_block_number": target.max_observed_block_number, + })).collect::>(), + })), + check("manifest_declared_targets_present", report.manifest_declared_targets_present(), json!({ + "missing_address_target_count": report.manifest_targets_missing_address.len(), + "missing_address_targets": report.manifest_targets_missing_address.iter().take(20).map(|target| json!({ + "chain": target.chain.as_str(), + "address": target.address.as_str(), + "source_family": target.source_family.as_str(), + "active_from_block_number": target.active_from_block_number, + "max_observed_block_number": target.max_observed_block_number, + })).collect::>(), + "missing_proxy_implementation_edge_count": report.manifest_proxy_implementations_missing_edge.len(), + "missing_proxy_implementation_edges": report.manifest_proxy_implementations_missing_edge.iter().take(20).map(|target| json!({ + "chain": target.chain.as_str(), + "address": target.address.as_str(), + "source_family": target.source_family.as_str(), + "active_from_block_number": target.active_from_block_number, + "max_observed_block_number": target.max_observed_block_number, + })).collect::>(), + })), + check("discovery_targets_present", report.discovery_targets_present(), json!({ + "missing_address_target_count": report.discovery_targets_missing_address.len(), + "missing_address_targets": report.discovery_targets_missing_address.iter().take(20).map(|target| json!({ + "chain": target.chain.as_str(), + "source_family": target.source_family.as_str(), + "contract_instance_id": target.contract_instance_id, + })).collect::>(), + "missing_target_manifest_count": report.discovery_targets_missing_manifest.len(), + "missing_target_manifests": report.discovery_targets_missing_manifest.iter().take(20).map(|target| json!({ + "chain": target.chain.as_str(), + "source_family": target.source_family.as_str(), + "contract_instance_id": target.contract_instance_id, + })).collect::>(), + })), + check("active_event_lineage_retained", report.active_event_lineage_retained(), json!({ + "manifest_sources_with_missing_lineage": report.active_manifest_sources_with_missing_lineage.iter().map(|entry| json!({ + "manifest_id": entry.manifest_id, + "manifest_version": entry.manifest_version, + "chain": entry.chain.as_str(), + "namespace": entry.namespace.as_str(), + "source_family": entry.source_family.as_str(), + "missing_canonical_lineage_count": entry.missing_canonical_lineage_count, + })).collect::>(), + })), + check("active_raw_logs_retained", report.active_raw_logs_retained(), json!({ + "retention_mode": report.retention_mode.as_str(), + "manifest_sources_with_missing_raw_logs": report.active_manifest_sources_with_missing_raw_logs.iter().map(|entry| json!({ + "manifest_id": entry.manifest_id, + "manifest_version": entry.manifest_version, + "chain": entry.chain.as_str(), + "namespace": entry.namespace.as_str(), + "source_family": entry.source_family.as_str(), + "missing_canonical_raw_log_count": entry.missing_canonical_raw_log_count, + })).collect::>(), + })), + check("normalization_no_failure", report.normalization_healthy(), json!({ + "active_deployment_profile": report.active_deployment_profile.as_deref(), + "failed_cursors": report.failed_replay_cursors.clone(), + })), + check("normalization_caught_up_to_raw_head", report.normalization_caught_up(), json!({ + "active_deployment_profile": report.active_deployment_profile.as_deref(), + "lagging_cursors": report.lagging_replay_cursors.iter().map(cursor_lag).collect::>(), + "chains_missing_raw_fact_cursor": report.chains_missing_raw_fact_cursor.clone(), + })), + check("projection_apply_drained", report.projection_drained(), json!({ + "lagging_cursors": report.lagging_projection_cursors.iter().map(cursor_lag).collect::>(), + "required_cursor": crate::projection_apply::NORMALIZED_EVENT_CURSOR, + "apply_cursor_missing_for_non_empty_change_log": report.projection_apply_cursor_missing, + "apply_cursor_ahead_of_retained_change_log_by": report.projection_apply_cursor_ahead_by, + })), + check("projection_invalidations_drained", report.projection_invalidations_drained(), json!({ + "pending_invalidation_count": report.pending_projection_invalidation_count, + })), + check("projection_no_dead_letters", report.projection_no_dead_letters(), json!({ + "dead_letter_count": report.projection_invalidation_dead_letter_count, + })), + check("projection_replay_complete", report.projection_replay_complete(), json!({ + "replay_version": report.projection_replay_version, + "required_replay_version": report.projection_replay_required_version, + "target_coverage_required": report.projection_replay_target_coverage_required, + "required_target_block": report.projection_replay_required_target_block, + "missing_projections": report.missing_projection_replay_markers.clone(), + })), + check("current_projection_content_present", report.projection_content_present(), json!({ + "tables": report.projection_content.tables.iter().map(|table| json!({ + "projection": table.projection.as_str(), + "scope_kind": table.scope_kind, + "raw_total_count": table.raw_total_count, + "raw_scoped_counts": table.raw_scoped_counts.iter().map(|entry| json!({ + "scope": entry.scope.as_str(), + "count": entry.count, + })).collect::>(), + "servable_total_count": table.servable_total_count, + "servable_scoped_counts": table.servable_scoped_counts.iter().map(|entry| json!({ + "scope": entry.scope.as_str(), + "count": entry.count, + })).collect::>(), + "expected_scopes": table.expected_scopes.clone(), + "missing_scopes": table.missing_scopes.clone(), + })).collect::>(), + })), + check("active_dataset_non_empty", report.active_dataset_non_empty(), json!({ + "normalized_event_total": report.normalized_event_total, + "name_current_total": report.name_current_total, + "manifest_sources_without_events": report.active_manifest_sources_without_events.iter().map(|entry| json!({ + "manifest_id": entry.manifest_id, + "manifest_version": entry.manifest_version, + "chain": entry.chain.as_str(), + "namespace": entry.namespace.as_str(), + "source_family": entry.source_family.as_str(), + })).collect::>(), + "namespaces_without_names": report.active_namespaces_without_names.clone(), + })), + check("normalized_events_chain_id_present", report.normalized_events_chain_id_present(), json!({ + "null_chain_id_count": report.normalized_events_null_chain_id_count, + })), + check("deferred_projection_indexes_present", report.deferred_projection_indexes_present(), json!({ + "missing_indexes": report.missing_deferred_projection_indexes.clone(), + })), + ], + "advisories": { + "manifest_corpus_unverified": !report.manifest_corpus.repository_supplied, + "foreign_chains": report.foreign_chains.clone(), + "ignored_replay_cursors": report.ignored_replay_cursors.clone(), + "pending_activation": report.pending_activation_targets.iter().map(|target| json!({ + "chain": target.chain.as_str(), + "address": target.address.as_str(), + "source_family": target.source_family.as_str(), + "active_from_block_number": target.active_from_block_number, + "canonical_head_block_number": target.canonical_head_block_number, + })).collect::>(), + "backfill_lifecycle": report.backfill_advisory.iter().map(|row| json!({ + "deployment_profile": row.deployment_profile.as_str(), + "failed_job_count": row.failed_job_count, + "failed_range_count": row.failed_range_count, + "incomplete_range_count": row.incomplete_range_count, + "expired_lease_range_count": row.expired_lease_range_count, + })).collect::>(), + }, + }) +} + +fn check(name: &'static str, status: CheckStatus, detail: Value) -> Value { + json!({ "name": name, "status": status.label(), "detail": detail }) +} + +fn cursor_lag(lag: &evaluate::CursorLag) -> Value { + json!({ "cursor": lag.label.as_str(), "behind_by": lag.behind_by }) +} diff --git a/apps/worker/src/inspect/data_completeness/backfill_coverage.rs b/apps/worker/src/inspect/data_completeness/backfill_coverage.rs new file mode 100644 index 00000000..57bad4ba --- /dev/null +++ b/apps/worker/src/inspect/data_completeness/backfill_coverage.rs @@ -0,0 +1,382 @@ +use std::collections::BTreeSet; + +use anyhow::{Context, Result}; +use bigname_manifests::{ + WATCHED_COVERAGE_VERIFICATION_CHUNK_BLOCKS, find_uncovered_watched_tuples, + load_active_manifest_topic0s_by_chain_and_source_families, load_log_producing_source_families, +}; +use bigname_storage::{DataCompletenessRead, ensure_backfill_family_topic_sets_undrifted}; +use sqlx::PgPool; + +const MAX_REPORTED_UNCOVERED_TUPLES: usize = 20; + +#[derive(Clone, Debug, Eq, PartialEq)] +pub(super) struct BackfillCoverageGap { + pub(super) chain: String, + pub(super) source_family: String, + pub(super) address: String, + pub(super) required_from_block: i64, + pub(super) required_to_block: i64, +} + +#[derive(Clone, Debug, Eq, PartialEq)] +pub(super) struct BackfillCoverageTopicDrift { + pub(super) chain: String, + pub(super) required_from_block: i64, + pub(super) required_to_block: i64, + pub(super) reason: String, +} + +#[derive(Clone, Debug, Default, Eq, PartialEq)] +pub(super) struct BackfillCoverageInspection { + pub(super) gaps: Vec, + pub(super) topic_drifts: Vec, +} + +/// Reconcile retained bounded-backfill spans against the same durable watched-tuple coverage +/// authority used by stored-lineage checkpoint promotion, including its exact active-manifest +/// topic-set drift guard before facts are trusted. Incomplete and failed job intervals remain in +/// scope because their retained lineage can be crash residue; facts from a later retry can satisfy +/// the interval. Evidence remains required after a checkpoint consumes the span: checkpoint +/// regression or database restore can make it promotion input again, and deleting the evidence +/// must not silently preserve a completeness pass. +pub(super) async fn load_backfill_coverage( + pool: &PgPool, + read: &DataCompletenessRead, +) -> Result { + let active_chains = read + .manifest_chain_namespaces + .iter() + .map(|entry| entry.chain.as_str()) + .collect::>(); + let mut gaps = Vec::new(); + let mut topic_drifts = Vec::new(); + + for chain in read + .chains + .iter() + .filter(|row| active_chains.contains(row.chain_id.as_str())) + { + let (Some(lineage_floor), Some(lineage_head)) = ( + chain.lineage_floor_block_number, + chain.lineage_head_block_number, + ) else { + continue; + }; + let backfill_jobs = bigname_storage::load_backfill_jobs_intersecting_range( + pool, + &chain.chain_id, + lineage_floor, + lineage_head, + ) + .await + .with_context(|| { + format!( + "failed to load backfill evidence for completeness coverage on {}", + chain.chain_id + ) + })?; + let retained_ranges = merged_retained_backfill_ranges( + backfill_jobs + .iter() + .map(|job| (job.range_start_block_number, job.range_end_block_number)), + lineage_floor, + lineage_head, + ); + if retained_ranges.is_empty() { + // Ordinary provider-fetched live lineage does not use backfill coverage facts. + // Persisted bounded jobs identify the retained spans that do. + continue; + } + let source_families = load_log_producing_source_families(pool, &chain.chain_id) + .await + .with_context(|| { + format!( + "failed to load log-producing source families for completeness coverage on {}", + chain.chain_id + ) + })?; + if source_families.is_empty() { + continue; + } + let current_topic0s_by_family = load_active_manifest_topic0s_by_chain_and_source_families( + pool, + &chain.chain_id, + &source_families, + ) + .await + .with_context(|| { + format!( + "failed to load current manifest topic sets for completeness coverage on {}", + chain.chain_id + ) + })?; + + for (from_block, through_block) in retained_ranges { + if let Err(reason) = ensure_backfill_family_topic_sets_undrifted( + pool, + &chain.chain_id, + ¤t_topic0s_by_family, + from_block, + through_block, + ) + .await + { + if topic_drifts.len() < MAX_REPORTED_UNCOVERED_TUPLES { + topic_drifts.push(BackfillCoverageTopicDrift { + chain: chain.chain_id.clone(), + required_from_block: from_block, + required_to_block: through_block, + reason, + }); + } + continue; + } + let mut chunk_from = from_block; + while chunk_from <= through_block && gaps.len() < MAX_REPORTED_UNCOVERED_TUPLES { + let chunk_through = chunk_from + .saturating_add(WATCHED_COVERAGE_VERIFICATION_CHUNK_BLOCKS - 1) + .min(through_block); + let remaining = MAX_REPORTED_UNCOVERED_TUPLES - gaps.len(); + let uncovered = find_uncovered_watched_tuples( + pool, + &chain.chain_id, + chunk_from, + chunk_through, + &source_families, + remaining as i64, + ) + .await + .with_context(|| { + format!( + "failed to reconcile completeness coverage for {} over {}..={}", + chain.chain_id, chunk_from, chunk_through + ) + })?; + gaps.extend(uncovered.into_iter().map(|tuple| BackfillCoverageGap { + chain: chain.chain_id.clone(), + source_family: tuple.source_family, + address: tuple.address, + required_from_block: tuple.required_from_block, + required_to_block: tuple.required_to_block, + })); + + let Some(next_chunk) = chunk_through.checked_add(1) else { + break; + }; + chunk_from = next_chunk; + } + if gaps.len() >= MAX_REPORTED_UNCOVERED_TUPLES { + break; + } + } + } + + Ok(BackfillCoverageInspection { gaps, topic_drifts }) +} + +fn merged_retained_backfill_ranges( + ranges: impl IntoIterator, + lineage_floor: i64, + lineage_head: i64, +) -> Vec<(i64, i64)> { + let mut ranges = ranges + .into_iter() + .filter_map(|(from_block, through_block)| { + let from_block = from_block.max(lineage_floor); + let through_block = through_block.min(lineage_head); + (from_block <= through_block).then_some((from_block, through_block)) + }) + .collect::>(); + ranges.sort_unstable(); + + let mut merged = Vec::<(i64, i64)>::new(); + for (from_block, through_block) in ranges { + if let Some((_, merged_through)) = merged.last_mut() + && from_block <= merged_through.saturating_add(1) + { + *merged_through = (*merged_through).max(through_block); + } else { + merged.push((from_block, through_block)); + } + } + merged +} + +#[cfg(test)] +mod tests { + use anyhow::Result; + use bigname_test_support::{TestDatabase, TestDatabaseConfig}; + + use super::{load_backfill_coverage, merged_retained_backfill_ranges}; + + #[test] + fn backfill_ranges_are_clamped_and_merged_inside_retained_lineage() { + assert_eq!( + merged_retained_backfill_ranges([(90, 101), (102, 120), (130, 160)], 100, 150), + vec![(100, 120), (130, 150)] + ); + } + + #[tokio::test] + async fn incomplete_backfill_residue_requires_durable_coverage_facts() -> Result<()> { + let database = TestDatabase::create_migrated( + TestDatabaseConfig::new("worker_data_completeness_backfill_coverage") + .admin_database("postgres") + .pool_max_connections(5) + .parse_context("failed to parse worker completeness test database URL") + .admin_connect_context("failed to connect worker completeness admin pool") + .pool_connect_context("failed to connect worker completeness test pool"), + &bigname_storage::MIGRATOR, + "failed to apply worker completeness test migrations", + ) + .await?; + let pool = database.pool(); + let manifest_id = sqlx::query_scalar::<_, i64>( + r#" + INSERT INTO manifest_versions + (manifest_version, namespace, source_family, chain, deployment_epoch, + rollout_status, normalizer_version, file_path, manifest_payload) + VALUES + (1, 'ens', 'ens_v2_registry_l1', 'ethereum-sepolia', + 'ens_v2_sepolia_dev', 'active', 'n', 'f', + '{"contracts":[{"role":"registry","address":"0xabc","start_block":1}], + "abi":{"events":[{"name":"ResolverChanged", + "fragment":"event ResolverChanged(bytes32 indexed node)"}]}}'::jsonb) + RETURNING manifest_id + "#, + ) + .fetch_one(pool) + .await?; + sqlx::raw_sql( + r#" + INSERT INTO contract_instances (contract_instance_id, chain_id, contract_kind) + VALUES ('11111111-1111-1111-1111-111111111111', 'ethereum-sepolia', 'contract') + "#, + ) + .execute(pool) + .await?; + sqlx::query( + r#" + INSERT INTO manifest_contract_instances + (manifest_id, declaration_kind, declaration_name, contract_instance_id, + declared_address, role, proxy_kind) + VALUES + ($1, 'contract', 'registry', '11111111-1111-1111-1111-111111111111', + '0xabc', 'registry', 'none') + "#, + ) + .bind(manifest_id) + .execute(pool) + .await?; + sqlx::raw_sql( + r#" + INSERT INTO contract_instance_addresses + (contract_instance_id, chain_id, address, active_from_block_number) + VALUES + ('11111111-1111-1111-1111-111111111111', 'ethereum-sepolia', '0xabc', 1); + + INSERT INTO chain_lineage + (chain_id, block_hash, parent_hash, block_number, block_timestamp, + canonicality_state) + VALUES + ('ethereum-sepolia', '0x100', '0x099', 100, now(), 'canonical'), + ('ethereum-sepolia', '0x101', '0x100', 101, now(), 'canonical'), + ('ethereum-sepolia', '0x102', '0x101', 102, now(), 'canonical'); + + INSERT INTO chain_checkpoints + (chain_id, canonical_block_hash, canonical_block_number) + VALUES ('ethereum-sepolia', '0x100', 100) + "#, + ) + .execute(pool) + .await?; + + let read = bigname_storage::load_data_completeness(pool).await?; + let inspection = load_backfill_coverage(pool, &read).await?; + assert!(inspection.gaps.is_empty()); + assert!(inspection.topic_drifts.is_empty()); + let backfill_job_id = sqlx::query_scalar::<_, i64>( + r#" + INSERT INTO backfill_jobs + (deployment_profile, chain_id, source_identity, scan_mode, + range_start_block_number, range_end_block_number, idempotency_key, + status) + VALUES + ('sepolia', 'ethereum-sepolia', '{}'::jsonb, 'hash_pinned', + 101, 102, 'missing-facts', 'running') + RETURNING backfill_job_id + "#, + ) + .fetch_one(pool) + .await?; + + sqlx::query( + r#" + UPDATE chain_checkpoints + SET canonical_block_hash = '0x102', canonical_block_number = 102 + WHERE chain_id = 'ethereum-sepolia' + "#, + ) + .execute(pool) + .await?; + let read = bigname_storage::load_data_completeness(pool).await?; + let inspection = load_backfill_coverage(pool, &read).await?; + assert_eq!(inspection.gaps.len(), 1); + assert_eq!(inspection.gaps[0].chain, "ethereum-sepolia"); + assert_eq!(inspection.gaps[0].source_family, "ens_v2_registry_l1"); + assert_eq!(inspection.gaps[0].address, "0xabc"); + assert_eq!(inspection.gaps[0].required_from_block, 101); + assert_eq!(inspection.gaps[0].required_to_block, 102); + assert!(inspection.topic_drifts.is_empty()); + + sqlx::query( + r#" + UPDATE backfill_jobs + SET status = 'completed', completed_at = now() + WHERE backfill_job_id = $1 + "#, + ) + .bind(backfill_job_id) + .execute(pool) + .await?; + sqlx::query( + r#" + INSERT INTO backfill_coverage_facts + (backfill_job_id, chain_id, source_family, scope, address, + covered_from_block, covered_to_block, derivation) + VALUES ($1, 'ethereum-sepolia', 'ens_v2_registry_l1', 'address', '0xabc', + 101, 102, 'job_completion') + "#, + ) + .bind(backfill_job_id) + .execute(pool) + .await?; + let inspection = load_backfill_coverage(pool, &read).await?; + assert!(inspection.gaps.is_empty()); + assert!(inspection.topic_drifts.is_empty()); + + sqlx::query( + r#" + UPDATE backfill_jobs + SET source_identity = '{ + "topic0s_by_source_family": {"ens_v2_registry_l1": []} + }'::jsonb + WHERE backfill_job_id = $1 + "#, + ) + .bind(backfill_job_id) + .execute(pool) + .await?; + let inspection = load_backfill_coverage(pool, &read).await?; + assert!(inspection.gaps.is_empty()); + assert_eq!(inspection.topic_drifts.len(), 1); + assert!( + inspection.topic_drifts[0] + .reason + .contains("manifest ABI topic0 set changed") + ); + + database.cleanup().await + } +} diff --git a/apps/worker/src/inspect/data_completeness/evaluate.rs b/apps/worker/src/inspect/data_completeness/evaluate.rs new file mode 100644 index 00000000..f858fdfc --- /dev/null +++ b/apps/worker/src/inspect/data_completeness/evaluate.rs @@ -0,0 +1,590 @@ +mod helpers; +mod report; + +pub(super) use report::{CheckStatus, CursorLag, DataCompletenessReport}; + +use crate::{ + projection_apply::NORMALIZED_EVENT_CURSOR, + replay::{ALL_CURRENT_PROJECTION_ORDER, CURRENT_PROJECTION_REPLAY_VERSION}, +}; +use bigname_manifests::WatchedContract; +use bigname_storage::{ + DEFERRED_NORMALIZED_EVENT_INDEXES, DataCompletenessRead, MAX_LIVE_CONTIGUOUS_GAP_FILL_BLOCKS, +}; +use helpers::{ + ActiveTargetInfo, ChainStartInfo, chain_frontier, cursor_label, latched_replay_lag, + missing_chain_frontier, replay_complete_lag, +}; +use report::{ + ChainWithoutFiniteStart, HistoryTruncation, MissingManifestContent, MissingManifestLineage, + MissingManifestRawLogs, PendingActivationTarget, UnobservedTarget, +}; +use std::collections::{BTreeMap, BTreeSet}; + +use super::{ + backfill_coverage::BackfillCoverageInspection, manifest_corpus::ManifestCorpusInspection, + projection_content::ProjectionContentInspection, +}; +use crate::inspect::RetentionMode; + +/// Blocks the stored canonical checkpoint may lead the reconciliation lineage head before the +/// frontier check fails. The opposite direction uses the writer's live contiguous gap limit. +pub(super) const DEFAULT_MAX_HEAD_LAG_BLOCKS: i64 = 8; + +pub(super) const RAW_FACT_NORMALIZED_EVENTS_CURSOR: &str = "raw_fact_normalized_events"; + +/// A chain that has this cursor ran closure/dependency replay, which latches the +/// `raw_fact_normalized_events` cursor's target permanently below the live head; newer logs +/// are swept by the backlog cursor and then live adapter sync. On such a chain the raw-fact +/// cursor is caught up when it reaches its own latched target, not the raw-log head. +pub(super) const POST_REPLAY_LIVE_ADAPTER_BACKLOG_CURSOR: &str = "post_replay_live_adapter_backlog"; + +pub(super) fn evaluate_data_completeness( + read: &DataCompletenessRead, + watched_contracts: &[WatchedContract], + backfill_coverage: &BackfillCoverageInspection, + manifest_corpus: &ManifestCorpusInspection, + projection_content: &ProjectionContentInspection, + max_head_lag_blocks: i64, + retention_mode: RetentionMode, +) -> DataCompletenessReport { + let observed = read + .observed_code_addresses + .iter() + .map(|entry| { + ( + (entry.chain_id.clone(), entry.address.to_ascii_lowercase()), + entry.max_observed_block_number, + ) + }) + .collect::>(); + let canonical_heads = read + .chains + .iter() + .filter_map(|chain| { + chain + .canonical_block_number + .map(|head| (chain.chain_id.as_str(), head)) + }) + .collect::>(); + + // Direct manifest declarations remain authority even if a partial restore lost the + // contract_instance_addresses row that normally materializes them into the watch view. + // Entries are deduplicated before deriving per-address coverage and per-chain history. + let mut active_target_entries = BTreeSet::<(String, String, String, Option)>::new(); + for contract in watched_contracts + .iter() + .filter(|contract| contract.active_to_block_number.is_none()) + { + active_target_entries.insert(( + contract.chain.clone(), + contract.address.to_ascii_lowercase(), + contract.source_family.clone(), + contract.active_from_block_number, + )); + } + for target in &read.manifest_declared_targets { + active_target_entries.insert(( + target.chain.clone(), + target.address.to_ascii_lowercase(), + target.source_family.clone(), + target.active_from_block_number, + )); + } + + let pending_activation_targets = active_target_entries + .iter() + .filter_map( + |(chain, address, source_family, active_from_block_number)| { + let start = active_from_block_number.as_ref().copied()?; + let canonical_head = canonical_heads.get(chain.as_str()).copied()?; + (start > canonical_head).then(|| PendingActivationTarget { + chain: chain.clone(), + address: address.clone(), + source_family: source_family.clone(), + active_from_block_number: start, + canonical_head_block_number: canonical_head, + }) + }, + ) + .collect::>(); + let pending_activation_entries = pending_activation_targets + .iter() + .map(|target| { + ( + target.chain.as_str(), + target.address.as_str(), + target.source_family.as_str(), + target.active_from_block_number, + ) + }) + .collect::>(); + let active_watched_target_count = active_target_entries + .iter() + .map(|(chain, address, _, _)| (chain, address)) + .collect::>() + .len(); + + // Coverage is address-scoped. Pre-activation declarations remain active authority but do + // not require an impossible code observation above the stored canonical head. If multiple + // current source entries share an address, the latest finite start is the strictest lower + // bound and proves every current entry was observed after its admission. + let mut active_targets = BTreeMap::<(String, String), ActiveTargetInfo>::new(); + for (chain, address, source_family, active_from_block_number) in &active_target_entries { + if active_from_block_number.is_some_and(|start| { + pending_activation_entries.contains(&( + chain.as_str(), + address.as_str(), + source_family.as_str(), + start, + )) + }) { + continue; + } + let target = active_targets + .entry((chain.clone(), address.clone())) + .or_insert_with(|| ActiveTargetInfo { + source_family: source_family.clone(), + active_from_block_number: *active_from_block_number, + }); + if active_from_block_number > &target.active_from_block_number { + target.source_family = source_family.clone(); + target.active_from_block_number = *active_from_block_number; + } + } + + let unobserved_targets = active_targets + .iter() + .filter_map(|((chain, address), target)| { + let max_observed_block_number = + observed.get(&(chain.clone(), address.clone())).copied(); + let covered = match target.active_from_block_number { + Some(start) => max_observed_block_number.is_some_and(|block| block >= start), + None => max_observed_block_number.is_some(), + }; + (!covered).then(|| UnobservedTarget { + chain: chain.clone(), + address: address.clone(), + source_family: target.source_family.clone(), + active_from_block_number: target.active_from_block_number, + max_observed_block_number, + }) + }) + .collect::>(); + + // Keep the structural address-materialization check separate from code observation. The + // direct manifest union above prevents a missing address row from shrinking coverage, while + // this diagnostic identifies the broken authority edge itself. + let manifest_targets_missing_address = read + .manifest_declared_targets_missing_address + .iter() + .map(|target| UnobservedTarget { + chain: target.chain.clone(), + address: target.address.clone(), + source_family: target.source_family.clone(), + active_from_block_number: target.active_from_block_number, + max_observed_block_number: observed + .get(&(target.chain.clone(), target.address.to_ascii_lowercase())) + .copied(), + }) + .collect::>(); + let manifest_proxy_implementations_missing_edge = read + .manifest_proxy_implementations_missing_edge + .iter() + .map(|target| UnobservedTarget { + chain: target.chain.clone(), + address: target.address.clone(), + source_family: target.source_family.clone(), + active_from_block_number: target.active_from_block_number, + max_observed_block_number: observed + .get(&(target.chain.clone(), target.address.to_ascii_lowercase())) + .copied(), + }) + .collect::>(); + + // Per-chain declared start information across the deduplicated active source entries. + let mut chain_starts = BTreeMap::::new(); + for (chain, _, _, active_from_block_number) in &active_target_entries { + let info = chain_starts.entry(chain.clone()).or_default(); + info.target_count += 1; + match active_from_block_number { + Some(start) => { + info.finite_min_start = Some( + info.finite_min_start + .map_or(*start, |current| current.min(*start)), + ); + } + None => info.open_ended_target_count += 1, + } + } + + let manifest_chains = read + .manifest_chain_namespaces + .iter() + .map(|entry| entry.chain.as_str()) + .collect::>(); + let mut active_chains = chain_starts.keys().cloned().collect::>(); + active_chains.extend(manifest_chains.iter().map(|chain| (*chain).to_owned())); + + let storage_chains = read + .chains + .iter() + .map(|chain| (chain.chain_id.as_str(), chain)) + .collect::>(); + + // Gating frontiers cover active chains only; a foreign or retired chain with residual + // storage rows is an advisory, not a permanent gate failure. + let mut frontiers = Vec::new(); + for chain_id in &active_chains { + match storage_chains.get(chain_id.as_str()) { + Some(row) => frontiers.push(chain_frontier(row)), + None => frontiers.push(missing_chain_frontier(chain_id)), + } + } + let foreign_chains = read + .chains + .iter() + .filter(|chain| !active_chains.contains(&chain.chain_id)) + .map(|chain| chain.chain_id.clone()) + .collect::>(); + let active_deployment_profile = read.active_deployment_profile.as_deref(); + let cursor_is_active = |cursor: &bigname_storage::ReplayCursorRow| { + active_deployment_profile == Some(cursor.deployment_profile.as_str()) + && active_chains.contains(&cursor.chain_id) + }; + let ignored_replay_cursors = read + .replay_cursors + .iter() + .filter(|cursor| !cursor_is_active(cursor)) + .map(cursor_label) + .collect::>(); + + // History: a finite declared start requires the lineage floor to reach it; a chain whose + // targets are all open-ended has no floor to check and fails closed. + let mut chains_history_truncated = Vec::new(); + let mut chains_without_finite_start = Vec::new(); + for (chain, info) in &chain_starts { + if info.target_count == 0 { + continue; + } + match info.finite_min_start { + Some(declared_start) => { + let floor = storage_chains + .get(chain.as_str()) + .and_then(|row| row.lineage_floor_block_number); + if !matches!(floor, Some(f) if f <= declared_start) { + chains_history_truncated.push(HistoryTruncation { + chain: chain.clone(), + declared_start_block: declared_start, + lineage_floor_block: floor, + }); + } + } + None => chains_without_finite_start.push(ChainWithoutFiniteStart { + chain: chain.clone(), + open_ended_target_count: info.open_ended_target_count, + }), + } + } + + let canonical_raw_log_head = read + .chains + .iter() + .map(|chain| { + ( + chain.chain_id.as_str(), + chain.canonical_raw_log_head_block_number, + ) + }) + .collect::>(); + + let closure_replay_chains = read + .manifest_chain_source_families + .iter() + .filter(|entry| { + bigname_adapters::source_family_preserves_normalized_replay_target(&entry.source_family) + }) + .map(|entry| entry.chain.as_str()) + .collect::>(); + + let raw_fact_targets = read + .replay_cursors + .iter() + .filter(|cursor| cursor_is_active(cursor)) + .filter(|cursor| cursor.cursor_kind == RAW_FACT_NORMALIZED_EVENTS_CURSOR) + .map(|cursor| { + ( + (cursor.deployment_profile.as_str(), cursor.chain_id.as_str()), + cursor.target_block_number, + ) + }) + .collect::>(); + + let failed_replay_cursors = read + .replay_cursors + .iter() + .filter(|cursor| cursor_is_active(cursor)) + .filter(|cursor| cursor.last_failure_reason.is_some()) + .map(cursor_label) + .collect::>(); + + let post_replay_backlog_cursor_keys = read + .replay_cursors + .iter() + .filter(|cursor| cursor_is_active(cursor)) + .filter(|cursor| cursor.cursor_kind == POST_REPLAY_LIVE_ADAPTER_BACKLOG_CURSOR) + .map(|cursor| (cursor.deployment_profile.as_str(), cursor.chain_id.as_str())) + .collect::>(); + + let lagging_replay_cursors = read + .replay_cursors + .iter() + .filter(|cursor| cursor_is_active(cursor)) + .filter_map(|cursor| match cursor.cursor_kind.as_str() { + RAW_FACT_NORMALIZED_EVENTS_CURSOR => { + if closure_replay_chains.contains(cursor.chain_id.as_str()) { + let head = canonical_raw_log_head + .get(cursor.chain_id.as_str()) + .copied() + .flatten(); + let key = (cursor.deployment_profile.as_str(), cursor.chain_id.as_str()); + latched_replay_lag(cursor, head, post_replay_backlog_cursor_keys.contains(&key)) + } else { + // Non-latched: replay must have completed its target and the target must + // have reached the canonical raw-log head. + let head = canonical_raw_log_head + .get(cursor.chain_id.as_str()) + .copied() + .flatten()?; + if let Some(lag) = replay_complete_lag(cursor) { + return Some(lag); + } + let target = cursor.target_block_number.unwrap_or(-1); + (target < head).then(|| CursorLag { + label: cursor_label(cursor), + behind_by: head - target, + }) + } + } + POST_REPLAY_LIVE_ADAPTER_BACKLOG_CURSOR => { + let key = (cursor.deployment_profile.as_str(), cursor.chain_id.as_str()); + let expected_start = raw_fact_targets + .get(&key) + .copied() + .flatten() + .and_then(|target| target.checked_add(1)); + match expected_start { + Some(expected) if cursor.range_start_block_number > expected => { + Some(CursorLag { + label: format!("{}:range_start", cursor_label(cursor)), + behind_by: cursor.range_start_block_number - expected, + }) + } + Some(_) => replay_complete_lag(cursor), + None => Some(CursorLag { + label: format!("{}:range_start", cursor_label(cursor)), + behind_by: -1, + }), + } + } + _ => None, + }) + .collect::>(); + + let chains_with_raw_fact_cursor = read + .replay_cursors + .iter() + .filter(|cursor| cursor_is_active(cursor)) + .filter(|cursor| cursor.cursor_kind == RAW_FACT_NORMALIZED_EVENTS_CURSOR) + .map(|cursor| cursor.chain_id.as_str()) + .collect::>(); + + let chains_missing_raw_fact_cursor = read + .chains + .iter() + .filter(|chain| active_chains.contains(&chain.chain_id)) + .filter(|chain| chain.canonical_raw_log_head_block_number.is_some()) + .filter(|chain| !chains_with_raw_fact_cursor.contains(chain.chain_id.as_str())) + .map(|chain| chain.chain_id.clone()) + .collect::>(); + + let expected_projection_cursor = read + .projection_apply_cursors + .iter() + .find(|cursor| cursor.cursor_name == NORMALIZED_EVENT_CURSOR); + let lagging_projection_cursors = expected_projection_cursor + .into_iter() + .filter_map(|cursor| { + let max_change_id = read.max_projection_change_id?; + (cursor.last_change_id < max_change_id).then(|| CursorLag { + label: cursor.cursor_name.clone(), + behind_by: max_change_id - cursor.last_change_id, + }) + }) + .collect::>(); + + let projection_apply_cursor_missing = + read.max_projection_change_id.is_some() && expected_projection_cursor.is_none(); + let projection_apply_cursor_ahead_by = expected_projection_cursor.and_then(|cursor| { + let retained_change_high_watermark = read.max_projection_change_id.unwrap_or(0); + (cursor.last_change_id > retained_change_high_watermark) + .then_some(cursor.last_change_id - retained_change_high_watermark) + }); + let projection_replay_target_coverage_required = !matches!( + (expected_projection_cursor, read.max_projection_change_id), + (Some(cursor), Some(high_watermark)) if cursor.last_change_id == high_watermark + ); + + // Projection replay markers: report the newest stored version for diagnosis, but require + // all current projections at this worker's replay version, exactly as automatic bootstrap + // does. Until a non-empty change log exactly corroborates the durable apply cursor, each + // marker must also cover the target automatic bootstrap would request now; after that + // handoff, apply/change/invalidation drain is the advancing authority and replay markers + // intentionally remain fixed. + let projection_replay_version = read + .projection_replay_markers + .iter() + .map(|marker| marker.replay_version) + .max(); + let present_projection_replay_markers = read + .projection_replay_markers + .iter() + .filter(|marker| { + marker.replay_version == CURRENT_PROJECTION_REPLAY_VERSION + && (!projection_replay_target_coverage_required + || match read.projection_replay_required_target_block { + Some(required) => marker + .completed_normalized_target_block + .is_some_and(|completed| completed >= required), + None => true, + }) + }) + .map(|marker| marker.projection.as_str()) + .collect::>(); + let missing_projection_replay_markers = ALL_CURRENT_PROJECTION_ORDER + .iter() + .filter(|projection| !present_projection_replay_markers.contains(*projection)) + .map(|projection| (*projection).to_owned()) + .collect(); + + // Content expectations come only from active manifest sources that declare normalized + // adapter output. Counts are joined to the exact source_manifest_id by storage, so rows + // from deprecated manifests cannot satisfy a newly active source. + let active_manifest_sources_without_events = read + .active_manifest_event_sources + .iter() + .filter(|source| source.normalized_event_count == 0) + .map(|source| MissingManifestContent { + manifest_id: source.manifest_id, + manifest_version: source.manifest_version, + chain: source.chain.clone(), + namespace: source.namespace.clone(), + source_family: source.source_family.clone(), + }) + .collect::>(); + let active_manifest_sources_with_missing_lineage = read + .active_manifest_event_sources + .iter() + .filter(|source| source.normalized_events_missing_canonical_lineage_count > 0) + .map(|source| MissingManifestLineage { + manifest_id: source.manifest_id, + manifest_version: source.manifest_version, + chain: source.chain.clone(), + namespace: source.namespace.clone(), + source_family: source.source_family.clone(), + missing_canonical_lineage_count: source + .normalized_events_missing_canonical_lineage_count, + }) + .collect::>(); + let active_manifest_sources_with_missing_raw_logs = read + .active_manifest_event_sources + .iter() + .filter(|source| source.normalized_events_missing_canonical_raw_log_count > 0) + .map(|source| MissingManifestRawLogs { + manifest_id: source.manifest_id, + manifest_version: source.manifest_version, + chain: source.chain.clone(), + namespace: source.namespace.clone(), + source_family: source.source_family.clone(), + missing_canonical_raw_log_count: source + .normalized_events_missing_canonical_raw_log_count, + }) + .collect::>(); + let names_by_namespace = read + .name_current_counts + .iter() + .map(|entry| (entry.namespace.as_str(), entry.count)) + .collect::>(); + let active_namespaces_without_names = read + .active_manifest_event_sources + .iter() + .map(|source| source.namespace.as_str()) + .collect::>() + .into_iter() + .filter(|namespace| names_by_namespace.get(namespace).copied().unwrap_or(0) == 0) + .map(str::to_owned) + .collect::>(); + + let present_indexes = read + .present_deferred_projection_indexes + .iter() + .map(String::as_str) + .collect::>(); + let missing_deferred_projection_indexes = DEFERRED_NORMALIZED_EVENT_INDEXES + .iter() + .filter(|index| !present_indexes.contains(*index)) + .map(|index| (*index).to_owned()) + .collect::>(); + + let normalized_event_total = read + .active_manifest_event_sources + .iter() + .map(|source| source.normalized_event_count) + .sum(); + let name_current_total = read.name_current_counts.iter().map(|e| e.count).sum(); + + DataCompletenessReport { + max_head_lag_blocks, + max_lineage_ahead_blocks: MAX_LIVE_CONTIGUOUS_GAP_FILL_BLOCKS, + retention_mode, + manifest_corpus: manifest_corpus.clone(), + projection_content: projection_content.clone(), + active_deployment_profile: read.active_deployment_profile.clone(), + frontiers, + foreign_chains, + ignored_replay_cursors, + active_watched_target_count, + unobserved_targets, + pending_activation_targets, + manifest_targets_missing_address, + manifest_proxy_implementations_missing_edge, + discovery_targets_missing_address: read.discovery_targets_missing_address.clone(), + discovery_targets_missing_manifest: read.discovery_targets_missing_manifest.clone(), + chains_history_truncated, + chains_without_finite_start, + backfill_coverage_gaps: backfill_coverage.gaps.clone(), + backfill_coverage_topic_drifts: backfill_coverage.topic_drifts.clone(), + failed_replay_cursors, + lagging_replay_cursors, + chains_missing_raw_fact_cursor, + lagging_projection_cursors, + projection_apply_cursor_missing, + projection_apply_cursor_ahead_by, + pending_projection_invalidation_count: read.pending_projection_invalidation_count, + projection_invalidation_dead_letter_count: read.projection_invalidation_dead_letter_count, + projection_replay_version, + projection_replay_required_version: CURRENT_PROJECTION_REPLAY_VERSION, + projection_replay_target_coverage_required, + projection_replay_required_target_block: read.projection_replay_required_target_block, + missing_projection_replay_markers, + active_manifest_sources_without_events, + active_manifest_sources_with_missing_lineage, + active_manifest_sources_with_missing_raw_logs, + active_namespaces_without_names, + normalized_events_null_chain_id_count: read.normalized_events_null_chain_id_count, + missing_deferred_projection_indexes, + backfill_advisory: read.backfill_lifecycle.clone(), + normalized_event_total, + name_current_total, + } +} diff --git a/apps/worker/src/inspect/data_completeness/evaluate/helpers.rs b/apps/worker/src/inspect/data_completeness/evaluate/helpers.rs new file mode 100644 index 00000000..c6dc264d --- /dev/null +++ b/apps/worker/src/inspect/data_completeness/evaluate/helpers.rs @@ -0,0 +1,103 @@ +use super::report::{ChainFrontier, CursorLag}; +use bigname_storage::{ChainCompletenessRow, ReplayCursorRow}; + +#[derive(Default)] +pub(super) struct ChainStartInfo { + pub(super) finite_min_start: Option, + pub(super) open_ended_target_count: usize, + pub(super) target_count: usize, +} + +#[derive(Clone)] +pub(super) struct ActiveTargetInfo { + pub(super) source_family: String, + pub(super) active_from_block_number: Option, +} + +/// Replay is complete for a cursor's target when `next_block_number > target_block_number`. +/// A reorg rewind lowers `next_block_number` below the target, so a stale high +/// `last_completed_block_number` no longer reads as caught up. A missing bound fails closed. +pub(super) fn replay_complete_lag(cursor: &ReplayCursorRow) -> Option { + match (cursor.next_block_number, cursor.target_block_number) { + (Some(next), Some(target)) if next > target => None, + (next, Some(target)) => Some(CursorLag { + label: cursor_label(cursor), + behind_by: target - next.unwrap_or(-1), + }), + (_, None) => Some(CursorLag { + label: cursor_label(cursor), + behind_by: -1, + }), + } +} + +pub(super) fn latched_replay_lag( + cursor: &ReplayCursorRow, + canonical_raw_log_head: Option, + backlog_cursor_present: bool, +) -> Option { + if let Some(lag) = replay_complete_lag(cursor) { + return Some(lag); + } + let target = cursor.target_block_number?; + match canonical_raw_log_head { + Some(head) if head > target && !backlog_cursor_present => Some(CursorLag { + label: format!("{}:missing_backlog", cursor_label(cursor)), + behind_by: head - target, + }), + _ => None, + } +} + +pub(super) fn cursor_label(cursor: &ReplayCursorRow) -> String { + format!( + "{}/{}/{}", + cursor.deployment_profile, cursor.chain_id, cursor.cursor_kind + ) +} + +pub(super) fn missing_chain_frontier(chain_id: &str) -> ChainFrontier { + ChainFrontier { + chain_id: chain_id.to_owned(), + canonical_block_number: None, + checkpoint_canonical_lineage_match: false, + lineage_head_block_number: None, + head_lag_blocks: None, + contiguous: false, + missing_block_count: 0, + duplicate_canonical_height_count: 0, + disconnected_canonical_parent_count: 0, + missing_from_storage: true, + } +} + +pub(super) fn chain_frontier(chain: &ChainCompletenessRow) -> ChainFrontier { + let head_lag_blocks = chain + .canonical_block_number + .zip(chain.lineage_head_block_number) + .map(|(canonical, lineage_head)| canonical - lineage_head); + + let expected_block_count = chain + .lineage_head_block_number + .zip(chain.lineage_floor_block_number) + .map(|(head, floor)| head - floor + 1); + let missing_block_count = expected_block_count + .map(|expected| expected - chain.lineage_canonical_block_count) + .unwrap_or_default(); + + ChainFrontier { + chain_id: chain.chain_id.clone(), + canonical_block_number: chain.canonical_block_number, + checkpoint_canonical_lineage_match: chain.checkpoint_canonical_lineage_match, + lineage_head_block_number: chain.lineage_head_block_number, + head_lag_blocks, + contiguous: expected_block_count.is_some() + && missing_block_count == 0 + && chain.duplicate_canonical_height_count == 0 + && chain.disconnected_canonical_parent_count == 0, + missing_block_count, + duplicate_canonical_height_count: chain.duplicate_canonical_height_count, + disconnected_canonical_parent_count: chain.disconnected_canonical_parent_count, + missing_from_storage: false, + } +} diff --git a/apps/worker/src/inspect/data_completeness/evaluate/report.rs b/apps/worker/src/inspect/data_completeness/evaluate/report.rs new file mode 100644 index 00000000..889fc41c --- /dev/null +++ b/apps/worker/src/inspect/data_completeness/evaluate/report.rs @@ -0,0 +1,324 @@ +use bigname_storage::{ + BackfillLifecycleRow, DiscoveryTargetMissingAddress, DiscoveryTargetMissingManifest, +}; + +use super::super::backfill_coverage::{BackfillCoverageGap, BackfillCoverageTopicDrift}; +use super::super::manifest_corpus::ManifestCorpusInspection; +use super::super::projection_content::ProjectionContentInspection; +use crate::inspect::RetentionMode; + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub(in crate::inspect::data_completeness) enum CheckStatus { + Pass, + Fail, +} + +impl CheckStatus { + pub(in crate::inspect::data_completeness) const fn label(self) -> &'static str { + match self { + Self::Pass => "pass", + Self::Fail => "fail", + } + } + + pub(super) const fn from_pass(pass: bool) -> Self { + if pass { Self::Pass } else { Self::Fail } + } +} + +#[derive(Clone, Debug, Eq, PartialEq)] +pub(in crate::inspect::data_completeness) struct ChainFrontier { + pub(in crate::inspect::data_completeness) chain_id: String, + pub(in crate::inspect::data_completeness) canonical_block_number: Option, + pub(in crate::inspect::data_completeness) checkpoint_canonical_lineage_match: bool, + pub(in crate::inspect::data_completeness) lineage_head_block_number: Option, + pub(in crate::inspect::data_completeness) head_lag_blocks: Option, + pub(in crate::inspect::data_completeness) contiguous: bool, + pub(in crate::inspect::data_completeness) missing_block_count: i64, + pub(in crate::inspect::data_completeness) duplicate_canonical_height_count: i64, + pub(in crate::inspect::data_completeness) disconnected_canonical_parent_count: i64, + pub(in crate::inspect::data_completeness) missing_from_storage: bool, +} + +#[derive(Clone, Debug, Eq, PartialEq)] +pub(in crate::inspect::data_completeness) struct UnobservedTarget { + pub(in crate::inspect::data_completeness) chain: String, + pub(in crate::inspect::data_completeness) address: String, + pub(in crate::inspect::data_completeness) source_family: String, + pub(in crate::inspect::data_completeness) active_from_block_number: Option, + pub(in crate::inspect::data_completeness) max_observed_block_number: Option, +} + +#[derive(Clone, Debug, Eq, PartialEq)] +pub(in crate::inspect::data_completeness) struct PendingActivationTarget { + pub(in crate::inspect::data_completeness) chain: String, + pub(in crate::inspect::data_completeness) address: String, + pub(in crate::inspect::data_completeness) source_family: String, + pub(in crate::inspect::data_completeness) active_from_block_number: i64, + pub(in crate::inspect::data_completeness) canonical_head_block_number: i64, +} + +#[derive(Clone, Debug, Eq, PartialEq)] +pub(in crate::inspect::data_completeness) struct HistoryTruncation { + pub(in crate::inspect::data_completeness) chain: String, + pub(in crate::inspect::data_completeness) declared_start_block: i64, + pub(in crate::inspect::data_completeness) lineage_floor_block: Option, +} + +/// An active chain all of whose targets have an open-ended (`NULL`) start block, so no history +/// floor can be established for it. +#[derive(Clone, Debug, Eq, PartialEq)] +pub(in crate::inspect::data_completeness) struct ChainWithoutFiniteStart { + pub(in crate::inspect::data_completeness) chain: String, + pub(in crate::inspect::data_completeness) open_ended_target_count: usize, +} + +#[derive(Clone, Debug, Eq, PartialEq)] +pub(in crate::inspect::data_completeness) struct MissingManifestContent { + pub(in crate::inspect::data_completeness) manifest_id: i64, + pub(in crate::inspect::data_completeness) manifest_version: i64, + pub(in crate::inspect::data_completeness) chain: String, + pub(in crate::inspect::data_completeness) namespace: String, + pub(in crate::inspect::data_completeness) source_family: String, +} + +#[derive(Clone, Debug, Eq, PartialEq)] +pub(in crate::inspect::data_completeness) struct MissingManifestLineage { + pub(in crate::inspect::data_completeness) manifest_id: i64, + pub(in crate::inspect::data_completeness) manifest_version: i64, + pub(in crate::inspect::data_completeness) chain: String, + pub(in crate::inspect::data_completeness) namespace: String, + pub(in crate::inspect::data_completeness) source_family: String, + pub(in crate::inspect::data_completeness) missing_canonical_lineage_count: i64, +} + +#[derive(Clone, Debug, Eq, PartialEq)] +pub(in crate::inspect::data_completeness) struct MissingManifestRawLogs { + pub(in crate::inspect::data_completeness) manifest_id: i64, + pub(in crate::inspect::data_completeness) manifest_version: i64, + pub(in crate::inspect::data_completeness) chain: String, + pub(in crate::inspect::data_completeness) namespace: String, + pub(in crate::inspect::data_completeness) source_family: String, + pub(in crate::inspect::data_completeness) missing_canonical_raw_log_count: i64, +} + +/// `behind_by` is the block or change-id distance to the applicable target, best-effort: +/// a missing bound is treated with a `-1` sentinel. +#[derive(Clone, Debug, Eq, PartialEq)] +pub(in crate::inspect::data_completeness) struct CursorLag { + pub(in crate::inspect::data_completeness) label: String, + pub(in crate::inspect::data_completeness) behind_by: i64, +} + +#[derive(Clone, Debug, Eq, PartialEq)] +pub(in crate::inspect::data_completeness) struct DataCompletenessReport { + pub(in crate::inspect::data_completeness) max_head_lag_blocks: i64, + pub(in crate::inspect::data_completeness) max_lineage_ahead_blocks: i64, + pub(in crate::inspect::data_completeness) retention_mode: RetentionMode, + pub(in crate::inspect::data_completeness) manifest_corpus: ManifestCorpusInspection, + pub(in crate::inspect::data_completeness) projection_content: ProjectionContentInspection, + pub(in crate::inspect::data_completeness) active_deployment_profile: Option, + pub(in crate::inspect::data_completeness) frontiers: Vec, + pub(in crate::inspect::data_completeness) foreign_chains: Vec, + pub(in crate::inspect::data_completeness) ignored_replay_cursors: Vec, + pub(in crate::inspect::data_completeness) active_watched_target_count: usize, + pub(in crate::inspect::data_completeness) unobserved_targets: Vec, + pub(in crate::inspect::data_completeness) pending_activation_targets: + Vec, + pub(in crate::inspect::data_completeness) manifest_targets_missing_address: + Vec, + pub(in crate::inspect::data_completeness) manifest_proxy_implementations_missing_edge: + Vec, + pub(in crate::inspect::data_completeness) discovery_targets_missing_address: + Vec, + pub(in crate::inspect::data_completeness) discovery_targets_missing_manifest: + Vec, + pub(in crate::inspect::data_completeness) chains_history_truncated: Vec, + pub(in crate::inspect::data_completeness) chains_without_finite_start: + Vec, + pub(in crate::inspect::data_completeness) backfill_coverage_gaps: Vec, + pub(in crate::inspect::data_completeness) backfill_coverage_topic_drifts: + Vec, + pub(in crate::inspect::data_completeness) failed_replay_cursors: Vec, + pub(in crate::inspect::data_completeness) lagging_replay_cursors: Vec, + pub(in crate::inspect::data_completeness) chains_missing_raw_fact_cursor: Vec, + pub(in crate::inspect::data_completeness) lagging_projection_cursors: Vec, + pub(in crate::inspect::data_completeness) projection_apply_cursor_missing: bool, + pub(in crate::inspect::data_completeness) projection_apply_cursor_ahead_by: Option, + pub(in crate::inspect::data_completeness) pending_projection_invalidation_count: i64, + pub(in crate::inspect::data_completeness) projection_invalidation_dead_letter_count: i64, + pub(in crate::inspect::data_completeness) projection_replay_version: Option, + pub(in crate::inspect::data_completeness) projection_replay_required_version: i32, + pub(in crate::inspect::data_completeness) projection_replay_target_coverage_required: bool, + pub(in crate::inspect::data_completeness) projection_replay_required_target_block: Option, + pub(in crate::inspect::data_completeness) missing_projection_replay_markers: Vec, + pub(in crate::inspect::data_completeness) active_manifest_sources_without_events: + Vec, + pub(in crate::inspect::data_completeness) active_manifest_sources_with_missing_lineage: + Vec, + pub(in crate::inspect::data_completeness) active_manifest_sources_with_missing_raw_logs: + Vec, + pub(in crate::inspect::data_completeness) active_namespaces_without_names: Vec, + pub(in crate::inspect::data_completeness) normalized_events_null_chain_id_count: i64, + pub(in crate::inspect::data_completeness) missing_deferred_projection_indexes: Vec, + pub(in crate::inspect::data_completeness) backfill_advisory: Vec, + pub(in crate::inspect::data_completeness) normalized_event_total: i64, + pub(in crate::inspect::data_completeness) name_current_total: i64, +} + +impl DataCompletenessReport { + pub(in crate::inspect::data_completeness) fn frontier_at_head(&self) -> CheckStatus { + CheckStatus::from_pass(self.frontiers.iter().all(|frontier| { + frontier.checkpoint_canonical_lineage_match + && frontier.head_lag_blocks.is_some_and(|lag| { + (-self.max_lineage_ahead_blocks..=self.max_head_lag_blocks).contains(&lag) + }) + })) + } + + pub(in crate::inspect::data_completeness) fn manifest_corpus_matches_repository( + &self, + ) -> CheckStatus { + CheckStatus::from_pass(self.manifest_corpus.complete()) + } + + pub(in crate::inspect::data_completeness) fn lineage_contiguous(&self) -> CheckStatus { + CheckStatus::from_pass(self.frontiers.iter().all(|frontier| frontier.contiguous)) + } + + pub(in crate::inspect::data_completeness) fn history_from_declared_start(&self) -> CheckStatus { + CheckStatus::from_pass( + self.chains_history_truncated.is_empty() && self.chains_without_finite_start.is_empty(), + ) + } + + pub(in crate::inspect::data_completeness) fn watch_set_observed(&self) -> CheckStatus { + CheckStatus::from_pass( + self.active_watched_target_count > 0 && self.unobserved_targets.is_empty(), + ) + } + + pub(in crate::inspect::data_completeness) fn stored_lineage_backfill_coverage( + &self, + ) -> CheckStatus { + CheckStatus::from_pass( + self.backfill_coverage_gaps.is_empty() + && self.backfill_coverage_topic_drifts.is_empty(), + ) + } + + pub(in crate::inspect::data_completeness) fn manifest_declared_targets_present( + &self, + ) -> CheckStatus { + CheckStatus::from_pass( + self.manifest_targets_missing_address.is_empty() + && self.manifest_proxy_implementations_missing_edge.is_empty(), + ) + } + + pub(in crate::inspect::data_completeness) fn discovery_targets_present(&self) -> CheckStatus { + CheckStatus::from_pass( + self.discovery_targets_missing_address.is_empty() + && self.discovery_targets_missing_manifest.is_empty(), + ) + } + + pub(in crate::inspect::data_completeness) fn active_event_lineage_retained( + &self, + ) -> CheckStatus { + CheckStatus::from_pass(self.active_manifest_sources_with_missing_lineage.is_empty()) + } + + pub(in crate::inspect::data_completeness) fn active_raw_logs_retained(&self) -> CheckStatus { + CheckStatus::from_pass( + self.retention_mode == RetentionMode::Minimal + || self + .active_manifest_sources_with_missing_raw_logs + .is_empty(), + ) + } + + pub(in crate::inspect::data_completeness) fn normalization_healthy(&self) -> CheckStatus { + CheckStatus::from_pass(self.failed_replay_cursors.is_empty()) + } + + pub(in crate::inspect::data_completeness) fn normalization_caught_up(&self) -> CheckStatus { + CheckStatus::from_pass( + self.active_deployment_profile.is_some() + && self.lagging_replay_cursors.is_empty() + && self.chains_missing_raw_fact_cursor.is_empty(), + ) + } + + pub(in crate::inspect::data_completeness) fn projection_drained(&self) -> CheckStatus { + CheckStatus::from_pass( + self.lagging_projection_cursors.is_empty() + && !self.projection_apply_cursor_missing + && self.projection_apply_cursor_ahead_by.is_none(), + ) + } + + pub(in crate::inspect::data_completeness) fn projection_invalidations_drained( + &self, + ) -> CheckStatus { + CheckStatus::from_pass(self.pending_projection_invalidation_count == 0) + } + + pub(in crate::inspect::data_completeness) fn projection_no_dead_letters(&self) -> CheckStatus { + CheckStatus::from_pass(self.projection_invalidation_dead_letter_count == 0) + } + + pub(in crate::inspect::data_completeness) fn projection_replay_complete(&self) -> CheckStatus { + CheckStatus::from_pass(self.missing_projection_replay_markers.is_empty()) + } + + pub(in crate::inspect::data_completeness) fn projection_content_present(&self) -> CheckStatus { + CheckStatus::from_pass(self.projection_content.complete()) + } + + pub(in crate::inspect::data_completeness) fn active_dataset_non_empty(&self) -> CheckStatus { + CheckStatus::from_pass( + self.active_manifest_sources_without_events.is_empty() + && self.active_namespaces_without_names.is_empty(), + ) + } + + pub(in crate::inspect::data_completeness) fn normalized_events_chain_id_present( + &self, + ) -> CheckStatus { + CheckStatus::from_pass(self.normalized_events_null_chain_id_count == 0) + } + + pub(in crate::inspect::data_completeness) fn deferred_projection_indexes_present( + &self, + ) -> CheckStatus { + CheckStatus::from_pass(self.missing_deferred_projection_indexes.is_empty()) + } + + pub(in crate::inspect::data_completeness) fn data_complete(&self) -> bool { + [ + self.frontier_at_head(), + self.manifest_corpus_matches_repository(), + self.lineage_contiguous(), + self.history_from_declared_start(), + self.stored_lineage_backfill_coverage(), + self.watch_set_observed(), + self.manifest_declared_targets_present(), + self.discovery_targets_present(), + self.active_event_lineage_retained(), + self.active_raw_logs_retained(), + self.normalization_healthy(), + self.normalization_caught_up(), + self.projection_drained(), + self.projection_invalidations_drained(), + self.projection_no_dead_letters(), + self.projection_replay_complete(), + self.projection_content_present(), + self.active_dataset_non_empty(), + self.normalized_events_chain_id_present(), + self.deferred_projection_indexes_present(), + ] + .iter() + .all(|status| *status == CheckStatus::Pass) + } +} diff --git a/apps/worker/src/inspect/data_completeness/manifest_corpus.rs b/apps/worker/src/inspect/data_completeness/manifest_corpus.rs new file mode 100644 index 00000000..052e945d --- /dev/null +++ b/apps/worker/src/inspect/data_completeness/manifest_corpus.rs @@ -0,0 +1,179 @@ +use std::{collections::BTreeMap, path::Path}; + +use anyhow::{Context, Result}; +use bigname_manifests::load_repository; +use serde::Serialize; +use serde_json::Value; +use sqlx::{PgPool, Row}; + +#[cfg(test)] +mod tests; + +#[derive(Clone, Debug, Eq, Ord, PartialEq, PartialOrd, Serialize)] +pub(super) struct ManifestIdentity { + pub(super) namespace: String, + pub(super) source_family: String, + pub(super) chain: String, + pub(super) deployment_epoch: String, + pub(super) manifest_version: u64, +} + +#[derive(Clone, Debug, Eq, PartialEq)] +struct ManifestEvidence { + identity: ManifestIdentity, + payload: Value, +} + +#[derive(Clone, Debug, Default, Eq, PartialEq)] +pub(super) struct ManifestCorpusInspection { + pub(super) repository_supplied: bool, + pub(super) verified: bool, + pub(super) repository_root: Option, + pub(super) repository_status: Option, + pub(super) repository_error: Option, + pub(super) expected_active_manifest_count: usize, + pub(super) database_active_manifest_count: usize, + pub(super) missing_active_manifests: Vec, + pub(super) unexpected_active_manifests: Vec, + pub(super) mismatched_manifest_payloads: Vec, +} + +impl ManifestCorpusInspection { + pub(super) fn complete(&self) -> bool { + !self.repository_supplied + || (self.verified + && self.missing_active_manifests.is_empty() + && self.unexpected_active_manifests.is_empty() + && self.mismatched_manifest_payloads.is_empty()) + } +} + +pub(super) async fn inspect_manifest_corpus( + pool: &PgPool, + manifests_root: Option<&Path>, +) -> Result { + let database = load_database_active_manifests(pool).await?; + let Some(root) = manifests_root else { + return Ok(ManifestCorpusInspection { + database_active_manifest_count: database.len(), + ..ManifestCorpusInspection::default() + }); + }; + + let repository = match load_repository(root) { + Ok(repository) => repository, + Err(error) => { + return Ok(ManifestCorpusInspection { + repository_supplied: true, + repository_root: Some(root.display().to_string()), + repository_error: Some(format!("{error:#}")), + database_active_manifest_count: database.len(), + ..ManifestCorpusInspection::default() + }); + } + }; + let repository_status = repository.summary().status.as_str().to_owned(); + let expected = repository + .manifests() + .iter() + .filter(|loaded| loaded.manifest.rollout_status.is_active()) + .map(|loaded| { + let manifest = &loaded.manifest; + Ok(ManifestEvidence { + identity: ManifestIdentity { + namespace: manifest.namespace.clone(), + source_family: manifest.source_family.clone(), + chain: manifest.chain.clone(), + deployment_epoch: manifest.deployment_epoch.clone(), + manifest_version: manifest.manifest_version, + }, + payload: serde_json::to_value(manifest) + .context("failed to serialize expected active manifest")?, + }) + }) + .collect::>>()?; + + let mut inspection = compare_manifests(expected, database); + inspection.repository_supplied = true; + inspection.verified = + repository.summary().status == bigname_manifests::ManifestLoadStatus::Loaded; + inspection.repository_root = Some(repository.root().display().to_string()); + inspection.repository_status = Some(repository_status); + Ok(inspection) +} + +fn compare_manifests( + expected: Vec, + database: Vec, +) -> ManifestCorpusInspection { + let expected_count = expected.len(); + let database_count = database.len(); + let expected = expected + .into_iter() + .map(|manifest| (manifest.identity.clone(), manifest.payload)) + .collect::>(); + let database = database + .into_iter() + .map(|manifest| (manifest.identity.clone(), manifest.payload)) + .collect::>(); + let missing_active_manifests = expected + .keys() + .filter(|identity| !database.contains_key(*identity)) + .cloned() + .collect(); + let unexpected_active_manifests = database + .keys() + .filter(|identity| !expected.contains_key(*identity)) + .cloned() + .collect(); + let mismatched_manifest_payloads = expected + .iter() + .filter(|(identity, payload)| { + database + .get(*identity) + .is_some_and(|database_payload| database_payload != *payload) + }) + .map(|(identity, _)| identity.clone()) + .collect(); + + ManifestCorpusInspection { + expected_active_manifest_count: expected_count, + database_active_manifest_count: database_count, + missing_active_manifests, + unexpected_active_manifests, + mismatched_manifest_payloads, + ..ManifestCorpusInspection::default() + } +} + +async fn load_database_active_manifests(pool: &PgPool) -> Result> { + let rows = sqlx::query( + r#" + SELECT namespace, source_family, chain, deployment_epoch, manifest_version, + manifest_payload + FROM manifest_versions + WHERE rollout_status = 'active'::manifest_rollout_status + ORDER BY namespace, source_family, chain, deployment_epoch, manifest_version + "#, + ) + .fetch_all(pool) + .await + .context("failed to load active database manifests for corpus inspection")?; + + rows.into_iter() + .map(|row| { + let manifest_version = row.try_get::("manifest_version")?; + Ok(ManifestEvidence { + identity: ManifestIdentity { + namespace: row.try_get("namespace")?, + source_family: row.try_get("source_family")?, + chain: row.try_get("chain")?, + deployment_epoch: row.try_get("deployment_epoch")?, + manifest_version: u64::try_from(manifest_version) + .context("active manifest version must be non-negative")?, + }, + payload: row.try_get("manifest_payload")?, + }) + }) + .collect() +} diff --git a/apps/worker/src/inspect/data_completeness/manifest_corpus/tests.rs b/apps/worker/src/inspect/data_completeness/manifest_corpus/tests.rs new file mode 100644 index 00000000..5aa7fe6e --- /dev/null +++ b/apps/worker/src/inspect/data_completeness/manifest_corpus/tests.rs @@ -0,0 +1,140 @@ +use std::{ + fs, + path::{Path, PathBuf}, + sync::atomic::{AtomicU64, Ordering}, + time::{SystemTime, UNIX_EPOCH}, +}; + +use anyhow::{Context, Result}; +use bigname_manifests::{load_repository, sync_repository}; +use bigname_test_support::{TestDatabase, TestDatabaseConfig}; + +use super::inspect_manifest_corpus; + +static NEXT_TEST_ID: AtomicU64 = AtomicU64::new(0); + +struct TestManifestRoot(PathBuf); + +impl TestManifestRoot { + fn new() -> Result { + let unique = SystemTime::now() + .duration_since(UNIX_EPOCH) + .context("system clock is before unix epoch")? + .as_nanos(); + let sequence = NEXT_TEST_ID.fetch_add(1, Ordering::Relaxed); + let root = std::env::temp_dir().join(format!( + "bigname-worker-manifest-corpus-{}-{unique}-{sequence}", + std::process::id() + )); + let directory = root.join("ens/ens_v2_registry_l1"); + fs::create_dir_all(&directory)?; + fs::write(directory.join("v1.toml"), manifest_contents())?; + Ok(Self(root)) + } + + fn path(&self) -> &Path { + &self.0 + } +} + +impl Drop for TestManifestRoot { + fn drop(&mut self) { + let _ = fs::remove_dir_all(&self.0); + } +} + +fn manifest_contents() -> &'static str { + r#" +manifest_version = 1 +namespace = "ens" +source_family = "ens_v2_registry_l1" +chain = "ethereum-sepolia" +deployment_epoch = "ens_v2_sepolia_dev" +rollout_status = "active" +normalizer_version = "ensip15@ens-normalize-0.1.1" + +[capability_flags] +declared_children = "supported" + +[[roots]] +name = "RootRegistry" +address = "0x0000000000000000000000000000000000000001" + +[[contracts]] +role = "registry" +address = "0x0000000000000000000000000000000000000002" +proxy_kind = "none" + +[[discovery_rules]] +edge_kind = "subregistry" +from_role = "registry" +admission = "reachable_from_root" +"# +} + +async fn test_database() -> Result { + TestDatabase::create_migrated( + TestDatabaseConfig::new("worker_manifest_corpus_inspection") + .admin_database("postgres") + .pool_max_connections(5) + .parse_context("failed to parse database URL for manifest corpus test") + .admin_connect_context("failed to connect manifest corpus admin pool") + .pool_connect_context("failed to connect manifest corpus test pool"), + &bigname_storage::MIGRATOR, + "failed to migrate manifest corpus test database", + ) + .await +} + +#[tokio::test] +async fn disk_and_database_active_manifest_corpus_match_bidirectionally() -> Result<()> { + let database = test_database().await?; + let root = TestManifestRoot::new()?; + + let missing = inspect_manifest_corpus(database.pool(), Some(root.path())).await?; + assert!(!missing.complete()); + assert_eq!(missing.missing_active_manifests.len(), 1); + assert!(missing.unexpected_active_manifests.is_empty()); + + sqlx::query( + r#" + INSERT INTO manifest_versions + (manifest_version, namespace, source_family, chain, deployment_epoch, + rollout_status, normalizer_version, file_path, manifest_payload) + VALUES + (1, 'basenames', 'unexpected', 'base-mainnet', 'unexpected', + 'active', 'n', 'unexpected.toml', '{}'::jsonb) + "#, + ) + .execute(database.pool()) + .await?; + let both_directions = inspect_manifest_corpus(database.pool(), Some(root.path())).await?; + assert!(!both_directions.complete()); + assert_eq!(both_directions.missing_active_manifests.len(), 1); + assert_eq!(both_directions.unexpected_active_manifests.len(), 1); + + sqlx::query("DELETE FROM manifest_versions") + .execute(database.pool()) + .await?; + let repository = load_repository(root.path())?; + sync_repository(database.pool(), &repository).await?; + let matching = inspect_manifest_corpus(database.pool(), Some(root.path())).await?; + assert!(matching.complete()); + assert!(matching.verified); + assert_eq!(matching.expected_active_manifest_count, 1); + assert_eq!(matching.database_active_manifest_count, 1); + + sqlx::query("UPDATE manifest_versions SET manifest_payload = '{}'::jsonb") + .execute(database.pool()) + .await?; + let mismatched = inspect_manifest_corpus(database.pool(), Some(root.path())).await?; + assert!(!mismatched.complete()); + assert_eq!(mismatched.mismatched_manifest_payloads.len(), 1); + + let unanchored = inspect_manifest_corpus(database.pool(), None).await?; + assert!(unanchored.complete()); + assert!(!unanchored.repository_supplied); + assert!(!unanchored.verified); + + database.cleanup().await +} diff --git a/apps/worker/src/inspect/data_completeness/projection_content.rs b/apps/worker/src/inspect/data_completeness/projection_content.rs new file mode 100644 index 00000000..fc226554 --- /dev/null +++ b/apps/worker/src/inspect/data_completeness/projection_content.rs @@ -0,0 +1,171 @@ +use std::collections::{BTreeMap, BTreeSet}; + +use anyhow::Result; +use bigname_storage::ActiveManifestEventSource; +use sqlx::PgPool; + +use crate::replay::ALL_CURRENT_PROJECTION_ORDER; + +#[cfg(test)] +mod tests; + +const CHILDREN_EVENT_KINDS: &[&str] = &[ + "ParentChanged", + "RegistrationGranted", + "RegistrationReleased", + "RegistrationRenewed", + "SubregistryChanged", +]; +const PERMISSIONS_EVENT_KINDS: &[&str] = &[ + "PermissionChanged", + "PermissionScopeChanged", + "RootPermissionChanged", +]; +const RECORD_INVENTORY_EVENT_KINDS: &[&str] = + &["RecordChanged", "RecordVersionChanged", "ResolverChanged"]; +const RESOLVER_EVENT_KINDS: &[&str] = &[ + "AliasChanged", + "PermissionChanged", + "PermissionScopeChanged", + "ResolverChanged", +]; +const ADDRESS_NAMES_EVENT_KINDS: &[&str] = &[ + "AuthorityEpochChanged", + "AuthorityTransferred", + "PermissionChanged", + "PermissionScopeChanged", + "RegistrationGranted", + "TokenControlTransferred", + "TokenRegenerated", +]; +const PRIMARY_NAME_EVENT_KINDS: &[&str] = &["ReverseChanged"]; + +#[derive(Clone, Debug, Eq, PartialEq)] +pub(super) struct ProjectionScopeCount { + pub(super) scope: String, + pub(super) count: i64, +} + +#[derive(Clone, Debug, Eq, PartialEq)] +pub(super) struct ProjectionTableContent { + pub(super) projection: String, + pub(super) scope_kind: &'static str, + pub(super) raw_total_count: i64, + pub(super) raw_scoped_counts: Vec, + pub(super) servable_total_count: i64, + pub(super) servable_scoped_counts: Vec, + pub(super) expected_scopes: Vec, + pub(super) missing_scopes: Vec, +} + +#[derive(Clone, Debug, Default, Eq, PartialEq)] +pub(super) struct ProjectionContentInspection { + pub(super) tables: Vec, +} + +impl ProjectionContentInspection { + pub(super) fn complete(&self) -> bool { + self.tables.len() == ALL_CURRENT_PROJECTION_ORDER.len() + && self + .tables + .iter() + .zip(ALL_CURRENT_PROJECTION_ORDER) + .all(|(table, expected)| { + table.projection == *expected && table.missing_scopes.is_empty() + }) + } +} + +pub(super) async fn load_projection_content( + pool: &PgPool, + sources: &[ActiveManifestEventSource], +) -> Result { + let mut tables = Vec::with_capacity(ALL_CURRENT_PROJECTION_ORDER.len()); + for projection in ALL_CURRENT_PROJECTION_ORDER { + tables.push(load_projection_table(pool, projection, sources).await?); + } + Ok(ProjectionContentInspection { tables }) +} + +async fn load_projection_table( + pool: &PgPool, + projection: &str, + sources: &[ActiveManifestEventSource], +) -> Result { + let counts = bigname_storage::load_projection_content_counts(pool, projection).await?; + let raw_scoped_counts = counts + .raw_scoped_counts + .into_iter() + .map(|entry| ProjectionScopeCount { + scope: entry.scope, + count: entry.count, + }) + .collect::>(); + let servable_scoped_counts = counts + .servable_scoped_counts + .into_iter() + .map(|entry| ProjectionScopeCount { + scope: entry.scope, + count: entry.count, + }) + .collect::>(); + + let expected_scopes = expected_scopes(projection, sources); + let counts_by_scope = servable_scoped_counts + .iter() + .map(|entry| (entry.scope.as_str(), entry.count)) + .collect::>(); + let missing_scopes = expected_scopes + .iter() + .filter(|scope| counts_by_scope.get(scope.as_str()).copied().unwrap_or(0) == 0) + .cloned() + .collect(); + + Ok(ProjectionTableContent { + projection: projection.to_owned(), + scope_kind: counts.scope_kind, + raw_total_count: counts.raw_total_count, + raw_scoped_counts, + servable_total_count: counts.servable_total_count, + servable_scoped_counts, + expected_scopes, + missing_scopes, + }) +} + +fn expected_scopes(projection: &str, sources: &[ActiveManifestEventSource]) -> Vec { + let feeds_projection = |source: &&ActiveManifestEventSource| match projection { + // `name_current` starts from canonical name surfaces. Every event-producing active + // source is therefore a namespace content expectation, matching the existing gate. + "name_current" => true, + "children_current" => source_declares_any(source, CHILDREN_EVENT_KINDS), + "permissions_current" => source_declares_any(source, PERMISSIONS_EVENT_KINDS), + "record_inventory_current" => source_declares_any(source, RECORD_INVENTORY_EVENT_KINDS), + "resolver_current" => source_declares_any(source, RESOLVER_EVENT_KINDS), + "address_names_current" => source_declares_any(source, ADDRESS_NAMES_EVENT_KINDS), + "primary_names_current" => source_declares_any(source, PRIMARY_NAME_EVENT_KINDS), + _ => false, + }; + let matching = sources.iter().filter(feeds_projection).collect::>(); + let scopes = match projection { + "name_current" | "children_current" | "address_names_current" | "primary_names_current" => { + matching + .into_iter() + .map(|source| source.namespace.clone()) + .collect::>() + } + "permissions_current" | "record_inventory_current" | "resolver_current" => matching + .into_iter() + .map(|source| source.chain.clone()) + .collect::>(), + _ => BTreeSet::new(), + }; + scopes.into_iter().collect() +} + +fn source_declares_any(source: &ActiveManifestEventSource, event_kinds: &[&str]) -> bool { + source + .normalized_event_kinds + .iter() + .any(|kind| event_kinds.contains(&kind.as_str())) +} diff --git a/apps/worker/src/inspect/data_completeness/projection_content/tests.rs b/apps/worker/src/inspect/data_completeness/projection_content/tests.rs new file mode 100644 index 00000000..f3dd9ec5 --- /dev/null +++ b/apps/worker/src/inspect/data_completeness/projection_content/tests.rs @@ -0,0 +1,318 @@ +use std::path::Path; + +use anyhow::Result; +use bigname_manifests::{load_repository, sync_repository}; +use bigname_storage::ActiveManifestEventSource; +use bigname_test_support::{TestDatabase, TestDatabaseConfig}; + +use super::load_projection_content; + +const CHAIN: &str = "ethereum-sepolia"; +const NAMESPACE: &str = "ens"; + +async fn test_database() -> Result { + TestDatabase::create_migrated( + TestDatabaseConfig::new("worker_projection_content_inspection") + .admin_database("postgres") + .pool_max_connections(5) + .parse_context("failed to parse projection content test database URL") + .admin_connect_context("failed to connect projection content admin pool") + .pool_connect_context("failed to connect projection content test pool"), + &bigname_storage::MIGRATOR, + "failed to migrate projection content test database", + ) + .await +} + +fn source(event_kinds: &[&str]) -> ActiveManifestEventSource { + ActiveManifestEventSource { + manifest_id: 1, + manifest_version: 1, + chain: CHAIN.to_owned(), + namespace: NAMESPACE.to_owned(), + source_family: "ens_v2_registry_l1".to_owned(), + normalized_event_kinds: event_kinds.iter().map(|kind| (*kind).to_owned()).collect(), + normalized_event_count: 1, + normalized_events_missing_canonical_lineage_count: 0, + normalized_events_missing_canonical_raw_log_count: 0, + } +} + +async fn seed_all_current_projections(database: &TestDatabase) -> Result<()> { + sqlx::raw_sql( + r#" + INSERT INTO name_surfaces + (logical_name_id, namespace, input_name, canonical_display_name, + normalized_name, dns_encoded_name, namehash, labelhashes, + normalizer_version, chain_id, block_hash, block_number, canonicality_state) + VALUES + ('ens:eth', 'ens', 'eth', 'eth', 'eth', '\x'::bytea, '0xparent', '{}', + 'test', 'ethereum-sepolia', '0xblock', 1, 'canonical'), + ('ens:alice.eth', 'ens', 'alice.eth', 'alice.eth', 'alice.eth', '\x'::bytea, + '0xchild', '{}', 'test', 'ethereum-sepolia', '0xblock', 1, 'canonical'); + + INSERT INTO resources + (resource_id, chain_id, block_hash, block_number, canonicality_state) + VALUES + ('11111111-1111-1111-1111-111111111111', 'ethereum-sepolia', + '0xblock', 1, 'canonical'); + + INSERT INTO surface_bindings + (surface_binding_id, logical_name_id, resource_id, binding_kind, active_from, + chain_id, block_hash, block_number, canonicality_state) + VALUES + ('22222222-2222-2222-2222-222222222222', 'ens:eth', + '11111111-1111-1111-1111-111111111111', 'declared_registry_path', now(), + 'ethereum-sepolia', '0xblock', 1, 'canonical'); + + INSERT INTO name_current + (logical_name_id, namespace, canonical_display_name, normalized_name, namehash, + manifest_version) + VALUES ('ens:eth', 'ens', 'eth', 'eth', '0xparent', 1); + + INSERT INTO children_current + (parent_logical_name_id, child_logical_name_id, namespace, + canonical_display_name, normalized_name, namehash, manifest_version) + VALUES ('ens:eth', 'ens:alice.eth', 'ens', 'alice.eth', 'alice.eth', '0xchild', 1); + + INSERT INTO permissions_current + (resource_id, subject, scope, scope_kind, manifest_version) + VALUES + ('11111111-1111-1111-1111-111111111111', '0xsubject', 'resource', 'resource', 1); + + INSERT INTO record_inventory_current + (resource_id, record_version_boundary_key, manifest_version) + VALUES ('11111111-1111-1111-1111-111111111111', 'boundary', 1); + + INSERT INTO resolver_current (chain_id, resolver_address, manifest_version) + VALUES ('ethereum-sepolia', '0xresolver', 1); + + INSERT INTO address_names_current + (address, logical_name_id, relation, namespace, canonical_display_name, + normalized_name, namehash, surface_binding_id, resource_id, binding_kind, + manifest_version) + VALUES + ('0xowner', 'ens:eth', 'effective_controller', 'ens', 'eth', 'eth', '0xparent', + '22222222-2222-2222-2222-222222222222', + '11111111-1111-1111-1111-111111111111', 'declared_registry_path', 1); + + INSERT INTO primary_names_current (address, coin_type, namespace) + VALUES ('0xowner', '60', 'ens'); + "#, + ) + .execute(database.pool()) + .await?; + Ok(()) +} + +#[tokio::test] +async fn truncating_one_non_name_projection_is_reported_by_table_and_scope() -> Result<()> { + let database = test_database().await?; + seed_all_current_projections(&database).await?; + let all_kinds = source(&[ + "SubregistryChanged", + "PermissionChanged", + "RecordChanged", + "ResolverChanged", + "RegistrationGranted", + "ReverseChanged", + ]); + + let complete = + load_projection_content(database.pool(), std::slice::from_ref(&all_kinds)).await?; + assert!(complete.complete()); + + sqlx::query("TRUNCATE resolver_current") + .execute(database.pool()) + .await?; + let truncated = load_projection_content(database.pool(), &[all_kinds]).await?; + assert!(!truncated.complete()); + let resolver = truncated + .tables + .iter() + .find(|table| table.projection == "resolver_current") + .expect("resolver_current table report"); + assert_eq!(resolver.raw_total_count, 0); + assert_eq!(resolver.servable_total_count, 0); + assert_eq!(resolver.missing_scopes, vec![CHAIN.to_owned()]); + + database.cleanup().await +} + +#[tokio::test] +async fn projection_without_declared_input_kind_may_be_empty() -> Result<()> { + let database = test_database().await?; + seed_all_current_projections(&database).await?; + sqlx::query("TRUNCATE permissions_current") + .execute(database.pool()) + .await?; + + let inspection = + load_projection_content(database.pool(), &[source(&["ResolverChanged"])]).await?; + assert!(inspection.complete()); + let permissions = inspection + .tables + .iter() + .find(|table| table.projection == "permissions_current") + .expect("permissions_current table report"); + assert_eq!(permissions.raw_total_count, 0); + assert_eq!(permissions.servable_total_count, 0); + assert!(permissions.expected_scopes.is_empty()); + assert!(permissions.missing_scopes.is_empty()); + + database.cleanup().await +} + +#[tokio::test] +async fn name_scope_requires_a_servable_surface_anchor() -> Result<()> { + let database = test_database().await?; + seed_all_current_projections(&database).await?; + let sources = [source(&["ResolverChanged"])]; + + sqlx::query( + "UPDATE name_surfaces SET canonicality_state = 'orphaned' WHERE logical_name_id = 'ens:eth'", + ) + .execute(database.pool()) + .await?; + let orphaned = load_projection_content(database.pool(), &sources).await?; + let names = orphaned + .tables + .iter() + .find(|table| table.projection == "name_current") + .expect("name_current table report"); + assert_eq!(names.raw_total_count, 1); + assert_eq!(names.servable_total_count, 0); + assert_eq!(names.missing_scopes, vec![NAMESPACE.to_owned()]); + assert!(!orphaned.complete()); + + sqlx::query( + "UPDATE name_surfaces SET canonicality_state = 'canonical' WHERE logical_name_id = 'ens:eth'", + ) + .execute(database.pool()) + .await?; + let canonical = load_projection_content(database.pool(), &sources).await?; + assert!(canonical.complete()); + + database.cleanup().await +} + +#[tokio::test] +async fn resource_projections_require_rows_on_each_expected_chain() -> Result<()> { + let database = test_database().await?; + seed_all_current_projections(&database).await?; + sqlx::raw_sql( + r#" + TRUNCATE permissions_current, record_inventory_current; + + INSERT INTO resources + (resource_id, chain_id, block_hash, block_number, canonicality_state) + VALUES + ('33333333-3333-3333-3333-333333333333', 'base-mainnet', + '0xforeign', 1, 'canonical'); + + INSERT INTO permissions_current + (resource_id, subject, scope, scope_kind, manifest_version) + VALUES + ('33333333-3333-3333-3333-333333333333', '0xsubject', 'resource', 'resource', 1); + + INSERT INTO record_inventory_current + (resource_id, record_version_boundary_key, manifest_version) + VALUES ('33333333-3333-3333-3333-333333333333', 'foreign-boundary', 1); + "#, + ) + .execute(database.pool()) + .await?; + let sources = [source(&["PermissionChanged", "RecordChanged"])]; + + let foreign_only = load_projection_content(database.pool(), &sources).await?; + for projection in ["permissions_current", "record_inventory_current"] { + let table = foreign_only + .tables + .iter() + .find(|table| table.projection == projection) + .expect("resource projection table report"); + assert_eq!(table.scope_kind, "chain"); + assert_eq!(table.missing_scopes, vec![CHAIN.to_owned()]); + assert_eq!(table.raw_scoped_counts[0].scope, "base-mainnet"); + assert_eq!(table.servable_scoped_counts[0].scope, "base-mainnet"); + } + assert!(!foreign_only.complete()); + + sqlx::raw_sql( + r#" + INSERT INTO permissions_current + (resource_id, subject, scope, scope_kind, manifest_version) + VALUES + ('11111111-1111-1111-1111-111111111111', '0xsubject', 'resource', 'resource', 1); + + INSERT INTO record_inventory_current + (resource_id, record_version_boundary_key, manifest_version) + VALUES ('11111111-1111-1111-1111-111111111111', 'active-boundary', 1); + "#, + ) + .execute(database.pool()) + .await?; + assert!( + load_projection_content(database.pool(), &sources) + .await? + .complete() + ); + + database.cleanup().await +} + +#[tokio::test] +async fn checked_in_mainnet_reverse_adapter_requires_primary_name_content() -> Result<()> { + let database = test_database().await?; + let root = Path::new(env!("CARGO_MANIFEST_DIR")).join("../../manifests/mainnet"); + let repository = load_repository(&root)?; + sync_repository(database.pool(), &repository).await?; + let adapter_kinds = bigname_adapters::adapter_normalized_event_kind_declarations(); + let read = bigname_storage::load_data_completeness_with_adapter_event_kinds( + database.pool(), + &adapter_kinds, + ) + .await?; + let reverse_source = read + .active_manifest_event_sources + .iter() + .find(|source| source.source_family == "ens_v1_reverse_l1") + .expect("checked-in ENS reverse source"); + assert_eq!( + reverse_source.normalized_event_kinds, + vec!["ReverseChanged"] + ); + for source_family in [ + "ens_v1_registrar_l1", + "ens_v1_wrapper_l1", + "basenames_base_registrar", + ] { + let source = read + .active_manifest_event_sources + .iter() + .find(|source| source.source_family == source_family) + .expect("checked-in block-derived source"); + assert!( + source + .normalized_event_kinds + .iter() + .any(|kind| kind == "PreimageObserved") + ); + } + + let inspection = + load_projection_content(database.pool(), &read.active_manifest_event_sources).await?; + let primary_names = inspection + .tables + .iter() + .find(|table| table.projection == "primary_names_current") + .expect("primary_names_current table report"); + assert!(!primary_names.expected_scopes.is_empty()); + assert!(primary_names.expected_scopes.contains(&"ens".to_owned())); + assert!(primary_names.missing_scopes.contains(&"ens".to_owned())); + assert_eq!(primary_names.raw_total_count, 0); + assert_eq!(primary_names.servable_total_count, 0); + assert!(!inspection.complete()); + + database.cleanup().await +} diff --git a/apps/worker/src/inspect/data_completeness/tests.rs b/apps/worker/src/inspect/data_completeness/tests.rs new file mode 100644 index 00000000..3a3a1c65 --- /dev/null +++ b/apps/worker/src/inspect/data_completeness/tests.rs @@ -0,0 +1,1451 @@ +use super::backfill_coverage::{ + BackfillCoverageGap, BackfillCoverageInspection, BackfillCoverageTopicDrift, +}; +use super::evaluate::{CheckStatus, DEFAULT_MAX_HEAD_LAG_BLOCKS, evaluate_data_completeness}; +use super::manifest_corpus::ManifestCorpusInspection; +use super::projection_content::{ProjectionContentInspection, ProjectionTableContent}; +use crate::inspect::RetentionMode; +use crate::replay::{ALL_CURRENT_PROJECTION_ORDER, CURRENT_PROJECTION_REPLAY_VERSION}; +use bigname_manifests::{WatchedContract, WatchedContractSource}; +use bigname_storage::{ + ActiveManifestEventSource, BackfillLifecycleRow, ChainCompletenessRow, + DEFERRED_NORMALIZED_EVENT_INDEXES, DataCompletenessRead, DiscoveryTargetMissingAddress, + DiscoveryTargetMissingManifest, ManifestChainNamespace, ManifestChainSourceFamily, + ManifestDeclaredTarget, NameCurrentCount, ObservedCodeAddress, ProjectionApplyCursorRow, + ProjectionReplayMarker, ReplayCursorRow, +}; +use uuid::Uuid; + +const CHAIN: &str = "ethereum-sepolia"; +const NAMESPACE: &str = "ens"; +const REGISTRY: &str = "0x796fff2e907449be8d5921bcc215b1b76d89d080"; +const RESOLVER: &str = "0xe99638b40e4fff0129d56f03b55b6bbc4bbe49b5"; +const APPLY_CURSOR: &str = "normalized_events_to_projection_invalidations"; +const DEPLOYMENT_PROFILE: &str = "sepolia"; + +fn manifest_ns(chain: &str, namespace: &str) -> ManifestChainNamespace { + ManifestChainNamespace { + chain: chain.to_owned(), + namespace: namespace.to_owned(), + } +} + +fn all_projection_markers(version: i32) -> Vec { + ALL_CURRENT_PROJECTION_ORDER + .iter() + .map(|projection| ProjectionReplayMarker { + replay_version: version, + projection: (*projection).to_owned(), + completed_normalized_target_block: Some(1_000), + }) + .collect() +} + +fn all_deferred_indexes() -> Vec { + DEFERRED_NORMALIZED_EVENT_INDEXES + .iter() + .map(|name| (*name).to_owned()) + .collect() +} + +fn complete_projection_content() -> ProjectionContentInspection { + ProjectionContentInspection { + tables: ALL_CURRENT_PROJECTION_ORDER + .iter() + .map(|projection| ProjectionTableContent { + projection: (*projection).to_owned(), + scope_kind: "global", + raw_total_count: 1, + raw_scoped_counts: vec![], + servable_total_count: 1, + servable_scoped_counts: vec![], + expected_scopes: vec![], + missing_scopes: vec![], + }) + .collect(), + } +} + +fn watched(address: &str, source: WatchedContractSource) -> WatchedContract { + WatchedContract { + chain: CHAIN.to_owned(), + source_family: "ens_v2_registry_l1".to_owned(), + address: address.to_owned(), + contract_instance_id: Uuid::nil(), + source, + source_manifest_id: None, + active_from_block_number: Some(1), + active_to_block_number: None, + } +} + +fn observed(address: &str) -> ObservedCodeAddress { + ObservedCodeAddress { + chain_id: CHAIN.to_owned(), + address: address.to_owned(), + max_observed_block_number: 1_000, + } +} + +fn manifest_target(address: &str, start: Option) -> ManifestDeclaredTarget { + ManifestDeclaredTarget { + chain: CHAIN.to_owned(), + source_family: "ens_v2_registry_l1".to_owned(), + address: address.to_owned(), + active_from_block_number: start, + } +} + +fn active_event_source( + manifest_id: i64, + source_family: &str, + normalized_event_count: i64, +) -> ActiveManifestEventSource { + ActiveManifestEventSource { + manifest_id, + manifest_version: 1, + chain: CHAIN.to_owned(), + namespace: NAMESPACE.to_owned(), + source_family: source_family.to_owned(), + normalized_event_kinds: vec!["ResolverChanged".to_owned()], + normalized_event_count, + normalized_events_missing_canonical_lineage_count: 0, + normalized_events_missing_canonical_raw_log_count: 0, + } +} + +fn names(namespace: &str, count: i64) -> NameCurrentCount { + NameCurrentCount { + namespace: namespace.to_owned(), + count, + } +} + +fn chain_row( + canonical: i64, + lineage_head: i64, + floor: i64, + block_count: i64, +) -> ChainCompletenessRow { + ChainCompletenessRow { + chain_id: CHAIN.to_owned(), + canonical_block_number: Some(canonical), + checkpoint_canonical_lineage_match: true, + lineage_head_block_number: Some(lineage_head), + lineage_floor_block_number: Some(floor), + lineage_canonical_block_count: block_count, + duplicate_canonical_height_count: 0, + disconnected_canonical_parent_count: 0, + canonical_raw_log_head_block_number: Some(floor), + raw_log_head_block_number: Some(floor), + } +} + +// A raw-fact cursor caught up to `target`: replay is complete when next > target. +fn replay_cursor(target: i64, failure: Option<&str>) -> ReplayCursorRow { + ReplayCursorRow { + deployment_profile: DEPLOYMENT_PROFILE.to_owned(), + chain_id: CHAIN.to_owned(), + cursor_kind: "raw_fact_normalized_events".to_owned(), + range_start_block_number: 0, + next_block_number: Some(target + 1), + target_block_number: Some(target), + last_completed_block_number: Some(target), + last_failure_reason: failure.map(str::to_owned), + } +} + +// A cursor whose `next` was rewound below `target` while `last_completed` stays high. +fn rewound_cursor(next: i64, target: i64, last_completed: i64) -> ReplayCursorRow { + ReplayCursorRow { + deployment_profile: DEPLOYMENT_PROFILE.to_owned(), + chain_id: CHAIN.to_owned(), + cursor_kind: "raw_fact_normalized_events".to_owned(), + range_start_block_number: 0, + next_block_number: Some(next), + target_block_number: Some(target), + last_completed_block_number: Some(last_completed), + last_failure_reason: None, + } +} + +// A backlog cursor caught up to `target` when `next > target`. +fn backlog_cursor(range_start: i64, next: i64, target: i64) -> ReplayCursorRow { + ReplayCursorRow { + deployment_profile: DEPLOYMENT_PROFILE.to_owned(), + chain_id: CHAIN.to_owned(), + cursor_kind: "post_replay_live_adapter_backlog".to_owned(), + range_start_block_number: range_start, + next_block_number: Some(next), + target_block_number: Some(target), + last_completed_block_number: Some(next - 1), + last_failure_reason: None, + } +} + +fn apply_cursor(last_change_id: i64) -> ProjectionApplyCursorRow { + ProjectionApplyCursorRow { + cursor_name: APPLY_CURSOR.to_owned(), + last_change_id, + } +} + +fn healthy_read() -> DataCompletenessRead { + DataCompletenessRead { + chains: vec![chain_row(1_000, 1_000, 1, 1_000)], + active_deployment_profile: Some(DEPLOYMENT_PROFILE.to_owned()), + replay_cursors: vec![replay_cursor(1, None)], + projection_apply_cursors: vec![apply_cursor(42)], + max_projection_change_id: Some(42), + pending_projection_invalidation_count: 0, + projection_invalidation_dead_letter_count: 0, + observed_code_addresses: vec![observed(REGISTRY)], + manifest_declared_targets: vec![manifest_target(REGISTRY, Some(1))], + active_manifest_event_sources: vec![active_event_source(1, "ens_v2_registry_l1", 100)], + name_current_counts: vec![names(NAMESPACE, 10)], + normalized_events_null_chain_id_count: 0, + projection_replay_markers: all_projection_markers(CURRENT_PROJECTION_REPLAY_VERSION), + projection_replay_required_target_block: Some(1_000), + backfill_lifecycle: vec![], + present_deferred_projection_indexes: all_deferred_indexes(), + manifest_chain_namespaces: vec![manifest_ns(CHAIN, NAMESPACE)], + manifest_chain_source_families: vec![ManifestChainSourceFamily { + chain: CHAIN.to_owned(), + source_family: "ens_v2_registry_l1".to_owned(), + }], + manifest_declared_targets_missing_address: vec![], + manifest_proxy_implementations_missing_edge: vec![], + discovery_targets_missing_address: vec![], + discovery_targets_missing_manifest: vec![], + } +} + +fn evaluate( + read: &DataCompletenessRead, + watched_contracts: &[WatchedContract], +) -> super::evaluate::DataCompletenessReport { + evaluate_with_retention(read, watched_contracts, RetentionMode::Minimal) +} + +fn evaluate_with_retention( + read: &DataCompletenessRead, + watched_contracts: &[WatchedContract], + retention_mode: RetentionMode, +) -> super::evaluate::DataCompletenessReport { + evaluate_data_completeness( + read, + watched_contracts, + &BackfillCoverageInspection::default(), + &ManifestCorpusInspection::default(), + &complete_projection_content(), + DEFAULT_MAX_HEAD_LAG_BLOCKS, + retention_mode, + ) +} + +fn registry_only() -> Vec { + vec![watched(REGISTRY, WatchedContractSource::ManifestContract)] +} + +#[test] +fn healthy_database_is_data_complete() { + assert!(evaluate(&healthy_read(), ®istry_only()).data_complete()); +} + +/// Durable fetch coverage remains a promotion precondition for retained bounded-backfill +/// lineage, even when every relative storage/content check is otherwise green. +#[test] +fn uncovered_retained_backfill_span_fails() { + let read = healthy_read(); + let coverage = BackfillCoverageInspection { + gaps: vec![BackfillCoverageGap { + chain: CHAIN.to_owned(), + source_family: "ens_v2_registry_l1".to_owned(), + address: REGISTRY.to_owned(), + required_from_block: 992, + required_to_block: 1_000, + }], + topic_drifts: vec![], + }; + let report = evaluate_data_completeness( + &read, + ®istry_only(), + &coverage, + &ManifestCorpusInspection::default(), + &complete_projection_content(), + DEFAULT_MAX_HEAD_LAG_BLOCKS, + RetentionMode::Minimal, + ); + + assert_eq!(report.stored_lineage_backfill_coverage(), CheckStatus::Fail); + assert_eq!(report.backfill_coverage_gaps, coverage.gaps); + assert!(!report.data_complete()); +} + +/// Coverage facts written under a stale topic-filtered ABI cannot prove the current watched +/// input, even when the tuple interval itself is fully contained. +#[test] +fn drifted_backfill_topic_set_fails() { + let read = healthy_read(); + let coverage = BackfillCoverageInspection { + gaps: vec![], + topic_drifts: vec![BackfillCoverageTopicDrift { + chain: CHAIN.to_owned(), + required_from_block: 992, + required_to_block: 1_000, + reason: "manifest ABI topic0 set changed".to_owned(), + }], + }; + let report = evaluate_data_completeness( + &read, + ®istry_only(), + &coverage, + &ManifestCorpusInspection::default(), + &complete_projection_content(), + DEFAULT_MAX_HEAD_LAG_BLOCKS, + RetentionMode::Minimal, + ); + + assert_eq!(report.stored_lineage_backfill_coverage(), CheckStatus::Fail); + assert_eq!(report.backfill_coverage_topic_drifts, coverage.topic_drifts); + assert!(!report.data_complete()); +} + +/// The 2026-07-06 shape: every cursor reports "done" because each measures itself against the +/// previous stage's frontier, while the watch set silently excludes discovered targets. +#[test] +fn discovered_target_without_code_observation_fails_watch_set_coverage() { + let watched_contracts = vec![ + watched(REGISTRY, WatchedContractSource::ManifestContract), + watched(RESOLVER, WatchedContractSource::DiscoveryEdge), + ]; + let report = evaluate(&healthy_read(), &watched_contracts); + + assert_eq!(report.watch_set_observed(), CheckStatus::Fail); + assert_eq!(report.unobserved_targets.len(), 1); + assert_eq!(report.unobserved_targets[0].address, RESOLVER); + assert!(!report.data_complete()); + + assert_eq!(report.normalization_healthy(), CheckStatus::Pass); + assert_eq!(report.normalization_caught_up(), CheckStatus::Pass); + assert_eq!(report.projection_drained(), CheckStatus::Pass); + assert_eq!(report.active_dataset_non_empty(), CheckStatus::Pass); +} + +#[test] +fn inactive_watched_target_is_not_required_to_be_observed() { + let mut retired = watched(RESOLVER, WatchedContractSource::DiscoveryEdge); + retired.active_to_block_number = Some(500); + let watched_contracts = vec![ + watched(REGISTRY, WatchedContractSource::ManifestContract), + retired, + ]; + + assert!(evaluate(&healthy_read(), &watched_contracts).data_complete()); +} + +#[test] +fn watch_set_coverage_matches_addresses_case_insensitively() { + let watched_contracts = vec![watched( + ®ISTRY.to_ascii_uppercase(), + WatchedContractSource::ManifestContract, + )]; + assert!(evaluate(&healthy_read(), &watched_contracts).data_complete()); +} + +/// A code observation from before the target's active range does not prove the target was +/// watched after admission. +#[test] +fn pre_admission_code_observation_does_not_cover_active_target() { + let mut read = healthy_read(); + read.observed_code_addresses[0].max_observed_block_number = 99; + read.manifest_declared_targets[0].active_from_block_number = Some(100); + let mut registry = watched(REGISTRY, WatchedContractSource::ManifestContract); + registry.active_from_block_number = Some(100); + let report = evaluate(&read, &[registry]); + + assert_eq!(report.watch_set_observed(), CheckStatus::Fail); + assert_eq!(report.unobserved_targets.len(), 1); + assert!(!report.data_complete()); +} + +#[test] +fn preactivation_target_passes_coverage_with_pending_advisory() { + let mut read = healthy_read(); + read.observed_code_addresses.clear(); + read.manifest_declared_targets[0].active_from_block_number = Some(1_500); + let mut registry = watched(REGISTRY, WatchedContractSource::ManifestContract); + registry.active_from_block_number = Some(1_500); + let report = evaluate(&read, &[registry]); + + assert_eq!(report.watch_set_observed(), CheckStatus::Pass); + assert!(report.unobserved_targets.is_empty()); + assert_eq!(report.pending_activation_targets.len(), 1); + assert_eq!( + report.pending_activation_targets[0].canonical_head_block_number, + 1_000 + ); + assert!(report.data_complete()); +} + +#[test] +fn activated_target_without_observation_still_fails_coverage() { + let mut read = healthy_read(); + read.observed_code_addresses.clear(); + read.manifest_declared_targets[0].active_from_block_number = Some(500); + let mut registry = watched(REGISTRY, WatchedContractSource::ManifestContract); + registry.active_from_block_number = Some(500); + let report = evaluate(&read, &[registry]); + + assert_eq!(report.watch_set_observed(), CheckStatus::Fail); + assert!(report.pending_activation_targets.is_empty()); + assert_eq!(report.unobserved_targets.len(), 1); +} + +#[test] +fn future_start_without_canonical_head_is_not_exempted() { + let mut read = healthy_read(); + read.observed_code_addresses.clear(); + read.chains[0].canonical_block_number = None; + read.manifest_declared_targets[0].active_from_block_number = Some(1_500); + let mut registry = watched(REGISTRY, WatchedContractSource::ManifestContract); + registry.active_from_block_number = Some(1_500); + let report = evaluate(&read, &[registry]); + + assert_eq!(report.watch_set_observed(), CheckStatus::Fail); + assert!(report.pending_activation_targets.is_empty()); + assert_eq!(report.unobserved_targets.len(), 1); +} + +/// Direct manifest declarations remain coverage authority if a partial restore loses the +/// address-range row that normally materializes them into `load_watched_contracts`. +#[test] +fn manifest_declared_target_missing_from_watch_view_fails_coverage() { + let mut read = healthy_read(); + read.manifest_declared_targets + .push(manifest_target(RESOLVER, Some(1))); + let report = evaluate(&read, ®istry_only()); + + assert_eq!(report.watch_set_observed(), CheckStatus::Fail); + assert!( + report + .unobserved_targets + .iter() + .any(|target| target.address == RESOLVER) + ); + assert!(!report.data_complete()); +} + +/// The direct manifest union keeps code coverage authoritative even when the materialized +/// address row is missing. The separately named check still identifies that structural gap. +#[test] +fn manifest_declared_target_missing_matching_address_fails_named_check() { + let mut read = healthy_read(); + let resolver = manifest_target(RESOLVER, Some(1)); + read.manifest_declared_targets.push(resolver.clone()); + read.manifest_declared_targets_missing_address = vec![resolver]; + read.observed_code_addresses.push(observed(RESOLVER)); + let report = evaluate(&read, ®istry_only()); + + assert_eq!(report.watch_set_observed(), CheckStatus::Pass); + assert_eq!( + report.manifest_declared_targets_present(), + CheckStatus::Fail + ); + assert_eq!(report.manifest_targets_missing_address.len(), 1); + assert_eq!(report.manifest_targets_missing_address[0].address, RESOLVER); + assert!(!report.data_complete()); +} + +/// The proxy implementation address fallback cannot substitute for the managed source-graph +/// edge that makes the implementation part of the runtime watch view. +#[test] +fn manifest_proxy_implementation_missing_edge_fails_named_check() { + let mut read = healthy_read(); + let implementation = manifest_target(RESOLVER, Some(1)); + read.manifest_declared_targets.push(implementation.clone()); + read.manifest_proxy_implementations_missing_edge = vec![implementation]; + read.observed_code_addresses.push(observed(RESOLVER)); + let report = evaluate(&read, ®istry_only()); + + assert_eq!(report.watch_set_observed(), CheckStatus::Pass); + assert_eq!( + report.manifest_declared_targets_present(), + CheckStatus::Fail + ); + assert_eq!( + report.manifest_proxy_implementations_missing_edge[0].address, + RESOLVER + ); + assert!(!report.data_complete()); +} + +/// An active discovery edge remains watch authority even if its target address row vanished. +/// The runtime view cannot render an address, so the structural check must fail explicitly. +#[test] +fn discovery_target_missing_address_fails_named_check() { + let mut read = healthy_read(); + read.discovery_targets_missing_address = vec![DiscoveryTargetMissingAddress { + chain: CHAIN.to_owned(), + source_family: "ens_v2_resolver_l1".to_owned(), + contract_instance_id: Uuid::from_u128(42), + }]; + let report = evaluate(&read, ®istry_only()); + + assert_eq!(report.discovery_targets_present(), CheckStatus::Fail); + assert_eq!(report.discovery_targets_missing_address.len(), 1); + assert!(!report.data_complete()); +} + +/// An admitted resolver endpoint remains discovery authority when its active registry source +/// survives, even if the resolver-family target manifest needed by the watch view vanished. +#[test] +fn discovery_target_missing_resolver_manifest_fails_named_check() { + let mut read = healthy_read(); + read.discovery_targets_missing_manifest = vec![DiscoveryTargetMissingManifest { + chain: CHAIN.to_owned(), + source_family: "ens_v2_resolver_l1".to_owned(), + contract_instance_id: Uuid::from_u128(42), + }]; + let report = evaluate(&read, ®istry_only()); + + assert_eq!(report.discovery_targets_present(), CheckStatus::Fail); + assert_eq!(report.discovery_targets_missing_manifest.len(), 1); + assert!(!report.data_complete()); +} + +/// A manifest-less restore reads zero watched contracts. The coverage check is the +/// load-bearing one, so it must not pass vacuously when there is nothing to observe. +#[test] +fn empty_watch_set_fails_coverage() { + let mut read = healthy_read(); + read.manifest_declared_targets.clear(); + let report = evaluate(&read, &[]); + + assert_eq!(report.active_watched_target_count, 0); + assert_eq!(report.watch_set_observed(), CheckStatus::Fail); + assert!(!report.data_complete()); +} + +#[test] +fn reconciliation_frontier_behind_head_beyond_tolerance_fails() { + let mut read = healthy_read(); + read.chains = vec![chain_row(1_000, 900, 1, 900)]; + let report = evaluate(&read, ®istry_only()); + + assert_eq!(report.frontier_at_head(), CheckStatus::Fail); + assert_eq!(report.frontiers[0].head_lag_blocks, Some(100)); + assert!(!report.data_complete()); +} + +#[test] +fn reconciliation_frontier_within_tolerance_passes() { + let mut read = healthy_read(); + read.chains = vec![chain_row(1_004, 1_000, 1, 1_000)]; + let report = evaluate(&read, ®istry_only()); + + assert_eq!(report.frontier_at_head(), CheckStatus::Pass); +} + +#[test] +fn lineage_gap_fails_contiguity() { + let mut read = healthy_read(); + read.chains = vec![chain_row(1_000, 1_000, 1, 999)]; + let report = evaluate(&read, ®istry_only()); + + assert_eq!(report.lineage_contiguous(), CheckStatus::Fail); + assert_eq!(report.frontiers[0].missing_block_count, 1); + assert!(!report.data_complete()); +} + +/// The LabelReserved crash-loop: the cursor stops advancing and records why. +#[test] +fn replay_cursor_failure_reason_fails_normalization() { + let mut read = healthy_read(); + read.replay_cursors = vec![replay_cursor(1, Some("LabelReserved expiry exceeds i64"))]; + let report = evaluate(&read, ®istry_only()); + + assert_eq!(report.normalization_healthy(), CheckStatus::Fail); + assert_eq!(report.failed_replay_cursors.len(), 1); + assert!(!report.data_complete()); +} + +/// Failed cursor state for a retired chain is advisory state outside the active serving set. +#[test] +fn failed_replay_cursor_on_inactive_chain_is_ignored() { + let mut read = healthy_read(); + read.replay_cursors.push(ReplayCursorRow { + deployment_profile: DEPLOYMENT_PROFILE.to_owned(), + chain_id: "retired-chain".to_owned(), + cursor_kind: "raw_fact_normalized_events".to_owned(), + range_start_block_number: 0, + next_block_number: Some(1), + target_block_number: Some(10), + last_completed_block_number: None, + last_failure_reason: Some("retired failure".to_owned()), + }); + let report = evaluate(&read, ®istry_only()); + + assert_eq!(report.normalization_healthy(), CheckStatus::Pass); + assert!(report.failed_replay_cursors.is_empty()); + assert_eq!(report.ignored_replay_cursors.len(), 1); + assert!(report.data_complete()); +} + +/// A caught-up cursor from a different deployment profile cannot stand in for the profile +/// selected by the active manifest corpus. +#[test] +fn stale_profile_cursor_does_not_satisfy_active_profile() { + let mut read = healthy_read(); + read.replay_cursors[0].deployment_profile = "retired-profile".to_owned(); + let report = evaluate(&read, ®istry_only()); + + assert_eq!(report.normalization_caught_up(), CheckStatus::Fail); + assert_eq!( + report.chains_missing_raw_fact_cursor, + vec![CHAIN.to_owned()] + ); + assert_eq!(report.ignored_replay_cursors.len(), 1); + assert!(!report.data_complete()); +} + +/// An unclassifiable active manifest corpus cannot select the writer's replay cursor profile. +#[test] +fn unresolved_active_deployment_profile_fails_normalization() { + let mut read = healthy_read(); + read.active_deployment_profile = None; + let report = evaluate(&read, ®istry_only()); + + assert_eq!(report.normalization_caught_up(), CheckStatus::Fail); + assert!(!report.data_complete()); +} + +/// Backlog cursor residue cannot latch a stateless chain; current source-family policy is the +/// writer's target-refresh authority. +#[test] +fn stateless_replay_cursor_behind_head_fails_despite_backlog_residue() { + let mut read = healthy_read(); + read.manifest_chain_source_families[0].source_family = "ens_v1_reverse_l1".to_owned(); + read.chains = vec![ChainCompletenessRow { + canonical_raw_log_head_block_number: Some(900), + raw_log_head_block_number: Some(900), + ..chain_row(1_000, 1_000, 1, 1_000) + }]; + read.replay_cursors = vec![replay_cursor(800, None), backlog_cursor(801, 901, 900)]; + let report = evaluate(&read, ®istry_only()); + + assert_eq!(report.normalization_caught_up(), CheckStatus::Fail); + assert_eq!(report.lagging_replay_cursors[0].behind_by, 100); +} + +/// Replay bounds require the raw log's lineage block to be canonical. The gate must compare +/// against the canonical raw-log head, not the non-orphaned head, so trailing `observed` +/// logs that replay cannot yet consume do not read as permanent lag. +#[test] +fn replay_cursor_at_canonical_head_passes_despite_trailing_observed_logs() { + let mut read = healthy_read(); + read.chains = vec![ChainCompletenessRow { + canonical_raw_log_head_block_number: Some(1_000), + raw_log_head_block_number: Some(1_100), + ..chain_row(1_000, 1_000, 1, 1_000) + }]; + read.replay_cursors = vec![replay_cursor(1_000, None)]; + let report = evaluate(&read, ®istry_only()); + + assert_eq!(report.normalization_caught_up(), CheckStatus::Pass); + assert!(report.lagging_replay_cursors.is_empty()); +} + +/// A truncated restore can retain raw logs while dropping the replay cursor row. A missing +/// cursor produces no lag entry, so the gate must check the cursor exists. +#[test] +fn chain_with_raw_logs_but_no_replay_cursor_fails() { + let mut read = healthy_read(); + read.chains = vec![ChainCompletenessRow { + canonical_raw_log_head_block_number: Some(1_000), + raw_log_head_block_number: Some(1_000), + ..chain_row(1_000, 1_000, 1, 1_000) + }]; + read.replay_cursors = vec![]; + let report = evaluate(&read, ®istry_only()); + + assert_eq!(report.normalization_caught_up(), CheckStatus::Fail); + assert_eq!( + report.chains_missing_raw_fact_cursor, + vec![CHAIN.to_owned()] + ); + assert!(!report.data_complete()); +} + +/// A chain with no retained canonical raw logs has no cursor frontier to compare. The separate +/// event-lineage check still rejects retained active normalized events without their anchors. +#[test] +fn chain_without_canonical_raw_logs_does_not_require_a_cursor() { + let mut read = healthy_read(); + read.chains = vec![ChainCompletenessRow { + canonical_raw_log_head_block_number: None, + raw_log_head_block_number: None, + ..chain_row(1_000, 1_000, 1, 1_000) + }]; + read.replay_cursors = vec![]; + let report = evaluate(&read, ®istry_only()); + + assert_eq!(report.normalization_caught_up(), CheckStatus::Pass); + assert!(report.chains_missing_raw_fact_cursor.is_empty()); +} + +/// Derived active content is not reorg-safe when its exact canonical lineage anchors vanished. +#[test] +fn active_normalized_events_missing_lineage_fail() { + let mut read = healthy_read(); + read.active_manifest_event_sources[0].normalized_events_missing_canonical_lineage_count = 3; + let report = evaluate(&read, ®istry_only()); + + assert_eq!(report.active_event_lineage_retained(), CheckStatus::Fail); + assert_eq!( + report.active_manifest_sources_with_missing_lineage[0].missing_canonical_lineage_count, + 3 + ); + assert!(!report.data_complete()); +} + +#[test] +fn missing_raw_log_is_mode_dependent() { + let mut read = healthy_read(); + read.active_manifest_event_sources[0].normalized_events_missing_canonical_raw_log_count = 1; + + let minimal = evaluate_with_retention(&read, ®istry_only(), RetentionMode::Minimal); + assert_eq!(minimal.active_raw_logs_retained(), CheckStatus::Pass); + assert!(minimal.data_complete()); + + let log_audit = evaluate_with_retention(&read, ®istry_only(), RetentionMode::LogAudit); + assert_eq!(log_audit.active_raw_logs_retained(), CheckStatus::Fail); + assert_eq!( + log_audit.active_manifest_sources_with_missing_raw_logs[0].missing_canonical_raw_log_count, + 1 + ); + assert!(!log_audit.data_complete()); +} + +#[test] +fn projection_apply_cursor_behind_max_change_fails() { + let mut read = healthy_read(); + read.projection_apply_cursors = vec![apply_cursor(40)]; + read.max_projection_change_id = Some(42); + let report = evaluate(&read, ®istry_only()); + + assert_eq!(report.projection_drained(), CheckStatus::Fail); + assert_eq!(report.lagging_projection_cursors[0].behind_by, 2); + assert!(!report.data_complete()); +} + +/// The derive worker scans only change IDs above its cursor. A restored cursor above the +/// retained high watermark would skip every retained row at or below that stale watermark. +#[test] +fn projection_apply_cursor_ahead_of_change_log_fails() { + let mut read = healthy_read(); + read.projection_apply_cursors = vec![apply_cursor(50)]; + read.max_projection_change_id = Some(42); + let report = evaluate(&read, ®istry_only()); + + assert_eq!(report.projection_apply_cursor_ahead_by, Some(8)); + assert_eq!(report.projection_drained(), CheckStatus::Fail); + assert!(!report.data_complete()); +} + +#[test] +fn nonzero_projection_apply_cursor_with_empty_change_log_fails() { + let mut read = healthy_read(); + read.projection_apply_cursors = vec![apply_cursor(50)]; + read.max_projection_change_id = None; + let report = evaluate(&read, ®istry_only()); + + assert_eq!(report.projection_apply_cursor_ahead_by, Some(50)); + assert_eq!(report.projection_drained(), CheckStatus::Fail); + assert!(!report.data_complete()); +} + +/// A non-empty change log with no apply cursor row means nothing has consumed it; the old +/// CROSS JOIN returned no rows and passed vacuously. +#[test] +fn non_empty_change_log_without_apply_cursor_fails() { + let mut read = healthy_read(); + read.projection_apply_cursors = vec![]; + read.max_projection_change_id = Some(42); + let report = evaluate(&read, ®istry_only()); + + assert!(report.projection_apply_cursor_missing); + assert_eq!(report.projection_drained(), CheckStatus::Fail); + assert!(!report.data_complete()); +} + +/// The continuous projection worker consumes only its named cursor. A stale cursor for some +/// other consumer cannot prove that normalized-event changes have been scanned. +#[test] +fn unrelated_apply_cursor_does_not_satisfy_non_empty_change_log() { + let mut read = healthy_read(); + read.projection_apply_cursors = vec![ProjectionApplyCursorRow { + cursor_name: "retired_projection_consumer".to_owned(), + last_change_id: 42, + }]; + read.max_projection_change_id = Some(42); + let report = evaluate(&read, ®istry_only()); + + assert!(report.projection_apply_cursor_missing); + assert_eq!(report.projection_drained(), CheckStatus::Fail); + assert!(!report.data_complete()); +} + +/// Cursor equal to the derive-scan frontier only means the scan finished. Pending +/// invalidations are unapplied projection work. +#[test] +fn pending_projection_invalidation_fails() { + let mut read = healthy_read(); + read.pending_projection_invalidation_count = 1; + let report = evaluate(&read, ®istry_only()); + + // The derive scan finished (cursor == max change id), but invalidations remain unapplied. + assert_eq!(report.projection_drained(), CheckStatus::Pass); + assert_eq!(report.projection_invalidations_drained(), CheckStatus::Fail); + assert!(!report.data_complete()); +} + +#[test] +fn projection_invalidation_dead_letter_fails() { + let mut read = healthy_read(); + read.projection_invalidation_dead_letter_count = 1; + let report = evaluate(&read, ®istry_only()); + + assert_eq!(report.projection_no_dead_letters(), CheckStatus::Fail); + assert!(!report.data_complete()); +} + +/// An empty database drains every queue trivially; only the content check catches it. +#[test] +fn empty_projections_fail_even_when_every_cursor_is_drained() { + let mut read = healthy_read(); + read.active_manifest_event_sources[0].normalized_event_count = 0; + read.name_current_counts = vec![]; + read.projection_apply_cursors = vec![apply_cursor(0)]; + read.max_projection_change_id = None; + let report = evaluate(&read, ®istry_only()); + + assert_eq!(report.projection_drained(), CheckStatus::Pass); + assert_eq!(report.active_dataset_non_empty(), CheckStatus::Fail); + assert_eq!( + report.active_manifest_sources_without_events[0].chain, + CHAIN + ); + assert!(!report.data_complete()); +} + +/// The wave-2 zero floor false-failed live databases, where reconcile commits canonical +/// lineage before advancing the checkpoint. A small negative lag is tolerated; a larger one +/// (a genuinely stale checkpoint writer) still fails. +#[test] +fn small_negative_head_lag_is_tolerated() { + let mut read = healthy_read(); + read.chains = vec![chain_row(996, 1_000, 1, 1_000)]; + let report = evaluate(&read, ®istry_only()); + + assert_eq!(report.frontiers[0].head_lag_blocks, Some(-4)); + assert_eq!(report.frontier_at_head(), CheckStatus::Pass); +} + +/// A checkpoint number at the lineage head is not sufficient when its stored hash does not +/// resolve to the retained serving-canonical branch anchor used by later reconciliation. +#[test] +fn checkpoint_hash_without_canonical_lineage_anchor_fails_frontier() { + let mut read = healthy_read(); + read.chains[0].checkpoint_canonical_lineage_match = false; + let report = evaluate(&read, ®istry_only()); + + assert_eq!(report.frontiers[0].head_lag_blocks, Some(0)); + assert!(!report.frontiers[0].checkpoint_canonical_lineage_match); + assert_eq!(report.frontier_at_head(), CheckStatus::Fail); + assert!(!report.data_complete()); +} + +/// Reconcile may commit a full live contiguous gap before advancing the checkpoint, so the +/// writer's admitted gap-fill range is a healthy negative lag. +#[test] +fn lineage_ahead_within_writer_gap_limit_passes_frontier() { + let mut read = healthy_read(); + read.chains = vec![chain_row(900, 1_000, 1, 1_000)]; + let report = evaluate(&read, ®istry_only()); + + assert_eq!(report.frontiers[0].head_lag_blocks, Some(-100)); + assert_eq!(report.frontier_at_head(), CheckStatus::Pass); +} + +#[test] +fn lineage_ahead_beyond_writer_gap_limit_fails_frontier() { + let mut read = healthy_read(); + let lineage_head = bigname_storage::MAX_LIVE_CONTIGUOUS_GAP_FILL_BLOCKS + 2; + read.chains = vec![chain_row(1, lineage_head, 1, lineage_head)]; + let report = evaluate(&read, ®istry_only()); + + assert_eq!( + report.frontiers[0].head_lag_blocks, + Some(-(bigname_storage::MAX_LIVE_CONTIGUOUS_GAP_FILL_BLOCKS + 1)) + ); + assert_eq!(report.frontier_at_head(), CheckStatus::Fail); +} + +/// A chain the active watch set declares but that is absent from checkpoints and lineage +/// produces no storage row, so every per-chain check would pass by absence. +#[test] +fn manifest_declared_chain_without_storage_fails_frontier() { + let mut read = healthy_read(); + // Storage has a foreign chain; the watched registry chain has no row at all. + read.chains = vec![ChainCompletenessRow { + chain_id: "base-mainnet".to_owned(), + ..chain_row(1_000, 1_000, 1, 1_000) + }]; + let report = evaluate(&read, ®istry_only()); + + let synthesized = report + .frontiers + .iter() + .find(|frontier| frontier.chain_id == CHAIN) + .expect("synthesized frontier for the declared chain"); + assert!(synthesized.missing_from_storage); + assert_eq!(synthesized.head_lag_blocks, None); + assert_eq!(report.frontier_at_head(), CheckStatus::Fail); + assert!(!report.data_complete()); +} + +/// Two non-orphaned canonical hashes at one height is a canonicality violation the distinct +/// block-number contiguity count cannot see. +#[test] +fn duplicate_canonical_height_fails_contiguity() { + let mut read = healthy_read(); + read.chains = vec![ChainCompletenessRow { + duplicate_canonical_height_count: 1, + ..chain_row(1_000, 1_000, 1, 1_000) + }]; + let report = evaluate(&read, ®istry_only()); + + assert_eq!(report.lineage_contiguous(), CheckStatus::Fail); + assert_eq!(report.frontiers[0].duplicate_canonical_height_count, 1); + assert!(!report.data_complete()); +} + +/// A height-complete lineage can still be disconnected when a child points at a different +/// parent hash than the canonical row at the preceding height. +#[test] +fn disconnected_canonical_parent_fails_contiguity() { + let mut read = healthy_read(); + read.chains = vec![ChainCompletenessRow { + disconnected_canonical_parent_count: 1, + ..chain_row(1_000, 1_000, 1, 1_000) + }]; + let report = evaluate(&read, ®istry_only()); + + assert_eq!(report.lineage_contiguous(), CheckStatus::Fail); + assert!(!report.data_complete()); +} + +/// A live-tail-only database is internally consistent — contiguous span, caught-up cursors, +/// non-empty projections — but its lineage floor sits above the earliest declared start, so +/// history is truncated. +#[test] +fn lineage_floor_above_declared_start_fails_history() { + let mut read = healthy_read(); + read.chains = vec![chain_row(1_000, 1_000, 900, 101)]; + read.replay_cursors = vec![replay_cursor(1_000, None)]; + // Registry declares a start at block 500, below the retained lineage floor of 900. + read.manifest_declared_targets[0].active_from_block_number = Some(500); + let mut early = watched(REGISTRY, WatchedContractSource::ManifestContract); + early.active_from_block_number = Some(500); + let report = evaluate(&read, &[early]); + + assert_eq!(report.history_from_declared_start(), CheckStatus::Fail); + assert_eq!(report.chains_history_truncated.len(), 1); + assert_eq!(report.chains_history_truncated[0].declared_start_block, 500); + assert_eq!( + report.chains_history_truncated[0].lineage_floor_block, + Some(900) + ); + // The gate would otherwise pass: the truncated span is itself contiguous. + assert_eq!(report.lineage_contiguous(), CheckStatus::Pass); + assert!(!report.data_complete()); +} + +/// A materialized address row may narrow the runtime watch range, but it cannot raise the +/// manifest's historical start. History continues to use the direct declaration. +#[test] +fn watched_address_start_does_not_raise_manifest_history_start() { + let mut read = healthy_read(); + read.chains = vec![chain_row(1_000, 1_000, 500, 501)]; + read.replay_cursors = vec![replay_cursor(1_000, None)]; + read.manifest_declared_targets[0].active_from_block_number = Some(100); + let mut narrowed_watch = watched(REGISTRY, WatchedContractSource::ManifestContract); + narrowed_watch.active_from_block_number = Some(900); + let report = evaluate(&read, &[narrowed_watch]); + + assert_eq!(report.watch_set_observed(), CheckStatus::Pass); + assert_eq!(report.history_from_declared_start(), CheckStatus::Fail); + assert_eq!(report.chains_history_truncated[0].declared_start_block, 100); + assert_eq!( + report.chains_history_truncated[0].lineage_floor_block, + Some(500) + ); + assert!(!report.data_complete()); +} + +/// A chain whose active targets are all open-ended has no finite start to establish a floor, +/// so the history check fails closed rather than passing vacuously. +#[test] +fn chain_with_only_open_ended_starts_fails_history() { + let mut read = healthy_read(); + read.chains = vec![chain_row(1_000, 1_000, 900, 101)]; + read.replay_cursors = vec![replay_cursor(1_000, None)]; + read.manifest_declared_targets[0].active_from_block_number = None; + let mut open_ended = watched(REGISTRY, WatchedContractSource::ManifestContract); + open_ended.active_from_block_number = None; + let report = evaluate(&read, &[open_ended]); + + assert_eq!(report.history_from_declared_start(), CheckStatus::Fail); + assert_eq!(report.chains_without_finite_start[0].chain, CHAIN); + assert_eq!( + report.chains_without_finite_start[0].open_ended_target_count, + 1 + ); + assert!(!report.data_complete()); +} + +/// A chain with at least one finite start still uses that as the floor, ignoring open-ended +/// siblings. +#[test] +fn mixed_starts_use_the_finite_floor() { + let mut read = healthy_read(); + read.chains = vec![chain_row(1_000, 1_000, 1, 1_000)]; + let finite = watched(REGISTRY, WatchedContractSource::ManifestContract); + let mut open_ended = watched(RESOLVER, WatchedContractSource::DiscoveryEdge); + open_ended.active_from_block_number = None; + read.observed_code_addresses = vec![observed(REGISTRY), observed(RESOLVER)]; + let report = evaluate(&read, &[finite, open_ended]); + + assert_eq!(report.history_from_declared_start(), CheckStatus::Pass); + assert!(report.chains_without_finite_start.is_empty()); +} + +/// Rows from another chain satisfy a global count while a newly active chain has zero. The +/// content check must be scoped to the active dataset. +#[test] +fn foreign_chain_content_does_not_satisfy_an_empty_active_chain() { + let mut read = healthy_read(); + read.active_manifest_event_sources[0].normalized_event_count = 0; + read.active_manifest_event_sources + .push(ActiveManifestEventSource { + manifest_id: 2, + manifest_version: 1, + chain: "base-mainnet".to_owned(), + namespace: NAMESPACE.to_owned(), + source_family: "foreign_registry".to_owned(), + normalized_event_kinds: vec!["ResolverChanged".to_owned()], + normalized_event_count: 500, + normalized_events_missing_canonical_lineage_count: 0, + normalized_events_missing_canonical_raw_log_count: 0, + }); + read.name_current_counts = vec![names(NAMESPACE, 20)]; + let report = evaluate(&read, ®istry_only()); + + assert_eq!(report.active_dataset_non_empty(), CheckStatus::Fail); + assert_eq!( + report.active_manifest_sources_without_events[0].chain, + CHAIN + ); + assert_eq!( + report.active_manifest_sources_without_events[0].namespace, + NAMESPACE + ); + assert!(!report.data_complete()); +} + +/// Rows from a deprecated manifest ID cannot satisfy the exact active source that now owns +/// event intake, even when chain, namespace, and source family are unchanged. +#[test] +fn deprecated_manifest_events_do_not_satisfy_active_source() { + let mut read = healthy_read(); + read.active_manifest_event_sources = vec![active_event_source(2, "ens_v2_registry_l1", 0)]; + let report = evaluate(&read, ®istry_only()); + + assert_eq!(report.active_dataset_non_empty(), CheckStatus::Fail); + assert!(!report.data_complete()); +} + +/// An active chain with events in a namespace that has no name_current rows fails: names did +/// not materialize for that namespace. +#[test] +fn active_namespace_without_names_fails_content() { + let mut read = healthy_read(); + read.name_current_counts = vec![]; + let report = evaluate(&read, ®istry_only()); + + assert_eq!(report.active_dataset_non_empty(), CheckStatus::Fail); + assert_eq!( + report.active_namespaces_without_names, + vec![NAMESPACE.to_owned()] + ); + assert!(!report.data_complete()); +} + +fn latched_chain_read() -> DataCompletenessRead { + // Raw-fact replay is latched at block 1000, well below the live head at 2000; the backlog + // cursor and live sync carry the rest. The raw-log head is 2000, so a head comparison + // would mark this chain permanently behind. + let mut read = healthy_read(); + read.chains = vec![ChainCompletenessRow { + canonical_raw_log_head_block_number: Some(2_000), + raw_log_head_block_number: Some(2_000), + ..chain_row(2_000, 2_000, 1, 2_000) + }]; + read +} + +/// On a chain with closure-replay adapters the raw-fact cursor's target is latched below the +/// head. Caught-up means both the raw-fact cursor and the backlog cursor reached their +/// targets, not the raw-log head. +#[test] +fn latched_chain_at_targets_is_caught_up() { + let mut read = latched_chain_read(); + read.replay_cursors = vec![ + replay_cursor(1_000, None), + backlog_cursor(1_001, 2_001, 2_000), + ]; + let report = evaluate(&read, ®istry_only()); + + assert_eq!(report.normalization_caught_up(), CheckStatus::Pass); + assert!(report.lagging_replay_cursors.is_empty()); + assert!(report.data_complete()); +} + +/// A lost backlog cursor cannot be treated as a quiet latch while retained canonical logs prove +/// that the writer had tail work above the raw-fact target. +#[test] +fn latched_chain_with_tail_logs_and_no_backlog_cursor_fails() { + let mut read = latched_chain_read(); + read.replay_cursors = vec![replay_cursor(1_000, None)]; + let report = evaluate(&read, ®istry_only()); + + assert_eq!(report.normalization_caught_up(), CheckStatus::Fail); + assert_eq!(report.lagging_replay_cursors[0].behind_by, 1_000); + assert!( + report.lagging_replay_cursors[0] + .label + .ends_with(":missing_backlog") + ); + assert!(!report.data_complete()); +} + +/// The backlog writer writes no cursor when no canonical raw logs exist above the completed +/// latch. The raw-log head independently corroborates that quiet shape. +#[test] +fn quiet_latched_chain_at_raw_log_head_needs_no_backlog_cursor() { + let mut read = latched_chain_read(); + read.chains[0].canonical_raw_log_head_block_number = Some(1_000); + read.replay_cursors = vec![replay_cursor(1_000, None)]; + let report = evaluate(&read, ®istry_only()); + + assert_eq!(report.normalization_caught_up(), CheckStatus::Pass); + assert!(report.lagging_replay_cursors.is_empty()); + assert!(report.data_complete()); +} + +#[test] +fn latched_chain_with_backlog_short_of_target_fails() { + let mut read = latched_chain_read(); + read.replay_cursors = vec![ + replay_cursor(1_000, None), + backlog_cursor(1_001, 1_900, 2_000), + ]; + let report = evaluate(&read, ®istry_only()); + + assert_eq!(report.normalization_caught_up(), CheckStatus::Fail); + assert_eq!(report.lagging_replay_cursors[0].behind_by, 100); + assert!(!report.data_complete()); +} + +/// The backlog writer seeds its inclusive start to the completed raw-fact replay target plus +/// one. A later restored start leaves a normalization hole even when both cursors are complete. +#[test] +fn latched_chain_with_backlog_start_gap_fails() { + let mut read = latched_chain_read(); + read.replay_cursors = vec![ + replay_cursor(1_000, None), + backlog_cursor(1_500, 2_001, 2_000), + ]; + let report = evaluate(&read, ®istry_only()); + + assert_eq!(report.normalization_caught_up(), CheckStatus::Fail); + assert_eq!(report.lagging_replay_cursors[0].behind_by, 499); + assert!( + report.lagging_replay_cursors[0] + .label + .ends_with(":range_start") + ); + assert!(!report.data_complete()); +} + +/// A reorg rewind lowers `next_block_number` below the target while `last_completed` stays at +/// its high-water mark. The gate must read `next`/`target`, not `last_completed`. +#[test] +fn rewound_cursor_below_target_fails_even_with_high_last_completed() { + let mut read = healthy_read(); + read.chains = vec![ChainCompletenessRow { + canonical_raw_log_head_block_number: Some(1_000), + raw_log_head_block_number: Some(1_000), + ..chain_row(1_000, 1_000, 1, 1_000) + }]; + read.replay_cursors = vec![rewound_cursor(500, 1_000, 1_000)]; + let report = evaluate(&read, ®istry_only()); + + assert_eq!(report.normalization_caught_up(), CheckStatus::Fail); + assert_eq!(report.lagging_replay_cursors[0].behind_by, 500); + assert!(!report.data_complete()); +} + +/// A candidate mid projection-bootstrap has published name_current (first in order) but not +/// the other projections, so not all markers are present at the newest replay version. +#[test] +fn incomplete_projection_replay_markers_fail() { + let mut read = healthy_read(); + read.projection_replay_markers = vec![ProjectionReplayMarker { + replay_version: CURRENT_PROJECTION_REPLAY_VERSION, + projection: "name_current".to_owned(), + completed_normalized_target_block: Some(1_000), + }]; + let report = evaluate(&read, ®istry_only()); + + assert_eq!(report.projection_replay_complete(), CheckStatus::Fail); + assert!( + report + .missing_projection_replay_markers + .contains(&"children_current".to_owned()) + ); + assert!(!report.data_complete()); +} + +/// Complete markers written for an earlier bootstrap target do not cover a normalized replay +/// target that has since advanced. +#[test] +fn projection_replay_markers_below_required_target_fail() { + let mut read = healthy_read(); + read.projection_apply_cursors.clear(); + read.max_projection_change_id = None; + read.projection_replay_required_target_block = Some(1_001); + let report = evaluate(&read, ®istry_only()); + + assert_eq!(report.projection_replay_complete(), CheckStatus::Fail); + assert_eq!( + report.missing_projection_replay_markers.len(), + ALL_CURRENT_PROJECTION_ORDER.len() + ); + assert_eq!(report.projection_replay_required_target_block, Some(1_001)); + assert!(report.projection_replay_target_coverage_required); + assert!(!report.data_complete()); +} + +/// A restored apply cursor over an empty change log is not evidence that continuous apply took +/// over, so it cannot waive current replay-marker target coverage. +#[test] +fn empty_projection_change_log_does_not_waive_marker_target_coverage() { + let mut read = healthy_read(); + read.projection_apply_cursors = vec![apply_cursor(0)]; + read.max_projection_change_id = None; + read.projection_replay_required_target_block = Some(1_001); + let report = evaluate(&read, ®istry_only()); + + assert!(report.projection_replay_target_coverage_required); + assert_eq!(report.projection_replay_complete(), CheckStatus::Fail); + assert_eq!(report.projection_drained(), CheckStatus::Pass); + assert_eq!( + report.missing_projection_replay_markers.len(), + ALL_CURRENT_PROJECTION_ORDER.len() + ); + assert!(!report.data_complete()); +} + +/// Once a non-empty change log exactly corroborates the durable apply cursor, current-version +/// markers prove bootstrap handoff and the drained apply/change-log checks own later normalized +/// blocks. Marker targets do not move. +#[test] +fn drained_projection_apply_allows_target_to_advance_past_replay_markers() { + let mut read = healthy_read(); + read.projection_replay_required_target_block = Some(1_001); + let report = evaluate(&read, ®istry_only()); + + assert!(!report.projection_replay_target_coverage_required); + assert_eq!(report.projection_replay_complete(), CheckStatus::Pass); + assert_eq!(report.projection_drained(), CheckStatus::Pass); + assert!(report.data_complete()); +} + +/// No replay markers at all means projections were never rebuilt. +#[test] +fn no_projection_replay_markers_fail() { + let mut read = healthy_read(); + read.projection_replay_markers = vec![]; + let report = evaluate(&read, ®istry_only()); + + assert_eq!(report.projection_replay_version, None); + assert_eq!(report.projection_replay_complete(), CheckStatus::Fail); + assert!(!report.data_complete()); +} + +/// Complete markers from an older worker image do not satisfy the current worker's replay +/// version, even when every projection and target is otherwise present. +#[test] +fn complete_markers_at_older_version_fail() { + let mut read = healthy_read(); + let older_version = CURRENT_PROJECTION_REPLAY_VERSION - 1; + read.projection_replay_markers = all_projection_markers(older_version); + let report = evaluate(&read, ®istry_only()); + + assert_eq!(report.projection_replay_version, Some(older_version)); + assert_eq!( + report.projection_replay_required_version, + CURRENT_PROJECTION_REPLAY_VERSION + ); + assert_eq!(report.projection_replay_complete(), CheckStatus::Fail); + assert_eq!( + report.missing_projection_replay_markers.len(), + ALL_CURRENT_PROJECTION_ORDER.len() + ); + assert!(!report.data_complete()); +} + +/// A NULL `chain_id` normalized event is a data-integrity fault. +#[test] +fn null_chain_id_normalized_events_fail() { + let mut read = healthy_read(); + read.normalized_events_null_chain_id_count = 3; + let report = evaluate(&read, ®istry_only()); + + assert_eq!( + report.normalized_events_chain_id_present(), + CheckStatus::Fail + ); + assert!(!report.data_complete()); +} + +/// A fresh replay drops the deferred projection indexes; an absent one marks a mid-replay +/// candidate not yet ready to serve. +#[test] +fn missing_deferred_projection_index_fails() { + let mut read = healthy_read(); + read.present_deferred_projection_indexes = all_deferred_indexes() + .into_iter() + .filter(|name| name != "normalized_events_namespace_idx") + .collect(); + let report = evaluate(&read, ®istry_only()); + + assert_eq!( + report.deferred_projection_indexes_present(), + CheckStatus::Fail + ); + assert!( + report + .missing_deferred_projection_indexes + .contains(&"normalized_events_namespace_idx".to_owned()) + ); + assert!(!report.data_complete()); +} + +/// Two active event-producing manifest sources fail when one has no events, even though the +/// other does — the expectation comes from declared source identities, not observed events. +#[test] +fn declared_namespace_with_no_events_fails_content() { + let mut read = healthy_read(); + read.manifest_chain_namespaces = + vec![manifest_ns(CHAIN, "ens"), manifest_ns(CHAIN, "basenames")]; + let mut basenames_source = active_event_source(2, "basenames_registry", 0); + basenames_source.namespace = "basenames".to_owned(); + read.active_manifest_event_sources.push(basenames_source); + read.name_current_counts = vec![names("ens", 10)]; + let report = evaluate(&read, ®istry_only()); + + assert_eq!(report.active_dataset_non_empty(), CheckStatus::Fail); + assert!( + report + .active_manifest_sources_without_events + .iter() + .any(|entry| entry.namespace == "basenames") + ); + assert!(!report.data_complete()); +} + +/// Active execution/transport metadata manifests do not participate in event intake and must +/// not require normalized-event or name projection content. +#[test] +fn metadata_only_manifest_does_not_create_content_expectation() { + let mut read = healthy_read(); + read.manifest_chain_namespaces = vec![ + manifest_ns(CHAIN, NAMESPACE), + manifest_ns("ethereum-mainnet", "basenames"), + ]; + let report = evaluate(&read, ®istry_only()); + + assert_eq!(report.active_dataset_non_empty(), CheckStatus::Pass); +} + +/// A chain declared only by an active manifest version (no watched-contract rows, e.g. a +/// partial restore losing contract_instance_addresses) still gets a gating frontier. +#[test] +fn manifest_only_chain_gets_a_frontier() { + let mut read = healthy_read(); + read.chains = vec![]; + read.manifest_chain_namespaces = vec![manifest_ns(CHAIN, NAMESPACE)]; + let report = evaluate(&read, ®istry_only()); + + let frontier = report + .frontiers + .iter() + .find(|frontier| frontier.chain_id == CHAIN) + .expect("frontier for the manifest-declared chain"); + assert!(frontier.missing_from_storage); + assert_eq!(report.frontier_at_head(), CheckStatus::Fail); +} + +/// A foreign or retired chain with residual storage rows is an advisory, not a gate failure. +#[test] +fn foreign_chain_is_advisory_not_gating() { + let mut read = healthy_read(); + read.chains = vec![ + chain_row(1_000, 1_000, 1, 1_000), + ChainCompletenessRow { + chain_id: "retired-chain".to_owned(), + ..chain_row(1_000, 1_000, 1, 1_000) + }, + ]; + let report = evaluate(&read, ®istry_only()); + + assert_eq!(report.foreign_chains, vec!["retired-chain".to_owned()]); + assert!( + report + .frontiers + .iter() + .all(|frontier| frontier.chain_id != "retired-chain") + ); + assert!(report.data_complete()); +} + +/// Backfill failures are surfaced as an advisory with counts, not gated. +#[test] +fn backfill_failures_are_advisory() { + let mut read = healthy_read(); + read.backfill_lifecycle = vec![BackfillLifecycleRow { + deployment_profile: DEPLOYMENT_PROFILE.to_owned(), + failed_job_count: 22, + failed_range_count: 22, + incomplete_range_count: 274, + expired_lease_range_count: 1, + }]; + let report = evaluate(&read, ®istry_only()); + + assert_eq!(report.backfill_advisory[0].failed_job_count, 22); + // Advisory only: backfill failures do not fail the gate. + assert!(report.data_complete()); +} diff --git a/apps/worker/src/main_tests.rs b/apps/worker/src/main_tests.rs index b52d70f1..2b62addb 100644 --- a/apps/worker/src/main_tests.rs +++ b/apps/worker/src/main_tests.rs @@ -187,6 +187,33 @@ fn inspect_backfill_job_cli_is_available() { } } +#[test] +fn inspect_data_completeness_accepts_manifest_and_retention_authority() { + let cli = Cli::parse_from([ + "bigname-worker", + "inspect", + "data-completeness", + "--manifests-root", + "manifests/sepolia", + "--retention-mode", + "log-audit", + ]); + + match cli.command { + Command::Inspect(args) => match args.command { + inspect::InspectCommand::DataCompleteness(args) => { + assert_eq!( + args.manifests_root.as_deref(), + Some(std::path::Path::new("manifests/sepolia")) + ); + assert_eq!(args.retention_mode, inspect::RetentionMode::LogAudit); + } + other => panic!("expected data-completeness inspect command, got {other:?}"), + }, + other => panic!("expected inspect command, got {other:?}"), + } +} + #[test] fn inspect_execution_trace_cli_is_available() { let trace_id = "0e7ec7ac-e000-0000-0000-000000000abc"; diff --git a/apps/worker/src/projection_apply.rs b/apps/worker/src/projection_apply.rs index b134ebf5..019dae3a 100644 --- a/apps/worker/src/projection_apply.rs +++ b/apps/worker/src/projection_apply.rs @@ -16,7 +16,7 @@ use crate::record_inventory; pub(crate) use derive::{normalized_event_cursor_exists, seed_normalized_event_cursor_if_absent}; -const NORMALIZED_EVENT_CURSOR: &str = "normalized_events_to_projection_invalidations"; +pub(crate) const NORMALIZED_EVENT_CURSOR: &str = "normalized_events_to_projection_invalidations"; const NORMALIZED_EVENT_DERIVE_BATCH_LIMIT: i64 = 5_000; const PROJECTION_APPLY_BATCH_LIMIT: i64 = 25; const FAILURE_RETRY_DELAY: &str = "60 seconds"; diff --git a/crates/adapters/src/block_derived_normalized_events.rs b/crates/adapters/src/block_derived_normalized_events.rs index 77587511..e7524314 100644 --- a/crates/adapters/src/block_derived_normalized_events.rs +++ b/crates/adapters/src/block_derived_normalized_events.rs @@ -22,6 +22,12 @@ mod types; use crate::normalized_event_support::{ NormalizedEventSyncCounts, count_events_by_kind, upsert_normalized_events_with_counts, }; +use constants::{ + EVENT_KIND_PREIMAGE_OBSERVED, SOURCE_FAMILY_BASENAMES_BASE_REGISTRAR, + SOURCE_FAMILY_ENS_V1_REGISTRAR_L1, SOURCE_FAMILY_ENS_V1_WRAPPER_L1, + SOURCE_FAMILY_ENS_V2_REGISTRAR_L1, SOURCE_FAMILY_ENS_V2_REGISTRY_L1, + SOURCE_FAMILY_ENS_V2_RESOLVER_L1, SOURCE_FAMILY_ENS_V2_ROOT_L1, +}; use event_builders::build_preimage_observed_events; use loading::{RawLogCanonicalityFilter, load_scanned_log_count, load_watched_raw_logs}; @@ -29,6 +35,37 @@ pub use types::{ BlockDerivedNormalizedEventKindSyncSummary, BlockDerivedNormalizedEventSyncSummary, }; +pub(crate) const NORMALIZED_EVENT_KIND_DECLARATIONS: &[(&str, &[&str])] = &[ + ( + SOURCE_FAMILY_ENS_V1_REGISTRAR_L1, + &[EVENT_KIND_PREIMAGE_OBSERVED], + ), + ( + SOURCE_FAMILY_ENS_V1_WRAPPER_L1, + &[EVENT_KIND_PREIMAGE_OBSERVED], + ), + ( + SOURCE_FAMILY_BASENAMES_BASE_REGISTRAR, + &[EVENT_KIND_PREIMAGE_OBSERVED], + ), + ( + SOURCE_FAMILY_ENS_V2_ROOT_L1, + &[EVENT_KIND_PREIMAGE_OBSERVED], + ), + ( + SOURCE_FAMILY_ENS_V2_REGISTRY_L1, + &[EVENT_KIND_PREIMAGE_OBSERVED], + ), + ( + SOURCE_FAMILY_ENS_V2_REGISTRAR_L1, + &[EVENT_KIND_PREIMAGE_OBSERVED], + ), + ( + SOURCE_FAMILY_ENS_V2_RESOLVER_L1, + &[EVENT_KIND_PREIMAGE_OBSERVED], + ), +]; + #[cfg(test)] use crate::evm_abi::keccak_signature_hex; #[cfg(test)] diff --git a/crates/adapters/src/ens_v1_reverse_claim.rs b/crates/adapters/src/ens_v1_reverse_claim.rs index 5de2080d..57bbd4d1 100644 --- a/crates/adapters/src/ens_v1_reverse_claim.rs +++ b/crates/adapters/src/ens_v1_reverse_claim.rs @@ -35,6 +35,17 @@ const ENS_NATIVE_COIN_TYPE: &str = "60"; const BASE_NATIVE_COIN_TYPE: &str = "2147492101"; const CONTRACT_ROLE_REVERSE_REGISTRAR: &str = "reverse_registrar"; +pub(crate) const NORMALIZED_EVENT_KIND_DECLARATIONS: &[(&str, &[&str])] = &[ + ( + SOURCE_FAMILY_ENS_V1_REVERSE_L1, + &[EVENT_KIND_REVERSE_CHANGED], + ), + ( + SOURCE_FAMILY_BASENAMES_BASE_PRIMARY, + &[EVENT_KIND_REVERSE_CHANGED, EVENT_KIND_RECORD_CHANGED], + ), +]; + #[derive(Clone, Debug, Eq, PartialEq)] pub struct EnsV1ReverseClaimSyncSummary { pub scanned_log_count: usize, diff --git a/crates/adapters/src/lib.rs b/crates/adapters/src/lib.rs index 57d339d5..c4528015 100644 --- a/crates/adapters/src/lib.rs +++ b/crates/adapters/src/lib.rs @@ -16,6 +16,7 @@ mod ens_v2_resolver; mod evm_abi; mod manifest_normalized_events; mod normalized_event_support; +mod normalized_replay_policy; mod registry_migration_cache; /// Current adapter bootstrap status. @@ -62,6 +63,25 @@ pub use manifest_normalized_events::{ ManifestNormalizedEventKindSyncSummary, ManifestNormalizedEventSyncSummary, sync_manifest_normalized_events, }; +pub use normalized_replay_policy::{ + CLOSURE_OR_DEPENDENCY_REPLAY_SOURCE_FAMILIES, source_family_preserves_normalized_replay_target, +}; + +/// Normalized event kinds emitted by adapters outside a one-to-one manifest ABI declaration. +/// +/// The declaration augments event-kind inspection for an already admitted active source family; +/// it does not admit sources or alter manifest capability and watch-plan authority. +pub fn adapter_normalized_event_kind_declarations() -> Vec<(&'static str, &'static str)> { + block_derived_normalized_events::NORMALIZED_EVENT_KIND_DECLARATIONS + .iter() + .chain(ens_v1_reverse_claim::NORMALIZED_EVENT_KIND_DECLARATIONS) + .flat_map(|(source_family, event_kinds)| { + event_kinds + .iter() + .map(move |event_kind| (*source_family, *event_kind)) + }) + .collect() +} pub async fn clear_replay_adapter_checkpoints( pool: &sqlx::PgPool, diff --git a/crates/adapters/src/normalized_replay_policy.rs b/crates/adapters/src/normalized_replay_policy.rs new file mode 100644 index 00000000..1d754542 --- /dev/null +++ b/crates/adapters/src/normalized_replay_policy.rs @@ -0,0 +1,49 @@ +/// Active source families that force automatic normalized replay to preserve its first +/// admitted target. These adapters need closure or contextual dependency replay, so advancing +/// the historical cursor to each new raw-log head would rerun the full closure indefinitely. +/// +/// The indexer uses this list to choose its target-refresh policy. Read-only operational tools +/// use the same list to interpret a completed cursor whose target is intentionally below head. +pub const CLOSURE_OR_DEPENDENCY_REPLAY_SOURCE_FAMILIES: &[&str] = &[ + "basenames_base_registrar", + "basenames_base_registry", + "basenames_base_resolver", + "ens_v1_registrar_l1", + "ens_v1_registry_l1", + "ens_v1_resolver_l1", + "ens_v1_wrapper_l1", + "ens_v2_registrar_l1", + "ens_v2_registry_l1", + "ens_v2_resolver_l1", + "ens_v2_root_l1", +]; + +pub const fn source_family_preserves_normalized_replay_target(source_family: &str) -> bool { + let mut index = 0; + while index < CLOSURE_OR_DEPENDENCY_REPLAY_SOURCE_FAMILIES.len() { + if const_str_eq( + source_family, + CLOSURE_OR_DEPENDENCY_REPLAY_SOURCE_FAMILIES[index], + ) { + return true; + } + index += 1; + } + false +} + +const fn const_str_eq(left: &str, right: &str) -> bool { + let left = left.as_bytes(); + let right = right.as_bytes(); + if left.len() != right.len() { + return false; + } + let mut index = 0; + while index < left.len() { + if left[index] != right[index] { + return false; + } + index += 1; + } + true +} diff --git a/crates/manifests/src/lib/views/abi.rs b/crates/manifests/src/lib/views/abi.rs index cc61f012..91136084 100644 --- a/crates/manifests/src/lib/views/abi.rs +++ b/crates/manifests/src/lib/views/abi.rs @@ -1,3 +1,5 @@ +use std::collections::{BTreeMap, BTreeSet}; + use anyhow::{Context, Result}; use serde::Deserialize; use sqlx::{PgPool, Row}; @@ -130,6 +132,30 @@ pub async fn load_active_manifest_abi_events_by_chain_and_source_families( active_manifest_abi_events_from_rows(rows).await } +/// Current topic0 set for every requested active source family on one chain. Stored-lineage +/// promotion and completeness inspection share this exact view before trusting topic-filtered +/// backfill coverage facts. +pub async fn load_active_manifest_topic0s_by_chain_and_source_families( + pool: &PgPool, + chain: &str, + source_families: &[String], +) -> Result>> { + let events = + load_active_manifest_abi_events_by_chain_and_source_families(pool, chain, source_families) + .await?; + let mut topic0s_by_family = BTreeMap::>::new(); + for event in events { + let Some(topic0) = event.topic0 else { + continue; + }; + topic0s_by_family + .entry(event.source_family) + .or_default() + .insert(topic0.to_ascii_lowercase()); + } + Ok(topic0s_by_family) +} + async fn active_manifest_abi_events_from_rows( rows: Vec, ) -> Result> { @@ -208,14 +234,10 @@ pub async fn load_log_producing_source_families(pool: &PgPool, chain: &str) -> R return Ok(Vec::new()); } - let events = - load_active_manifest_abi_events_by_chain_and_source_families(pool, chain, &source_families) - .await?; - Ok(events - .into_iter() - .filter(|event| event.topic0.is_some()) - .map(|event| event.source_family) - .collect::>() - .into_iter() - .collect()) + Ok( + load_active_manifest_topic0s_by_chain_and_source_families(pool, chain, &source_families) + .await? + .into_keys() + .collect(), + ) } diff --git a/crates/manifests/src/lib/views/watched.rs b/crates/manifests/src/lib/views/watched.rs index 86889c3a..701912c0 100644 --- a/crates/manifests/src/lib/views/watched.rs +++ b/crates/manifests/src/lib/views/watched.rs @@ -10,7 +10,10 @@ use sqlx::{PgPool, Row, postgres::PgRow}; use crate::{WatchedContract, WatchedContractSource, normalize_address}; -pub use coverage::{UncoveredWatchedTuple, find_uncovered_watched_tuples}; +pub use coverage::{ + UncoveredWatchedTuple, WATCHED_COVERAGE_VERIFICATION_CHUNK_BLOCKS, + find_uncovered_watched_tuples, +}; pub use scoped::{ load_watched_contracts_by_addresses, load_watched_contracts_by_source_family_and_addresses, }; diff --git a/crates/manifests/src/lib/views/watched/coverage.rs b/crates/manifests/src/lib/views/watched/coverage.rs index e1fe85dd..88dd3dfc 100644 --- a/crates/manifests/src/lib/views/watched/coverage.rs +++ b/crates/manifests/src/lib/views/watched/coverage.rs @@ -1,6 +1,10 @@ use anyhow::{Context, Result}; use sqlx::{PgPool, Row}; +/// Block chunk shared by stored-lineage promotion and read-only completeness inspection when +/// reconciling watched tuples against durable backfill coverage facts. +pub const WATCHED_COVERAGE_VERIFICATION_CHUNK_BLOCKS: i64 = 131_072; + /// A watched (source_family, address) tuple whose required interval within the /// evaluated block range is not fully contained in any single /// `backfill_coverage_facts` row (address-scoped or family-scoped). diff --git a/crates/storage/src/address_names.rs b/crates/storage/src/address_names.rs index 4826f4a3..1329447c 100644 --- a/crates/storage/src/address_names.rs +++ b/crates/storage/src/address_names.rs @@ -9,6 +9,8 @@ mod read; mod types; mod write; +pub(crate) use query::DEFAULT_ADDRESS_NAMES_CURRENT_READ_FILTER; + pub use address_replacement::{ AddressNamesCurrentAddressReplacement, begin_address_names_current_address_replacement, drop_address_names_current_address_replacement, diff --git a/crates/storage/src/address_names/query.rs b/crates/storage/src/address_names/query.rs index f2045614..9b77f2e1 100644 --- a/crates/storage/src/address_names/query.rs +++ b/crates/storage/src/address_names/query.rs @@ -6,7 +6,7 @@ use super::types::{ AddressNamesCurrentSort, AddressNamesCurrentSortedCursor, AddressNamesCurrentSortedCursorValue, }; -pub(super) const DEFAULT_ADDRESS_NAMES_CURRENT_READ_FILTER: &str = r#" +pub(crate) const DEFAULT_ADDRESS_NAMES_CURRENT_READ_FILTER: &str = r#" AND surface.canonicality_state IN ( 'canonical'::canonicality_state, 'safe'::canonicality_state, diff --git a/crates/storage/src/backfill_jobs.rs b/crates/storage/src/backfill_jobs.rs index ba6e5844..fb55a5b8 100644 --- a/crates/storage/src/backfill_jobs.rs +++ b/crates/storage/src/backfill_jobs.rs @@ -6,6 +6,7 @@ mod fail; mod lease; mod read; mod sql; +mod topic_drift; mod types; mod validate; @@ -20,8 +21,10 @@ pub use create::create_backfill_job; pub use fail::{fail_backfill_job, fail_backfill_range}; pub use lease::{advance_backfill_range, reserve_backfill_range}; pub use read::{ - load_backfill_job, load_backfill_ranges, load_completed_backfill_jobs_intersecting_range, + load_backfill_job, load_backfill_jobs_intersecting_range, load_backfill_ranges, + load_completed_backfill_jobs_intersecting_range, }; +pub use topic_drift::ensure_backfill_family_topic_sets_undrifted; pub use types::{ BackfillJob, BackfillJobCreate, BackfillJobRecord, BackfillLifecycleStatus, BackfillRange, BackfillRangeSpec, diff --git a/crates/storage/src/backfill_jobs/read.rs b/crates/storage/src/backfill_jobs/read.rs index a7b031a5..6d1a2561 100644 --- a/crates/storage/src/backfill_jobs/read.rs +++ b/crates/storage/src/backfill_jobs/read.rs @@ -22,13 +22,43 @@ pub async fn load_completed_backfill_jobs_intersecting_range( from_block: i64, to_block: i64, ) -> Result> { + load_backfill_jobs_intersecting_range_inner(pool, chain_id, from_block, to_block, true).await +} + +/// Load every persisted backfill job for a chain whose declared block range intersects +/// `[from_block, to_block]`. Unlike the completed-only promotion helper, this includes incomplete +/// and failed jobs so inspection can recognize retained crash residue that still requires durable +/// coverage evidence from a successful retry. +pub async fn load_backfill_jobs_intersecting_range( + pool: &PgPool, + chain_id: &str, + from_block: i64, + to_block: i64, +) -> Result> { + load_backfill_jobs_intersecting_range_inner(pool, chain_id, from_block, to_block, false).await +} + +async fn load_backfill_jobs_intersecting_range_inner( + pool: &PgPool, + chain_id: &str, + from_block: i64, + to_block: i64, + completed_only: bool, +) -> Result> { + let status_filter = if completed_only { + "AND status = 'completed'::backfill_lifecycle_status" + } else { + "" + }; let select_sql = backfill_job_select_sql( - r#" + &format!( + r#" WHERE chain_id = $1 - AND status = 'completed'::backfill_lifecycle_status + {status_filter} AND range_start_block_number <= $3 AND range_end_block_number >= $2 - "#, + "# + ), "ORDER BY backfill_job_id", ); let rows = sqlx::query(&select_sql) @@ -36,12 +66,13 @@ pub async fn load_completed_backfill_jobs_intersecting_range( .bind(from_block) .bind(to_block) .fetch_all(pool) - .await - .with_context(|| { - format!( - "failed to load completed backfill jobs for chain {chain_id} intersecting {from_block}..={to_block}" - ) - })?; + .await + .with_context(|| { + let status = if completed_only { "completed " } else { "" }; + format!( + "failed to load {status}backfill jobs for chain {chain_id} intersecting {from_block}..={to_block}" + ) + })?; rows.into_iter().map(decode_backfill_job).collect() } diff --git a/apps/indexer/src/main/reconciliation/canonical/stored_lineage/coverage/topic_drift.rs b/crates/storage/src/backfill_jobs/topic_drift.rs similarity index 80% rename from apps/indexer/src/main/reconciliation/canonical/stored_lineage/coverage/topic_drift.rs rename to crates/storage/src/backfill_jobs/topic_drift.rs index a5704c52..a6924c47 100644 --- a/apps/indexer/src/main/reconciliation/canonical/stored_lineage/coverage/topic_drift.rs +++ b/crates/storage/src/backfill_jobs/topic_drift.rs @@ -1,21 +1,17 @@ -//! Topic-set drift guard: coverage facts assert topics-complete fetches, so -//! promotion must refuse when a family's manifest ABI topic0 set no longer -//! matches the set a completed topic-filtered job persisted. +//! Topic-set drift guard for durable coverage facts. use std::collections::{BTreeMap, BTreeSet}; -use bigname_storage::load_completed_backfill_jobs_intersecting_range; use serde_json::Value; -/// Fail closed on topic-set drift: coverage facts assert fetches that were -/// topics-complete relative to the family's manifest ABI event set at fetch -/// time. If a family's current topic0 set differs from the set persisted in -/// any completed topic-filtered job intersecting the evaluated range — or a -/// topic-filtered job did not persist its set at all — the facts may -/// overclaim relative to the current ABI, so promotion refuses naming the -/// family. Address-enumerated hash-pinned fetches are topic-unfiltered and -/// immune. -pub(super) async fn ensure_family_topic_sets_undrifted( +use super::load_completed_backfill_jobs_intersecting_range; + +/// Fail closed on topic-set drift: coverage facts assert fetches that were topics-complete +/// relative to the family's manifest ABI event set at fetch time. If a family's current topic0 +/// set differs from the set persisted in any completed topic-filtered job intersecting the +/// evaluated range—or a topic-filtered job did not persist its set—the facts may overclaim. +/// Address-enumerated hash-pinned fetches are topic-unfiltered and immune. +pub async fn ensure_backfill_family_topic_sets_undrifted( pool: &sqlx::PgPool, chain: &str, current_topic0s_by_family: &BTreeMap>, @@ -82,8 +78,8 @@ pub(super) async fn ensure_family_topic_sets_undrifted( } /// The persisted topic0 sets a topic-filtered job fetched under: nested in -/// `coinbase_sql_topic_plan` for Coinbase SQL jobs, or top-level for the -/// hash-pinned Basenames registry scan-all shape. +/// `coinbase_sql_topic_plan` for Coinbase SQL jobs, or top-level for the hash-pinned Basenames +/// registry scan-all shape. fn persisted_topic0s_by_source_family( source_identity: &Value, ) -> Option<&serde_json::Map> { @@ -98,9 +94,8 @@ fn persisted_topic0s_by_source_family( }) } -/// Families a job fetched through topic-filtered generic scans whose identity -/// does not persist the topic set in force (hash-pinned generic resolver -/// scans; identities with persisted topic sets are checked above). +/// Families a job fetched through topic-filtered generic scans whose identity does not persist +/// the topic set in force. Identities with persisted topic sets are checked above. fn topic_filtered_families_without_persisted_sets(source_identity: &Value) -> Vec { if persisted_topic0s_by_source_family(source_identity).is_some() { return Vec::new(); diff --git a/crates/storage/src/children.rs b/crates/storage/src/children.rs index 58bc6c70..54c827d7 100644 --- a/crates/storage/src/children.rs +++ b/crates/storage/src/children.rs @@ -20,7 +20,7 @@ pub use types::{ }; pub use writes::{clear_children_current, delete_children_current, upsert_children_current_rows}; -const DECLARED_SURFACE_CLASS: &str = "declared"; +pub(crate) const DECLARED_SURFACE_CLASS: &str = "declared"; const SUBREGISTRY_EVENT_KIND: &str = "SubregistryChanged"; const PARENT_EVENT_KIND: &str = "ParentChanged"; const REGISTRATION_GRANTED_EVENT_KIND: &str = "RegistrationGranted"; @@ -32,7 +32,7 @@ const ENSV1_SUBREGISTRY_SOURCE_FAMILY: &str = "ens_v1_registry_l1"; const BASENAMES_BASE_SUBREGISTRY_SOURCE_FAMILY: &str = "basenames_base_registry"; const ENSV2_ROOT_SOURCE_FAMILY: &str = "ens_v2_root_l1"; const ENSV2_REGISTRY_SOURCE_FAMILY: &str = "ens_v2_registry_l1"; -const DEFAULT_CHILDREN_CURRENT_READ_FILTER: &str = r#" +pub(crate) const DEFAULT_CHILDREN_CURRENT_READ_FILTER: &str = r#" AND parent.canonicality_state IN ( 'canonical'::canonicality_state, 'safe'::canonicality_state, diff --git a/crates/storage/src/data_completeness.rs b/crates/storage/src/data_completeness.rs new file mode 100644 index 00000000..1e32c812 --- /dev/null +++ b/crates/storage/src/data_completeness.rs @@ -0,0 +1,525 @@ +use anyhow::{Context, Result}; + +mod content; +mod manifest_targets; +mod projection_content; + +use content::*; +pub use manifest_targets::load_active_manifest_deployment_profile; +use manifest_targets::*; +pub use projection_content::{ + ProjectionContentCounts, ProjectionContentScopeCount, load_projection_content_counts, +}; + +#[cfg(test)] +mod tests; + +/// Per-chain intake frontiers. +/// +/// `lineage_canonical_block_count` counts distinct non-orphaned block numbers, so a +/// contiguous lineage satisfies `count == head - floor + 1`. +/// +/// `canonical_raw_log_head_block_number` is the head of the raw logs that normalized +/// replay is eligible to consume: it mirrors the replay bounds, which require both the +/// raw log and its lineage block to be canonical, safe, or finalized. The gate compares +/// replay progress against this head. `raw_log_head_block_number` is the non-orphaned +/// head including `observed` logs and is reported only, so a candidate whose newest logs +/// have not yet been promoted to canonical is not measured as lagging against them. +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct ChainCompletenessRow { + pub chain_id: String, + pub canonical_block_number: Option, + /// Whether the checkpoint's exact `(chain, block_hash, block_number)` resolves to a + /// canonical/safe/finalized lineage row. An empty checkpoint has no anchor to validate. + pub checkpoint_canonical_lineage_match: bool, + pub lineage_head_block_number: Option, + pub lineage_floor_block_number: Option, + pub lineage_canonical_block_count: i64, + /// Canonical/safe/finalized block heights carrying an additional non-orphaned hash. The + /// distinct-block-number contiguity count cannot see these competing forks, so they are + /// counted separately; a non-zero value is a canonicality violation, not a gap. + pub duplicate_canonical_height_count: i64, + /// Canonical/safe/finalized rows above the retained floor whose `parent_hash` does not + /// resolve to the canonical/safe/finalized row at the preceding height. + pub disconnected_canonical_parent_count: i64, + pub canonical_raw_log_head_block_number: Option, + pub raw_log_head_block_number: Option, +} + +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct ReplayCursorRow { + pub deployment_profile: String, + pub chain_id: String, + pub cursor_kind: String, + /// Inclusive start of the cursor's admitted range. The post-replay backlog writer seeds + /// this to the raw-fact replay target plus one so the two cursor ranges remain contiguous. + pub range_start_block_number: i64, + /// The completion authority. Replay is complete for a cursor's target when + /// `next_block_number > target_block_number`; a reorg rewind lowers `next_block_number` + /// but leaves `last_completed_block_number` at its high-water mark, so the gate reads the + /// `next`/`target` pair and treats `last_completed_block_number` as reporting detail only. + pub next_block_number: Option, + pub target_block_number: Option, + pub last_completed_block_number: Option, + pub last_failure_reason: Option, +} + +/// A completed `current_projection_replay_status` marker for one `(replay_version, projection)`. +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct ProjectionReplayMarker { + pub replay_version: i32, + pub projection: String, + pub completed_normalized_target_block: Option, +} + +/// Backfill lifecycle counts, scoped by deployment profile. Reported as an advisory rather +/// than gated: without coverage-fact reconciliation a `failed` range cannot be distinguished +/// from one superseded by a later successful retry. +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct BackfillLifecycleRow { + pub deployment_profile: String, + pub failed_job_count: i64, + pub failed_range_count: i64, + pub incomplete_range_count: i64, + pub expired_lease_range_count: i64, +} + +/// A `(chain, namespace)` an active manifest version declares. These declarations ensure +/// manifest-only chains remain in the gate's active-chain set even if a partial restore lost +/// their materialized watch rows. +#[derive(Clone, Debug, Eq, Ord, PartialEq, PartialOrd)] +pub struct ManifestChainNamespace { + pub chain: String, + pub namespace: String, +} + +/// A `(chain, source_family)` declared by an active manifest. Replay inspection uses this +/// external manifest authority to apply the same target-refresh policy as the writer. +#[derive(Clone, Debug, Eq, Ord, PartialEq, PartialOrd)] +pub struct ManifestChainSourceFamily { + pub chain: String, + pub source_family: String, +} + +/// A root, contract, or proxy-implementation address declared by an active manifest payload. +/// This remains authoritative even when a partial restore has lost its materialized +/// `manifest_contract_instances` or `contract_instance_addresses` row. +#[derive(Clone, Debug, Eq, Ord, PartialEq, PartialOrd)] +pub struct ManifestDeclaredTarget { + pub chain: String, + pub source_family: String, + pub address: String, + pub active_from_block_number: Option, +} + +/// An open discovery-edge endpoint whose contract instance has no matching open address row that +/// preserves the edge's admitted start. The edge remains authoritative even though the +/// materialized watch view cannot faithfully render it. +#[derive(Clone, Debug, Eq, Ord, PartialEq, PartialOrd)] +pub struct DiscoveryTargetMissingAddress { + pub chain: String, + pub source_family: String, + pub contract_instance_id: uuid::Uuid, +} + +/// An open resolver discovery edge whose registry source remains active while the resolver +/// target manifest required by the runtime watch view is absent. The edge remains admission +/// authority even though the materialized watch view can no longer assign its resolver family. +#[derive(Clone, Debug, Eq, Ord, PartialEq, PartialOrd)] +pub struct DiscoveryTargetMissingManifest { + pub chain: String, + pub source_family: String, + pub contract_instance_id: uuid::Uuid, +} + +/// One active manifest source that declares normalized adapter output through its ABI or the +/// admitted adapter's emitted-kind declaration, together with the count of matching +/// serving-canonical normalized events written under that exact manifest ID. +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct ActiveManifestEventSource { + pub manifest_id: i64, + pub manifest_version: i64, + pub chain: String, + pub namespace: String, + pub source_family: String, + /// Distinct normalized event kinds declared by the active manifest ABI and admitted adapter. + /// Projection-content inspection maps these declarations to the current projection writers + /// they can feed. + pub normalized_event_kinds: Vec, + pub normalized_event_count: i64, + /// Matching normalized events whose exact lineage anchor is absent or is no longer + /// canonical/safe/finalized. + pub normalized_events_missing_canonical_lineage_count: i64, + /// Matching serving-canonical events sourced from a raw log whose exact canonical raw-log + /// row, including its canonical lineage anchor, is absent. + pub normalized_events_missing_canonical_raw_log_count: i64, +} + +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct ProjectionApplyCursorRow { + pub cursor_name: String, + pub last_change_id: i64, +} + +/// A `(chain_id, lowercased address)` pair with at least one non-orphaned code observation +/// anchored to retained non-orphaned lineage. +#[derive(Clone, Debug, Eq, Ord, PartialEq, PartialOrd)] +pub struct ObservedCodeAddress { + pub chain_id: String, + pub address: String, + pub max_observed_block_number: i64, +} + +/// Non-empty `name_current` count for one namespace. `name_current` carries no chain, so a +/// namespace is the finest dimension a name projection can be scoped to. +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct NameCurrentCount { + pub namespace: String, + pub count: i64, +} + +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct DataCompletenessRead { + pub chains: Vec, + /// Deployment profile inferred from the active manifest corpus using the same authority as + /// replay admission (`mainnet` or `sepolia`). `None` is unresolved and fails closed. + pub active_deployment_profile: Option, + pub replay_cursors: Vec, + pub projection_apply_cursors: Vec, + /// `MAX(change_id)` over `projection_normalized_event_changes`, loaded independently of + /// the apply cursors so an absent cursor with a non-empty change log and a cursor ahead of + /// retained history are both detectable. An empty log has an effective high-water mark of 0. + pub max_projection_change_id: Option, + /// Rows still queued in `projection_invalidations`. A successful apply deletes the row, + /// so a fully applied projection queue is empty; a non-zero count is pending work. + pub pending_projection_invalidation_count: i64, + /// Rows in `projection_invalidation_dead_letters`: invalidations that exhausted their + /// retries. A non-zero count is a terminal projection failure. + pub projection_invalidation_dead_letter_count: i64, + pub observed_code_addresses: Vec, + /// Direct active-manifest payload targets, independently of the materialized watch view. + pub manifest_declared_targets: Vec, + /// Event-producing active manifest sources and exact-identity content counts. + pub active_manifest_event_sources: Vec, + /// `name_current` counts grouped by namespace. + pub name_current_counts: Vec, + /// Non-orphaned `normalized_events` rows with a NULL `chain_id` — a data-integrity fault + /// that would otherwise abort the per-chain read. + pub normalized_events_null_chain_id_count: i64, + /// Completed current-projection replay markers. The gate requires all current projections + /// present at the worker's current replay version with target coverage matching the worker's + /// bootstrap handoff. + pub projection_replay_markers: Vec, + /// The target a projection bootstrap would request now: the greater of the normalized + /// raw-fact replay target and the chain-checkpoint frontier. + pub projection_replay_required_target_block: Option, + /// Per-profile backfill lifecycle counts (advisory). + pub backfill_lifecycle: Vec, + /// Deferred `normalized_events` projection indexes that currently exist. A fresh replay + /// drops them and a later pass rebuilds them, so an absent index marks a mid-replay + /// candidate. + pub present_deferred_projection_indexes: Vec, + /// `(chain, namespace)` declared by active manifest versions — active-chain authority. + pub manifest_chain_namespaces: Vec, + /// `(chain, source_family)` declared by active manifests. This lets the gate interpret + /// latched replay cursors using the writer's shared adapter replay policy. + pub manifest_chain_source_families: Vec, + /// Active manifest payload targets whose declaration/implementation instance or live + /// address row does not match the payload's chain and address. A non-empty list is a + /// watch-authority gap. + pub manifest_declared_targets_missing_address: Vec, + /// Materialized manifest proxy/implementation pairs that lack the active managed discovery + /// edge consumed by the runtime watch view. + pub manifest_proxy_implementations_missing_edge: Vec, + /// Open discovery-edge endpoints with no range-preserving open address row on the edge's + /// chain. + pub discovery_targets_missing_address: Vec, + /// Open resolver edges whose active registry source has lost the active resolver manifest + /// required by the runtime watch view. + pub discovery_targets_missing_manifest: Vec, +} + +/// The deferred `normalized_events` projection indexes, owned by the replay drop/rebuild path +/// in `apps/indexer/src/main/normalized_replay_catchup/indexes.rs`. +pub const DEFERRED_NORMALIZED_EVENT_INDEXES: &[&str] = &[ + "normalized_events_namespace_idx", + "normalized_events_kind_idx", + "normalized_events_manifest_idx", + "normalized_events_chain_position_idx", + "normalized_events_name_projection_replay_idx", + "normalized_events_resource_projection_replay_idx", + "normalized_events_name_relevant_projection_idx", + "normalized_events_record_inventory_resource_replay_idx", +]; + +pub async fn load_data_completeness(pool: &sqlx::PgPool) -> Result { + load_data_completeness_with_adapter_event_kinds(pool, &[]).await +} + +pub async fn load_data_completeness_with_adapter_event_kinds( + pool: &sqlx::PgPool, + adapter_event_kind_declarations: &[(&str, &str)], +) -> Result { + Ok(DataCompletenessRead { + chains: load_chain_completeness(pool).await?, + active_deployment_profile: load_active_manifest_deployment_profile(pool).await?, + replay_cursors: load_replay_cursors(pool).await?, + projection_apply_cursors: load_projection_apply_cursors(pool).await?, + max_projection_change_id: load_max_projection_change_id(pool).await?, + pending_projection_invalidation_count: count_table(pool, "projection_invalidations") + .await?, + projection_invalidation_dead_letter_count: count_table( + pool, + "projection_invalidation_dead_letters", + ) + .await?, + observed_code_addresses: load_observed_code_addresses(pool).await?, + manifest_declared_targets: load_manifest_declared_targets(pool).await?, + active_manifest_event_sources: load_active_manifest_event_sources( + pool, + adapter_event_kind_declarations, + ) + .await?, + name_current_counts: load_name_current_counts(pool).await?, + normalized_events_null_chain_id_count: load_normalized_events_null_chain_id_count(pool) + .await?, + projection_replay_markers: load_projection_replay_markers(pool).await?, + projection_replay_required_target_block: load_projection_replay_required_target_block(pool) + .await?, + backfill_lifecycle: load_backfill_lifecycle(pool).await?, + present_deferred_projection_indexes: load_present_deferred_projection_indexes(pool).await?, + manifest_chain_namespaces: load_manifest_chain_namespaces(pool).await?, + manifest_chain_source_families: load_manifest_chain_source_families(pool).await?, + manifest_declared_targets_missing_address: load_manifest_declared_targets_missing_address( + pool, + ) + .await?, + manifest_proxy_implementations_missing_edge: + load_manifest_proxy_implementations_missing_edge(pool).await?, + discovery_targets_missing_address: load_discovery_targets_missing_address(pool).await?, + discovery_targets_missing_manifest: load_discovery_targets_missing_manifest(pool).await?, + }) +} + +async fn load_chain_completeness(pool: &sqlx::PgPool) -> Result> { + let rows = sqlx::query( + r#" + WITH known_chains AS ( + SELECT chain_id FROM chain_checkpoints + UNION + SELECT DISTINCT chain_id FROM chain_lineage + ), + canonical_lineage AS ( + SELECT chain_id, block_hash, parent_hash, block_number + FROM chain_lineage + WHERE canonicality_state IN ( + 'canonical'::canonicality_state, + 'safe'::canonicality_state, + 'finalized'::canonicality_state + ) + ), + lineage AS ( + SELECT + chain_id, + MAX(block_number) AS lineage_head_block_number, + MIN(block_number) AS lineage_floor_block_number, + COUNT(DISTINCT block_number) AS lineage_canonical_block_count + FROM canonical_lineage + GROUP BY chain_id + ), + canonical_raw_log_head AS ( + SELECT + raw_logs.chain_id, + MAX(raw_logs.block_number) AS canonical_raw_log_head_block_number + FROM raw_logs + JOIN chain_lineage + ON chain_lineage.chain_id = raw_logs.chain_id + AND chain_lineage.block_hash = raw_logs.block_hash + WHERE raw_logs.canonicality_state IN ( + 'canonical'::canonicality_state, + 'safe'::canonicality_state, + 'finalized'::canonicality_state + ) + AND chain_lineage.canonicality_state IN ( + 'canonical'::canonicality_state, + 'safe'::canonicality_state, + 'finalized'::canonicality_state + ) + GROUP BY raw_logs.chain_id + ), + raw_log_head AS ( + SELECT chain_id, MAX(block_number) AS raw_log_head_block_number + FROM raw_logs + WHERE canonicality_state <> 'orphaned'::canonicality_state + GROUP BY chain_id + ), + duplicate_canonical_height AS ( + SELECT chain_id, COUNT(*) AS duplicate_canonical_height_count + FROM ( + SELECT canonical.chain_id, canonical.block_number + FROM canonical_lineage canonical + JOIN chain_lineage sibling + ON sibling.chain_id = canonical.chain_id + AND sibling.block_number = canonical.block_number + AND sibling.canonicality_state <> 'orphaned'::canonicality_state + GROUP BY canonical.chain_id, canonical.block_number + HAVING COUNT(DISTINCT sibling.block_hash) > 1 + ) duplicated + GROUP BY chain_id + ), + disconnected_canonical_parent AS ( + SELECT child.chain_id, COUNT(*) AS disconnected_canonical_parent_count + FROM canonical_lineage child + JOIN lineage span ON span.chain_id = child.chain_id + LEFT JOIN canonical_lineage parent + ON parent.chain_id = child.chain_id + AND parent.block_hash = child.parent_hash + AND parent.block_number = child.block_number - 1 + WHERE child.block_number > span.lineage_floor_block_number + AND parent.block_hash IS NULL + GROUP BY child.chain_id + ) + SELECT + known_chains.chain_id, + chain_checkpoints.canonical_block_number, + CASE + WHEN chain_checkpoints.canonical_block_number IS NULL THEN TRUE + ELSE checkpoint_lineage.block_hash IS NOT NULL + END AS checkpoint_canonical_lineage_match, + lineage.lineage_head_block_number, + lineage.lineage_floor_block_number, + COALESCE(lineage.lineage_canonical_block_count, 0) AS lineage_canonical_block_count, + COALESCE(duplicate_canonical_height.duplicate_canonical_height_count, 0) + AS duplicate_canonical_height_count, + COALESCE(disconnected_canonical_parent.disconnected_canonical_parent_count, 0) + AS disconnected_canonical_parent_count, + canonical_raw_log_head.canonical_raw_log_head_block_number, + raw_log_head.raw_log_head_block_number + FROM known_chains + LEFT JOIN chain_checkpoints ON chain_checkpoints.chain_id = known_chains.chain_id + LEFT JOIN canonical_lineage checkpoint_lineage + ON checkpoint_lineage.chain_id = chain_checkpoints.chain_id + AND checkpoint_lineage.block_hash = chain_checkpoints.canonical_block_hash + AND checkpoint_lineage.block_number = chain_checkpoints.canonical_block_number + LEFT JOIN lineage ON lineage.chain_id = known_chains.chain_id + LEFT JOIN duplicate_canonical_height + ON duplicate_canonical_height.chain_id = known_chains.chain_id + LEFT JOIN disconnected_canonical_parent + ON disconnected_canonical_parent.chain_id = known_chains.chain_id + LEFT JOIN canonical_raw_log_head + ON canonical_raw_log_head.chain_id = known_chains.chain_id + LEFT JOIN raw_log_head ON raw_log_head.chain_id = known_chains.chain_id + ORDER BY known_chains.chain_id + "#, + ) + .fetch_all(pool) + .await + .context("failed to load chain completeness frontiers")?; + + rows.into_iter() + .map(|row| { + Ok(ChainCompletenessRow { + chain_id: crate::sql_row::get(&row, "chain_id")?, + canonical_block_number: crate::sql_row::get(&row, "canonical_block_number")?, + checkpoint_canonical_lineage_match: crate::sql_row::get( + &row, + "checkpoint_canonical_lineage_match", + )?, + lineage_head_block_number: crate::sql_row::get(&row, "lineage_head_block_number")?, + lineage_floor_block_number: crate::sql_row::get( + &row, + "lineage_floor_block_number", + )?, + lineage_canonical_block_count: crate::sql_row::get( + &row, + "lineage_canonical_block_count", + )?, + duplicate_canonical_height_count: crate::sql_row::get( + &row, + "duplicate_canonical_height_count", + )?, + disconnected_canonical_parent_count: crate::sql_row::get( + &row, + "disconnected_canonical_parent_count", + )?, + canonical_raw_log_head_block_number: crate::sql_row::get( + &row, + "canonical_raw_log_head_block_number", + )?, + raw_log_head_block_number: crate::sql_row::get(&row, "raw_log_head_block_number")?, + }) + }) + .collect() +} + +async fn load_replay_cursors(pool: &sqlx::PgPool) -> Result> { + let rows = sqlx::query( + r#" + SELECT + deployment_profile, + chain_id, + cursor_kind, + range_start_block_number, + next_block_number, + target_block_number, + last_completed_block_number, + NULLIF(last_failure_reason, '') AS last_failure_reason + FROM normalized_replay_cursors + ORDER BY deployment_profile, chain_id, cursor_kind + "#, + ) + .fetch_all(pool) + .await + .context("failed to load normalized replay cursors")?; + + rows.into_iter() + .map(|row| { + Ok(ReplayCursorRow { + deployment_profile: crate::sql_row::get(&row, "deployment_profile")?, + chain_id: crate::sql_row::get(&row, "chain_id")?, + cursor_kind: crate::sql_row::get(&row, "cursor_kind")?, + range_start_block_number: crate::sql_row::get(&row, "range_start_block_number")?, + next_block_number: crate::sql_row::get(&row, "next_block_number")?, + target_block_number: crate::sql_row::get(&row, "target_block_number")?, + last_completed_block_number: crate::sql_row::get( + &row, + "last_completed_block_number", + )?, + last_failure_reason: crate::sql_row::get(&row, "last_failure_reason")?, + }) + }) + .collect() +} + +async fn load_projection_apply_cursors( + pool: &sqlx::PgPool, +) -> Result> { + let rows = sqlx::query( + r#" + SELECT cursor_name, last_change_id + FROM projection_apply_cursors + ORDER BY cursor_name + "#, + ) + .fetch_all(pool) + .await + .context("failed to load projection apply cursors")?; + + rows.into_iter() + .map(|row| { + Ok(ProjectionApplyCursorRow { + cursor_name: crate::sql_row::get(&row, "cursor_name")?, + last_change_id: crate::sql_row::get(&row, "last_change_id")?, + }) + }) + .collect() +} + +async fn load_max_projection_change_id(pool: &sqlx::PgPool) -> Result> { + sqlx::query_scalar::<_, Option>( + "SELECT MAX(change_id) FROM projection_normalized_event_changes", + ) + .fetch_one(pool) + .await + .context("failed to load max projection change id") +} diff --git a/crates/storage/src/data_completeness/content.rs b/crates/storage/src/data_completeness/content.rs new file mode 100644 index 00000000..9cd1fd24 --- /dev/null +++ b/crates/storage/src/data_completeness/content.rs @@ -0,0 +1,443 @@ +use anyhow::{Context, Result}; + +use super::{ + ActiveManifestEventSource, BackfillLifecycleRow, DEFERRED_NORMALIZED_EVENT_INDEXES, + ManifestChainNamespace, ManifestChainSourceFamily, NameCurrentCount, ObservedCodeAddress, + ProjectionReplayMarker, +}; + +pub(super) async fn load_observed_code_addresses( + pool: &sqlx::PgPool, +) -> Result> { + let rows = sqlx::query( + r#" + SELECT + code.chain_id, + lower(code.contract_address) AS address, + MAX(code.block_number) AS max_observed_block_number + FROM raw_code_hashes code + JOIN chain_lineage lineage + ON lineage.chain_id = code.chain_id + AND lineage.block_hash = code.block_hash + AND lineage.block_number = code.block_number + AND lineage.canonicality_state <> 'orphaned'::canonicality_state + WHERE code.canonicality_state <> 'orphaned'::canonicality_state + GROUP BY code.chain_id, lower(code.contract_address) + ORDER BY chain_id, address + "#, + ) + .fetch_all(pool) + .await + .context("failed to load observed code-hash addresses")?; + + rows.into_iter() + .map(|row| { + Ok(ObservedCodeAddress { + chain_id: crate::sql_row::get(&row, "chain_id")?, + address: crate::sql_row::get(&row, "address")?, + max_observed_block_number: crate::sql_row::get(&row, "max_observed_block_number")?, + }) + }) + .collect() +} + +pub(super) async fn load_active_manifest_event_sources( + pool: &sqlx::PgPool, + adapter_event_kind_declarations: &[(&str, &str)], +) -> Result> { + let adapter_source_families = adapter_event_kind_declarations + .iter() + .map(|(source_family, _)| *source_family) + .collect::>(); + let adapter_event_kinds = adapter_event_kind_declarations + .iter() + .map(|(_, event_kind)| *event_kind) + .collect::>(); + let rows = sqlx::query( + r#" + WITH adapter_declared_kinds AS ( + SELECT declaration.source_family, declaration.event_kind + FROM UNNEST($1::TEXT[], $2::TEXT[]) + AS declaration(source_family, event_kind) + ), + active_event_kinds AS ( + SELECT + manifest.manifest_id, + manifest.manifest_version, + manifest.chain, + manifest.namespace, + manifest.source_family, + normalized_kind.event_kind + FROM manifest_versions manifest + CROSS JOIN LATERAL jsonb_array_elements( + COALESCE(manifest.manifest_payload #> '{abi,events}', '[]'::JSONB) + ) abi_event + CROSS JOIN LATERAL jsonb_array_elements_text( + COALESCE(abi_event -> 'normalized_events', '[]'::JSONB) + ) normalized_kind(event_kind) + WHERE manifest.rollout_status = 'active' + + UNION + + SELECT + manifest.manifest_id, + manifest.manifest_version, + manifest.chain, + manifest.namespace, + manifest.source_family, + adapter.event_kind + FROM manifest_versions manifest + JOIN adapter_declared_kinds adapter + ON adapter.source_family = manifest.source_family + WHERE manifest.rollout_status = 'active' + ), + active_event_sources AS ( + SELECT + manifest_id, + manifest_version, + chain, + namespace, + source_family, + ARRAY_AGG(DISTINCT event_kind ORDER BY event_kind) AS normalized_event_kinds + FROM active_event_kinds + GROUP BY + manifest_id, + manifest_version, + chain, + namespace, + source_family + ) + SELECT + source.manifest_id, + source.manifest_version, + source.chain, + source.namespace, + source.source_family, + source.normalized_event_kinds, + COUNT(event.normalized_event_id)::BIGINT AS normalized_event_count, + COUNT(event.normalized_event_id) FILTER ( + WHERE event.normalized_event_id IS NOT NULL + AND lineage.block_hash IS NULL + )::BIGINT AS normalized_events_missing_canonical_lineage_count, + COUNT(event.normalized_event_id) FILTER ( + WHERE event.normalized_event_id IS NOT NULL + AND event.raw_fact_ref ->> 'kind' = 'raw_log' + AND (raw_log.raw_log_id IS NULL OR lineage.block_hash IS NULL) + )::BIGINT AS normalized_events_missing_canonical_raw_log_count + FROM active_event_sources source + LEFT JOIN normalized_events event + ON event.source_manifest_id = source.manifest_id + AND event.manifest_version = source.manifest_version + AND event.chain_id = source.chain + AND event.namespace = source.namespace + AND event.source_family = source.source_family + AND event.event_kind = ANY(source.normalized_event_kinds) + AND event.canonicality_state IN ( + 'canonical'::canonicality_state, + 'safe'::canonicality_state, + 'finalized'::canonicality_state + ) + LEFT JOIN chain_lineage lineage + ON lineage.chain_id = event.chain_id + AND lineage.block_hash = event.block_hash + AND lineage.block_number = event.block_number + AND lineage.canonicality_state IN ( + 'canonical'::canonicality_state, + 'safe'::canonicality_state, + 'finalized'::canonicality_state + ) + LEFT JOIN raw_logs raw_log + ON event.raw_fact_ref ->> 'kind' = 'raw_log' + AND raw_log.chain_id = event.chain_id + AND raw_log.block_hash = event.block_hash + AND raw_log.block_number = event.block_number + AND raw_log.transaction_hash = event.transaction_hash + AND raw_log.log_index = event.log_index + AND raw_log.canonicality_state IN ( + 'canonical'::canonicality_state, + 'safe'::canonicality_state, + 'finalized'::canonicality_state + ) + GROUP BY + source.manifest_id, + source.manifest_version, + source.chain, + source.namespace, + source.source_family, + source.normalized_event_kinds + ORDER BY + source.chain, + source.namespace, + source.source_family, + source.manifest_version, + source.manifest_id + "#, + ) + .bind(&adapter_source_families) + .bind(&adapter_event_kinds) + .fetch_all(pool) + .await + .context("failed to load active manifest event-source counts")?; + + rows.into_iter() + .map(|row| { + Ok(ActiveManifestEventSource { + manifest_id: crate::sql_row::get(&row, "manifest_id")?, + manifest_version: crate::sql_row::get(&row, "manifest_version")?, + chain: crate::sql_row::get(&row, "chain")?, + namespace: crate::sql_row::get(&row, "namespace")?, + source_family: crate::sql_row::get(&row, "source_family")?, + normalized_event_kinds: crate::sql_row::get(&row, "normalized_event_kinds")?, + normalized_event_count: crate::sql_row::get(&row, "normalized_event_count")?, + normalized_events_missing_canonical_lineage_count: crate::sql_row::get( + &row, + "normalized_events_missing_canonical_lineage_count", + )?, + normalized_events_missing_canonical_raw_log_count: crate::sql_row::get( + &row, + "normalized_events_missing_canonical_raw_log_count", + )?, + }) + }) + .collect() +} + +pub(super) async fn count_table(pool: &sqlx::PgPool, table: &'static str) -> Result { + sqlx::query_scalar::<_, i64>(&format!("SELECT COUNT(*)::BIGINT FROM {table}")) + .fetch_one(pool) + .await + .with_context(|| format!("failed to count {table}")) +} + +pub(super) async fn load_name_current_counts(pool: &sqlx::PgPool) -> Result> { + let rows = sqlx::query( + r#" + SELECT namespace, COUNT(*)::BIGINT AS count + FROM name_current + GROUP BY namespace + ORDER BY namespace + "#, + ) + .fetch_all(pool) + .await + .context("failed to load name-current counts")?; + + rows.into_iter() + .map(|row| { + Ok(NameCurrentCount { + namespace: crate::sql_row::get(&row, "namespace")?, + count: crate::sql_row::get(&row, "count")?, + }) + }) + .collect() +} + +pub(super) async fn load_normalized_events_null_chain_id_count(pool: &sqlx::PgPool) -> Result { + sqlx::query_scalar::<_, i64>( + r#" + SELECT COUNT(*)::BIGINT + FROM normalized_events + WHERE chain_id IS NULL + AND canonicality_state <> 'orphaned'::canonicality_state + "#, + ) + .fetch_one(pool) + .await + .context("failed to count normalized events with a null chain id") +} + +pub(super) async fn load_projection_replay_markers( + pool: &sqlx::PgPool, +) -> Result> { + let rows = sqlx::query( + r#" + SELECT DISTINCT + replay_version, + projection, + completed_normalized_target_block + FROM current_projection_replay_status + ORDER BY replay_version, projection + "#, + ) + .fetch_all(pool) + .await + .context("failed to load current projection replay markers")?; + + rows.into_iter() + .map(|row| { + Ok(ProjectionReplayMarker { + replay_version: crate::sql_row::get(&row, "replay_version")?, + projection: crate::sql_row::get(&row, "projection")?, + completed_normalized_target_block: crate::sql_row::get( + &row, + "completed_normalized_target_block", + )?, + }) + }) + .collect() +} + +/// Mirrors the target passed by automatic projection bootstrap: the greater of the raw-fact +/// normalized replay target and every chain checkpoint's furthest canonicality frontier. +pub(super) async fn load_projection_replay_required_target_block( + pool: &sqlx::PgPool, +) -> Result> { + sqlx::query_scalar::<_, Option>( + r#" + SELECT NULLIF(MAX(target_block), -1) + FROM ( + SELECT COALESCE(MAX(target_block_number), -1) AS target_block + FROM normalized_replay_cursors + WHERE cursor_kind = 'raw_fact_normalized_events' + + UNION ALL + + SELECT COALESCE(MAX(GREATEST( + COALESCE(canonical_block_number, -1), + COALESCE(safe_block_number, -1), + COALESCE(finalized_block_number, -1) + )), -1) AS target_block + FROM chain_checkpoints + ) bootstrap_targets + "#, + ) + .fetch_one(pool) + .await + .context("failed to load required projection replay target block") +} + +pub(super) async fn load_backfill_lifecycle( + pool: &sqlx::PgPool, +) -> Result> { + let rows = sqlx::query( + r#" + WITH profiles AS ( + SELECT DISTINCT deployment_profile FROM backfill_jobs + ), + failed_jobs AS ( + SELECT deployment_profile, COUNT(*) AS failed_job_count + FROM backfill_jobs + WHERE status = 'failed' + GROUP BY deployment_profile + ), + ranges AS ( + SELECT + job.deployment_profile, + COUNT(*) FILTER (WHERE r.status = 'failed') AS failed_range_count, + COUNT(*) FILTER (WHERE r.status IN ('pending', 'reserved', 'running')) + AS incomplete_range_count, + COUNT(*) FILTER ( + WHERE r.status IN ('reserved', 'running') + AND r.lease_expires_at IS NOT NULL + AND r.lease_expires_at < now() + ) AS expired_lease_range_count + FROM backfill_ranges r + JOIN backfill_jobs job ON job.backfill_job_id = r.backfill_job_id + GROUP BY job.deployment_profile + ) + SELECT + profiles.deployment_profile, + COALESCE(failed_jobs.failed_job_count, 0)::BIGINT AS failed_job_count, + COALESCE(ranges.failed_range_count, 0)::BIGINT AS failed_range_count, + COALESCE(ranges.incomplete_range_count, 0)::BIGINT AS incomplete_range_count, + COALESCE(ranges.expired_lease_range_count, 0)::BIGINT AS expired_lease_range_count + FROM profiles + LEFT JOIN failed_jobs ON failed_jobs.deployment_profile = profiles.deployment_profile + LEFT JOIN ranges ON ranges.deployment_profile = profiles.deployment_profile + ORDER BY profiles.deployment_profile + "#, + ) + .fetch_all(pool) + .await + .context("failed to load backfill lifecycle counts")?; + + rows.into_iter() + .map(|row| { + Ok(BackfillLifecycleRow { + deployment_profile: crate::sql_row::get(&row, "deployment_profile")?, + failed_job_count: crate::sql_row::get(&row, "failed_job_count")?, + failed_range_count: crate::sql_row::get(&row, "failed_range_count")?, + incomplete_range_count: crate::sql_row::get(&row, "incomplete_range_count")?, + expired_lease_range_count: crate::sql_row::get(&row, "expired_lease_range_count")?, + }) + }) + .collect() +} + +pub(super) async fn load_present_deferred_projection_indexes( + pool: &sqlx::PgPool, +) -> Result> { + let expected = DEFERRED_NORMALIZED_EVENT_INDEXES + .iter() + .map(|name| (*name).to_owned()) + .collect::>(); + sqlx::query_scalar::<_, String>( + r#" + SELECT index_relation.relname + FROM pg_index index_state + JOIN pg_class index_relation ON index_relation.oid = index_state.indexrelid + JOIN pg_class table_relation ON table_relation.oid = index_state.indrelid + JOIN pg_namespace table_namespace ON table_namespace.oid = table_relation.relnamespace + WHERE table_namespace.nspname = 'public' + AND table_relation.relname = 'normalized_events' + AND index_relation.relname = ANY($1::TEXT[]) + AND index_state.indisvalid + AND index_state.indisready + ORDER BY index_relation.relname + "#, + ) + .bind(&expected) + .fetch_all(pool) + .await + .context("failed to load present deferred projection indexes") +} + +pub(super) async fn load_manifest_chain_namespaces( + pool: &sqlx::PgPool, +) -> Result> { + let rows = sqlx::query( + r#" + SELECT DISTINCT chain, namespace + FROM manifest_versions + WHERE rollout_status = 'active' + ORDER BY chain, namespace + "#, + ) + .fetch_all(pool) + .await + .context("failed to load active manifest chain namespaces")?; + + rows.into_iter() + .map(|row| { + Ok(ManifestChainNamespace { + chain: crate::sql_row::get(&row, "chain")?, + namespace: crate::sql_row::get(&row, "namespace")?, + }) + }) + .collect() +} + +pub(super) async fn load_manifest_chain_source_families( + pool: &sqlx::PgPool, +) -> Result> { + let rows = sqlx::query( + r#" + SELECT DISTINCT chain, source_family + FROM manifest_versions + WHERE rollout_status = 'active' + ORDER BY chain, source_family + "#, + ) + .fetch_all(pool) + .await + .context("failed to load active manifest chain source families")?; + + rows.into_iter() + .map(|row| { + Ok(ManifestChainSourceFamily { + chain: crate::sql_row::get(&row, "chain")?, + source_family: crate::sql_row::get(&row, "source_family")?, + }) + }) + .collect() +} diff --git a/crates/storage/src/data_completeness/manifest_targets.rs b/crates/storage/src/data_completeness/manifest_targets.rs new file mode 100644 index 00000000..c77027c0 --- /dev/null +++ b/crates/storage/src/data_completeness/manifest_targets.rs @@ -0,0 +1,369 @@ +use anyhow::{Context, Result}; + +use super::{ + DiscoveryTargetMissingAddress, DiscoveryTargetMissingManifest, ManifestDeclaredTarget, +}; + +const ACTIVE_MANIFEST_TARGETS_CTE: &str = r#" +WITH active_manifest_entries AS ( + SELECT + manifest.manifest_id, + manifest.chain, + manifest.source_family, + 'root'::TEXT AS declaration_kind, + entry ->> 'name' AS declaration_name, + lower(entry ->> 'address') AS declared_address, + (entry ->> 'start_block')::BIGINT AS start_block, + entry + FROM manifest_versions manifest + CROSS JOIN LATERAL jsonb_array_elements( + COALESCE(manifest.manifest_payload -> 'roots', '[]'::JSONB) + ) entry + WHERE manifest.rollout_status = 'active' + + UNION ALL + + SELECT + manifest.manifest_id, + manifest.chain, + manifest.source_family, + 'contract'::TEXT AS declaration_kind, + entry ->> 'role' AS declaration_name, + lower(entry ->> 'address') AS declared_address, + (entry ->> 'start_block')::BIGINT AS start_block, + entry + FROM manifest_versions manifest + CROSS JOIN LATERAL jsonb_array_elements( + COALESCE(manifest.manifest_payload -> 'contracts', '[]'::JSONB) + ) entry + WHERE manifest.rollout_status = 'active' +), +manifest_targets AS ( + SELECT + manifest_id, + chain, + source_family, + declaration_kind, + declaration_name, + declared_address, + 'declaration'::TEXT AS target_kind, + declared_address AS address, + start_block + FROM active_manifest_entries + + UNION ALL + + SELECT + manifest_id, + chain, + source_family, + declaration_kind, + declaration_name, + declared_address, + 'implementation'::TEXT AS target_kind, + lower(entry ->> 'implementation') AS address, + start_block + FROM active_manifest_entries + WHERE declaration_kind = 'contract' + AND entry ->> 'implementation' IS NOT NULL +) +"#; + +pub async fn load_active_manifest_deployment_profile( + pool: &sqlx::PgPool, +) -> Result> { + let rows = sqlx::query( + r#" + SELECT DISTINCT chain, deployment_epoch + FROM manifest_versions + WHERE rollout_status = 'active' + ORDER BY chain, deployment_epoch + "#, + ) + .fetch_all(pool) + .await + .context("failed to load active manifest corpus for deployment-profile inference")?; + + if rows.is_empty() { + return Ok(None); + } + + let rows = rows + .into_iter() + .map(|row| { + Ok(( + crate::sql_row::get::(&row, "chain")?, + crate::sql_row::get::(&row, "deployment_epoch")?, + )) + }) + .collect::>>()?; + if rows.iter().all(|(chain, _)| chain.ends_with("-mainnet")) { + return Ok(Some("mainnet".to_owned())); + } + if rows.iter().all(|(chain, deployment_epoch)| { + chain.ends_with("-sepolia") && deployment_epoch.ends_with("_sepolia_dev") + }) { + return Ok(Some("sepolia".to_owned())); + } + + Ok(None) +} + +pub(super) async fn load_manifest_declared_targets( + pool: &sqlx::PgPool, +) -> Result> { + let query = format!( + "{ACTIVE_MANIFEST_TARGETS_CTE} + SELECT DISTINCT + chain, + source_family, + address, + start_block AS active_from_block_number + FROM manifest_targets + ORDER BY chain, source_family, address, active_from_block_number" + ); + load_manifest_targets( + pool, + &query, + "failed to load active manifest-declared targets", + ) + .await +} + +pub(super) async fn load_manifest_declared_targets_missing_address( + pool: &sqlx::PgPool, +) -> Result> { + let query = format!( + "{ACTIVE_MANIFEST_TARGETS_CTE} + SELECT DISTINCT + target.chain, + target.source_family, + target.address, + target.start_block AS active_from_block_number + FROM manifest_targets target + WHERE NOT EXISTS ( + SELECT 1 + FROM manifest_contract_instances declaration + JOIN contract_instance_addresses address + ON address.contract_instance_id = CASE target.target_kind + WHEN 'declaration' THEN declaration.contract_instance_id + ELSE declaration.implementation_contract_instance_id + END + WHERE declaration.manifest_id = target.manifest_id + AND declaration.declaration_kind = target.declaration_kind + AND declaration.declaration_name = target.declaration_name + AND lower(declaration.declared_address) = target.declared_address + AND address.deactivated_at IS NULL + AND address.active_to_block_number IS NULL + AND address.chain_id = target.chain + AND lower(address.address) = target.address + AND ( + address.active_from_block_number IS NULL + OR address.active_from_block_number <= target.start_block + ) + AND ( + target.target_kind = 'declaration' + OR lower(declaration.declared_implementation_address) = target.address + ) + ) + ORDER BY target.chain, target.source_family, target.address, active_from_block_number" + ); + load_manifest_targets( + pool, + &query, + "failed to load manifest-declared targets missing a range-preserving live address row", + ) + .await +} + +pub(super) async fn load_manifest_proxy_implementations_missing_edge( + pool: &sqlx::PgPool, +) -> Result> { + let query = format!( + "{ACTIVE_MANIFEST_TARGETS_CTE} + SELECT DISTINCT + target.chain, + target.source_family, + target.address, + target.start_block AS active_from_block_number + FROM manifest_targets target + JOIN manifest_contract_instances declaration + ON declaration.manifest_id = target.manifest_id + AND declaration.declaration_kind = target.declaration_kind + AND declaration.declaration_name = target.declaration_name + AND lower(declaration.declared_address) = target.declared_address + AND lower(declaration.declared_implementation_address) = target.address + WHERE target.target_kind = 'implementation' + AND declaration.implementation_contract_instance_id IS NOT NULL + AND NOT EXISTS ( + SELECT 1 + FROM discovery_edges edge + WHERE edge.chain_id = target.chain + AND edge.edge_kind = 'proxy_implementation' + AND edge.from_contract_instance_id = declaration.contract_instance_id + AND edge.to_contract_instance_id = + declaration.implementation_contract_instance_id + AND edge.discovery_source = 'manifest_declared_proxy' + AND edge.source_manifest_id = target.manifest_id + AND edge.admission = 'manifest_declared' + AND edge.deactivated_at IS NULL + AND edge.active_to_block_number IS NULL + AND ( + edge.active_from_block_number IS NULL + OR edge.active_from_block_number <= target.start_block + ) + ) + ORDER BY target.chain, target.source_family, target.address, active_from_block_number" + ); + load_manifest_targets( + pool, + &query, + "failed to load manifest proxy implementations missing their managed edge", + ) + .await +} + +pub(super) async fn load_discovery_targets_missing_manifest( + pool: &sqlx::PgPool, +) -> Result> { + let rows = sqlx::query( + r#" + SELECT DISTINCT + edge.chain_id AS chain, + CASE source_manifest.source_family + WHEN 'ens_v1_registry_l1' THEN 'ens_v1_resolver_l1' + WHEN 'ens_v2_registry_l1' THEN 'ens_v2_resolver_l1' + WHEN 'basenames_base_registry' THEN 'basenames_base_resolver' + END AS source_family, + edge.to_contract_instance_id AS contract_instance_id + FROM discovery_edges edge + JOIN manifest_versions source_manifest + ON source_manifest.manifest_id = edge.source_manifest_id + AND source_manifest.rollout_status = 'active' + LEFT JOIN manifest_versions target_manifest + ON target_manifest.rollout_status = 'active' + AND target_manifest.namespace = source_manifest.namespace + AND target_manifest.chain = edge.chain_id + AND target_manifest.deployment_epoch = source_manifest.deployment_epoch + AND target_manifest.source_family = CASE source_manifest.source_family + WHEN 'ens_v1_registry_l1' THEN 'ens_v1_resolver_l1' + WHEN 'ens_v2_registry_l1' THEN 'ens_v2_resolver_l1' + WHEN 'basenames_base_registry' THEN 'basenames_base_resolver' + END + WHERE edge.edge_kind = 'resolver' + AND source_manifest.source_family IN ( + 'ens_v1_registry_l1', + 'ens_v2_registry_l1', + 'basenames_base_registry' + ) + AND edge.deactivated_at IS NULL + AND edge.active_to_block_number IS NULL + AND target_manifest.manifest_id IS NULL + ORDER BY chain, source_family, contract_instance_id + "#, + ) + .fetch_all(pool) + .await + .context("failed to load active resolver discovery targets missing their target manifest")?; + + rows.into_iter() + .map(|row| { + Ok(DiscoveryTargetMissingManifest { + chain: crate::sql_row::get(&row, "chain")?, + source_family: crate::sql_row::get(&row, "source_family")?, + contract_instance_id: crate::sql_row::get(&row, "contract_instance_id")?, + }) + }) + .collect() +} + +pub(super) async fn load_discovery_targets_missing_address( + pool: &sqlx::PgPool, +) -> Result> { + let rows = sqlx::query( + r#" + SELECT DISTINCT + edge.chain_id AS chain, + COALESCE(target_manifest.source_family, source_manifest.source_family) + AS source_family, + edge.to_contract_instance_id AS contract_instance_id + FROM discovery_edges edge + JOIN manifest_versions source_manifest + ON source_manifest.manifest_id = edge.source_manifest_id + LEFT JOIN manifest_versions target_manifest + ON target_manifest.rollout_status = 'active' + AND target_manifest.namespace = source_manifest.namespace + AND target_manifest.chain = edge.chain_id + AND target_manifest.deployment_epoch = source_manifest.deployment_epoch + AND target_manifest.source_family = CASE + WHEN edge.edge_kind = 'resolver' + AND source_manifest.source_family = 'ens_v1_registry_l1' + THEN 'ens_v1_resolver_l1' + WHEN edge.edge_kind = 'resolver' + AND source_manifest.source_family = 'ens_v2_registry_l1' + THEN 'ens_v2_resolver_l1' + WHEN edge.edge_kind = 'resolver' + AND source_manifest.source_family = 'basenames_base_registry' + THEN 'basenames_base_resolver' + ELSE NULL + END + WHERE source_manifest.rollout_status = 'active' + AND edge.deactivated_at IS NULL + AND edge.active_to_block_number IS NULL + AND edge.edge_kind <> 'migration' + AND ( + edge.edge_kind <> 'resolver' + OR source_manifest.source_family NOT IN ( + 'ens_v1_registry_l1', + 'ens_v2_registry_l1', + 'basenames_base_registry' + ) + OR target_manifest.manifest_id IS NOT NULL + ) + AND NOT EXISTS ( + SELECT 1 + FROM contract_instance_addresses address + WHERE address.contract_instance_id = edge.to_contract_instance_id + AND address.chain_id = edge.chain_id + AND address.deactivated_at IS NULL + AND address.active_to_block_number IS NULL + AND ( + address.active_from_block_number IS NULL + OR address.active_from_block_number <= edge.active_from_block_number + ) + ) + ORDER BY chain, source_family, contract_instance_id + "#, + ) + .fetch_all(pool) + .await + .context("failed to load active discovery targets missing range-preserving live addresses")?; + + rows.into_iter() + .map(|row| { + Ok(DiscoveryTargetMissingAddress { + chain: crate::sql_row::get(&row, "chain")?, + source_family: crate::sql_row::get(&row, "source_family")?, + contract_instance_id: crate::sql_row::get(&row, "contract_instance_id")?, + }) + }) + .collect() +} + +async fn load_manifest_targets( + pool: &sqlx::PgPool, + query: &str, + context: &'static str, +) -> Result> { + let rows = sqlx::query(query).fetch_all(pool).await.context(context)?; + rows.into_iter() + .map(|row| { + Ok(ManifestDeclaredTarget { + chain: crate::sql_row::get(&row, "chain")?, + source_family: crate::sql_row::get(&row, "source_family")?, + address: crate::sql_row::get(&row, "address")?, + active_from_block_number: crate::sql_row::get(&row, "active_from_block_number")?, + }) + }) + .collect() +} diff --git a/crates/storage/src/data_completeness/projection_content.rs b/crates/storage/src/data_completeness/projection_content.rs new file mode 100644 index 00000000..05446e4b --- /dev/null +++ b/crates/storage/src/data_completeness/projection_content.rs @@ -0,0 +1,190 @@ +use anyhow::{Context, Result, bail}; +use sqlx::{PgPool, Row}; + +use crate::{ + address_names::DEFAULT_ADDRESS_NAMES_CURRENT_READ_FILTER, + children::{DECLARED_SURFACE_CLASS, DEFAULT_CHILDREN_CURRENT_READ_FILTER}, + name_current::DEFAULT_NAME_CURRENT_READ_FILTER, + permissions::DEFAULT_PERMISSIONS_CURRENT_READ_FILTER, + record_inventory::DEFAULT_RECORD_INVENTORY_CURRENT_READ_FILTER, +}; + +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct ProjectionContentScopeCount { + pub scope: String, + pub count: i64, +} + +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct ProjectionContentCounts { + pub scope_kind: &'static str, + pub raw_total_count: i64, + pub raw_scoped_counts: Vec, + pub servable_total_count: i64, + pub servable_scoped_counts: Vec, +} + +/// Count one current projection both as stored and through the same supporting-row validity +/// rules used by its normal serving reads. +pub async fn load_projection_content_counts( + pool: &PgPool, + projection: &str, +) -> Result { + let (scope_kind, raw_query, servable_query) = projection_queries(projection)?; + let raw_scoped_counts = load_scoped_counts(pool, projection, "raw", raw_query.as_str()).await?; + let servable_scoped_counts = + load_scoped_counts(pool, projection, "servable", servable_query.as_str()).await?; + + Ok(ProjectionContentCounts { + scope_kind, + raw_total_count: raw_scoped_counts.iter().map(|entry| entry.count).sum(), + raw_scoped_counts, + servable_total_count: servable_scoped_counts.iter().map(|entry| entry.count).sum(), + servable_scoped_counts, + }) +} + +fn projection_queries(projection: &str) -> Result<(&'static str, String, String)> { + let queries = match projection { + "name_current" => ( + "namespace", + grouped_query("name_current", "namespace"), + format!( + r#" + SELECT nc.namespace AS scope, COUNT(*)::BIGINT AS count + FROM name_current nc + JOIN name_surfaces surface + ON surface.logical_name_id = nc.logical_name_id + LEFT JOIN resources resource + ON resource.resource_id = nc.resource_id + LEFT JOIN surface_bindings binding + ON binding.surface_binding_id = nc.surface_binding_id + LEFT JOIN token_lineages token_lineage + ON token_lineage.token_lineage_id = nc.token_lineage_id + WHERE TRUE + {DEFAULT_NAME_CURRENT_READ_FILTER} + GROUP BY nc.namespace + ORDER BY nc.namespace + "#, + ), + ), + "children_current" => ( + "namespace", + grouped_query("children_current", "namespace"), + format!( + r#" + SELECT cc.namespace AS scope, COUNT(*)::BIGINT AS count + FROM children_current cc + JOIN name_surfaces parent + ON parent.logical_name_id = cc.parent_logical_name_id + LEFT JOIN name_surfaces child + ON child.logical_name_id = cc.child_logical_name_id + WHERE cc.surface_class = '{DECLARED_SURFACE_CLASS}' + {DEFAULT_CHILDREN_CURRENT_READ_FILTER} + GROUP BY cc.namespace + ORDER BY cc.namespace + "#, + ), + ), + "permissions_current" => ( + "chain", + resource_grouped_query("permissions_current", "pc"), + format!( + r#" + SELECT resource.chain_id AS scope, COUNT(*)::BIGINT AS count + FROM permissions_current pc + JOIN resources resource + ON resource.resource_id = pc.resource_id + WHERE TRUE + {DEFAULT_PERMISSIONS_CURRENT_READ_FILTER} + GROUP BY resource.chain_id + ORDER BY resource.chain_id + "#, + ), + ), + "record_inventory_current" => ( + "chain", + resource_grouped_query("record_inventory_current", "ric"), + format!( + r#" + SELECT resource.chain_id AS scope, COUNT(*)::BIGINT AS count + FROM record_inventory_current ric + JOIN resources resource + ON resource.resource_id = ric.resource_id + WHERE TRUE + {DEFAULT_RECORD_INVENTORY_CURRENT_READ_FILTER} + GROUP BY resource.chain_id + ORDER BY resource.chain_id + "#, + ), + ), + "resolver_current" => { + let query = grouped_query("resolver_current", "chain_id"); + ("chain", query.clone(), query) + } + "address_names_current" => ( + "namespace", + grouped_query("address_names_current", "namespace"), + format!( + r#" + SELECT anc.namespace AS scope, COUNT(*)::BIGINT AS count + FROM address_names_current anc + JOIN name_surfaces surface + ON surface.logical_name_id = anc.logical_name_id + JOIN resources resource + ON resource.resource_id = anc.resource_id + JOIN surface_bindings binding + ON binding.surface_binding_id = anc.surface_binding_id + LEFT JOIN token_lineages token_lineage + ON token_lineage.token_lineage_id = anc.token_lineage_id + WHERE TRUE + {DEFAULT_ADDRESS_NAMES_CURRENT_READ_FILTER} + GROUP BY anc.namespace + ORDER BY anc.namespace + "#, + ), + ), + "primary_names_current" => { + let query = grouped_query("primary_names_current", "namespace"); + ("namespace", query.clone(), query) + } + _ => bail!("current projection {projection} has no content-count rule"), + }; + Ok(queries) +} + +fn grouped_query(projection: &str, scope_column: &str) -> String { + format!( + "SELECT {scope_column} AS scope, COUNT(*)::BIGINT AS count \ + FROM {projection} GROUP BY {scope_column} ORDER BY {scope_column}" + ) +} + +fn resource_grouped_query(projection: &str, alias: &str) -> String { + format!( + "SELECT resource.chain_id AS scope, COUNT(*)::BIGINT AS count \ + FROM {projection} {alias} JOIN resources resource \ + ON resource.resource_id = {alias}.resource_id \ + GROUP BY resource.chain_id ORDER BY resource.chain_id" + ) +} + +async fn load_scoped_counts( + pool: &PgPool, + projection: &str, + count_kind: &str, + query: &str, +) -> Result> { + let rows = sqlx::query(query) + .fetch_all(pool) + .await + .with_context(|| format!("failed to load {count_kind} content counts for {projection}"))?; + rows.into_iter() + .map(|row| { + Ok(ProjectionContentScopeCount { + scope: row.try_get("scope")?, + count: row.try_get("count")?, + }) + }) + .collect() +} diff --git a/crates/storage/src/data_completeness/tests.rs b/crates/storage/src/data_completeness/tests.rs new file mode 100644 index 00000000..48241ab1 --- /dev/null +++ b/crates/storage/src/data_completeness/tests.rs @@ -0,0 +1,1492 @@ +use anyhow::Result; +use bigname_test_support::{TestDatabase, TestDatabaseConfig}; + +use super::{load_data_completeness, load_data_completeness_with_adapter_event_kinds}; + +async fn test_database() -> Result { + TestDatabase::create_migrated( + TestDatabaseConfig::new("bigname_storage_data_completeness_test") + .admin_database("postgres") + .pool_max_connections(5) + .parse_context("failed to parse database URL for data_completeness tests") + .admin_connect_context("failed to connect admin pool for data_completeness tests") + .pool_connect_context("failed to connect data_completeness test pool"), + &crate::MIGRATOR, + "failed to apply migrations for data_completeness tests", + ) + .await +} + +/// A serving-canonical height with any non-orphaned sibling is an ambiguous promotion path the +/// distinct-block-number count cannot see; the dedicated CTE must count the height. +#[tokio::test] +async fn non_orphaned_same_height_forks_are_counted() -> Result<()> { + let database = test_database().await?; + let pool = database.pool(); + sqlx::query( + r#" + INSERT INTO chain_lineage + (chain_id, block_hash, block_number, block_timestamp, canonicality_state) + VALUES + ('ethereum-sepolia', '0xaa', 100, now(), 'canonical'::canonicality_state), + ('ethereum-sepolia', '0xbb', 100, now(), 'observed'::canonicality_state), + ('ethereum-sepolia', '0xcc', 101, now(), 'canonical'::canonicality_state) + "#, + ) + .execute(pool) + .await?; + + let read = load_data_completeness(pool).await?; + let chain = read + .chains + .iter() + .find(|chain| chain.chain_id == "ethereum-sepolia") + .expect("chain row"); + assert_eq!(chain.duplicate_canonical_height_count, 1); + // Distinct block numbers are 100 and 101, so the span-based contiguity count is blind. + assert_eq!(chain.lineage_canonical_block_count, 2); + + database.cleanup().await +} + +/// A complete set of heights is not a connected branch when a child does not point to the +/// canonical hash at the preceding height. +#[tokio::test] +async fn disconnected_canonical_parents_are_counted() -> Result<()> { + let database = test_database().await?; + let pool = database.pool(); + sqlx::query( + r#" + INSERT INTO chain_lineage + (chain_id, block_hash, parent_hash, block_number, block_timestamp, + canonicality_state) + VALUES + ('ethereum-sepolia', '0xaa', '0x99', 100, now(), 'canonical'), + ('ethereum-sepolia', '0xbb', '0xdead', 101, now(), 'canonical') + "#, + ) + .execute(pool) + .await?; + + let read = load_data_completeness(pool).await?; + let chain = read + .chains + .iter() + .find(|chain| chain.chain_id == "ethereum-sepolia") + .expect("chain row"); + assert_eq!(chain.lineage_canonical_block_count, 2); + assert_eq!(chain.disconnected_canonical_parent_count, 1); + + database.cleanup().await +} + +/// Checkpoint progress is anchored by both number and hash. A same-height lineage row on a +/// different hash, or an orphaned exact row, cannot prove the checkpoint's branch anchor. +#[tokio::test] +async fn checkpoint_hash_requires_exact_canonical_lineage_anchor() -> Result<()> { + let database = test_database().await?; + let pool = database.pool(); + sqlx::query( + r#" + INSERT INTO chain_lineage + (chain_id, block_hash, block_number, block_timestamp, canonicality_state) + VALUES ('ethereum-sepolia', '0xRIGHT', 100, now(), 'canonical') + "#, + ) + .execute(pool) + .await?; + sqlx::query( + r#" + INSERT INTO chain_checkpoints + (chain_id, canonical_block_hash, canonical_block_number) + VALUES ('ethereum-sepolia', '0xWRONG', 100) + "#, + ) + .execute(pool) + .await?; + + let read = load_data_completeness(pool).await?; + assert!(!read.chains[0].checkpoint_canonical_lineage_match); + + sqlx::query( + "UPDATE chain_checkpoints SET canonical_block_hash = '0xRIGHT' WHERE chain_id = 'ethereum-sepolia'", + ) + .execute(pool) + .await?; + let read = load_data_completeness(pool).await?; + assert!(read.chains[0].checkpoint_canonical_lineage_match); + + sqlx::query( + "UPDATE chain_lineage SET canonicality_state = 'orphaned' WHERE chain_id = 'ethereum-sepolia'", + ) + .execute(pool) + .await?; + let read = load_data_completeness(pool).await?; + assert!(!read.chains[0].checkpoint_canonical_lineage_match); + + database.cleanup().await +} + +/// Code coverage needs the observation height so the evaluator can reject observations from +/// before a target's inclusive active start. +#[tokio::test] +async fn latest_code_observation_block_is_loaded() -> Result<()> { + let database = test_database().await?; + let pool = database.pool(); + sqlx::query( + r#" + INSERT INTO chain_lineage + (chain_id, block_hash, block_number, block_timestamp, canonicality_state) + VALUES + ('ethereum-sepolia', '0xaa', 10, now(), 'canonical'), + ('ethereum-sepolia', '0xbb', 20, now(), 'canonical'), + ('ethereum-sepolia', '0xee', 50, now(), 'orphaned') + "#, + ) + .execute(pool) + .await?; + sqlx::query( + r#" + INSERT INTO raw_code_hashes + (chain_id, block_hash, block_number, contract_address, code_hash, + code_byte_length, canonicality_state) + VALUES + ('ethereum-sepolia', '0xaa', 10, '0xabc', '0x01', 1, 'canonical'), + ('ethereum-sepolia', '0xbb', 20, '0xAbC', '0x02', 1, 'canonical'), + ('ethereum-sepolia', '0xcc', 30, '0xabc', '0x03', 1, 'orphaned'), + ('ethereum-sepolia', '0xdd', 40, '0xabc', '0x04', 1, 'canonical'), + ('ethereum-sepolia', '0xee', 50, '0xabc', '0x05', 1, 'canonical') + "#, + ) + .execute(pool) + .await?; + + let read = load_data_completeness(pool).await?; + assert_eq!(read.observed_code_addresses.len(), 1); + assert_eq!( + read.observed_code_addresses[0].max_observed_block_number, + 20 + ); + + database.cleanup().await +} + +/// Direct manifest declarations are loaded without depending on the materialized +/// `contract_instance_addresses` row. +#[tokio::test] +async fn manifest_declared_target_is_loaded_without_address_row() -> Result<()> { + let database = test_database().await?; + let pool = database.pool(); + let manifest_id = sqlx::query_scalar::<_, i64>( + r#" + INSERT INTO manifest_versions + (manifest_version, namespace, source_family, chain, deployment_epoch, + rollout_status, normalizer_version, file_path, manifest_payload) + VALUES + (1, 'ens', 'ens_v2_registry_l1', 'ethereum-sepolia', 'e', 'active', + 'n', 'f', '{"contracts":[{"role":"registry","address":"0xAbC","start_block":42}]}'::jsonb) + RETURNING manifest_id + "#, + ) + .fetch_one(pool) + .await?; + let contract_instance_id = uuid::Uuid::parse_str("00000000-0000-0000-0000-000000000042")?; + sqlx::query( + r#" + INSERT INTO contract_instances + (contract_instance_id, chain_id, contract_kind) + VALUES ($1, 'ethereum-sepolia', 'registry') + "#, + ) + .bind(contract_instance_id) + .execute(pool) + .await?; + sqlx::query( + r#" + INSERT INTO manifest_contract_instances + (manifest_id, declaration_kind, declaration_name, contract_instance_id, + declared_address, role, proxy_kind) + VALUES ($1, 'contract', 'registry', $2, '0xAbC', 'registry', 'none') + "#, + ) + .bind(manifest_id) + .bind(contract_instance_id) + .execute(pool) + .await?; + + let read = load_data_completeness(pool).await?; + assert_eq!(read.manifest_declared_targets.len(), 1); + assert_eq!(read.manifest_declared_targets[0].address, "0xabc"); + assert_eq!( + read.manifest_declared_targets[0].active_from_block_number, + Some(42) + ); + + // Direct history authority stays at the payload start, and the materialized row must not + // narrow the runtime watch/coverage interval above it. + sqlx::query( + r#" + INSERT INTO contract_instance_addresses + (contract_instance_id, chain_id, address, active_from_block_number) + VALUES ($1, 'ethereum-sepolia', '0xAbC', 900) + "#, + ) + .bind(contract_instance_id) + .execute(pool) + .await?; + let read = load_data_completeness(pool).await?; + assert_eq!( + read.manifest_declared_targets[0].active_from_block_number, + Some(42) + ); + assert_eq!(read.manifest_declared_targets_missing_address.len(), 1); + + sqlx::query( + r#" + UPDATE contract_instance_addresses + SET active_from_block_number = 42 + WHERE contract_instance_id = $1 + "#, + ) + .bind(contract_instance_id) + .execute(pool) + .await?; + let read = load_data_completeness(pool).await?; + assert!(read.manifest_declared_targets_missing_address.is_empty()); + + database.cleanup().await +} + +/// A live address row only satisfies a manifest declaration when both its chain and address +/// match and remain open. Deactivated, closed, and mismatched rows remain explicit authority gaps. +#[tokio::test] +async fn manifest_declared_targets_require_matching_live_address_rows() -> Result<()> { + let database = test_database().await?; + let pool = database.pool(); + let manifest_id = sqlx::query_scalar::<_, i64>( + r#" + INSERT INTO manifest_versions + (manifest_version, namespace, source_family, chain, deployment_epoch, + rollout_status, normalizer_version, file_path, manifest_payload) + VALUES + (1, 'ens', 'ens_v2_registry_l1', 'ethereum-sepolia', 'e', 'active', + 'n', 'f', '{"contracts":[ + {"role":"exact","address":"0xEXACT"}, + {"role":"deactivated","address":"0xGONE"}, + {"role":"closed","address":"0xCLOSED"}, + {"role":"wrong_address","address":"0xDECLARED"}, + {"role":"wrong_chain","address":"0xCHAIN"}, + {"role":"late_start","address":"0xLATE","start_block":100}, + {"role":"unknown_start","address":"0xUNKNOWN"} + ]}'::jsonb) + RETURNING manifest_id + "#, + ) + .fetch_one(pool) + .await?; + sqlx::query( + r#" + INSERT INTO contract_instances (contract_instance_id, chain_id, contract_kind) + VALUES + ('11111111-1111-1111-1111-111111111111', 'ethereum-sepolia', 'contract'), + ('22222222-2222-2222-2222-222222222222', 'ethereum-sepolia', 'contract'), + ('55555555-5555-5555-5555-555555555555', 'ethereum-sepolia', 'contract'), + ('33333333-3333-3333-3333-333333333333', 'ethereum-sepolia', 'contract'), + ('44444444-4444-4444-4444-444444444444', 'ethereum-sepolia', 'contract'), + ('66666666-6666-6666-6666-666666666666', 'ethereum-sepolia', 'contract'), + ('77777777-7777-7777-7777-777777777777', 'ethereum-sepolia', 'contract') + "#, + ) + .execute(pool) + .await?; + sqlx::query( + r#" + INSERT INTO manifest_contract_instances + (manifest_id, declaration_kind, declaration_name, contract_instance_id, + declared_address, role, proxy_kind) + VALUES + ($1, 'contract', 'exact', + '11111111-1111-1111-1111-111111111111', '0xEXACT', 'exact', 'none'), + ($1, 'contract', 'deactivated', + '22222222-2222-2222-2222-222222222222', '0xGONE', 'deactivated', 'none'), + ($1, 'contract', 'closed', + '55555555-5555-5555-5555-555555555555', '0xCLOSED', 'closed', 'none'), + ($1, 'contract', 'wrong_address', + '33333333-3333-3333-3333-333333333333', '0xDECLARED', 'wrong_address', 'none'), + ($1, 'contract', 'wrong_chain', + '44444444-4444-4444-4444-444444444444', '0xCHAIN', 'wrong_chain', 'none'), + ($1, 'contract', 'late_start', + '66666666-6666-6666-6666-666666666666', '0xLATE', 'late_start', 'none'), + ($1, 'contract', 'unknown_start', + '77777777-7777-7777-7777-777777777777', '0xUNKNOWN', 'unknown_start', 'none') + "#, + ) + .bind(manifest_id) + .execute(pool) + .await?; + sqlx::query( + r#" + INSERT INTO contract_instance_addresses + (contract_instance_id, chain_id, address, deactivated_at, + active_from_block_number, active_to_block_number) + VALUES + ('11111111-1111-1111-1111-111111111111', 'ethereum-sepolia', '0xexact', NULL, NULL, NULL), + ('22222222-2222-2222-2222-222222222222', 'ethereum-sepolia', '0xgone', now(), NULL, NULL), + ('55555555-5555-5555-5555-555555555555', 'ethereum-sepolia', '0xclosed', NULL, NULL, 99), + ('33333333-3333-3333-3333-333333333333', 'ethereum-sepolia', '0xOTHER', NULL, NULL, NULL), + ('44444444-4444-4444-4444-444444444444', 'base-sepolia', '0xchain', NULL, NULL, NULL), + ('66666666-6666-6666-6666-666666666666', 'ethereum-sepolia', '0xlate', NULL, 900, NULL), + ('77777777-7777-7777-7777-777777777777', 'ethereum-sepolia', '0xunknown', NULL, 900, NULL) + "#, + ) + .execute(pool) + .await?; + + let read = load_data_completeness(pool).await?; + let missing = read + .manifest_declared_targets_missing_address + .iter() + .map(|target| target.address.as_str()) + .collect::>(); + assert_eq!( + missing, + [ + "0xchain", + "0xclosed", + "0xdeclared", + "0xgone", + "0xlate", + "0xunknown" + ] + .into_iter() + .collect() + ); + assert!(!missing.contains("0xexact")); + + sqlx::raw_sql( + r#" + UPDATE contract_instance_addresses + SET active_from_block_number = 50 + WHERE contract_instance_id = '66666666-6666-6666-6666-666666666666'; + + UPDATE contract_instance_addresses + SET active_from_block_number = NULL + WHERE contract_instance_id = '77777777-7777-7777-7777-777777777777' + "#, + ) + .execute(pool) + .await?; + let read = load_data_completeness(pool).await?; + let missing = read + .manifest_declared_targets_missing_address + .iter() + .map(|target| target.address.as_str()) + .collect::>(); + assert!(!missing.contains("0xlate")); + assert!(!missing.contains("0xunknown")); + + database.cleanup().await +} + +/// The manifest payload is authority for direct declarations. Losing a materialized +/// `manifest_contract_instances` child must leave the payload target in both the coverage +/// universe and the explicit materialization-gap report. +#[tokio::test] +async fn manifest_payload_target_survives_missing_contract_instance_row() -> Result<()> { + let database = test_database().await?; + let pool = database.pool(); + let manifest_id = sqlx::query_scalar::<_, i64>( + r#" + INSERT INTO manifest_versions + (manifest_version, namespace, source_family, chain, deployment_epoch, + rollout_status, normalizer_version, file_path, manifest_payload) + VALUES + (1, 'ens', 'ens_v2_registry_l1', 'ethereum-sepolia', 'e', 'active', + 'n', 'f', '{ + "roots":[{"name":"root","address":"0xROOT","start_block":7}], + "contracts":[{"role":"registry","address":"0xREGISTRY","start_block":11}] + }'::jsonb) + RETURNING manifest_id + "#, + ) + .fetch_one(pool) + .await?; + sqlx::query( + r#" + INSERT INTO contract_instances (contract_instance_id, chain_id, contract_kind) + VALUES ('55555555-5555-5555-5555-555555555555', 'ethereum-sepolia', 'root') + "#, + ) + .execute(pool) + .await?; + sqlx::query( + r#" + INSERT INTO manifest_contract_instances + (manifest_id, declaration_kind, declaration_name, contract_instance_id, + declared_address) + VALUES + ($1, 'root', 'root', + '55555555-5555-5555-5555-555555555555', '0xROOT') + "#, + ) + .bind(manifest_id) + .execute(pool) + .await?; + sqlx::query( + r#" + INSERT INTO contract_instance_addresses + (contract_instance_id, chain_id, address) + VALUES + ('55555555-5555-5555-5555-555555555555', 'ethereum-sepolia', '0xROOT') + "#, + ) + .execute(pool) + .await?; + + let read = load_data_completeness(pool).await?; + let targets = read + .manifest_declared_targets + .iter() + .map(|target| (target.address.as_str(), target.active_from_block_number)) + .collect::>(); + assert_eq!( + targets, + [("0xregistry", Some(11)), ("0xroot", Some(7))] + .into_iter() + .collect() + ); + assert_eq!(read.manifest_declared_targets_missing_address.len(), 1); + assert_eq!( + read.manifest_declared_targets_missing_address[0].address, + "0xregistry" + ); + + database.cleanup().await +} + +/// A manifest-declared proxy implementation is a separate admitted and watched instance. Its +/// payload address must remain a direct coverage target and require its own live address row. +#[tokio::test] +async fn manifest_proxy_implementation_is_a_declared_target() -> Result<()> { + let database = test_database().await?; + let pool = database.pool(); + let manifest_id = sqlx::query_scalar::<_, i64>( + r#" + INSERT INTO manifest_versions + (manifest_version, namespace, source_family, chain, deployment_epoch, + rollout_status, normalizer_version, file_path, manifest_payload) + VALUES + (1, 'ens', 'ens_v2_registry_l1', 'ethereum-sepolia', 'e', 'active', + 'n', 'f', '{"contracts":[{ + "role":"registry", + "address":"0xPROXY", + "proxy_kind":"uups", + "implementation":"0xIMPLEMENTATION", + "start_block":17 + }]}'::jsonb) + RETURNING manifest_id + "#, + ) + .fetch_one(pool) + .await?; + sqlx::query( + r#" + INSERT INTO contract_instances (contract_instance_id, chain_id, contract_kind) + VALUES + ('66666666-6666-6666-6666-666666666666', 'ethereum-sepolia', 'contract'), + ('77777777-7777-7777-7777-777777777777', 'ethereum-sepolia', 'contract') + "#, + ) + .execute(pool) + .await?; + sqlx::query( + r#" + INSERT INTO manifest_contract_instances + (manifest_id, declaration_kind, declaration_name, contract_instance_id, + declared_address, role, proxy_kind, implementation_contract_instance_id, + declared_implementation_address) + VALUES + ($1, 'contract', 'registry', + '66666666-6666-6666-6666-666666666666', '0xPROXY', 'registry', 'uups', + '77777777-7777-7777-7777-777777777777', '0xIMPLEMENTATION') + "#, + ) + .bind(manifest_id) + .execute(pool) + .await?; + sqlx::query( + r#" + INSERT INTO contract_instance_addresses + (contract_instance_id, chain_id, address) + VALUES + ('66666666-6666-6666-6666-666666666666', 'ethereum-sepolia', '0xPROXY') + "#, + ) + .execute(pool) + .await?; + + let read = load_data_completeness(pool).await?; + let targets = read + .manifest_declared_targets + .iter() + .map(|target| (target.address.as_str(), target.active_from_block_number)) + .collect::>(); + assert_eq!( + targets, + [("0ximplementation", Some(17)), ("0xproxy", Some(17))] + .into_iter() + .collect() + ); + assert_eq!(read.manifest_declared_targets_missing_address.len(), 1); + assert_eq!( + read.manifest_declared_targets_missing_address[0].address, + "0ximplementation" + ); + + sqlx::query( + r#" + INSERT INTO contract_instance_addresses + (contract_instance_id, chain_id, address) + VALUES + ('77777777-7777-7777-7777-777777777777', + 'ethereum-sepolia', '0xIMPLEMENTATION') + "#, + ) + .execute(pool) + .await?; + let read = load_data_completeness(pool).await?; + assert!(read.manifest_declared_targets_missing_address.is_empty()); + assert_eq!(read.manifest_proxy_implementations_missing_edge.len(), 1); + assert_eq!( + read.manifest_proxy_implementations_missing_edge[0].address, + "0ximplementation" + ); + + sqlx::query( + r#" + INSERT INTO discovery_edges + (chain_id, edge_kind, from_contract_instance_id, to_contract_instance_id, + discovery_source, source_manifest_id, admission) + VALUES + ('ethereum-sepolia', 'proxy_implementation', + '66666666-6666-6666-6666-666666666666', + '77777777-7777-7777-7777-777777777777', + 'manifest_declared_proxy', $1, 'manifest_declared') + "#, + ) + .bind(manifest_id) + .execute(pool) + .await?; + let read = load_data_completeness(pool).await?; + assert!(read.manifest_proxy_implementations_missing_edge.is_empty()); + + sqlx::query( + r#" + UPDATE discovery_edges + SET active_from_block_number = 99 + WHERE source_manifest_id = $1 + AND edge_kind = 'proxy_implementation' + "#, + ) + .bind(manifest_id) + .execute(pool) + .await?; + let read = load_data_completeness(pool).await?; + assert_eq!(read.manifest_proxy_implementations_missing_edge.len(), 1); + + sqlx::query( + r#" + UPDATE discovery_edges + SET active_from_block_number = 10 + WHERE source_manifest_id = $1 + AND edge_kind = 'proxy_implementation' + "#, + ) + .bind(manifest_id) + .execute(pool) + .await?; + let read = load_data_completeness(pool).await?; + assert!(read.manifest_proxy_implementations_missing_edge.is_empty()); + + sqlx::query( + r#" + UPDATE discovery_edges + SET active_to_block_number = 99 + WHERE source_manifest_id = $1 + AND edge_kind = 'proxy_implementation' + "#, + ) + .bind(manifest_id) + .execute(pool) + .await?; + let read = load_data_completeness(pool).await?; + assert_eq!(read.manifest_proxy_implementations_missing_edge.len(), 1); + assert_eq!( + read.manifest_proxy_implementations_missing_edge[0].address, + "0ximplementation" + ); + + database.cleanup().await +} + +/// An open discovery edge remains current watch authority only while its target also has an open +/// address. Bounded edges are retained history and must not create current-authority failures. +#[tokio::test] +async fn discovery_target_without_live_address_is_surfaced() -> Result<()> { + let database = test_database().await?; + let pool = database.pool(); + let manifest_id = sqlx::query_scalar::<_, i64>( + r#" + INSERT INTO manifest_versions + (manifest_version, namespace, source_family, chain, deployment_epoch, + rollout_status, normalizer_version, file_path, manifest_payload) + VALUES + (1, 'ens', 'registry', 'ethereum-sepolia', 'ens_v2_sepolia_dev', + 'active', 'n', 'f', '{}'::jsonb) + RETURNING manifest_id + "#, + ) + .fetch_one(pool) + .await?; + sqlx::query( + r#" + INSERT INTO contract_instances (contract_instance_id, chain_id, contract_kind) + VALUES + ('88888888-8888-8888-8888-888888888888', 'ethereum-sepolia', 'contract'), + ('99999999-9999-9999-9999-999999999999', 'ethereum-sepolia', 'contract') + "#, + ) + .execute(pool) + .await?; + sqlx::query( + r#" + INSERT INTO discovery_edges + (chain_id, edge_kind, from_contract_instance_id, to_contract_instance_id, + discovery_source, source_manifest_id, admission, active_from_block_number) + VALUES + ('ethereum-sepolia', 'subregistry', + '88888888-8888-8888-8888-888888888888', + '99999999-9999-9999-9999-999999999999', + 'registry_event', $1, 'reachable_from_root', 100) + "#, + ) + .bind(manifest_id) + .execute(pool) + .await?; + + let read = load_data_completeness(pool).await?; + assert_eq!(read.discovery_targets_missing_address.len(), 1); + assert_eq!( + read.discovery_targets_missing_address[0].contract_instance_id, + uuid::Uuid::parse_str("99999999-9999-9999-9999-999999999999")? + ); + + sqlx::query( + r#" + INSERT INTO contract_instance_addresses + (contract_instance_id, chain_id, address, active_to_block_number) + VALUES + ('99999999-9999-9999-9999-999999999999', 'ethereum-sepolia', '0xDISCOVERED', 99) + "#, + ) + .execute(pool) + .await?; + let read = load_data_completeness(pool).await?; + assert_eq!(read.discovery_targets_missing_address.len(), 1); + + sqlx::query( + r#" + UPDATE contract_instance_addresses + SET active_to_block_number = NULL + WHERE contract_instance_id = '99999999-9999-9999-9999-999999999999' + "#, + ) + .execute(pool) + .await?; + let read = load_data_completeness(pool).await?; + assert!(read.discovery_targets_missing_address.is_empty()); + + sqlx::query( + r#" + UPDATE contract_instance_addresses + SET active_from_block_number = 900 + WHERE contract_instance_id = '99999999-9999-9999-9999-999999999999' + "#, + ) + .execute(pool) + .await?; + let read = load_data_completeness(pool).await?; + assert_eq!(read.discovery_targets_missing_address.len(), 1); + + sqlx::query( + r#" + UPDATE contract_instance_addresses + SET active_from_block_number = 50 + WHERE contract_instance_id = '99999999-9999-9999-9999-999999999999' + "#, + ) + .execute(pool) + .await?; + let read = load_data_completeness(pool).await?; + assert!(read.discovery_targets_missing_address.is_empty()); + + sqlx::query( + r#" + UPDATE discovery_edges + SET active_from_block_number = NULL + WHERE source_manifest_id = $1 + "#, + ) + .bind(manifest_id) + .execute(pool) + .await?; + sqlx::query( + r#" + UPDATE contract_instance_addresses + SET active_from_block_number = 900 + WHERE contract_instance_id = '99999999-9999-9999-9999-999999999999' + "#, + ) + .execute(pool) + .await?; + let read = load_data_completeness(pool).await?; + assert_eq!(read.discovery_targets_missing_address.len(), 1); + + sqlx::query( + r#" + UPDATE contract_instance_addresses + SET active_from_block_number = NULL + WHERE contract_instance_id = '99999999-9999-9999-9999-999999999999' + "#, + ) + .execute(pool) + .await?; + let read = load_data_completeness(pool).await?; + assert!(read.discovery_targets_missing_address.is_empty()); + + sqlx::query( + r#" + UPDATE discovery_edges + SET active_to_block_number = 100 + WHERE source_manifest_id = $1 + "#, + ) + .bind(manifest_id) + .execute(pool) + .await?; + sqlx::query( + r#" + UPDATE contract_instance_addresses + SET active_to_block_number = 100 + WHERE contract_instance_id = '99999999-9999-9999-9999-999999999999' + "#, + ) + .execute(pool) + .await?; + let read = load_data_completeness(pool).await?; + assert!(read.discovery_targets_missing_address.is_empty()); + + database.cleanup().await +} + +/// Resolver discovery from an active registry source requires the matching active resolver +/// manifest used by the runtime watch view. Losing that target manifest must not erase the edge +/// from completeness authority. +#[tokio::test] +async fn resolver_discovery_target_without_active_manifest_is_surfaced() -> Result<()> { + let database = test_database().await?; + let pool = database.pool(); + let source_manifest_id = sqlx::query_scalar::<_, i64>( + r#" + INSERT INTO manifest_versions + (manifest_version, namespace, source_family, chain, deployment_epoch, + rollout_status, normalizer_version, file_path, manifest_payload) + VALUES + (1, 'ens', 'ens_v2_registry_l1', 'ethereum-sepolia', 'ens_v2_sepolia_dev', + 'active', 'n', 'registry', '{}'::jsonb) + RETURNING manifest_id + "#, + ) + .fetch_one(pool) + .await?; + sqlx::raw_sql( + r#" + INSERT INTO contract_instances (contract_instance_id, chain_id, contract_kind) + VALUES + ('aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa', 'ethereum-sepolia', 'contract'), + ('bbbbbbbb-bbbb-bbbb-bbbb-bbbbbbbbbbbb', 'ethereum-sepolia', 'contract'); + + INSERT INTO contract_instance_addresses + (contract_instance_id, chain_id, address) + VALUES + ('bbbbbbbb-bbbb-bbbb-bbbb-bbbbbbbbbbbb', 'ethereum-sepolia', '0xRESOLVER') + "#, + ) + .execute(pool) + .await?; + sqlx::query( + r#" + INSERT INTO discovery_edges + (chain_id, edge_kind, from_contract_instance_id, to_contract_instance_id, + discovery_source, source_manifest_id, admission, active_from_block_number) + VALUES + ('ethereum-sepolia', 'resolver', + 'aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa', + 'bbbbbbbb-bbbb-bbbb-bbbb-bbbbbbbbbbbb', + 'registry_event', $1, 'reachable_from_root', 100) + "#, + ) + .bind(source_manifest_id) + .execute(pool) + .await?; + + let read = load_data_completeness(pool).await?; + assert_eq!(read.discovery_targets_missing_manifest.len(), 1); + assert_eq!( + read.discovery_targets_missing_manifest[0].source_family, + "ens_v2_resolver_l1" + ); + assert!(read.discovery_targets_missing_address.is_empty()); + + sqlx::query( + r#" + INSERT INTO manifest_versions + (manifest_version, namespace, source_family, chain, deployment_epoch, + rollout_status, normalizer_version, file_path, manifest_payload) + VALUES + (1, 'ens', 'ens_v2_resolver_l1', 'ethereum-sepolia', 'ens_v2_sepolia_dev', + 'active', 'n', 'resolver', '{}'::jsonb) + "#, + ) + .execute(pool) + .await?; + let read = load_data_completeness(pool).await?; + assert!(read.discovery_targets_missing_manifest.is_empty()); + assert!(read.discovery_targets_missing_address.is_empty()); + + database.cleanup().await +} + +/// Only active manifests that declare normalized adapter output form content expectations, +/// and residual rows from a deprecated manifest ID do not count for the active identity. +#[tokio::test] +async fn active_event_sources_are_counted_by_exact_manifest_identity() -> Result<()> { + let database = test_database().await?; + let pool = database.pool(); + let active_event_manifest_id = sqlx::query_scalar::<_, i64>( + r#" + INSERT INTO manifest_versions + (manifest_version, namespace, source_family, chain, deployment_epoch, + rollout_status, normalizer_version, file_path, manifest_payload) + VALUES + (2, 'ens', 'registry', 'ethereum-sepolia', 'e', 'active', 'n', 'active', + '{"abi":{"events":[{"normalized_events":["ResolverChanged"]}]}}'::jsonb) + RETURNING manifest_id + "#, + ) + .fetch_one(pool) + .await?; + let deprecated_manifest_id = sqlx::query_scalar::<_, i64>( + r#" + INSERT INTO manifest_versions + (manifest_version, namespace, source_family, chain, deployment_epoch, + rollout_status, normalizer_version, file_path, manifest_payload) + VALUES + (1, 'ens', 'registry', 'ethereum-sepolia', 'e', 'deprecated', 'n', 'old', + '{"abi":{"events":[{"normalized_events":["ResolverChanged"]}]}}'::jsonb) + RETURNING manifest_id + "#, + ) + .fetch_one(pool) + .await?; + sqlx::query( + r#" + INSERT INTO manifest_versions + (manifest_version, namespace, source_family, chain, deployment_epoch, + rollout_status, normalizer_version, file_path, manifest_payload) + VALUES + (1, 'basenames', 'basenames_execution', 'ethereum-mainnet', 'e', + 'active', 'n', 'metadata', '{}'::jsonb) + "#, + ) + .execute(pool) + .await?; + sqlx::query( + r#" + INSERT INTO normalized_events + (event_identity, namespace, event_kind, source_family, manifest_version, + source_manifest_id, chain_id, derivation_kind, canonicality_state) + VALUES + ('stale', 'ens', 'ResolverChanged', 'registry', 1, $1, + 'ethereum-sepolia', 'raw_log', 'canonical') + "#, + ) + .bind(deprecated_manifest_id) + .execute(pool) + .await?; + sqlx::query( + r#" + INSERT INTO chain_lineage + (chain_id, block_hash, block_number, block_timestamp, canonicality_state) + VALUES ('ethereum-sepolia', '0xOBSERVED', 42, now(), 'canonical') + "#, + ) + .execute(pool) + .await?; + sqlx::query( + r#" + INSERT INTO normalized_events + (event_identity, namespace, event_kind, source_family, manifest_version, + source_manifest_id, chain_id, block_number, block_hash, derivation_kind, + canonicality_state) + VALUES + ('observed-active', 'ens', 'ResolverChanged', 'registry', 2, $1, + 'ethereum-sepolia', 42, '0xOBSERVED', 'raw_log', 'observed') + "#, + ) + .bind(active_event_manifest_id) + .execute(pool) + .await?; + + let read = load_data_completeness(pool).await?; + assert_eq!(read.active_manifest_event_sources.len(), 1); + let source = &read.active_manifest_event_sources[0]; + assert_eq!(source.manifest_id, active_event_manifest_id); + assert_eq!(source.normalized_event_kinds, vec!["ResolverChanged"]); + assert_eq!(source.normalized_event_count, 0); + + database.cleanup().await +} + +/// Adapter-owned kinds extend only an already-active manifest source and participate in the +/// same exact manifest identity, content, and lineage checks as ABI-declared kinds. +#[tokio::test] +async fn adapter_declared_kinds_extend_active_event_source_universe() -> Result<()> { + let database = test_database().await?; + let pool = database.pool(); + let manifest_id = sqlx::query_scalar::<_, i64>( + r#" + INSERT INTO manifest_versions + (manifest_version, namespace, source_family, chain, deployment_epoch, + rollout_status, normalizer_version, file_path, manifest_payload) + VALUES + (1, 'ens', 'ens_v1_reverse_l1', 'ethereum-mainnet', 'ens_v1', + 'active', 'n', 'reverse', '{}'::jsonb) + RETURNING manifest_id + "#, + ) + .fetch_one(pool) + .await?; + + assert!( + load_data_completeness(pool) + .await? + .active_manifest_event_sources + .is_empty() + ); + + let declarations = [ + ("ens_v1_reverse_l1", "ReverseChanged"), + ("ens_v1_reverse_l1", "RecordChanged"), + ]; + let declared = load_data_completeness_with_adapter_event_kinds(pool, &declarations).await?; + assert_eq!(declared.active_manifest_event_sources.len(), 1); + assert_eq!( + declared.active_manifest_event_sources[0].normalized_event_kinds, + vec!["RecordChanged", "ReverseChanged"] + ); + assert_eq!( + declared.active_manifest_event_sources[0].normalized_event_count, + 0 + ); + + sqlx::raw_sql( + r#" + INSERT INTO chain_lineage + (chain_id, block_hash, block_number, block_timestamp, canonicality_state) + VALUES ('ethereum-mainnet', '0xREVERSE', 42, now(), 'canonical'); + "#, + ) + .execute(pool) + .await?; + sqlx::query( + r#" + INSERT INTO normalized_events + (event_identity, namespace, event_kind, source_family, manifest_version, + source_manifest_id, chain_id, block_number, block_hash, derivation_kind, + canonicality_state) + VALUES + ('reverse', 'ens', 'ReverseChanged', 'ens_v1_reverse_l1', 1, $1, + 'ethereum-mainnet', 42, '0xREVERSE', 'ens_v1_reverse_claim', 'canonical') + "#, + ) + .bind(manifest_id) + .execute(pool) + .await?; + + let populated = load_data_completeness_with_adapter_event_kinds(pool, &declarations).await?; + assert_eq!( + populated.active_manifest_event_sources[0].normalized_event_count, + 1 + ); + assert_eq!( + populated.active_manifest_event_sources[0] + .normalized_events_missing_canonical_lineage_count, + 0 + ); + + database.cleanup().await +} + +/// Active normalized content must retain its exact canonical lineage anchors so reorg repair +/// remains possible after restore, including when minimal retention compacted raw-log staging. +#[tokio::test] +async fn active_event_sources_report_missing_canonical_lineage() -> Result<()> { + let database = test_database().await?; + let pool = database.pool(); + let manifest_id = sqlx::query_scalar::<_, i64>( + r#" + INSERT INTO manifest_versions + (manifest_version, namespace, source_family, chain, deployment_epoch, + rollout_status, normalizer_version, file_path, manifest_payload) + VALUES + (1, 'ens', 'registry', 'ethereum-sepolia', 'ens_v2_sepolia_dev', + 'active', 'n', 'active', + '{"abi":{"events":[{"normalized_events":["ResolverChanged"]}]}}'::jsonb) + RETURNING manifest_id + "#, + ) + .fetch_one(pool) + .await?; + sqlx::query( + r#" + INSERT INTO normalized_events + (event_identity, namespace, event_kind, source_family, manifest_version, + source_manifest_id, chain_id, block_number, block_hash, transaction_hash, + log_index, raw_fact_ref, derivation_kind, canonicality_state) + VALUES + ('active-event', 'ens', 'ResolverChanged', 'registry', 1, $1, + 'ethereum-sepolia', 42, '0xBLOCK', '0xTX', 3, + '{"kind":"raw_log"}'::jsonb, 'raw_log', 'canonical'), + ('boundary-event', 'ens', 'ResolverChanged', 'registry', 1, $1, + 'ethereum-sepolia', 42, '0xBLOCK', NULL, NULL, + '{"kind":"raw_block"}'::jsonb, 'synthetic_boundary', 'canonical') + "#, + ) + .bind(manifest_id) + .execute(pool) + .await?; + + let read = load_data_completeness(pool).await?; + assert_eq!( + read.active_manifest_event_sources[0].normalized_events_missing_canonical_lineage_count, + 2 + ); + assert_eq!( + read.active_manifest_event_sources[0].normalized_event_count, + 2 + ); + + sqlx::query( + r#" + INSERT INTO chain_lineage + (chain_id, block_hash, block_number, block_timestamp, canonicality_state) + VALUES + ('ethereum-sepolia', '0xBLOCK', 42, now(), 'orphaned') + "#, + ) + .execute(pool) + .await?; + let read = load_data_completeness(pool).await?; + assert_eq!( + read.active_manifest_event_sources[0].normalized_events_missing_canonical_lineage_count, + 2 + ); + + sqlx::query( + r#" + UPDATE chain_lineage + SET canonicality_state = 'canonical' + WHERE chain_id = 'ethereum-sepolia' + AND block_hash = '0xBLOCK' + "#, + ) + .execute(pool) + .await?; + let read = load_data_completeness(pool).await?; + assert_eq!( + read.active_manifest_event_sources[0].normalized_events_missing_canonical_lineage_count, + 0 + ); + + database.cleanup().await +} + +/// Log-audit verification requires both the exact serving-canonical raw-log row and its exact +/// serving-canonical lineage anchor. An observed raw-log row or an orphaned anchor cannot prove +/// retained audit input. +#[tokio::test] +async fn active_raw_log_events_report_missing_canonical_raw_log_with_anchor_transitivity() +-> Result<()> { + let database = test_database().await?; + let pool = database.pool(); + let manifest_id = sqlx::query_scalar::<_, i64>( + r#" + INSERT INTO manifest_versions + (manifest_version, namespace, source_family, chain, deployment_epoch, + rollout_status, normalizer_version, file_path, manifest_payload) + VALUES + (1, 'ens', 'registry', 'ethereum-sepolia', 'ens_v2_sepolia_dev', + 'active', 'n', 'active', + '{"abi":{"events":[{"normalized_events":["ResolverChanged"]}]}}'::jsonb) + RETURNING manifest_id + "#, + ) + .fetch_one(pool) + .await?; + sqlx::query( + r#" + INSERT INTO chain_lineage + (chain_id, block_hash, block_number, block_timestamp, canonicality_state) + VALUES ('ethereum-sepolia', '0xBLOCK', 42, now(), 'canonical') + "#, + ) + .execute(pool) + .await?; + sqlx::query( + r#" + INSERT INTO normalized_events + (event_identity, namespace, event_kind, source_family, manifest_version, + source_manifest_id, chain_id, block_number, block_hash, transaction_hash, + log_index, raw_fact_ref, derivation_kind, canonicality_state) + VALUES + ('active-event', 'ens', 'ResolverChanged', 'registry', 1, $1, + 'ethereum-sepolia', 42, '0xBLOCK', '0xTX', 3, + '{"kind":"raw_log"}'::jsonb, 'raw_log', 'canonical') + "#, + ) + .bind(manifest_id) + .execute(pool) + .await?; + + let read = load_data_completeness(pool).await?; + assert_eq!( + read.active_manifest_event_sources[0].normalized_events_missing_canonical_raw_log_count, + 1 + ); + assert_eq!(read.chains[0].canonical_raw_log_head_block_number, None); + + sqlx::query( + r#" + INSERT INTO raw_logs + (chain_id, block_hash, block_number, transaction_hash, transaction_index, + log_index, emitting_address, canonicality_state) + VALUES + ('ethereum-sepolia', '0xBLOCK', 42, '0xTX', 0, 3, '0xresolver', 'observed') + "#, + ) + .execute(pool) + .await?; + let read = load_data_completeness(pool).await?; + assert_eq!( + read.active_manifest_event_sources[0].normalized_events_missing_canonical_raw_log_count, + 1 + ); + + sqlx::query("UPDATE raw_logs SET canonicality_state = 'canonical'") + .execute(pool) + .await?; + let read = load_data_completeness(pool).await?; + assert_eq!( + read.active_manifest_event_sources[0].normalized_events_missing_canonical_raw_log_count, + 0 + ); + assert_eq!(read.chains[0].canonical_raw_log_head_block_number, Some(42)); + + sqlx::query("UPDATE chain_lineage SET canonicality_state = 'orphaned'") + .execute(pool) + .await?; + let read = load_data_completeness(pool).await?; + assert_eq!( + read.active_manifest_event_sources[0].normalized_events_missing_canonical_raw_log_count, + 1 + ); + assert_eq!(read.chains[0].canonical_raw_log_head_block_number, None); + + database.cleanup().await +} + +/// Orphaned events and NULL-chain events must not satisfy the per-chain content check; the +/// NULL rows are counted separately as a data-integrity signal. +#[tokio::test] +async fn orphaned_and_null_chain_events_excluded_from_counts() -> Result<()> { + let database = test_database().await?; + let pool = database.pool(); + let manifest_id = sqlx::query_scalar::<_, i64>( + r#" + INSERT INTO manifest_versions + (manifest_version, namespace, source_family, chain, deployment_epoch, + rollout_status, normalizer_version, file_path, manifest_payload) + VALUES + (1, 'ens', 'sf', 'ethereum-sepolia', 'e', 'active', 'n', 'f', + '{"abi":{"events":[{"normalized_events":["ResolverChanged"]}]}}'::jsonb) + RETURNING manifest_id + "#, + ) + .fetch_one(pool) + .await?; + sqlx::query( + r#" + INSERT INTO normalized_events + (event_identity, namespace, event_kind, source_family, + manifest_version, source_manifest_id, derivation_kind, chain_id, + canonicality_state) + VALUES + ('e1', 'ens', 'ResolverChanged', 'sf', 1, $1, 'd', + 'ethereum-sepolia', 'canonical'::canonicality_state), + ('e2', 'ens', 'ResolverChanged', 'sf', 1, $1, 'd', + 'ethereum-sepolia', 'orphaned'::canonicality_state), + ('e3', 'ens', 'ResolverChanged', 'sf', 1, $1, 'd', + NULL, 'canonical'::canonicality_state) + "#, + ) + .bind(manifest_id) + .execute(pool) + .await?; + + let read = load_data_completeness(pool).await?; + assert_eq!(read.active_manifest_event_sources.len(), 1); + assert_eq!( + read.active_manifest_event_sources[0].normalized_event_count, + 1 + ); + assert_eq!(read.normalized_events_null_chain_id_count, 1); + + database.cleanup().await +} + +/// The cursor read must carry `next_block_number` so the evaluator can detect a rewind where +/// `next` dropped below `target` while `last_completed` stayed high. +#[tokio::test] +async fn rewound_cursor_next_below_target_is_loaded() -> Result<()> { + let database = test_database().await?; + let pool = database.pool(); + sqlx::query( + r#" + INSERT INTO normalized_replay_cursors + (deployment_profile, chain_id, cursor_kind, range_start_block_number, + next_block_number, target_block_number, last_completed_block_number) + VALUES + ('sepolia', 'ethereum-sepolia', 'raw_fact_normalized_events', 0, 500, 1000, 1000) + "#, + ) + .execute(pool) + .await?; + + let read = load_data_completeness(pool).await?; + let cursor = read + .replay_cursors + .iter() + .find(|cursor| cursor.cursor_kind == "raw_fact_normalized_events") + .expect("cursor"); + assert_eq!(cursor.range_start_block_number, 0); + assert_eq!(cursor.next_block_number, Some(500)); + assert_eq!(cursor.target_block_number, Some(1000)); + assert_eq!(cursor.last_completed_block_number, Some(1000)); + + database.cleanup().await +} + +/// Projection replay inspection carries both the writer's completed target and the global target +/// a bootstrap would request now. Per-chain foreign-state advisories do not scope this marker: +/// automatic projection replay uses the same unscoped checkpoint maximum. +#[tokio::test] +async fn projection_replay_marker_and_required_target_are_loaded() -> Result<()> { + let database = test_database().await?; + let pool = database.pool(); + sqlx::query( + r#" + INSERT INTO normalized_replay_cursors + (deployment_profile, chain_id, cursor_kind, range_start_block_number, + next_block_number, target_block_number) + VALUES + ('sepolia', 'ethereum-sepolia', 'raw_fact_normalized_events', 0, 121, 120) + "#, + ) + .execute(pool) + .await?; + sqlx::query( + r#" + INSERT INTO manifest_versions + (manifest_version, namespace, source_family, chain, deployment_epoch, + rollout_status, normalizer_version, file_path, manifest_payload) + VALUES + (1, 'ens', 'registry', 'ethereum-sepolia', 'e', 'active', 'n', 'active', + '{}'::jsonb) + "#, + ) + .execute(pool) + .await?; + sqlx::query( + r#" + INSERT INTO chain_checkpoints + (chain_id, canonical_block_hash, canonical_block_number, + safe_block_hash, safe_block_number) + VALUES + ('ethereum-sepolia', '0x100', 100, '0x140', 140), + ('retired-chain', '0x200', 200, NULL, NULL) + "#, + ) + .execute(pool) + .await?; + sqlx::query( + r#" + INSERT INTO current_projection_replay_status + (projection, replay_version, completed_normalized_target_block, + requested_key_count, upserted_row_count, deleted_row_count) + VALUES ('name_current', 6, 130, 0, 0, 0) + "#, + ) + .execute(pool) + .await?; + + let read = load_data_completeness(pool).await?; + assert_eq!(read.projection_replay_required_target_block, Some(200)); + assert_eq!(read.projection_replay_markers.len(), 1); + assert_eq!( + read.projection_replay_markers[0].completed_normalized_target_block, + Some(130) + ); + + database.cleanup().await +} + +/// Only active manifest versions form the expected content set; a manifest-declared chain with +/// no checkpoint or lineage row still appears so the evaluator can gate it. +#[tokio::test] +async fn active_manifest_chain_namespaces_are_loaded() -> Result<()> { + let database = test_database().await?; + let pool = database.pool(); + sqlx::query( + r#" + INSERT INTO manifest_versions + (manifest_version, namespace, source_family, chain, deployment_epoch, + rollout_status, normalizer_version, file_path, manifest_payload) + VALUES + (1, 'ens', 'sf', 'ethereum-sepolia', 'e', 'active', 'n', 'f1', '{}'::jsonb), + (1, 'basenames', 'sf', 'base-mainnet', 'e', 'deprecated', 'n', 'f2', '{}'::jsonb) + "#, + ) + .execute(pool) + .await?; + + let read = load_data_completeness(pool).await?; + assert_eq!(read.manifest_chain_namespaces.len(), 1); + assert_eq!(read.manifest_chain_namespaces[0].chain, "ethereum-sepolia"); + assert_eq!(read.manifest_chain_namespaces[0].namespace, "ens"); + assert_eq!(read.manifest_chain_source_families.len(), 1); + assert_eq!(read.manifest_chain_source_families[0].source_family, "sf"); + // The declared chain has no checkpoint/lineage row, so it is absent from the storage chains. + assert!( + read.chains + .iter() + .all(|chain| chain.chain_id != "ethereum-sepolia") + ); + + database.cleanup().await +} + +/// Replay cursor selection uses the deployment profile implied by the active manifest corpus, +/// matching the indexer's replay-admission writer boundary. +#[tokio::test] +async fn active_manifest_deployment_profile_is_inferred() -> Result<()> { + let database = test_database().await?; + let pool = database.pool(); + sqlx::query( + r#" + INSERT INTO manifest_versions + (manifest_version, namespace, source_family, chain, deployment_epoch, + rollout_status, normalizer_version, file_path, manifest_payload) + VALUES + (1, 'ens', 'registry', 'ethereum-sepolia', 'ens_v2_sepolia_dev', + 'active', 'n', 'f', '{}'::jsonb) + "#, + ) + .execute(pool) + .await?; + + let read = load_data_completeness(pool).await?; + assert_eq!(read.active_deployment_profile.as_deref(), Some("sepolia")); + + database.cleanup().await +} + +/// A migrated database has the deferred projection indexes; the read reports them present. +#[tokio::test] +async fn deferred_projection_indexes_present_on_migrated_database() -> Result<()> { + let database = test_database().await?; + let read = load_data_completeness(database.pool()).await?; + assert_eq!( + read.present_deferred_projection_indexes.len(), + super::DEFERRED_NORMALIZED_EVENT_INDEXES.len() + ); + + database.cleanup().await +} + +/// A failed concurrent build leaves a named `pg_indexes` entry with `indisvalid = false`. +/// The completeness read must not report that unusable index as present. +#[tokio::test] +async fn invalid_deferred_projection_index_is_not_present() -> Result<()> { + let database = test_database().await?; + let pool = database.pool(); + sqlx::query("DROP INDEX normalized_events_namespace_idx") + .execute(pool) + .await?; + sqlx::query( + r#" + INSERT INTO normalized_events + (event_identity, namespace, event_kind, source_family, + manifest_version, derivation_kind) + VALUES + ('invalid-index-1', 'ens', 'k', 'sf', 1, 'd'), + ('invalid-index-2', 'ens', 'k', 'sf', 1, 'd') + "#, + ) + .execute(pool) + .await?; + + let build_error = sqlx::query( + "CREATE UNIQUE INDEX CONCURRENTLY normalized_events_namespace_idx \ + ON normalized_events (namespace)", + ) + .execute(pool) + .await; + assert!( + build_error.is_err(), + "duplicate rows must fail the unique build" + ); + + let catalog_state = sqlx::query_scalar::<_, bool>( + r#" + SELECT index_state.indisvalid + FROM pg_index index_state + WHERE index_state.indexrelid = 'normalized_events_namespace_idx'::regclass + "#, + ) + .fetch_one(pool) + .await?; + assert!( + !catalog_state, + "failed build must leave an invalid catalog entry" + ); + + let read = load_data_completeness(pool).await?; + assert!( + !read + .present_deferred_projection_indexes + .contains(&"normalized_events_namespace_idx".to_owned()) + ); + + database.cleanup().await +} diff --git a/crates/storage/src/lib.rs b/crates/storage/src/lib.rs index 991dafe2..317d6274 100644 --- a/crates/storage/src/lib.rs +++ b/crates/storage/src/lib.rs @@ -8,6 +8,7 @@ mod backfill_jobs; mod base_normalized_rederive; mod checkpoints; mod children; +mod data_completeness; mod evm_primitives; mod execution; mod history; @@ -72,8 +73,9 @@ pub use backfill_jobs::{ BackfillCoverageFactDerivation, BackfillCoverageFactScope, BackfillCoverageFactWrite, BackfillJob, BackfillJobCreate, BackfillJobRecord, BackfillLifecycleStatus, BackfillRange, BackfillRangeSpec, advance_backfill_range, complete_backfill_job, complete_backfill_range, - complete_backfill_range_recording_coverage, create_backfill_job, fail_backfill_job, - fail_backfill_range, load_backfill_coverage_fact_counts, load_backfill_job, + complete_backfill_range_recording_coverage, create_backfill_job, + ensure_backfill_family_topic_sets_undrifted, fail_backfill_job, fail_backfill_range, + load_backfill_coverage_fact_counts, load_backfill_job, load_backfill_jobs_intersecting_range, load_backfill_ranges, load_completed_backfill_jobs_intersecting_range, reserve_backfill_range, write_backfill_coverage_facts, }; @@ -113,6 +115,15 @@ pub use children::{ load_children_current_page, load_children_current_summaries, stream_canonical_declared_child_sources, upsert_children_current_rows, }; +pub use data_completeness::{ + ActiveManifestEventSource, BackfillLifecycleRow, ChainCompletenessRow, + DEFERRED_NORMALIZED_EVENT_INDEXES, DataCompletenessRead, DiscoveryTargetMissingAddress, + DiscoveryTargetMissingManifest, ManifestChainNamespace, ManifestChainSourceFamily, + ManifestDeclaredTarget, NameCurrentCount, ObservedCodeAddress, ProjectionApplyCursorRow, + ProjectionContentCounts, ProjectionContentScopeCount, ProjectionReplayMarker, ReplayCursorRow, + load_active_manifest_deployment_profile, load_data_completeness, + load_data_completeness_with_adapter_event_kinds, load_projection_content_counts, +}; pub use evm_primitives::{normalize_evm_address, normalize_evm_b256}; pub use execution::{ ExecutionBoundaryInvalidation, ExecutionCacheKey, ExecutionManifestInvalidation, @@ -168,11 +179,11 @@ pub use label_preimages::{ upsert_label_preimages_from_normalized_events, upsert_label_preimages_in_transaction, }; pub use lineage::{ - CanonicalityState, ChainLineageBlock, chain_lineage_contains_ancestor, - chain_lineage_contains_canonical_ancestor_position, load_chain_lineage_block, - load_chain_lineage_canonical_child_path, load_highest_canonical_chain_lineage_block, - mark_chain_lineage_range_orphaned, upsert_chain_lineage_blocks, - upsert_chain_lineage_blocks_recanonicalizing_orphaned, + CanonicalityState, ChainLineageBlock, MAX_LIVE_CONTIGUOUS_GAP_FILL_BLOCKS, + chain_lineage_contains_ancestor, chain_lineage_contains_canonical_ancestor_position, + load_chain_lineage_block, load_chain_lineage_canonical_child_path, + load_highest_canonical_chain_lineage_block, mark_chain_lineage_range_orphaned, + upsert_chain_lineage_blocks, upsert_chain_lineage_blocks_recanonicalizing_orphaned, upsert_chain_lineage_blocks_without_snapshots, upsert_chain_lineage_blocks_without_snapshots_recanonicalizing_orphaned, }; diff --git a/crates/storage/src/lineage.rs b/crates/storage/src/lineage.rs index 5908d3ea..f9a6cd1d 100644 --- a/crates/storage/src/lineage.rs +++ b/crates/storage/src/lineage.rs @@ -6,6 +6,10 @@ mod types; mod upserts; mod validation; +/// Largest canonical-lineage gap the live reconciler may persist before advancing the stored +/// checkpoint. Larger gaps are handed to bounded catch-up or hash-pinned backfill. +pub const MAX_LIVE_CONTIGUOUS_GAP_FILL_BLOCKS: i64 = 1_024; + pub use orphaning::mark_chain_lineage_range_orphaned; pub use reads::{ chain_lineage_contains_ancestor, chain_lineage_contains_canonical_ancestor_position, diff --git a/crates/storage/src/permissions.rs b/crates/storage/src/permissions.rs index 92349da2..1df82671 100644 --- a/crates/storage/src/permissions.rs +++ b/crates/storage/src/permissions.rs @@ -5,6 +5,8 @@ mod types; mod validation; mod writes; +pub(crate) use reads::DEFAULT_PERMISSIONS_CURRENT_READ_FILTER; + pub use paging::{ load_permissions_current_account_resource_page, load_permissions_current_account_resource_page_count_summary, load_permissions_current_page, diff --git a/crates/storage/src/permissions/reads.rs b/crates/storage/src/permissions/reads.rs index b4fa883f..047a5dd9 100644 --- a/crates/storage/src/permissions/reads.rs +++ b/crates/storage/src/permissions/reads.rs @@ -9,7 +9,7 @@ use super::{ types::{PermissionScope, PermissionsCurrentRow}, }; -pub(super) const DEFAULT_PERMISSIONS_CURRENT_READ_FILTER: &str = r#" +pub(crate) const DEFAULT_PERMISSIONS_CURRENT_READ_FILTER: &str = r#" AND resource.canonicality_state IN ( 'canonical'::canonicality_state, 'safe'::canonicality_state, diff --git a/crates/storage/src/record_inventory.rs b/crates/storage/src/record_inventory.rs index 50457d7c..2ac8f536 100644 --- a/crates/storage/src/record_inventory.rs +++ b/crates/storage/src/record_inventory.rs @@ -6,6 +6,7 @@ mod snapshot_reads; mod validation; pub(crate) use boundary_key::record_version_boundary_storage_key; +pub(crate) use snapshot_reads::DEFAULT_RECORD_INVENTORY_CURRENT_READ_FILTER; pub use batch_upsert::upsert_record_inventory_current_rows; pub use counts::count_record_inventory_selectors_by_lookup_keys; diff --git a/crates/storage/src/record_inventory/snapshot_reads.rs b/crates/storage/src/record_inventory/snapshot_reads.rs index d5057cd0..a9738acb 100644 --- a/crates/storage/src/record_inventory/snapshot_reads.rs +++ b/crates/storage/src/record_inventory/snapshot_reads.rs @@ -15,7 +15,7 @@ use super::{ row_decode::{RecordInventoryCurrentRow, decode_record_inventory_current_row}, }; -pub(super) const DEFAULT_RECORD_INVENTORY_CURRENT_READ_FILTER: &str = r#" +pub(crate) const DEFAULT_RECORD_INVENTORY_CURRENT_READ_FILTER: &str = r#" AND resource.canonicality_state IN ( 'canonical'::canonicality_state, 'safe'::canonicality_state, diff --git a/docs/deployment.md b/docs/deployment.md index 604ba826..f63fd910 100644 --- a/docs/deployment.md +++ b/docs/deployment.md @@ -22,6 +22,18 @@ docker run --rm ghcr.io/tateb/bigname:latest bigname-api print-openapi docker run --rm ghcr.io/tateb/bigname:latest bigname-worker inspect watch-plan --json ``` +Before promoting a database or cutting traffic over to one, check that it is +data-complete. The command is read-only and exits non-zero with +`--fail-on-incomplete`; `docs/runbooks/data-completeness.md` describes each check and how +to read a failure. It is a database-level gate, not a route-level one, and `/v1/status` +cannot substitute for it: that endpoint reports an empty projection queue as caught up to +head, so an incomplete database looks caught up. + +```sh +docker run --rm ghcr.io/tateb/bigname:latest \ + bigname-worker inspect data-completeness --json --fail-on-incomplete +``` + ## Fresh Server Compose 1. Install Docker and Docker Compose. diff --git a/docs/manifests.md b/docs/manifests.md index 872e90e7..230ac15a 100644 --- a/docs/manifests.md +++ b/docs/manifests.md @@ -49,6 +49,13 @@ For `[[discovery_rules]]`, the only authorable `admission` value is `reachable_f `[abi]` is optional. When present, it declares the Solidity ABI fragments that this manifest version authorizes for adapter, execution, or watch-plan use. ABI entries are source-family metadata; they do not by themselves graduate public capability support. +Manifest `normalized_events` declarations are combined with adapter-owned emitted-kind declarations +when operational tooling enumerates the active normalized-event universe. The manifest remains the +authority that admits the source family, chain, namespace, and version; the adapter declaration only +adds normalized kinds that the admitted adapter can synthesize even when no one ABI event carries +that kind, such as reverse-claim or block-derived output. Adapter declarations do not admit a source, +change capability status, or expand the watch plan. + ### `capability_flags` Each flag carries a name, a status (`unsupported` | `shadow` | `supported`), and optional notes. @@ -237,6 +244,15 @@ Live manifest drift and proxy-upgrade alerting is a worker-owned operational loo `bigname-worker manifest-drift audit --json` computes candidates, persists alert observations, and renders the persisted view alongside live counts. `--fail-on-alert --json` returns nonzero when actionable persisted alerts remain. `bigname-worker inspect manifest-drift --json` is read-only over already persisted observations. +Cutover inspection can anchor the active database corpus back to this filesystem authority with +`bigname-worker inspect data-completeness --manifests-root `. The comparison is +bidirectional over active manifests: disk requires a database row with the same namespace, +source family, chain, deployment epoch, manifest version, and complete serialized payload, and +every active database row requires the corresponding disk manifest. A missing, empty, or invalid +supplied root fails. Omitting the argument preserves database-only diagnosis but reports +`manifest_corpus_unverified: true`; promotion automation must supply the selected profile root so +a partial restore cannot shrink its own manifest-derived expectations. + ## Watch-plan expansion Watch-plan expansion starts from active manifest roots by `contract_instance_id` and traverses active discovery edges by ID. diff --git a/docs/runbooks/data-completeness.md b/docs/runbooks/data-completeness.md new file mode 100644 index 00000000..a7fdb211 --- /dev/null +++ b/docs/runbooks/data-completeness.md @@ -0,0 +1,493 @@ +# Data Completeness Runbook + +`bigname-worker inspect data-completeness` answers one question: **is this database +complete enough to serve, or to cut traffic over to?** + +It is read-only. It opens the inspect connection with +`default_transaction_read_only = on`, reads raw facts, cursors, projections, and the +runtime watch plan, and writes nothing. + +## Command + +```sh +bigname-worker inspect data-completeness --json +``` + +Gate a promotion or cutover on it: + +```sh +bigname-worker inspect data-completeness \ + --manifests-root manifests/mainnet \ + --retention-mode minimal \ + --json --fail-on-incomplete +``` + +Exit `0` means every check passed and `data_complete` is `true`. With +`--fail-on-incomplete`, exit `1` means at least one check failed. Without the flag the +command always exits `0` and is a report. + +Point it at any database with `--database-url`, or `BIGNAME_DATABASE_URL` / +`DATABASE_URL`. Comparing a warmed candidate database against the serving one is the +intended use. + +`--manifests-root` is optional so the command remains usable against a database without a +local checkout, but promotion automation should always supply the same profile root as the +indexer. With a root, the on-disk active manifests are an external expectation and must match +the database bidirectionally, including version and complete serialized payload. Without one, +the report keeps database-derived behavior and sets `advisories.manifest_corpus_unverified` to +`true`. + +`--retention-mode` selects the storage contract to verify. It defaults to `minimal`, where +compacted raw-log staging is valid. `log-audit` additionally requires the exact retained +serving-canonical raw log and lineage anchor for every active normalized event whose +`raw_fact_ref.kind` is `raw_log`. + +## Why `/v1/status` cannot do this + +`/v1/status` reports `projection_lag_blocks` from the projection queue. When the queue is +empty it reports `latest_projected_block` as the stored canonical checkpoint, so an +*empty work queue* reads as *caught up to head*. Each cursor in the pipeline measures +itself against the previous stage's frontier, so if an upstream stage stalls, or the watch +set silently excludes targets, every cursor still reports "done". + +An empty database and a complete database are indistinguishable to that endpoint. Do not +gate a cutover on it. + +## Checks + +Order is dependency order. Later checks are meaningless if earlier ones fail. + +The active chain set is the authority for every per-chain check: active manifest-payload roots, +contracts, and proxy implementations; the materialized watch view (manifest-declared contracts +unioned with active discovery edges); and every chain an active `manifest_versions` row declares +directly. A partial restore that lost `manifest_contract_instances` or +`contract_instance_addresses` rows therefore cannot delete a chain or directly declared target +from its own expectations. A chain present in storage but not in that active set is a foreign or +retired chain and is reported as an advisory, not gated for the per-chain checks. + +| Check | Passes when | +| --- | --- | +| `manifest_corpus_matches_repository` | when `--manifests-root` is supplied, every active on-disk manifest has an active database row with the same identity, version, and payload, and every active database row has an on-disk counterpart; without the argument the check is non-gating and explicitly unverified | +| `reconciliation_frontier_at_head` | every active chain has a stored frontier, the checkpoint's exact hash and number resolve to a canonical/safe/finalized lineage row, checkpoint-ahead lag is at most `--max-head-lag-blocks` (default 8), and lineage-ahead lag is at most the live reconciler's shared contiguous gap-fill limit (currently 1,024 blocks) | +| `reconciliation_lineage_contiguous` | distinct canonical/safe/finalized block numbers equal `head - floor + 1`, no retained canonical height has an additional non-orphaned hash, and every row above the retained floor points by `parent_hash` to the canonical/safe/finalized row at the preceding height | +| `reconciliation_history_from_declared_start` | for every active watched chain, the retained lineage floor is at or below the earliest finite start block its active targets declare; a chain whose targets are all open-ended fails, since no floor can be established | +| `stored_lineage_backfill_coverage` | every active log-producing watched tuple intersecting a persisted bounded-backfill range inside retained lineage is contained in a durable address- or family-scoped `backfill_coverage_facts` row over each shared promotion-verification chunk, and each topic-filtered completed job in that range recorded the exact topic set the active manifest ABI still declares | +| `watch_set_code_observation_coverage` | there is at least one active target, and every target whose inclusive start is at or below its chain's stored canonical head — direct active-manifest declarations unioned with the materialized watch view and active discovery edges — has a non-orphaned `raw_code_hashes` observation with an exact retained non-orphaned lineage anchor at or after that start; future-start targets are reported as `pending_activation` | +| `manifest_declared_targets_present` | every active manifest-payload root, contract, and proxy implementation has its matching materialized instance and open address whose start does not narrow the payload start; every proxy implementation also has the exact open managed `proxy_implementation` edge, with a range-preserving start, consumed by the watch view | +| `discovery_targets_present` | every target endpoint of an open non-migration discovery edge has an open `contract_instance_addresses` row on the edge's chain whose start does not narrow the edge's admitted start, and every open resolver edge from an active registry source retains its matching active resolver target manifest; bounded edges are historical and impose no current target requirement | +| `active_event_lineage_retained` | every matching canonical/safe/finalized normalized event for an active event-producing source still resolves by exact chain, block hash, and block number to a canonical/safe/finalized `chain_lineage` anchor; event kinds are the union of manifest ABI and adapter-owned declarations | +| `active_raw_logs_retained` | in `minimal` mode, always; in `log-audit` mode, every active serving-canonical normalized event sourced from a raw log retains the exact canonical/safe/finalized `raw_logs` row and exact serving-canonical lineage anchor | +| `normalization_no_failure` | no replay cursor for an active chain under the deployment profile inferred from the active manifest corpus carries a `last_failure_reason` | +| `normalization_caught_up_to_raw_head` | the active manifest corpus resolves to one supported deployment profile; every active chain with retained canonical raw logs has that profile's `raw_fact_normalized_events` cursor, each applicable cursor has reached its target, and canonical tail logs above a closure-replay latch require a complete post-replay backlog that starts no later than the raw-fact target plus one (see below) | +| `projection_apply_drained` | the `normalized_events_to_projection_invalidations` cursor, when present, exactly equals the retained change-log high-water mark (`max(change_id)`, or zero for an empty log); a non-empty log also requires the cursor, and unrelated cursor rows do not count | +| `projection_invalidations_drained` | `projection_invalidations` is empty — every enqueued invalidation has been applied and deleted | +| `projection_no_dead_letters` | `projection_invalidation_dead_letters` is empty — no invalidation exhausted its retries | +| `projection_replay_complete` | every current projection has a `current_projection_replay_status` marker at this worker's `CURRENT_PROJECTION_REPLAY_VERSION`; until a non-empty retained projection change log exactly corroborates the durable apply cursor, each marker must also cover the target bootstrap would request now | +| `current_projection_content_present` | all seven projections in the worker's `ALL_CURRENT_PROJECTION_ORDER` were counted through their serving validity rules, and every namespace or chain whose active manifest-plus-adapter event declarations can feed that projection has at least one servable row; raw counts remain diagnostic | +| `active_dataset_non_empty` | every active manifest source with non-empty manifest- or adapter-declared normalized output has a canonical/safe/finalized matching event under that exact `source_manifest_id`, manifest version, chain, namespace, source family, and declared event kind; every such namespace also has `name_current` rows | +| `normalized_events_chain_id_present` | no non-orphaned `normalized_events` row has a NULL `chain_id` | +| `deferred_projection_indexes_present` | the eight deferred `normalized_events` projection indexes all exist on `public.normalized_events` and have `pg_index.indisvalid = true` and `indisready = true` (a fresh replay drops them and rebuilds them after catch-up) | + +`watch_set_code_observation_coverage` is the check with teeth, and the reason the command +exists. Most others are *relative* invariants: each compares a stage to the stage before +it, so they stay green while the pipeline faithfully processes an incomplete input. +`reconciliation_history_from_declared_start`, `stored_lineage_backfill_coverage`, +`manifest_corpus_matches_repository`, `active_event_lineage_retained`, +`active_dataset_non_empty`, `current_projection_content_present`, and +`projection_replay_complete` also compare against external or retained truth, the declared +world, or the pipeline's own handoff authority rather than the previous stage. + +`projection_apply_drained` only proves the derive scan finished — that the apply cursor +exactly matches the change-log frontier. A cursor behind the frontier has not derived all +changes; a cursor ahead of it proves retained change-log history was lost, because derive only +scans IDs greater than the cursor. It does not prove the resulting invalidations were +applied, because those move through a separate claim/apply queue. `projection_invalidations_drained` +and `projection_no_dead_letters` close that gap: the queue must be empty (a successful +apply deletes the row) and nothing may have dead-lettered. All three must pass for +projections to be fully applied. + +### Replay targets: raw-log head vs latched target + +`normalization_caught_up_to_raw_head` compares each chain's `raw_fact_normalized_events` +cursor against the **canonical** raw-log head — the newest raw log whose lineage block is +canonical, safe, or finalized, mirroring the bounds replay actually consumes. It does not +use the non-orphaned head, which includes `observed` logs replay cannot yet touch; that +head is reported only, so trailing unpromoted logs do not read as permanent lag. + +Cursor selection is keyed by both chain and the deployment profile inferred from the active +manifest corpus, using the same `mainnet` / `sepolia` classification as replay admission. +A cursor left by another profile cannot satisfy an active chain, and failures or lag on an +inactive chain or a non-active profile are reported in `advisories.ignored_replay_cursors` +instead of gating the serving corpus. + +A chain with an active closure- or dependency-replay source family is an exception. The writer +uses the shared source-family policy to preserve that chain's first raw-fact cursor target below +the live head; newer logs are carried by a one-shot `post_replay_live_adapter_backlog` sweep and +then live adapter sync. The gate uses the same policy. A quiet chain whose canonical raw-log head +is at or below the latched target may complete without a backlog row, matching a sweep that found +no tail logs and therefore wrote no cursor. If the canonical raw-log head is above the latch, a +backlog cursor must exist, both cursors must reach their own targets, and the backlog's inclusive +range start must be no later than the raw-fact target plus one. An earlier start is a safe overlap; +a later one leaves an unnormalized gap that completed cursors cannot reveal. The backlog row is +therefore optional only when retained canonical logs independently corroborate the quiet latch. +**Limitation:** the live tail beyond the backlog target has no cursor, so the gate cannot verify +it. This is one reason a passing gate is necessary but not sufficient (see Scope). + +The catch-up writer clears `last_failure_reason` and `last_failure_at` after either a successful +advance or a successful idle iteration. A transient failure therefore remains gating until the +same chain completes a healthy poll, but it cannot remain stuck forever on an already-complete +latched cursor. + +Completion is read from the cursor's `next_block_number > target_block_number` pair, the +same authority the catch-up loop uses, not from `last_completed_block_number`. A reorg +rewind lowers `next_block_number` back below the target while leaving +`last_completed_block_number` at its high-water mark, so a gate that trusted +`last_completed_block_number` would read a rewound cursor as still caught up. +`last_completed_block_number` is reported only. + +The check also requires the cursor to exist: a chain with retained canonical raw logs but +no `raw_fact_normalized_events` cursor row — a truncated restore, or a chain absent from +catch-up configuration — fails with a `chains_missing_raw_fact_cursor` entry rather than +passing because there was no cursor to measure. + +It works because code observations are keyed on the watch set rather than on activity: a +watched target acquires a code observation from the live tailer's baseline pass even if it +never emits a log. Coverage preserves the latest non-orphaned observation block with a matching +non-orphaned `chain_lineage` row per `(chain, address)` and compares it with the target's +inclusive active start; an unanchored observation or one retained from a pre-admission range +cannot satisfy a newly active target. Direct active +manifest payload targets, including proxy implementation addresses, are read independently of +`manifest_contract_instances` and `contract_instance_addresses`, so losing a materialized +declaration or address row makes the target unobserved instead of removing it from the expected +set. When several active entries share an address, coverage uses the latest finite start, the +strictest lower bound across those entries. + +A finite start above the chain checkpoint's canonical block is a pre-activation declaration. +It remains part of the active target and history authority, but coverage cannot require an +observation from a block the chain has not reached. The entry is excluded from only the code +observation comparison and appears in `advisories.pending_activation` with its declared start +and current canonical head. Once the head reaches the start, the same target becomes gating. +If the chain has no stored canonical checkpoint, the gate cannot prove pre-activation and does +not skip the target. + +`manifest_declared_targets_present` diagnoses the materialization edge separately. Every payload +root or contract must have the matching `manifest_contract_instances` declaration; a proxy +implementation must also have the matching implementation instance. An open address row only +satisfies a target when its `chain_id` and lowercased address match the payload; a row for the +same instance on another chain or at another address does not count. An address with a finite +`active_to_block_number` is closed even when `deactivated_at` is null. Proxy implementations +must additionally retain the exact open managed discovery edge that the source-graph writer +creates; the direct payload fallback cannot substitute for the runtime watch-plan edge. Both the +address row and managed edge must start no later than the payload target (or remain open-ended +when the payload start is unknown). Otherwise the runtime watch view's `GREATEST` start silently +narrows the manifest-declared interval. + +`discovery_targets_present` closes the corresponding gap for dynamic discovery. The active edge +and its target `contract_instance_id` remain authority if a restore loses the target's open +address row or the active resolver manifest that assigns the target's source family, even though +`load_watched_contracts` can no longer render that target. Resolver target manifests are matched +to the active registry source by namespace, chain, deployment epoch, and the registry-to-resolver +family mapping. Only open +edges impose this current-authority requirement: a finite `active_to_block_number` is legitimate +retained discovery history. The named check fails on a missing or closed current materialization +rather than allowing coverage and history to shrink. When the edge has a finite admitted start, +the open address row must have no start or an equal/earlier start. When the edge start is unknown, +the address start must also remain unknown; inventing a finite materialization start would silently +discard an unbounded earlier interval. A later address start would otherwise make the runtime watch +view take `GREATEST(edge_start, address_start)` and discard the edge's earlier required interval, so +it is reported as a materialization gap. +See [`chain-intake.md`](../chain-intake.md). + +### Frontier, history, and content against the declared world + +Three checks measure the chain and its content against what the watch set declares, not +only against the previous pipeline stage: + +- **Frontier.** `reconciliation_frontier_at_head` applies asymmetric writer bounds. A positive + lag, where the checkpoint is ahead of retained lineage, may not exceed + `--max-head-lag-blocks` (default 8). Reconcile commits an entire canonical lineage path before + advancing the checkpoint, so a negative lag may reach the live reconciler's shared contiguous + gap-fill limit (currently 1,024 blocks). Beyond that limit the writer itself refuses live gap + fill and requires bounded catch-up or hash-pinned backfill. A zero numeric lag still + fails when the checkpoint's `canonical_block_hash` does not join the canonical/safe/finalized + lineage row at that exact height; later reconciliation uses that hash as its branch anchor. The + check also + synthesizes a failing frontier row for any active chain that has no checkpoint or lineage + row at all, so a declared chain missing from storage (reported with + `missing_from_storage: true`) cannot pass by absence. +- **History.** `reconciliation_history_from_declared_start` compares each active watched + chain's retained lineage floor against the earliest finite start block its active targets + declare. A floor above that start means history was truncated — the shape of a + live-tail-only restore, where the truncated span is itself contiguous, cursors are caught + up over the short raw set, and projections are non-empty, so every other check passes. A + target whose start block is open-ended (`active_from_block_number` is null) imposes no + floor and is skipped, matching bootstrap's own authority; but a chain whose targets are + *all* open-ended has no floor to establish and fails closed. Direct manifest-payload starts + remain part of this union even when a materialized address row narrows its runtime + `active_from_block_number`; the address row cannot raise the manifest's historical floor. +- **Stored-path fetch coverage.** `stored_lineage_backfill_coverage` runs the same indexed + watched-tuple anti-join as stored-lineage checkpoint promotion over merged persisted backfill-job + ranges intersected with retained lineage. The job is the durable signal that a span came from + bounded backfill rather than ordinary provider-fetched live reconciliation. Incomplete and failed + jobs remain in scope because retained lineage can be crash residue that the writer refuses; + durable facts from a later successful retry can satisfy the interval. Evidence remains required + after the checkpoint consumes the span: a checkpoint regression or database restore can make it + promotion input again, and deleting facts must not preserve a completeness pass. The check uses + the writer's shared `131072`-block verification chunks and active + log-producing source-family authority. Each required tuple interval within a chunk must fit + inside one durable address-scoped fact for that exact family/address or one family-scoped fact. + Before those facts are trusted, the inspector invokes the same topic-set drift guard as + stored-lineage promotion. For every intersecting completed topic-filtered job, its persisted + per-family topic set must exactly equal the active manifest ABI's current set; a missing + persisted set also fails closed. Address-enumerated, topic-unfiltered jobs are unaffected. + Facts from a later successful retry can satisfy the interval, so failed job/range lifecycle rows + remain advisory. A chain with no bounded-backfill range inside retained lineage has no bounded + fetch span requiring facts and passes this check vacuously. +- **Content.** `active_dataset_non_empty` derives expected event-producing sources from the union + of active manifest ABI `normalized_events` entries and adapter-owned emitted-kind declarations + for those admitted source families. The adapter declaration covers normalized output synthesized + outside a one-to-one manifest ABI event, including reverse-claim and block-derived events; it + does not admit a source or widen its watch scope. Execution- or transport-only manifests with no + declaration from either authority do not create event-content expectations. Each expected source + must have a canonical, safe, or finalized event matching its exact + `source_manifest_id`, manifest version, chain, namespace, source family, and one of its + declared event kinds. Rows from a deprecated manifest version therefore cannot make a newly + active source pass. Each expected namespace must also have `name_current` rows. + `name_current` carries no chain column, so names are scoped to the namespace — the finest + dimension a name projection has; a name in a namespace shared across chains is not + attributed to a specific one. Failed sources are reported in + `manifest_sources_without_events` with their manifest IDs and source families. This exact + identity rule intentionally makes a manifest-version rollout fail until normalized events + have been re-derived under the newly active `source_manifest_id`; residual rows from the + previous version are not proof that the new declaration was indexed. +- **Retained event lineage.** `active_event_lineage_retained` joins every matching active + canonical/safe/finalized normalized event, including synthetic block-boundary events, back to + `chain_lineage` by exact chain, block hash, and block number and requires that anchor to remain + canonical, safe, or finalized. An `observed` event does not count as active serving content, + because projection readers consume only serving-canonical states. This is the durable + reorg-repair authority. In the documented minimal retention + mode, `raw_logs` are replay staging and may be compacted after normalized replay and downstream + durability; that valid compaction does not fail this check. With `--retention-mode log-audit`, + `active_raw_logs_retained` additionally joins every active normalized raw-log event to its + exact `(chain, block hash, block number, transaction hash, log index)` raw-log row and requires + both that row and its exact lineage anchor to be canonical, safe, or finalized. + +### Projections rebuilt, and structural integrity + +`current_projection_content_present` enumerates the same seven-family +`ALL_CURRENT_PROJECTION_ORDER` that replay publishes. It reports raw and servable total/scoped +row counts for every family and evaluates missing scopes only from servable counts. Expectations +come from the union of active manifest and adapter-owned normalized event-kind declarations, never +from the projection rows being checked: + +| Projection | Required scope | Serving validity mirrored by the count | A scope is expected when an active source declares | +| --- | --- | --- | --- | +| `name_current` | namespace | canonical/safe/finalized surface; when bound, canonical/safe/finalized resource and binding, open binding, and canonical/safe/finalized token lineage when present (`DEFAULT_NAME_CURRENT_READ_FILTER`) | any normalized event output | +| `children_current` | namespace | declared rows with a canonical/safe/finalized parent and a missing, label-preimage-backed, or canonical/safe/finalized child surface (`DEFAULT_CHILDREN_CURRENT_READ_FILTER`) | `SubregistryChanged`, `ParentChanged`, `RegistrationGranted`, `RegistrationRenewed`, or `RegistrationReleased` | +| `permissions_current` | resource chain | joined resource is canonical/safe/finalized (`DEFAULT_PERMISSIONS_CURRENT_READ_FILTER`) | `PermissionChanged`, `RootPermissionChanged`, or `PermissionScopeChanged` | +| `record_inventory_current` | resource chain | joined resource is canonical/safe/finalized (`DEFAULT_RECORD_INVENTORY_CURRENT_READ_FILTER`) | `RecordChanged`, `RecordVersionChanged`, or `ResolverChanged` | +| `resolver_current` | chain | no additional consumer-side identity/canonicality filter | `ResolverChanged`, `AliasChanged`, `PermissionChanged`, or `PermissionScopeChanged` (resolver-scoped permission rows are resolver rebuild targets) | +| `address_names_current` | namespace | canonical/safe/finalized surface, resource, and binding; open binding; canonical/safe/finalized token lineage when present (`DEFAULT_ADDRESS_NAMES_CURRENT_READ_FILTER`) | `RegistrationGranted`, `TokenControlTransferred`, `AuthorityTransferred`, `AuthorityEpochChanged`, `PermissionChanged`, `PermissionScopeChanged`, or `TokenRegenerated` | +| `primary_names_current` | namespace | no additional consumer-side identity/canonicality filter | `ReverseChanged` | + +An unlisted input does not create an expectation for that projection. This lets a small healthy +corpus with no declared permission events keep an empty `permissions_current`, while truncating +`resolver_current` for a chain whose active sources declare `ResolverChanged` fails by chain. +Permissions and record inventory are resource-keyed, so residue attached only to a foreign chain's +resources cannot satisfy an active chain. Likewise, a raw projection row whose supporting identity +row is orphaned or whose binding is closed remains visible in diagnostics but cannot satisfy a +servable scope. + +`projection_replay_complete` requires every current projection to have a marker in +`current_projection_replay_status` at the inspecting worker's +`CURRENT_PROJECTION_REPLAY_VERSION`. The newest stored version is still reported as +`replay_version` for diagnosis, while `required_replay_version` reports the version the gate +requires. Each current-version marker's `completed_normalized_target_block` must also reach the +target automatic bootstrap would use now — +`max(global raw_fact_normalized_events target, furthest persisted chain checkpoint)` — until a +durable `normalized_events_to_projection_invalidations` apply cursor is exactly corroborated by a +non-empty retained `projection_normalized_event_changes` high-water mark. Bootstrap +publishes projections in order and `name_current` is first, so a candidate mid-bootstrap has +`name_current` but not the rest. After handoff, the worker deliberately stops advancing replay +markers: the apply cursor/change log and invalidation queue own later normalized blocks. At that +point current-version marker presence plus the separately drained apply/change/invalidation +checks is the writer's authority; comparing the old bootstrap marker target with a newer chain +frontier would false-fail a healthy live database. Complete markers from an older worker image +are never accepted. A cursor over an empty change log is treated as restore residue and does not +waive marker target coverage, even when its stored position is zero. + +When target coverage is required before apply handoff, the replay target is intentionally global +even though foreign or retired chains are advisory for the per-chain frontier, history, +normalization, and coverage checks. Current projections and their replay marker are global, and +the automatic replay writer computes the same unscoped maximum across persisted raw-fact cursors +and chain checkpoints. Scoping only the inspector to active chains could approve a marker below +the target that the writer would request. + +`deferred_projection_indexes_present` checks that the eight deferred `normalized_events` +projection indexes exist on the expected table and are valid and ready in `pg_index`. A failed +`CREATE INDEX CONCURRENTLY` can leave a named but invalid catalog entry that PostgreSQL cannot +use; that entry fails the check just like an absent index. A fresh replay drops the indexes for +speed and a later pass rebuilds them after catch-up, so a missing or invalid index marks a +candidate whose replay has not finished rebuilding them — complete data, but not yet ready to +serve efficiently. + +`normalized_events_chain_id_present` fails on any non-orphaned `normalized_events` row with a +NULL `chain_id`. Those rows are excluded from the per-chain content counts (and would +otherwise abort the read), so they are surfaced here as a data-integrity fault. + +## Interpreting failures + +- **`manifest_corpus_matches_repository` fails.** The supplied profile root is missing or + invalid, an active on-disk manifest is absent or payload-mismatched in the database, or the + database retains an active manifest the root does not contain. Run manifest sync from the + intended root and investigate unexpected active rows; do not accept a smaller surviving + database corpus as its own expectation. +- **`watch_set_code_observation_coverage` fails, everything else passes.** The runtime + watch scope is narrower than the manifests and discovery edges declare. The reported + `unobserved_targets` were never watched, so their logs were never indexed and their + derived state is missing, silently. Check the indexer's + `BIGNAME_INDEXER_HASH_PINNED_BACKFILL_ADAPTER_SYNC` and the resulting + `RuntimeWatchScope`; `manifest_declared_only` excludes discovery edges. Verify with + `inspect watch-plan --json`. +- **`manifest_declared_targets_present` fails.** An active payload target has no matching + materialized declaration/implementation instance or no open address row matching both its + declared chain and address, or a materialized proxy/implementation pair lacks its open managed + edge. A missing declaration, missing proxy implementation, missing or bounded proxy edge, + a later-than-declared address or edge start, deactivated, bounded, wrong-chain, or wrong-address + row breaks the watch-view authority even if + retained code observations let the separate coverage check pass. Reseed manifests or repair + the materialized graph before promoting. +- **`discovery_targets_present` fails.** An open discovery edge points to a contract instance + with no open address on that chain, or its address start is later than the edge's admitted + start, or an open resolver edge from an active registry has lost its matching active resolver + target manifest. Restore or rederive the range-preserving target address or manifest before promoting; closing or + deleting a current edge merely to silence the check changes admission authority. +- **`stored_lineage_backfill_coverage` fails.** A bounded-backfill range inside retained lineage + contains an active log-producing watched tuple whose required interval is not contained in one + durable coverage fact, or a completed topic-filtered job's persisted topic set differs from the + current manifest ABI (or was not persisted). Run hash-pinned or Coinbase SQL backfill for the reported tuple/range on the current manifest, or + derive facts for a compatible completed legacy job, before retrying promotion. +- **`active_event_lineage_retained` fails.** Active normalized content has lost one or more exact + canonical lineage anchors. Restore `chain_lineage` and run reorg/projection repair as needed; + raw-log compaction by itself is not this failure. +- **`active_raw_logs_retained` fails.** The operator selected `log-audit`, but an active + normalized raw-log event has lost its exact serving-canonical raw-log row or exact canonical + lineage anchor. Restore or refetch and verify the retained audit fact. Select `minimal` only + when compaction is the deployment's actual retention policy, not merely to silence the check. +- **Coverage fails on a database warmed by backfill alone.** Backfill only observes a + block's selected log emitters. A watched target that never emits acquires its single + baseline observation from the live tailer, on canonical head reconciliation. Let the + tailer reconcile a head before treating coverage as authoritative on a freshly + backfilled database. +- **`normalization_no_failure` fails.** The adapter pass is crash-looping; the cursor + records why. Normalization has stopped at the first affected block, so every downstream + count is stale even though the cursors look drained. +- **`normalization_caught_up_to_raw_head` reports no active deployment profile.** The active + manifest corpus is empty or mixes chains/epochs that do not classify as the writer's + `mainnet` or `sepolia` profile. Correct manifest rollout state before interpreting cursor + progress. +- **`reconciliation_frontier_at_head` fails.** Head reconciliation has stalled, or the + checkpoint and lineage exceed the applicable asymmetric bound. A positive + `head_lag_blocks` beyond `--max-head-lag-blocks` means lineage trails the checkpoint. A + negative value is allowed through the reconciler's live gap-fill limit because lineage is + committed before checkpoint advancement; beyond that limit the checkpoint writer is stale + or the restore is mixed. A `missing_from_storage: true` chain is declared by the watch set but has no + frontier row. `checkpoint_canonical_lineage_match: false` with zero numeric lag means the + checkpoint hash is missing, orphaned, or points at a different branch at that height. +- **`reconciliation_lineage_contiguous` fails with a non-zero `duplicate_canonical_height_count`.** + A retained canonical/safe/finalized height also has another non-orphaned lineage hash. That is + a canonicality violation — every competing hash at a retained canonical height must be + orphaned — not a gap, and it points at a + reorg-repair or canonicality-assignment bug rather than missing intake. +- **`reconciliation_lineage_contiguous` fails with a non-zero + `disconnected_canonical_parent_count`.** The retained heights are present, but one or more + child rows do not point to the canonical/safe/finalized hash at the preceding height. Treat + this as a broken restored or repaired branch, even if `missing_block_count` is zero. +- **`reconciliation_history_from_declared_start` fails.** The chain's lineage floor is above + the earliest block its watched targets declare, so history is truncated. Common on a + restore that kept only a recent window; the reported `declared_start_block` and + `lineage_floor_block` bound the missing span. +- **`active_dataset_non_empty` fails with an entry in + `manifest_sources_without_events`.** The exact active event-producing manifest source has no + matching canonical/safe/finalized normalized event. Observed rows, rows from another chain or + source family, and deprecated manifest identities do not satisfy it, even if the global event + table is non-empty. +- **`projection_replay_complete` fails with entries in `missing_projections`.** The candidate + is mid projection-bootstrap, its marker belongs to a version other than + `required_replay_version`, or (before apply handoff) it completed below + `required_target_block`. `target_coverage_required` states whether the target comparison is + active. `name_current` is published first, so a candidate with only it is early in the rebuild; + old-version markers always require replay, while stale-target markers require replay only + before the durable apply cursor takes over. +- **`current_projection_content_present` fails.** At least one expected namespace, chain, or + global current-projection table is empty even though active manifest event declarations can + feed it. Use the reported table and `missing_scopes`; replay markers and drained cursors are + not substitutes for the missing serving rows. Rebuild the named projection from canonical + inputs before promotion. +- **`deferred_projection_indexes_present` fails.** A fresh replay dropped the listed indexes, + the rebuild pass has not run, or a concurrent build left a named but invalid catalog entry. + Data may be complete, but the database is not serve-ready; rebuild valid indexes before + promoting. +- **`normalized_events_chain_id_present` fails.** Non-orphaned normalized events have a NULL + `chain_id` — a decode or write fault. Those rows are not attributable to a chain. +- **`projection_invalidations_drained` or `projection_no_dead_letters` fails while + `projection_apply_drained` passes.** The derive scan finished but the resulting projection + writes have not landed: invalidations are still queued, or some exhausted their retries and + dead-lettered. Projections are stale or partial even though the apply cursor looks caught + up. +- **`projection_apply_drained` fails because the apply cursor is ahead.** Retained rows were + removed from `projection_normalized_event_changes` without rewinding or rebuilding the cursor. + Restore the missing change log or rebuild projections and reseed the handoff; do not advance or + delete rows merely to make the cursor and log agree. +- **`normalization_caught_up_to_raw_head` fails on a `:range_start` cursor label.** The + post-replay backlog starts after `raw_fact_normalized_events.target_block_number + 1`, or the + raw-fact target needed to establish continuity is absent. Recreate the backlog from the replay + target so the normalized ranges overlap or meet before promotion. +- **`watch_set_code_observation_coverage` fails with zero active watched targets.** The + database has no manifests loaded — a restore that dropped or never applied them. The check + fails rather than passing vacuously, since an ENS deployment always watches at least the + registry. +- **`active_dataset_non_empty` fails while every cursor is drained.** The pipeline is healthy + and there is nothing in the active dataset. This is the empty-database case, and it is the + shape prod presented on 2026-07-06. + +## Advisories + +The `advisories` block reports state that is worth an operator's attention but does not gate: + +- **`manifest_corpus_unverified`** — no `--manifests-root` was supplied, so all manifest-derived + expectations still come from surviving database rows. This is acceptable for diagnosis, not + for a promotion gate that must detect a partially restored manifest corpus. +- **`pending_activation`** — active targets whose finite start is above their chain's stored + canonical head. They do not yet require code-observation coverage; they become gating when + the checkpoint reaches the declared start. +- **`foreign_chains`** — chains with residual checkpoint or lineage rows that the active watch + set does not cover (a retired chain, or one whose manifest was removed while its chain state + remained). Their per-chain checks are not gated; the listing is for cleanup. +- **`ignored_replay_cursors`** — cursor labels outside the inferred active deployment profile or + active chain set. Their failures and lag do not gate serving, but the rows remain visible for + cleanup and profile-rollout diagnosis. +- **`backfill_lifecycle`** — per-profile counts of `failed`, incomplete, and expired-lease + backfill jobs and ranges. Backfill lifecycle failures are *not* gated because a later successful + retry may supply the durable facts for the same required tuples; the separate + `stored_lineage_backfill_coverage` check gates the resulting promotion evidence. Treat a + non-zero `failed_range_count` or + `expired_lease_range_count` as a prompt to investigate before promoting. + +## Caveats + +- **Coinbase SQL sample-mode candidates.** In `sample` validation mode a Coinbase SQL backfill + persists lineage only for the blocks whose logs it returned, while completing the whole + range. The reconciled span is then sparse, so `reconciliation_lineage_contiguous` fails on a + candidate that is legitimately complete for its sampled coverage. A sample-backed candidate + is not gate-promotable without a full-mode pass over the same range; coverage facts cannot + substitute for the missing lineage rows. + +## Scope + +This gate is database-level. It does not exercise HTTP routes, compare name counts across +two databases, or spot-check GraphQL and REST answers. It also cannot verify the live tail +beyond a latched chain's backlog target, which has no cursor, and it does not reconcile +Coinbase SQL sample-mode coverage (see Caveats). Those remain a separate, explicitly deferred +layer on top of this command; a passing gate is a necessary condition for a cutover, not a +sufficient one. diff --git a/docs/storage.md b/docs/storage.md index d80faef0..3e022ce9 100644 --- a/docs/storage.md +++ b/docs/storage.md @@ -380,6 +380,12 @@ There is no deployed object-storage layer in the current schema or compose stack - **log-audit** — the same rows remain durable audit facts and may keep heavier indexes for historical raw-fact replay. Switching modes is operational policy. It does not change route coverage, projection truth, canonicality semantics, manifest rollout, or consumer-replacement meaning. +The database does not persist the selected mode. Promotion tooling therefore supplies it +explicitly to `bigname-worker inspect data-completeness --retention-mode `. +In `log-audit` mode the inspector requires every active serving-canonical normalized event whose +`raw_fact_ref.kind` is `raw_log` to retain the exact serving-canonical `raw_logs` row and exact +serving-canonical `chain_lineage` anchor. In `minimal` mode the same missing raw-log staging row +is allowed, while the normalized event's lineage anchor remains mandatory. Live polling may retain selected `raw_transactions` and `raw_receipts` for successful direct transactions to configured event-silent resolver addresses even when those transactions do not emit selected logs. Intake copies the chain id, resolver address, block number/hash, transaction hash/index, and canonicality into `event_silent_resolver_call_observations` before those staging rows become compactable. The durable observation row is the projection-invalidation trigger for explicitly documented hydration repairs, such as legacy ENSv1 reverse-resolver primary-name hydration. It does not authorize adapters to synthesize normalized events from calldata or receipts, does not make raw facts an API fallback, and does not change minimal/log-audit compaction boundaries once downstream normalized replay and projection inputs are durable. @@ -893,6 +899,7 @@ Worker-owned, read-only operational tooling reads storage audit helpers and rend - `bigname-worker inspect stored-lineage-range --chain-id --from --to ` — lists only lineage rows already stored for the requested chain and finite block range, ordered by `(block_number, block_hash)`. Renders chain id, block number, block hash, parent hash, canonicality state, timestamp, and stored promotion markers per observed block. Nullable fields render as `null`. Does not infer missing heights, gaps, span-wide canonicality, or completeness. - `bigname-worker inspect backfill-job --backfill-job-id ` — resolves one persisted job and its child ranges. Renders job lifecycle, declared range, selector kind, resolved source identity, idempotency key, timestamps, failure metadata, and a `ranges` array sorted by range bounds and id. - `bigname-worker inspect execution-trace --execution-trace-id ` — reads `execution_traces`, `execution_steps`, and trace-attachment metadata for one stored trace. +- `bigname-worker inspect data-completeness [--manifests-root ] [--retention-mode ]` — checks active manifest authority, chain intake, replay completion, every current projection family, and the selected retention contract without writing. Supplying a manifest root anchors active database manifests bidirectionally to disk; omitting it is reported as unverified. Active event kinds are the union of manifest ABI declarations and adapter-owned emitted-kind declarations for admitted active source families. Projection content is counted through the same identity and canonicality filters as its serving storage reads, while raw row counts remain diagnostic. - Manifest-drift and proxy-alert inspection — joins stored alert observations to manifest/discovery identifiers, code-hash facts, proxy/implementation edges, and derived watch-target metadata. Does not fetch fresh chain state, create alerts, mutate alert lifecycle, mutate manifest truth, or change capability flags. ## Migration rules diff --git a/scripts/rust-file-size-baseline.toml b/scripts/rust-file-size-baseline.toml index c9e36275..8ed20fe9 100644 --- a/scripts/rust-file-size-baseline.toml +++ b/scripts/rust-file-size-baseline.toml @@ -32,7 +32,7 @@ justification = "Phase 7 allowlist for an oversized production file; future chan [[files]] path = "apps/indexer/src/main/reconciliation/canonical.rs" -loc = 613 +loc = 611 justification = "Phase 7 allowlist for an oversized production file; future changes should split or shrink it." [[files]]