diff --git a/apps/indexer/src/main/normalized_replay_catchup.rs b/apps/indexer/src/main/normalized_replay_catchup.rs index 9e66bbbf..86f23556 100644 --- a/apps/indexer/src/main/normalized_replay_catchup.rs +++ b/apps/indexer/src/main/normalized_replay_catchup.rs @@ -17,7 +17,8 @@ use crate::{ HeaderAuditMode, RawFactNormalizedEventReplayOutcome, RawFactNormalizedEventReplayRequest, RawFactNormalizedEventReplaySelection, active_closure_or_dependency_replay_adapters, chain_has_closure_or_dependency_replay_adapter, replay_raw_fact_normalized_events, - sync_full_closure_normalized_events_from_persisted_raw_payloads, + select_log_bounded_replay_to_block, + sync_automatic_two_phase_full_closure_normalized_events, unsupported_closure_replay_adapters, }, }; @@ -35,7 +36,10 @@ mod sources; mod test_hook; #[cfg(test)] -pub(crate) use test_hook::install_after_rewind as install_after_rewind_test_hook; +pub(crate) use test_hook::{ + install_after_coverage_recovery as install_after_coverage_recovery_test_hook, + install_after_rewind as install_after_rewind_test_hook, +}; use coverage_recovery::replay_full_closure_with_coverage_recovery; use cursors::{ @@ -46,7 +50,7 @@ use indexes::{ ensure_projection_indexes_after_catchup, prepare_deferred_projection_indexes_for_fresh_replay, restore_deferred_projection_indexes, }; -use sources::{load_canonical_raw_log_bounds, select_log_bounded_replay_to_block}; +use sources::load_canonical_raw_log_bounds; pub(crate) const DEFAULT_NORMALIZED_REPLAY_CATCHUP_CHUNK_BLOCKS: i64 = 262_144; pub(crate) const DEFAULT_NORMALIZED_REPLAY_CATCHUP_MAX_LOGS_PER_CHUNK: usize = 100_000; @@ -520,6 +524,7 @@ async fn replay_full_closure_or_dependency_normalized_events( chain: &str, from_block: i64, to_block: i64, + stateless_ranges: &[(i64, i64)], max_raw_logs_per_page: usize, ) -> Result { let adapters = active_closure_or_dependency_replay_adapters(pool, chain).await?; @@ -536,35 +541,41 @@ async fn replay_full_closure_or_dependency_normalized_events( chain, from_block, to_block, + stateless_range_count = stateless_ranges.len(), + stateless_ranges = ?stateless_ranges, max_raw_logs_per_page, adapter_count = adapters.len(), adapters = ?adapters, - "full closure normalized-event replay session started" + "two-phase full closure normalized-event replay session started" ); - let replay = sync_full_closure_normalized_events_from_persisted_raw_payloads( + let replay = sync_automatic_two_phase_full_closure_normalized_events( pool, deployment_profile, chain, CURSOR_KIND_RAW_FACT_NORMALIZED_EVENTS, from_block, to_block, + stateless_ranges, &adapters, max_raw_logs_per_page, ) .await?; - let summary = replay.summary; + let stateless = replay.stateless; + let closure = replay.closure; Ok(RawFactNormalizedEventReplayOutcome { deployment_profile: deployment_profile.to_owned(), chain: chain.to_owned(), - selection_kind: "full_closure", + selection_kind: "two_phase_full_closure", source_scope_target_count: adapters.len(), - selected_block_count: 0, - canonical_raw_log_count: summary.scanned_log_count, - scanned_raw_log_count: summary.scanned_log_count, - matched_raw_log_count: summary.matched_log_count, - normalized_event_synced_count: summary.total_synced_count, - normalized_event_inserted_count: summary.total_inserted_count, + selected_block_count: stateless.selected_block_count, + canonical_raw_log_count: stateless.canonical_raw_log_count, + scanned_raw_log_count: stateless.scanned_raw_log_count + closure.scanned_log_count, + matched_raw_log_count: stateless.matched_raw_log_count + closure.matched_log_count, + normalized_event_synced_count: stateless.normalized_event_synced_count + + closure.total_synced_count, + normalized_event_inserted_count: stateless.normalized_event_inserted_count + + closure.total_inserted_count, }) } diff --git a/apps/indexer/src/main/normalized_replay_catchup/coverage_recovery.rs b/apps/indexer/src/main/normalized_replay_catchup/coverage_recovery.rs index bfdd5d3a..6fc5f8ba 100644 --- a/apps/indexer/src/main/normalized_replay_catchup/coverage_recovery.rs +++ b/apps/indexer/src/main/normalized_replay_catchup/coverage_recovery.rs @@ -9,7 +9,7 @@ use crate::{ provider::ChainProviderOps, reconciliation::{ EnsV2LiveCoverageRecoveryStatus, HeaderAuditMode, RawFactNormalizedEventReplayOutcome, - recover_ens_v2_live_coverage_requirement, + automatic_stateless_replay_completed, recover_ens_v2_live_coverage_requirement, }, }; @@ -31,6 +31,7 @@ pub(super) async fn replay_full_closure_with_coverage_recovery( RawLogStagingInputVersion, )> { let mut recovery_attempt = 0_usize; + let mut stateless_ranges = vec![(from_block, to_block)]; loop { let replay_error = match replay_full_closure_or_dependency_normalized_events( pool, @@ -38,6 +39,7 @@ pub(super) async fn replay_full_closure_with_coverage_recovery( chain, from_block, to_block, + &stateless_ranges, max_raw_logs_per_page, ) .await @@ -45,6 +47,7 @@ pub(super) async fn replay_full_closure_with_coverage_recovery( Ok(outcome) => return Ok((outcome, raw_log_input_version)), Err(error) => error, }; + let stateless_replay_completed = automatic_stateless_replay_completed(&replay_error); let Some(requirement) = bigname_adapters::ens_v2_missing_coverage(&replay_error).cloned() else { return Err(replay_error); @@ -83,8 +86,60 @@ pub(super) async fn replay_full_closure_with_coverage_recovery( )); } - raw_log_input_version = + // Preserve the original full span when preflight validation prevented + // phase one from running. Once phase one completed, retain only every + // exact span fetched by later recovery attempts. The stateful adapter + // pass still restarts over its complete span. + if stateless_replay_completed { + stateless_ranges.clear(); + } + include_stateless_range( + &mut stateless_ranges, + requirement.required_from_block, + requirement.required_to_block, + ); + #[cfg(test)] + super::test_hook::pause_after_coverage_recovery(pool, deployment_profile, chain).await; + + let observed_raw_log_input_version = bigname_storage::load_raw_log_staging_input_version(pool, chain).await?; + if observed_raw_log_input_version.retention_generation + != raw_log_input_version.retention_generation + { + return Err(replay_error.context(format!( + "raw-log retention generation changed during normalized replay coverage recovery: expected {}, observed {}; replan the replay from current authority", + raw_log_input_version.retention_generation, + observed_raw_log_input_version.retention_generation, + ))); + } + if from_block > 0 + && raw_log_changed_outside_stateless_ranges( + pool, + chain, + raw_log_input_version.revision, + &stateless_ranges, + 0, + from_block - 1, + ) + .await? + { + return Err(replay_error.context(format!( + "raw-log staging input changed below normalized replay range start {from_block} during coverage recovery; replan from the durable cursor" + ))); + } + let widened_for_concurrent_input = raw_log_changed_outside_stateless_ranges( + pool, + chain, + raw_log_input_version.revision, + &stateless_ranges, + from_block, + to_block, + ) + .await?; + if widened_for_concurrent_input { + include_stateless_range(&mut stateless_ranges, from_block, to_block); + } + raw_log_input_version = observed_raw_log_input_version; info!( service = "indexer", command = "run", @@ -96,7 +151,76 @@ pub(super) async fn replay_full_closure_with_coverage_recovery( to_block = requirement.required_to_block, retention_generation = requirement.retention_generation, recovery_attempt, + widened_for_concurrent_input, + stateless_range_count = stateless_ranges.len(), + stateless_ranges = ?stateless_ranges, "retrying unchanged normalized replay after exact generation-bound coverage recovery" ); } } + +async fn raw_log_changed_outside_stateless_ranges( + pool: &sqlx::PgPool, + chain: &str, + revision: i64, + stateless_ranges: &[(i64, i64)], + inspected_from_block: i64, + inspected_to_block: i64, +) -> Result { + let mut next_uncovered_block = inspected_from_block; + for &(range_from_block, range_to_block) in stateless_ranges { + if range_to_block < next_uncovered_block { + continue; + } + if range_from_block > inspected_to_block { + break; + } + let covered_from_block = range_from_block.max(inspected_from_block); + if next_uncovered_block < covered_from_block + && bigname_storage::raw_log_staging_block_range_changed_since( + pool, + chain, + revision, + next_uncovered_block, + covered_from_block - 1, + ) + .await? + { + return Ok(true); + } + let Some(after_covered_block) = range_to_block.checked_add(1) else { + return Ok(false); + }; + next_uncovered_block = next_uncovered_block.max(after_covered_block); + if next_uncovered_block > inspected_to_block { + return Ok(false); + } + } + bigname_storage::raw_log_staging_block_range_changed_since( + pool, + chain, + revision, + next_uncovered_block, + inspected_to_block, + ) + .await +} + +fn include_stateless_range(ranges: &mut Vec<(i64, i64)>, from_block: i64, to_block: i64) { + debug_assert!(from_block <= to_block); + ranges.push((from_block, to_block)); + ranges.sort_unstable(); + + let mut merged = Vec::<(i64, i64)>::with_capacity(ranges.len()); + for (from_block, to_block) in ranges.drain(..) { + if let Some((_, merged_to_block)) = merged.last_mut() + && (from_block <= *merged_to_block + || merged_to_block.checked_add(1) == Some(from_block)) + { + *merged_to_block = (*merged_to_block).max(to_block); + } else { + merged.push((from_block, to_block)); + } + } + *ranges = merged; +} diff --git a/apps/indexer/src/main/normalized_replay_catchup/sources.rs b/apps/indexer/src/main/normalized_replay_catchup/sources.rs index 3f81a2f4..06ba9e5c 100644 --- a/apps/indexer/src/main/normalized_replay_catchup/sources.rs +++ b/apps/indexer/src/main/normalized_replay_catchup/sources.rs @@ -81,75 +81,3 @@ pub(super) async fn load_canonical_raw_log_bounds( target_block, })) } - -pub(super) async fn select_log_bounded_replay_to_block( - pool: &PgPool, - chain: &str, - from_block: i64, - hard_to_block: i64, - max_raw_logs_per_chunk: usize, -) -> Result { - if from_block >= hard_to_block { - return Ok(hard_to_block); - } - let max_raw_logs_per_chunk = i64::try_from(max_raw_logs_per_chunk) - .context("normalized replay max logs per chunk does not fit in i64")?; - - sqlx::query_scalar::<_, i64>( - r#" - WITH ordered_logs AS ( - SELECT raw_logs.block_number - FROM raw_logs - JOIN chain_lineage AS lineage - ON lineage.chain_id = raw_logs.chain_id - AND lineage.block_hash = raw_logs.block_hash - WHERE raw_logs.chain_id = $1 - AND raw_logs.block_number >= $2 - AND raw_logs.block_number <= $3 - AND lineage.canonicality_state IN ( - 'canonical'::canonicality_state, - 'safe'::canonicality_state, - 'finalized'::canonicality_state - ) - AND raw_logs.canonicality_state IN ( - 'canonical'::canonicality_state, - 'safe'::canonicality_state, - 'finalized'::canonicality_state - ) - ORDER BY raw_logs.block_number ASC, raw_logs.log_index ASC - LIMIT ($4 + 1) - ), - numbered_logs AS ( - SELECT block_number, ROW_NUMBER() OVER () AS ordinal - FROM ordered_logs - ), - overflow AS ( - SELECT block_number - FROM numbered_logs - WHERE ordinal = $4 + 1 - ), - bounded AS ( - SELECT block_number - FROM numbered_logs - WHERE NOT EXISTS (SELECT 1 FROM overflow) - OR block_number < (SELECT block_number FROM overflow) - UNION ALL - SELECT MIN(block_number) - FROM numbered_logs - ) - SELECT COALESCE(MAX(block_number), $3) - FROM bounded - "#, - ) - .bind(chain) - .bind(from_block) - .bind(hard_to_block) - .bind(max_raw_logs_per_chunk) - .fetch_one(pool) - .await - .with_context(|| { - format!( - "failed to select log-bounded normalized replay range for chain {chain} range {from_block}..={hard_to_block}" - ) - }) -} diff --git a/apps/indexer/src/main/normalized_replay_catchup/test_hook.rs b/apps/indexer/src/main/normalized_replay_catchup/test_hook.rs index c1e67b16..141bd4ea 100644 --- a/apps/indexer/src/main/normalized_replay_catchup/test_hook.rs +++ b/apps/indexer/src/main/normalized_replay_catchup/test_hook.rs @@ -9,6 +9,11 @@ pub(crate) struct NormalizedReplayAfterRewindTestHook { _registration: ScopedTestHookGuard, } +pub(crate) struct NormalizedReplayAfterCoverageRecoveryTestHook { + state: NormalizedReplayAfterRewindTestHookState, + _registration: ScopedTestHookGuard, +} + #[derive(Clone)] struct NormalizedReplayAfterRewindTestHookState { after_rewind: Arc, @@ -25,16 +30,36 @@ impl NormalizedReplayAfterRewindTestHook { } } +impl NormalizedReplayAfterCoverageRecoveryTestHook { + pub(crate) async fn wait_until_after_coverage_recovery(&self) { + self.state.after_rewind.notified().await; + } + + pub(crate) fn resume(&self) { + self.state.resume.notify_one(); + } +} + impl Drop for NormalizedReplayAfterRewindTestHook { fn drop(&mut self) { self.state.resume.notify_one(); } } +impl Drop for NormalizedReplayAfterCoverageRecoveryTestHook { + fn drop(&mut self) { + self.state.resume.notify_one(); + } +} + type HookKey = (String, String, String); static HOOKS: ScopedTestHookRegistry = ScopedTestHookRegistry::new(); +static COVERAGE_RECOVERY_HOOKS: ScopedTestHookRegistry< + HookKey, + NormalizedReplayAfterRewindTestHookState, +> = ScopedTestHookRegistry::new(); pub(crate) async fn install_after_rewind( pool: &PgPool, @@ -58,6 +83,28 @@ pub(crate) async fn install_after_rewind( } } +pub(crate) async fn install_after_coverage_recovery( + pool: &PgPool, + deployment_profile: &str, + chain: &str, +) -> NormalizedReplayAfterCoverageRecoveryTestHook { + let database = current_test_database(pool) + .await + .expect("normalized replay test hook must identify its database"); + let state = NormalizedReplayAfterRewindTestHookState { + after_rewind: Arc::new(Notify::new()), + resume: Arc::new(Notify::new()), + }; + let registration = COVERAGE_RECOVERY_HOOKS.install( + (database, deployment_profile.to_owned(), chain.to_owned()), + state.clone(), + ); + NormalizedReplayAfterCoverageRecoveryTestHook { + state, + _registration: registration, + } +} + pub(super) async fn pause_after_rewind(pool: &PgPool, deployment_profile: &str, chain: &str) { let database = current_test_database(pool) .await @@ -68,3 +115,19 @@ pub(super) async fn pause_after_rewind(pool: &PgPool, deployment_profile: &str, hook.resume.notified().await; } } + +pub(super) async fn pause_after_coverage_recovery( + pool: &PgPool, + deployment_profile: &str, + chain: &str, +) { + let database = current_test_database(pool) + .await + .expect("normalized replay test hook must identify its database"); + let hook = + COVERAGE_RECOVERY_HOOKS.take(&(database, deployment_profile.to_owned(), chain.to_owned())); + if let Some(hook) = hook { + hook.after_rewind.notify_one(); + hook.resume.notified().await; + } +} diff --git a/apps/indexer/src/main/reconciliation.rs b/apps/indexer/src/main/reconciliation.rs index 8254d102..8d1edb6d 100644 --- a/apps/indexer/src/main/reconciliation.rs +++ b/apps/indexer/src/main/reconciliation.rs @@ -19,16 +19,19 @@ mod types; #[allow(unused_imports)] pub(crate) use adapter_sync::{ - BacklogHandoffStatus, sync_adapter_state_from_persisted_raw_payloads, + AutomaticTwoPhaseFullClosureSyncResult, BacklogHandoffStatus, + automatic_stateless_replay_completed, sync_adapter_state_from_persisted_raw_payloads, sync_adapter_state_from_scoped_persisted_raw_payloads, - sync_full_closure_normalized_events_from_persisted_raw_payloads, + sync_automatic_two_phase_full_closure_normalized_events, sync_live_adapter_backlog_after_normalized_replay, sync_live_adapter_state_from_persisted_raw_payloads, validate_chain_handoff_while_guarded, }; #[cfg(test)] pub(crate) use adapter_sync::{ - install_backlog_after_adapter_sync_test_hook, install_ownership_release_test_hook, - install_post_discovery_mutation_failure_for_test, + install_after_stateless_failure, install_backlog_after_adapter_sync_test_hook, + install_ownership_release_test_hook, install_post_discovery_mutation_failure_for_test, + install_stateless_page_observer, + sync_full_closure_normalized_events_from_persisted_raw_payloads, sync_manual_full_closure_normalized_events_from_persisted_raw_payloads, }; #[allow(unused_imports)] @@ -74,7 +77,8 @@ pub(crate) use replay::{ chain_has_closure_or_dependency_replay_adapter, ensure_full_closure_retention_authority_for_adapters, ensure_legacy_registry_closure_retention_authority_for_adapters, - replay_raw_fact_normalized_events, unsupported_closure_replay_adapters, + replay_raw_fact_normalized_events, select_log_bounded_replay_to_block, + unsupported_closure_replay_adapters, }; #[allow(unused_imports)] pub(crate) use types::{ diff --git a/apps/indexer/src/main/reconciliation/adapter_sync.rs b/apps/indexer/src/main/reconciliation/adapter_sync.rs index 04d67ca2..2d87701f 100644 --- a/apps/indexer/src/main/reconciliation/adapter_sync.rs +++ b/apps/indexer/src/main/reconciliation/adapter_sync.rs @@ -13,7 +13,6 @@ use crate::runtime::{ use super::replay::NormalizedEventReplayAdapter; use super::types::PersistedRawPayloadAdapterSyncSummary; - #[path = "adapter_sync/backlog.rs"] mod backlog; #[path = "adapter_sync/ens_v1_subregistry.rs"] @@ -48,11 +47,16 @@ pub(crate) use entrypoints::{ sync_live_adapter_state_from_persisted_raw_payloads_after_reorg, sync_replay_normalized_events_from_persisted_raw_payloads, }; +pub(crate) use full_closure::{ + AutomaticTwoPhaseFullClosureSyncResult, automatic_stateless_replay_completed, + sync_automatic_two_phase_full_closure_normalized_events, + sync_manual_full_closure_normalized_events_from_persisted_raw_payloads, +}; #[cfg(test)] -pub(crate) use full_closure::install_ownership_release_test_hook; pub(crate) use full_closure::{ + install_after_stateless_failure, install_ownership_release_test_hook, + install_stateless_page_observer, sync_full_closure_normalized_events_from_persisted_raw_payloads, - sync_manual_full_closure_normalized_events_from_persisted_raw_payloads, }; use mode::{PersistedRawPayloadAdapterSyncMode, ensure_raw_fact_adapter_allowed}; use scope::load_live_adapter_source_scope; diff --git a/apps/indexer/src/main/reconciliation/adapter_sync/full_closure.rs b/apps/indexer/src/main/reconciliation/adapter_sync/full_closure.rs index 3bdb98a0..622846b2 100644 --- a/apps/indexer/src/main/reconciliation/adapter_sync/full_closure.rs +++ b/apps/indexer/src/main/reconciliation/adapter_sync/full_closure.rs @@ -1,4 +1,4 @@ -use std::time::Instant; +use std::{future::Future, time::Instant}; use anyhow::{Context, Result, ensure}; use bigname_storage::{ @@ -25,11 +25,19 @@ use crate::reconciliation::{ types::PersistedRawPayloadAdapterSyncSummary, }; +#[path = "full_closure/automatic.rs"] +mod automatic; #[path = "full_closure/ownership.rs"] mod ownership; #[path = "full_closure/reverse_claim.rs"] mod reverse_claim; +pub(crate) use automatic::{ + AutomaticTwoPhaseFullClosureSyncResult, automatic_stateless_replay_completed, + sync_automatic_two_phase_full_closure_normalized_events, +}; +#[cfg(test)] +pub(crate) use automatic::{install_after_stateless_failure, install_stateless_page_observer}; #[cfg(test)] pub(crate) use ownership::install_ownership_release_test_hook; use ownership::with_full_closure_replay_lock; @@ -40,6 +48,7 @@ unsafe extern "C" { fn malloc_trim(pad: usize) -> i32; } +#[cfg(test)] #[expect(clippy::too_many_arguments)] pub(crate) async fn sync_full_closure_normalized_events_from_persisted_raw_payloads( pool: &sqlx::PgPool, @@ -113,6 +122,39 @@ async fn sync_full_closure_with_checkpoint_completion( max_raw_logs_per_page: usize, checkpoint_completion: FullClosureCheckpointCompletion, ) -> Result { + let (result, ()) = sync_full_closure_with_checkpoint_completion_and_prelude( + pool, + deployment_profile, + chain, + checkpoint_cursor_kind, + range_start_block_number, + target_block_number, + adapters, + max_raw_logs_per_page, + checkpoint_completion, + || async { Ok(()) }, + ) + .await?; + Ok(result) +} + +#[expect(clippy::too_many_arguments)] +async fn sync_full_closure_with_checkpoint_completion_and_prelude( + pool: &sqlx::PgPool, + deployment_profile: &str, + chain: &str, + checkpoint_cursor_kind: &str, + range_start_block_number: i64, + target_block_number: i64, + adapters: &[NormalizedEventReplayAdapter], + max_raw_logs_per_page: usize, + checkpoint_completion: FullClosureCheckpointCompletion, + prelude: Prelude, +) -> Result<(FullClosureSyncResult, T)> +where + Prelude: FnOnce() -> PreludeFuture, + PreludeFuture: Future>, +{ ensure!( !checkpoint_cursor_kind.trim().is_empty(), "full-closure replay checkpoint cursor kind must not be empty" @@ -132,6 +174,16 @@ async fn sync_full_closure_with_checkpoint_completion( ) .await?; } + let prelude_output = prelude().await?; + if !adapters.is_empty() { + ensure_full_closure_retention_authority_for_adapters( + pool, + chain, + adapters, + target_block_number, + ) + .await?; + } let summary = sync_full_closure_normalized_events_without_lock( pool, deployment_profile, @@ -177,7 +229,7 @@ async fn sync_full_closure_with_checkpoint_completion( if let Some(guard) = raw_log_guard { guard.release().await?; } - Ok(FullClosureSyncResult { summary }) + Ok((FullClosureSyncResult { summary }, prelude_output)) }) .await } diff --git a/apps/indexer/src/main/reconciliation/adapter_sync/full_closure/automatic.rs b/apps/indexer/src/main/reconciliation/adapter_sync/full_closure/automatic.rs new file mode 100644 index 00000000..54d00a7d --- /dev/null +++ b/apps/indexer/src/main/reconciliation/adapter_sync/full_closure/automatic.rs @@ -0,0 +1,231 @@ +use std::{ + fmt, + sync::{ + Arc, + atomic::{AtomicBool, Ordering}, + }, +}; + +use anyhow::{Context, Result, ensure}; +use tracing::info; + +use crate::reconciliation::{ + replay::{ + NormalizedEventReplayAdapter, replay_stateless_normalized_events_before_full_closure, + select_log_bounded_replay_to_block, + }, + types::{ + PersistedRawPayloadAdapterSyncSummary, RawFactNormalizedEventReplayOutcome, + RawFactNormalizedEventReplayRequest, RawFactNormalizedEventReplaySelection, + }, +}; + +use super::{ + FullClosureCheckpointCompletion, sync_full_closure_with_checkpoint_completion_and_prelude, +}; + +#[cfg(test)] +#[path = "automatic/test_hook.rs"] +mod test_hook; +#[cfg(test)] +pub(crate) use test_hook::{install_after_stateless_failure, install_stateless_page_observer}; + +pub(crate) struct AutomaticTwoPhaseFullClosureSyncResult { + pub(crate) stateless: RawFactNormalizedEventReplayOutcome, + pub(crate) closure: PersistedRawPayloadAdapterSyncSummary, +} + +#[derive(Debug)] +struct AutomaticStatelessReplayCompleted; + +impl fmt::Display for AutomaticStatelessReplayCompleted { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + formatter.write_str("automatic stateless replay phase completed before failure") + } +} + +impl std::error::Error for AutomaticStatelessReplayCompleted {} + +pub(crate) fn automatic_stateless_replay_completed(error: &anyhow::Error) -> bool { + error + .downcast_ref::() + .is_some() +} + +#[expect(clippy::too_many_arguments)] +pub(crate) async fn sync_automatic_two_phase_full_closure_normalized_events( + pool: &sqlx::PgPool, + deployment_profile: &str, + chain: &str, + checkpoint_cursor_kind: &str, + range_start_block_number: i64, + target_block_number: i64, + stateless_ranges: &[(i64, i64)], + adapters: &[NormalizedEventReplayAdapter], + max_raw_logs_per_page: usize, +) -> Result { + info!( + service = "indexer", + replay_cursor_kind = checkpoint_cursor_kind, + chain, + range_start_block_number, + target_block_number, + stateless_range_count = stateless_ranges.len(), + stateless_ranges = ?stateless_ranges, + "automatic two-phase normalized-event replay session started" + ); + + let stateless_replay_completed = Arc::new(AtomicBool::new(false)); + let stateless_replay_completed_in_prelude = Arc::clone(&stateless_replay_completed); + let sync = sync_full_closure_with_checkpoint_completion_and_prelude( + pool, + deployment_profile, + chain, + checkpoint_cursor_kind, + range_start_block_number, + target_block_number, + adapters, + max_raw_logs_per_page, + FullClosureCheckpointCompletion::Retain, + move || async move { + let stateless = replay_stateless_normalized_events_for_ranges( + pool, + deployment_profile, + chain, + stateless_ranges, + max_raw_logs_per_page, + ) + .await?; + stateless_replay_completed_in_prelude.store(true, Ordering::Release); + #[cfg(test)] + test_hook::fail_after_stateless(pool, deployment_profile, chain).await?; + Ok(stateless) + }, + ) + .await; + let (closure, stateless) = match sync { + Ok(result) => result, + Err(error) if stateless_replay_completed.load(Ordering::Acquire) => { + return Err(error.context(AutomaticStatelessReplayCompleted)); + } + Err(error) => return Err(error), + }; + + Ok(AutomaticTwoPhaseFullClosureSyncResult { + stateless, + closure: closure.summary, + }) +} + +async fn replay_stateless_normalized_events_for_ranges( + pool: &sqlx::PgPool, + deployment_profile: &str, + chain: &str, + stateless_ranges: &[(i64, i64)], + max_raw_logs_per_page: usize, +) -> Result { + ensure!( + !stateless_ranges.is_empty(), + "automatic stateless replay requires at least one block range" + ); + let mut aggregate = RawFactNormalizedEventReplayOutcome { + deployment_profile: deployment_profile.to_owned(), + chain: chain.to_owned(), + selection_kind: "block_range", + source_scope_target_count: 0, + selected_block_count: 0, + canonical_raw_log_count: 0, + scanned_raw_log_count: 0, + matched_raw_log_count: 0, + normalized_event_synced_count: 0, + normalized_event_inserted_count: 0, + }; + for &(range_start_block_number, target_block_number) in stateless_ranges { + let range = replay_stateless_normalized_events_in_pages( + pool, + deployment_profile, + chain, + range_start_block_number, + target_block_number, + max_raw_logs_per_page, + ) + .await?; + aggregate.selected_block_count += range.selected_block_count; + aggregate.canonical_raw_log_count += range.canonical_raw_log_count; + aggregate.scanned_raw_log_count += range.scanned_raw_log_count; + aggregate.matched_raw_log_count += range.matched_raw_log_count; + aggregate.normalized_event_synced_count += range.normalized_event_synced_count; + aggregate.normalized_event_inserted_count += range.normalized_event_inserted_count; + } + Ok(aggregate) +} + +async fn replay_stateless_normalized_events_in_pages( + pool: &sqlx::PgPool, + deployment_profile: &str, + chain: &str, + range_start_block_number: i64, + target_block_number: i64, + max_raw_logs_per_page: usize, +) -> Result { + ensure!( + range_start_block_number <= target_block_number, + "automatic stateless replay range start {range_start_block_number} is after target {target_block_number}" + ); + ensure!( + max_raw_logs_per_page > 0, + "automatic stateless replay max logs per page must be positive" + ); + let mut aggregate = RawFactNormalizedEventReplayOutcome { + deployment_profile: deployment_profile.to_owned(), + chain: chain.to_owned(), + selection_kind: "block_range", + source_scope_target_count: 0, + selected_block_count: 0, + canonical_raw_log_count: 0, + scanned_raw_log_count: 0, + matched_raw_log_count: 0, + normalized_event_synced_count: 0, + normalized_event_inserted_count: 0, + }; + let mut from_block = range_start_block_number; + loop { + let to_block = select_log_bounded_replay_to_block( + pool, + chain, + from_block, + target_block_number, + max_raw_logs_per_page, + ) + .await?; + #[cfg(test)] + test_hook::record_stateless_page(pool, deployment_profile, chain, from_block, to_block) + .await?; + let page = replay_stateless_normalized_events_before_full_closure( + pool, + RawFactNormalizedEventReplayRequest { + deployment_profile: deployment_profile.to_owned(), + chain: chain.to_owned(), + selection: RawFactNormalizedEventReplaySelection::BlockRange { + from_block, + to_block, + }, + }, + ) + .await?; + aggregate.selected_block_count += page.selected_block_count; + aggregate.canonical_raw_log_count += page.canonical_raw_log_count; + aggregate.scanned_raw_log_count += page.scanned_raw_log_count; + aggregate.matched_raw_log_count += page.matched_raw_log_count; + aggregate.normalized_event_synced_count += page.normalized_event_synced_count; + aggregate.normalized_event_inserted_count += page.normalized_event_inserted_count; + + if to_block == target_block_number { + break; + } + from_block = to_block + .checked_add(1) + .context("automatic stateless replay page boundary overflowed")?; + } + Ok(aggregate) +} diff --git a/apps/indexer/src/main/reconciliation/adapter_sync/full_closure/automatic/test_hook.rs b/apps/indexer/src/main/reconciliation/adapter_sync/full_closure/automatic/test_hook.rs new file mode 100644 index 00000000..9cfa48a3 --- /dev/null +++ b/apps/indexer/src/main/reconciliation/adapter_sync/full_closure/automatic/test_hook.rs @@ -0,0 +1,85 @@ +use std::sync::{Arc, Mutex}; + +use anyhow::Result; +use bigname_test_support::{ScopedTestHookGuard, ScopedTestHookRegistry, current_test_database}; + +type HookKey = (String, String, String); + +static FAILURES: ScopedTestHookRegistry = ScopedTestHookRegistry::new(); +type PageRanges = Arc>>; +static PAGE_OBSERVERS: ScopedTestHookRegistry = ScopedTestHookRegistry::new(); + +pub(crate) struct AutomaticStatelessPageTestHook { + page_ranges: PageRanges, + _registration: ScopedTestHookGuard, +} + +impl AutomaticStatelessPageTestHook { + pub(crate) fn page_ranges(&self) -> Vec<(i64, i64)> { + self.page_ranges + .lock() + .expect("automatic stateless page observer mutex poisoned") + .clone() + } +} + +pub(crate) async fn install_after_stateless_failure( + pool: &sqlx::PgPool, + deployment_profile: &str, + chain: &str, +) -> Result> { + let database = current_test_database(pool).await?; + Ok(FAILURES.install( + (database, deployment_profile.to_owned(), chain.to_owned()), + (), + )) +} + +pub(crate) async fn install_stateless_page_observer( + pool: &sqlx::PgPool, + deployment_profile: &str, + chain: &str, +) -> Result { + let database = current_test_database(pool).await?; + let key = (database, deployment_profile.to_owned(), chain.to_owned()); + let page_ranges = Arc::new(Mutex::new(Vec::new())); + let registration = PAGE_OBSERVERS.install(key, Arc::clone(&page_ranges)); + Ok(AutomaticStatelessPageTestHook { + page_ranges, + _registration: registration, + }) +} + +pub(super) async fn record_stateless_page( + pool: &sqlx::PgPool, + deployment_profile: &str, + chain: &str, + from_block: i64, + to_block: i64, +) -> Result<()> { + let database = current_test_database(pool).await?; + if let Some(page_ranges) = + PAGE_OBSERVERS.get_cloned(&(database, deployment_profile.to_owned(), chain.to_owned())) + { + page_ranges + .lock() + .expect("automatic stateless page observer mutex poisoned") + .push((from_block, to_block)); + } + Ok(()) +} + +pub(super) async fn fail_after_stateless( + pool: &sqlx::PgPool, + deployment_profile: &str, + chain: &str, +) -> Result<()> { + let database = current_test_database(pool).await?; + if FAILURES + .take(&(database, deployment_profile.to_owned(), chain.to_owned())) + .is_some() + { + anyhow::bail!("injected failure after automatic stateless replay phase"); + } + Ok(()) +} diff --git a/apps/indexer/src/main/reconciliation/replay.rs b/apps/indexer/src/main/reconciliation/replay.rs index 1cd165da..12528f62 100644 --- a/apps/indexer/src/main/reconciliation/replay.rs +++ b/apps/indexer/src/main/reconciliation/replay.rs @@ -7,6 +7,8 @@ use tracing::info; #[path = "replay/classification.rs"] mod classification; +#[path = "replay/paging.rs"] +mod paging; #[path = "replay/profile_scope.rs"] mod profile_scope; #[path = "replay/scoped.rs"] @@ -23,6 +25,7 @@ use super::{ }, }; use classification::classify_raw_fact_replay_contract; +pub(crate) use paging::select_log_bounded_replay_to_block; use profile_scope::{ ensure_replay_matches_deployment_profile_scope, load_replay_adapter_source_scopes, }; @@ -38,9 +41,57 @@ pub(crate) use classification::{ source_scope_includes_adapter, unsupported_closure_replay_adapters, }; +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +enum RawFactReplayExecution { + Complete, + StatelessBeforeFullClosure, +} + +impl RawFactReplayExecution { + const fn as_str(self) -> &'static str { + match self { + Self::Complete => "complete", + Self::StatelessBeforeFullClosure => "stateless_before_full_closure", + } + } + + const fn runs_full_closure(self) -> bool { + matches!(self, Self::Complete) + } + + const fn validates_request_profile(self) -> bool { + matches!(self, Self::Complete) + } +} + pub(crate) async fn replay_raw_fact_normalized_events( pool: &sqlx::PgPool, request: RawFactNormalizedEventReplayRequest, +) -> Result { + replay_raw_fact_normalized_events_with_execution( + pool, + request, + RawFactReplayExecution::Complete, + ) + .await +} + +pub(crate) async fn replay_stateless_normalized_events_before_full_closure( + pool: &sqlx::PgPool, + request: RawFactNormalizedEventReplayRequest, +) -> Result { + replay_raw_fact_normalized_events_with_execution( + pool, + request, + RawFactReplayExecution::StatelessBeforeFullClosure, + ) + .await +} + +async fn replay_raw_fact_normalized_events_with_execution( + pool: &sqlx::PgPool, + request: RawFactNormalizedEventReplayRequest, + execution: RawFactReplayExecution, ) -> Result { if request.deployment_profile.trim().is_empty() { bail!("deployment_profile must not be empty"); @@ -54,7 +105,10 @@ pub(crate) async fn replay_raw_fact_normalized_events( let load_selection_ms = selection_started.elapsed().as_millis(); let profile_scope_started = Instant::now(); - ensure_replay_matches_deployment_profile_scope(pool, &request, raw_log_selection.range).await?; + if execution.validates_request_profile() { + ensure_replay_matches_deployment_profile_scope(pool, &request, raw_log_selection.range) + .await?; + } let profile_scope_ms = profile_scope_started.elapsed().as_millis(); let source_scope_started = Instant::now(); @@ -68,14 +122,27 @@ pub(crate) async fn replay_raw_fact_normalized_events( let source_scope_ms = source_scope_started.elapsed().as_millis(); let source_scope = &source_scopes.execution; - let replay_contract_plan = classify_raw_fact_replay_contract( - pool, - &request, - &raw_log_selection, - source_scope, - &source_scopes.closure_validation, - ) - .await?; + let replay_contract_plan = match execution { + RawFactReplayExecution::Complete => { + classify_raw_fact_replay_contract( + pool, + &request, + &raw_log_selection, + source_scope, + &source_scopes.closure_validation, + ) + .await? + } + // Automatic catch-up has already admitted the manifest corpus, and + // its cursor profile is an operational ownership identity rather than + // a manual replay request. It validates retained history immediately + // before phase two. Phase one uses the same restricted adapter + // selection as manual whole-range replay, so it runs only producers + // that phase two does not re-emit. + RawFactReplayExecution::StatelessBeforeFullClosure => { + RawFactReplayContractPlan::full_closure() + } + }; let adapter_sync_started = Instant::now(); let mut normalized_event_summary = if raw_log_selection.block_hashes.is_empty() { @@ -100,7 +167,7 @@ pub(crate) async fn replay_raw_fact_normalized_events( ) .await? }; - if replay_contract_plan.permits_nonstateless_adapters() { + if execution.runs_full_closure() && replay_contract_plan.permits_nonstateless_adapters() { let RawFactNormalizedEventReplaySelection::BlockRange { from_block, to_block, @@ -139,6 +206,7 @@ pub(crate) async fn replay_raw_fact_normalized_events( info!( service = "indexer", replay_cursor_kind = "raw_fact_normalized_events", + replay_phase = execution.as_str(), deployment_profile = %request.deployment_profile, chain = %request.chain, selection_kind, @@ -146,7 +214,8 @@ pub(crate) async fn replay_raw_fact_normalized_events( selected_block_count = raw_log_selection.block_hashes.len(), address_target_count = raw_log_selection.address_targets.len(), replay_source_scope_target_count = source_scope.len(), - closure_or_dependency_replay = replay_contract_plan.permits_nonstateless_adapters(), + closure_or_dependency_replay = execution.runs_full_closure() + && replay_contract_plan.permits_nonstateless_adapters(), canonical_raw_log_count = raw_log_selection.canonical_raw_log_count, scanned_raw_log_count = normalized_event_summary.scanned_log_count, matched_raw_log_count = normalized_event_summary.matched_log_count, diff --git a/apps/indexer/src/main/reconciliation/replay/classification.rs b/apps/indexer/src/main/reconciliation/replay/classification.rs index 1e19be89..249bca5c 100644 --- a/apps/indexer/src/main/reconciliation/replay/classification.rs +++ b/apps/indexer/src/main/reconciliation/replay/classification.rs @@ -330,14 +330,18 @@ pub(crate) async fn classify_raw_fact_replay_contract( ); } - let closure_source_families = closure_source_families_for_contracts(&nonstateless_contracts); - ensure_full_closure_retention_authority( + let nonstateless_adapters = nonstateless_contracts + .iter() + .map(|contract| contract.adapter) + .collect::>(); + ensure_full_closure_retention_authority_for_adapters( pool, &request.chain, - &closure_source_families, + &nonstateless_adapters, *to_block, ) .await?; + let closure_source_families = closure_source_families_for_contracts(&nonstateless_contracts); let closure_start_block = earliest_required_raw_fact_block( pool, &request.chain, @@ -457,6 +461,10 @@ pub(crate) async fn ensure_full_closure_retention_authority_for_adapters( adapters: &[NormalizedEventReplayAdapter], through_block: i64, ) -> Result<()> { + if adapters.contains(&NormalizedEventReplayAdapter::EnsV2RegistryResourceSurface) { + bigname_adapters::ensure_ens_v2_retained_history_proof_through(pool, chain, through_block) + .await?; + } let contracts = adapters .iter() .copied() diff --git a/apps/indexer/src/main/reconciliation/replay/classification/closure_boundary.rs b/apps/indexer/src/main/reconciliation/replay/classification/closure_boundary.rs index 1605f702..f2075d36 100644 --- a/apps/indexer/src/main/reconciliation/replay/classification/closure_boundary.rs +++ b/apps/indexer/src/main/reconciliation/replay/classification/closure_boundary.rs @@ -15,7 +15,8 @@ use crate::ens_v1_resolver::{ use crate::reconciliation::canonical::stored_lineage::find_uncovered_generation_bound_coverage_with_current_topics; use super::{ - SOURCE_FAMILY_BASENAMES_BASE_REGISTRY, SOURCE_FAMILY_ENS_V1_REGISTRY_L1, source_family_in, + SOURCE_FAMILY_BASENAMES_BASE_REGISTRY, SOURCE_FAMILY_ENS_V1_REGISTRY_L1, + SOURCE_FAMILY_ENS_V2_RESOLVER_L1, source_family_in, }; #[derive(Clone, Copy, Debug, Eq, PartialEq)] @@ -175,6 +176,21 @@ pub(super) async fn ensure_full_closure_retention_authority( generation_covered_families.join(", ") ); if !proof.uncovered.is_empty() { + if let Some(uncovered) = proof + .uncovered + .iter() + .find(|tuple| tuple.source_family == SOURCE_FAMILY_ENS_V2_RESOLVER_L1) + { + return Err(bigname_adapters::EnsV2MissingCoverage { + chain: chain.to_owned(), + retention_generation, + source_family: uncovered.source_family.clone(), + address: uncovered.address.clone(), + required_from_block: uncovered.required_from_block, + required_to_block: uncovered.required_to_block, + } + .into()); + } let listed = render_uncovered_tuples(&proof.uncovered); let suffix = elided_gap_suffix(&proof.uncovered); anyhow::bail!( @@ -428,65 +444,5 @@ pub(super) async fn earliest_required_raw_fact_block( } #[cfg(test)] -mod tests { - use bigname_test_support::{TestDatabase, TestDatabaseConfig}; - - use super::*; - use crate::reconciliation::replay::classification::SOURCE_FAMILY_ENS_V2_REGISTRY_L1; - - #[test] - fn legacy_registry_closure_has_generation_bound_coverage_strategy() { - assert_eq!( - retention_closure_authority_kind(ENS_V2_RETAINED_HISTORY_SOURCE_FAMILIES), - RetentionClosureAuthorityKind::EnsV2Proof - ); - for source_families in [ - &[SOURCE_FAMILY_ENS_V1_REGISTRY_L1][..], - &[SOURCE_FAMILY_BASENAMES_BASE_REGISTRY][..], - &[ - SOURCE_FAMILY_ENS_V1_REGISTRY_L1, - SOURCE_FAMILY_BASENAMES_BASE_REGISTRY, - ][..], - ] { - assert_eq!( - retention_closure_authority_kind(source_families), - RetentionClosureAuthorityKind::LegacyRegistryCoverage - ); - } - assert_eq!( - retention_closure_authority_kind(&[ - SOURCE_FAMILY_ENS_V1_REGISTRY_L1, - SOURCE_FAMILY_ENS_V1_RESOLVER_L1, - ]), - RetentionClosureAuthorityKind::Unsupported - ); - } - - #[tokio::test] - async fn full_closure_fails_closed_without_retention_authority_state() -> Result<()> { - let database = TestDatabase::create_migrated( - TestDatabaseConfig::new("indexer_closure_boundary_missing_authority"), - &bigname_storage::MIGRATOR, - "failed to apply migrations for closure-boundary test", - ) - .await?; - - let error = ensure_full_closure_retention_authority( - database.pool(), - "unconfigured-testnet", - &[SOURCE_FAMILY_ENS_V2_REGISTRY_L1], - 1, - ) - .await - .expect_err("full closure without durable retention authority must fail closed"); - - assert!( - error - .to_string() - .contains("has no raw-log retention authority state"), - "unexpected missing-authority error: {error:#}" - ); - - database.cleanup().await - } -} +#[path = "closure_boundary/tests.rs"] +mod tests; diff --git a/apps/indexer/src/main/reconciliation/replay/classification/closure_boundary/tests.rs b/apps/indexer/src/main/reconciliation/replay/classification/closure_boundary/tests.rs new file mode 100644 index 00000000..1072e6eb --- /dev/null +++ b/apps/indexer/src/main/reconciliation/replay/classification/closure_boundary/tests.rs @@ -0,0 +1,60 @@ +use bigname_test_support::{TestDatabase, TestDatabaseConfig}; + +use super::*; +use crate::reconciliation::replay::classification::SOURCE_FAMILY_ENS_V2_REGISTRY_L1; + +#[test] +fn legacy_registry_closure_has_generation_bound_coverage_strategy() { + assert_eq!( + retention_closure_authority_kind(ENS_V2_RETAINED_HISTORY_SOURCE_FAMILIES), + RetentionClosureAuthorityKind::EnsV2Proof + ); + for source_families in [ + &[SOURCE_FAMILY_ENS_V1_REGISTRY_L1][..], + &[SOURCE_FAMILY_BASENAMES_BASE_REGISTRY][..], + &[ + SOURCE_FAMILY_ENS_V1_REGISTRY_L1, + SOURCE_FAMILY_BASENAMES_BASE_REGISTRY, + ][..], + ] { + assert_eq!( + retention_closure_authority_kind(source_families), + RetentionClosureAuthorityKind::LegacyRegistryCoverage + ); + } + assert_eq!( + retention_closure_authority_kind(&[ + SOURCE_FAMILY_ENS_V1_REGISTRY_L1, + SOURCE_FAMILY_ENS_V1_RESOLVER_L1, + ]), + RetentionClosureAuthorityKind::Unsupported + ); +} + +#[tokio::test] +async fn full_closure_fails_closed_without_retention_authority_state() -> Result<()> { + let database = TestDatabase::create_migrated( + TestDatabaseConfig::new("indexer_closure_boundary_missing_authority"), + &bigname_storage::MIGRATOR, + "failed to apply migrations for closure-boundary test", + ) + .await?; + + let error = ensure_full_closure_retention_authority( + database.pool(), + "unconfigured-testnet", + &[SOURCE_FAMILY_ENS_V2_REGISTRY_L1], + 1, + ) + .await + .expect_err("full closure without durable retention authority must fail closed"); + + assert!( + error + .to_string() + .contains("has no raw-log retention authority state"), + "unexpected missing-authority error: {error:#}" + ); + + database.cleanup().await +} diff --git a/apps/indexer/src/main/reconciliation/replay/paging.rs b/apps/indexer/src/main/reconciliation/replay/paging.rs new file mode 100644 index 00000000..f8a3fe29 --- /dev/null +++ b/apps/indexer/src/main/reconciliation/replay/paging.rs @@ -0,0 +1,74 @@ +use anyhow::{Context, Result}; +use sqlx::PgPool; + +pub(crate) async fn select_log_bounded_replay_to_block( + pool: &PgPool, + chain: &str, + from_block: i64, + hard_to_block: i64, + max_raw_logs_per_page: usize, +) -> Result { + if from_block >= hard_to_block { + return Ok(hard_to_block); + } + let max_raw_logs_per_page = i64::try_from(max_raw_logs_per_page) + .context("normalized replay max logs per page does not fit in i64")?; + + sqlx::query_scalar::<_, i64>( + r#" + WITH ordered_logs AS ( + SELECT raw_logs.block_number + FROM raw_logs + JOIN chain_lineage AS lineage + ON lineage.chain_id = raw_logs.chain_id + AND lineage.block_hash = raw_logs.block_hash + WHERE raw_logs.chain_id = $1 + AND raw_logs.block_number >= $2 + AND raw_logs.block_number <= $3 + AND lineage.canonicality_state IN ( + 'canonical'::canonicality_state, + 'safe'::canonicality_state, + 'finalized'::canonicality_state + ) + AND raw_logs.canonicality_state IN ( + 'canonical'::canonicality_state, + 'safe'::canonicality_state, + 'finalized'::canonicality_state + ) + ORDER BY raw_logs.block_number ASC, raw_logs.log_index ASC + LIMIT ($4 + 1) + ), + numbered_logs AS ( + SELECT block_number, ROW_NUMBER() OVER () AS ordinal + FROM ordered_logs + ), + overflow AS ( + SELECT block_number + FROM numbered_logs + WHERE ordinal = $4 + 1 + ), + bounded AS ( + SELECT block_number + FROM numbered_logs + WHERE NOT EXISTS (SELECT 1 FROM overflow) + OR block_number < (SELECT block_number FROM overflow) + UNION ALL + SELECT MIN(block_number) + FROM numbered_logs + ) + SELECT COALESCE(MAX(block_number), $3) + FROM bounded + "#, + ) + .bind(chain) + .bind(from_block) + .bind(hard_to_block) + .bind(max_raw_logs_per_page) + .fetch_one(pool) + .await + .with_context(|| { + format!( + "failed to select log-bounded normalized replay range for chain {chain} range {from_block}..={hard_to_block}" + ) + }) +} diff --git a/apps/indexer/src/main/tests/normalized_replay_catchup.rs b/apps/indexer/src/main/tests/normalized_replay_catchup.rs index 238341aa..f5bbbdd6 100644 --- a/apps/indexer/src/main/tests/normalized_replay_catchup.rs +++ b/apps/indexer/src/main/tests/normalized_replay_catchup.rs @@ -418,6 +418,8 @@ async fn normalized_replay_catchup_does_not_use_log_bound_as_stateful_boundary() 1, 1, )?; + let stateless_pages = + install_stateless_page_observer(database.pool(), "mainnet", chain).await?; let status = normalized_replay_catchup::run_normalized_replay_catchup_iteration( database.pool(), &config, @@ -428,6 +430,11 @@ async fn normalized_replay_catchup_does_not_use_log_bound_as_stateful_boundary() status, normalized_replay_catchup::CatchupIterationStatus::Progressed ); + assert_eq!( + stateless_pages.page_ranges(), + vec![(30, 30), (31, 31)], + "the stateless prelude must honor the configured whole-block raw-log page cap" + ); let cursor_kind = "raw_fact_normalized_events"; let (last_completed, next_block) = sqlx::query_as::<_, (Option, i64)>( @@ -450,6 +457,19 @@ async fn normalized_replay_catchup_does_not_use_log_bound_as_stateful_boundary() .await? > 0 ); + assert_eq!( + sqlx::query_scalar::<_, i64>( + "SELECT COUNT(*)::BIGINT FROM normalized_events \ + WHERE event_kind = 'PreimageObserved' \ + AND derivation_kind = 'raw_log_preimage_observation' \ + AND block_hash = $1" + ) + .bind(&wrapper_block.block_hash) + .fetch_one(database.pool()) + .await?, + 1, + "automatic full-closure catch-up must run the stateless preimage pass" + ); assert!( sqlx::query_scalar::<_, bool>( "SELECT to_regclass('normalized_events_namespace_idx') IS NOT NULL" @@ -463,6 +483,197 @@ async fn normalized_replay_catchup_does_not_use_log_bound_as_stateful_boundary() Ok(()) } +#[tokio::test] +async fn normalized_replay_catchup_validates_retention_before_stateless_phase() -> Result<()> { + let database = TestDatabase::new().await?; + create_normalized_replay_cursor_table(database.pool()).await?; + create_ops_catchup_backfill_job_tables(database.pool()).await?; + let deployment_profile = "retention-validation-test"; + let chain = "ethereum-mainnet"; + let wrapper_address = "0x0000000000000000000000000000000000000138"; + let block = provider_block( + "0x3838383838383838383838383838383838383838383838383838383838383838", + Some("0x3737373737373737373737373737373737373737373737373737373737373737"), + 38, + ); + + insert_active_replay_watched_contract( + database.pool(), + 398, + chain, + Uuid::from_u128(0x398), + wrapper_address, + ) + .await?; + insert_raw_name_wrapped_log( + database.pool(), + chain, + &block, + wrapper_address, + 0, + CanonicalityState::Canonical, + ) + .await?; + upsert_raw_staging_input_version_for_handoff_test(database.pool(), chain, 2, 1).await?; + + let config = normalized_replay_catchup::NormalizedReplayCatchupConfig::new( + deployment_profile.to_owned(), + vec![chain.to_owned()], + 1_000, + 1_000, + 1, + )?; + let stateless_pages = + install_stateless_page_observer(database.pool(), deployment_profile, chain).await?; + let error = normalized_replay_catchup::run_normalized_replay_catchup_iteration( + database.pool(), + &config, + chain, + ) + .await + .expect_err("incomplete retention authority must fail closed"); + assert!( + format!("{error:#}").contains("retention generation 1"), + "unexpected retention validation error: {error:#}" + ); + assert_eq!( + stateless_pages.page_ranges(), + Vec::<(i64, i64)>::new(), + "retention validation must abort before any stateless replay page" + ); + assert_eq!( + sqlx::query_scalar::<_, i64>( + "SELECT COUNT(*)::BIGINT FROM normalized_events \ + WHERE event_kind = 'PreimageObserved' \ + AND derivation_kind = 'raw_log_preimage_observation'" + ) + .fetch_one(database.pool()) + .await?, + 0, + "retention validation failure must attempt zero stateless upserts" + ); + + database.cleanup().await +} + +#[tokio::test] +async fn normalized_replay_catchup_retries_full_closure_after_stateless_phase_failure() +-> Result<()> { + let database = TestDatabase::new().await?; + create_normalized_replay_cursor_table(database.pool()).await?; + let deployment_profile = "automatic-catchup-test"; + let chain = "ethereum-mainnet"; + let wrapper_address = "0x0000000000000000000000000000000000000139"; + let block = provider_block( + "0x3939393939393939393939393939393939393939393939393939393939393939", + Some("0x3838383838383838383838383838383838383838383838383838383838383838"), + 39, + ); + + insert_active_replay_watched_contract( + database.pool(), + 399, + chain, + Uuid::from_u128(0x399), + wrapper_address, + ) + .await?; + insert_raw_name_wrapped_log( + database.pool(), + chain, + &block, + wrapper_address, + 0, + CanonicalityState::Canonical, + ) + .await?; + + let config = normalized_replay_catchup::NormalizedReplayCatchupConfig::new( + deployment_profile.to_owned(), + vec![chain.to_owned()], + 1_000, + 1_000, + 1, + )?; + let _failure = + install_after_stateless_failure(database.pool(), deployment_profile, chain).await?; + let error = normalized_replay_catchup::run_normalized_replay_catchup_iteration( + database.pool(), + &config, + chain, + ) + .await + .expect_err("the injected phase-boundary failure must stop catch-up"); + assert!( + format!("{error:#}").contains("injected failure after automatic stateless replay phase"), + "unexpected phase-boundary error: {error:#}" + ); + assert_eq!( + sqlx::query_as::<_, (i64, Option)>( + "SELECT next_block_number, last_completed_block_number \ + FROM normalized_replay_cursors \ + WHERE deployment_profile = $1 \ + AND chain_id = $2 \ + AND cursor_kind = 'raw_fact_normalized_events'" + ) + .bind(deployment_profile) + .bind(chain) + .fetch_one(database.pool()) + .await?, + (block.block_number, None), + "the shared completion cursor must stay pending between phases" + ); + assert_eq!( + sqlx::query_scalar::<_, i64>( + "SELECT COUNT(*)::BIGINT FROM normalized_events \ + WHERE event_kind = 'PreimageObserved' \ + AND derivation_kind = 'raw_log_preimage_observation'" + ) + .fetch_one(database.pool()) + .await?, + 1, + "phase one must durably publish the stateless preimage before the injected failure" + ); + + assert_eq!( + normalized_replay_catchup::run_normalized_replay_catchup_iteration( + database.pool(), + &config, + chain, + ) + .await?, + normalized_replay_catchup::CatchupIterationStatus::Progressed + ); + assert_eq!( + sqlx::query_as::<_, (i64, Option)>( + "SELECT next_block_number, last_completed_block_number \ + FROM normalized_replay_cursors \ + WHERE deployment_profile = $1 \ + AND chain_id = $2 \ + AND cursor_kind = 'raw_fact_normalized_events'" + ) + .bind(deployment_profile) + .bind(chain) + .fetch_one(database.pool()) + .await?, + (block.block_number + 1, Some(block.block_number)), + "retry must complete phase two before publishing the shared cursor" + ); + assert_eq!( + sqlx::query_scalar::<_, i64>( + "SELECT COUNT(*)::BIGINT FROM normalized_events \ + WHERE event_kind = 'PreimageObserved' \ + AND derivation_kind = 'raw_log_preimage_observation'" + ) + .fetch_one(database.pool()) + .await?, + 1, + "retrying phase one must be identity-idempotent" + ); + + database.cleanup().await +} + #[tokio::test] async fn normalized_replay_catchup_accepts_current_generation_ensv1_full_history_coverage() -> Result<()> { @@ -1185,7 +1396,7 @@ async fn normalized_replay_catchup_accepts_commit_strictly_after_latched_closure } #[tokio::test] -async fn normalized_replay_catchup_recovers_durable_ensv2_missing_coverage_without_early_cursor_advance() +async fn normalized_replay_catchup_recovers_multiple_new_ensv2_emitters_with_scoped_phase_one() -> Result<()> { let database = TestDatabase::new().await?; create_normalized_replay_cursor_table(database.pool()).await?; @@ -1193,11 +1404,15 @@ async fn normalized_replay_catchup_recovers_durable_ensv2_missing_coverage_witho let chain = "ethereum-sepolia"; let root_manifest_id = 51_300; let registry_manifest_id = 51_301; + let resolver_manifest_id = 51_302; let root_contract_instance_id = Uuid::from_u128(51_300); let registry_contract_instance_id = Uuid::from_u128(51_301); + let resolver_contract_instance_id = Uuid::from_u128(51_302); let root_address = "0x0000000000000000000000000000000000005130"; let registry_address = "0x0000000000000000000000000000000000005131"; + let historical_child_address = "0x0000000000000000000000000000000000005100"; let child_address = "0x0000000000000000000000000000000000005132"; + let resolver_address = "0x0000000000000000000000000000000000005133"; insert_normalized_replay_ens_v2_registry_manifests( database.pool(), chain, @@ -1209,6 +1424,14 @@ async fn normalized_replay_catchup_recovers_durable_ensv2_missing_coverage_witho registry_address, ) .await?; + insert_normalized_replay_ens_v2_resolver_manifest( + database.pool(), + chain, + resolver_manifest_id, + resolver_contract_instance_id, + resolver_address, + ) + .await?; sqlx::query( "INSERT INTO discovery_admission_epochs (chain_id, epoch) VALUES ($1, 0) ON CONFLICT (chain_id) DO NOTHING", ) @@ -1218,6 +1441,11 @@ async fn normalized_replay_catchup_recovers_durable_ensv2_missing_coverage_witho let block_1 = provider_block(&format!("0x{:064x}", 51_301), None, 1); let block_2 = provider_block(&format!("0x{:064x}", 51_302), Some(&block_1.block_hash), 2); + let block_3 = provider_block( + &format!("0x{:064x}", 51_303), + Some(&block_2.block_hash), + 3, + ); insert_chain_lineage_for_block( database.pool(), chain, @@ -1225,6 +1453,13 @@ async fn normalized_replay_catchup_recovers_durable_ensv2_missing_coverage_witho CanonicalityState::Finalized, ) .await?; + insert_chain_lineage_for_block( + database.pool(), + chain, + &block_3, + CanonicalityState::Finalized, + ) + .await?; insert_chain_lineage_for_block( database.pool(), chain, @@ -1234,11 +1469,11 @@ async fn normalized_replay_catchup_recovers_durable_ensv2_missing_coverage_witho .await?; upsert_raw_blocks( database.pool(), - &[provider_block_to_raw_block( - chain, - &block_2, - CanonicalityState::Finalized, - )], + &[ + provider_block_to_raw_block(chain, &block_1, CanonicalityState::Finalized), + provider_block_to_raw_block(chain, &block_2, CanonicalityState::Finalized), + provider_block_to_raw_block(chain, &block_3, CanonicalityState::Finalized), + ], ) .await?; upsert_raw_logs( @@ -1246,9 +1481,28 @@ async fn normalized_replay_catchup_recovers_durable_ensv2_missing_coverage_witho &[ RawLog { chain_id: chain.to_owned(), - block_hash: block_2.block_hash.clone(), - block_number: block_2.block_number, - transaction_hash: transaction_hash_for_block(&block_2), + block_hash: block_1.block_hash.clone(), + block_number: block_1.block_number, + transaction_hash: transaction_hash_for_block(&block_1), + transaction_index: 0, + log_index: 0, + emitting_address: registry_address.to_owned(), + topics: vec![ + keccak256_hex(b"SubregistryUpdated(uint256,address,address)"), + hex_string(&abi_word_u64(2)), + hex_string(&abi_word_address(historical_child_address)), + hex_string(&abi_word_address( + "0x0000000000000000000000000000000000000dad", + )), + ], + data: Vec::new(), + canonicality_state: CanonicalityState::Finalized, + }, + RawLog { + chain_id: chain.to_owned(), + block_hash: block_3.block_hash.clone(), + block_number: block_3.block_number, + transaction_hash: transaction_hash_for_block(&block_3), transaction_index: 0, log_index: 0, emitting_address: registry_address.to_owned(), @@ -1263,15 +1517,36 @@ async fn normalized_replay_catchup_recovers_durable_ensv2_missing_coverage_witho data: decode_hex_string(&encode_ens_v2_label_registered_log_data( "alice", "0x0000000000000000000000000000000000000aaa", - block_2.block_timestamp_unix_secs + 31_536_000, + block_3.block_timestamp_unix_secs + 31_536_000, )), canonicality_state: CanonicalityState::Finalized, }, RawLog { chain_id: chain.to_owned(), - block_hash: block_2.block_hash.clone(), - block_number: block_2.block_number, - transaction_hash: transaction_hash_for_block(&block_2), + block_hash: block_1.block_hash.clone(), + block_number: block_1.block_number, + transaction_hash: transaction_hash_for_block(&block_1), + transaction_index: 0, + log_index: 1, + emitting_address: registry_address.to_owned(), + topics: vec![ + keccak256_hex(b"SubregistryUpdated(uint256,address,address)"), + hex_string(&abi_word_u64(2)), + hex_string(&abi_word_address( + "0x0000000000000000000000000000000000000000", + )), + hex_string(&abi_word_address( + "0x0000000000000000000000000000000000000dad", + )), + ], + data: Vec::new(), + canonicality_state: CanonicalityState::Finalized, + }, + RawLog { + chain_id: chain.to_owned(), + block_hash: block_3.block_hash.clone(), + block_number: block_3.block_number, + transaction_hash: transaction_hash_for_block(&block_3), transaction_index: 0, log_index: 1, emitting_address: registry_address.to_owned(), @@ -1289,11 +1564,26 @@ async fn normalized_replay_catchup_recovers_durable_ensv2_missing_coverage_witho ], ) .await?; + sqlx::query( + r#" + UPDATE raw_log_staging_input_revisions + SET retention_generation = 1, + retained_history_complete = false, + incomplete_since = clock_timestamp(), + proven_retention_generation = NULL, + proven_discovery_admission_epoch = NULL, + proven_through_block = NULL + WHERE chain_id = $1 + "#, + ) + .bind(chain) + .execute(database.pool()) + .await?; insert_completed_backfill_range_coverage_for_source_family( database.pool(), chain, 1, - 2, + 3, "ens_v2_root_l1", &[root_address], ) @@ -1302,7 +1592,16 @@ async fn normalized_replay_catchup_recovers_durable_ensv2_missing_coverage_witho database.pool(), chain, 1, - 2, + 3, + "ens_v2_resolver_l1", + &[resolver_address], + ) + .await?; + insert_completed_backfill_range_coverage_for_source_family( + database.pool(), + chain, + 1, + 3, "ens_v2_registry_l1", &[registry_address], ) @@ -1312,11 +1611,11 @@ async fn normalized_replay_catchup_recovers_durable_ensv2_missing_coverage_witho UPDATE raw_log_staging_input_revisions SET retained_history_complete = true, incomplete_since = NULL, - proven_retention_generation = 0, + proven_retention_generation = 1, proven_discovery_admission_epoch = 0, - proven_through_block = 2 + proven_through_block = 3 WHERE chain_id = $1 - AND retention_generation = 0 + AND retention_generation = 1 "#, ) .bind(chain) @@ -1331,75 +1630,137 @@ async fn normalized_replay_catchup_recovers_durable_ensv2_missing_coverage_witho 1, )? .with_defer_projection_indexes(false); - let error = normalized_replay_catchup::run_normalized_replay_catchup_iteration( + let stateless_pages = + install_stateless_page_observer(database.pool(), "sepolia", chain).await?; + let recovery_hook = normalized_replay_catchup::install_after_coverage_recovery_test_hook( database.pool(), - &config, + "sepolia", chain, ) - .await - .expect_err("new discovery without a configured provider must fail closed"); - assert_eq!( - bigname_adapters::ens_v2_missing_coverage(&error), - Some(&bigname_adapters::EnsV2MissingCoverage { - chain: chain.to_owned(), - retention_generation: 0, - source_family: "ens_v2_registry_l1".to_owned(), - address: child_address.to_owned(), - required_from_block: 2, - required_to_block: 2, - }), - "the newly discovered requirement must survive catch-up context as an exact typed tuple" - ); - assert_eq!( - sqlx::query_as::<_, (i64, Option)>( - r#" - SELECT next_block_number, last_completed_block_number - FROM normalized_replay_cursors - WHERE deployment_profile = 'sepolia' - AND chain_id = $1 - AND cursor_kind = 'raw_fact_normalized_events' - "#, - ) - .bind(chain) - .fetch_one(database.pool()) - .await?, - (2, None), - "failed recovery must not advance the replay cursor or completion checkpoint" - ); - assert_eq!( - sqlx::query_scalar::<_, i64>( - "SELECT COUNT(*)::BIGINT FROM normalized_replay_adapter_checkpoints", - ) - .fetch_one(database.pool()) - .await?, - 0, - "failed ENSv2 replay must not publish an adapter checkpoint" - ); - assert_eq!( - sqlx::query_scalar::<_, i64>( - "SELECT COUNT(*)::BIGINT FROM discovery_edges WHERE chain_id = $1 AND edge_kind = 'subregistry' AND deactivated_at IS NULL", - ) - .bind(chain) - .fetch_one(database.pool()) - .await?, - 1, - "the restart pass must begin from the already-durable child admission" - ); - - let (provider, server) = bundle_provider_with_fixtures(vec![ProviderBlockFixture { - block: block_2.clone(), - logs: Vec::new(), - }]) + .await; + let (provider, server) = bundle_provider_with_fixtures(vec![ + ProviderBlockFixture { + block: block_1.clone(), + logs: vec![rpc_ens_v2_label_registered_log_payload( + &block_1, + historical_child_address, + 2, + "historical-recovered", + 3, + )], + }, + ProviderBlockFixture { + block: block_3.clone(), + logs: vec![rpc_ens_v2_label_registered_log_payload( + &block_3, + child_address, + 2, + "recovered", + 2, + )], + }, + ]) .await?; - assert_eq!( - normalized_replay_catchup::run_normalized_replay_catchup_iteration_with_provider_for_test( - database.pool(), - &config, + let pool = database.pool().clone(); + let task_config = config.clone(); + let task_provider = provider.clone(); + let replay = tokio::spawn(async move { + normalized_replay_catchup::run_normalized_replay_catchup_iteration_with_provider_for_test( + &pool, + &task_config, chain, - &provider, + &task_provider, HeaderAuditMode::Minimal, ) - .await?, + .await + }); + tokio::time::timeout( + std::time::Duration::from_secs(10), + recovery_hook.wait_until_after_coverage_recovery(), + ) + .await + .context("normalized replay did not reach its post-coverage-recovery barrier")?; + let version_before_concurrent = + bigname_storage::load_raw_log_staging_input_version(database.pool(), chain).await?; + upsert_raw_logs( + database.pool(), + &[RawLog { + chain_id: chain.to_owned(), + block_hash: block_2.block_hash.clone(), + block_number: block_2.block_number, + transaction_hash: transaction_hash_for_block(&block_2), + transaction_index: 0, + log_index: 9, + emitting_address: registry_address.to_owned(), + topics: vec![ + ens_v2_label_registered_topic0(), + hex_string(&abi_word_u64(1)), + labelhash_hex("concurrent"), + hex_string(&abi_word_address( + "0x0000000000000000000000000000000000000dad", + )), + ], + data: decode_hex_string(&encode_ens_v2_label_registered_log_data( + "concurrent", + "0x0000000000000000000000000000000000000aaa", + block_2.block_timestamp_unix_secs + 31_536_000, + )), + canonicality_state: CanonicalityState::Finalized, + }], + ) + .await?; + let concurrent_revision = sqlx::query_scalar::<_, i64>( + "UPDATE raw_log_staging_input_revisions \ + SET revision = revision + 1 \ + WHERE chain_id = $1 \ + RETURNING revision", + ) + .bind(chain) + .fetch_one(database.pool()) + .await?; + sqlx::query( + r#" + INSERT INTO raw_log_staging_block_revisions ( + chain_id, block_hash, block_number, revision + ) + VALUES ($1, $2, $3, $4) + "#, + ) + .bind(chain) + .bind(&block_2.block_hash) + .bind(block_2.block_number) + .bind(concurrent_revision) + .execute(database.pool()) + .await?; + let version_after_concurrent = + bigname_storage::load_raw_log_staging_input_version(database.pool(), chain).await?; + assert!( + version_after_concurrent.revision > version_before_concurrent.revision, + "concurrent insertion did not advance revision: before={version_before_concurrent:?}, after={version_after_concurrent:?}, row_count={}", + sqlx::query_scalar::<_, i64>( + "SELECT COUNT(*)::BIGINT FROM raw_logs WHERE chain_id = $1 AND block_hash = $2 AND log_index = 9" + ) + .bind(chain) + .bind(&block_2.block_hash) + .fetch_one(database.pool()) + .await? + ); + assert!( + bigname_storage::raw_log_staging_block_range_changed_since( + database.pool(), + chain, + version_before_concurrent.revision, + 2, + 2, + ) + .await? + ); + recovery_hook.resume(); + assert_eq!( + tokio::time::timeout(std::time::Duration::from_secs(10), replay) + .await + .context("normalized replay did not resume after coverage recovery")? + .context("normalized replay task panicked")??, normalized_replay_catchup::CatchupIterationStatus::Progressed ); assert_eq!( @@ -1426,11 +1787,11 @@ async fn normalized_replay_catchup_recovers_durable_ensv2_missing_coverage_witho ( "ens_v2_registry_l1".to_owned(), child_address.to_owned(), - 2, - 2, - 0, + 3, + 3, + 1, ), - "restart recovery must persist only exact generation-zero address coverage" + "restart recovery must persist only exact current-generation address coverage" ); assert_eq!( sqlx::query_as::<_, (i64, Option)>( @@ -1445,111 +1806,749 @@ async fn normalized_replay_catchup_recovers_durable_ensv2_missing_coverage_witho .bind(chain) .fetch_one(database.pool()) .await?, - (3, Some(2)), + (4, Some(3)), "the unchanged replay may advance only after provider-backed recovery succeeds" ); + assert_eq!( + stateless_pages.page_ranges(), + vec![(1, 3), (1, 3)], + "coverage recovery must widen phase one when another raw-log writer changes the saved span" + ); + assert_eq!( + sqlx::query_as::<_, (bool, Option, Option, Option, i64)>( + r#" + SELECT + retained.retained_history_complete, + retained.proven_retention_generation, + retained.proven_discovery_admission_epoch, + retained.proven_through_block, + admission.epoch + FROM raw_log_staging_input_revisions retained + JOIN discovery_admission_epochs admission + ON admission.chain_id = retained.chain_id + WHERE retained.chain_id = $1 + "#, + ) + .bind(chain) + .fetch_one(database.pool()) + .await?, + (true, Some(1), Some(1), Some(3), 1), + "coverage recovery must rebuild the generation-one proof at the discovered epoch" + ); + assert_eq!( + sqlx::query_scalar::<_, i64>( + "SELECT COUNT(*)::BIGINT FROM normalized_events \ + WHERE event_kind = 'PreimageObserved' \ + AND derivation_kind = 'raw_log_preimage_observation' \ + AND block_hash = $1 \ + AND log_index = 2" + ) + .bind(&block_3.block_hash) + .fetch_one(database.pool()) + .await?, + 1, + "phase one must derive a preimage row from the recovered raw log" + ); + assert_eq!( + sqlx::query_scalar::<_, i64>( + "SELECT COUNT(*)::BIGINT FROM raw_logs \ + WHERE chain_id = $1 \ + AND block_hash = $2 \ + AND log_index = 3 \ + AND emitting_address = $3" + ) + .bind(chain) + .bind(&block_1.block_hash) + .bind(historical_child_address) + .fetch_one(database.pool()) + .await?, + 1, + "the first exact recovery must persist its raw log" + ); + assert_eq!( + sqlx::query_scalar::<_, i64>( + "SELECT COUNT(*)::BIGINT FROM normalized_events \ + WHERE event_kind = 'PreimageObserved' \ + AND derivation_kind = 'raw_log_preimage_observation' \ + AND block_hash = $1 \ + AND log_index = 3" + ) + .bind(&block_1.block_hash) + .fetch_one(database.pool()) + .await?, + 1, + "phase one must not drop a prior recovered span while validation finds the next gap" + ); + assert_eq!( + sqlx::query_scalar::<_, i64>( + "SELECT COUNT(*)::BIGINT FROM normalized_events \ + WHERE event_kind = 'PreimageObserved' \ + AND derivation_kind = 'raw_log_preimage_observation' \ + AND block_hash = $1 \ + AND log_index = 9" + ) + .bind(&block_2.block_hash) + .fetch_one(database.pool()) + .await?, + 1, + "a concurrent in-span raw-log write must be included before its revision is acknowledged" + ); server.abort(); database.cleanup().await } #[tokio::test] -async fn normalized_replay_handoff_classifies_completed_cursor_input_versions() -> Result<()> { - let database = TestDatabase::new().await?; - create_normalized_replay_cursor_table(database.pool()).await?; - let target = 200; - - let generation_chain = "generation-drift"; - insert_completed_replay_cursor_for_handoff_test( - database.pool(), - generation_chain, - target, - 5, - 0, +async fn normalized_replay_coverage_recovery_rejects_raw_log_change_below_replay_span() +-> Result<()> { + let outcome = run_normalized_replay_coverage_fence_test( + NormalizedReplayCoverageFenceMutation::RawLogBelowReplaySpan, ) .await?; - upsert_raw_staging_input_version_for_handoff_test(database.pool(), generation_chain, 6, 1) - .await?; + + assert_eq!( + bigname_adapters::ens_v2_missing_coverage(&outcome.error), + Some(&bigname_adapters::EnsV2MissingCoverage { + chain: NORMALIZED_REPLAY_COVERAGE_FENCE_CHAIN.to_owned(), + retention_generation: 1, + source_family: "ens_v2_registry_l1".to_owned(), + address: NORMALIZED_REPLAY_COVERAGE_FENCE_HISTORICAL_CHILD.to_owned(), + required_from_block: 1, + required_to_block: 1, + }), + "the fail-closed fence must preserve the typed recovery requirement" + ); + let error = format!("{:#}", outcome.error); assert!( - !normalized_replay_catchup::normalized_replay_cursors_complete( - database.pool(), - "mainnet", - &[generation_chain.to_owned()], - ) - .await?, - "a completed cursor from an older retention generation must not hand off" + error.contains( + "raw-log staging input changed below normalized replay range start 1 during coverage recovery; replan from the durable cursor" + ), + "unexpected below-span recovery fence error: {error}" + ); + assert_eq!( + outcome.input_version_after.retention_generation, + outcome.input_version_before.retention_generation, + "the injected raw-log mutation must remain in the original retention generation" + ); + assert!( + outcome.input_version_after.revision > outcome.input_version_before.revision, + "the injected below-span raw log must advance the committed input revision" + ); + assert!( + outcome.changed_below_replay_span, + "the injected raw-log revision must be visible strictly below from_block" + ); + assert_eq!( + ( + outcome.cursor_after.range_start_block_number, + outcome.cursor_after.next_block_number, + outcome.cursor_after.target_block_number, + outcome.cursor_after.last_completed_block_number, + ), + ( + outcome.cursor_before.range_start_block_number, + outcome.cursor_before.next_block_number, + outcome.cursor_before.target_block_number, + outcome.cursor_before.last_completed_block_number, + ), + "the durable cursor must not advance after a below-span recovery fence failure" + ); + assert_eq!( + ( + outcome.cursor_after.raw_log_input_revision, + outcome.cursor_after.raw_log_retention_generation, + ), + ( + outcome.cursor_before.raw_log_input_revision, + outcome.cursor_before.raw_log_retention_generation, + ), + "the failed attempt must not acknowledge any part of the newer raw-log input version" ); - let post_target_chain = "strictly-post-target"; - insert_completed_replay_cursor_for_handoff_test( - database.pool(), - post_target_chain, - target, - 5, - 0, - ) - .await?; - upsert_raw_staging_input_version_for_handoff_test(database.pool(), post_target_chain, 6, 0) - .await?; - upsert_raw_staging_block_revision_for_handoff_test( - database.pool(), - post_target_chain, - "0xpost-target", - target + 1, - 6, + Ok(()) +} + +#[tokio::test] +async fn normalized_replay_coverage_recovery_rejects_retention_generation_change() -> Result<()> { + let outcome = run_normalized_replay_coverage_fence_test( + NormalizedReplayCoverageFenceMutation::RetentionGeneration, ) .await?; + + assert_eq!( + bigname_adapters::ens_v2_missing_coverage(&outcome.error), + Some(&bigname_adapters::EnsV2MissingCoverage { + chain: NORMALIZED_REPLAY_COVERAGE_FENCE_CHAIN.to_owned(), + retention_generation: 1, + source_family: "ens_v2_registry_l1".to_owned(), + address: NORMALIZED_REPLAY_COVERAGE_FENCE_HISTORICAL_CHILD.to_owned(), + required_from_block: 1, + required_to_block: 1, + }), + "the fail-closed fence must preserve the typed recovery requirement" + ); + let error = format!("{:#}", outcome.error); assert!( - normalized_replay_catchup::normalized_replay_cursors_complete( - database.pool(), - "mainnet", - &[post_target_chain.to_owned()], - ) - .await?, - "a witnessed revision strictly after the replay target remains backlog-owned" + error.contains( + "raw-log retention generation changed during normalized replay coverage recovery: expected 1, observed 2; replan the replay from current authority" + ), + "unexpected retention-generation recovery fence error: {error}" + ); + assert_eq!( + outcome.input_version_after.revision, + outcome.input_version_before.revision, + "rotating retention authority must not manufacture a raw-log input revision" + ); + assert_eq!( + outcome.input_version_after.retention_generation, + outcome.input_version_before.retention_generation + 1, + "the injected retention change must advance the generation exactly once" + ); + assert_eq!( + ( + outcome.cursor_after.range_start_block_number, + outcome.cursor_after.next_block_number, + outcome.cursor_after.target_block_number, + outcome.cursor_after.last_completed_block_number, + ), + ( + outcome.cursor_before.range_start_block_number, + outcome.cursor_before.next_block_number, + outcome.cursor_before.target_block_number, + outcome.cursor_before.last_completed_block_number, + ), + "the durable cursor must not advance after a retention-generation fence failure" + ); + assert_eq!( + ( + outcome.cursor_after.raw_log_input_revision, + outcome.cursor_after.raw_log_retention_generation, + ), + ( + outcome.cursor_before.raw_log_input_revision, + outcome.cursor_before.raw_log_retention_generation, + ), + "the failed attempt must not acknowledge any part of the changed raw-log input version" ); - let missing_evidence_chain = "missing-evidence"; - insert_completed_replay_cursor_for_handoff_test( + Ok(()) +} + +#[tokio::test] +async fn normalized_replay_recovery_preserves_full_stateless_span_after_preflight_gap() +-> Result<()> { + let database = TestDatabase::new().await?; + create_normalized_replay_cursor_table(database.pool()).await?; + create_ops_catchup_backfill_job_tables(database.pool()).await?; + let chain = "ethereum-sepolia"; + let root_manifest_id = 52_300; + let registry_manifest_id = 52_301; + let resolver_manifest_id = 52_302; + let root_contract_instance_id = Uuid::from_u128(52_300); + let registry_contract_instance_id = Uuid::from_u128(52_301); + let resolver_contract_instance_id = Uuid::from_u128(52_302); + let child_contract_instance_id = Uuid::from_u128(52_303); + let root_address = "0x0000000000000000000000000000000000005230"; + let registry_address = "0x0000000000000000000000000000000000005231"; + let child_address = "0x0000000000000000000000000000000000005232"; + let resolver_address = "0x0000000000000000000000000000000000005233"; + insert_normalized_replay_ens_v2_registry_manifests( database.pool(), - missing_evidence_chain, - target, - 5, - 0, + chain, + root_manifest_id, + registry_manifest_id, + root_contract_instance_id, + registry_contract_instance_id, + root_address, + registry_address, ) .await?; - upsert_raw_staging_input_version_for_handoff_test( + insert_normalized_replay_ens_v2_resolver_manifest( database.pool(), - missing_evidence_chain, - 6, - 0, + chain, + resolver_manifest_id, + resolver_contract_instance_id, + resolver_address, ) .await?; - let missing_evidence_error = normalized_replay_catchup::normalized_replay_cursors_complete( - database.pool(), - "mainnet", - &[missing_evidence_chain.to_owned()], + sqlx::query( + "INSERT INTO discovery_admission_epochs (chain_id, epoch) VALUES ($1, 0) ON CONFLICT (chain_id) DO NOTHING", ) - .await - .expect_err("an advanced revision without block evidence is an integrity error"); - assert!(format!("{missing_evidence_error:#}").contains("without per-block revision evidence")); + .bind(chain) + .execute(database.pool()) + .await?; - let rollback_chain = "revision-rollback"; - insert_completed_replay_cursor_for_handoff_test(database.pool(), rollback_chain, target, 6, 0) - .await?; - upsert_raw_staging_input_version_for_handoff_test(database.pool(), rollback_chain, 5, 0) + let block_1 = provider_block(&format!("0x{:064x}", 52_301), None, 1); + let block_2 = provider_block(&format!("0x{:064x}", 52_302), Some(&block_1.block_hash), 2); + let block_3 = provider_block( + &format!("0x{:064x}", 52_303), + Some(&block_2.block_hash), + 3, + ); + for block in [&block_1, &block_2, &block_3] { + insert_chain_lineage_for_block( + database.pool(), + chain, + block, + CanonicalityState::Finalized, + ) .await?; - let rollback_error = normalized_replay_catchup::normalized_replay_cursors_complete( + } + upsert_raw_blocks( database.pool(), - "mainnet", - &[rollback_chain.to_owned()], + &[ + provider_block_to_raw_block(chain, &block_1, CanonicalityState::Finalized), + provider_block_to_raw_block(chain, &block_2, CanonicalityState::Finalized), + provider_block_to_raw_block(chain, &block_3, CanonicalityState::Finalized), + ], ) - .await - .expect_err("a raw-log revision rollback is an integrity error"); - assert!(format!("{rollback_error:#}").contains("revision moved backwards")); - - database.cleanup().await -} + .await?; + upsert_raw_logs( + database.pool(), + &[ + RawLog { + chain_id: chain.to_owned(), + block_hash: block_1.block_hash.clone(), + block_number: block_1.block_number, + transaction_hash: transaction_hash_for_block(&block_1), + transaction_index: 0, + log_index: 0, + emitting_address: registry_address.to_owned(), + topics: vec![ + ens_v2_label_registered_topic0(), + hex_string(&abi_word_u64(1)), + labelhash_hex("full-span"), + hex_string(&abi_word_address( + "0x0000000000000000000000000000000000000dad", + )), + ], + data: decode_hex_string(&encode_ens_v2_label_registered_log_data( + "full-span", + "0x0000000000000000000000000000000000000aaa", + block_1.block_timestamp_unix_secs + 31_536_000, + )), + canonicality_state: CanonicalityState::Finalized, + }, + RawLog { + chain_id: chain.to_owned(), + block_hash: block_3.block_hash.clone(), + block_number: block_3.block_number, + transaction_hash: transaction_hash_for_block(&block_3), + transaction_index: 0, + log_index: 0, + emitting_address: "0x00000000000000000000000000000000000052ff".to_owned(), + topics: vec![keccak256_hex(b"Unrelated(uint256)")], + data: Vec::new(), + canonicality_state: CanonicalityState::Finalized, + }, + ], + ) + .await?; + sqlx::query( + r#" + UPDATE raw_log_staging_input_revisions + SET retention_generation = 1, + retained_history_complete = false, + incomplete_since = clock_timestamp(), + proven_retention_generation = NULL, + proven_discovery_admission_epoch = NULL, + proven_through_block = NULL + WHERE chain_id = $1 + "#, + ) + .bind(chain) + .execute(database.pool()) + .await?; + insert_completed_backfill_range_coverage_for_source_family( + database.pool(), + chain, + 1, + 3, + "ens_v2_root_l1", + &[root_address], + ) + .await?; + insert_completed_backfill_range_coverage_for_source_family( + database.pool(), + chain, + 1, + 3, + "ens_v2_resolver_l1", + &[resolver_address], + ) + .await?; + insert_completed_backfill_range_coverage_for_source_family( + database.pool(), + chain, + 1, + 3, + "ens_v2_registry_l1", + &[registry_address], + ) + .await?; + sqlx::query( + r#" + UPDATE raw_log_staging_input_revisions + SET retained_history_complete = true, + incomplete_since = NULL, + proven_retention_generation = 1, + proven_discovery_admission_epoch = 0, + proven_through_block = 3 + WHERE chain_id = $1 + AND retention_generation = 1 + "#, + ) + .bind(chain) + .execute(database.pool()) + .await?; + + insert_contract_instance(database.pool(), child_contract_instance_id, chain, "contract") + .await?; + insert_active_contract_instance_address( + database.pool(), + child_contract_instance_id, + chain, + child_address, + Some(registry_manifest_id), + ) + .await?; + sqlx::query( + "UPDATE contract_instance_addresses \ + SET active_from_block_number = 3, active_to_block_number = 3, deactivated_at = now() \ + WHERE contract_instance_id = $1 AND chain_id = $2", + ) + .bind(child_contract_instance_id) + .bind(chain) + .execute(database.pool()) + .await?; + insert_active_discovery_edge_with_range( + database.pool(), + chain, + "subregistry", + registry_contract_instance_id, + child_contract_instance_id, + Some(registry_manifest_id), + Some(3), + Some(3), + ) + .await?; + sqlx::query( + "UPDATE discovery_edges \ + SET deactivated_at = now() \ + WHERE chain_id = $1 AND to_contract_instance_id = $2", + ) + .bind(chain) + .bind(child_contract_instance_id) + .execute(database.pool()) + .await?; + sqlx::query("UPDATE discovery_admission_epochs SET epoch = 1 WHERE chain_id = $1") + .bind(chain) + .execute(database.pool()) + .await?; + + let config = normalized_replay_catchup::NormalizedReplayCatchupConfig::new( + "sepolia".to_owned(), + vec![chain.to_owned()], + 1_000, + 1_000, + 1, + )? + .with_defer_projection_indexes(false); + let stateless_pages = + install_stateless_page_observer(database.pool(), "sepolia", chain).await?; + let error = normalized_replay_catchup::run_normalized_replay_catchup_iteration( + database.pool(), + &config, + chain, + ) + .await + .expect_err("stale retained-history proof must fail before phase one without a provider"); + assert_eq!( + bigname_adapters::ens_v2_missing_coverage(&error), + Some(&bigname_adapters::EnsV2MissingCoverage { + chain: chain.to_owned(), + retention_generation: 1, + source_family: "ens_v2_registry_l1".to_owned(), + address: child_address.to_owned(), + required_from_block: 3, + required_to_block: 3, + }) + ); + assert_eq!( + stateless_pages.page_ranges(), + Vec::<(i64, i64)>::new(), + "preflight coverage validation must fail before any stateless replay page" + ); + + let (provider, server) = bundle_provider_with_fixtures(vec![ProviderBlockFixture { + block: block_3.clone(), + logs: vec![rpc_ens_v2_label_registered_log_payload( + &block_3, + child_address, + 2, + "recovered-preflight", + 2, + )], + }]) + .await?; + assert_eq!( + normalized_replay_catchup::run_normalized_replay_catchup_iteration_with_provider_for_test( + database.pool(), + &config, + chain, + &provider, + HeaderAuditMode::Minimal, + ) + .await?, + normalized_replay_catchup::CatchupIterationStatus::Progressed + ); + assert_eq!( + stateless_pages.page_ranges(), + vec![(1, 3)], + "a preflight recovery must preserve the original full stateless span" + ); + assert_eq!( + sqlx::query_scalar::<_, i64>( + "SELECT COUNT(*)::BIGINT FROM normalized_events \ + WHERE event_kind = 'PreimageObserved' \ + AND derivation_kind = 'raw_log_preimage_observation' \ + AND block_hash = $1 \ + AND log_index = 0" + ) + .bind(&block_1.block_hash) + .fetch_one(database.pool()) + .await?, + 1, + "the original full-span log must still receive its stateless preimage" + ); + assert_eq!( + sqlx::query_scalar::<_, i64>( + "SELECT COUNT(*)::BIGINT FROM normalized_events \ + WHERE event_kind = 'PreimageObserved' \ + AND derivation_kind = 'raw_log_preimage_observation' \ + AND block_hash = $1 \ + AND log_index = 2" + ) + .bind(&block_3.block_hash) + .fetch_one(database.pool()) + .await?, + 1, + "the recovered log must receive its stateless preimage" + ); + + server.abort(); + database.cleanup().await +} + +#[tokio::test] +async fn normalized_replay_retention_authority_keeps_durable_ensv2_resolver_gap_retryable() +-> Result<()> { + let database = TestDatabase::new().await?; + create_ops_catchup_backfill_job_tables(database.pool()).await?; + let chain = "ethereum-sepolia"; + let root_address = "0x0000000000000000000000000000000000005330"; + let registry_address = "0x0000000000000000000000000000000000005331"; + let resolver_address = "0x0000000000000000000000000000000000005332"; + insert_normalized_replay_ens_v2_registry_manifests( + database.pool(), + chain, + 53_300, + 53_301, + Uuid::from_u128(53_300), + Uuid::from_u128(53_301), + root_address, + registry_address, + ) + .await?; + insert_normalized_replay_ens_v2_resolver_manifest( + database.pool(), + chain, + 53_302, + Uuid::from_u128(53_302), + resolver_address, + ) + .await?; + sqlx::query( + "INSERT INTO discovery_admission_epochs (chain_id, epoch) VALUES ($1, 0)", + ) + .bind(chain) + .execute(database.pool()) + .await?; + sqlx::query( + r#" + INSERT INTO raw_log_staging_input_revisions ( + chain_id, revision, retention_generation, + retained_history_complete, incomplete_since, + proven_retention_generation, proven_discovery_admission_epoch, + proven_through_block + ) + VALUES ($1, 0, 1, true, NULL, 1, 0, 3) + "#, + ) + .bind(chain) + .execute(database.pool()) + .await?; + insert_completed_backfill_range_coverage_for_source_family( + database.pool(), + chain, + 1, + 3, + "ens_v2_root_l1", + &[root_address], + ) + .await?; + insert_completed_backfill_range_coverage_for_source_family( + database.pool(), + chain, + 1, + 3, + "ens_v2_registry_l1", + &[registry_address], + ) + .await?; + + let adapters = [ + NormalizedEventReplayAdapter::EnsV2RegistryResourceSurface, + NormalizedEventReplayAdapter::EnsV2Resolver, + ]; + let error = ensure_full_closure_retention_authority_for_adapters( + database.pool(), + chain, + &adapters, + 3, + ) + .await + .expect_err("durable resolver coverage gap must remain provider-retryable"); + assert_eq!( + bigname_adapters::ens_v2_missing_coverage(&error), + Some(&bigname_adapters::EnsV2MissingCoverage { + chain: chain.to_owned(), + retention_generation: 1, + source_family: "ens_v2_resolver_l1".to_owned(), + address: resolver_address.to_owned(), + required_from_block: 1, + required_to_block: 3, + }), + "a restarted catch-up iteration must still recognize the durable resolver gap" + ); + + insert_completed_backfill_range_coverage_for_source_family( + database.pool(), + chain, + 1, + 3, + "ens_v2_resolver_l1", + &[resolver_address], + ) + .await?; + ensure_full_closure_retention_authority_for_adapters( + database.pool(), + chain, + &adapters, + 3, + ) + .await?; + + database.cleanup().await +} + +#[tokio::test] +async fn normalized_replay_handoff_classifies_completed_cursor_input_versions() -> Result<()> { + let database = TestDatabase::new().await?; + create_normalized_replay_cursor_table(database.pool()).await?; + let target = 200; + + let generation_chain = "generation-drift"; + insert_completed_replay_cursor_for_handoff_test( + database.pool(), + generation_chain, + target, + 5, + 0, + ) + .await?; + upsert_raw_staging_input_version_for_handoff_test(database.pool(), generation_chain, 6, 1) + .await?; + assert!( + !normalized_replay_catchup::normalized_replay_cursors_complete( + database.pool(), + "mainnet", + &[generation_chain.to_owned()], + ) + .await?, + "a completed cursor from an older retention generation must not hand off" + ); + + let post_target_chain = "strictly-post-target"; + insert_completed_replay_cursor_for_handoff_test( + database.pool(), + post_target_chain, + target, + 5, + 0, + ) + .await?; + upsert_raw_staging_input_version_for_handoff_test(database.pool(), post_target_chain, 6, 0) + .await?; + upsert_raw_staging_block_revision_for_handoff_test( + database.pool(), + post_target_chain, + "0xpost-target", + target + 1, + 6, + ) + .await?; + assert!( + normalized_replay_catchup::normalized_replay_cursors_complete( + database.pool(), + "mainnet", + &[post_target_chain.to_owned()], + ) + .await?, + "a witnessed revision strictly after the replay target remains backlog-owned" + ); + + let missing_evidence_chain = "missing-evidence"; + insert_completed_replay_cursor_for_handoff_test( + database.pool(), + missing_evidence_chain, + target, + 5, + 0, + ) + .await?; + upsert_raw_staging_input_version_for_handoff_test( + database.pool(), + missing_evidence_chain, + 6, + 0, + ) + .await?; + let missing_evidence_error = normalized_replay_catchup::normalized_replay_cursors_complete( + database.pool(), + "mainnet", + &[missing_evidence_chain.to_owned()], + ) + .await + .expect_err("an advanced revision without block evidence is an integrity error"); + assert!(format!("{missing_evidence_error:#}").contains("without per-block revision evidence")); + + let rollback_chain = "revision-rollback"; + insert_completed_replay_cursor_for_handoff_test(database.pool(), rollback_chain, target, 6, 0) + .await?; + upsert_raw_staging_input_version_for_handoff_test(database.pool(), rollback_chain, 5, 0) + .await?; + let rollback_error = normalized_replay_catchup::normalized_replay_cursors_complete( + database.pool(), + "mainnet", + &[rollback_chain.to_owned()], + ) + .await + .expect_err("a raw-log revision rollback is an integrity error"); + assert!(format!("{rollback_error:#}").contains("revision moved backwards")); + + database.cleanup().await +} #[tokio::test] async fn normalized_replay_handoff_waits_for_rewind_after_completed_prefix_replacement() @@ -1954,29 +2953,128 @@ async fn normalized_replay_catchup_refuses_pending_base_rederive_below_boundary_ completed_at, updated_at ) - VALUES ('base-rederive-reset-pending', 'mainnet', $1, $2, 'completed', now(), now()) + VALUES ('base-rederive-reset-pending', 'mainnet', $1, $2, 'completed', now(), now()) + "#, + ) + .bind(chain) + .bind(target) + .execute(database.pool()) + .await?; + insert_chain_lineage_for_block( + database.pool(), + chain, + &older_block, + CanonicalityState::Canonical, + ) + .await?; + insert_raw_reverse_claimed_log( + database.pool(), + chain, + &older_block, + "0x0000000000000000000000000000000000000141", + "0x0000000000000000000000000000000000000051", + CanonicalityState::Canonical, + ) + .await?; + + let config = normalized_replay_catchup::NormalizedReplayCatchupConfig::new( + "mainnet".to_owned(), + vec![chain.to_owned()], + 1_000, + 1_000, + 1, + )?; + let error = normalized_replay_catchup::run_normalized_replay_catchup_iteration( + database.pool(), + &config, + chain, + ) + .await + .expect_err("pending Base correction replay must not widen below reviewed boundary"); + assert!( + format!("{error:?}").contains("would widen below reviewed boundary"), + "unexpected error: {error:?}" + ); + assert_eq!( + sqlx::query_as::<_, (i64, i64, i64)>( + r#" + SELECT range_start_block_number, next_block_number, target_block_number + FROM normalized_replay_cursors + WHERE deployment_profile = 'mainnet' + AND chain_id = $1 + AND cursor_kind = 'raw_fact_normalized_events' + "#, + ) + .bind(chain) + .fetch_one(database.pool()) + .await?, + (boundary, boundary, target), + "catch-up must bail before ensure_cursor widens the pending correction cursor" + ); + assert_eq!( + normalized_replay_catchup::ensure_cursor_for_test( + database.pool(), + "mainnet", + chain, + boundary - 1, + target, + false, + ) + .await?, + (boundary - 1, boundary - 1, target), + "normal ensure_cursor behavior remains able to widen when invoked directly" + ); + + database.cleanup().await?; + Ok(()) +} + +#[tokio::test] +async fn normalized_replay_catchup_refuses_pending_base_rederive_without_raw_log_bounds() +-> Result<()> { + let database = TestDatabase::new().await?; + create_normalized_replay_cursor_table(database.pool()).await?; + create_base_normalized_rederive_run_table(database.pool()).await?; + let chain = "base-mainnet"; + let boundary = bigname_storage::BASE_NORMALIZED_REDERIVE_REPLAY_START_BLOCK; + let target = boundary + 100; + + sqlx::query( + r#" + INSERT INTO normalized_replay_cursors ( + deployment_profile, + chain_id, + cursor_kind, + range_start_block_number, + next_block_number, + target_block_number + ) + VALUES ('mainnet', $1, 'raw_fact_normalized_events', $2, $2, $3) + "#, + ) + .bind(chain) + .bind(boundary) + .bind(target) + .execute(database.pool()) + .await?; + sqlx::query( + r#" + INSERT INTO base_normalized_rederive_runs ( + run_id, + deployment_profile, + chain_id, + replay_target_block, + status, + completed_at, + updated_at + ) + VALUES ('base-rederive-reset-pending-no-logs', 'mainnet', $1, $2, 'completed', now(), now()) "#, ) .bind(chain) .bind(target) .execute(database.pool()) .await?; - insert_chain_lineage_for_block( - database.pool(), - chain, - &older_block, - CanonicalityState::Canonical, - ) - .await?; - insert_raw_reverse_claimed_log( - database.pool(), - chain, - &older_block, - "0x0000000000000000000000000000000000000141", - "0x0000000000000000000000000000000000000051", - CanonicalityState::Canonical, - ) - .await?; let config = normalized_replay_catchup::NormalizedReplayCatchupConfig::new( "mainnet".to_owned(), @@ -1991,226 +3089,575 @@ async fn normalized_replay_catchup_refuses_pending_base_rederive_below_boundary_ chain, ) .await - .expect_err("pending Base correction replay must not widen below reviewed boundary"); + .expect_err("pending Base correction replay must not idle without retained raw-log bounds"); assert!( - format!("{error:?}").contains("would widen below reviewed boundary"), + format!("{error:?}").contains("no retained canonical raw-log bounds"), "unexpected error: {error:?}" ); + + database.cleanup().await?; + Ok(()) +} + +#[tokio::test] +async fn normalized_replay_catchup_rebuilds_deferred_indexes_when_configured_chain_has_no_logs() +-> Result<()> { + let database = TestDatabase::new().await?; + create_normalized_replay_cursor_table(database.pool()).await?; + sqlx::query("CREATE TABLE invalid_projection_index_fixture (duplicate_value INTEGER NOT NULL)") + .execute(database.pool()) + .await + .context("failed to create invalid-index fixture table")?; + sqlx::query("INSERT INTO invalid_projection_index_fixture (duplicate_value) VALUES (1), (1)") + .execute(database.pool()) + .await + .context("failed to seed invalid-index fixture rows")?; + sqlx::query( + "CREATE UNIQUE INDEX CONCURRENTLY normalized_events_namespace_idx \ + ON invalid_projection_index_fixture (duplicate_value)", + ) + .execute(database.pool()) + .await + .expect_err("duplicate fixture rows must leave an invalid concurrent index remnant"); + let (is_valid, is_ready) = sqlx::query_as::<_, (bool, bool)>( + r#" + SELECT index.indisvalid, index.indisready + FROM pg_index AS index + WHERE index.indexrelid = to_regclass('normalized_events_namespace_idx') + "#, + ) + .fetch_one(database.pool()) + .await + .context("failed to inspect invalid deferred projection index")?; + assert!(!is_valid || !is_ready); + sqlx::query("DROP INDEX IF EXISTS normalized_events_record_inventory_resource_replay_idx") + .execute(database.pool()) + .await + .context("failed to drop record inventory replay index for test")?; + sqlx::query( + r#" + INSERT INTO normalized_replay_cursors ( + deployment_profile, + chain_id, + cursor_kind, + range_start_block_number, + next_block_number, + target_block_number, + last_completed_block_number + ) + VALUES ('mainnet', 'ethereum-mainnet', 'raw_fact_normalized_events', 1, 2, 1, 1) + "#, + ) + .execute(database.pool()) + .await + .context("failed to insert complete normalized replay cursor for test")?; + + let config = normalized_replay_catchup::NormalizedReplayCatchupConfig::new( + "mainnet".to_owned(), + vec!["ethereum-mainnet".to_owned(), "base-mainnet".to_owned()], + 1_000, + 1_000, + 1, + )?; + // TestDatabase is capped at two connections. Retain one exactly as the + // process-lifetime Base rederive writer guard does in production; index + // restoration must run all fenced catalog work on the DDL guard's own + // second connection instead of waiting forever for a third. + let outer_runtime_guard = database + .pool() + .acquire() + .await + .context("failed to retain simulated runtime writer-guard connection")?; assert_eq!( - sqlx::query_as::<_, (i64, i64, i64)>( + normalized_replay_catchup::run_normalized_replay_catchup_iteration( + database.pool(), + &config, + "base-mainnet", + ) + .await?, + normalized_replay_catchup::CatchupIterationStatus::Idle + ); + + assert!( + sqlx::query_scalar::<_, bool>( r#" - SELECT range_start_block_number, next_block_number, target_block_number - FROM normalized_replay_cursors - WHERE deployment_profile = 'mainnet' - AND chain_id = $1 - AND cursor_kind = 'raw_fact_normalized_events' + SELECT index.indisvalid AND index.indisready + FROM pg_index AS index + WHERE index.indexrelid = to_regclass('normalized_events_namespace_idx') + AND index.indrelid = 'normalized_events'::regclass "#, ) - .bind(chain) .fetch_one(database.pool()) .await?, - (boundary, boundary, target), - "catch-up must bail before ensure_cursor widens the pending correction cursor" + "catch-up must replace invalid remnants with a ready index on normalized_events" ); - assert_eq!( - normalized_replay_catchup::ensure_cursor_for_test( - database.pool(), - "mainnet", - chain, - boundary - 1, - target, - false, - ) - .await?, - (boundary - 1, boundary - 1, target), - "normal ensure_cursor behavior remains able to widen when invoked directly" + let record_inventory_predicate = sqlx::query_scalar::<_, String>( + r#" + SELECT pg_get_expr(index.indpred, index.indrelid) + FROM pg_index AS index + JOIN pg_class AS class ON class.oid = index.indexrelid + WHERE class.relname = 'normalized_events_record_inventory_resource_replay_idx' + "#, + ) + .fetch_one(database.pool()) + .await + .context("failed to inspect restored record inventory replay index predicate")?; + assert!( + record_inventory_predicate.contains("ens_v2_registry_resource_surface"), + "restored record inventory replay index must cover ENSv2 registry resources: {record_inventory_predicate}" ); + drop(outer_runtime_guard); database.cleanup().await?; Ok(()) } -#[tokio::test] -async fn normalized_replay_catchup_refuses_pending_base_rederive_without_raw_log_bounds() --> Result<()> { +const NORMALIZED_REPLAY_COVERAGE_FENCE_CHAIN: &str = "ethereum-sepolia"; +const NORMALIZED_REPLAY_COVERAGE_FENCE_HISTORICAL_CHILD: &str = + "0x0000000000000000000000000000000000005400"; + +#[derive(Clone, Copy)] +enum NormalizedReplayCoverageFenceMutation { + RawLogBelowReplaySpan, + RetentionGeneration, +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +struct NormalizedReplayCoverageFenceCursor { + range_start_block_number: i64, + next_block_number: i64, + target_block_number: i64, + last_completed_block_number: Option, + raw_log_input_revision: i64, + raw_log_retention_generation: i64, +} + +struct NormalizedReplayCoverageFenceOutcome { + error: anyhow::Error, + cursor_before: NormalizedReplayCoverageFenceCursor, + cursor_after: NormalizedReplayCoverageFenceCursor, + input_version_before: bigname_storage::RawLogStagingInputVersion, + input_version_after: bigname_storage::RawLogStagingInputVersion, + changed_below_replay_span: bool, +} + +async fn run_normalized_replay_coverage_fence_test( + mutation: NormalizedReplayCoverageFenceMutation, +) -> Result { let database = TestDatabase::new().await?; create_normalized_replay_cursor_table(database.pool()).await?; - create_base_normalized_rederive_run_table(database.pool()).await?; - let chain = "base-mainnet"; - let boundary = bigname_storage::BASE_NORMALIZED_REDERIVE_REPLAY_START_BLOCK; - let target = boundary + 100; + create_ops_catchup_backfill_job_tables(database.pool()).await?; + let chain = NORMALIZED_REPLAY_COVERAGE_FENCE_CHAIN; + let root_manifest_id = 54_300; + let registry_manifest_id = 54_301; + let resolver_manifest_id = 54_302; + let root_contract_instance_id = Uuid::from_u128(54_300); + let registry_contract_instance_id = Uuid::from_u128(54_301); + let resolver_contract_instance_id = Uuid::from_u128(54_302); + let root_address = "0x0000000000000000000000000000000000005430"; + let registry_address = "0x0000000000000000000000000000000000005431"; + let child_address = "0x0000000000000000000000000000000000005432"; + let resolver_address = "0x0000000000000000000000000000000000005433"; + insert_normalized_replay_ens_v2_registry_manifests( + database.pool(), + chain, + root_manifest_id, + registry_manifest_id, + root_contract_instance_id, + registry_contract_instance_id, + root_address, + registry_address, + ) + .await?; + insert_normalized_replay_ens_v2_resolver_manifest( + database.pool(), + chain, + resolver_manifest_id, + resolver_contract_instance_id, + resolver_address, + ) + .await?; + sqlx::query( + "INSERT INTO discovery_admission_epochs (chain_id, epoch) VALUES ($1, 0) ON CONFLICT (chain_id) DO NOTHING", + ) + .bind(chain) + .execute(database.pool()) + .await?; + let block_1 = provider_block(&format!("0x{:064x}", 54_301), None, 1); + let block_2 = provider_block(&format!("0x{:064x}", 54_302), Some(&block_1.block_hash), 2); + let block_3 = provider_block( + &format!("0x{:064x}", 54_303), + Some(&block_2.block_hash), + 3, + ); + for block in [&block_1, &block_2, &block_3] { + insert_chain_lineage_for_block( + database.pool(), + chain, + block, + CanonicalityState::Finalized, + ) + .await?; + } + upsert_raw_blocks( + database.pool(), + &[ + provider_block_to_raw_block(chain, &block_1, CanonicalityState::Finalized), + provider_block_to_raw_block(chain, &block_2, CanonicalityState::Finalized), + provider_block_to_raw_block(chain, &block_3, CanonicalityState::Finalized), + ], + ) + .await?; + upsert_raw_logs( + database.pool(), + &[ + RawLog { + chain_id: chain.to_owned(), + block_hash: block_1.block_hash.clone(), + block_number: block_1.block_number, + transaction_hash: transaction_hash_for_block(&block_1), + transaction_index: 0, + log_index: 0, + emitting_address: registry_address.to_owned(), + topics: vec![ + keccak256_hex(b"SubregistryUpdated(uint256,address,address)"), + hex_string(&abi_word_u64(2)), + hex_string(&abi_word_address( + NORMALIZED_REPLAY_COVERAGE_FENCE_HISTORICAL_CHILD, + )), + hex_string(&abi_word_address( + "0x0000000000000000000000000000000000000dad", + )), + ], + data: Vec::new(), + canonicality_state: CanonicalityState::Finalized, + }, + RawLog { + chain_id: chain.to_owned(), + block_hash: block_3.block_hash.clone(), + block_number: block_3.block_number, + transaction_hash: transaction_hash_for_block(&block_3), + transaction_index: 0, + log_index: 0, + emitting_address: registry_address.to_owned(), + topics: vec![ + ens_v2_label_registered_topic0(), + hex_string(&abi_word_u64(1)), + labelhash_hex("alice"), + hex_string(&abi_word_address( + "0x0000000000000000000000000000000000000dad", + )), + ], + data: decode_hex_string(&encode_ens_v2_label_registered_log_data( + "alice", + "0x0000000000000000000000000000000000000aaa", + block_3.block_timestamp_unix_secs + 31_536_000, + )), + canonicality_state: CanonicalityState::Finalized, + }, + RawLog { + chain_id: chain.to_owned(), + block_hash: block_1.block_hash.clone(), + block_number: block_1.block_number, + transaction_hash: transaction_hash_for_block(&block_1), + transaction_index: 0, + log_index: 1, + emitting_address: registry_address.to_owned(), + topics: vec![ + keccak256_hex(b"SubregistryUpdated(uint256,address,address)"), + hex_string(&abi_word_u64(2)), + hex_string(&abi_word_address( + "0x0000000000000000000000000000000000000000", + )), + hex_string(&abi_word_address( + "0x0000000000000000000000000000000000000dad", + )), + ], + data: Vec::new(), + canonicality_state: CanonicalityState::Finalized, + }, + RawLog { + chain_id: chain.to_owned(), + block_hash: block_3.block_hash.clone(), + block_number: block_3.block_number, + transaction_hash: transaction_hash_for_block(&block_3), + transaction_index: 0, + log_index: 1, + emitting_address: registry_address.to_owned(), + topics: vec![ + keccak256_hex(b"SubregistryUpdated(uint256,address,address)"), + hex_string(&abi_word_u64(1)), + hex_string(&abi_word_address(child_address)), + hex_string(&abi_word_address( + "0x0000000000000000000000000000000000000dad", + )), + ], + data: Vec::new(), + canonicality_state: CanonicalityState::Finalized, + }, + ], + ) + .await?; sqlx::query( r#" - INSERT INTO normalized_replay_cursors ( - deployment_profile, - chain_id, - cursor_kind, - range_start_block_number, - next_block_number, - target_block_number - ) - VALUES ('mainnet', $1, 'raw_fact_normalized_events', $2, $2, $3) + UPDATE raw_log_staging_input_revisions + SET retention_generation = 1, + retained_history_complete = false, + incomplete_since = clock_timestamp(), + proven_retention_generation = NULL, + proven_discovery_admission_epoch = NULL, + proven_through_block = NULL + WHERE chain_id = $1 "#, ) .bind(chain) - .bind(boundary) - .bind(target) .execute(database.pool()) .await?; + insert_completed_backfill_range_coverage_for_source_family( + database.pool(), + chain, + 1, + 3, + "ens_v2_root_l1", + &[root_address], + ) + .await?; + insert_completed_backfill_range_coverage_for_source_family( + database.pool(), + chain, + 1, + 3, + "ens_v2_resolver_l1", + &[resolver_address], + ) + .await?; + insert_completed_backfill_range_coverage_for_source_family( + database.pool(), + chain, + 1, + 3, + "ens_v2_registry_l1", + &[registry_address], + ) + .await?; sqlx::query( r#" - INSERT INTO base_normalized_rederive_runs ( - run_id, - deployment_profile, - chain_id, - replay_target_block, - status, - completed_at, - updated_at - ) - VALUES ('base-rederive-reset-pending-no-logs', 'mainnet', $1, $2, 'completed', now(), now()) + UPDATE raw_log_staging_input_revisions + SET retained_history_complete = true, + incomplete_since = NULL, + proven_retention_generation = 1, + proven_discovery_admission_epoch = 0, + proven_through_block = 3 + WHERE chain_id = $1 + AND retention_generation = 1 "#, ) .bind(chain) - .bind(target) .execute(database.pool()) .await?; let config = normalized_replay_catchup::NormalizedReplayCatchupConfig::new( - "mainnet".to_owned(), + "sepolia".to_owned(), vec![chain.to_owned()], 1_000, 1_000, 1, - )?; - let error = normalized_replay_catchup::run_normalized_replay_catchup_iteration( + )? + .with_defer_projection_indexes(false); + let recovery_hook = normalized_replay_catchup::install_after_coverage_recovery_test_hook( database.pool(), - &config, + "sepolia", chain, ) + .await; + let (provider, server) = bundle_provider_with_fixtures(vec![ + ProviderBlockFixture { + block: block_1.clone(), + logs: vec![rpc_ens_v2_label_registered_log_payload( + &block_1, + NORMALIZED_REPLAY_COVERAGE_FENCE_HISTORICAL_CHILD, + 2, + "historical-recovered", + 3, + )], + }, + ProviderBlockFixture { + block: block_3.clone(), + logs: vec![rpc_ens_v2_label_registered_log_payload( + &block_3, + child_address, + 2, + "recovered", + 2, + )], + }, + ]) + .await?; + let pool = database.pool().clone(); + let task_config = config.clone(); + let replay = tokio::spawn(async move { + normalized_replay_catchup::run_normalized_replay_catchup_iteration_with_provider_for_test( + &pool, + &task_config, + chain, + &provider, + HeaderAuditMode::Minimal, + ) + .await + }); + tokio::time::timeout( + std::time::Duration::from_secs(10), + recovery_hook.wait_until_after_coverage_recovery(), + ) .await - .expect_err("pending Base correction replay must not idle without retained raw-log bounds"); - assert!( - format!("{error:?}").contains("no retained canonical raw-log bounds"), - "unexpected error: {error:?}" - ); + .context("normalized replay did not reach its post-coverage-recovery barrier")?; + + let cursor_before = + load_normalized_replay_coverage_fence_cursor(database.pool(), chain).await?; + let input_version_before = + bigname_storage::load_raw_log_staging_input_version(database.pool(), chain).await?; + let changed_below_replay_span = match mutation { + NormalizedReplayCoverageFenceMutation::RawLogBelowReplaySpan => { + let block_0 = provider_block(&format!("0x{:064x}", 54_300), None, 0); + insert_chain_lineage_for_block( + database.pool(), + chain, + &block_0, + CanonicalityState::Finalized, + ) + .await?; + upsert_raw_blocks( + database.pool(), + &[provider_block_to_raw_block( + chain, + &block_0, + CanonicalityState::Finalized, + )], + ) + .await?; + upsert_raw_logs( + database.pool(), + &[RawLog { + chain_id: chain.to_owned(), + block_hash: block_0.block_hash.clone(), + block_number: block_0.block_number, + transaction_hash: transaction_hash_for_block(&block_0), + transaction_index: 0, + log_index: 9, + emitting_address: registry_address.to_owned(), + topics: vec![keccak256_hex(b"Unrelated(uint256)")], + data: Vec::new(), + canonicality_state: CanonicalityState::Finalized, + }], + ) + .await?; + let revision = sqlx::query_scalar::<_, i64>( + "UPDATE raw_log_staging_input_revisions \ + SET revision = revision + 1 \ + WHERE chain_id = $1 \ + RETURNING revision", + ) + .bind(chain) + .fetch_one(database.pool()) + .await?; + sqlx::query( + r#" + INSERT INTO raw_log_staging_block_revisions ( + chain_id, block_hash, block_number, revision + ) + VALUES ($1, $2, $3, $4) + "#, + ) + .bind(chain) + .bind(&block_0.block_hash) + .bind(block_0.block_number) + .bind(revision) + .execute(database.pool()) + .await?; + bigname_storage::raw_log_staging_block_range_changed_since( + database.pool(), + chain, + input_version_before.revision, + 0, + 0, + ) + .await? + } + NormalizedReplayCoverageFenceMutation::RetentionGeneration => { + sqlx::query( + r#" + UPDATE raw_log_staging_input_revisions + SET retention_generation = retention_generation + 1, + retained_history_complete = false, + incomplete_since = clock_timestamp(), + proven_retention_generation = NULL, + proven_discovery_admission_epoch = NULL, + proven_through_block = NULL + WHERE chain_id = $1 + "#, + ) + .bind(chain) + .execute(database.pool()) + .await?; + false + } + }; + let input_version_after = + bigname_storage::load_raw_log_staging_input_version(database.pool(), chain).await?; + + recovery_hook.resume(); + let error = tokio::time::timeout(std::time::Duration::from_secs(10), replay) + .await + .context("normalized replay did not resume after coverage recovery")? + .context("normalized replay task panicked")? + .expect_err("the injected recovery-fence change must fail the replay attempt"); + let cursor_after = + load_normalized_replay_coverage_fence_cursor(database.pool(), chain).await?; + drop(recovery_hook); + server.abort(); database.cleanup().await?; - Ok(()) + Ok(NormalizedReplayCoverageFenceOutcome { + error, + cursor_before, + cursor_after, + input_version_before, + input_version_after, + changed_below_replay_span, + }) } -#[tokio::test] -async fn normalized_replay_catchup_rebuilds_deferred_indexes_when_configured_chain_has_no_logs() --> Result<()> { - let database = TestDatabase::new().await?; - create_normalized_replay_cursor_table(database.pool()).await?; - sqlx::query("CREATE TABLE invalid_projection_index_fixture (duplicate_value INTEGER NOT NULL)") - .execute(database.pool()) - .await - .context("failed to create invalid-index fixture table")?; - sqlx::query("INSERT INTO invalid_projection_index_fixture (duplicate_value) VALUES (1), (1)") - .execute(database.pool()) - .await - .context("failed to seed invalid-index fixture rows")?; - sqlx::query( - "CREATE UNIQUE INDEX CONCURRENTLY normalized_events_namespace_idx \ - ON invalid_projection_index_fixture (duplicate_value)", - ) - .execute(database.pool()) - .await - .expect_err("duplicate fixture rows must leave an invalid concurrent index remnant"); - let (is_valid, is_ready) = sqlx::query_as::<_, (bool, bool)>( - r#" - SELECT index.indisvalid, index.indisready - FROM pg_index AS index - WHERE index.indexrelid = to_regclass('normalized_events_namespace_idx') - "#, - ) - .fetch_one(database.pool()) - .await - .context("failed to inspect invalid deferred projection index")?; - assert!(!is_valid || !is_ready); - sqlx::query("DROP INDEX IF EXISTS normalized_events_record_inventory_resource_replay_idx") - .execute(database.pool()) - .await - .context("failed to drop record inventory replay index for test")?; - sqlx::query( +async fn load_normalized_replay_coverage_fence_cursor( + pool: &PgPool, + chain: &str, +) -> Result { + let row = sqlx::query_as::<_, (i64, i64, i64, Option, i64, i64)>( r#" - INSERT INTO normalized_replay_cursors ( - deployment_profile, - chain_id, - cursor_kind, + SELECT range_start_block_number, next_block_number, target_block_number, - last_completed_block_number - ) - VALUES ('mainnet', 'ethereum-mainnet', 'raw_fact_normalized_events', 1, 2, 1, 1) - "#, - ) - .execute(database.pool()) - .await - .context("failed to insert complete normalized replay cursor for test")?; - - let config = normalized_replay_catchup::NormalizedReplayCatchupConfig::new( - "mainnet".to_owned(), - vec!["ethereum-mainnet".to_owned(), "base-mainnet".to_owned()], - 1_000, - 1_000, - 1, - )?; - // TestDatabase is capped at two connections. Retain one exactly as the - // process-lifetime Base rederive writer guard does in production; index - // restoration must run all fenced catalog work on the DDL guard's own - // second connection instead of waiting forever for a third. - let outer_runtime_guard = database - .pool() - .acquire() - .await - .context("failed to retain simulated runtime writer-guard connection")?; - assert_eq!( - normalized_replay_catchup::run_normalized_replay_catchup_iteration( - database.pool(), - &config, - "base-mainnet", - ) - .await?, - normalized_replay_catchup::CatchupIterationStatus::Idle - ); - - assert!( - sqlx::query_scalar::<_, bool>( - r#" - SELECT index.indisvalid AND index.indisready - FROM pg_index AS index - WHERE index.indexrelid = to_regclass('normalized_events_namespace_idx') - AND index.indrelid = 'normalized_events'::regclass - "#, - ) - .fetch_one(database.pool()) - .await?, - "catch-up must replace invalid remnants with a ready index on normalized_events" - ); - let record_inventory_predicate = sqlx::query_scalar::<_, String>( - r#" - SELECT pg_get_expr(index.indpred, index.indrelid) - FROM pg_index AS index - JOIN pg_class AS class ON class.oid = index.indexrelid - WHERE class.relname = 'normalized_events_record_inventory_resource_replay_idx' + last_completed_block_number, + raw_log_input_revision, + raw_log_retention_generation + FROM normalized_replay_cursors + WHERE deployment_profile = 'sepolia' + AND chain_id = $1 + AND cursor_kind = 'raw_fact_normalized_events' "#, ) - .fetch_one(database.pool()) - .await - .context("failed to inspect restored record inventory replay index predicate")?; - assert!( - record_inventory_predicate.contains("ens_v2_registry_resource_surface"), - "restored record inventory replay index must cover ENSv2 registry resources: {record_inventory_predicate}" - ); - - drop(outer_runtime_guard); - database.cleanup().await?; - Ok(()) + .bind(chain) + .fetch_one(pool) + .await?; + Ok(NormalizedReplayCoverageFenceCursor { + range_start_block_number: row.0, + next_block_number: row.1, + target_block_number: row.2, + last_completed_block_number: row.3, + raw_log_input_revision: row.4, + raw_log_retention_generation: row.5, + }) } async fn insert_completed_replay_cursor_for_handoff_test( @@ -2370,6 +3817,62 @@ async fn insert_normalized_replay_ens_v2_registry_manifests( .await } +async fn insert_normalized_replay_ens_v2_resolver_manifest( + pool: &PgPool, + chain: &str, + manifest_id: i64, + contract_instance_id: Uuid, + address: &str, +) -> Result<()> { + let manifest_payload = json!({ + "roots": [], + "contracts": [{ + "role": "resolver", + "address": address, + "start_block": 1 + }], + "abi": {"events": test_manifest_abi_events()}, + }); + sqlx::query( + r#" + INSERT INTO manifest_versions ( + manifest_id, + namespace, + source_family, + chain, + rollout_status, + manifest_payload + ) + VALUES ($1, 'ens', 'ens_v2_resolver_l1', $2, 'active', $3::jsonb) + "#, + ) + .bind(manifest_id) + .bind(chain) + .bind(serde_json::to_string(&manifest_payload)?) + .execute(pool) + .await?; + insert_contract_instance(pool, contract_instance_id, chain, "contract").await?; + insert_active_contract_instance_address( + pool, + contract_instance_id, + chain, + address, + Some(manifest_id), + ) + .await?; + insert_manifest_contract_instance( + pool, + manifest_id, + "resolver", + contract_instance_id, + address, + "none", + None, + None, + ) + .await +} + async fn insert_ready_replay_and_backlog_cursors_for_handoff_test( pool: &PgPool, chain: &str, diff --git a/apps/indexer/src/main/tests/replay.rs b/apps/indexer/src/main/tests/replay.rs index ecf2b4a5..bf43fbfa 100644 --- a/apps/indexer/src/main/tests/replay.rs +++ b/apps/indexer/src/main/tests/replay.rs @@ -1329,16 +1329,19 @@ async fn replay_normalized_events_block_hash_replay_fails_closed_for_ens_v2_regi } #[tokio::test] -async fn replay_normalized_events_block_range_runs_ens_v2_closure_over_closed_emitter() -> Result<()> -{ +async fn replay_normalized_events_block_range_rebuilds_ens_v2_proof_for_closed_emitter() +-> Result<()> { let database = TestDatabase::new().await?; let chain = "ethereum-mainnet"; + create_ops_catchup_backfill_job_tables(database.pool()).await?; create_complete_raw_log_staging_input_fixture(database.pool(), chain, 121).await?; let manifest_id = 121; let current_registry_id = Uuid::from_u128(0x1210); let retired_registry_id = Uuid::from_u128(0x1211); + let resolver_id = Uuid::from_u128(0x1212); let current_registry = "0x00000000000000000000000000000000000002a1"; let retired_registry = "0x00000000000000000000000000000000000002a2"; + let resolver = "0x00000000000000000000000000000000000002a3"; let block = provider_block( "0xa1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a121", Some("0x1111111111111111111111111111111111111111111111111111111111111111"), @@ -1362,6 +1365,14 @@ async fn replay_normalized_events_block_range_runs_ens_v2_closure_over_closed_em .bind(test_manifest_payload()) .execute(database.pool()) .await?; + insert_normalized_replay_ens_v2_resolver_manifest( + database.pool(), + chain, + manifest_id + 1, + resolver_id, + resolver, + ) + .await?; insert_contract_instance(database.pool(), retired_registry_id, chain, "contract").await?; insert_active_contract_instance_address( database.pool(), @@ -1425,6 +1436,39 @@ async fn replay_normalized_events_block_range_runs_ens_v2_closure_over_closed_em CanonicalityState::Canonical, ) .await?; + sqlx::query( + r#" + UPDATE raw_log_staging_input_revisions + SET retention_generation = 1, + retained_history_complete = false, + incomplete_since = clock_timestamp(), + proven_retention_generation = NULL, + proven_discovery_admission_epoch = NULL, + proven_through_block = NULL + WHERE chain_id = $1 + "#, + ) + .bind(chain) + .execute(database.pool()) + .await?; + insert_completed_backfill_range_coverage_for_source_family( + database.pool(), + chain, + 0, + block.block_number, + "ens_v2_registry_l1", + &[current_registry, retired_registry], + ) + .await?; + insert_completed_backfill_range_coverage_for_source_family( + database.pool(), + chain, + 0, + block.block_number, + "ens_v2_resolver_l1", + &[resolver], + ) + .await?; let outcome = replay_raw_fact_normalized_events( database.pool(), @@ -1444,6 +1488,23 @@ async fn replay_normalized_events_block_range_runs_ens_v2_closure_over_closed_em outcome.scanned_raw_log_count, 2, "the stateless boundary and ENSv2 full-closure boundary must each see the retained closed-emitter fact" ); + assert_eq!( + sqlx::query_as::<_, (bool, Option, Option)>( + r#" + SELECT + retained_history_complete, + proven_retention_generation, + proven_through_block + FROM raw_log_staging_input_revisions + WHERE chain_id = $1 + "#, + ) + .bind(chain) + .fetch_one(database.pool()) + .await?, + (true, Some(1), Some(block.block_number)), + "manual full-closure replay must rebuild its stale proof from current-generation coverage" + ); database.cleanup().await } @@ -2963,13 +3024,14 @@ async fn insert_active_replay_watched_contract_with_source_family( 'active', 'ensip15@ens-normalize-0.1.1', ('manifests/ens/' || $3 || '/v1.toml'), - DEFAULT + $4 ) "#, ) .bind(manifest_id) .bind(chain) .bind(source_family) + .bind(test_manifest_payload()) .execute(pool) .await .context("failed to insert manifest_versions for replay test")?; diff --git a/crates/adapters/src/block_derived_normalized_events/loading.rs b/crates/adapters/src/block_derived_normalized_events/loading.rs index 26f68061..f50beda8 100644 --- a/crates/adapters/src/block_derived_normalized_events/loading.rs +++ b/crates/adapters/src/block_derived_normalized_events/loading.rs @@ -5,8 +5,11 @@ use anyhow::{Context, Result}; use sqlx::PgPool; use super::event_topics::PreimageObservedEventTopics; -use super::source_selection::{load_active_emitters, normalized_source_scope_targets}; -use super::types::WatchedRawLogRow; +use super::source_selection::{ + RawLogSourceScopeIndex, load_active_emitters, normalized_source_scope_targets, + select_emitter_for_block, +}; +use super::types::{ActiveEmitter, WatchedRawLogRow}; #[derive(Clone, Debug)] pub(super) struct WatchedRawLogLoad { @@ -83,6 +86,10 @@ pub(super) async fn load_watched_raw_logs( event_topics: PreimageObservedEventTopics::default(), }); } + let source_scope_index = source_scope + .as_deref() + .map(RawLogSourceScopeIndex::new) + .transpose()?; let scoped_emitter_identities = source_scope.as_ref().map(|source_scope| { source_scope .iter() @@ -101,11 +108,15 @@ pub(super) async fn load_watched_raw_logs( let event_topics = PreimageObservedEventTopics::load(pool, &active_emitters).await?; let preimage_topic0s = event_topics.query_topic0s(); - let emitters_by_address = active_emitters - .into_iter() - .map(|emitter| (emitter.address.clone(), emitter)) - .collect::>(); - let watched_addresses = emitters_by_address.keys().cloned().collect::>(); + let mut emitters_by_address = HashMap::>::new(); + for emitter in active_emitters { + emitters_by_address + .entry(emitter.address.clone()) + .or_default() + .push(emitter); + } + let mut watched_addresses = emitters_by_address.keys().cloned().collect::>(); + watched_addresses.sort(); let rows = if let Some(source_scope) = &source_scope { let scoped_addresses = source_scope @@ -245,19 +256,31 @@ pub(super) async fn load_watched_raw_logs( .map(|row| { let emitting_address = sql_row::get::(&row, "emitting_address")?; let normalized_emitting_address = emitting_address.to_ascii_lowercase(); - let active_emitter = emitters_by_address + let block_number = sql_row::get::(&row, "block_number")?; + let candidate_emitters = emitters_by_address .get(&normalized_emitting_address) .with_context(|| { format!( - "missing active emitter attribution for chain {} address {}", - chain, emitting_address + "missing block-derived emitter candidates for chain {chain} address {emitting_address}" ) })?; + let active_emitter = select_emitter_for_block( + chain, + &emitting_address, + block_number, + candidate_emitters, + source_scope_index.as_ref(), + )? + .with_context(|| { + format!( + "missing block-derived emitter attribution for chain {chain} address {emitting_address} at block {block_number}" + ) + })?; Ok(WatchedRawLogRow { chain_id: sql_row::get(&row, "chain_id")?, block_hash: sql_row::get(&row, "block_hash")?, - block_number: sql_row::get(&row, "block_number")?, + block_number, transaction_hash: sql_row::get(&row, "transaction_hash")?, transaction_index: sql_row::get(&row, "transaction_index")?, log_index: sql_row::get(&row, "log_index")?, diff --git a/crates/adapters/src/block_derived_normalized_events/source_selection.rs b/crates/adapters/src/block_derived_normalized_events/source_selection.rs index f8a029a1..c2b41d47 100644 --- a/crates/adapters/src/block_derived_normalized_events/source_selection.rs +++ b/crates/adapters/src/block_derived_normalized_events/source_selection.rs @@ -1,7 +1,7 @@ use std::collections::{HashMap, HashSet}; -use anyhow::{Context, Result}; -use bigname_manifests::{load_manifest_declared_watched_contracts, load_watched_contracts}; +use anyhow::{Context, Result, bail}; +use bigname_manifests::{load_historical_watched_contracts_by_chain, load_watched_contracts}; use sqlx::PgPool; use crate::adapter_manifest::{ @@ -29,15 +29,76 @@ pub(super) fn normalized_source_scope_targets( .collect() } +#[derive(Clone, Debug, Default)] +pub(super) struct RawLogSourceScopeIndex { + ranges_by_family_and_address: HashMap>>, +} + +impl RawLogSourceScopeIndex { + pub(super) fn new(targets: &[RawLogSourceScopeTarget]) -> Result { + let mut ranges_by_family_and_address = + HashMap::>>::new(); + for target in targets { + if target.effective_from_block > target.effective_to_block { + bail!( + "block-derived source scope for {}/{} has inverted interval {}..={}", + target.source_family, + target.address, + target.effective_from_block, + target.effective_to_block, + ); + } + ranges_by_family_and_address + .entry(target.source_family.clone()) + .or_default() + .entry(target.address.clone()) + .or_default() + .push((target.effective_from_block, target.effective_to_block)); + } + + for ranges_by_address in ranges_by_family_and_address.values_mut() { + for ranges in ranges_by_address.values_mut() { + ranges.sort_unstable(); + let mut merged = Vec::<(i64, i64)>::with_capacity(ranges.len()); + for (from_block, to_block) in ranges.drain(..) { + match merged.last_mut() { + Some((_, merged_to)) if from_block <= merged_to.saturating_add(1) => { + *merged_to = (*merged_to).max(to_block); + } + _ => merged.push((from_block, to_block)), + } + } + *ranges = merged; + } + } + + Ok(Self { + ranges_by_family_and_address, + }) + } + + pub(super) fn contains(&self, source_family: &str, address: &str, block_number: i64) -> bool { + let Some(ranges) = self + .ranges_by_family_and_address + .get(source_family) + .and_then(|ranges_by_address| ranges_by_address.get(address)) + else { + return false; + }; + let insertion = ranges.partition_point(|(from_block, _)| *from_block <= block_number); + insertion > 0 && ranges[insertion - 1].1 >= block_number + } +} + pub(super) async fn load_active_emitters( pool: &PgPool, chain: &str, scoped_emitter_identities: Option<&HashSet<(String, String)>>, ) -> Result> { let watched_contracts = if scoped_emitter_identities.is_some() { - load_manifest_declared_watched_contracts(pool) + load_historical_watched_contracts_by_chain(pool, chain) .await - .context("failed to load manifest-declared watched contracts for scoped adapter emitter attribution")? + .context("failed to load historical watched contracts for scoped adapter emitter attribution")? } else { load_watched_contracts(pool) .await @@ -60,7 +121,7 @@ pub(super) async fn load_active_emitters( let active_manifests = load_active_manifest_metadata(pool, &manifest_ids, "watched contracts").await?; - let mut emitters_by_address = HashMap::::new(); + let mut emitters = Vec::with_capacity(watched_contracts.len()); for watched_contract in watched_contracts { let (source_manifest_id, manifest) = active_manifest_for_watched_contract(&active_manifests, &watched_contract)?; @@ -74,27 +135,84 @@ pub(super) async fn load_active_emitters( source_family: manifest.source_family.clone(), manifest_version: manifest.manifest_version, source_rank: source_rank(watched_contract.source), + active_from_block_number: watched_contract.active_from_block_number, + active_to_block_number: watched_contract.active_to_block_number, }; - - match emitters_by_address.get(&candidate.address) { - Some(current) if !candidate_precedes(&candidate, current) => {} - _ => { - emitters_by_address.insert(candidate.address.clone(), candidate); - } - } + emitters.push(candidate); } - let mut emitters = emitters_by_address.into_values().collect::>(); emitters.sort_by(|left, right| { left.address .cmp(&right.address) .then(left.source_rank.cmp(&right.source_rank)) .then(left.source_manifest_id.cmp(&right.source_manifest_id)) .then(left.contract_instance_id.cmp(&right.contract_instance_id)) + .then( + left.active_from_block_number + .cmp(&right.active_from_block_number), + ) + .then( + left.active_to_block_number + .cmp(&right.active_to_block_number), + ) }); Ok(emitters) } +pub(super) fn select_emitter_for_block<'a>( + chain: &str, + address: &str, + block_number: i64, + emitters: &'a [ActiveEmitter], + source_scope: Option<&RawLogSourceScopeIndex>, +) -> Result> { + let mut applicable = emitters.iter().filter(|emitter| { + emitter_covers_block(emitter, block_number) + && source_scope.is_none_or(|index| { + index.contains(&emitter.source_family, &emitter.address, block_number) + }) + }); + let Some(mut selected) = applicable.next() else { + return Ok(None); + }; + + for candidate in applicable { + if candidate.source_rank == selected.source_rank + && !same_emitter_attribution(candidate, selected) + { + bail!( + "ambiguous block-derived emitter attribution for {chain} address {address} at block {block_number}: {} manifest_id {} conflicts with {} manifest_id {} at equal source rank {}", + selected.source_family, + selected.source_manifest_id, + candidate.source_family, + candidate.source_manifest_id, + selected.source_rank, + ); + } + if candidate_precedes(candidate, selected) { + selected = candidate; + } + } + + Ok(Some(selected)) +} + +fn emitter_covers_block(emitter: &ActiveEmitter, block_number: i64) -> bool { + emitter + .active_from_block_number + .is_none_or(|from_block| block_number >= from_block) + && emitter + .active_to_block_number + .is_none_or(|to_block| block_number <= to_block) +} + +fn same_emitter_attribution(left: &ActiveEmitter, right: &ActiveEmitter) -> bool { + left.source_manifest_id == right.source_manifest_id + && left.namespace == right.namespace + && left.source_family == right.source_family + && left.manifest_version == right.manifest_version +} + fn candidate_precedes(candidate: &ActiveEmitter, current: &ActiveEmitter) -> bool { ( candidate.source_rank, diff --git a/crates/adapters/src/block_derived_normalized_events/tests.rs b/crates/adapters/src/block_derived_normalized_events/tests.rs index 4c60c178..56cb68e2 100644 --- a/crates/adapters/src/block_derived_normalized_events/tests.rs +++ b/crates/adapters/src/block_derived_normalized_events/tests.rs @@ -1,5 +1,5 @@ use std::{ - collections::HashMap, + collections::{HashMap, HashSet}, str::FromStr, sync::atomic::{AtomicU64, Ordering}, time::{SystemTime, UNIX_EPOCH}, @@ -1816,6 +1816,358 @@ async fn sync_block_derived_normalized_events_replays_scoped_selected_logs_witho database.cleanup().await } +#[tokio::test] +async fn scoped_historical_emitters_retain_disjoint_same_address_families() -> Result<()> { + let _permit = crate::acquire_test_db_permit().await; + let database = TestDatabase::new().await?; + let chain = "ethereum-sepolia"; + let address = "0x00000000000000000000000000000000000000aa"; + + let registry_manifest_id = insert_manifest_version( + database.pool(), + ManifestVersionSeed { + manifest_version: 1, + namespace: "ens", + source_family: SOURCE_FAMILY_ENS_V2_REGISTRY_L1, + chain, + deployment_epoch: "ens_v2_registry", + rollout_status: "active", + normalizer_version: "ensip15@ens-normalize-0.1.1", + file_path: "manifests/ens/ens_v2_registry_l1/v1.toml", + }, + ) + .await?; + let resolver_manifest_id = insert_manifest_version( + database.pool(), + ManifestVersionSeed { + manifest_version: 1, + namespace: "ens", + source_family: SOURCE_FAMILY_ENS_V2_RESOLVER_L1, + chain, + deployment_epoch: "ens_v2_resolver", + rollout_status: "active", + normalizer_version: "ensip15@ens-normalize-0.1.1", + file_path: "manifests/ens/ens_v2_resolver_l1/v1.toml", + }, + ) + .await?; + let registry_contract_instance_id = Uuid::new_v4(); + let resolver_contract_instance_id = Uuid::new_v4(); + for contract_instance_id in [registry_contract_instance_id, resolver_contract_instance_id] { + insert_contract_instance(database.pool(), contract_instance_id, chain, "contract").await?; + } + insert_manifest_contract_instance( + database.pool(), + ManifestContractInstanceSeed { + manifest_id: registry_manifest_id, + declaration_kind: "contract", + declaration_name: "registry", + contract_instance_id: registry_contract_instance_id, + declared_address: address, + role: Some("registry"), + proxy_kind: Some("none"), + implementation_contract_instance_id: None, + declared_implementation_address: None, + }, + ) + .await?; + insert_manifest_contract_instance( + database.pool(), + ManifestContractInstanceSeed { + manifest_id: resolver_manifest_id, + declaration_kind: "contract", + declaration_name: "resolver", + contract_instance_id: resolver_contract_instance_id, + declared_address: address, + role: Some("resolver"), + proxy_kind: Some("none"), + implementation_contract_instance_id: None, + declared_implementation_address: None, + }, + ) + .await?; + insert_contract_instance_address( + database.pool(), + registry_contract_instance_id, + chain, + address, + registry_manifest_id, + ) + .await?; + sqlx::query( + r#" + UPDATE contract_instance_addresses + SET active_from_block_number = 1, + active_to_block_number = 10, + deactivated_at = now() + WHERE contract_instance_id = $1 + "#, + ) + .bind(registry_contract_instance_id) + .execute(database.pool()) + .await?; + insert_contract_instance_address( + database.pool(), + resolver_contract_instance_id, + chain, + address, + resolver_manifest_id, + ) + .await?; + sqlx::query( + r#" + UPDATE contract_instance_addresses + SET active_from_block_number = 20, + active_to_block_number = 30, + deactivated_at = now() + WHERE contract_instance_id = $1 + "#, + ) + .bind(resolver_contract_instance_id) + .execute(database.pool()) + .await?; + + let scoped_identities = HashSet::from([ + ( + SOURCE_FAMILY_ENS_V2_REGISTRY_L1.to_owned(), + address.to_owned(), + ), + ( + SOURCE_FAMILY_ENS_V2_RESOLVER_L1.to_owned(), + address.to_owned(), + ), + ]); + let emitters = + source_selection::load_active_emitters(database.pool(), chain, Some(&scoped_identities)) + .await?; + assert_eq!(emitters.len(), 2); + assert!( + emitters + .iter() + .any(|emitter| emitter.source_family == SOURCE_FAMILY_ENS_V2_REGISTRY_L1) + ); + assert!( + emitters + .iter() + .any(|emitter| emitter.source_family == SOURCE_FAMILY_ENS_V2_RESOLVER_L1) + ); + + let source_scope = source_selection::normalized_source_scope_targets(&[ + ( + SOURCE_FAMILY_ENS_V2_REGISTRY_L1.to_owned(), + address.to_owned(), + 1, + 10, + ), + ( + SOURCE_FAMILY_ENS_V2_RESOLVER_L1.to_owned(), + address.to_owned(), + 20, + 30, + ), + ]); + let source_scope_index = source_selection::RawLogSourceScopeIndex::new(&source_scope)?; + let early = source_selection::select_emitter_for_block( + chain, + address, + 5, + &emitters, + Some(&source_scope_index), + )? + .expect("registry interval should cover block 5"); + assert_eq!(early.source_family, SOURCE_FAMILY_ENS_V2_REGISTRY_L1); + assert!( + source_selection::select_emitter_for_block( + chain, + address, + 15, + &emitters, + Some(&source_scope_index), + )? + .is_none(), + "the inactive gap must not inherit either family" + ); + let late = source_selection::select_emitter_for_block( + chain, + address, + 25, + &emitters, + Some(&source_scope_index), + )? + .expect("resolver interval should cover block 25"); + assert_eq!(late.source_family, SOURCE_FAMILY_ENS_V2_RESOLVER_L1); + + let mut ranked_overlap = emitters.clone(); + let resolver = ranked_overlap + .iter_mut() + .find(|emitter| emitter.source_family == SOURCE_FAMILY_ENS_V2_RESOLVER_L1) + .expect("resolver emitter exists"); + resolver.active_from_block_number = Some(5); + resolver.source_rank += 1; + let overlap_scope = source_selection::normalized_source_scope_targets(&[ + ( + SOURCE_FAMILY_ENS_V2_REGISTRY_L1.to_owned(), + address.to_owned(), + 1, + 10, + ), + ( + SOURCE_FAMILY_ENS_V2_RESOLVER_L1.to_owned(), + address.to_owned(), + 5, + 30, + ), + ]); + let overlap_scope_index = source_selection::RawLogSourceScopeIndex::new(&overlap_scope)?; + let preferred = source_selection::select_emitter_for_block( + chain, + address, + 5, + &ranked_overlap, + Some(&overlap_scope_index), + )? + .expect("overlap should retain the higher-priority emitter"); + assert_eq!(preferred.source_family, SOURCE_FAMILY_ENS_V2_REGISTRY_L1); + + let resolver = ranked_overlap + .iter_mut() + .find(|emitter| emitter.source_family == SOURCE_FAMILY_ENS_V2_RESOLVER_L1) + .expect("resolver emitter exists"); + resolver.source_rank -= 1; + let error = source_selection::select_emitter_for_block( + chain, + address, + 5, + &ranked_overlap, + Some(&overlap_scope_index), + ) + .expect_err("equal-priority overlapping families remain ambiguous"); + assert!( + format!("{error:#}").contains("ambiguous block-derived emitter attribution"), + "unexpected ambiguity error: {error:#}" + ); + + let early_block_hash = "0xaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"; + let late_block_hash = "0xbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb"; + upsert_raw_blocks( + database.pool(), + &[ + RawBlock { + chain_id: chain.to_owned(), + block_hash: early_block_hash.to_owned(), + parent_hash: None, + block_number: 5, + block_timestamp: OffsetDateTime::UNIX_EPOCH, + logs_bloom: None, + transactions_root: None, + receipts_root: None, + state_root: None, + canonicality_state: CanonicalityState::Canonical, + }, + RawBlock { + chain_id: chain.to_owned(), + block_hash: late_block_hash.to_owned(), + parent_hash: None, + block_number: 25, + block_timestamp: OffsetDateTime::UNIX_EPOCH, + logs_bloom: None, + transactions_root: None, + receipts_root: None, + state_root: None, + canonicality_state: CanonicalityState::Canonical, + }, + ], + ) + .await?; + let late_dns_name = dns_encoded_name(&["late", "eth"]); + upsert_raw_logs( + database.pool(), + &[ + RawLog { + chain_id: chain.to_owned(), + block_hash: early_block_hash.to_owned(), + block_number: 5, + transaction_hash: "0xearly".to_owned(), + transaction_index: 0, + log_index: 0, + emitting_address: address.to_owned(), + topics: vec![ + keccak_signature_hex( + "LabelRegistered(uint256,bytes32,string,address,uint64,address)", + ), + hex_string(&abi_word_u64(1)), + keccak256_hex(b"early"), + hex_string(&abi_word_address( + "0x00000000000000000000000000000000000000cc", + )), + ], + data: encode_ens_v2_label_registered_data( + "early", + "0x00000000000000000000000000000000000000bb", + 2_000_000_000, + ), + canonicality_state: CanonicalityState::Canonical, + }, + RawLog { + chain_id: chain.to_owned(), + block_hash: late_block_hash.to_owned(), + block_number: 25, + transaction_hash: "0xlate".to_owned(), + transaction_index: 0, + log_index: 0, + emitting_address: address.to_owned(), + topics: vec![ + keccak_signature_hex("NamedResource(uint256,bytes)"), + hex_string(&abi_word_u64(42)), + ], + data: encode_single_dynamic_bytes(&late_dns_name), + canonicality_state: CanonicalityState::Canonical, + }, + ], + ) + .await?; + + let summary = sync_block_derived_normalized_events( + database.pool(), + chain, + &[early_block_hash.to_owned(), late_block_hash.to_owned()], + Some(&[ + ( + SOURCE_FAMILY_ENS_V2_REGISTRY_L1.to_owned(), + address.to_owned(), + 1, + 10, + ), + ( + SOURCE_FAMILY_ENS_V2_RESOLVER_L1.to_owned(), + address.to_owned(), + 20, + 30, + ), + ]), + ) + .await?; + assert_eq!(summary.scanned_log_count, 2); + assert_eq!(summary.matched_log_count, 2); + assert_eq!(summary.total_synced_count, 2); + + let events = load_normalized_events_by_namespace(database.pool(), "ens").await?; + let early = events + .iter() + .find(|event| event.after_state["decoded_name"] == "early") + .expect("early registry preimage event exists"); + assert_eq!(early.source_manifest_id, Some(registry_manifest_id)); + assert_eq!(early.source_family, SOURCE_FAMILY_ENS_V2_REGISTRY_L1); + let late = events + .iter() + .find(|event| event.after_state["decoded_name"] == "late.eth") + .expect("late resolver preimage event exists"); + assert_eq!(late.source_manifest_id, Some(resolver_manifest_id)); + assert_eq!(late.source_family, SOURCE_FAMILY_ENS_V2_RESOLVER_L1); + + database.cleanup().await +} + #[tokio::test] async fn sync_block_derived_normalized_events_uses_active_manifest_after_reactivation_gap() -> Result<()> { diff --git a/crates/adapters/src/block_derived_normalized_events/types.rs b/crates/adapters/src/block_derived_normalized_events/types.rs index 75e8e5b6..3268b490 100644 --- a/crates/adapters/src/block_derived_normalized_events/types.rs +++ b/crates/adapters/src/block_derived_normalized_events/types.rs @@ -46,6 +46,8 @@ pub(super) struct ActiveEmitter { pub(super) source_family: String, pub(super) manifest_version: i64, pub(super) source_rank: i32, + pub(super) active_from_block_number: Option, + pub(super) active_to_block_number: Option, } #[derive(Clone, Debug, Eq, PartialEq)] diff --git a/crates/adapters/src/ens_v2_registry.rs b/crates/adapters/src/ens_v2_registry.rs index 6cbce658..1a578dbe 100644 --- a/crates/adapters/src/ens_v2_registry.rs +++ b/crates/adapters/src/ens_v2_registry.rs @@ -48,7 +48,8 @@ use types::*; use util::normalize_address; pub use live::{ - record_ens_v2_live_selected_raw_log_coverage, sync_ens_v2_registry_resource_surface_live_poll, + ensure_ens_v2_retained_history_proof_through, record_ens_v2_live_selected_raw_log_coverage, + sync_ens_v2_registry_resource_surface_live_poll, }; pub use recovery::{EnsV2MissingCoverage, ens_v2_missing_coverage, is_ens_v2_missing_coverage}; diff --git a/crates/adapters/src/ens_v2_registry/live.rs b/crates/adapters/src/ens_v2_registry/live.rs index 70a2bc3c..02d648be 100644 --- a/crates/adapters/src/ens_v2_registry/live.rs +++ b/crates/adapters/src/ens_v2_registry/live.rs @@ -109,6 +109,31 @@ pub async fn record_ens_v2_live_selected_raw_log_coverage( } } +/// Rebuild the current ENSv2 retained-history proof from already-durable, +/// generation-bound coverage without running projection reconciliation. +/// Missing coverage remains a typed recovery requirement for the caller. +pub async fn ensure_ens_v2_retained_history_proof_through( + pool: &PgPool, + chain: &str, + through_block: i64, +) -> Result<()> { + if !has_authoritative_ens_v2_closure_through(pool, chain, through_block).await? { + return Ok(()); + } + let guard = FullSourceRawLogHistoryGuard::acquire( + acquire_registry_sync_fence(pool, chain).await?, + chain, + ) + .await?; + match guard.ensure_proof_through(pool, through_block).await { + Ok(_) => guard.release().await, + Err(error) => { + let _ = guard.abort().await; + Err(error) + } + } +} + /// Apply an ordinary ENSv2 live poll from a best-effort process-local /// lifecycle-state cache. Hydration is restricted to the exact retained /// ancestor path of one selected target hash. Cache reuse additionally diff --git a/crates/adapters/src/lib.rs b/crates/adapters/src/lib.rs index c87f88a7..556f378d 100644 --- a/crates/adapters/src/lib.rs +++ b/crates/adapters/src/lib.rs @@ -52,8 +52,9 @@ pub use ens_v2_registrar::{ }; pub use ens_v2_registry::{ EnsV2MissingCoverage, EnsV2RegistryResourceSurfaceSyncSummary, ens_v2_missing_coverage, - is_ens_v2_missing_coverage, record_ens_v2_live_selected_raw_log_coverage, - sync_ens_v2_registry_resource_surface, sync_ens_v2_registry_resource_surface_live_poll, + ensure_ens_v2_retained_history_proof_through, is_ens_v2_missing_coverage, + record_ens_v2_live_selected_raw_log_coverage, sync_ens_v2_registry_resource_surface, + sync_ens_v2_registry_resource_surface_live_poll, sync_ens_v2_registry_resource_surface_through_block, }; pub use ens_v2_resolver::{ diff --git a/docs/deployment.md b/docs/deployment.md index e8d8c8f8..371cc3a3 100644 --- a/docs/deployment.md +++ b/docs/deployment.md @@ -840,6 +840,88 @@ Use `RUST_LOG=info,sqlx::query=error` for these runs; otherwise SQLx slow-query warnings can print huge generated INSERT statements for dense chunks and waste time on logging instead of ingest. +### Single-phase to two-phase normalized replay upgrade + +The `raw_fact_normalized_events` cursor has no phase-version field. Its state at +the image change determines the upgrade procedure: + +- A cursor that completed under a pre-two-phase image has + `next_block_number > target_block_number`. The upgraded catch-up loop reports + `Idle`; it does not run phase 1 retroactively, so any stateless + [label-preimage](glossary.md) omission persists across that cursor's completed + span. For each affected + chain, use the manual stateless replay stopgap over the cursor's exact saved + range (with the normal database environment configured): + + ```sh + bigname-indexer replay normalized-events \ + --deployment-profile \ + --chain \ + --from-block \ + --to-block + ``` + + This manual range replay runs the stateless producers before its full-closure + pass; the completed automatic cursor itself is not reset or treated as proof + that the older image ran phase 1. + + Cursor completion may already have allowed raw-log staging compaction. Before + the manual replay, inspect the chain's retention generation: + + ```sql + SELECT + retention_generation, + retained_history_complete, + incomplete_since, + proven_retention_generation, + proven_discovery_admission_epoch, + proven_through_block + FROM raw_log_staging_input_revisions + WHERE chain_id = ''; + ``` + + A missing row is a hard stop. Generation zero means the staging corpus has + never been destructively rotated. A later generation means compaction has + occurred, and cursor completion is not retained-history authority. The replay + command performs the authoritative source-family coverage/proof check before + phase 1; if it reports incomplete or stale retained history, stop and restore + raw facts before retrying. + + When all required targets remain selectable, restore the saved range with a + provider-backed, raw-only hash-pinned backfill in the current retention + generation. Use a fresh idempotency key containing that generation so an old + completed job cannot be reused: + + ```sh + bigname-indexer backfill \ + --deployment-profile \ + --chain \ + --from-block \ + --to-block \ + --idempotency-key two-phase-upgrade--g \ + --hash-pinned-adapter-sync raw-only + ``` + + Run it with the normal manifest root, database, and validation-provider + configuration, then retry manual normalized-event replay. If retention + validation still names a closed historical discovery interval that the + standalone current watch selector cannot refetch, the supported recovery is + a clean database rebootstrap: apply the checked-in migrations to a new + database, configure the same manifests and historical provider, and let + generation-zero historical bootstrap plus two-phase normalized replay finish + before cutover. Never weaken the retention check or mark the old cursor + pending without restoring its raw facts. +- A cursor still in progress at upgrade has + `next_block_number <= target_block_number`. The two-phase image runs one full + phase-1 pass over the saved range and latched target, then resumes phase 2 from + the existing closure checkpoints. Those checkpoints carry over because both + images use the same deployment profile, chain, cursor kind, range, and target + as their replay checkpoint context. + +Where feasible, deploy the two-phase image before in-flight cursors complete. +This converts the omission into the expected one-time full phase-1 cost and +avoids a later manual replay of already completed spans. + ### Streamed full-closure discovery finalize The ENSv1/Basenames registry full-closure replay finalizes its discovery-edge diff --git a/docs/storage.md b/docs/storage.md index 906f5af5..80f94c2c 100644 --- a/docs/storage.md +++ b/docs/storage.md @@ -400,7 +400,7 @@ Raw-log inserts and semantic updates advance a commit-ordered per-chain [input r ENSv2 full-source reconciliation is allowed only when the stored proof generation and discovery epoch still match and the requested target is not above the proven-through block. Recovery requires gap-free coverage facts from completed backfill jobs that captured the exact current retention generation for every authoritative `ens_v2_root_l1` and `ens_v2_registry_l1` address interval, including closed historical discovery intervals. Retained ENSv2 normalized-event and discovery provenance on `canonical`, `safe`, or `finalized` lineage is checked for a matching readable raw-log witness within that source-and-block boundary. Orphaned events and discovery anchors from losing branches remain audit truth, but they neither require bytes that canonical backfill cannot recreate nor define canonical closure authority. This anti-join is only a consistency check; generation-bound fetch coverage is the absence proof. Recovery reads the watched requirements while holding a shared lock on the concrete discovery-admission epoch row and persists the new proof tuple in the same short transaction. Full-source replay then uses one long transaction for the chain's registry-sync serializer, a chain-scoped raw-log semantic-mutation advisory fence, and an `ACCESS SHARE` table lock that blocks global truncation but permits ordinary row writes. Raw-log insert, semantic-update, and delete triggers take the same advisory key for every affected old or new chain in sorted order before advancing revisions; an `observed_at`-only update takes no semantic fence. Therefore another chain's intake remains writable while same-chain semantic mutation and global truncation wait through raw-log loading, discovery reconciliation, and adapter persistence. The destructive discovery writer compares the expected epoch after acquiring its writer fence before it may deactivate anything. -Before a complete live-path extension, the adapter captures the union of authoritative ENSv2 root and registry address intervals through the new target. At proof advance it reloads that union under the post-sync discovery epoch, requires current-generation backfill coverage for every newly admitted, reopened, or earlier interval portion, and rechecks raw-log witnesses against the complete post-sync union. An unchanged interval extended through the target was part of the complete fetch and needs no second historical job; resolver-only admission does not enter this root/registry closure proof. Missing current-generation coverage for one exact required interval is a distinct typed recovery condition whether the requirement was admitted during the current reconciliation or was already durable when a later pass or process restart began. ENSv2 automatic startup and normalized-event catch-up can therefore backfill the named root, registry, or resolver tuple and retry a bounded fixed point without treating unrelated adapter failures as recoverable. Successful complete extensions advance the proven-through block, so a later process restart can hydrate through the advanced live target without rerunning historical bootstrap. A chain with no authoritative ENSv2 root or registry closure is an ENSv2 no-op and does not create retained-history state. If any generation, epoch, coverage, lineage, or witness check fails, reconciliation fails closed. +Before a complete live-path extension, the adapter captures the union of authoritative ENSv2 root and registry address intervals through the new target. At proof advance it reloads that union under the post-sync discovery epoch, requires current-generation backfill coverage for every newly admitted, reopened, or earlier interval portion, and rechecks raw-log witnesses against the complete post-sync union. An unchanged interval extended through the target was part of the complete fetch and needs no second historical job; resolver-only admission does not enter this root/registry closure proof. Missing current-generation coverage for one exact required interval is a distinct typed recovery condition whether the requirement was admitted during the current reconciliation or was already durable when a later pass or process restart began. ENSv2 automatic startup and normalized-event catch-up can therefore backfill the named root, registry, or resolver tuple and retry a bounded fixed point without treating unrelated adapter failures as recoverable. Before phase 1, normalized-event catch-up rebuilds a stale [retained-history proof](glossary.md) from already-durable current-generation coverage; an uncovered interval remains the same typed recovery condition and is fetched before any stateless work. Recovery at that preflight keeps the original full replay span pending, so phase 1 still covers the complete saved range once validation reaches a fixed point. When validation instead exposes a gap after phase 1 completed, catch-up retains and coalesces every exact recovered block span across further attempts and reruns the stateless producers only over those spans before restarting the full stateful adapter pass. Before it accepts the recovery's newer raw-log input revision, it checks for other committed mutations: another changed block inside the saved replay span widens the next phase 1 to that full span, while a change below the span fails the attempt so the durable cursor can replan. Newly fetched and concurrent in-span logs therefore receive their stateless rows before their revision is acknowledged. The fixed point remains bounded to 32 recovery attempts. Successful complete extensions advance the proven-through block, so a later process restart can hydrate through the advanced live target without rerunning historical bootstrap. A chain with no authoritative ENSv2 root or registry closure is an ENSv2 no-op and does not create retained-history state. If any generation, epoch, coverage, lineage, or witness check fails, reconciliation fails closed. After compaction, `chain_lineage` and any retained compact block-anchor metadata remain the block-hash path for losing-branch repair, and `normalized_events` carry the block identity, source identity, event identity, and provenance needed by projection rebuilds and history reads. If raw-log staging rows are already gone, reorg repair marks normalized events and identity rows orphaned from lineage and updates zero raw-log rows for that range. Historical adapter replay from compacted ranges is an explicit backfill/refetch operation against the configured provider/cache substrate or requires log-audit retention; it is not an implicit API fallback. @@ -865,6 +865,23 @@ Raw-fact normalized-event replay is indexer-owned orchestration over the adapter Whole-range replay is the default. Automatic bootstrap and automatic catch-up share one all-source chain cursor over persisted canonical raw facts in block order — adapter-owned identity histories combine registry, registrar, wrapper, resolver, and reverse-claim signals into one storage write boundary, so independent per-source-family cursors would tear those histories. +For a chain that requires stateful or dependency-ordered replay, automatic catch-up runs two ordered phases while holding the existing deployment-profile/chain replay ownership fence. First it runs the block-derived stateless producers that the following full-history pass does not re-emit, using the same selected canonical raw logs, manifest/source identity, decoder, event identity, provenance, and canonicality rules as live adapter sync. That stateless work uses the configured canonical raw-log candidate cap and whole-block boundaries for physical pages, but the ownership and raw-log input-version fences remain session-wide. It then runs the existing stateful and dependency-ordered pass through the latched target. The all-source cursor is the completion fence for both phases: it advances only after the second phase succeeds and the raw-log input version is accepted. A process exit after the first phase leaves that cursor pending, so restart repeats the idempotent stateless upserts before resuming the second phase; projection workers and live-poll handoff therefore cannot observe an intermediate phase as completed replay. + +That completion marker does not record whether the binary that advanced it ran one +phase or two. A `raw_fact_normalized_events` cursor completed by a pre-two-phase +indexer remains complete after upgrade, so the upgraded loop is idle and does not +backfill stateless label-preimage rows omitted from that completed span. Operators +must use the [manual normalized-event replay stopgap](deployment.md#single-phase-to-two-phase-normalized-replay-upgrade) +for the exact completed range. Because cursor completion may already have +permitted raw-log compaction, that stopgap first requires current-generation +retained-history authority; operators must restore it with provider-backed +backfill or rebuild a clean generation-zero database before replay. A pending +cursor upgraded in place does run the full stateless phase before resuming +stateful closure from the existing adapter checkpoints because both images use +the same replay checkpoint context. Deploying the two-phase image before an +in-progress cursor completes avoids the manual stopgap, at the cost of that one +full stateless pass. + Normalized events are adapter-owned semantic transition rows, not guaranteed-stateless decorations on individual raw logs. Some adapters can derive every emitted row from the selected raw fact alone; those stateless adapters may be replayed from a block-hash selection. Stateful adapters derive `before_state`, resource continuity, authority metadata, resolver state, wrapper state, registrar expiry, and permission provenance from the chronological adapter history. For those adapters, replay that emits or compares transition rows must start from a valid closure boundary and carry adapter state across every physical page in the replay. Where a stateful adapter's replay may start depends on whether raw-log history @@ -912,7 +929,7 @@ If a process exits after a flush but before the matching checkpoint save, restar Full-closure replay chooses physical pages by canonical raw-log event candidate count while preserving whole-block boundaries; adapter routing may then filter that page down to the watched or generic source events that the closure pass consumes. Implementation scan guards may limit one database range probe, but they are throughput guards rather than semantic 512-block replay windows. If a single block exceeds the configured candidate-log cap, the full block is still replayed as one page; the cap is not allowed to split a block or create a replay cursor. When a scan guard is reached before the candidate-log cap, the page may advance through empty or low-density whole blocks because no semantic boundary is created until the closure target completes. -The global `raw_fact_normalized_events` cursor advances only after the closure adapter finalizes its adapter-owned writes through the requested target block. Automatic full-closure catch-up latches that requested target when the cursor is created and does not widen the same closure pass just because newer live raw facts arrive while it is running or after it completes; a later closure target requires an explicit cursor rewind/reset or a documented adapter-state snapshot boundary. +The global `raw_fact_normalized_events` cursor advances only after the stateless phase and every stateful or dependency-ordered adapter finalize their adapter-owned writes through the requested target block. Automatic full-closure catch-up latches that requested target when the cursor is created and does not widen the same closure pass just because newer live raw facts arrive while it is running or after it completes; a later closure target requires an explicit cursor rewind/reset or a documented adapter-state snapshot boundary. A completed automatic catch-up may be followed by a separate `post_replay_live_adapter_backlog` operational cursor that live-normalizes canonical raw-log blocks already persisted after the latched replay target before normal live polling resumes adapter sync. That cursor scopes adapter routing from the selected raw-log emitters, is not a closure replay cursor, does not change the full-closure target, and remains replay-safe because it uses the same deterministic adapter-owned upsert path as live polling. Backlog normalization never replaces provider-backed live intake; the following live reconciliation still admits raw payloads for canonical blocks that were not already persisted. diff --git a/tests/e2e/README.md b/tests/e2e/README.md index b769fd0c..fbbfbd1b 100644 --- a/tests/e2e/README.md +++ b/tests/e2e/README.md @@ -586,13 +586,10 @@ route inventory or a claim that every protocol transition is covered. - `catchup_equivalence::automatic_catchup_matches_live_ingestion_outputs` — compares a finalized ENSv1 registration, addr/text records, and registry-only child ingested from the same chain by live polling and forced automatic - catch-up. Route snapshots and all common normalized-event rows are exact; - the temporary #157 containment requires the live-only delta to be exactly one - finalized registrar `PreimageObserved` for `catchupeq.eth`, with no catch-up- - only or other live-only rows. The scenario's contract constant switches to - full row equality when two-phase automatic replay adds the omitted stateless - label-preimage pass. The result—only the #157-class divergence, with no - #161-style incremental-versus-replay under-derivation observed—is certified + catch-up. Route snapshots and all normalized-event rows are exact. The full + equality arm also requires the receipt-reconstructed finalized registrar + `PreimageObserved` for `catchupeq.eth` to exist identically in both corpora, + so equality cannot pass if both paths omit the event. The result is certified for the current corpus only: one finalized ENSv1 `.eth` registration with addr/text records and a registry-only child. Wrapper events, reverse/primary claims, renewals, expiry/grace, ENSv2, Basenames/multi-chain, non-finalized diff --git a/tests/e2e/src/harness/perturb.rs b/tests/e2e/src/harness/perturb.rs index 3a700bb7..3eb9222b 100644 --- a/tests/e2e/src/harness/perturb.rs +++ b/tests/e2e/src/harness/perturb.rs @@ -92,17 +92,22 @@ pub fn assert_snapshots_equal(expected: &RouteSnapshots, actual: &RouteSnapshots /// Automatic catch-up parity is full /// [normalized-event](../../../../docs/glossary.md) row equality after /// normalizing per-corpus [contract-instance](../../../../docs/glossary.md) -/// ids. The temporary #157 contract permits only the exact expected finalized, -/// raw-log-derived [label-preimage](../../../../docs/glossary.md) events to be -/// live-only; catch-up-only or any other live-only row fails. +/// ids. The receipt-reconstructed [label-preimage](../../../../docs/glossary.md) +/// events authenticate both paths under the full contract. The temporary #157 +/// containment contract permits only those exact events to be live-only; +/// catch-up-only or any other live-only row fails. pub async fn assert_catchup_normalized_event_parity( live: &sqlx::PgPool, catchup: &sqlx::PgPool, contract: CatchupEquivalenceContract, - expected_missing_preimages: &[StatelessLabelPreimage], + expected_preimages: &[StatelessLabelPreimage], ) -> Result<()> { let live_rows = normalized_event_rows(live, None).await?; let catchup_rows = normalized_event_rows(catchup, None).await?; + ensure!( + !expected_preimages.is_empty(), + "catch-up equivalence must name at least one receipt-reconstructed stateless label-preimage event" + ); if contract == CatchupEquivalenceContract::Full { if live_rows != catchup_rows { bail!( @@ -112,13 +117,19 @@ pub async fn assert_catchup_normalized_event_parity( first_line_diff(&live_rows.join("\n"), &catchup_rows.join("\n")) ); } + assert_expected_stateless_label_preimages( + &live_rows, + expected_preimages, + "live ingestion", + )?; + assert_expected_stateless_label_preimages( + &catchup_rows, + expected_preimages, + "automatic catch-up", + )?; return Ok(()); } - ensure!( - !expected_missing_preimages.is_empty(), - "the contained catch-up preimage divergence must name at least one expected event" - ); let catchup_only = multiset_difference(&catchup_rows, &live_rows); if !catchup_only.is_empty() { bail!( @@ -134,7 +145,7 @@ pub async fn assert_catchup_normalized_event_parity( .map(|row| stateless_label_preimage(row)) .collect::>>()?; observed_missing_preimages.sort(); - let mut expected_missing_preimages = expected_missing_preimages.to_vec(); + let mut expected_missing_preimages = expected_preimages.to_vec(); expected_missing_preimages.sort(); if observed_missing_preimages != expected_missing_preimages { bail!( @@ -144,6 +155,26 @@ pub async fn assert_catchup_normalized_event_parity( Ok(()) } +fn assert_expected_stateless_label_preimages( + rows: &[String], + expected_preimages: &[StatelessLabelPreimage], + corpus: &str, +) -> Result<()> { + let expected_rows = expected_preimages + .iter() + .map(|preimage| preimage.0.clone()) + .collect::>(); + let missing = multiset_difference(&expected_rows, rows); + if !missing.is_empty() { + bail!( + "{corpus} omitted {} receipt-reconstructed stateless label-preimage event(s):\n{}", + missing.len(), + missing.join("\n") + ); + } + Ok(()) +} + fn stateless_label_preimage(row: &str) -> Result { let event: Value = serde_json::from_str(row)?; ensure!( diff --git a/tests/e2e/src/scenarios/catchup_equivalence.rs b/tests/e2e/src/scenarios/catchup_equivalence.rs index d2ff15f2..bbd871aa 100644 --- a/tests/e2e/src/scenarios/catchup_equivalence.rs +++ b/tests/e2e/src/scenarios/catchup_equivalence.rs @@ -17,8 +17,8 @@ const YEAR: u64 = 365 * 24 * 60 * 60; // Anvil's finalized anchor trails the head by 64 blocks, so 66 is load-bearing: // fixture events must finalize for `live_ready_sql`, and the post-handoff cold-start // safe/finalized anchor must land above the last fixture event so live adapter sync -// never touches fixture blocks. The roughly two-block headroom prevents -// `PreimageObserved` from appearing in the catch-up corpus as an inscrutable parity flake. +// never touches fixture blocks. The roughly two-block headroom prevents post-handoff +// live sync from masking a replay omission in the catch-up corpus. const FINALITY_MARGIN_BLOCKS: u64 = 66; const REGISTRAR_MANIFEST_VERSION: i64 = 1; const REGISTRAR_SOURCE_MANIFEST_ID: i64 = 1; @@ -26,10 +26,8 @@ const REGISTRAR_SOURCE_MANIFEST_ID: i64 = 1; const NAME_REGISTERED_EVENT_SIGNATURE: &str = "NameRegistered(string,bytes32,address,uint256,uint256,uint256,bytes32)"; -// #157 seam: change this one line to `Full` when automatic catch-up runs the stateless -// [label-preimage](../../../../docs/glossary.md) pass. const CATCHUP_EQUIVALENCE_CONTRACT: perturb::CatchupEquivalenceContract = - perturb::CatchupEquivalenceContract::MissingStatelessLabelPreimages; + perturb::CatchupEquivalenceContract::Full; struct CatchupChain { deployment: ens_v1::EnsV1Deployment, @@ -47,7 +45,7 @@ struct PreparedCorpus { struct CatchupFixture { last_event_block: u64, - missing_preimage: perturb::StatelessLabelPreimage, + expected_preimage: perturb::StatelessLabelPreimage, } impl CatchupChain { @@ -196,7 +194,7 @@ async fn add_rich_name_fixture(anvil: &Anvil, chain: &CatchupChain) -> Result Result Result<()> { &live.db.pool, &catchup.db.pool, CATCHUP_EQUIVALENCE_CONTRACT, - &[fixture.missing_preimage], + &[fixture.expected_preimage], ) .await?;