From b17aba0ee4a924d0c4e85354dea0d9ec11382dcd Mon Sep 17 00:00:00 2001 From: TateB Date: Thu, 23 Jul 2026 01:49:37 +0000 Subject: [PATCH 01/12] fix: complete heartbeat coverage audit Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01C9RTnbCyhqEUSPc7xCK76p --- Cargo.lock | 1 + apps/indexer/src/main/backfill.rs | 7 +- .../src/main/backfill/concurrent_execution.rs | 23 +- .../src/main/backfill/coverage_facts.rs | 633 +++++++----------- .../src/main/backfill/coverage_facts/tests.rs | 319 +++++++++ .../main/backfill/reservation_execution.rs | 311 ++------- .../reservation_execution/coinbase_sql.rs | 4 +- .../reservation_execution/creation.rs | 239 +++++++ .../backfill/reservation_execution/digest.rs | 76 +++ .../generic_topic_identity.rs | 112 +++- .../reservation_execution/identity.rs | 117 ++++ .../reservation_execution/progress.rs | 46 ++ apps/indexer/src/main/bootstrap_backfill.rs | 564 +--------------- .../main/bootstrap_backfill/checkpoints.rs | 10 - .../src/main/bootstrap_backfill/identity.rs | 16 +- .../src/main/bootstrap_backfill/logging.rs | 38 ++ .../main/bootstrap_backfill/orchestration.rs | 591 ++++++++++++++++ .../src/main/bootstrap_backfill/planning.rs | 7 + .../bootstrap_backfill/planning/progress.rs | 213 ++++++ .../src/main/bootstrap_backfill/progress.rs | 225 +++++++ .../src/main/bootstrap_backfill/recovery.rs | 211 +++++- .../src/main/normalized_replay_catchup.rs | 169 ++--- .../coverage_recovery.rs | 5 +- .../main/normalized_replay_catchup/cursors.rs | 2 + .../normalized_replay_catchup/execution.rs | 107 +++ .../normalized_replay_catchup/test_hook.rs | 119 ++++ apps/indexer/src/main/reconciliation.rs | 21 +- .../src/main/reconciliation/adapter_sync.rs | 193 ++---- .../reconciliation/adapter_sync/backlog.rs | 82 ++- .../adapter_sync/block_scoped.rs | 91 +++ .../adapter_sync/ens_v1_subregistry.rs | 142 +++- .../adapter_sync/ens_v2_registry.rs | 58 +- .../adapter_sync/ens_v2_tail.rs | 173 +++++ .../adapter_sync/entrypoints.rs | 47 +- .../adapter_sync/full_closure.rs | 215 ++++-- .../adapter_sync/full_closure/automatic.rs | 15 +- .../adapter_sync/full_closure/heartbeat.rs | 54 ++ .../full_closure/reverse_claim.rs | 34 +- .../reconciliation/adapter_sync/progress.rs | 31 + .../reconciliation/adapter_sync/stateless.rs | 141 +++- .../src/main/reconciliation/canonical.rs | 133 ++-- .../reconciliation/canonical/checkpoints.rs | 13 + .../canonical/contiguous_gap.rs | 44 +- .../canonical/ens_v2_coverage_recovery.rs | 127 ++-- .../reconciliation/canonical/orphaning.rs | 216 +++++- .../src/main/reconciliation/canonical/poll.rs | 126 +++- .../main/reconciliation/canonical/progress.rs | 156 +++++ .../canonical/stored_lineage.rs | 19 + .../canonical/stored_lineage/coverage.rs | 268 ++++---- .../stored_lineage/coverage/verification.rs | 127 ++++ .../src/main/reconciliation/persistence.rs | 324 +++++++-- .../reconciliation/persistence/code_hashes.rs | 224 +++++-- .../persistence/code_hashes/config.rs | 26 + .../reconciliation/persistence/raw_blocks.rs | 89 +++ .../indexer/src/main/reconciliation/replay.rs | 27 +- .../src/main/resolver_profile_convergence.rs | 9 +- .../resolver_profile_convergence/authority.rs | 5 +- .../authority/journal.rs | 117 +++- .../resolver_profile_convergence/drain.rs | 122 +++- .../invalidations.rs | 216 +----- .../invalidations/capture.rs | 431 ++++++++++++ .../resolver_profile_convergence/tests.rs | 108 ++- apps/indexer/src/main/run.rs | 110 ++- apps/indexer/src/main/run/subtask_tests.rs | 206 +++++- apps/indexer/src/main/runtime.rs | 23 +- apps/indexer/src/main/runtime/adapter_sync.rs | 30 +- apps/indexer/src/main/runtime/intake.rs | 139 +++- apps/indexer/src/main/runtime/manifest.rs | 152 ++++- apps/indexer/src/main/runtime/poll_loop.rs | 194 ++++-- .../runtime/poll_loop/discovery_refresh.rs | 82 ++- .../main/runtime/poll_loop/replay_handoff.rs | 93 ++- apps/indexer/src/main/runtime/refresh.rs | 148 +++- apps/indexer/src/main/startup_heartbeat.rs | 98 ++- apps/indexer/src/main/tests/adapter_sync.rs | 66 +- apps/indexer/src/main/tests/backfill.rs | 49 +- apps/indexer/src/main/tests/cli.rs | 2 +- .../main/tests/normalized_replay_catchup.rs | 265 ++++++++ apps/indexer/src/main/tests/reconcile.rs | 49 +- apps/indexer/src/main/tests/reorg.rs | 23 +- apps/indexer/src/main/tests/replay.rs | 1 + .../indexer/src/main/tests/runtime_refresh.rs | 37 +- apps/indexer/src/main/tests/support.rs | 173 ++++- apps/worker/src/address_names.rs | 5 +- apps/worker/src/address_names/rebuild.rs | 38 +- .../worker/src/automatic_projection_replay.rs | 41 +- .../invalidation_derive_loop.rs | 209 +++++- .../primary_hydration_loop.rs | 15 +- apps/worker/src/children.rs | 135 ++-- apps/worker/src/name_current.rs | 24 +- apps/worker/src/permissions.rs | 5 +- apps/worker/src/primary_name/projection.rs | 5 +- apps/worker/src/projection_apply.rs | 17 +- apps/worker/src/projection_apply/apply.rs | 398 ++++++----- .../src/projection_apply/apply/completion.rs | 130 ++++ .../src/projection_apply/apply/tests.rs | 155 +++++ apps/worker/src/rebuild_heartbeat.rs | 57 +- .../worker/src/record_inventory/projection.rs | 10 +- apps/worker/src/resolver.rs | 14 +- apps/worker/src/resolver/summary_json.rs | 13 + .../src/block_derived_normalized_events.rs | 141 +++- crates/adapters/src/checkpoint_context.rs | 9 + crates/adapters/src/ens_v1_reverse_claim.rs | 111 ++- .../src/ens_v1_subregistry_discovery.rs | 102 ++- .../application.rs | 34 + .../ens_v1_subregistry_discovery/emitter.rs | 8 +- .../entrypoints.rs | 100 +++ .../loader/active_emitters.rs | 50 +- .../loader/active_emitters/rows.rs | 167 ++++- .../loader/raw_logs.rs | 7 +- .../reconciliation.rs | 129 +++- .../ens_v1_subregistry_discovery/replay.rs | 19 + .../src/ens_v1_unwrapped_authority.rs | 1 + .../ens_v1_unwrapped_authority/checkpoint.rs | 77 +-- .../checkpoint/items.rs | 203 ++++++ .../checkpoint/startup_events.rs | 40 +- .../src/ens_v1_unwrapped_authority/loading.rs | 2 +- .../loading/active_emitters.rs | 192 ++---- .../loading/active_emitters/manifest.rs | 142 ++++ .../loading/blocks.rs | 63 ++ .../ens_v1_unwrapped_authority/pipeline.rs | 92 ++- .../pipeline/entrypoints.rs | 60 ++ .../pipeline/flush.rs | 37 +- .../pipeline/profile_stream.rs | 2 + .../resolver_profile_reconciliation.rs | 156 ++++- .../targets.rs | 129 +++- .../reverse_claims.rs | 174 +++-- crates/adapters/src/ens_v2_common.rs | 146 ++-- crates/adapters/src/ens_v2_permissions.rs | 57 +- .../adapters/src/ens_v2_permissions/load.rs | 101 ++- crates/adapters/src/ens_v2_registrar.rs | 57 +- .../src/ens_v2_registrar/active_emitters.rs | 67 +- crates/adapters/src/ens_v2_registry.rs | 250 +++---- .../adapters/src/ens_v2_registry/emitters.rs | 94 ++- .../src/ens_v2_registry/entrypoints.rs | 168 +++++ crates/adapters/src/ens_v2_registry/live.rs | 291 ++++---- .../src/ens_v2_registry/live/completeness.rs | 487 ++------------ .../live/completeness/finish.rs | 172 ++++- .../live/completeness/guard.rs | 591 ++++++++++++++++ .../live/completeness/requirements.rs | 254 +++---- .../completeness/requirements/coverage.rs | 67 ++ .../completeness/requirements/witnesses.rs | 297 ++++++++ .../src/ens_v2_registry/live/reuse.rs | 102 +++ crates/adapters/src/ens_v2_resolver.rs | 49 +- .../adapters/src/ens_v2_resolver/queries.rs | 7 +- crates/adapters/src/lib.rs | 25 +- .../src/manifest_normalized_events.rs | 95 ++- .../manifest_normalized_events/builders.rs | 21 +- .../drift_alerts.rs | 39 +- .../src/manifest_normalized_events/loading.rs | 34 +- .../src/manifest_normalized_events/tests.rs | 25 +- .../adapters/src/normalized_event_support.rs | 21 + crates/adapters/src/startup_progress.rs | 19 + crates/manifests/Cargo.toml | 1 + crates/manifests/src/lib.rs | 2 +- crates/manifests/src/lib/discovery.rs | 7 +- crates/manifests/src/lib/discovery/loading.rs | 201 ++++-- .../src/lib/discovery/loading/progress.rs | 204 ++++++ .../src/lib/discovery/reconciliation.rs | 5 +- .../src/lib/discovery/reconciliation/bulk.rs | 4 +- .../reconciliation/bulk/historical.rs | 24 +- .../lib/discovery/reconciliation/streamed.rs | 400 ++++++----- .../discovery/reconciliation/streamed/diff.rs | 322 +++++---- .../reconciliation/streamed/diff/retention.rs | 11 +- .../reconciliation/streamed/guard.rs | 77 +++ .../reconciliation/streamed/options.rs | 35 + .../reconciliation/streamed/progress.rs | 79 +++ .../reconciliation/streamed/staging.rs | 56 +- .../reconciliation/streamed/walk_pages.rs | 24 +- crates/manifests/src/lib/managed_edges.rs | 3 +- .../src/lib/managed_edges/active_addresses.rs | 410 +++++++----- .../src/lib/managed_edges/source_graph.rs | 129 +++- crates/manifests/src/lib/model/watched.rs | 2 +- crates/manifests/src/lib/sync.rs | 42 +- crates/manifests/src/lib/tests.rs | 62 +- crates/manifests/src/lib/views/drift.rs | 190 ++++-- .../src/lib/views/drift/code_hashes.rs | 55 +- crates/manifests/src/lib/views/types.rs | 2 +- crates/manifests/src/lib/views/watched.rs | 179 +++-- .../src/lib/views/watched/coverage.rs | 576 +++++++--------- .../lib/views/watched/coverage/uncovered.rs | 337 ++++++++++ .../src/lib/views/watched/frontier.rs | 538 +++++++++------ .../views/watched/frontier/source_pages.rs | 73 ++ .../src/lib/views/watched/frontier/tests.rs | 197 ++++++ .../src/lib/views/watched/historical.rs | 371 +++++++--- .../src/lib/views/watched/intervals.rs | 45 +- .../src/lib/views/watched/selection.rs | 164 ++++- .../lib/views/watched/selection/progress.rs | 259 +++++++ crates/storage/src/backfill_jobs.rs | 4 +- crates/storage/src/backfill_jobs/complete.rs | 123 +++- .../src/backfill_jobs/coverage_facts.rs | 81 ++- crates/storage/src/execution.rs | 2 + crates/storage/src/execution/invalidation.rs | 89 ++- crates/storage/src/execution/tests.rs | 28 +- crates/storage/src/lib.rs | 46 +- crates/storage/src/long_operation_exports.rs | 25 + .../src/resolver_profile_authority_journal.rs | 1 + .../advance.rs | 93 ++- .../advance/diff.rs | 324 ++++++--- .../advance/mutations.rs | 3 + .../tests.rs | 24 +- crates/storage/src/stored_lineage_coverage.rs | 193 ++++-- .../src/stored_lineage_coverage/integrity.rs | 181 ++++- docs/deployment.md | 6 +- docs/storage.md | 76 +++ scripts/rust-file-size-baseline.toml | 2 +- 205 files changed, 17917 insertions(+), 5853 deletions(-) create mode 100644 apps/indexer/src/main/backfill/coverage_facts/tests.rs create mode 100644 apps/indexer/src/main/backfill/reservation_execution/creation.rs create mode 100644 apps/indexer/src/main/backfill/reservation_execution/identity.rs create mode 100644 apps/indexer/src/main/backfill/reservation_execution/progress.rs create mode 100644 apps/indexer/src/main/bootstrap_backfill/logging.rs create mode 100644 apps/indexer/src/main/bootstrap_backfill/orchestration.rs create mode 100644 apps/indexer/src/main/bootstrap_backfill/planning/progress.rs create mode 100644 apps/indexer/src/main/bootstrap_backfill/progress.rs create mode 100644 apps/indexer/src/main/normalized_replay_catchup/execution.rs create mode 100644 apps/indexer/src/main/reconciliation/adapter_sync/block_scoped.rs create mode 100644 apps/indexer/src/main/reconciliation/adapter_sync/ens_v2_tail.rs create mode 100644 apps/indexer/src/main/reconciliation/adapter_sync/full_closure/heartbeat.rs create mode 100644 apps/indexer/src/main/reconciliation/adapter_sync/progress.rs create mode 100644 apps/indexer/src/main/reconciliation/canonical/progress.rs create mode 100644 apps/indexer/src/main/reconciliation/canonical/stored_lineage/coverage/verification.rs create mode 100644 apps/indexer/src/main/reconciliation/persistence/code_hashes/config.rs create mode 100644 apps/indexer/src/main/reconciliation/persistence/raw_blocks.rs create mode 100644 apps/indexer/src/main/resolver_profile_convergence/invalidations/capture.rs create mode 100644 apps/worker/src/projection_apply/apply/completion.rs create mode 100644 crates/adapters/src/ens_v1_subregistry_discovery/application.rs create mode 100644 crates/adapters/src/ens_v1_unwrapped_authority/loading/active_emitters/manifest.rs create mode 100644 crates/adapters/src/ens_v2_registry/entrypoints.rs create mode 100644 crates/adapters/src/ens_v2_registry/live/completeness/guard.rs create mode 100644 crates/adapters/src/ens_v2_registry/live/completeness/requirements/witnesses.rs create mode 100644 crates/adapters/src/ens_v2_registry/live/reuse.rs create mode 100644 crates/manifests/src/lib/discovery/loading/progress.rs create mode 100644 crates/manifests/src/lib/discovery/reconciliation/streamed/guard.rs create mode 100644 crates/manifests/src/lib/discovery/reconciliation/streamed/options.rs create mode 100644 crates/manifests/src/lib/discovery/reconciliation/streamed/progress.rs create mode 100644 crates/manifests/src/lib/views/watched/coverage/uncovered.rs create mode 100644 crates/manifests/src/lib/views/watched/frontier/source_pages.rs create mode 100644 crates/manifests/src/lib/views/watched/frontier/tests.rs create mode 100644 crates/manifests/src/lib/views/watched/selection/progress.rs create mode 100644 crates/storage/src/long_operation_exports.rs diff --git a/Cargo.lock b/Cargo.lock index c7457e69..23aa2c61 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1774,6 +1774,7 @@ dependencies = [ "bigname-domain", "bigname-storage", "bigname-test-support", + "futures-util", "serde", "serde_json", "sqlx", diff --git a/apps/indexer/src/main/backfill.rs b/apps/indexer/src/main/backfill.rs index 2dc011e9..eac74000 100644 --- a/apps/indexer/src/main/backfill.rs +++ b/apps/indexer/src/main/backfill.rs @@ -50,9 +50,10 @@ pub(crate) use reservation_execution::coinbase_sql_backfill_job_source_identity_ pub(crate) use reservation_execution::effective_hash_pinned_adapter_sync_mode; pub(crate) use reservation_execution::{ DEFAULT_HASH_PINNED_BACKFILL_CHUNK_BLOCKS, backfill_job_source_identity_payload, - create_hash_pinned_backfill_job, effective_coinbase_sql_adapter_sync_mode, - hash_pinned_backfill_range_specs, run_precreated_hash_pinned_backfill_job, - run_resumable_coinbase_sql_backfill_job, run_resumable_hash_pinned_backfill_job, + backfill_job_source_identity_payload_with_progress, create_hash_pinned_backfill_job, + effective_coinbase_sql_adapter_sync_mode, hash_pinned_backfill_range_specs, + run_precreated_hash_pinned_backfill_job, run_resumable_coinbase_sql_backfill_job, + run_resumable_hash_pinned_backfill_job, run_resumable_hash_pinned_backfill_job_with_progress, }; #[cfg(test)] pub(crate) use selection::SelectedTargetIntervalIndex; diff --git a/apps/indexer/src/main/backfill/concurrent_execution.rs b/apps/indexer/src/main/backfill/concurrent_execution.rs index fd2b9bec..5e228cc9 100644 --- a/apps/indexer/src/main/backfill/concurrent_execution.rs +++ b/apps/indexer/src/main/backfill/concurrent_execution.rs @@ -15,7 +15,9 @@ use super::{ coinbase_sql::load_backfill_topic_plan, reservation_execution::{ backfill_lease_duration_secs, create_coinbase_sql_backfill_job_with_ranges, - create_hash_pinned_backfill_job_with_ranges, effective_coinbase_sql_adapter_sync_mode, + create_hash_pinned_backfill_job_with_ranges, + create_hash_pinned_backfill_job_with_ranges_with_progress, + effective_coinbase_sql_adapter_sync_mode, ensure_coinbase_sql_registry_range_start_is_replay_safe, refreshed_backfill_lease_expires_at, run_reserved_coinbase_sql_backfill_range, run_reserved_hash_pinned_backfill_range_with_progress, validate_hash_pinned_chunk_blocks, @@ -84,8 +86,23 @@ async fn run_resumable_hash_pinned_backfill_job_concurrently_inner( ); validate_hash_pinned_chunk_blocks(config.hash_pinned_chunk_blocks)?; let watched_chain = &source_plan.watched_chain_plan; - let record = - create_hash_pinned_backfill_job_with_ranges(pool, source_plan, &config, ranges).await?; + let record = match heartbeat.as_mut() { + Some((heartbeat, chain_ids)) => { + let mut progress = + crate::run::startup_heartbeat::StartupAdapterHeartbeat::new(heartbeat, chain_ids); + create_hash_pinned_backfill_job_with_ranges_with_progress( + pool, + source_plan, + &config, + ranges, + &mut progress, + ) + .await? + } + None => { + create_hash_pinned_backfill_job_with_ranges(pool, source_plan, &config, ranges).await? + } + }; config .idempotency_key .clone_from(&record.job.idempotency_key); diff --git a/apps/indexer/src/main/backfill/coverage_facts.rs b/apps/indexer/src/main/backfill/coverage_facts.rs index 8c8208bc..f1e4e970 100644 --- a/apps/indexer/src/main/backfill/coverage_facts.rs +++ b/apps/indexer/src/main/backfill/coverage_facts.rs @@ -1,11 +1,13 @@ -use std::collections::{BTreeMap, BTreeSet}; +use std::collections::{BTreeMap, BTreeSet, btree_map, btree_set}; use anyhow::Result; +use bigname_adapters::StartupAdapterProgress; use bigname_domain::block_interval::{InclusiveBlockInterval, coalesce_inclusive_block_intervals}; -use bigname_manifests::WatchedSourceSelectorPlan; +use bigname_manifests::{WatchedBackfillTarget, WatchedSourceSelectorPlan}; use bigname_storage::{ - BackfillCoverageFactScope, BackfillCoverageFactWrite, BackfillRange, - complete_backfill_range_recording_coverage, + BackfillCoverageFactScope, BackfillCoverageFactStreamItem, BackfillCoverageFactWrite, + BackfillCoverageProgress, BackfillCoverageProgressFuture, BackfillRange, + complete_backfill_range_recording_coverage_with_progress, }; use crate::{ @@ -19,11 +21,33 @@ use super::{ }; const SOURCE_FAMILY_BASENAMES_BASE_REGISTRY: &str = "basenames_base_registry"; +const COVERAGE_PLAN_PROGRESS_TARGETS: usize = 1_000; + +struct BackfillCoverageHeartbeat<'a, 'b> { + pool: &'a sqlx::PgPool, + progress_sender: Option<&'a tokio::sync::mpsc::UnboundedSender<()>>, + service_progress: &'a mut Option<&'b mut dyn StartupAdapterProgress>, +} + +impl BackfillCoverageProgress for BackfillCoverageHeartbeat<'_, '_> { + fn record<'a>(&'a mut self) -> BackfillCoverageProgressFuture<'a> { + Box::pin(async move { + if let Some(progress_sender) = self.progress_sender { + let _ = progress_sender.send(()); + } + if let Some(progress) = self.service_progress.as_deref_mut() { + progress.record(self.pool).await?; + } + Ok(()) + }) + } +} /// Complete a reserved range, recording plan-derived coverage facts in the /// job-flip transaction when this range completion also completes the job. /// Completion failures are persisted as reserved-range failure state, matching /// the other reserved-range phases. +#[expect(clippy::too_many_arguments)] pub(super) async fn complete_reserved_range_recording_plan_coverage( pool: &sqlx::PgPool, active_range: &BackfillRange, @@ -31,21 +55,31 @@ pub(super) async fn complete_reserved_range_recording_plan_coverage( source_plan: &WatchedSourceSelectorPlan, uses_basenames_registry_scan_all: bool, failure_reason: &'static str, + progress_sender: Option<&tokio::sync::mpsc::UnboundedSender<()>>, + service_progress: &mut Option<&mut dyn StartupAdapterProgress>, ) -> Result<()> { - let completion = complete_backfill_range_recording_coverage( - pool, - active_range.backfill_range_id, - &config.lease_token, - |job| { - job_completion_coverage_facts( - source_plan, - uses_basenames_registry_scan_all, - job.range_start_block_number, - job.range_end_block_number, - ) - }, - ) - .await; + let completion = { + let mut progress = BackfillCoverageHeartbeat { + pool, + progress_sender, + service_progress, + }; + complete_backfill_range_recording_coverage_with_progress( + pool, + active_range.backfill_range_id, + &config.lease_token, + |job| { + job_completion_coverage_fact_stream( + source_plan, + uses_basenames_registry_scan_all, + job.range_start_block_number, + job.range_end_block_number, + ) + }, + &mut progress, + ) + .await + }; if let Err(error) = completion { return Err(record_reserved_range_failure(ReservedRangeFailure { pool, @@ -73,12 +107,31 @@ pub(super) async fn complete_reserved_range_recording_plan_coverage( /// per-address targets are excluded because the fetch was not /// address-enumerated. Every other selected target yields an address-scope /// fact clamped to the job range, skipping empty intersections. +#[cfg(test)] fn job_completion_coverage_facts<'a>( source_plan: &'a WatchedSourceSelectorPlan, uses_basenames_registry_scan_all: bool, job_start_block: i64, job_end_block: i64, ) -> impl Iterator + 'a { + job_completion_coverage_fact_stream( + source_plan, + uses_basenames_registry_scan_all, + job_start_block, + job_end_block, + ) + .filter_map(|item| match item { + BackfillCoverageFactStreamItem::Fact(fact) => Some(fact), + BackfillCoverageFactStreamItem::Progress => None, + }) +} + +fn job_completion_coverage_fact_stream( + source_plan: &WatchedSourceSelectorPlan, + uses_basenames_registry_scan_all: bool, + job_start_block: i64, + job_end_block: i64, +) -> JobCompletionCoverageFactStream<'_> { let mut family_scan_source_families = BTreeSet::new(); if watched_source_plan_uses_generic_resolver_scope(source_plan) { family_scan_source_families.insert(SOURCE_FAMILY_ENS_V1_RESOLVER_L1); @@ -87,54 +140,188 @@ fn job_completion_coverage_facts<'a>( family_scan_source_families.insert(SOURCE_FAMILY_BASENAMES_BASE_REGISTRY); } - let mut family_scan_windows = BTreeMap::<&'a str, Vec<(i64, i64)>>::new(); - for target in &source_plan.selected_targets { - if !family_scan_source_families.contains(target.source_family.as_str()) { - continue; - } - family_scan_windows - .entry(target.source_family.as_str()) - .or_default() - .push((target.effective_from_block, target.effective_to_block)); + JobCompletionCoverageFactStream { + all_targets: &source_plan.selected_targets, + targets: source_plan.selected_targets.iter(), + address_targets: None, + family_scan_source_families, + family_scan_windows: BTreeMap::new(), + family_sources: None, + family_merge: None, + pending_fact: None, + source_examined: 0, + address_examined: 0, + job_start_block, + job_end_block, } - let family_facts = family_scan_windows - .into_iter() - .flat_map(move |(source_family, windows)| { - merged_covered_block_segments(windows, job_start_block, job_end_block) - .into_iter() - .map( - move |(covered_from_block, covered_to_block)| BackfillCoverageFactWrite { - source_family: source_family.to_owned(), - scope: BackfillCoverageFactScope::Family, - address: None, - covered_from_block, - covered_to_block, - }, - ) - }); - let address_facts = source_plan - .selected_targets - .iter() - .filter_map(move |target| { - if family_scan_source_families.contains(target.source_family.as_str()) { - return None; +} + +struct FamilyWindowMerge { + source_family: String, + windows: btree_set::IntoIter<(i64, i64)>, + current: Option<(i64, i64)>, + examined: usize, +} + +struct JobCompletionCoverageFactStream<'a> { + all_targets: &'a [WatchedBackfillTarget], + targets: std::slice::Iter<'a, WatchedBackfillTarget>, + address_targets: Option>, + family_scan_source_families: BTreeSet<&'static str>, + family_scan_windows: BTreeMap>, + family_sources: Option>>, + family_merge: Option, + pending_fact: Option, + source_examined: usize, + address_examined: usize, + job_start_block: i64, + job_end_block: i64, +} + +impl Iterator for JobCompletionCoverageFactStream<'_> { + type Item = BackfillCoverageFactStreamItem; + + fn next(&mut self) -> Option { + if let Some(fact) = self.pending_fact.take() { + return Some(BackfillCoverageFactStreamItem::Fact(fact)); + } + + if let Some(address_targets) = self.address_targets.as_mut() { + loop { + let target = address_targets.next()?; + self.address_examined += 1; + let fact = if self + .family_scan_source_families + .contains(target.source_family.as_str()) + { + None + } else { + covered_block_interval( + target.effective_from_block, + target.effective_to_block, + self.job_start_block, + self.job_end_block, + ) + .map(|(covered_from_block, covered_to_block)| { + BackfillCoverageFactWrite { + source_family: target.source_family.clone(), + scope: BackfillCoverageFactScope::Address, + address: Some(target.address.to_ascii_lowercase()), + covered_from_block, + covered_to_block, + } + }) + }; + if self + .address_examined + .is_multiple_of(COVERAGE_PLAN_PROGRESS_TARGETS) + { + self.pending_fact = fact; + return Some(BackfillCoverageFactStreamItem::Progress); + } + if let Some(fact) = fact { + return Some(BackfillCoverageFactStreamItem::Fact(fact)); + } } - let (covered_from_block, covered_to_block) = covered_block_interval( + } + + for target in self.targets.by_ref() { + self.source_examined += 1; + let interval = covered_block_interval( target.effective_from_block, target.effective_to_block, - job_start_block, - job_end_block, - )?; - Some(BackfillCoverageFactWrite { - source_family: target.source_family.clone(), - scope: BackfillCoverageFactScope::Address, - address: Some(target.address.to_ascii_lowercase()), - covered_from_block, - covered_to_block, - }) - }); + self.job_start_block, + self.job_end_block, + ); + if self + .family_scan_source_families + .contains(target.source_family.as_str()) + && let Some(interval) = interval + { + self.family_scan_windows + .entry(target.source_family.clone()) + .or_default() + .insert(interval); + } + if self + .source_examined + .is_multiple_of(COVERAGE_PLAN_PROGRESS_TARGETS) + { + return Some(BackfillCoverageFactStreamItem::Progress); + } + } - family_facts.chain(address_facts) + if self.family_sources.is_none() { + self.family_sources = Some(std::mem::take(&mut self.family_scan_windows).into_iter()); + } + loop { + if self.family_merge.is_none() { + let Some((source_family, windows)) = self.family_sources.as_mut()?.next() else { + self.address_targets = Some(self.all_targets.iter()); + return self.next(); + }; + self.family_merge = Some(FamilyWindowMerge { + source_family, + windows: windows.into_iter(), + current: None, + examined: 0, + }); + } + let merge = self + .family_merge + .as_mut() + .expect("family merge initialized"); + match merge.windows.next() { + Some((from_block, to_block)) => { + merge.examined += 1; + let completed = match merge.current.replace((from_block, to_block)) { + Some((current_from, current_to)) + if from_block <= current_to.saturating_add(1) => + { + merge.current = Some((current_from, current_to.max(to_block))); + None + } + previous => previous, + }; + let fact = completed.map(|(covered_from_block, covered_to_block)| { + BackfillCoverageFactWrite { + source_family: merge.source_family.clone(), + scope: BackfillCoverageFactScope::Family, + address: None, + covered_from_block, + covered_to_block, + } + }); + if merge + .examined + .is_multiple_of(COVERAGE_PLAN_PROGRESS_TARGETS) + { + self.pending_fact = fact; + return Some(BackfillCoverageFactStreamItem::Progress); + } + if let Some(fact) = fact { + return Some(BackfillCoverageFactStreamItem::Fact(fact)); + } + } + None => { + let final_interval = merge.current.take(); + let source_family = merge.source_family.clone(); + self.family_merge = None; + if let Some((covered_from_block, covered_to_block)) = final_interval { + return Some(BackfillCoverageFactStreamItem::Fact( + BackfillCoverageFactWrite { + source_family, + scope: BackfillCoverageFactScope::Family, + address: None, + covered_from_block, + covered_to_block, + }, + )); + } + } + } + } + } } /// Intersect a target's effective window with the job range; `None` when the @@ -174,325 +361,5 @@ pub(crate) fn merged_covered_block_segments( } #[cfg(test)] -mod tests { - use bigname_manifests::{ - WatchedBackfillTarget, WatchedChainPlan, WatchedSourceSelectorKind, - WatchedSourceSelectorPlan, - }; - use bigname_storage::BackfillCoverageFactScope; - use sqlx::types::Uuid; - - use super::*; - - #[test] - fn covered_block_interval_clamps_and_rejects_empty_intersections() { - assert_eq!(covered_block_interval(10, 20, 12, 18), Some((12, 18))); - assert_eq!(covered_block_interval(12, 18, 10, 20), Some((12, 18))); - assert_eq!(covered_block_interval(10, 20, 15, 30), Some((15, 20))); - assert_eq!(covered_block_interval(15, 30, 10, 20), Some((15, 20))); - assert_eq!(covered_block_interval(10, 10, 10, 10), Some((10, 10))); - assert_eq!(covered_block_interval(10, 20, 20, 25), Some((20, 20))); - assert_eq!(covered_block_interval(10, 20, 21, 25), None); - assert_eq!(covered_block_interval(21, 25, 10, 20), None); - assert_eq!(covered_block_interval(0, 0, 1, 1), None); - } - - #[test] - fn job_completion_facts_clamp_targets_and_skip_out_of_range_targets() { - let source_plan = source_plan( - "base-mainnet", - vec![ - target( - "basenames_base_registry", - 1, - "0xABCDEFabcdefABCDEFabcdefabcdefABCDEFabcd", - 10, - 30, - ), - target( - "basenames_base_registrar", - 2, - "0x2222222222222222222222222222222222222222", - 25, - 60, - ), - target( - "basenames_base_resolver", - 3, - "0x3333333333333333333333333333333333333333", - 1, - 19, - ), - ], - ); - - let facts = job_completion_coverage_facts(&source_plan, false, 20, 40).collect::>(); - - assert_eq!(facts.len(), 2); - assert_eq!(facts[0].source_family, "basenames_base_registry"); - assert_eq!(facts[0].scope, BackfillCoverageFactScope::Address); - assert_eq!( - facts[0].address.as_deref(), - Some("0xabcdefabcdefabcdefabcdefabcdefabcdefabcd") - ); - assert_eq!( - (facts[0].covered_from_block, facts[0].covered_to_block), - (20, 30) - ); - assert_eq!(facts[1].source_family, "basenames_base_registrar"); - assert_eq!( - (facts[1].covered_from_block, facts[1].covered_to_block), - (25, 40) - ); - } - - #[test] - fn generic_resolver_scan_yields_family_fact_and_excludes_resolver_targets() { - let source_plan = source_plan( - "ethereum-mainnet", - vec![ - target( - "ens_v1_registry_l1", - 1, - "0x1111111111111111111111111111111111111111", - 5, - 50, - ), - target( - "ens_v1_resolver_l1", - 2, - "0x2222222222222222222222222222222222222222", - 12, - 18, - ), - ], - ); - - let facts = job_completion_coverage_facts(&source_plan, false, 10, 20).collect::>(); - - assert_eq!(facts.len(), 2); - assert_eq!(facts[0].source_family, "ens_v1_resolver_l1"); - assert_eq!(facts[0].scope, BackfillCoverageFactScope::Family); - assert_eq!(facts[0].address, None); - assert_eq!( - (facts[0].covered_from_block, facts[0].covered_to_block), - (12, 18), - "family fact must be clamped to the resolver targets' effective span" - ); - assert_eq!(facts[1].source_family, "ens_v1_registry_l1"); - assert_eq!(facts[1].scope, BackfillCoverageFactScope::Address); - assert_eq!( - (facts[1].covered_from_block, facts[1].covered_to_block), - (10, 20) - ); - } - - #[test] - fn family_scans_without_matching_targets_or_overlap_yield_no_family_fact() { - let mut no_resolver_targets = source_plan( - "ethereum-mainnet", - vec![target( - "ens_v1_registry_l1", - 1, - "0x1111111111111111111111111111111111111111", - 5, - 50, - )], - ); - no_resolver_targets.selector_kind = WatchedSourceSelectorKind::SourceFamily; - no_resolver_targets.source_family = Some("ens_v1_resolver_l1".to_owned()); - assert!( - job_completion_coverage_facts(&no_resolver_targets, false, 10, 20) - .all(|fact| fact.scope != BackfillCoverageFactScope::Family), - "a family scan with no selected targets must not claim family coverage" - ); - - let disjoint_span = source_plan( - "base-mainnet", - vec![target( - "basenames_base_registry", - 1, - "0x1111111111111111111111111111111111111111", - 30, - 40, - )], - ); - assert_eq!( - job_completion_coverage_facts(&disjoint_span, true, 10, 20).count(), - 0, - "a family span disjoint from the job range must not claim coverage" - ); - } - - #[test] - fn family_targets_individually_disjoint_from_the_job_range_claim_nothing() { - // Both windows miss the job range, but their hull [5, 40] spans it: - // hull-then-intersect would mint a family fact over blocks that were - // never fetched. - let source_plan = source_plan( - "base-mainnet", - vec![ - target( - "basenames_base_registry", - 1, - "0x1111111111111111111111111111111111111111", - 5, - 8, - ), - target( - "basenames_base_registry", - 2, - "0x2222222222222222222222222222222222222222", - 30, - 40, - ), - ], - ); - - assert_eq!( - job_completion_coverage_facts(&source_plan, true, 10, 20).count(), - 0, - "windows individually disjoint from the job range must not claim family coverage" - ); - } - - #[test] - fn family_facts_split_on_interior_gaps_between_target_windows() { - let source_plan = source_plan( - "base-mainnet", - vec![ - target( - "basenames_base_registry", - 1, - "0x1111111111111111111111111111111111111111", - 10, - 12, - ), - target( - "basenames_base_registry", - 2, - "0x2222222222222222222222222222222222222222", - 16, - 20, - ), - target( - "basenames_base_registry", - 3, - "0x3333333333333333333333333333333333333333", - 13, - 13, - ), - ], - ); - - let facts = job_completion_coverage_facts(&source_plan, true, 5, 25).collect::>(); - - assert_eq!( - facts - .iter() - .map(|fact| (fact.scope, (fact.covered_from_block, fact.covered_to_block))) - .collect::>(), - vec![ - (BackfillCoverageFactScope::Family, (10, 13)), - (BackfillCoverageFactScope::Family, (16, 20)), - ], - "adjacent windows merge but the unfetched gap between segments must stay unclaimed" - ); - } - - #[test] - fn merged_covered_block_segments_clamp_before_merging() { - assert_eq!( - merged_covered_block_segments(vec![(5, 8), (30, 40)], 10, 20), - Vec::<(i64, i64)>::new() - ); - assert_eq!( - merged_covered_block_segments(vec![(16, 30), (1, 12), (13, 14)], 10, 20), - vec![(10, 14), (16, 20)] - ); - assert_eq!( - merged_covered_block_segments(vec![(1, 100)], 10, 20), - vec![(10, 20)] - ); - assert_eq!( - merged_covered_block_segments(std::iter::empty(), 10, 20), - Vec::<(i64, i64)>::new() - ); - let max = i64::MAX; - assert_eq!( - merged_covered_block_segments([(max, max), (max - 1, max - 1)], max - 1, max), - vec![(max - 1, max)] - ); - } - - #[test] - fn basenames_registry_scan_all_yields_only_the_registry_family_fact() { - let source_plan = source_plan( - "base-mainnet", - vec![ - target( - "basenames_base_registry", - 1, - "0x1111111111111111111111111111111111111111", - 12, - 18, - ), - target( - "basenames_base_registry", - 2, - "0x2222222222222222222222222222222222222222", - 14, - 30, - ), - ], - ); - - let facts = job_completion_coverage_facts(&source_plan, true, 10, 20).collect::>(); - - assert_eq!(facts.len(), 1); - assert_eq!(facts[0].source_family, "basenames_base_registry"); - assert_eq!(facts[0].scope, BackfillCoverageFactScope::Family); - assert_eq!(facts[0].address, None); - assert_eq!( - (facts[0].covered_from_block, facts[0].covered_to_block), - (12, 20), - "family fact must span the union of registry target windows clamped to the job range" - ); - } - - fn target( - source_family: &str, - id: u128, - address: &str, - effective_from_block: i64, - effective_to_block: i64, - ) -> WatchedBackfillTarget { - WatchedBackfillTarget { - source_family: source_family.to_owned(), - contract_instance_id: Uuid::from_u128(id), - address: address.to_owned(), - effective_from_block, - effective_to_block, - } - } - - fn source_plan( - chain: &str, - selected_targets: Vec, - ) -> WatchedSourceSelectorPlan { - WatchedSourceSelectorPlan { - chain: chain.to_owned(), - selector_kind: WatchedSourceSelectorKind::WatchedTargetSet, - source_family: None, - requested_watched_targets: Vec::new(), - selected_targets, - watched_chain_plan: WatchedChainPlan { - chain: chain.to_owned(), - addresses: Vec::new(), - manifest_root_entry_count: 0, - manifest_contract_entry_count: 0, - discovery_edge_entry_count: 0, - }, - } - } -} +#[path = "coverage_facts/tests.rs"] +mod tests; diff --git a/apps/indexer/src/main/backfill/coverage_facts/tests.rs b/apps/indexer/src/main/backfill/coverage_facts/tests.rs new file mode 100644 index 00000000..34c812bf --- /dev/null +++ b/apps/indexer/src/main/backfill/coverage_facts/tests.rs @@ -0,0 +1,319 @@ +use bigname_manifests::{ + WatchedBackfillTarget, WatchedChainPlan, WatchedSourceSelectorKind, WatchedSourceSelectorPlan, +}; +use bigname_storage::BackfillCoverageFactScope; +use sqlx::types::Uuid; + +use super::*; + +#[test] +fn covered_block_interval_clamps_and_rejects_empty_intersections() { + assert_eq!(covered_block_interval(10, 20, 12, 18), Some((12, 18))); + assert_eq!(covered_block_interval(12, 18, 10, 20), Some((12, 18))); + assert_eq!(covered_block_interval(10, 20, 15, 30), Some((15, 20))); + assert_eq!(covered_block_interval(15, 30, 10, 20), Some((15, 20))); + assert_eq!(covered_block_interval(10, 10, 10, 10), Some((10, 10))); + assert_eq!(covered_block_interval(10, 20, 20, 25), Some((20, 20))); + assert_eq!(covered_block_interval(10, 20, 21, 25), None); + assert_eq!(covered_block_interval(21, 25, 10, 20), None); + assert_eq!(covered_block_interval(0, 0, 1, 1), None); +} + +#[test] +fn job_completion_facts_clamp_targets_and_skip_out_of_range_targets() { + let source_plan = source_plan( + "base-mainnet", + vec![ + target( + "basenames_base_registry", + 1, + "0xABCDEFabcdefABCDEFabcdefabcdefABCDEFabcd", + 10, + 30, + ), + target( + "basenames_base_registrar", + 2, + "0x2222222222222222222222222222222222222222", + 25, + 60, + ), + target( + "basenames_base_resolver", + 3, + "0x3333333333333333333333333333333333333333", + 1, + 19, + ), + ], + ); + + let facts = job_completion_coverage_facts(&source_plan, false, 20, 40).collect::>(); + + assert_eq!(facts.len(), 2); + assert_eq!(facts[0].source_family, "basenames_base_registry"); + assert_eq!(facts[0].scope, BackfillCoverageFactScope::Address); + assert_eq!( + facts[0].address.as_deref(), + Some("0xabcdefabcdefabcdefabcdefabcdefabcdefabcd") + ); + assert_eq!( + (facts[0].covered_from_block, facts[0].covered_to_block), + (20, 30) + ); + assert_eq!(facts[1].source_family, "basenames_base_registrar"); + assert_eq!( + (facts[1].covered_from_block, facts[1].covered_to_block), + (25, 40) + ); +} + +#[test] +fn generic_resolver_scan_yields_family_fact_and_excludes_resolver_targets() { + let source_plan = source_plan( + "ethereum-mainnet", + vec![ + target( + "ens_v1_registry_l1", + 1, + "0x1111111111111111111111111111111111111111", + 5, + 50, + ), + target( + "ens_v1_resolver_l1", + 2, + "0x2222222222222222222222222222222222222222", + 12, + 18, + ), + ], + ); + + let facts = job_completion_coverage_facts(&source_plan, false, 10, 20).collect::>(); + + assert_eq!(facts.len(), 2); + assert_eq!(facts[0].source_family, "ens_v1_resolver_l1"); + assert_eq!(facts[0].scope, BackfillCoverageFactScope::Family); + assert_eq!(facts[0].address, None); + assert_eq!( + (facts[0].covered_from_block, facts[0].covered_to_block), + (12, 18), + "family fact must be clamped to the resolver targets' effective span" + ); + assert_eq!(facts[1].source_family, "ens_v1_registry_l1"); + assert_eq!(facts[1].scope, BackfillCoverageFactScope::Address); + assert_eq!( + (facts[1].covered_from_block, facts[1].covered_to_block), + (10, 20) + ); +} + +#[test] +fn family_scans_without_matching_targets_or_overlap_yield_no_family_fact() { + let mut no_resolver_targets = source_plan( + "ethereum-mainnet", + vec![target( + "ens_v1_registry_l1", + 1, + "0x1111111111111111111111111111111111111111", + 5, + 50, + )], + ); + no_resolver_targets.selector_kind = WatchedSourceSelectorKind::SourceFamily; + no_resolver_targets.source_family = Some("ens_v1_resolver_l1".to_owned()); + assert!( + job_completion_coverage_facts(&no_resolver_targets, false, 10, 20) + .all(|fact| fact.scope != BackfillCoverageFactScope::Family), + "a family scan with no selected targets must not claim family coverage" + ); + + let disjoint_span = source_plan( + "base-mainnet", + vec![target( + "basenames_base_registry", + 1, + "0x1111111111111111111111111111111111111111", + 30, + 40, + )], + ); + assert_eq!( + job_completion_coverage_facts(&disjoint_span, true, 10, 20).count(), + 0, + "a family span disjoint from the job range must not claim coverage" + ); +} + +#[test] +fn family_targets_individually_disjoint_from_the_job_range_claim_nothing() { + // Both windows miss the job range, but their hull [5, 40] spans it: + // hull-then-intersect would mint a family fact over blocks that were + // never fetched. + let source_plan = source_plan( + "base-mainnet", + vec![ + target( + "basenames_base_registry", + 1, + "0x1111111111111111111111111111111111111111", + 5, + 8, + ), + target( + "basenames_base_registry", + 2, + "0x2222222222222222222222222222222222222222", + 30, + 40, + ), + ], + ); + + assert_eq!( + job_completion_coverage_facts(&source_plan, true, 10, 20).count(), + 0, + "windows individually disjoint from the job range must not claim family coverage" + ); +} + +#[test] +fn family_facts_split_on_interior_gaps_between_target_windows() { + let source_plan = source_plan( + "base-mainnet", + vec![ + target( + "basenames_base_registry", + 1, + "0x1111111111111111111111111111111111111111", + 10, + 12, + ), + target( + "basenames_base_registry", + 2, + "0x2222222222222222222222222222222222222222", + 16, + 20, + ), + target( + "basenames_base_registry", + 3, + "0x3333333333333333333333333333333333333333", + 13, + 13, + ), + ], + ); + + let facts = job_completion_coverage_facts(&source_plan, true, 5, 25).collect::>(); + + assert_eq!( + facts + .iter() + .map(|fact| (fact.scope, (fact.covered_from_block, fact.covered_to_block))) + .collect::>(), + vec![ + (BackfillCoverageFactScope::Family, (10, 13)), + (BackfillCoverageFactScope::Family, (16, 20)), + ], + "adjacent windows merge but the unfetched gap between segments must stay unclaimed" + ); +} + +#[test] +fn merged_covered_block_segments_clamp_before_merging() { + assert_eq!( + merged_covered_block_segments(vec![(5, 8), (30, 40)], 10, 20), + Vec::<(i64, i64)>::new() + ); + assert_eq!( + merged_covered_block_segments(vec![(16, 30), (1, 12), (13, 14)], 10, 20), + vec![(10, 14), (16, 20)] + ); + assert_eq!( + merged_covered_block_segments(vec![(1, 100)], 10, 20), + vec![(10, 20)] + ); + assert_eq!( + merged_covered_block_segments(std::iter::empty(), 10, 20), + Vec::<(i64, i64)>::new() + ); + let max = i64::MAX; + assert_eq!( + merged_covered_block_segments([(max, max), (max - 1, max - 1)], max - 1, max), + vec![(max - 1, max)] + ); +} + +#[test] +fn basenames_registry_scan_all_yields_only_the_registry_family_fact() { + let source_plan = source_plan( + "base-mainnet", + vec![ + target( + "basenames_base_registry", + 1, + "0x1111111111111111111111111111111111111111", + 12, + 18, + ), + target( + "basenames_base_registry", + 2, + "0x2222222222222222222222222222222222222222", + 14, + 30, + ), + ], + ); + + let facts = job_completion_coverage_facts(&source_plan, true, 10, 20).collect::>(); + + assert_eq!(facts.len(), 1); + assert_eq!(facts[0].source_family, "basenames_base_registry"); + assert_eq!(facts[0].scope, BackfillCoverageFactScope::Family); + assert_eq!(facts[0].address, None); + assert_eq!( + (facts[0].covered_from_block, facts[0].covered_to_block), + (12, 20), + "family fact must span the union of registry target windows clamped to the job range" + ); +} + +fn target( + source_family: &str, + id: u128, + address: &str, + effective_from_block: i64, + effective_to_block: i64, +) -> WatchedBackfillTarget { + WatchedBackfillTarget { + source_family: source_family.to_owned(), + contract_instance_id: Uuid::from_u128(id), + address: address.to_owned(), + effective_from_block, + effective_to_block, + } +} + +fn source_plan( + chain: &str, + selected_targets: Vec, +) -> WatchedSourceSelectorPlan { + WatchedSourceSelectorPlan { + chain: chain.to_owned(), + selector_kind: WatchedSourceSelectorKind::WatchedTargetSet, + source_family: None, + requested_watched_targets: Vec::new(), + selected_targets, + watched_chain_plan: WatchedChainPlan { + chain: chain.to_owned(), + addresses: Vec::new(), + manifest_root_entry_count: 0, + manifest_contract_entry_count: 0, + discovery_edge_entry_count: 0, + }, + } +} diff --git a/apps/indexer/src/main/backfill/reservation_execution.rs b/apps/indexer/src/main/backfill/reservation_execution.rs index d0b360ab..62c339dd 100644 --- a/apps/indexer/src/main/backfill/reservation_execution.rs +++ b/apps/indexer/src/main/backfill/reservation_execution.rs @@ -1,11 +1,17 @@ #[path = "reservation_execution/coinbase_sql.rs"] mod coinbase_sql_execution; +#[path = "reservation_execution/creation.rs"] +mod creation; #[path = "reservation_execution/digest.rs"] mod digest; #[path = "reservation_execution/generic_topic_identity.rs"] mod generic_topic_identity; +#[path = "reservation_execution/identity.rs"] +mod identity; #[path = "reservation_execution/lease_heartbeat.rs"] mod lease_heartbeat; +#[path = "reservation_execution/progress.rs"] +mod progress; #[path = "reservation_execution/scan_all.rs"] mod scan_all; #[path = "reservation_execution/startup_progress.rs"] @@ -15,7 +21,8 @@ mod startup_progress; mod tests; use anyhow::{Context, Result, bail}; -use bigname_manifests::{WatchedSourceSelectorKind, WatchedSourceSelectorPlan}; +use bigname_adapters::StartupAdapterProgress; +use bigname_manifests::WatchedSourceSelectorPlan; use bigname_storage::{ BackfillJobCreate, BackfillJobRecord, BackfillLifecycleStatus, BackfillRange, BackfillRangeSpec, advance_backfill_range, create_backfill_job, @@ -42,6 +49,10 @@ use super::{ }; use digest::keccak256_json_digest; use generic_topic_identity::{generic_topic_scan_source_identity_payload, selected_targets_sample}; +use identity::requested_watched_targets_value_with_progress; +pub(crate) use identity::{ + backfill_job_source_identity_payload, backfill_job_source_identity_payload_with_progress, +}; pub(crate) use coinbase_sql_execution::{ effective_coinbase_sql_adapter_sync_mode, @@ -52,6 +63,8 @@ pub(super) use lease_heartbeat::{ backfill_lease_duration_secs, refreshed_backfill_lease_expires_at, run_with_backfill_lease_heartbeat, validate_hash_pinned_chunk_blocks, }; +pub(super) use progress::run_reserved_hash_pinned_backfill_range_with_progress; +pub(crate) use progress::run_resumable_hash_pinned_backfill_job_with_progress; pub(crate) use scan_all::effective_hash_pinned_adapter_sync_mode; use scan_all::{ basenames_registry_scan_all_topics_source_identity_payload, @@ -64,227 +77,14 @@ pub(crate) const COINBASE_SQL_BACKFILL_SCAN_MODE: &str = "coinbase_sql_hash_pinn pub(crate) const DEFAULT_HASH_PINNED_BACKFILL_CHUNK_BLOCKS: i64 = 1_024; pub(crate) const COMPACT_SOURCE_IDENTITY_SELECTED_TARGET_THRESHOLD: usize = 10_000; -pub(crate) async fn create_hash_pinned_backfill_job( - pool: &sqlx::PgPool, - source_plan: &WatchedSourceSelectorPlan, - config: &BackfillJobRunConfig, -) -> Result { - let ranges = vec![BackfillRangeSpec { - range_start_block_number: config.range.from_block, - range_end_block_number: config.range.to_block, - }]; - create_hash_pinned_backfill_job_with_ranges(pool, source_plan, config, ranges).await -} - -pub(crate) async fn create_hash_pinned_backfill_job_with_ranges( - pool: &sqlx::PgPool, - source_plan: &WatchedSourceSelectorPlan, - config: &BackfillJobRunConfig, - ranges: Vec, -) -> Result { - let request = BackfillJobCreate { - deployment_profile: config.deployment_profile.clone(), - chain_id: source_plan.watched_chain_plan.chain.clone(), - source_identity: backfill_job_source_identity_payload(source_plan)?, - scan_mode: HASH_PINNED_BACKFILL_SCAN_MODE.to_owned(), - range_start_block_number: config.range.from_block, - range_end_block_number: config.range.to_block, - idempotency_key: config.idempotency_key.clone(), - ranges, - }; - if config.scope_idempotency_to_raw_log_retention_generation { - create_generation_scoped_backfill_job(pool, &request).await - } else { - create_backfill_job(pool, &request).await - } -} - -pub(crate) async fn create_coinbase_sql_backfill_job( - pool: &sqlx::PgPool, - source_plan: &WatchedSourceSelectorPlan, - config: &BackfillJobRunConfig, - coinbase_config: &CoinbaseSqlBackfillConfig, - topic_plan: &BackfillTopicPlan, -) -> Result { - let ranges = vec![BackfillRangeSpec { - range_start_block_number: config.range.from_block, - range_end_block_number: config.range.to_block, - }]; - create_coinbase_sql_backfill_job_with_ranges( - pool, - source_plan, - config, - coinbase_config, - topic_plan, - ranges, - ) - .await -} - -pub(crate) async fn create_coinbase_sql_backfill_job_with_ranges( - pool: &sqlx::PgPool, - source_plan: &WatchedSourceSelectorPlan, - config: &BackfillJobRunConfig, - coinbase_config: &CoinbaseSqlBackfillConfig, - topic_plan: &BackfillTopicPlan, - ranges: Vec, -) -> Result { - let request = BackfillJobCreate { - deployment_profile: config.deployment_profile.clone(), - chain_id: source_plan.watched_chain_plan.chain.clone(), - source_identity: coinbase_sql_backfill_job_source_identity_payload( - source_plan, - coinbase_config, - topic_plan, - )?, - scan_mode: COINBASE_SQL_BACKFILL_SCAN_MODE.to_owned(), - range_start_block_number: config.range.from_block, - range_end_block_number: config.range.to_block, - idempotency_key: config.idempotency_key.clone(), - ranges, - }; - if config.scope_idempotency_to_raw_log_retention_generation { - create_generation_scoped_backfill_job(pool, &request).await - } else { - create_backfill_job(pool, &request).await - } -} - -pub(crate) fn hash_pinned_backfill_range_specs( - range: BackfillBlockRange, - range_blocks: i64, -) -> Result> { - if range_blocks <= 0 { - bail!("hash-pinned backfill range blocks must be positive, got {range_blocks}"); - } - - let mut ranges = Vec::new(); - let mut range_start = range.from_block; - while range_start <= range.to_block { - let range_end = range_start - .checked_add(range_blocks - 1) - .unwrap_or(range.to_block) - .min(range.to_block); - ranges.push(BackfillRangeSpec { - range_start_block_number: range_start, - range_end_block_number: range_end, - }); - if range_end == range.to_block { - break; - } - range_start = range_end - .checked_add(1) - .context("hash-pinned backfill range start overflowed while partitioning")?; - } - - Ok(ranges) -} - -pub(crate) fn backfill_job_source_identity_payload( - source_plan: &WatchedSourceSelectorPlan, -) -> Result { - if watched_source_plan_uses_basenames_registry_scan_all(source_plan) { - return basenames_registry_scan_all_topics_source_identity_payload(source_plan); - } - if watched_source_plan_uses_generic_resolver_scope(source_plan) { - return generic_topic_scan_source_identity_payload(source_plan); - } - - if source_plan.selector_kind != WatchedSourceSelectorKind::SourceFamily - || source_plan.selected_targets.len() <= COMPACT_SOURCE_IDENTITY_SELECTED_TARGET_THRESHOLD - { - return Ok(source_plan.source_identity_payload()); - } - - let selected_targets_digest = keccak256_json_digest(&source_plan.selected_targets) - .context("failed to digest compact backfill source selected targets")?; - let mut payload = json!({ - "selector_kind": source_plan.selector_kind.as_str(), - "source_family": &source_plan.source_family, - "requested_watched_targets": &source_plan.requested_watched_targets, - "selected_target_count": source_plan.selected_targets.len(), - "selected_targets_digest_algorithm": "keccak256", - "selected_targets_digest": selected_targets_digest, - "selected_targets_sample": selected_targets_sample(&source_plan.selected_targets), - "source_identity_payload_format": "selected_targets_digest_v1", - }); - let source_identity_hash = - keccak256_json_digest(&payload).context("failed to digest backfill source identity")?; - payload - .as_object_mut() - .expect("compact source identity payload must be an object") - .insert( - "source_identity_hash".to_owned(), - Value::String(source_identity_hash), - ); - Ok(payload) -} - -pub(crate) fn coinbase_sql_backfill_job_source_identity_payload( - source_plan: &WatchedSourceSelectorPlan, - coinbase_config: &CoinbaseSqlBackfillConfig, - topic_plan: &BackfillTopicPlan, -) -> Result { - let mut payload = if coinbase_sql_uses_basenames_registry_scan_all(source_plan, topic_plan) { - coinbase_sql_basenames_registry_scan_all_source_identity_payload(source_plan)? - } else { - if watched_source_plan_uses_basenames_registry_scan_all(source_plan) { - // A registry-family plan whose Coinbase SQL topic plan is empty - // would fetch address-scoped, so the hash-pinned scan-all - // identity (which asserts a topics-complete scan) must not be - // minted for it. - bail!( - "Coinbase SQL registry-family backfill has an empty topic plan; \ - refusing to mint a scan-all identity for an address-scoped fetch" - ); - } - backfill_job_source_identity_payload(source_plan)? - }; - let object = payload - .as_object_mut() - .context("backfill source identity payload must be an object")?; - object.insert( - "backfill_provider".to_owned(), - Value::String("coinbase_cdp_sql".to_owned()), - ); - object.insert( - "scan_mode".to_owned(), - Value::String(COINBASE_SQL_BACKFILL_SCAN_MODE.to_owned()), - ); - object.insert( - "coinbase_sql_plan_version".to_owned(), - Value::String("base_logs_v2".to_owned()), - ); - object.insert("validation_provider_required".to_owned(), Value::Bool(true)); - object.insert( - "coinbase_sql_validation_mode".to_owned(), - Value::String(coinbase_config.validation_mode.as_str().to_owned()), - ); - object.insert( - "topic_filtering".to_owned(), - Value::String("manifest_abi_topic0_union_v1".to_owned()), - ); - object.insert( - "coinbase_sql_topic_plan".to_owned(), - topic_plan.source_identity_payload()?, - ); - payload - .as_object_mut() - .context("backfill source identity payload must be an object")? - .remove("source_identity_hash"); - let source_identity_hash = - keccak256_json_digest(&payload).context("failed to digest Coinbase SQL source identity")?; - payload - .as_object_mut() - .context("backfill source identity payload must be an object")? - .insert( - "source_identity_hash".to_owned(), - Value::String(source_identity_hash), - ); - - Ok(payload) -} - +#[cfg(test)] +pub(crate) use creation::coinbase_sql_backfill_job_source_identity_payload; +pub(crate) use creation::{ + create_coinbase_sql_backfill_job, create_coinbase_sql_backfill_job_with_ranges, + create_hash_pinned_backfill_job, create_hash_pinned_backfill_job_with_progress, + create_hash_pinned_backfill_job_with_ranges, + create_hash_pinned_backfill_job_with_ranges_with_progress, hash_pinned_backfill_range_specs, +}; pub(crate) async fn run_resumable_hash_pinned_backfill_job( pool: &sqlx::PgPool, source_plan: &WatchedSourceSelectorPlan, @@ -295,7 +95,15 @@ pub(crate) async fn run_resumable_hash_pinned_backfill_job( effective_hash_pinned_adapter_sync_mode(source_plan, config.adapter_sync_mode); validate_hash_pinned_chunk_blocks(config.hash_pinned_chunk_blocks)?; let record = create_hash_pinned_backfill_job(pool, source_plan, &config).await?; - run_precreated_hash_pinned_backfill_job_inner(pool, source_plan, provider, config, record).await + run_precreated_hash_pinned_backfill_job_inner( + pool, + source_plan, + provider, + config, + record, + &mut None, + ) + .await } pub(crate) async fn run_precreated_hash_pinned_backfill_job( @@ -308,7 +116,15 @@ pub(crate) async fn run_precreated_hash_pinned_backfill_job( config.adapter_sync_mode = effective_hash_pinned_adapter_sync_mode(source_plan, config.adapter_sync_mode); validate_hash_pinned_chunk_blocks(config.hash_pinned_chunk_blocks)?; - run_precreated_hash_pinned_backfill_job_inner(pool, source_plan, provider, config, record).await + run_precreated_hash_pinned_backfill_job_inner( + pool, + source_plan, + provider, + config, + record, + &mut None, + ) + .await } async fn run_precreated_hash_pinned_backfill_job_inner( @@ -317,6 +133,7 @@ async fn run_precreated_hash_pinned_backfill_job_inner( provider: &(impl ChainProviderOps + ?Sized), mut config: BackfillJobRunConfig, record: BackfillJobRecord, + progress: &mut Option<&mut dyn StartupAdapterProgress>, ) -> Result { let watched_chain = &source_plan.watched_chain_plan; config @@ -358,13 +175,15 @@ async fn run_precreated_hash_pinned_backfill_job_inner( }; outcome.reserved_range_count += 1; - run_reserved_hash_pinned_backfill_range( + run_reserved_hash_pinned_backfill_range_inner( pool, source_plan, provider, &config, &reserved_range, &mut outcome, + None, + progress, ) .await?; outcome.completed_range_count += 1; @@ -404,6 +223,7 @@ async fn run_precreated_hash_pinned_backfill_job_inner( ); } +#[allow(dead_code)] pub(super) async fn run_reserved_hash_pinned_backfill_range( pool: &sqlx::PgPool, source_plan: &WatchedSourceSelectorPlan, @@ -420,31 +240,12 @@ pub(super) async fn run_reserved_hash_pinned_backfill_range( reserved_range, aggregate, None, + &mut None, ) .await } -pub(super) async fn run_reserved_hash_pinned_backfill_range_with_progress( - pool: &sqlx::PgPool, - source_plan: &WatchedSourceSelectorPlan, - provider: &(impl ChainProviderOps + ?Sized), - config: &BackfillJobRunConfig, - reserved_range: &BackfillRange, - aggregate: &mut BackfillJobRunOutcome, - progress: &tokio::sync::mpsc::UnboundedSender<()>, -) -> Result<()> { - run_reserved_hash_pinned_backfill_range_inner( - pool, - source_plan, - provider, - config, - reserved_range, - aggregate, - Some(progress), - ) - .await -} - +#[expect(clippy::too_many_arguments)] async fn run_reserved_hash_pinned_backfill_range_inner( pool: &sqlx::PgPool, source_plan: &WatchedSourceSelectorPlan, @@ -452,7 +253,8 @@ async fn run_reserved_hash_pinned_backfill_range_inner( config: &BackfillJobRunConfig, reserved_range: &BackfillRange, aggregate: &mut BackfillJobRunOutcome, - progress: Option<&tokio::sync::mpsc::UnboundedSender<()>>, + progress_sender: Option<&tokio::sync::mpsc::UnboundedSender<()>>, + service_progress: &mut Option<&mut dyn StartupAdapterProgress>, ) -> Result<()> { let mut active_range = reserved_range.clone(); let mut block_number = active_range @@ -490,8 +292,10 @@ async fn run_reserved_hash_pinned_backfill_range_inner( .unwrap_or(active_range.range_end_block_number) .min(active_range.range_end_block_number); let chunk_range = BackfillBlockRange::new(block_number, chunk_end)?; - let progress_ranges = - startup_progress::heartbeat_progress_ranges(chunk_range, progress.is_some())?; + let progress_ranges = startup_progress::heartbeat_progress_ranges( + chunk_range, + progress_sender.is_some() || service_progress.is_some(), + )?; for progress_range in progress_ranges { let selected_target_addresses = scan_all::chunk_addresses_for_plan( source_plan, @@ -530,8 +334,11 @@ async fn run_reserved_hash_pinned_backfill_range_inner( Err(failure) => return Err(record_reserved_range_failure(failure).await), }; aggregate.add_range_outcome(&outcome); - if let Some(progress) = progress { - let _ = progress.send(()); + if let Some(progress_sender) = progress_sender { + let _ = progress_sender.send(()); + } + if let Some(progress) = service_progress.as_deref_mut() { + progress.record(pool).await?; } } @@ -573,6 +380,8 @@ async fn run_reserved_hash_pinned_backfill_range_inner( source_plan, watched_source_plan_uses_basenames_registry_scan_all(source_plan), "backfill range completion failed", + progress_sender, + service_progress, ) .await } diff --git a/apps/indexer/src/main/backfill/reservation_execution/coinbase_sql.rs b/apps/indexer/src/main/backfill/reservation_execution/coinbase_sql.rs index ff12c2d7..a876faf3 100644 --- a/apps/indexer/src/main/backfill/reservation_execution/coinbase_sql.rs +++ b/apps/indexer/src/main/backfill/reservation_execution/coinbase_sql.rs @@ -145,7 +145,6 @@ pub(crate) async fn run_resumable_coinbase_sql_backfill_job( ); } -// Reserved-range execution keeps lease, source, provider, and aggregate inputs explicit. #[expect(clippy::too_many_arguments)] pub(crate) async fn run_reserved_coinbase_sql_backfill_range( pool: &sqlx::PgPool, @@ -298,6 +297,8 @@ pub(crate) async fn run_reserved_coinbase_sql_backfill_range( source_plan, super::coinbase_sql_uses_basenames_registry_scan_all(source_plan, topic_plan), "Coinbase SQL backfill range completion failed", + None, + &mut None, ) .await } @@ -318,7 +319,6 @@ fn next_coinbase_sql_window_blocks( } } -// Window execution keeps selection, validation, evidence, and job inputs explicit. #[expect(clippy::too_many_arguments)] async fn run_coinbase_sql_backfill_window( pool: &sqlx::PgPool, diff --git a/apps/indexer/src/main/backfill/reservation_execution/creation.rs b/apps/indexer/src/main/backfill/reservation_execution/creation.rs new file mode 100644 index 00000000..2b373fe8 --- /dev/null +++ b/apps/indexer/src/main/backfill/reservation_execution/creation.rs @@ -0,0 +1,239 @@ +use super::*; + +pub(crate) async fn create_hash_pinned_backfill_job( + pool: &sqlx::PgPool, + source_plan: &WatchedSourceSelectorPlan, + config: &BackfillJobRunConfig, +) -> Result { + let ranges = vec![BackfillRangeSpec { + range_start_block_number: config.range.from_block, + range_end_block_number: config.range.to_block, + }]; + create_hash_pinned_backfill_job_with_ranges(pool, source_plan, config, ranges).await +} + +pub(crate) async fn create_hash_pinned_backfill_job_with_progress( + pool: &sqlx::PgPool, + source_plan: &WatchedSourceSelectorPlan, + config: &BackfillJobRunConfig, + progress: &mut dyn StartupAdapterProgress, +) -> Result { + let ranges = vec![BackfillRangeSpec { + range_start_block_number: config.range.from_block, + range_end_block_number: config.range.to_block, + }]; + create_hash_pinned_backfill_job_with_ranges_with_progress( + pool, + source_plan, + config, + ranges, + progress, + ) + .await +} + +pub(crate) async fn create_hash_pinned_backfill_job_with_ranges( + pool: &sqlx::PgPool, + source_plan: &WatchedSourceSelectorPlan, + config: &BackfillJobRunConfig, + ranges: Vec, +) -> Result { + let source_identity = backfill_job_source_identity_payload(source_plan)?; + create_hash_pinned_backfill_job_from_identity( + pool, + source_plan, + config, + ranges, + source_identity, + ) + .await +} + +pub(crate) async fn create_hash_pinned_backfill_job_with_ranges_with_progress( + pool: &sqlx::PgPool, + source_plan: &WatchedSourceSelectorPlan, + config: &BackfillJobRunConfig, + ranges: Vec, + progress: &mut dyn StartupAdapterProgress, +) -> Result { + let source_identity = + backfill_job_source_identity_payload_with_progress(pool, source_plan, progress).await?; + create_hash_pinned_backfill_job_from_identity( + pool, + source_plan, + config, + ranges, + source_identity, + ) + .await +} + +async fn create_hash_pinned_backfill_job_from_identity( + pool: &sqlx::PgPool, + source_plan: &WatchedSourceSelectorPlan, + config: &BackfillJobRunConfig, + ranges: Vec, + source_identity: Value, +) -> Result { + let request = BackfillJobCreate { + deployment_profile: config.deployment_profile.clone(), + chain_id: source_plan.watched_chain_plan.chain.clone(), + source_identity, + scan_mode: HASH_PINNED_BACKFILL_SCAN_MODE.to_owned(), + range_start_block_number: config.range.from_block, + range_end_block_number: config.range.to_block, + idempotency_key: config.idempotency_key.clone(), + ranges, + }; + if config.scope_idempotency_to_raw_log_retention_generation { + create_generation_scoped_backfill_job(pool, &request).await + } else { + create_backfill_job(pool, &request).await + } +} + +pub(crate) async fn create_coinbase_sql_backfill_job( + pool: &sqlx::PgPool, + source_plan: &WatchedSourceSelectorPlan, + config: &BackfillJobRunConfig, + coinbase_config: &CoinbaseSqlBackfillConfig, + topic_plan: &BackfillTopicPlan, +) -> Result { + let ranges = vec![BackfillRangeSpec { + range_start_block_number: config.range.from_block, + range_end_block_number: config.range.to_block, + }]; + create_coinbase_sql_backfill_job_with_ranges( + pool, + source_plan, + config, + coinbase_config, + topic_plan, + ranges, + ) + .await +} + +pub(crate) async fn create_coinbase_sql_backfill_job_with_ranges( + pool: &sqlx::PgPool, + source_plan: &WatchedSourceSelectorPlan, + config: &BackfillJobRunConfig, + coinbase_config: &CoinbaseSqlBackfillConfig, + topic_plan: &BackfillTopicPlan, + ranges: Vec, +) -> Result { + let request = BackfillJobCreate { + deployment_profile: config.deployment_profile.clone(), + chain_id: source_plan.watched_chain_plan.chain.clone(), + source_identity: coinbase_sql_backfill_job_source_identity_payload( + source_plan, + coinbase_config, + topic_plan, + )?, + scan_mode: COINBASE_SQL_BACKFILL_SCAN_MODE.to_owned(), + range_start_block_number: config.range.from_block, + range_end_block_number: config.range.to_block, + idempotency_key: config.idempotency_key.clone(), + ranges, + }; + if config.scope_idempotency_to_raw_log_retention_generation { + create_generation_scoped_backfill_job(pool, &request).await + } else { + create_backfill_job(pool, &request).await + } +} + +pub(crate) fn hash_pinned_backfill_range_specs( + range: BackfillBlockRange, + range_blocks: i64, +) -> Result> { + if range_blocks <= 0 { + bail!("hash-pinned backfill range blocks must be positive, got {range_blocks}"); + } + + let mut ranges = Vec::new(); + let mut range_start = range.from_block; + while range_start <= range.to_block { + let range_end = range_start + .checked_add(range_blocks - 1) + .unwrap_or(range.to_block) + .min(range.to_block); + ranges.push(BackfillRangeSpec { + range_start_block_number: range_start, + range_end_block_number: range_end, + }); + if range_end == range.to_block { + break; + } + range_start = range_end + .checked_add(1) + .context("hash-pinned backfill range start overflowed while partitioning")?; + } + + Ok(ranges) +} + +pub(crate) fn coinbase_sql_backfill_job_source_identity_payload( + source_plan: &WatchedSourceSelectorPlan, + coinbase_config: &CoinbaseSqlBackfillConfig, + topic_plan: &BackfillTopicPlan, +) -> Result { + let mut payload = if coinbase_sql_uses_basenames_registry_scan_all(source_plan, topic_plan) { + coinbase_sql_basenames_registry_scan_all_source_identity_payload(source_plan)? + } else { + if watched_source_plan_uses_basenames_registry_scan_all(source_plan) { + // A registry-family plan whose Coinbase SQL topic plan is empty + // would fetch address-scoped, so the hash-pinned scan-all + // identity (which asserts a topics-complete scan) must not be + // minted for it. + bail!( + "Coinbase SQL registry-family backfill has an empty topic plan; \ + refusing to mint a scan-all identity for an address-scoped fetch" + ); + } + backfill_job_source_identity_payload(source_plan)? + }; + let object = payload + .as_object_mut() + .context("backfill source identity payload must be an object")?; + object.insert( + "backfill_provider".to_owned(), + Value::String("coinbase_cdp_sql".to_owned()), + ); + object.insert( + "scan_mode".to_owned(), + Value::String(COINBASE_SQL_BACKFILL_SCAN_MODE.to_owned()), + ); + object.insert( + "coinbase_sql_plan_version".to_owned(), + Value::String("base_logs_v2".to_owned()), + ); + object.insert("validation_provider_required".to_owned(), Value::Bool(true)); + object.insert( + "coinbase_sql_validation_mode".to_owned(), + Value::String(coinbase_config.validation_mode.as_str().to_owned()), + ); + object.insert( + "topic_filtering".to_owned(), + Value::String("manifest_abi_topic0_union_v1".to_owned()), + ); + object.insert( + "coinbase_sql_topic_plan".to_owned(), + topic_plan.source_identity_payload()?, + ); + payload + .as_object_mut() + .context("backfill source identity payload must be an object")? + .remove("source_identity_hash"); + let source_identity_hash = + keccak256_json_digest(&payload).context("failed to digest Coinbase SQL source identity")?; + payload + .as_object_mut() + .context("backfill source identity payload must be an object")? + .insert( + "source_identity_hash".to_owned(), + Value::String(source_identity_hash), + ); + + Ok(payload) +} diff --git a/apps/indexer/src/main/backfill/reservation_execution/digest.rs b/apps/indexer/src/main/backfill/reservation_execution/digest.rs index 6f3eda78..87725ad7 100644 --- a/apps/indexer/src/main/backfill/reservation_execution/digest.rs +++ b/apps/indexer/src/main/backfill/reservation_execution/digest.rs @@ -4,6 +4,11 @@ use alloy_primitives::{Keccak256, hex}; use anyhow::{Context, Result}; use serde::Serialize; +use bigname_adapters::StartupAdapterProgress; +use bigname_manifests::WatchedBackfillTarget; + +const DIGEST_PROGRESS_ITEMS: usize = 1_000; + pub(super) fn keccak256_json_digest(value: &T) -> Result where T: Serialize + ?Sized, @@ -13,6 +18,77 @@ where Ok(format!("keccak256:{}", hex_string(&writer.finalize()))) } +pub(super) async fn keccak256_selected_targets_digest_with_progress( + pool: &sqlx::PgPool, + items: &[WatchedBackfillTarget], + excluded_source_family: Option<&str>, + progress: &mut dyn StartupAdapterProgress, +) -> Result { + let mut writer = Keccak256Writer::default(); + writer.write_all(b"[")?; + let mut count = 0usize; + for item in items { + if excluded_source_family.is_some_and(|family| item.source_family == family) { + continue; + } + if count > 0 { + writer.write_all(b",")?; + } + serde_json::to_writer(&mut writer, item) + .context("failed to serialize JSON digest array item")?; + count += 1; + if count.is_multiple_of(DIGEST_PROGRESS_ITEMS) { + progress.record(pool).await?; + } + } + writer.write_all(b"]")?; + if count > 0 && !count.is_multiple_of(DIGEST_PROGRESS_ITEMS) { + progress.record(pool).await?; + } + Ok(format!("keccak256:{}", hex_string(&writer.finalize()))) +} + +pub(super) async fn keccak256_json_value_digest_with_progress( + pool: &sqlx::PgPool, + value: &serde_json::Value, + progress: &mut dyn StartupAdapterProgress, +) -> Result { + let Some(fields) = value.as_object() else { + return keccak256_json_digest(value); + }; + let mut writer = Keccak256Writer::default(); + writer.write_all(b"{")?; + for (field_index, (key, value)) in fields.iter().enumerate() { + if field_index > 0 { + writer.write_all(b",")?; + } + serde_json::to_writer(&mut writer, key).context("failed to serialize JSON digest key")?; + writer.write_all(b":")?; + if let Some(items) = value.as_array() { + writer.write_all(b"[")?; + for (item_index, item) in items.iter().enumerate() { + if item_index > 0 { + writer.write_all(b",")?; + } + serde_json::to_writer(&mut writer, item) + .context("failed to serialize JSON digest array item")?; + if (item_index + 1).is_multiple_of(DIGEST_PROGRESS_ITEMS) { + progress.record(pool).await?; + } + } + writer.write_all(b"]")?; + if !items.is_empty() && !items.len().is_multiple_of(DIGEST_PROGRESS_ITEMS) { + progress.record(pool).await?; + } + } else { + serde_json::to_writer(&mut writer, value) + .context("failed to serialize JSON digest value")?; + } + } + writer.write_all(b"}")?; + Ok(format!("keccak256:{}", hex_string(&writer.finalize()))) +} + #[derive(Default)] struct Keccak256Writer { hasher: Keccak256, diff --git a/apps/indexer/src/main/backfill/reservation_execution/generic_topic_identity.rs b/apps/indexer/src/main/backfill/reservation_execution/generic_topic_identity.rs index d23f159f..e0660616 100644 --- a/apps/indexer/src/main/backfill/reservation_execution/generic_topic_identity.rs +++ b/apps/indexer/src/main/backfill/reservation_execution/generic_topic_identity.rs @@ -1,4 +1,5 @@ use anyhow::{Context, Result}; +use bigname_adapters::StartupAdapterProgress; use bigname_manifests::{ WatchedBackfillTarget, WatchedSourceSelectorKind, WatchedSourceSelectorPlan, }; @@ -6,7 +7,116 @@ use serde_json::{Value, json}; use crate::ens_v1_resolver::{SOURCE_FAMILY_ENS_V1_RESOLVER_L1, generic_resolver_record_topic0s}; -use super::{COMPACT_SOURCE_IDENTITY_SELECTED_TARGET_THRESHOLD, digest::keccak256_json_digest}; +use super::{ + COMPACT_SOURCE_IDENTITY_SELECTED_TARGET_THRESHOLD, + digest::{ + keccak256_json_digest, keccak256_json_value_digest_with_progress, + keccak256_selected_targets_digest_with_progress, + }, + requested_watched_targets_value_with_progress, +}; + +pub(super) async fn generic_topic_scan_source_identity_payload_with_progress( + pool: &sqlx::PgPool, + source_plan: &WatchedSourceSelectorPlan, + progress: &mut dyn StartupAdapterProgress, +) -> Result { + const PROGRESS_TARGETS: usize = 1_000; + let mut selected_count = 0usize; + let mut selected_first = None; + let mut selected_last = None; + for (index, target) in source_plan.selected_targets.iter().enumerate() { + if target.source_family != SOURCE_FAMILY_ENS_V1_RESOLVER_L1 { + selected_count += 1; + selected_first.get_or_insert(target); + selected_last = Some(target); + } + if (index + 1).is_multiple_of(PROGRESS_TARGETS) { + progress.record(pool).await?; + } + } + if !source_plan.selected_targets.is_empty() + && !source_plan + .selected_targets + .len() + .is_multiple_of(PROGRESS_TARGETS) + { + progress.record(pool).await?; + } + let requested_watched_targets = + requested_watched_targets_value_with_progress(pool, source_plan, progress).await?; + let generic_topic0s = generic_resolver_record_topic0s(); + let generic_topic_scans = json!([{ + "source_family": SOURCE_FAMILY_ENS_V1_RESOLVER_L1, + "source_identity_payload_format": "generic_resolver_event_topics_v1" + }]); + + let mut payload = if source_plan.selector_kind == WatchedSourceSelectorKind::SourceFamily + && source_plan.source_family.as_deref() == Some(SOURCE_FAMILY_ENS_V1_RESOLVER_L1) + { + json!({ + "selector_kind": source_plan.selector_kind.as_str(), + "source_family": &source_plan.source_family, + "requested_watched_targets": requested_watched_targets, + "source_identity_payload_format": "generic_resolver_event_topics_v1", + "topic0s_by_source_family": { + SOURCE_FAMILY_ENS_V1_RESOLVER_L1: generic_topic0s, + }, + }) + } else if selected_count <= COMPACT_SOURCE_IDENTITY_SELECTED_TARGET_THRESHOLD { + let mut selected_targets = Vec::with_capacity(selected_count); + for target in &source_plan.selected_targets { + if target.source_family != SOURCE_FAMILY_ENS_V1_RESOLVER_L1 { + selected_targets.push(target.clone()); + } + } + json!({ + "selector_kind": source_plan.selector_kind.as_str(), + "source_family": &source_plan.source_family, + "requested_watched_targets": requested_watched_targets, + "selected_targets": selected_targets, + "generic_topic_scans": generic_topic_scans, + "source_identity_payload_format": "selected_targets_with_generic_topic_scans_v1", + "topic0s_by_source_family": { + SOURCE_FAMILY_ENS_V1_RESOLVER_L1: generic_topic0s, + }, + }) + } else { + let selected_targets_digest = keccak256_selected_targets_digest_with_progress( + pool, + &source_plan.selected_targets, + Some(SOURCE_FAMILY_ENS_V1_RESOLVER_L1), + progress, + ) + .await + .context("failed to digest compact generic-topic-scan source selected targets")?; + json!({ + "selector_kind": source_plan.selector_kind.as_str(), + "source_family": &source_plan.source_family, + "requested_watched_targets": requested_watched_targets, + "selected_target_count": selected_count, + "selected_targets_digest_algorithm": "keccak256", + "selected_targets_digest": selected_targets_digest, + "selected_targets_sample": {"first": selected_first, "last": selected_last}, + "generic_topic_scans": generic_topic_scans, + "source_identity_payload_format": "selected_targets_digest_with_generic_topic_scans_v1", + "topic0s_by_source_family": { + SOURCE_FAMILY_ENS_V1_RESOLVER_L1: generic_topic0s, + }, + }) + }; + let source_identity_hash = keccak256_json_value_digest_with_progress(pool, &payload, progress) + .await + .context("failed to digest generic-topic-scan backfill source identity")?; + payload + .as_object_mut() + .expect("generic-topic-scan source identity payload must be an object") + .insert( + "source_identity_hash".to_owned(), + Value::String(source_identity_hash), + ); + Ok(payload) +} pub(super) fn generic_topic_scan_source_identity_payload( source_plan: &WatchedSourceSelectorPlan, diff --git a/apps/indexer/src/main/backfill/reservation_execution/identity.rs b/apps/indexer/src/main/backfill/reservation_execution/identity.rs new file mode 100644 index 00000000..38e286c8 --- /dev/null +++ b/apps/indexer/src/main/backfill/reservation_execution/identity.rs @@ -0,0 +1,117 @@ +use super::digest::{ + keccak256_json_value_digest_with_progress, keccak256_selected_targets_digest_with_progress, +}; +use super::*; + +pub(crate) fn backfill_job_source_identity_payload( + source_plan: &WatchedSourceSelectorPlan, +) -> Result { + if watched_source_plan_uses_basenames_registry_scan_all(source_plan) { + return basenames_registry_scan_all_topics_source_identity_payload(source_plan); + } + if watched_source_plan_uses_generic_resolver_scope(source_plan) { + return generic_topic_scan_source_identity_payload(source_plan); + } + + if source_plan.selected_targets.len() <= COMPACT_SOURCE_IDENTITY_SELECTED_TARGET_THRESHOLD { + return Ok(source_plan.source_identity_payload()); + } + + let selected_targets_digest = keccak256_json_digest(&source_plan.selected_targets) + .context("failed to digest compact backfill source selected targets")?; + let mut payload = json!({ + "selector_kind": source_plan.selector_kind.as_str(), + "source_family": &source_plan.source_family, + "requested_watched_targets": &source_plan.requested_watched_targets, + "selected_target_count": source_plan.selected_targets.len(), + "selected_targets_digest_algorithm": "keccak256", + "selected_targets_digest": selected_targets_digest, + "selected_targets_sample": selected_targets_sample(&source_plan.selected_targets), + "source_identity_payload_format": "selected_targets_digest_v1", + }); + let source_identity_hash = + keccak256_json_digest(&payload).context("failed to digest backfill source identity")?; + payload + .as_object_mut() + .expect("compact source identity payload must be an object") + .insert( + "source_identity_hash".to_owned(), + Value::String(source_identity_hash), + ); + Ok(payload) +} + +pub(crate) async fn backfill_job_source_identity_payload_with_progress( + pool: &sqlx::PgPool, + source_plan: &WatchedSourceSelectorPlan, + progress: &mut dyn StartupAdapterProgress, +) -> Result { + if watched_source_plan_uses_basenames_registry_scan_all(source_plan) { + return basenames_registry_scan_all_topics_source_identity_payload(source_plan); + } + if watched_source_plan_uses_generic_resolver_scope(source_plan) { + return generic_topic_identity::generic_topic_scan_source_identity_payload_with_progress( + pool, + source_plan, + progress, + ) + .await; + } + if source_plan.selected_targets.len() <= COMPACT_SOURCE_IDENTITY_SELECTED_TARGET_THRESHOLD { + return Ok(source_plan.source_identity_payload()); + } + + let requested_watched_targets = + requested_watched_targets_value_with_progress(pool, source_plan, progress).await?; + let selected_targets_digest = keccak256_selected_targets_digest_with_progress( + pool, + &source_plan.selected_targets, + None, + progress, + ) + .await + .context("failed to digest compact backfill source selected targets")?; + let mut payload = json!({ + "selector_kind": source_plan.selector_kind.as_str(), + "source_family": &source_plan.source_family, + "requested_watched_targets": requested_watched_targets, + "selected_target_count": source_plan.selected_targets.len(), + "selected_targets_digest_algorithm": "keccak256", + "selected_targets_digest": selected_targets_digest, + "selected_targets_sample": selected_targets_sample(&source_plan.selected_targets), + "source_identity_payload_format": "selected_targets_digest_v1", + }); + let source_identity_hash = keccak256_json_value_digest_with_progress(pool, &payload, progress) + .await + .context("failed to digest backfill source identity")?; + payload + .as_object_mut() + .expect("compact source identity payload must be an object") + .insert( + "source_identity_hash".to_owned(), + Value::String(source_identity_hash), + ); + Ok(payload) +} + +pub(super) async fn requested_watched_targets_value_with_progress( + pool: &sqlx::PgPool, + source_plan: &WatchedSourceSelectorPlan, + progress: &mut dyn StartupAdapterProgress, +) -> Result { + const PROGRESS_TARGETS: usize = 1_000; + let mut targets = Vec::with_capacity(source_plan.requested_watched_targets.len()); + for target in &source_plan.requested_watched_targets { + targets.push( + serde_json::to_value(target) + .context("failed to serialize requested watched target identity")?, + ); + if targets.len().is_multiple_of(PROGRESS_TARGETS) { + progress.record(pool).await?; + } + } + if !targets.is_empty() && !targets.len().is_multiple_of(PROGRESS_TARGETS) { + progress.record(pool).await?; + } + Ok(Value::Array(targets)) +} diff --git a/apps/indexer/src/main/backfill/reservation_execution/progress.rs b/apps/indexer/src/main/backfill/reservation_execution/progress.rs new file mode 100644 index 00000000..0012e828 --- /dev/null +++ b/apps/indexer/src/main/backfill/reservation_execution/progress.rs @@ -0,0 +1,46 @@ +use super::*; + +pub(crate) async fn run_resumable_hash_pinned_backfill_job_with_progress( + pool: &sqlx::PgPool, + source_plan: &WatchedSourceSelectorPlan, + provider: &(impl ChainProviderOps + ?Sized), + mut config: BackfillJobRunConfig, + progress: &mut dyn StartupAdapterProgress, +) -> Result { + config.adapter_sync_mode = + effective_hash_pinned_adapter_sync_mode(source_plan, config.adapter_sync_mode); + validate_hash_pinned_chunk_blocks(config.hash_pinned_chunk_blocks)?; + let record = + create_hash_pinned_backfill_job_with_progress(pool, source_plan, &config, progress).await?; + run_precreated_hash_pinned_backfill_job_inner( + pool, + source_plan, + provider, + config, + record, + &mut Some(progress), + ) + .await +} + +pub(crate) async fn run_reserved_hash_pinned_backfill_range_with_progress( + pool: &sqlx::PgPool, + source_plan: &WatchedSourceSelectorPlan, + provider: &(impl ChainProviderOps + ?Sized), + config: &BackfillJobRunConfig, + reserved_range: &BackfillRange, + aggregate: &mut BackfillJobRunOutcome, + progress: &tokio::sync::mpsc::UnboundedSender<()>, +) -> Result<()> { + run_reserved_hash_pinned_backfill_range_inner( + pool, + source_plan, + provider, + config, + reserved_range, + aggregate, + Some(progress), + &mut None, + ) + .await +} diff --git a/apps/indexer/src/main/bootstrap_backfill.rs b/apps/indexer/src/main/bootstrap_backfill.rs index 16684e7f..a33c59ad 100644 --- a/apps/indexer/src/main/bootstrap_backfill.rs +++ b/apps/indexer/src/main/bootstrap_backfill.rs @@ -1,17 +1,15 @@ -use std::path::Path; +use std::{collections::BTreeSet, path::Path}; use anyhow::{Context, Result}; use bigname_manifests::{ - load_ens_v2_authoritative_discovery_bootstrap_targets, - load_ens_v2_retained_history_recovery_targets, load_manifest_declared_bootstrap_targets, - load_manifest_skipped_bootstrap_targets, + load_manifest_declared_bootstrap_targets, load_manifest_skipped_bootstrap_targets, }; use tracing::{info, warn}; use crate::{ backfill::{ - BackfillAdapterSyncMode, BackfillBlockRange, backfill_job_source_identity_payload, - hash_pinned_backfill_range_specs, run_resumable_hash_pinned_backfill_job_concurrently, + BackfillAdapterSyncMode, BackfillBlockRange, hash_pinned_backfill_range_specs, + run_resumable_hash_pinned_backfill_job_concurrently, run_resumable_hash_pinned_backfill_job_concurrently_with_heartbeat, }, backfill_lease_expires_at, default_backfill_lease_owner, deployment_profile_from_manifest_root, @@ -22,7 +20,7 @@ use crate::{ RawFactNormalizedEventReplaySelection, log_raw_fact_normalized_event_replay_outcome, replay_raw_fact_normalized_events, }, - run::startup_heartbeat::StartupHeartbeat, + run::startup_heartbeat::{StartupAdapterHeartbeat, StartupHeartbeat}, runtime::{IntakeChainTask, validate_provider_registry_for_intake_tasks}, }; @@ -32,15 +30,18 @@ mod checkpoints; mod entrypoints; #[path = "bootstrap_backfill/identity.rs"] mod identity; +#[path = "bootstrap_backfill/logging.rs"] +mod logging; +#[path = "bootstrap_backfill/orchestration.rs"] +mod orchestration; #[path = "bootstrap_backfill/planning.rs"] mod planning; +#[path = "bootstrap_backfill/progress.rs"] +mod progress; #[path = "bootstrap_backfill/recovery.rs"] mod recovery; -use checkpoints::{ - bootstrap_segment_target_ids, load_bootstrap_segment_checkpoint, - load_bootstrap_target_checkpoint, -}; +use checkpoints::{load_bootstrap_segment_checkpoint, load_bootstrap_target_checkpoint}; #[cfg(test)] pub(crate) use entrypoints::run_startup_bootstrap_backfills; pub(crate) use entrypoints::{ @@ -49,21 +50,29 @@ pub(crate) use entrypoints::{ pub(crate) use identity::bootstrap_backfill_idempotency_key; use identity::{ partitioned_bootstrap_backfill_idempotency_key, replay_source_scope_from_source_plan, - source_identity_hash_for_backfill, }; +use orchestration::run_startup_bootstrap_backfills_inner; use planning::{ - BootstrapBackfillTargetRange, bootstrap_target_range, - effective_bootstrap_backfill_worker_count, narrow_manifest_bootstrap_source_plan, - plan_bootstrap_backfill_segments, + BootstrapBackfillTargetRange, bootstrap_target_range, effective_bootstrap_backfill_worker_count, }; pub(crate) use planning::{ bootstrap_finalized_head_block, resolve_bootstrap_backfill_worker_count, }; +use progress::{ + bootstrap_segment_target_ids_with_optional_progress, + bootstrap_source_identity_with_optional_progress, extend_bootstrap_targets_with_progress, + load_bootstrap_source_plan_with_optional_progress, + load_discovery_bootstrap_targets_with_optional_progress, + load_retained_recovery_targets_with_optional_progress, + narrow_bootstrap_source_plan_with_optional_progress, + plan_bootstrap_segments_with_optional_progress, record_bootstrap_progress, +}; #[cfg(test)] pub(crate) use recovery::install_forced_retention_rotation; -use recovery::load_bootstrap_source_plan; use recovery::{ BootstrapConvergenceTracker, BootstrapPassStatus, finish_bootstrap_convergence_pass, + finish_bootstrap_convergence_pass_with_progress, + load_bootstrap_retention_snapshot_with_progress, }; pub(crate) use recovery::{ automatic_backfill_retention_snapshot_is_stable, @@ -72,526 +81,3 @@ pub(crate) use recovery::{ const BOOTSTRAP_BACKFILL_LEASE_DURATION_SECS: u64 = 300; pub(crate) const DEFAULT_BOOTSTRAP_BACKFILL_WORKERS: usize = 0; pub(crate) const DEFAULT_BOOTSTRAP_BACKFILL_RANGE_BLOCKS: i64 = 50_000; -#[expect(clippy::too_many_arguments)] -async fn run_startup_bootstrap_backfills_inner( - pool: &sqlx::PgPool, - manifests_root: &Path, - intake_chain_tasks: &[IntakeChainTask], - provider_registry: &ProviderRegistry, - hash_pinned_chunk_blocks: i64, - adapter_sync_mode: BackfillAdapterSyncMode, - replay_completed_raw_ranges: bool, - header_audit_mode: HeaderAuditMode, - bootstrap_backfill_workers: usize, - bootstrap_backfill_range_blocks: i64, - mut heartbeat: Option<&mut StartupHeartbeat>, -) -> Result { - validate_provider_registry_for_intake_tasks(intake_chain_tasks, provider_registry)?; - let heartbeat_chain_ids = intake_chain_tasks - .iter() - .map(|task| task.chain.clone()) - .collect::>(); - record_bootstrap_progress(pool, &mut heartbeat, &heartbeat_chain_ids).await?; - let backfill_adapter_sync_mode = adapter_sync_mode.startup_hash_pinned_backfill_mode(); - let deployment_profile = deployment_profile_from_manifest_root(manifests_root); - let lease_owner = format!("{}:bootstrap-backfill", default_backfill_lease_owner()); - let requested_worker_count = - resolve_bootstrap_backfill_worker_count(bootstrap_backfill_workers); - let effective_worker_count = effective_bootstrap_backfill_worker_count( - requested_worker_count, - backfill_adapter_sync_mode, - ); - let mut outcome = BootstrapBackfillOutcome { - active_chain_count: intake_chain_tasks.len(), - requested_worker_count, - effective_worker_count, - range_partition_block_count: bootstrap_backfill_range_blocks, - ..BootstrapBackfillOutcome::default() - }; - - for task in intake_chain_tasks { - let skipped_unknown_start_targets = - load_manifest_skipped_bootstrap_targets(pool, &task.chain).await?; - outcome.skipped_unknown_start_target_count += skipped_unknown_start_targets.len(); - for skipped_target in &skipped_unknown_start_targets { - info!( - service = "indexer", - command = "run", - bootstrap_backfill_status = "skipped_unknown_start_target", - chain = %task.chain, - source_family = %skipped_target.source_family, - contract_instance_id = %skipped_target.contract_instance_id, - address = %skipped_target.address, - skip_reason = %skipped_target.skip_reason, - "manifest-declared bootstrap target is skipped because it has no declared start block" - ); - } - outcome - .skipped_unknown_start_targets - .extend(skipped_unknown_start_targets); - - let Some(provider) = provider_registry.provider_for(&task.chain) else { - outcome.missing_provider_chain_count += 1; - warn!( - service = "indexer", - command = "run", - bootstrap_backfill_status = "idle_missing_provider", - chain = %task.chain, - intake_address_count = task.addresses.len(), - "no provider source is configured for an active bootstrap chain; automatic bootstrap backfill will stay idle for this chain" - ); - continue; - }; - outcome.provider_configured_chain_count += 1; - - let heads = provider.fetch_chain_heads().await.with_context(|| { - format!( - "failed to fetch provider heads for bootstrap backfill on chain {}", - task.chain - ) - })?; - let provider_finalized_head_block = bootstrap_finalized_head_block(&task.chain, &heads)?; - outcome.latched_finalized_heads.insert( - task.chain.clone(), - heads - .finalized - .clone() - .expect("validated bootstrap heads must include finalized"), - ); - let mut convergence_tracker = BootstrapConvergenceTracker::default(); - loop { - let retention_snapshot = - load_bootstrap_retention_snapshot(pool, &task.chain, provider_finalized_head_block) - .await?; - let mut bootstrap_targets = - load_manifest_declared_bootstrap_targets(pool, &task.chain).await?; - let discovery_targets = load_ens_v2_authoritative_discovery_bootstrap_targets( - pool, - &task.chain, - provider_finalized_head_block, - ) - .await?; - let include_historical_bootstrap_targets = !discovery_targets.is_empty() - || retention_snapshot.requires_ens_v2_history_recovery; - bootstrap_targets.extend(discovery_targets); - if retention_snapshot.requires_ens_v2_history_recovery { - bootstrap_targets.extend( - load_ens_v2_retained_history_recovery_targets( - pool, - &task.chain, - provider_finalized_head_block, - ) - .await?, - ); - bootstrap_targets.sort(); - bootstrap_targets.dedup(); - } - let eligible_target_count = bootstrap_targets.len(); - let mut skipped_future_target_count = 0_usize; - - info!( - service = "indexer", - command = "run", - bootstrap_backfill_status = "planning", - chain = %task.chain, - provider_finalized_head_block, - bootstrap_backfill_range_policy = "authoritative_known_start_to_provider_finalized_head", - hash_pinned_chunk_blocks, - bootstrap_backfill_workers = requested_worker_count, - effective_bootstrap_backfill_workers = effective_worker_count, - bootstrap_backfill_range_blocks, - eligible_bootstrap_target_count = eligible_target_count, - raw_log_retention_generation = retention_snapshot.generation, - ens_v2_retained_history_recovery = retention_snapshot - .requires_ens_v2_history_recovery, - skipped_unknown_start_target_count = outcome.skipped_unknown_start_target_count, - "automatic bootstrap targets loaded" - ); - - let mut target_ranges = Vec::new(); - for target in bootstrap_targets { - let Some(range) = bootstrap_target_range(&target, provider_finalized_head_block)? - else { - skipped_future_target_count += 1; - info!( - service = "indexer", - command = "run", - bootstrap_backfill_status = "skipped_future_target", - chain = %task.chain, - source_family = %target.source_family, - contract_instance_id = %target.contract_instance_id, - address = %target.address, - effective_from_block = target.effective_from_block, - effective_to_block = target.effective_to_block, - provider_finalized_head_block, - bootstrap_backfill_range_policy = "authoritative_known_start_to_provider_finalized_head", - "automatic bootstrap target starts after the provider finalized bootstrap head" - ); - continue; - }; - - let mut range = range; - let mut checkpoint_source_plan = load_bootstrap_source_plan( - pool, - &task.chain, - std::slice::from_ref(&target), - range, - include_historical_bootstrap_targets, - ) - .await - .with_context(|| { - format!( - "failed to build bootstrap watched-target checkpoint source plan for chain {} target {} range {}..={}", - task.chain, - target.contract_instance_id, - range.from_block, - range.to_block - ) - })?; - narrow_manifest_bootstrap_source_plan( - &mut checkpoint_source_plan, - std::slice::from_ref(&target), - range, - )?; - let checkpoint_source_identity = - backfill_job_source_identity_payload(&checkpoint_source_plan)?; - if let Some(stored_checkpoint) = load_bootstrap_target_checkpoint( - pool, - &deployment_profile, - &task.chain, - &checkpoint_source_identity, - range, - &target.contract_instance_id.to_string(), - retention_snapshot.generation, - ) - .await? - { - if stored_checkpoint >= range.to_block { - info!( - service = "indexer", - command = "run", - bootstrap_backfill_status = "skipped_target_stored_checkpoint", - chain = %task.chain, - source_family = %target.source_family, - contract_instance_id = %target.contract_instance_id, - address = %target.address, - from_block = range.from_block, - to_block = range.to_block, - stored_checkpoint_block = stored_checkpoint, - "automatic bootstrap target already has stored backfill checkpoint coverage" - ); - continue; - } - if stored_checkpoint >= range.from_block { - let resumed_from_block = stored_checkpoint.checked_add(1).with_context(|| { - format!( - "stored bootstrap checkpoint {stored_checkpoint} overflowed while resuming target" - ) - })?; - info!( - service = "indexer", - command = "run", - bootstrap_backfill_status = "resuming_target_after_stored_checkpoint", - chain = %task.chain, - source_family = %target.source_family, - contract_instance_id = %target.contract_instance_id, - address = %target.address, - from_block = range.from_block, - resumed_from_block, - to_block = range.to_block, - stored_checkpoint_block = stored_checkpoint, - "automatic bootstrap target resumes after stored backfill checkpoint" - ); - range = BackfillBlockRange::new(resumed_from_block, range.to_block)?; - } - } - - target_ranges.push(BootstrapBackfillTargetRange { target, range }); - } - - target_ranges.sort_by(|left, right| { - ( - &left.target.source_family, - left.target.contract_instance_id, - &left.target.address, - left.range.from_block, - left.range.to_block, - ) - .cmp(&( - &right.target.source_family, - right.target.contract_instance_id, - &right.target.address, - right.range.from_block, - right.range.to_block, - )) - }); - target_ranges.dedup_by(|left, right| { - left.target.source_family == right.target.source_family - && left.target.contract_instance_id == right.target.contract_instance_id - && left.target.address == right.target.address - && left.range == right.range - }); - - for segment in plan_bootstrap_backfill_segments(target_ranges)? { - let segment_target_ids = bootstrap_segment_target_ids(&segment.targets); - let mut checkpoint_source_plan = load_bootstrap_source_plan( - pool, - &task.chain, - &segment.targets, - segment.range, - include_historical_bootstrap_targets, - ) - .await - .with_context(|| { - format!( - "failed to build bootstrap segment checkpoint source plan for chain {} range {}..={}", - task.chain, segment.range.from_block, segment.range.to_block - ) - })?; - narrow_manifest_bootstrap_source_plan( - &mut checkpoint_source_plan, - &segment.targets, - segment.range, - )?; - let checkpoint_source_identity = - backfill_job_source_identity_payload(&checkpoint_source_plan)?; - let mut segment_range = segment.range; - if let Some(stored_checkpoint) = load_bootstrap_segment_checkpoint( - pool, - &deployment_profile, - &task.chain, - &checkpoint_source_identity, - segment.range, - &segment_target_ids, - retention_snapshot.generation, - ) - .await? - { - if stored_checkpoint >= segment_range.to_block { - info!( - service = "indexer", - command = "run", - bootstrap_backfill_status = "skipped_stored_checkpoint", - chain = %task.chain, - from_block = segment.range.from_block, - to_block = segment.range.to_block, - stored_checkpoint_block = stored_checkpoint, - selected_target_count = segment.targets.len(), - "automatic bootstrap segment already has stored backfill checkpoint coverage" - ); - continue; - } - if stored_checkpoint >= segment_range.from_block { - let resumed_from_block = stored_checkpoint.checked_add(1).with_context(|| { - format!( - "stored bootstrap checkpoint {stored_checkpoint} overflowed while resuming" - ) - })?; - info!( - service = "indexer", - command = "run", - bootstrap_backfill_status = "resuming_after_stored_checkpoint", - chain = %task.chain, - from_block = segment.range.from_block, - resumed_from_block, - to_block = segment.range.to_block, - stored_checkpoint_block = stored_checkpoint, - selected_target_count = segment.targets.len(), - "automatic bootstrap segment resumes after stored backfill checkpoint" - ); - segment_range = - BackfillBlockRange::new(resumed_from_block, segment_range.to_block)?; - } - } - - let mut source_plan = load_bootstrap_source_plan( - pool, - &task.chain, - &segment.targets, - segment_range, - include_historical_bootstrap_targets, - ) - .await - .with_context(|| { - format!( - "failed to build bootstrap watched-target source plan for chain {} range {}..={}", - task.chain, segment_range.from_block, segment_range.to_block - ) - })?; - narrow_manifest_bootstrap_source_plan( - &mut source_plan, - &segment.targets, - segment_range, - )?; - - let source_identity_hash = source_identity_hash_for_backfill(&source_plan)?; - let range_specs = hash_pinned_backfill_range_specs( - segment_range, - bootstrap_backfill_range_blocks, - )?; - let idempotency_key = if range_specs.len() == 1 { - bootstrap_backfill_idempotency_key( - &deployment_profile, - &task.chain, - &source_identity_hash, - segment_range, - ) - } else { - partitioned_bootstrap_backfill_idempotency_key( - &deployment_profile, - &task.chain, - &source_identity_hash, - segment_range, - bootstrap_backfill_range_blocks, - ) - }; - let config = crate::backfill::BackfillJobRunConfig { - deployment_profile: deployment_profile.clone(), - idempotency_key, - scope_idempotency_to_raw_log_retention_generation: true, - range: segment_range, - lease_owner: lease_owner.clone(), - lease_token: generated_backfill_lease_token()?, - lease_expires_at: backfill_lease_expires_at( - BOOTSTRAP_BACKFILL_LEASE_DURATION_SECS, - )?, - hash_pinned_chunk_blocks, - adapter_sync_mode: backfill_adapter_sync_mode, - header_audit_mode, - }; - - let job_outcome = if let Some(heartbeat) = heartbeat.as_deref_mut() { - run_resumable_hash_pinned_backfill_job_concurrently_with_heartbeat( - pool, - &source_plan, - provider, - config, - range_specs, - effective_worker_count, - heartbeat, - &heartbeat_chain_ids, - ) - .await? - } else { - run_resumable_hash_pinned_backfill_job_concurrently( - pool, - &source_plan, - provider, - config, - range_specs, - effective_worker_count, - ) - .await? - }; - outcome.add_job(&job_outcome); - if replay_completed_raw_ranges && job_outcome.raw_log_count > 0 { - let replay_outcome = replay_raw_fact_normalized_events( - pool, - RawFactNormalizedEventReplayRequest { - deployment_profile: deployment_profile.clone(), - chain: task.chain.clone(), - selection: RawFactNormalizedEventReplaySelection::ScopedBlockRange { - from_block: job_outcome.from_block, - to_block: job_outcome.to_block, - source_scope: replay_source_scope_from_source_plan( - &source_plan, - job_outcome.from_block, - job_outcome.to_block, - ), - }, - }, - ) - .await - .with_context(|| { - format!( - "failed to replay normalized events after bootstrap raw backfill for chain {} range {}..={}", - task.chain, job_outcome.from_block, job_outcome.to_block - ) - })?; - log_raw_fact_normalized_event_replay_outcome(&replay_outcome); - outcome.normalized_replay_job_count += 1; - outcome.normalized_replay_synced_count += - replay_outcome.normalized_event_synced_count; - outcome.normalized_replay_inserted_count += - replay_outcome.normalized_event_inserted_count; - } - record_bootstrap_progress(pool, &mut heartbeat, &heartbeat_chain_ids).await?; - } - - let pass_status = finish_bootstrap_convergence_pass( - pool, - &task.chain, - provider_finalized_head_block, - retention_snapshot, - adapter_sync_mode, - ) - .await?; - if pass_status == BootstrapPassStatus::Stable { - outcome.eligible_target_count += eligible_target_count; - outcome.skipped_future_target_count += skipped_future_target_count; - break; - } - convergence_tracker - .record_retry( - pool, - &task.chain, - provider_finalized_head_block, - pass_status, - ) - .await?; - warn!( - service = "indexer", - command = "run", - bootstrap_backfill_status = "retry_retention_authority_changed", - chain = %task.chain, - planned_raw_log_retention_generation = retention_snapshot.generation, - planned_discovery_admission_epoch = retention_snapshot.discovery_admission_epoch, - pass_status = ?pass_status, - "raw-log retention or ENSv2 discovery authority changed during automatic bootstrap; retrying the complete chain planning pass" - ); - } - } - - info!( - service = "indexer", - command = "run", - bootstrap_backfill_status = "drained", - active_chain_count = outcome.active_chain_count, - provider_configured_chain_count = outcome.provider_configured_chain_count, - missing_provider_chain_count = outcome.missing_provider_chain_count, - eligible_bootstrap_target_count = outcome.eligible_target_count, - skipped_unknown_start_target_count = outcome.skipped_unknown_start_target_count, - drained_bootstrap_job_count = outcome.drained_job_count, - skipped_future_target_count = outcome.skipped_future_target_count, - bootstrap_backfill_range_policy = "authoritative_known_start_to_provider_finalized_head", - hash_pinned_chunk_blocks, - bootstrap_backfill_workers = outcome.requested_worker_count, - effective_bootstrap_backfill_workers = outcome.effective_worker_count, - bootstrap_backfill_range_blocks = outcome.range_partition_block_count, - reserved_range_count = outcome.reserved_range_count, - completed_range_count = outcome.completed_range_count, - resolved_block_count = outcome.resolved_block_count, - raw_block_count = outcome.raw_block_count, - raw_transaction_count = outcome.raw_transaction_count, - raw_receipt_count = outcome.raw_receipt_count, - raw_log_count = outcome.raw_log_count, - raw_code_hash_count = outcome.raw_code_hash_count, - normalized_replay_job_count = outcome.normalized_replay_job_count, - normalized_replay_synced_count = outcome.normalized_replay_synced_count, - normalized_replay_inserted_count = outcome.normalized_replay_inserted_count, - "startup bootstrap backfill jobs drained before live polling" - ); - - Ok(outcome) -} - -async fn record_bootstrap_progress( - pool: &sqlx::PgPool, - heartbeat: &mut Option<&mut StartupHeartbeat>, - chain_ids: &[String], -) -> Result<()> { - if let Some(heartbeat) = heartbeat.as_deref_mut() { - heartbeat.record_if_due(pool, chain_ids).await?; - } - Ok(()) -} diff --git a/apps/indexer/src/main/bootstrap_backfill/checkpoints.rs b/apps/indexer/src/main/bootstrap_backfill/checkpoints.rs index 5126a4e3..c6201a6b 100644 --- a/apps/indexer/src/main/bootstrap_backfill/checkpoints.rs +++ b/apps/indexer/src/main/bootstrap_backfill/checkpoints.rs @@ -1,7 +1,6 @@ use std::collections::BTreeSet; use anyhow::{Context, Result}; -use bigname_manifests::ManifestBootstrapTarget; use serde_json::Value; use sqlx::Row; @@ -142,15 +141,6 @@ pub(super) async fn load_bootstrap_target_checkpoint( ) } -pub(super) fn bootstrap_segment_target_ids( - targets: &[ManifestBootstrapTarget], -) -> BTreeSet { - targets - .iter() - .map(|target| target.contract_instance_id.to_string()) - .collect() -} - fn source_identity_requested_target_ids(source_identity: &Value) -> Option> { let requested_targets = source_identity .get("requested_watched_targets") diff --git a/apps/indexer/src/main/bootstrap_backfill/identity.rs b/apps/indexer/src/main/bootstrap_backfill/identity.rs index 9587c43c..04c4b2fb 100644 --- a/apps/indexer/src/main/bootstrap_backfill/identity.rs +++ b/apps/indexer/src/main/bootstrap_backfill/identity.rs @@ -1,8 +1,5 @@ -use anyhow::{Context, Result}; - use crate::{ - backfill::{BackfillBlockRange, backfill_job_source_identity_payload}, - reconciliation::RawFactNormalizedEventReplaySourceScope, + backfill::BackfillBlockRange, reconciliation::RawFactNormalizedEventReplaySourceScope, source_scope::SourceScope, }; @@ -14,17 +11,6 @@ pub(super) fn replay_source_scope_from_source_plan( SourceScope::from_watched_source_plan(source_plan, from_block, to_block).into_targets() } -pub(super) fn source_identity_hash_for_backfill( - source_plan: &bigname_manifests::WatchedSourceSelectorPlan, -) -> Result { - let payload = backfill_job_source_identity_payload(source_plan)?; - payload - .get("source_identity_hash") - .and_then(serde_json::Value::as_str) - .map(ToOwned::to_owned) - .context("backfill source identity payload is missing source_identity_hash") -} - pub(crate) fn bootstrap_backfill_idempotency_key( deployment_profile: &str, chain: &str, diff --git a/apps/indexer/src/main/bootstrap_backfill/logging.rs b/apps/indexer/src/main/bootstrap_backfill/logging.rs new file mode 100644 index 00000000..f0e2e4f7 --- /dev/null +++ b/apps/indexer/src/main/bootstrap_backfill/logging.rs @@ -0,0 +1,38 @@ +use tracing::info; + +use super::entrypoints::BootstrapBackfillOutcome; + +pub(super) fn log_bootstrap_backfill_outcome( + outcome: &BootstrapBackfillOutcome, + hash_pinned_chunk_blocks: i64, +) { + info!( + service = "indexer", + command = "run", + bootstrap_backfill_status = "drained", + active_chain_count = outcome.active_chain_count, + provider_configured_chain_count = outcome.provider_configured_chain_count, + missing_provider_chain_count = outcome.missing_provider_chain_count, + eligible_bootstrap_target_count = outcome.eligible_target_count, + skipped_unknown_start_target_count = outcome.skipped_unknown_start_target_count, + drained_bootstrap_job_count = outcome.drained_job_count, + skipped_future_target_count = outcome.skipped_future_target_count, + bootstrap_backfill_range_policy = "authoritative_known_start_to_provider_finalized_head", + hash_pinned_chunk_blocks, + bootstrap_backfill_workers = outcome.requested_worker_count, + effective_bootstrap_backfill_workers = outcome.effective_worker_count, + bootstrap_backfill_range_blocks = outcome.range_partition_block_count, + reserved_range_count = outcome.reserved_range_count, + completed_range_count = outcome.completed_range_count, + resolved_block_count = outcome.resolved_block_count, + raw_block_count = outcome.raw_block_count, + raw_transaction_count = outcome.raw_transaction_count, + raw_receipt_count = outcome.raw_receipt_count, + raw_log_count = outcome.raw_log_count, + raw_code_hash_count = outcome.raw_code_hash_count, + normalized_replay_job_count = outcome.normalized_replay_job_count, + normalized_replay_synced_count = outcome.normalized_replay_synced_count, + normalized_replay_inserted_count = outcome.normalized_replay_inserted_count, + "startup bootstrap backfill jobs drained before live polling" + ); +} diff --git a/apps/indexer/src/main/bootstrap_backfill/orchestration.rs b/apps/indexer/src/main/bootstrap_backfill/orchestration.rs new file mode 100644 index 00000000..c27f5ed1 --- /dev/null +++ b/apps/indexer/src/main/bootstrap_backfill/orchestration.rs @@ -0,0 +1,591 @@ +use super::*; + +#[expect(clippy::too_many_arguments)] +pub(super) async fn run_startup_bootstrap_backfills_inner( + pool: &sqlx::PgPool, + manifests_root: &Path, + intake_chain_tasks: &[IntakeChainTask], + provider_registry: &ProviderRegistry, + hash_pinned_chunk_blocks: i64, + adapter_sync_mode: BackfillAdapterSyncMode, + replay_completed_raw_ranges: bool, + header_audit_mode: HeaderAuditMode, + bootstrap_backfill_workers: usize, + bootstrap_backfill_range_blocks: i64, + mut heartbeat: Option<&mut StartupHeartbeat>, +) -> Result { + validate_provider_registry_for_intake_tasks(intake_chain_tasks, provider_registry)?; + let heartbeat_chain_ids = intake_chain_tasks + .iter() + .map(|task| task.chain.clone()) + .collect::>(); + record_bootstrap_progress(pool, &mut heartbeat, &heartbeat_chain_ids).await?; + let backfill_adapter_sync_mode = adapter_sync_mode.startup_hash_pinned_backfill_mode(); + let deployment_profile = deployment_profile_from_manifest_root(manifests_root); + let lease_owner = format!("{}:bootstrap-backfill", default_backfill_lease_owner()); + let requested_worker_count = + resolve_bootstrap_backfill_worker_count(bootstrap_backfill_workers); + let effective_worker_count = effective_bootstrap_backfill_worker_count( + requested_worker_count, + backfill_adapter_sync_mode, + ); + let mut outcome = BootstrapBackfillOutcome { + active_chain_count: intake_chain_tasks.len(), + requested_worker_count, + effective_worker_count, + range_partition_block_count: bootstrap_backfill_range_blocks, + ..BootstrapBackfillOutcome::default() + }; + + for task in intake_chain_tasks { + let skipped_unknown_start_targets = + load_manifest_skipped_bootstrap_targets(pool, &task.chain).await?; + outcome.skipped_unknown_start_target_count += skipped_unknown_start_targets.len(); + for skipped_target in &skipped_unknown_start_targets { + info!( + service = "indexer", + command = "run", + bootstrap_backfill_status = "skipped_unknown_start_target", + chain = %task.chain, + source_family = %skipped_target.source_family, + contract_instance_id = %skipped_target.contract_instance_id, + address = %skipped_target.address, + skip_reason = %skipped_target.skip_reason, + "manifest-declared bootstrap target is skipped because it has no declared start block" + ); + } + outcome + .skipped_unknown_start_targets + .extend(skipped_unknown_start_targets); + + let Some(provider) = provider_registry.provider_for(&task.chain) else { + outcome.missing_provider_chain_count += 1; + warn!( + service = "indexer", + command = "run", + bootstrap_backfill_status = "idle_missing_provider", + chain = %task.chain, + intake_address_count = task.addresses.len(), + "no provider source is configured for an active bootstrap chain; automatic bootstrap backfill will stay idle for this chain" + ); + continue; + }; + outcome.provider_configured_chain_count += 1; + + let heads = provider.fetch_chain_heads().await.with_context(|| { + format!( + "failed to fetch provider heads for bootstrap backfill on chain {}", + task.chain + ) + })?; + let provider_finalized_head_block = bootstrap_finalized_head_block(&task.chain, &heads)?; + record_bootstrap_progress(pool, &mut heartbeat, &heartbeat_chain_ids).await?; + outcome.latched_finalized_heads.insert( + task.chain.clone(), + heads + .finalized + .clone() + .expect("validated bootstrap heads must include finalized"), + ); + let mut convergence_tracker = BootstrapConvergenceTracker::default(); + loop { + let retention_snapshot = match heartbeat.as_deref_mut() { + Some(heartbeat) => { + let mut progress = + StartupAdapterHeartbeat::new(heartbeat, &heartbeat_chain_ids); + load_bootstrap_retention_snapshot_with_progress( + pool, + &task.chain, + provider_finalized_head_block, + &mut progress, + ) + .await? + } + None => { + load_bootstrap_retention_snapshot( + pool, + &task.chain, + provider_finalized_head_block, + ) + .await? + } + }; + record_bootstrap_progress(pool, &mut heartbeat, &heartbeat_chain_ids).await?; + let mut bootstrap_targets = load_manifest_declared_bootstrap_targets(pool, &task.chain) + .await? + .into_iter() + .collect::>(); + let discovery_targets = load_discovery_bootstrap_targets_with_optional_progress( + pool, + &task.chain, + provider_finalized_head_block, + &mut heartbeat, + &heartbeat_chain_ids, + ) + .await?; + let include_historical_bootstrap_targets = !discovery_targets.is_empty() + || retention_snapshot.requires_ens_v2_history_recovery; + extend_bootstrap_targets_with_progress( + pool, + &mut bootstrap_targets, + discovery_targets, + &mut heartbeat, + &heartbeat_chain_ids, + ) + .await?; + if retention_snapshot.requires_ens_v2_history_recovery { + let recovery_targets = load_retained_recovery_targets_with_optional_progress( + pool, + &task.chain, + provider_finalized_head_block, + &mut heartbeat, + &heartbeat_chain_ids, + ) + .await?; + extend_bootstrap_targets_with_progress( + pool, + &mut bootstrap_targets, + recovery_targets, + &mut heartbeat, + &heartbeat_chain_ids, + ) + .await?; + } + let eligible_target_count = bootstrap_targets.len(); + let mut skipped_future_target_count = 0_usize; + + info!( + service = "indexer", + command = "run", + bootstrap_backfill_status = "planning", + chain = %task.chain, + provider_finalized_head_block, + bootstrap_backfill_range_policy = "authoritative_known_start_to_provider_finalized_head", + hash_pinned_chunk_blocks, + bootstrap_backfill_workers = requested_worker_count, + effective_bootstrap_backfill_workers = effective_worker_count, + bootstrap_backfill_range_blocks, + eligible_bootstrap_target_count = eligible_target_count, + raw_log_retention_generation = retention_snapshot.generation, + ens_v2_retained_history_recovery = retention_snapshot + .requires_ens_v2_history_recovery, + skipped_unknown_start_target_count = outcome.skipped_unknown_start_target_count, + "automatic bootstrap targets loaded" + ); + + let mut target_ranges = Vec::new(); + for target in bootstrap_targets { + let Some(range) = bootstrap_target_range(&target, provider_finalized_head_block)? + else { + skipped_future_target_count += 1; + info!( + service = "indexer", + command = "run", + bootstrap_backfill_status = "skipped_future_target", + chain = %task.chain, + source_family = %target.source_family, + contract_instance_id = %target.contract_instance_id, + address = %target.address, + effective_from_block = target.effective_from_block, + effective_to_block = target.effective_to_block, + provider_finalized_head_block, + bootstrap_backfill_range_policy = "authoritative_known_start_to_provider_finalized_head", + "automatic bootstrap target starts after the provider finalized bootstrap head" + ); + record_bootstrap_progress(pool, &mut heartbeat, &heartbeat_chain_ids).await?; + continue; + }; + + let mut range = range; + let mut checkpoint_source_plan = load_bootstrap_source_plan_with_optional_progress( + pool, + &task.chain, + std::slice::from_ref(&target), + range, + include_historical_bootstrap_targets, + &mut heartbeat, + &heartbeat_chain_ids, + ) + .await + .with_context(|| { + format!( + "failed to build bootstrap watched-target checkpoint source plan for chain {} target {} range {}..={}", + task.chain, + target.contract_instance_id, + range.from_block, + range.to_block + ) + })?; + narrow_bootstrap_source_plan_with_optional_progress( + pool, + &mut checkpoint_source_plan, + std::slice::from_ref(&target), + range, + &mut heartbeat, + &heartbeat_chain_ids, + ) + .await?; + let checkpoint_source_identity = bootstrap_source_identity_with_optional_progress( + pool, + &checkpoint_source_plan, + &mut heartbeat, + &heartbeat_chain_ids, + ) + .await?; + if let Some(stored_checkpoint) = load_bootstrap_target_checkpoint( + pool, + &deployment_profile, + &task.chain, + &checkpoint_source_identity, + range, + &target.contract_instance_id.to_string(), + retention_snapshot.generation, + ) + .await? + { + if stored_checkpoint >= range.to_block { + info!( + service = "indexer", + command = "run", + bootstrap_backfill_status = "skipped_target_stored_checkpoint", + chain = %task.chain, + source_family = %target.source_family, + contract_instance_id = %target.contract_instance_id, + address = %target.address, + from_block = range.from_block, + to_block = range.to_block, + stored_checkpoint_block = stored_checkpoint, + "automatic bootstrap target already has stored backfill checkpoint coverage" + ); + record_bootstrap_progress(pool, &mut heartbeat, &heartbeat_chain_ids) + .await?; + continue; + } + if stored_checkpoint >= range.from_block { + let resumed_from_block = stored_checkpoint.checked_add(1).with_context(|| { + format!( + "stored bootstrap checkpoint {stored_checkpoint} overflowed while resuming target" + ) + })?; + info!( + service = "indexer", + command = "run", + bootstrap_backfill_status = "resuming_target_after_stored_checkpoint", + chain = %task.chain, + source_family = %target.source_family, + contract_instance_id = %target.contract_instance_id, + address = %target.address, + from_block = range.from_block, + resumed_from_block, + to_block = range.to_block, + stored_checkpoint_block = stored_checkpoint, + "automatic bootstrap target resumes after stored backfill checkpoint" + ); + range = BackfillBlockRange::new(resumed_from_block, range.to_block)?; + } + } + + target_ranges.push(BootstrapBackfillTargetRange { target, range }); + record_bootstrap_progress(pool, &mut heartbeat, &heartbeat_chain_ids).await?; + } + + let segments = plan_bootstrap_segments_with_optional_progress( + pool, + target_ranges, + &mut heartbeat, + &heartbeat_chain_ids, + ) + .await?; + for segment in segments { + let segment_target_ids = bootstrap_segment_target_ids_with_optional_progress( + pool, + &segment.targets, + &mut heartbeat, + &heartbeat_chain_ids, + ) + .await?; + let mut checkpoint_source_plan = load_bootstrap_source_plan_with_optional_progress( + pool, + &task.chain, + &segment.targets, + segment.range, + include_historical_bootstrap_targets, + &mut heartbeat, + &heartbeat_chain_ids, + ) + .await + .with_context(|| { + format!( + "failed to build bootstrap segment checkpoint source plan for chain {} range {}..={}", + task.chain, segment.range.from_block, segment.range.to_block + ) + })?; + narrow_bootstrap_source_plan_with_optional_progress( + pool, + &mut checkpoint_source_plan, + &segment.targets, + segment.range, + &mut heartbeat, + &heartbeat_chain_ids, + ) + .await?; + let checkpoint_source_identity = bootstrap_source_identity_with_optional_progress( + pool, + &checkpoint_source_plan, + &mut heartbeat, + &heartbeat_chain_ids, + ) + .await?; + let mut segment_range = segment.range; + if let Some(stored_checkpoint) = load_bootstrap_segment_checkpoint( + pool, + &deployment_profile, + &task.chain, + &checkpoint_source_identity, + segment.range, + &segment_target_ids, + retention_snapshot.generation, + ) + .await? + { + if stored_checkpoint >= segment_range.to_block { + info!( + service = "indexer", + command = "run", + bootstrap_backfill_status = "skipped_stored_checkpoint", + chain = %task.chain, + from_block = segment.range.from_block, + to_block = segment.range.to_block, + stored_checkpoint_block = stored_checkpoint, + selected_target_count = segment.targets.len(), + "automatic bootstrap segment already has stored backfill checkpoint coverage" + ); + continue; + } + if stored_checkpoint >= segment_range.from_block { + let resumed_from_block = stored_checkpoint.checked_add(1).with_context(|| { + format!( + "stored bootstrap checkpoint {stored_checkpoint} overflowed while resuming" + ) + })?; + info!( + service = "indexer", + command = "run", + bootstrap_backfill_status = "resuming_after_stored_checkpoint", + chain = %task.chain, + from_block = segment.range.from_block, + resumed_from_block, + to_block = segment.range.to_block, + stored_checkpoint_block = stored_checkpoint, + selected_target_count = segment.targets.len(), + "automatic bootstrap segment resumes after stored backfill checkpoint" + ); + segment_range = + BackfillBlockRange::new(resumed_from_block, segment_range.to_block)?; + } + } + + let mut source_plan = load_bootstrap_source_plan_with_optional_progress( + pool, + &task.chain, + &segment.targets, + segment_range, + include_historical_bootstrap_targets, + &mut heartbeat, + &heartbeat_chain_ids, + ) + .await + .with_context(|| { + format!( + "failed to build bootstrap watched-target source plan for chain {} range {}..={}", + task.chain, segment_range.from_block, segment_range.to_block + ) + })?; + narrow_bootstrap_source_plan_with_optional_progress( + pool, + &mut source_plan, + &segment.targets, + segment_range, + &mut heartbeat, + &heartbeat_chain_ids, + ) + .await?; + + let source_identity = bootstrap_source_identity_with_optional_progress( + pool, + &source_plan, + &mut heartbeat, + &heartbeat_chain_ids, + ) + .await?; + let source_identity_hash = source_identity + .get("source_identity_hash") + .and_then(serde_json::Value::as_str) + .map(ToOwned::to_owned) + .context("backfill source identity payload is missing source_identity_hash")?; + let range_specs = hash_pinned_backfill_range_specs( + segment_range, + bootstrap_backfill_range_blocks, + )?; + let idempotency_key = if range_specs.len() == 1 { + bootstrap_backfill_idempotency_key( + &deployment_profile, + &task.chain, + &source_identity_hash, + segment_range, + ) + } else { + partitioned_bootstrap_backfill_idempotency_key( + &deployment_profile, + &task.chain, + &source_identity_hash, + segment_range, + bootstrap_backfill_range_blocks, + ) + }; + let config = crate::backfill::BackfillJobRunConfig { + deployment_profile: deployment_profile.clone(), + idempotency_key, + scope_idempotency_to_raw_log_retention_generation: true, + range: segment_range, + lease_owner: lease_owner.clone(), + lease_token: generated_backfill_lease_token()?, + lease_expires_at: backfill_lease_expires_at( + BOOTSTRAP_BACKFILL_LEASE_DURATION_SECS, + )?, + hash_pinned_chunk_blocks, + adapter_sync_mode: backfill_adapter_sync_mode, + header_audit_mode, + }; + + let job_outcome = if let Some(heartbeat) = heartbeat.as_deref_mut() { + run_resumable_hash_pinned_backfill_job_concurrently_with_heartbeat( + pool, + &source_plan, + provider, + config, + range_specs, + effective_worker_count, + heartbeat, + &heartbeat_chain_ids, + ) + .await? + } else { + run_resumable_hash_pinned_backfill_job_concurrently( + pool, + &source_plan, + provider, + config, + range_specs, + effective_worker_count, + ) + .await? + }; + outcome.add_job(&job_outcome); + if replay_completed_raw_ranges && job_outcome.raw_log_count > 0 { + let replay_outcome = replay_raw_fact_normalized_events( + pool, + RawFactNormalizedEventReplayRequest { + deployment_profile: deployment_profile.clone(), + chain: task.chain.clone(), + selection: RawFactNormalizedEventReplaySelection::ScopedBlockRange { + from_block: job_outcome.from_block, + to_block: job_outcome.to_block, + source_scope: replay_source_scope_from_source_plan( + &source_plan, + job_outcome.from_block, + job_outcome.to_block, + ), + }, + }, + ) + .await + .with_context(|| { + format!( + "failed to replay normalized events after bootstrap raw backfill for chain {} range {}..={}", + task.chain, job_outcome.from_block, job_outcome.to_block + ) + })?; + log_raw_fact_normalized_event_replay_outcome(&replay_outcome); + outcome.normalized_replay_job_count += 1; + outcome.normalized_replay_synced_count += + replay_outcome.normalized_event_synced_count; + outcome.normalized_replay_inserted_count += + replay_outcome.normalized_event_inserted_count; + } + record_bootstrap_progress(pool, &mut heartbeat, &heartbeat_chain_ids).await?; + } + + let pass_status = match heartbeat.as_deref_mut() { + Some(heartbeat) => { + let mut progress = + StartupAdapterHeartbeat::new(heartbeat, &heartbeat_chain_ids); + finish_bootstrap_convergence_pass_with_progress( + pool, + &task.chain, + provider_finalized_head_block, + retention_snapshot, + adapter_sync_mode, + &mut progress, + ) + .await? + } + None => { + finish_bootstrap_convergence_pass( + pool, + &task.chain, + provider_finalized_head_block, + retention_snapshot, + adapter_sync_mode, + ) + .await? + } + }; + if pass_status == BootstrapPassStatus::Stable { + outcome.eligible_target_count += eligible_target_count; + outcome.skipped_future_target_count += skipped_future_target_count; + break; + } + match heartbeat.as_deref_mut() { + Some(heartbeat) => { + let mut progress = + StartupAdapterHeartbeat::new(heartbeat, &heartbeat_chain_ids); + convergence_tracker + .record_retry( + pool, + &task.chain, + provider_finalized_head_block, + pass_status, + Some(&mut progress), + ) + .await?; + } + None => { + convergence_tracker + .record_retry( + pool, + &task.chain, + provider_finalized_head_block, + pass_status, + None, + ) + .await?; + } + } + warn!( + service = "indexer", + command = "run", + bootstrap_backfill_status = "retry_retention_authority_changed", + chain = %task.chain, + planned_raw_log_retention_generation = retention_snapshot.generation, + planned_discovery_admission_epoch = retention_snapshot.discovery_admission_epoch, + pass_status = ?pass_status, + "raw-log retention or ENSv2 discovery authority changed during automatic bootstrap; retrying the complete chain planning pass" + ); + } + } + + super::logging::log_bootstrap_backfill_outcome(&outcome, hash_pinned_chunk_blocks); + + Ok(outcome) +} diff --git a/apps/indexer/src/main/bootstrap_backfill/planning.rs b/apps/indexer/src/main/bootstrap_backfill/planning.rs index e6f11295..a4f61f52 100644 --- a/apps/indexer/src/main/bootstrap_backfill/planning.rs +++ b/apps/indexer/src/main/bootstrap_backfill/planning.rs @@ -14,6 +14,13 @@ use crate::{ use super::DEFAULT_BOOTSTRAP_BACKFILL_WORKERS; +#[path = "planning/progress.rs"] +mod progress; +pub(super) use progress::{ + narrow_manifest_bootstrap_source_plan_with_progress, + plan_bootstrap_backfill_segments_with_progress, +}; + const MAX_AUTOMATIC_BOOTSTRAP_BACKFILL_WORKERS: usize = 4; #[derive(Clone, Debug, Eq, PartialEq)] diff --git a/apps/indexer/src/main/bootstrap_backfill/planning/progress.rs b/apps/indexer/src/main/bootstrap_backfill/planning/progress.rs new file mode 100644 index 00000000..c0ea9289 --- /dev/null +++ b/apps/indexer/src/main/bootstrap_backfill/planning/progress.rs @@ -0,0 +1,213 @@ +use std::collections::BTreeSet; + +use anyhow::{Context, Result}; +use bigname_adapters::StartupAdapterProgress; +use bigname_manifests::{ + ManifestBootstrapTarget, WatchedBackfillTarget, WatchedSourceSelectorKind, + WatchedSourceSelectorPlan, +}; + +use crate::{backfill::BackfillBlockRange, ens_v1_resolver::SOURCE_FAMILY_ENS_V1_RESOLVER_L1}; + +use super::{BootstrapBackfillSegment, BootstrapBackfillTargetRange}; + +const PLANNING_PROGRESS_TARGETS: usize = 1_000; + +pub(in crate::bootstrap_backfill) async fn plan_bootstrap_backfill_segments_with_progress( + pool: &sqlx::PgPool, + target_ranges: Vec, + progress: &mut dyn StartupAdapterProgress, +) -> Result> { + let mut max_end_block = None::; + let mut boundaries = BTreeSet::new(); + let mut resolver_range = None::; + for (index, target_range) in target_ranges.iter().enumerate() { + max_end_block = Some( + max_end_block.map_or(target_range.range.to_block, |current| { + current.max(target_range.range.to_block) + }), + ); + if target_range.target.source_family == SOURCE_FAMILY_ENS_V1_RESOLVER_L1 { + resolver_range = Some(match resolver_range { + Some(range) => BackfillBlockRange { + from_block: range.from_block.min(target_range.range.from_block), + to_block: range.to_block.max(target_range.range.to_block), + }, + None => target_range.range, + }); + } else { + boundaries.insert(target_range.range.from_block); + if target_range.range.to_block < i64::MAX { + boundaries.insert(target_range.range.to_block.checked_add(1).with_context( + || { + format!( + "bootstrap target range end {} overflowed while planning segments", + target_range.range.to_block + ) + }, + )?); + } + } + record_every(pool, progress, index + 1).await?; + } + record_tail(pool, progress, target_ranges.len()).await?; + let Some(max_end_block) = max_end_block else { + return Ok(Vec::new()); + }; + + if let Some(resolver_range) = resolver_range { + boundaries.insert(resolver_range.from_block); + if resolver_range.to_block < i64::MAX { + boundaries.insert(resolver_range.to_block.checked_add(1).with_context(|| { + format!( + "bootstrap resolver range end {} overflowed while planning segments", + resolver_range.to_block + ) + })?); + } + } + + let mut boundaries_vec = Vec::with_capacity(boundaries.len()); + for boundary in boundaries { + boundaries_vec.push(boundary); + record_every(pool, progress, boundaries_vec.len()).await?; + } + record_tail(pool, progress, boundaries_vec.len()).await?; + + let mut segments = Vec::new(); + for (index, segment_start) in boundaries_vec.iter().copied().enumerate() { + if segment_start > max_end_block { + break; + } + let segment_end = boundaries_vec + .get(index + 1) + .map(|next_start| *next_start - 1) + .unwrap_or(max_end_block) + .min(max_end_block); + if segment_start > segment_end { + continue; + } + + let mut targets = Vec::new(); + for (target_index, target_range) in target_ranges.iter().enumerate() { + let selected = if target_range.target.source_family == SOURCE_FAMILY_ENS_V1_RESOLVER_L1 + { + target_range.range.from_block <= segment_end + && segment_start <= target_range.range.to_block + } else { + target_range.range.from_block <= segment_start + && segment_end <= target_range.range.to_block + }; + if selected { + targets.push(target_range.target.clone()); + } + record_every(pool, progress, target_index + 1).await?; + } + record_tail(pool, progress, target_ranges.len()).await?; + if !targets.is_empty() { + segments.push(BootstrapBackfillSegment { + range: BackfillBlockRange::new(segment_start, segment_end)?, + targets, + }); + } + } + Ok(segments) +} + +pub(in crate::bootstrap_backfill) async fn narrow_manifest_bootstrap_source_plan_with_progress( + pool: &sqlx::PgPool, + source_plan: &mut WatchedSourceSelectorPlan, + targets: &[ManifestBootstrapTarget], + range: BackfillBlockRange, + progress: &mut dyn StartupAdapterProgress, +) -> Result<()> { + anyhow::ensure!( + source_plan.selector_kind == WatchedSourceSelectorKind::WatchedTargetSet, + "bootstrap source plan for range {}..={} used selector kind {} instead of watched_target_set", + range.from_block, + range.to_block, + source_plan.selector_kind.as_str() + ); + + let mut selected_ids = BTreeSet::new(); + let mut selected_exact = BTreeSet::new(); + for (index, selected) in source_plan.selected_targets.iter().enumerate() { + selected_ids.insert(( + selected.source_family.clone(), + selected.contract_instance_id, + )); + selected_exact.insert(( + selected.source_family.clone(), + selected.contract_instance_id, + selected.address.clone(), + selected.effective_from_block, + selected.effective_to_block, + )); + record_every(pool, progress, index + 1).await?; + } + record_tail(pool, progress, source_plan.selected_targets.len()).await?; + + let mut narrowed = BTreeSet::new(); + for (index, target) in targets.iter().enumerate() { + let is_resolver = target.source_family == SOURCE_FAMILY_ENS_V1_RESOLVER_L1; + let selected = if is_resolver { + selected_ids.contains(&(target.source_family.clone(), target.contract_instance_id)) + } else { + selected_exact.contains(&( + target.source_family.clone(), + target.contract_instance_id, + target.address.clone(), + range.from_block, + range.to_block, + )) + }; + anyhow::ensure!( + selected || is_resolver, + "bootstrap source plan for range {}..={} did not select authoritative contract_instance_id {} with the segmented address/range", + range.from_block, + range.to_block, + target.contract_instance_id + ); + narrowed.insert(WatchedBackfillTarget { + source_family: target.source_family.clone(), + contract_instance_id: target.contract_instance_id, + address: target.address.clone(), + effective_from_block: range.from_block, + effective_to_block: range.to_block, + }); + record_every(pool, progress, index + 1).await?; + } + record_tail(pool, progress, targets.len()).await?; + + let narrowed_len = narrowed.len(); + let mut narrowed_targets = Vec::with_capacity(narrowed_len); + for target in narrowed { + narrowed_targets.push(target); + record_every(pool, progress, narrowed_targets.len()).await?; + } + record_tail(pool, progress, narrowed_len).await?; + source_plan.selected_targets = narrowed_targets; + Ok(()) +} + +async fn record_every( + pool: &sqlx::PgPool, + progress: &mut dyn StartupAdapterProgress, + completed: usize, +) -> Result<()> { + if completed.is_multiple_of(PLANNING_PROGRESS_TARGETS) { + progress.record(pool).await?; + } + Ok(()) +} + +async fn record_tail( + pool: &sqlx::PgPool, + progress: &mut dyn StartupAdapterProgress, + completed: usize, +) -> Result<()> { + if completed > 0 && !completed.is_multiple_of(PLANNING_PROGRESS_TARGETS) { + progress.record(pool).await?; + } + Ok(()) +} diff --git a/apps/indexer/src/main/bootstrap_backfill/progress.rs b/apps/indexer/src/main/bootstrap_backfill/progress.rs new file mode 100644 index 00000000..aa5cfccd --- /dev/null +++ b/apps/indexer/src/main/bootstrap_backfill/progress.rs @@ -0,0 +1,225 @@ +use std::collections::BTreeSet; + +use anyhow::Result; +use bigname_manifests::ManifestBootstrapTarget; + +use crate::{ + backfill::{ + BackfillBlockRange, backfill_job_source_identity_payload, + backfill_job_source_identity_payload_with_progress, + }, + run::startup_heartbeat::{StartupAdapterHeartbeat, StartupHeartbeat}, +}; + +use super::{ + planning::{ + BootstrapBackfillSegment, BootstrapBackfillTargetRange, + narrow_manifest_bootstrap_source_plan, narrow_manifest_bootstrap_source_plan_with_progress, + plan_bootstrap_backfill_segments, plan_bootstrap_backfill_segments_with_progress, + }, + recovery::{load_bootstrap_source_plan, load_bootstrap_source_plan_with_progress}, +}; + +const TARGET_ACCUMULATION_PROGRESS_ROWS: usize = 1_000; + +pub(super) async fn load_discovery_bootstrap_targets_with_optional_progress( + pool: &sqlx::PgPool, + chain: &str, + through_block: i64, + heartbeat: &mut Option<&mut StartupHeartbeat>, + chain_ids: &[String], +) -> Result> { + match heartbeat.as_deref_mut() { + Some(heartbeat) => { + let mut progress = StartupAdapterHeartbeat::new(heartbeat, chain_ids); + bigname_manifests::load_ens_v2_authoritative_discovery_bootstrap_targets_with_progress( + pool, + chain, + through_block, + &mut progress, + ) + .await + } + None => { + bigname_manifests::load_ens_v2_authoritative_discovery_bootstrap_targets( + pool, + chain, + through_block, + ) + .await + } + } +} + +pub(super) async fn bootstrap_source_identity_with_optional_progress( + pool: &sqlx::PgPool, + source_plan: &bigname_manifests::WatchedSourceSelectorPlan, + heartbeat: &mut Option<&mut StartupHeartbeat>, + chain_ids: &[String], +) -> Result { + match heartbeat.as_deref_mut() { + Some(heartbeat) => { + let mut progress = StartupAdapterHeartbeat::new(heartbeat, chain_ids); + backfill_job_source_identity_payload_with_progress(pool, source_plan, &mut progress) + .await + } + None => backfill_job_source_identity_payload(source_plan), + } +} + +pub(super) async fn bootstrap_segment_target_ids_with_optional_progress( + pool: &sqlx::PgPool, + targets: &[ManifestBootstrapTarget], + heartbeat: &mut Option<&mut StartupHeartbeat>, + chain_ids: &[String], +) -> Result> { + let mut target_ids = BTreeSet::new(); + for (index, target) in targets.iter().enumerate() { + target_ids.insert(target.contract_instance_id.to_string()); + if (index + 1).is_multiple_of(TARGET_ACCUMULATION_PROGRESS_ROWS) { + record_bootstrap_progress(pool, heartbeat, chain_ids).await?; + } + } + if !targets.is_empty() + && !targets + .len() + .is_multiple_of(TARGET_ACCUMULATION_PROGRESS_ROWS) + { + record_bootstrap_progress(pool, heartbeat, chain_ids).await?; + } + Ok(target_ids) +} + +pub(super) async fn narrow_bootstrap_source_plan_with_optional_progress( + pool: &sqlx::PgPool, + source_plan: &mut bigname_manifests::WatchedSourceSelectorPlan, + targets: &[ManifestBootstrapTarget], + range: BackfillBlockRange, + heartbeat: &mut Option<&mut StartupHeartbeat>, + chain_ids: &[String], +) -> Result<()> { + match heartbeat.as_deref_mut() { + Some(heartbeat) => { + let mut progress = StartupAdapterHeartbeat::new(heartbeat, chain_ids); + narrow_manifest_bootstrap_source_plan_with_progress( + pool, + source_plan, + targets, + range, + &mut progress, + ) + .await + } + None => narrow_manifest_bootstrap_source_plan(source_plan, targets, range), + } +} + +pub(super) async fn load_retained_recovery_targets_with_optional_progress( + pool: &sqlx::PgPool, + chain: &str, + through_block: i64, + heartbeat: &mut Option<&mut StartupHeartbeat>, + chain_ids: &[String], +) -> Result> { + match heartbeat.as_deref_mut() { + Some(heartbeat) => { + let mut progress = StartupAdapterHeartbeat::new(heartbeat, chain_ids); + bigname_manifests::load_ens_v2_retained_history_recovery_targets_with_progress( + pool, + chain, + through_block, + &mut progress, + ) + .await + } + None => { + bigname_manifests::load_ens_v2_retained_history_recovery_targets( + pool, + chain, + through_block, + ) + .await + } + } +} + +pub(super) async fn load_bootstrap_source_plan_with_optional_progress( + pool: &sqlx::PgPool, + chain: &str, + targets: &[ManifestBootstrapTarget], + range: BackfillBlockRange, + include_historical_recovery_targets: bool, + heartbeat: &mut Option<&mut StartupHeartbeat>, + chain_ids: &[String], +) -> Result { + match heartbeat.as_deref_mut() { + Some(heartbeat) => { + let mut progress = StartupAdapterHeartbeat::new(heartbeat, chain_ids); + load_bootstrap_source_plan_with_progress( + pool, + chain, + targets, + range, + include_historical_recovery_targets, + &mut progress, + ) + .await + } + None => { + load_bootstrap_source_plan( + pool, + chain, + targets, + range, + include_historical_recovery_targets, + ) + .await + } + } +} + +pub(super) async fn plan_bootstrap_segments_with_optional_progress( + pool: &sqlx::PgPool, + target_ranges: Vec, + heartbeat: &mut Option<&mut StartupHeartbeat>, + chain_ids: &[String], +) -> Result> { + match heartbeat.as_deref_mut() { + Some(heartbeat) => { + let mut progress = StartupAdapterHeartbeat::new(heartbeat, chain_ids); + plan_bootstrap_backfill_segments_with_progress(pool, target_ranges, &mut progress).await + } + None => plan_bootstrap_backfill_segments(target_ranges), + } +} + +pub(super) async fn extend_bootstrap_targets_with_progress( + pool: &sqlx::PgPool, + targets: &mut BTreeSet, + additions: Vec, + heartbeat: &mut Option<&mut StartupHeartbeat>, + chain_ids: &[String], +) -> Result<()> { + let addition_count = additions.len(); + for (index, target) in additions.into_iter().enumerate() { + targets.insert(target); + if (index + 1).is_multiple_of(TARGET_ACCUMULATION_PROGRESS_ROWS) { + record_bootstrap_progress(pool, heartbeat, chain_ids).await?; + } + } + if addition_count > 0 && !addition_count.is_multiple_of(TARGET_ACCUMULATION_PROGRESS_ROWS) { + record_bootstrap_progress(pool, heartbeat, chain_ids).await?; + } + Ok(()) +} + +pub(super) async fn record_bootstrap_progress( + pool: &sqlx::PgPool, + heartbeat: &mut Option<&mut StartupHeartbeat>, + chain_ids: &[String], +) -> Result<()> { + if let Some(heartbeat) = heartbeat.as_deref_mut() { + heartbeat.record_if_due(pool, chain_ids).await?; + } + Ok(()) +} diff --git a/apps/indexer/src/main/bootstrap_backfill/recovery.rs b/apps/indexer/src/main/bootstrap_backfill/recovery.rs index 604944f2..77a1c6a4 100644 --- a/apps/indexer/src/main/bootstrap_backfill/recovery.rs +++ b/apps/indexer/src/main/bootstrap_backfill/recovery.rs @@ -1,9 +1,12 @@ use anyhow::{Context, Result}; use bigname_manifests::{ - ManifestBootstrapTarget, WatchedSourceSelector, WatchedTargetIdentity, + ManifestBootstrapTarget, ManifestRuntimeProgress, WatchedSourceSelector, WatchedTargetIdentity, + has_required_watched_tuples, has_required_watched_tuples_with_progress, load_discovery_admission_epoch, load_ens_v2_authoritative_discovery_bootstrap_targets, + load_ens_v2_authoritative_discovery_bootstrap_targets_with_progress, load_historical_watched_source_selector_plan, - load_manifest_declared_watched_source_selector_plan, load_required_watched_tuples, + load_historical_watched_source_selector_plan_with_progress, + load_manifest_declared_watched_source_selector_plan, }; use sqlx::Row; @@ -102,6 +105,7 @@ impl BootstrapConvergenceTracker { chain: &str, through_block: i64, status: BootstrapPassStatus, + mut progress: Option<&mut dyn ManifestRuntimeProgress>, ) -> Result<()> { match status { BootstrapPassStatus::Stable => Ok(()), @@ -118,16 +122,39 @@ impl BootstrapConvergenceTracker { } BootstrapPassStatus::DiscoveryExpanded => { self.consecutive_retention_authority_retries = 0; - let snapshot = - load_bootstrap_retention_snapshot(pool, chain, through_block).await?; + let snapshot = match progress.as_mut() { + Some(progress) => { + load_bootstrap_retention_snapshot_with_progress( + pool, + chain, + through_block, + *progress, + ) + .await? + } + None => load_bootstrap_retention_snapshot(pool, chain, through_block).await?, + }; let progress = EnsV2DiscoveryProgress { generation: snapshot.generation, - targets: load_ens_v2_authoritative_discovery_bootstrap_targets( - pool, - chain, - through_block, - ) - .await?, + targets: match progress.as_mut() { + Some(progress) => { + load_ens_v2_authoritative_discovery_bootstrap_targets_with_progress( + pool, + chain, + through_block, + *progress, + ) + .await? + } + None => { + load_ens_v2_authoritative_discovery_bootstrap_targets( + pool, + chain, + through_block, + ) + .await? + } + }, }; self.record_discovery_progress(chain, progress) } @@ -162,15 +189,42 @@ pub(crate) async fn load_bootstrap_retention_snapshot( pool: &sqlx::PgPool, chain: &str, through_block: i64, +) -> Result { + load_bootstrap_retention_snapshot_inner(pool, chain, through_block, None).await +} + +pub(super) async fn load_bootstrap_retention_snapshot_with_progress( + pool: &sqlx::PgPool, + chain: &str, + through_block: i64, + progress: &mut dyn ManifestRuntimeProgress, +) -> Result { + load_bootstrap_retention_snapshot_inner(pool, chain, through_block, Some(progress)).await +} + +async fn load_bootstrap_retention_snapshot_inner( + pool: &sqlx::PgPool, + chain: &str, + through_block: i64, + progress: Option<&mut dyn ManifestRuntimeProgress>, ) -> Result { let source_families = bigname_manifests::ENS_V2_RETAINED_HISTORY_SOURCE_FAMILIES .iter() .map(|source_family| (*source_family).to_owned()) .collect::>(); - let has_ens_v2_history_requirements = - !load_required_watched_tuples(pool, chain, 0, through_block, &source_families) - .await? - .is_empty(); + let has_ens_v2_history_requirements = if let Some(progress) = progress { + has_required_watched_tuples_with_progress( + pool, + chain, + 0, + through_block, + &source_families, + progress, + ) + .await? + } else { + has_required_watched_tuples(pool, chain, 0, through_block, &source_families).await? + }; let discovery_admission_epoch = load_discovery_admission_epoch(pool, chain).await?; let row = sqlx::query( r#" @@ -277,6 +331,35 @@ pub(crate) async fn converge_ens_v2_retained_history_through_block( } } +pub(super) async fn converge_ens_v2_retained_history_through_block_with_progress( + pool: &sqlx::PgPool, + chain: &str, + through_block: i64, + has_ens_v2_history_requirements: bool, + progress: &mut dyn bigname_adapters::StartupAdapterProgress, +) -> Result { + if !has_ens_v2_history_requirements { + return Ok(false); + } + + match bigname_adapters::sync_ens_v2_registry_resource_surface_through_block_with_progress( + pool, + chain, + through_block, + progress, + ) + .await + { + Ok(_) => Ok(false), + Err(error) if bigname_adapters::is_ens_v2_missing_coverage(&error) => Ok(true), + Err(error) => Err(error).with_context(|| { + format!( + "failed to converge ENSv2 full-source authority on chain {chain} through block {through_block}" + ) + }), + } +} + pub(super) async fn finish_bootstrap_convergence_pass( pool: &sqlx::PgPool, chain: &str, @@ -301,20 +384,72 @@ pub(super) async fn finish_bootstrap_convergence_pass( }; let current = load_bootstrap_retention_snapshot(pool, chain, through_block).await?; + Ok(classify_bootstrap_pass( + planned, + current, + requested_adapter_sync_mode, + newly_required_coverage, + )) +} + +pub(super) async fn finish_bootstrap_convergence_pass_with_progress

( + pool: &sqlx::PgPool, + chain: &str, + through_block: i64, + planned: BootstrapRetentionSnapshot, + requested_adapter_sync_mode: BackfillAdapterSyncMode, + progress: &mut P, +) -> Result +where + P: bigname_adapters::StartupAdapterProgress + ManifestRuntimeProgress, +{ + #[cfg(test)] + maybe_force_retention_rotation(pool, chain).await?; + + let newly_required_coverage = if requested_adapter_sync_mode == BackfillAdapterSyncMode::RawOnly + { + false + } else { + converge_ens_v2_retained_history_through_block_with_progress( + pool, + chain, + through_block, + planned.has_ens_v2_history_requirements, + progress, + ) + .await? + }; + let current = + load_bootstrap_retention_snapshot_with_progress(pool, chain, through_block, progress) + .await?; + Ok(classify_bootstrap_pass( + planned, + current, + requested_adapter_sync_mode, + newly_required_coverage, + )) +} + +fn classify_bootstrap_pass( + planned: BootstrapRetentionSnapshot, + current: BootstrapRetentionSnapshot, + requested_adapter_sync_mode: BackfillAdapterSyncMode, + newly_required_coverage: bool, +) -> BootstrapPassStatus { if !newly_required_coverage && (requested_adapter_sync_mode == BackfillAdapterSyncMode::RawOnly || !current.requires_ens_v2_history_recovery) && retention_snapshots_are_stable(planned, current) { - Ok(BootstrapPassStatus::Stable) + BootstrapPassStatus::Stable } else if current.generation != planned.generation { - Ok(BootstrapPassStatus::RetentionAuthorityChanged) + BootstrapPassStatus::RetentionAuthorityChanged } else if newly_required_coverage || current.discovery_admission_epoch != planned.discovery_admission_epoch { - Ok(BootstrapPassStatus::DiscoveryExpanded) + BootstrapPassStatus::DiscoveryExpanded } else { - Ok(BootstrapPassStatus::RetentionAuthorityChanged) + BootstrapPassStatus::RetentionAuthorityChanged } } @@ -354,6 +489,46 @@ pub(super) async fn load_bootstrap_source_plan( } } +pub(super) async fn load_bootstrap_source_plan_with_progress( + pool: &sqlx::PgPool, + chain: &str, + targets: &[ManifestBootstrapTarget], + range: BackfillBlockRange, + include_historical_recovery_targets: bool, + progress: &mut dyn ManifestRuntimeProgress, +) -> Result { + let selector = WatchedSourceSelector::WatchedTargetSet( + targets + .iter() + .map(|target| WatchedTargetIdentity { + contract_instance_id: target.contract_instance_id, + }) + .collect(), + ); + if include_historical_recovery_targets { + load_historical_watched_source_selector_plan_with_progress( + pool, + chain, + selector, + range.from_block, + range.to_block, + progress, + ) + .await + } else { + let plan = load_manifest_declared_watched_source_selector_plan( + pool, + chain, + selector, + range.from_block, + range.to_block, + ) + .await?; + progress.record(pool).await?; + Ok(plan) + } +} + #[cfg(test)] mod tests { use super::*; diff --git a/apps/indexer/src/main/normalized_replay_catchup.rs b/apps/indexer/src/main/normalized_replay_catchup.rs index 1b9f55bb..43cbf227 100644 --- a/apps/indexer/src/main/normalized_replay_catchup.rs +++ b/apps/indexer/src/main/normalized_replay_catchup.rs @@ -14,19 +14,20 @@ use crate::provider::ChainProvider; use crate::{ provider::{ChainProviderOps, ProviderRegistry}, reconciliation::{ - HeaderAuditMode, RawFactNormalizedEventReplayOutcome, RawFactNormalizedEventReplayRequest, - RawFactNormalizedEventReplaySelection, active_closure_or_dependency_replay_adapters, - chain_has_closure_or_dependency_replay_adapter, replay_raw_fact_normalized_events, + HeaderAuditMode, RawFactNormalizedEventReplayRequest, + RawFactNormalizedEventReplaySelection, chain_has_closure_or_dependency_replay_adapter, + replay_raw_fact_normalized_events, replay_raw_fact_normalized_events_with_progress, select_log_bounded_replay_to_block, - sync_automatic_two_phase_full_closure_normalized_events, - unsupported_closure_replay_adapters, }, + run::startup_heartbeat::{NormalizedReplayHeartbeat, RequiredSubtaskActivity}, }; #[path = "normalized_replay_catchup/coverage_recovery.rs"] mod coverage_recovery; #[path = "normalized_replay_catchup/cursors.rs"] mod cursors; +#[path = "normalized_replay_catchup/execution.rs"] +mod execution; #[path = "normalized_replay_catchup/indexes.rs"] mod indexes; #[path = "normalized_replay_catchup/sources.rs"] @@ -38,7 +39,9 @@ mod test_hook; #[cfg(test)] pub(crate) use test_hook::{ install_after_coverage_recovery as install_after_coverage_recovery_test_hook, + install_after_progress as install_after_progress_test_hook, install_after_rewind as install_after_rewind_test_hook, + install_before_cursor_failure_record as install_before_cursor_failure_record_test_hook, }; use coverage_recovery::replay_full_closure_with_coverage_recovery; @@ -46,6 +49,9 @@ use cursors::{ advance_cursor, ensure_cursor, record_cursor_failure, rewind_cursor_for_newly_observed_older_logs, }; +use execution::{ + record_normalized_replay_progress, replay_full_closure_or_dependency_normalized_events, +}; use indexes::{ ensure_projection_indexes_after_catchup, prepare_deferred_projection_indexes_for_fresh_replay, restore_deferred_projection_indexes, @@ -227,6 +233,8 @@ pub(crate) async fn run_normalized_replay_catchup( config: NormalizedReplayCatchupConfig, provider_registry: ProviderRegistry, header_audit_mode: HeaderAuditMode, + heartbeat: NormalizedReplayHeartbeat, + activity: RequiredSubtaskActivity, ) -> Result<()> { info!( service = "indexer", @@ -244,12 +252,15 @@ pub(crate) async fn run_normalized_replay_catchup( loop { let mut progressed = false; for chain in &config.chains { - match run_normalized_replay_catchup_iteration_with_provider( + let mut progress = heartbeat.clone(); + match run_required_normalized_replay_catchup_iteration( &pool, &config, chain, provider_registry.provider_for(chain), header_audit_mode, + &mut progress, + &activity, ) .await { @@ -258,7 +269,6 @@ pub(crate) async fn run_normalized_replay_catchup( } Ok(CatchupIterationStatus::Idle) => {} Err(error) => { - record_cursor_failure(&pool, &config.deployment_profile, chain, &error).await?; warn!( service = "indexer", command = "run", @@ -298,6 +308,7 @@ pub(crate) async fn run_normalized_replay_catchup_iteration( chain, provider, HeaderAuditMode::Minimal, + &mut None, ) .await } @@ -316,16 +327,64 @@ pub(crate) async fn run_normalized_replay_catchup_iteration_with_provider_for_te chain, Some(provider), header_audit_mode, + &mut None, + ) + .await +} + +#[cfg(test)] +pub(crate) async fn run_required_normalized_replay_catchup_iteration_for_test( + pool: &PgPool, + config: &NormalizedReplayCatchupConfig, + chain: &str, + progress: &mut NormalizedReplayHeartbeat, + activity: &RequiredSubtaskActivity, +) -> Result { + let provider: Option<&ChainProvider> = None; + run_required_normalized_replay_catchup_iteration( + pool, + config, + chain, + provider, + HeaderAuditMode::Minimal, + progress, + activity, ) .await } +async fn run_required_normalized_replay_catchup_iteration( + pool: &PgPool, + config: &NormalizedReplayCatchupConfig, + chain: &str, + provider: Option<&(impl ChainProviderOps + ?Sized)>, + header_audit_mode: HeaderAuditMode, + progress: &mut NormalizedReplayHeartbeat, + activity: &RequiredSubtaskActivity, +) -> Result { + let _activity = activity.begin().await; + let result = run_normalized_replay_catchup_iteration_with_provider( + pool, + config, + chain, + provider, + header_audit_mode, + &mut Some(progress), + ) + .await; + if let Err(error) = &result { + record_cursor_failure(pool, &config.deployment_profile, chain, error).await?; + } + result +} + async fn run_normalized_replay_catchup_iteration_with_provider( pool: &PgPool, config: &NormalizedReplayCatchupConfig, chain: &str, provider: Option<&(impl ChainProviderOps + ?Sized)>, header_audit_mode: HeaderAuditMode, + progress: &mut Option<&mut NormalizedReplayHeartbeat>, ) -> Result { let pending_base_rederive_replay_target = bigname_storage::pending_base_normalized_rederive_replay_target( @@ -448,24 +507,31 @@ async fn run_normalized_replay_catchup_iteration_with_provider( provider, header_audit_mode, rewind_inspection_input_version, + progress, ) .await? } else { - let outcome = replay_raw_fact_normalized_events( - pool, - RawFactNormalizedEventReplayRequest { - deployment_profile: config.deployment_profile.clone(), - chain: chain.to_owned(), - selection: RawFactNormalizedEventReplaySelection::BlockRange { - from_block, - to_block, - }, + let request = RawFactNormalizedEventReplayRequest { + deployment_profile: config.deployment_profile.clone(), + chain: chain.to_owned(), + selection: RawFactNormalizedEventReplaySelection::BlockRange { + from_block, + to_block, }, - ) - .await?; + }; + let outcome = match progress.as_deref_mut() { + Some(progress) => { + replay_raw_fact_normalized_events_with_progress(pool, request, progress).await? + } + None => replay_raw_fact_normalized_events(pool, request).await?, + }; (outcome, rewind_inspection_input_version) }; + record_normalized_replay_progress(pool, progress).await?; + #[cfg(test)] + test_hook::pause_after_progress(pool, &config.deployment_profile, chain).await; + advance_cursor( pool, &config.deployment_profile, @@ -481,6 +547,7 @@ async fn run_normalized_replay_catchup_iteration_with_provider( raw_log_input_version, ) .await?; + record_normalized_replay_progress(pool, progress).await?; if closure_or_dependency_replay { bigname_adapters::clear_replay_adapter_checkpoints( pool, @@ -489,6 +556,7 @@ async fn run_normalized_replay_catchup_iteration_with_provider( CURSOR_KIND_RAW_FACT_NORMALIZED_EVENTS, ) .await?; + record_normalized_replay_progress(pool, progress).await?; } if config.defer_projection_indexes { ensure_projection_indexes_after_catchup(pool, &config.deployment_profile, &config.chains) @@ -517,68 +585,3 @@ async fn run_normalized_replay_catchup_iteration_with_provider( Ok(CatchupIterationStatus::Progressed) } - -async fn replay_full_closure_or_dependency_normalized_events( - pool: &PgPool, - deployment_profile: &str, - chain: &str, - from_block: i64, - to_block: i64, - stateless_ranges: &[(i64, i64)], - max_raw_logs_per_page: usize, -) -> Result { - let adapters = active_closure_or_dependency_replay_adapters(pool, chain).await?; - let unsupported = unsupported_closure_replay_adapters(&adapters); - if !unsupported.is_empty() { - bail!( - "normalized-event replay selected closure/context-dependent adapter(s) {}; full closure replay is not implemented for these adapters", - unsupported.join(", ") - ); - } - info!( - service = "indexer", - replay_cursor_kind = CURSOR_KIND_RAW_FACT_NORMALIZED_EVENTS, - chain, - from_block, - to_block, - stateless_range_count = stateless_ranges.len(), - stateless_ranges = ?stateless_ranges, - max_raw_logs_per_page, - adapter_count = adapters.len(), - adapters = ?adapters, - "two-phase full closure normalized-event replay session started" - ); - - let replay = sync_automatic_two_phase_full_closure_normalized_events( - pool, - deployment_profile, - chain, - CURSOR_KIND_RAW_FACT_NORMALIZED_EVENTS, - from_block, - to_block, - stateless_ranges, - &adapters, - max_raw_logs_per_page, - ) - .await?; - let stateless = replay.stateless; - let closure = replay.closure; - let mut stateless_replay_authority = stateless.stateless_replay_authority.clone(); - stateless_replay_authority.add(&closure.stateless_replay_authority); - - Ok(RawFactNormalizedEventReplayOutcome { - deployment_profile: deployment_profile.to_owned(), - chain: chain.to_owned(), - selection_kind: "two_phase_full_closure", - source_scope_target_count: adapters.len(), - selected_block_count: stateless.selected_block_count, - canonical_raw_log_count: stateless.canonical_raw_log_count, - scanned_raw_log_count: stateless.scanned_raw_log_count + closure.scanned_log_count, - matched_raw_log_count: stateless.matched_raw_log_count + closure.matched_log_count, - normalized_event_synced_count: stateless.normalized_event_synced_count - + closure.total_synced_count, - normalized_event_inserted_count: stateless.normalized_event_inserted_count - + closure.total_inserted_count, - stateless_replay_authority, - }) -} diff --git a/apps/indexer/src/main/normalized_replay_catchup/coverage_recovery.rs b/apps/indexer/src/main/normalized_replay_catchup/coverage_recovery.rs index 6fc5f8ba..9c26f483 100644 --- a/apps/indexer/src/main/normalized_replay_catchup/coverage_recovery.rs +++ b/apps/indexer/src/main/normalized_replay_catchup/coverage_recovery.rs @@ -3,7 +3,8 @@ use bigname_storage::RawLogStagingInputVersion; use tracing::info; use super::{ - CURSOR_KIND_RAW_FACT_NORMALIZED_EVENTS, replay_full_closure_or_dependency_normalized_events, + CURSOR_KIND_RAW_FACT_NORMALIZED_EVENTS, NormalizedReplayHeartbeat, + replay_full_closure_or_dependency_normalized_events, }; use crate::{ provider::ChainProviderOps, @@ -26,6 +27,7 @@ pub(super) async fn replay_full_closure_with_coverage_recovery( provider: Option<&(impl ChainProviderOps + ?Sized)>, header_audit_mode: HeaderAuditMode, mut raw_log_input_version: RawLogStagingInputVersion, + progress: &mut Option<&mut NormalizedReplayHeartbeat>, ) -> Result<( RawFactNormalizedEventReplayOutcome, RawLogStagingInputVersion, @@ -41,6 +43,7 @@ pub(super) async fn replay_full_closure_with_coverage_recovery( to_block, &stateless_ranges, max_raw_logs_per_page, + progress, ) .await { diff --git a/apps/indexer/src/main/normalized_replay_catchup/cursors.rs b/apps/indexer/src/main/normalized_replay_catchup/cursors.rs index cbfab490..3bd1a20d 100644 --- a/apps/indexer/src/main/normalized_replay_catchup/cursors.rs +++ b/apps/indexer/src/main/normalized_replay_catchup/cursors.rs @@ -394,6 +394,8 @@ pub(super) async fn record_cursor_failure( chain: &str, error: &anyhow::Error, ) -> Result<()> { + #[cfg(test)] + super::test_hook::pause_before_cursor_failure_record(pool, deployment_profile, chain).await; let failure_reason = postgres_text_safe(&format!("{error:#}")); sqlx::query( r#" diff --git a/apps/indexer/src/main/normalized_replay_catchup/execution.rs b/apps/indexer/src/main/normalized_replay_catchup/execution.rs new file mode 100644 index 00000000..9bb2ba91 --- /dev/null +++ b/apps/indexer/src/main/normalized_replay_catchup/execution.rs @@ -0,0 +1,107 @@ +use anyhow::{Result, bail}; +use sqlx::PgPool; +use tracing::info; + +use super::{CURSOR_KIND_RAW_FACT_NORMALIZED_EVENTS, NormalizedReplayHeartbeat}; +use crate::reconciliation::{ + RawFactNormalizedEventReplayOutcome, active_closure_or_dependency_replay_adapters, + sync_automatic_two_phase_full_closure_normalized_events, unsupported_closure_replay_adapters, +}; + +#[expect(clippy::too_many_arguments)] +pub(super) async fn replay_full_closure_or_dependency_normalized_events( + pool: &PgPool, + deployment_profile: &str, + chain: &str, + from_block: i64, + to_block: i64, + stateless_ranges: &[(i64, i64)], + max_raw_logs_per_page: usize, + progress: &mut Option<&mut NormalizedReplayHeartbeat>, +) -> Result { + let adapters = active_closure_or_dependency_replay_adapters(pool, chain).await?; + let unsupported = unsupported_closure_replay_adapters(&adapters); + if !unsupported.is_empty() { + bail!( + "normalized-event replay selected closure/context-dependent adapter(s) {}; full closure replay is not implemented for these adapters", + unsupported.join(", ") + ); + } + info!( + service = "indexer", + replay_cursor_kind = CURSOR_KIND_RAW_FACT_NORMALIZED_EVENTS, + chain, + from_block, + to_block, + stateless_range_count = stateless_ranges.len(), + stateless_ranges = ?stateless_ranges, + max_raw_logs_per_page, + adapter_count = adapters.len(), + adapters = ?adapters, + "two-phase full closure normalized-event replay session started" + ); + + let mut stateless_progress = progress.as_deref().cloned(); + let mut closure_progress = progress.as_deref().cloned(); + let mut noop_stateless_progress = NoopNormalizedReplayProgress; + let mut noop_closure_progress = NoopNormalizedReplayProgress; + let replay = sync_automatic_two_phase_full_closure_normalized_events( + pool, + deployment_profile, + chain, + CURSOR_KIND_RAW_FACT_NORMALIZED_EVENTS, + from_block, + to_block, + stateless_ranges, + &adapters, + max_raw_logs_per_page, + stateless_progress + .as_mut() + .map_or(&mut noop_stateless_progress, |progress| progress), + closure_progress + .as_mut() + .map_or(&mut noop_closure_progress, |progress| progress), + ) + .await?; + let stateless = replay.stateless; + let closure = replay.closure; + let mut stateless_replay_authority = stateless.stateless_replay_authority.clone(); + stateless_replay_authority.add(&closure.stateless_replay_authority); + + Ok(RawFactNormalizedEventReplayOutcome { + deployment_profile: deployment_profile.to_owned(), + chain: chain.to_owned(), + selection_kind: "two_phase_full_closure", + source_scope_target_count: adapters.len(), + selected_block_count: stateless.selected_block_count, + canonical_raw_log_count: stateless.canonical_raw_log_count, + scanned_raw_log_count: stateless.scanned_raw_log_count + closure.scanned_log_count, + matched_raw_log_count: stateless.matched_raw_log_count + closure.matched_log_count, + normalized_event_synced_count: stateless.normalized_event_synced_count + + closure.total_synced_count, + normalized_event_inserted_count: stateless.normalized_event_inserted_count + + closure.total_inserted_count, + stateless_replay_authority, + }) +} + +struct NoopNormalizedReplayProgress; + +impl bigname_adapters::StartupAdapterProgress for NoopNormalizedReplayProgress { + fn record<'a>( + &'a mut self, + _pool: &'a PgPool, + ) -> bigname_adapters::StartupAdapterProgressFuture<'a> { + Box::pin(std::future::ready(Ok(()))) + } +} + +pub(super) async fn record_normalized_replay_progress( + pool: &PgPool, + progress: &mut Option<&mut NormalizedReplayHeartbeat>, +) -> Result<()> { + if let Some(progress) = progress.as_deref_mut() { + bigname_adapters::StartupAdapterProgress::record(progress, pool).await?; + } + Ok(()) +} diff --git a/apps/indexer/src/main/normalized_replay_catchup/test_hook.rs b/apps/indexer/src/main/normalized_replay_catchup/test_hook.rs index 141bd4ea..ddd3e7a8 100644 --- a/apps/indexer/src/main/normalized_replay_catchup/test_hook.rs +++ b/apps/indexer/src/main/normalized_replay_catchup/test_hook.rs @@ -14,6 +14,16 @@ pub(crate) struct NormalizedReplayAfterCoverageRecoveryTestHook { _registration: ScopedTestHookGuard, } +pub(crate) struct NormalizedReplayAfterProgressTestHook { + state: NormalizedReplayAfterRewindTestHookState, + _registration: ScopedTestHookGuard, +} + +pub(crate) struct NormalizedReplayBeforeCursorFailureRecordTestHook { + state: NormalizedReplayAfterRewindTestHookState, + _registration: ScopedTestHookGuard, +} + #[derive(Clone)] struct NormalizedReplayAfterRewindTestHookState { after_rewind: Arc, @@ -40,6 +50,26 @@ impl NormalizedReplayAfterCoverageRecoveryTestHook { } } +impl NormalizedReplayAfterProgressTestHook { + pub(crate) async fn wait_until_after_progress(&self) { + self.state.after_rewind.notified().await; + } + + pub(crate) fn resume(&self) { + self.state.resume.notify_one(); + } +} + +impl NormalizedReplayBeforeCursorFailureRecordTestHook { + pub(crate) async fn wait_until_before_record(&self) { + self.state.after_rewind.notified().await; + } + + pub(crate) fn resume(&self) { + self.state.resume.notify_one(); + } +} + impl Drop for NormalizedReplayAfterRewindTestHook { fn drop(&mut self) { self.state.resume.notify_one(); @@ -52,6 +82,18 @@ impl Drop for NormalizedReplayAfterCoverageRecoveryTestHook { } } +impl Drop for NormalizedReplayAfterProgressTestHook { + fn drop(&mut self) { + self.state.resume.notify_one(); + } +} + +impl Drop for NormalizedReplayBeforeCursorFailureRecordTestHook { + fn drop(&mut self) { + self.state.resume.notify_one(); + } +} + type HookKey = (String, String, String); static HOOKS: ScopedTestHookRegistry = @@ -60,6 +102,12 @@ static COVERAGE_RECOVERY_HOOKS: ScopedTestHookRegistry< HookKey, NormalizedReplayAfterRewindTestHookState, > = ScopedTestHookRegistry::new(); +static PROGRESS_HOOKS: ScopedTestHookRegistry = + ScopedTestHookRegistry::new(); +static CURSOR_FAILURE_HOOKS: ScopedTestHookRegistry< + HookKey, + NormalizedReplayAfterRewindTestHookState, +> = ScopedTestHookRegistry::new(); pub(crate) async fn install_after_rewind( pool: &PgPool, @@ -105,6 +153,50 @@ pub(crate) async fn install_after_coverage_recovery( } } +pub(crate) async fn install_after_progress( + pool: &PgPool, + deployment_profile: &str, + chain: &str, +) -> NormalizedReplayAfterProgressTestHook { + let database = current_test_database(pool) + .await + .expect("normalized replay test hook must identify its database"); + let state = NormalizedReplayAfterRewindTestHookState { + after_rewind: Arc::new(Notify::new()), + resume: Arc::new(Notify::new()), + }; + let registration = PROGRESS_HOOKS.install( + (database, deployment_profile.to_owned(), chain.to_owned()), + state.clone(), + ); + NormalizedReplayAfterProgressTestHook { + state, + _registration: registration, + } +} + +pub(crate) async fn install_before_cursor_failure_record( + pool: &PgPool, + deployment_profile: &str, + chain: &str, +) -> NormalizedReplayBeforeCursorFailureRecordTestHook { + let database = current_test_database(pool) + .await + .expect("normalized replay test hook must identify its database"); + let state = NormalizedReplayAfterRewindTestHookState { + after_rewind: Arc::new(Notify::new()), + resume: Arc::new(Notify::new()), + }; + let registration = CURSOR_FAILURE_HOOKS.install( + (database, deployment_profile.to_owned(), chain.to_owned()), + state.clone(), + ); + NormalizedReplayBeforeCursorFailureRecordTestHook { + state, + _registration: registration, + } +} + pub(super) async fn pause_after_rewind(pool: &PgPool, deployment_profile: &str, chain: &str) { let database = current_test_database(pool) .await @@ -131,3 +223,30 @@ pub(super) async fn pause_after_coverage_recovery( hook.resume.notified().await; } } + +pub(super) async fn pause_after_progress(pool: &PgPool, deployment_profile: &str, chain: &str) { + let database = current_test_database(pool) + .await + .expect("normalized replay test hook must identify its database"); + let hook = PROGRESS_HOOKS.take(&(database, deployment_profile.to_owned(), chain.to_owned())); + if let Some(hook) = hook { + hook.after_rewind.notify_one(); + hook.resume.notified().await; + } +} + +pub(super) async fn pause_before_cursor_failure_record( + pool: &PgPool, + deployment_profile: &str, + chain: &str, +) { + let database = current_test_database(pool) + .await + .expect("normalized replay test hook must identify its database"); + let hook = + CURSOR_FAILURE_HOOKS.take(&(database, deployment_profile.to_owned(), chain.to_owned())); + if let Some(hook) = hook { + hook.after_rewind.notify_one(); + hook.resume.notified().await; + } +} diff --git a/apps/indexer/src/main/reconciliation.rs b/apps/indexer/src/main/reconciliation.rs index 79d0f3e7..535806c6 100644 --- a/apps/indexer/src/main/reconciliation.rs +++ b/apps/indexer/src/main/reconciliation.rs @@ -23,8 +23,10 @@ pub(crate) use adapter_sync::{ automatic_stateless_replay_completed, sync_adapter_state_from_persisted_raw_payloads, sync_adapter_state_from_scoped_persisted_raw_payloads, sync_automatic_two_phase_full_closure_normalized_events, - sync_live_adapter_backlog_after_normalized_replay, - sync_live_adapter_state_from_persisted_raw_payloads, validate_chain_handoff_while_guarded, + sync_live_adapter_backlog_after_normalized_replay_with_progress, + sync_live_adapter_state_from_persisted_raw_payloads, + sync_live_adapter_state_from_persisted_raw_payloads_with_progress, + validate_chain_handoff_while_guarded, }; #[cfg(test)] pub(crate) use adapter_sync::{ @@ -32,15 +34,18 @@ pub(crate) use adapter_sync::{ install_ownership_release_test_hook, install_post_discovery_mutation_failure_for_test, install_stateless_page_observer, sync_full_closure_normalized_events_from_persisted_raw_payloads, + sync_live_adapter_backlog_after_normalized_replay, sync_manual_full_closure_normalized_events_from_persisted_raw_payloads, }; +#[cfg(test)] +pub(crate) use canonical::reconcile_canonical_head_with_adapter_progress; #[allow(unused_imports)] pub(crate) use canonical::{ ChainCoverageFrontiers, EnsV2LiveCoverageRecoveryStatus, RawCodeBaselineFrontier, orphan_canonical_branch, orphan_reorg_losing_branch_payloads, poll_provider_heads, - poll_provider_heads_with_adapter_sync, reconcile_canonical_head, reconcile_fetched_heads, - reconcile_fetched_heads_with_adapter_sync, reconcile_intake_chain_task, - recover_ens_v2_live_coverage_requirement, + poll_provider_heads_with_adapter_sync, poll_provider_heads_with_adapter_sync_and_progress, + reconcile_canonical_head, reconcile_fetched_heads, reconcile_fetched_heads_with_adapter_sync, + reconcile_intake_chain_task, recover_ens_v2_live_coverage_requirement, }; #[allow(unused_imports)] pub(crate) use lineage::{ @@ -77,8 +82,10 @@ pub(crate) use replay::{ chain_has_closure_or_dependency_replay_adapter, ensure_full_closure_retention_authority_for_adapters, ensure_legacy_registry_closure_retention_authority_for_adapters, - replay_raw_fact_normalized_events, replay_stateless_only_raw_fact_normalized_events, - select_log_bounded_replay_to_block, unsupported_closure_replay_adapters, + replay_raw_fact_normalized_events, replay_raw_fact_normalized_events_with_progress, + replay_stateless_normalized_events_before_full_closure_with_progress, + replay_stateless_only_raw_fact_normalized_events, select_log_bounded_replay_to_block, + unsupported_closure_replay_adapters, }; #[allow(unused_imports)] pub(crate) use types::{ diff --git a/apps/indexer/src/main/reconciliation/adapter_sync.rs b/apps/indexer/src/main/reconciliation/adapter_sync.rs index a8d5f8f8..e147ebc4 100644 --- a/apps/indexer/src/main/reconciliation/adapter_sync.rs +++ b/apps/indexer/src/main/reconciliation/adapter_sync.rs @@ -1,29 +1,34 @@ use std::time::Instant; use anyhow::{Result, ensure}; +use bigname_adapters::StartupAdapterProgress; use tracing::info; -use crate::resolver_profile_convergence::journal_resolver_profile_authority_if_epoch_changed; use crate::runtime::{ log_ens_v1_subregistry_discovery_sync_summary, log_ens_v1_unwrapped_authority_sync_summary, - log_ens_v2_permissions_sync_summary, log_ens_v2_registrar_sync_summary, - log_ens_v2_registry_resource_surface_sync_summary, log_ens_v2_resolver_sync_summary, + log_ens_v2_registrar_sync_summary, log_ens_v2_registry_resource_surface_sync_summary, }; use super::replay::NormalizedEventReplayAdapter; use super::types::PersistedRawPayloadAdapterSyncSummary; #[path = "adapter_sync/backlog.rs"] mod backlog; +#[path = "adapter_sync/block_scoped.rs"] +mod block_scoped; #[path = "adapter_sync/ens_v1_subregistry.rs"] mod ens_v1_subregistry; #[path = "adapter_sync/ens_v2_registry.rs"] mod ens_v2_registry; +#[path = "adapter_sync/ens_v2_tail.rs"] +mod ens_v2_tail; #[path = "adapter_sync/entrypoints.rs"] mod entrypoints; #[path = "adapter_sync/full_closure.rs"] mod full_closure; #[path = "adapter_sync/mode.rs"] mod mode; +#[path = "adapter_sync/progress.rs"] +mod progress; #[path = "adapter_sync/scope.rs"] mod scope; #[path = "adapter_sync/stateless.rs"] @@ -35,18 +40,25 @@ mod sync_logging; mod test_hooks; #[cfg(test)] pub(crate) use backlog::install_backlog_after_adapter_sync_test_hook; +#[cfg(test)] +pub(crate) use backlog::sync_live_adapter_backlog_after_normalized_replay; pub(crate) use backlog::{ - BacklogHandoffStatus, sync_live_adapter_backlog_after_normalized_replay, + BacklogHandoffStatus, sync_live_adapter_backlog_after_normalized_replay_with_progress, validate_chain_handoff_while_guarded, }; +use block_scoped::{sync_ens_v1_unwrapped_authority_for_scope, sync_ens_v2_registrar_for_scope}; use ens_v1_subregistry::{ens_v1_subregistry_sync_operation, sync_ens_v1_subregistry_for_mode}; use ens_v2_registry::{ens_v2_registry_sync_operation, sync_ens_v2_registry_for_mode}; +use ens_v2_tail::sync_ens_v2_tail_adapters; +#[allow(unused_imports)] pub(crate) use entrypoints::{ sync_adapter_state_from_persisted_raw_payloads, sync_adapter_state_from_scoped_persisted_raw_payloads, sync_live_adapter_state_from_persisted_raw_payloads, sync_live_adapter_state_from_persisted_raw_payloads_after_reorg, - sync_replay_normalized_events_from_persisted_raw_payloads, + sync_live_adapter_state_from_persisted_raw_payloads_after_reorg_with_progress, + sync_live_adapter_state_from_persisted_raw_payloads_with_progress, + sync_replay_normalized_events_from_persisted_raw_payloads_with_progress, }; pub(crate) use full_closure::{ AutomaticTwoPhaseFullClosureSyncResult, automatic_stateless_replay_completed, @@ -60,6 +72,7 @@ pub(crate) use full_closure::{ sync_full_closure_normalized_events_from_persisted_raw_payloads, }; use mode::{PersistedRawPayloadAdapterSyncMode, ensure_raw_fact_adapter_allowed}; +use progress::{journal_authority_epoch_with_progress, record_adapter_progress}; use scope::load_live_adapter_source_scope; use stateless::{sync_block_derived_for_mode, sync_reverse_claim_for_mode}; use sync_logging::{log_adapter_call_timing, log_live_poll_adapter_sync_completion}; @@ -78,6 +91,7 @@ async fn sync_adapter_state_from_persisted_raw_payloads_with_mode( mode: PersistedRawPayloadAdapterSyncMode, reload_live_source_scope: bool, full_source_reconciliation: entrypoints::FullSourceReconciliationScope, + progress: &mut Option<&mut dyn StartupAdapterProgress>, ) -> Result { if matches!(mode, PersistedRawPayloadAdapterSyncMode::LivePoll) { ensure!( @@ -94,7 +108,7 @@ async fn sync_adapter_state_from_persisted_raw_payloads_with_mode( let ens_v2_full_source = full_source_reconciliation.reconciles_ens_v2_registry(); let mut aggregate = PersistedRawPayloadAdapterSyncSummary::default(); if !mode.uses_stateless_replay_authority() { - let epoch_guard = journal_resolver_profile_authority_if_epoch_changed(pool, chain).await?; + let epoch_guard = journal_authority_epoch_with_progress(pool, chain, progress).await?; aggregate.resolver_profile_authority_epoch_guard_count += epoch_guard.epoch_guard_count; aggregate.resolver_profile_authority_scan_count += epoch_guard.authority_scan_count; } @@ -106,6 +120,7 @@ async fn sync_adapter_state_from_persisted_raw_payloads_with_mode( active_source_scope.as_deref(), mode, &mut aggregate, + progress, ) .await?; if legacy_full_source @@ -138,6 +153,7 @@ async fn sync_adapter_state_from_persisted_raw_payloads_with_mode( active_source_scope.as_deref(), mode, legacy_full_source, + progress, ) .await?; log_adapter_call_timing( @@ -160,6 +176,7 @@ async fn sync_adapter_state_from_persisted_raw_payloads_with_mode( subregistry_discovery_summary.total_normalized_event_inserted_count, ); aggregate.add_stateless_replay_authority(&stateless_replay_authority); + record_adapter_progress(pool, progress).await?; let discovery_mutated = subregistry_discovery_summary.inserted_edge_count > 0 || subregistry_discovery_summary.deactivated_edge_count > 0; #[cfg(test)] @@ -167,8 +184,7 @@ async fn sync_adapter_state_from_persisted_raw_payloads_with_mode( fail_after_discovery_mutation_for_test(pool).await?; } if !mode.uses_stateless_replay_authority() { - let epoch_guard = - journal_resolver_profile_authority_if_epoch_changed(pool, chain).await?; + let epoch_guard = journal_authority_epoch_with_progress(pool, chain, progress).await?; aggregate.resolver_profile_authority_epoch_guard_count += epoch_guard.epoch_guard_count; aggregate.resolver_profile_authority_scan_count += epoch_guard.authority_scan_count; } @@ -185,6 +201,7 @@ async fn sync_adapter_state_from_persisted_raw_payloads_with_mode( source_scope, mode, &mut aggregate, + progress, ) .await?; if mode.selects_adapter( @@ -207,22 +224,14 @@ async fn sync_adapter_state_from_persisted_raw_payloads_with_mode( adapter_sync_mode = ?mode, "adapter sync call started" ); - let unwrapped_authority_summary = if let Some(source_scope) = source_scope { - bigname_adapters::EnsV1UnwrappedAuthoritySyncSummary::sync_for_block_hashes_with_source_scope( - pool, - chain, - block_hashes, - source_scope, - ) - .await? - } else { - bigname_adapters::EnsV1UnwrappedAuthoritySyncSummary::sync_for_block_hashes( - pool, - chain, - block_hashes, - ) - .await? - }; + let unwrapped_authority_summary = sync_ens_v1_unwrapped_authority_for_scope( + pool, + chain, + block_hashes, + source_scope, + progress, + ) + .await?; log_adapter_call_timing( chain, "ens_v1_unwrapped_authority", @@ -242,6 +251,7 @@ async fn sync_adapter_state_from_persisted_raw_payloads_with_mode( unwrapped_authority_summary.total_normalized_event_count, unwrapped_authority_summary.total_normalized_event_inserted_count, ); + record_adapter_progress(pool, progress).await?; } if !mode.selects_adapter( source_scope, @@ -282,6 +292,7 @@ async fn sync_adapter_state_from_persisted_raw_payloads_with_mode( source_scope, mode, ens_v2_full_source, + progress, ) .await?; log_adapter_call_timing( @@ -303,6 +314,7 @@ async fn sync_adapter_state_from_persisted_raw_payloads_with_mode( ens_v2_registry_summary.total_normalized_event_count, ens_v2_registry_summary.total_normalized_event_inserted_count, ); + record_adapter_progress(pool, progress).await?; if reload_live_source_scope && (ens_v2_registry_summary.inserted_edge_count > 0 || ens_v2_registry_summary.deactivated_edge_count > 0) @@ -328,14 +340,14 @@ async fn sync_adapter_state_from_persisted_raw_payloads_with_mode( adapter_sync_mode = ?mode, "adapter sync call started" ); - let ens_v2_registrar_summary = - bigname_adapters::EnsV2RegistrarSyncSummary::sync_for_block_hashes_with_source_scope( - pool, - chain, - block_hashes, - source_scope, - ) - .await?; + let ens_v2_registrar_summary = sync_ens_v2_registrar_for_scope( + pool, + chain, + block_hashes, + Some(source_scope), + progress, + ) + .await?; log_adapter_call_timing( chain, "ens_v2_registrar", @@ -355,6 +367,7 @@ async fn sync_adapter_state_from_persisted_raw_payloads_with_mode( ens_v2_registrar_summary.total_synced_count, ens_v2_registrar_summary.total_inserted_count, ); + record_adapter_progress(pool, progress).await?; } else if source_scope.is_none() { ensure_raw_fact_adapter_allowed(mode, NormalizedEventReplayAdapter::EnsV2Registrar)?; let adapter_started = Instant::now(); @@ -369,12 +382,7 @@ async fn sync_adapter_state_from_persisted_raw_payloads_with_mode( "adapter sync call started" ); let ens_v2_registrar_summary = - bigname_adapters::EnsV2RegistrarSyncSummary::sync_for_block_hashes( - pool, - chain, - block_hashes, - ) - .await?; + sync_ens_v2_registrar_for_scope(pool, chain, block_hashes, None, progress).await?; log_adapter_call_timing( chain, "ens_v2_registrar", @@ -394,105 +402,18 @@ async fn sync_adapter_state_from_persisted_raw_payloads_with_mode( ens_v2_registrar_summary.total_synced_count, ens_v2_registrar_summary.total_inserted_count, ); + record_adapter_progress(pool, progress).await?; } - if mode.selects_adapter(source_scope, NormalizedEventReplayAdapter::EnsV2Resolver) { - ensure_raw_fact_adapter_allowed(mode, NormalizedEventReplayAdapter::EnsV2Resolver)?; - let adapter_started = Instant::now(); - info!( - service = "indexer", - command = "adapter-sync", - chain, - adapter = "ens_v2_resolver", - block_hash_count = block_hashes.len(), - source_scope_target_count, - adapter_sync_mode = ?mode, - "adapter sync call started" - ); - let ens_v2_resolver_summary = if let Some(source_scope) = source_scope { - bigname_adapters::EnsV2ResolverSyncSummary::sync_for_block_hashes_with_source_scope( - pool, - chain, - block_hashes, - source_scope, - ) - .await? - } else { - bigname_adapters::EnsV2ResolverSyncSummary::sync_for_block_hashes( - pool, - chain, - block_hashes, - ) - .await? - }; - log_adapter_call_timing( - chain, - "ens_v2_resolver", - "sync_for_block_hashes", - block_hashes.len(), - source_scope_target_count, - ens_v2_resolver_summary.scanned_log_count, - ens_v2_resolver_summary.matched_log_count, - ens_v2_resolver_summary.total_synced_count, - ens_v2_resolver_summary.total_inserted_count, - adapter_started.elapsed().as_millis(), - ); - log_ens_v2_resolver_sync_summary(chain, &ens_v2_resolver_summary); - aggregate.add_counts( - ens_v2_resolver_summary.scanned_log_count, - ens_v2_resolver_summary.matched_log_count, - ens_v2_resolver_summary.total_synced_count, - ens_v2_resolver_summary.total_inserted_count, - ); - } - if mode.selects_adapter(source_scope, NormalizedEventReplayAdapter::EnsV2Permissions) { - ensure_raw_fact_adapter_allowed(mode, NormalizedEventReplayAdapter::EnsV2Permissions)?; - let adapter_started = Instant::now(); - info!( - service = "indexer", - command = "adapter-sync", - chain, - adapter = "ens_v2_permissions", - block_hash_count = block_hashes.len(), - source_scope_target_count, - adapter_sync_mode = ?mode, - "adapter sync call started" - ); - let ens_v2_permissions_summary = if let Some(source_scope) = source_scope { - bigname_adapters::EnsV2PermissionsSyncSummary::sync_for_block_hashes_with_source_scope( - pool, - chain, - block_hashes, - source_scope, - ) - .await? - } else { - bigname_adapters::EnsV2PermissionsSyncSummary::sync_for_block_hashes( - pool, - chain, - block_hashes, - ) - .await? - }; - log_adapter_call_timing( - chain, - "ens_v2_permissions", - "sync_for_block_hashes", - block_hashes.len(), - source_scope_target_count, - ens_v2_permissions_summary.scanned_log_count, - ens_v2_permissions_summary.matched_log_count, - ens_v2_permissions_summary.total_synced_count, - ens_v2_permissions_summary.total_inserted_count, - adapter_started.elapsed().as_millis(), - ); - log_ens_v2_permissions_sync_summary(chain, &ens_v2_permissions_summary); - aggregate.add_counts( - ens_v2_permissions_summary.scanned_log_count, - ens_v2_permissions_summary.matched_log_count, - ens_v2_permissions_summary.total_synced_count, - ens_v2_permissions_summary.total_inserted_count, - ); - } + sync_ens_v2_tail_adapters( + pool, + chain, + block_hashes, + source_scope, + mode, + &mut aggregate, + progress, + ) + .await?; if mode == PersistedRawPayloadAdapterSyncMode::LivePoll { log_live_poll_adapter_sync_completion( chain, diff --git a/apps/indexer/src/main/reconciliation/adapter_sync/backlog.rs b/apps/indexer/src/main/reconciliation/adapter_sync/backlog.rs index e23b4df9..d1ee8ffc 100644 --- a/apps/indexer/src/main/reconciliation/adapter_sync/backlog.rs +++ b/apps/indexer/src/main/reconciliation/adapter_sync/backlog.rs @@ -1,7 +1,10 @@ use anyhow::{Context, Result}; use tracing::info; -use super::sync_live_adapter_state_from_persisted_raw_payloads; +use super::{ + sync_live_adapter_state_from_persisted_raw_payloads, + sync_live_adapter_state_from_persisted_raw_payloads_with_progress, +}; #[path = "backlog/cursor.rs"] mod cursor; @@ -28,10 +31,41 @@ pub(crate) struct LiveAdapterBacklogSyncSummary { pub(crate) normalized_event_inserted_count: usize, } +#[cfg(test)] pub(crate) async fn sync_live_adapter_backlog_after_normalized_replay( pool: &sqlx::PgPool, deployment_profile: &str, chains: &[String], +) -> Result { + sync_live_adapter_backlog_after_normalized_replay_inner( + pool, + deployment_profile, + chains, + &mut None, + ) + .await +} + +pub(crate) async fn sync_live_adapter_backlog_after_normalized_replay_with_progress( + pool: &sqlx::PgPool, + deployment_profile: &str, + chains: &[String], + progress: &mut dyn bigname_adapters::StartupAdapterProgress, +) -> Result { + sync_live_adapter_backlog_after_normalized_replay_inner( + pool, + deployment_profile, + chains, + &mut Some(progress), + ) + .await +} + +async fn sync_live_adapter_backlog_after_normalized_replay_inner( + pool: &sqlx::PgPool, + deployment_profile: &str, + chains: &[String], + progress: &mut Option<&mut dyn bigname_adapters::StartupAdapterProgress>, ) -> Result { let mut aggregate = LiveAdapterBacklogSyncSummary::default(); @@ -95,11 +129,12 @@ pub(crate) async fn sync_live_adapter_backlog_after_normalized_replay( block_hash_count = hashes.len(), "post-replay live raw payload adapter backlog page selected" ); - let summary = sync_live_adapter_state_from_persisted_raw_payloads( + let summary = sync_backlog_page( pool, deployment_profile, chain, &hashes, + progress, ) .await .with_context(|| { @@ -129,6 +164,7 @@ pub(crate) async fn sync_live_adapter_backlog_after_normalized_replay( aggregate.matched_log_count += summary.matched_log_count; aggregate.normalized_event_synced_count += summary.total_synced_count; aggregate.normalized_event_inserted_count += summary.total_inserted_count; + record_backlog_progress(pool, progress).await?; } BacklogCursorPublication::Retry(next_cursor) => { cursor = next_cursor; @@ -162,3 +198,45 @@ pub(crate) async fn sync_live_adapter_backlog_after_normalized_replay( Ok(aggregate) } + +async fn sync_backlog_page( + pool: &sqlx::PgPool, + deployment_profile: &str, + chain: &str, + hashes: &[String], + progress: &mut Option<&mut dyn bigname_adapters::StartupAdapterProgress>, +) -> Result { + match progress.as_mut() { + Some(progress) => { + let mut page_progress = + Some(&mut **progress as &mut dyn bigname_adapters::StartupAdapterProgress); + sync_live_adapter_state_from_persisted_raw_payloads_with_progress( + pool, + deployment_profile, + chain, + hashes, + &mut page_progress, + ) + .await + } + None => { + sync_live_adapter_state_from_persisted_raw_payloads( + pool, + deployment_profile, + chain, + hashes, + ) + .await + } + } +} + +async fn record_backlog_progress( + pool: &sqlx::PgPool, + progress: &mut Option<&mut dyn bigname_adapters::StartupAdapterProgress>, +) -> Result<()> { + if let Some(progress) = progress.as_mut() { + progress.record(pool).await?; + } + Ok(()) +} diff --git a/apps/indexer/src/main/reconciliation/adapter_sync/block_scoped.rs b/apps/indexer/src/main/reconciliation/adapter_sync/block_scoped.rs new file mode 100644 index 00000000..0c7482de --- /dev/null +++ b/apps/indexer/src/main/reconciliation/adapter_sync/block_scoped.rs @@ -0,0 +1,91 @@ +use anyhow::Result; +use bigname_adapters::{ + EnsV1UnwrappedAuthoritySyncSummary, EnsV2RegistrarSyncSummary, StartupAdapterProgress, +}; + +type SourceScope<'a> = &'a [(String, String, i64, i64)]; + +pub(super) async fn sync_ens_v1_unwrapped_authority_for_scope( + pool: &sqlx::PgPool, + chain: &str, + block_hashes: &[String], + source_scope: Option>, + progress: &mut Option<&mut dyn StartupAdapterProgress>, +) -> Result { + match (source_scope, progress.as_deref_mut()) { + (Some(source_scope), Some(progress)) => { + EnsV1UnwrappedAuthoritySyncSummary::sync_for_block_hashes_with_source_scope_and_progress( + pool, + chain, + block_hashes, + source_scope, + progress, + ) + .await + } + (Some(source_scope), None) => { + EnsV1UnwrappedAuthoritySyncSummary::sync_for_block_hashes_with_source_scope( + pool, + chain, + block_hashes, + source_scope, + ) + .await + } + (None, Some(progress)) => { + EnsV1UnwrappedAuthoritySyncSummary::sync_for_block_hashes_with_progress( + pool, + chain, + block_hashes, + progress, + ) + .await + } + (None, None) => { + EnsV1UnwrappedAuthoritySyncSummary::sync_for_block_hashes(pool, chain, block_hashes) + .await + } + } +} + +pub(super) async fn sync_ens_v2_registrar_for_scope( + pool: &sqlx::PgPool, + chain: &str, + block_hashes: &[String], + source_scope: Option>, + progress: &mut Option<&mut dyn StartupAdapterProgress>, +) -> Result { + match (source_scope, progress.as_deref_mut()) { + (Some(source_scope), Some(progress)) => { + EnsV2RegistrarSyncSummary::sync_for_block_hashes_with_source_scope_and_progress( + pool, + chain, + block_hashes, + source_scope, + progress, + ) + .await + } + (Some(source_scope), None) => { + EnsV2RegistrarSyncSummary::sync_for_block_hashes_with_source_scope( + pool, + chain, + block_hashes, + source_scope, + ) + .await + } + (None, Some(progress)) => { + EnsV2RegistrarSyncSummary::sync_for_block_hashes_with_progress( + pool, + chain, + block_hashes, + progress, + ) + .await + } + (None, None) => { + EnsV2RegistrarSyncSummary::sync_for_block_hashes(pool, chain, block_hashes).await + } + } +} diff --git a/apps/indexer/src/main/reconciliation/adapter_sync/ens_v1_subregistry.rs b/apps/indexer/src/main/reconciliation/adapter_sync/ens_v1_subregistry.rs index 0ef5c1c3..7bddfbdb 100644 --- a/apps/indexer/src/main/reconciliation/adapter_sync/ens_v1_subregistry.rs +++ b/apps/indexer/src/main/reconciliation/adapter_sync/ens_v1_subregistry.rs @@ -1,4 +1,5 @@ use anyhow::Result; +use bigname_adapters::StartupAdapterProgress; use bigname_storage::acquire_raw_log_staging_read_guard; use crate::reconciliation::guard_release::prioritize_operation_error; @@ -17,6 +18,7 @@ pub(super) async fn sync_ens_v1_subregistry_for_mode( source_scope: Option<&[(String, String, i64, i64)]>, mode: PersistedRawPayloadAdapterSyncMode, reconcile_full_source: bool, + progress: &mut Option<&mut dyn StartupAdapterProgress>, ) -> Result<( bigname_adapters::EnsV1SubregistryDiscoverySyncSummary, bigname_storage::NormalizedEventReplayAuthoritySummary, @@ -24,6 +26,7 @@ pub(super) async fn sync_ens_v1_subregistry_for_mode( if reconcile_full_source { let target_block_number = load_live_adapter_target_block_number(pool, chain, block_hashes).await?; + record_progress(pool, progress).await?; // The same-chain raw-log mutation fence spans coverage proof, complete // loading, absence-aware discovery persistence, and normalized-event // persistence. Releasing it earlier would let a late raw fact land @@ -38,13 +41,24 @@ pub(super) async fn sync_ens_v1_subregistry_for_mode( target_block_number, ) .await?; - bigname_adapters::sync_ens_v1_subregistry_discovery_through_block_with_expected_admission_epoch( - pool, - chain, - target_block_number, - expected_admission_epoch, - ) - .await + record_progress(pool, progress).await?; + match progress.as_deref_mut() { + Some(progress) => bigname_adapters::sync_ens_v1_subregistry_discovery_through_block_with_expected_admission_epoch_and_progress( + pool, + chain, + target_block_number, + expected_admission_epoch, + progress, + ) + .await, + None => bigname_adapters::sync_ens_v1_subregistry_discovery_through_block_with_expected_admission_epoch( + pool, + chain, + target_block_number, + expected_admission_epoch, + ) + .await, + } } .await; let release_result = raw_log_guard.release().await; @@ -59,21 +73,43 @@ pub(super) async fn sync_ens_v1_subregistry_for_mode( match mode { PersistedRawPayloadAdapterSyncMode::LivePoll | PersistedRawPayloadAdapterSyncMode::LiveOrBackfill => { - if let Some(source_scope) = source_scope { - bigname_adapters::EnsV1SubregistryDiscoverySyncSummary::sync_for_block_hashes_with_source_scope( - pool, - chain, - block_hashes, - source_scope, - ) - .await - } else { - bigname_adapters::EnsV1SubregistryDiscoverySyncSummary::sync_for_block_hashes_without_discovery_reconciliation( - pool, - chain, - block_hashes, - ) - .await + match (source_scope, progress.as_deref_mut()) { + (Some(source_scope), Some(progress)) => { + bigname_adapters::EnsV1SubregistryDiscoverySyncSummary::sync_for_block_hashes_with_source_scope_and_progress( + pool, + chain, + block_hashes, + source_scope, + progress, + ) + .await + } + (Some(source_scope), None) => { + bigname_adapters::EnsV1SubregistryDiscoverySyncSummary::sync_for_block_hashes_with_source_scope( + pool, + chain, + block_hashes, + source_scope, + ) + .await + } + (None, Some(progress)) => { + bigname_adapters::EnsV1SubregistryDiscoverySyncSummary::sync_for_block_hashes_without_discovery_reconciliation_and_progress( + pool, + chain, + block_hashes, + progress, + ) + .await + } + (None, None) => { + bigname_adapters::EnsV1SubregistryDiscoverySyncSummary::sync_for_block_hashes_without_discovery_reconciliation( + pool, + chain, + block_hashes, + ) + .await + } } .map(|summary| { ( @@ -83,21 +119,43 @@ pub(super) async fn sync_ens_v1_subregistry_for_mode( }) } PersistedRawPayloadAdapterSyncMode::RawFactReplay { .. } => { - if let Some(source_scope) = source_scope { - bigname_adapters::EnsV1SubregistryDiscoverySyncSummary::sync_for_block_hashes_with_source_scope_without_discovery_reconciliation( - pool, - chain, - block_hashes, - source_scope, - ) - .await - } else { - bigname_adapters::EnsV1SubregistryDiscoverySyncSummary::sync_for_block_hashes_without_discovery_reconciliation( - pool, - chain, - block_hashes, - ) - .await + match (source_scope, progress.as_deref_mut()) { + (Some(source_scope), Some(progress)) => { + bigname_adapters::EnsV1SubregistryDiscoverySyncSummary::sync_for_block_hashes_with_source_scope_without_discovery_reconciliation_and_progress( + pool, + chain, + block_hashes, + source_scope, + progress, + ) + .await + } + (Some(source_scope), None) => { + bigname_adapters::EnsV1SubregistryDiscoverySyncSummary::sync_for_block_hashes_with_source_scope_without_discovery_reconciliation( + pool, + chain, + block_hashes, + source_scope, + ) + .await + } + (None, Some(progress)) => { + bigname_adapters::EnsV1SubregistryDiscoverySyncSummary::sync_for_block_hashes_without_discovery_reconciliation_and_progress( + pool, + chain, + block_hashes, + progress, + ) + .await + } + (None, None) => { + bigname_adapters::EnsV1SubregistryDiscoverySyncSummary::sync_for_block_hashes_without_discovery_reconciliation( + pool, + chain, + block_hashes, + ) + .await + } } .map(|summary| { ( @@ -109,6 +167,16 @@ pub(super) async fn sync_ens_v1_subregistry_for_mode( } } +async fn record_progress( + pool: &sqlx::PgPool, + progress: &mut Option<&mut dyn StartupAdapterProgress>, +) -> Result<()> { + if let Some(progress) = progress.as_deref_mut() { + progress.record(pool).await?; + } + Ok(()) +} + pub(super) const fn ens_v1_subregistry_sync_operation(reconcile_full_source: bool) -> &'static str { if reconcile_full_source { "sync_ens_v1_subregistry_discovery_through_block" diff --git a/apps/indexer/src/main/reconciliation/adapter_sync/ens_v2_registry.rs b/apps/indexer/src/main/reconciliation/adapter_sync/ens_v2_registry.rs index 07704d72..b21d8c40 100644 --- a/apps/indexer/src/main/reconciliation/adapter_sync/ens_v2_registry.rs +++ b/apps/indexer/src/main/reconciliation/adapter_sync/ens_v2_registry.rs @@ -4,6 +4,7 @@ use super::{ mode::PersistedRawPayloadAdapterSyncMode, scope::load_live_adapter_target_block_number, }; +#[expect(clippy::too_many_arguments)] pub(super) async fn sync_ens_v2_registry_for_mode( pool: &sqlx::PgPool, live_deployment_profile: Option<&str>, @@ -12,16 +13,30 @@ pub(super) async fn sync_ens_v2_registry_for_mode( source_scope: Option<&[(String, String, i64, i64)]>, mode: PersistedRawPayloadAdapterSyncMode, reconcile_full_source: bool, + progress: &mut Option<&mut dyn bigname_adapters::StartupAdapterProgress>, ) -> Result { if reconcile_full_source { let target_block_number = load_live_adapter_target_block_number(pool, chain, block_hashes).await?; - return bigname_adapters::sync_ens_v2_registry_resource_surface_through_block( - pool, - chain, - target_block_number, - ) - .await; + return match progress.as_deref_mut() { + Some(progress) => { + bigname_adapters::sync_ens_v2_registry_resource_surface_through_block_with_progress( + pool, + chain, + target_block_number, + progress, + ) + .await + } + None => { + bigname_adapters::sync_ens_v2_registry_resource_surface_through_block( + pool, + chain, + target_block_number, + ) + .await + } + }; } match (mode, source_scope) { (PersistedRawPayloadAdapterSyncMode::LivePoll, _) => { @@ -29,14 +44,29 @@ pub(super) async fn sync_ens_v2_registry_for_mode( .context("ENSv2 live-poll adapter sync is missing its deployment profile")?; let target_block_number = load_live_adapter_target_block_number(pool, chain, block_hashes).await?; - bigname_adapters::sync_ens_v2_registry_resource_surface_live_poll( - pool, - deployment_profile, - chain, - target_block_number, - block_hashes, - ) - .await + match progress.as_deref_mut() { + Some(progress) => { + bigname_adapters::sync_ens_v2_registry_resource_surface_live_poll_with_progress( + pool, + deployment_profile, + chain, + target_block_number, + block_hashes, + progress, + ) + .await + } + None => { + bigname_adapters::sync_ens_v2_registry_resource_surface_live_poll( + pool, + deployment_profile, + chain, + target_block_number, + block_hashes, + ) + .await + } + } } (PersistedRawPayloadAdapterSyncMode::RawFactReplay { .. }, Some(source_scope)) => { bigname_adapters::EnsV2RegistryResourceSurfaceSyncSummary::sync_for_block_hashes_with_source_scope_canonical_only( diff --git a/apps/indexer/src/main/reconciliation/adapter_sync/ens_v2_tail.rs b/apps/indexer/src/main/reconciliation/adapter_sync/ens_v2_tail.rs new file mode 100644 index 00000000..4c9d6e6c --- /dev/null +++ b/apps/indexer/src/main/reconciliation/adapter_sync/ens_v2_tail.rs @@ -0,0 +1,173 @@ +use std::time::Instant; + +use anyhow::Result; +use bigname_adapters::StartupAdapterProgress; +use tracing::info; + +use crate::runtime::{log_ens_v2_permissions_sync_summary, log_ens_v2_resolver_sync_summary}; + +use super::super::{ + replay::NormalizedEventReplayAdapter, types::PersistedRawPayloadAdapterSyncSummary, +}; +use super::{ + mode::{PersistedRawPayloadAdapterSyncMode, ensure_raw_fact_adapter_allowed}, + progress::record_adapter_progress, + sync_logging::log_adapter_call_timing, +}; + +pub(super) async fn sync_ens_v2_tail_adapters( + pool: &sqlx::PgPool, + chain: &str, + block_hashes: &[String], + source_scope: Option<&[(String, String, i64, i64)]>, + mode: PersistedRawPayloadAdapterSyncMode, + aggregate: &mut PersistedRawPayloadAdapterSyncSummary, + progress: &mut Option<&mut dyn StartupAdapterProgress>, +) -> Result<()> { + let source_scope_target_count = source_scope.map_or(0, <[_]>::len); + if mode.selects_adapter(source_scope, NormalizedEventReplayAdapter::EnsV2Resolver) { + ensure_raw_fact_adapter_allowed(mode, NormalizedEventReplayAdapter::EnsV2Resolver)?; + let adapter_started = Instant::now(); + info!( + service = "indexer", + command = "adapter-sync", + chain, + adapter = "ens_v2_resolver", + block_hash_count = block_hashes.len(), + source_scope_target_count, + adapter_sync_mode = ?mode, + "adapter sync call started" + ); + let summary = match (source_scope, progress.as_deref_mut()) { + (Some(source_scope), Some(progress)) => { + bigname_adapters::EnsV2ResolverSyncSummary::sync_for_block_hashes_with_source_scope_and_progress( + pool, + chain, + block_hashes, + source_scope, + progress, + ) + .await? + } + (Some(source_scope), None) => { + bigname_adapters::EnsV2ResolverSyncSummary::sync_for_block_hashes_with_source_scope( + pool, + chain, + block_hashes, + source_scope, + ) + .await? + } + (None, Some(progress)) => { + bigname_adapters::EnsV2ResolverSyncSummary::sync_for_block_hashes_with_progress( + pool, + chain, + block_hashes, + progress, + ) + .await? + } + (None, None) => { + bigname_adapters::EnsV2ResolverSyncSummary::sync_for_block_hashes( + pool, + chain, + block_hashes, + ) + .await? + } + }; + log_adapter_call_timing( + chain, + "ens_v2_resolver", + "sync_for_block_hashes", + block_hashes.len(), + source_scope_target_count, + summary.scanned_log_count, + summary.matched_log_count, + summary.total_synced_count, + summary.total_inserted_count, + adapter_started.elapsed().as_millis(), + ); + log_ens_v2_resolver_sync_summary(chain, &summary); + aggregate.add_counts( + summary.scanned_log_count, + summary.matched_log_count, + summary.total_synced_count, + summary.total_inserted_count, + ); + record_adapter_progress(pool, progress).await?; + } + if mode.selects_adapter(source_scope, NormalizedEventReplayAdapter::EnsV2Permissions) { + ensure_raw_fact_adapter_allowed(mode, NormalizedEventReplayAdapter::EnsV2Permissions)?; + let adapter_started = Instant::now(); + info!( + service = "indexer", + command = "adapter-sync", + chain, + adapter = "ens_v2_permissions", + block_hash_count = block_hashes.len(), + source_scope_target_count, + adapter_sync_mode = ?mode, + "adapter sync call started" + ); + let summary = match (source_scope, progress.as_deref_mut()) { + (Some(source_scope), Some(progress)) => { + bigname_adapters::EnsV2PermissionsSyncSummary::sync_for_block_hashes_with_source_scope_and_progress( + pool, + chain, + block_hashes, + source_scope, + progress, + ) + .await? + } + (Some(source_scope), None) => { + bigname_adapters::EnsV2PermissionsSyncSummary::sync_for_block_hashes_with_source_scope( + pool, + chain, + block_hashes, + source_scope, + ) + .await? + } + (None, Some(progress)) => { + bigname_adapters::EnsV2PermissionsSyncSummary::sync_for_block_hashes_with_progress( + pool, + chain, + block_hashes, + progress, + ) + .await? + } + (None, None) => { + bigname_adapters::EnsV2PermissionsSyncSummary::sync_for_block_hashes( + pool, + chain, + block_hashes, + ) + .await? + } + }; + log_adapter_call_timing( + chain, + "ens_v2_permissions", + "sync_for_block_hashes", + block_hashes.len(), + source_scope_target_count, + summary.scanned_log_count, + summary.matched_log_count, + summary.total_synced_count, + summary.total_inserted_count, + adapter_started.elapsed().as_millis(), + ); + log_ens_v2_permissions_sync_summary(chain, &summary); + aggregate.add_counts( + summary.scanned_log_count, + summary.matched_log_count, + summary.total_synced_count, + summary.total_inserted_count, + ); + record_adapter_progress(pool, progress).await?; + } + Ok(()) +} diff --git a/apps/indexer/src/main/reconciliation/adapter_sync/entrypoints.rs b/apps/indexer/src/main/reconciliation/adapter_sync/entrypoints.rs index 4f858f0b..34c77776 100644 --- a/apps/indexer/src/main/reconciliation/adapter_sync/entrypoints.rs +++ b/apps/indexer/src/main/reconciliation/adapter_sync/entrypoints.rs @@ -75,6 +75,7 @@ pub(crate) async fn sync_adapter_state_from_persisted_raw_payloads( PersistedRawPayloadAdapterSyncMode::LiveOrBackfill, true, FullSourceReconciliationScope::default(), + &mut None, ) .await } @@ -91,10 +92,12 @@ pub(crate) async fn sync_live_adapter_state_from_persisted_raw_payloads( chain, block_hashes, false, + &mut None, ) .await } +#[allow(dead_code)] pub(crate) async fn sync_live_adapter_state_from_persisted_raw_payloads_after_reorg( pool: &sqlx::PgPool, deployment_profile: &str, @@ -107,6 +110,43 @@ pub(crate) async fn sync_live_adapter_state_from_persisted_raw_payloads_after_re chain, block_hashes, true, + &mut None, + ) + .await +} + +pub(crate) async fn sync_live_adapter_state_from_persisted_raw_payloads_with_progress( + pool: &sqlx::PgPool, + deployment_profile: &str, + chain: &str, + block_hashes: &[String], + progress: &mut Option<&mut dyn bigname_adapters::StartupAdapterProgress>, +) -> Result { + sync_live_adapter_state_from_persisted_raw_payloads_with_reorg_repair( + pool, + deployment_profile, + chain, + block_hashes, + false, + progress, + ) + .await +} + +pub(crate) async fn sync_live_adapter_state_from_persisted_raw_payloads_after_reorg_with_progress( + pool: &sqlx::PgPool, + deployment_profile: &str, + chain: &str, + block_hashes: &[String], + progress: &mut Option<&mut dyn bigname_adapters::StartupAdapterProgress>, +) -> Result { + sync_live_adapter_state_from_persisted_raw_payloads_with_reorg_repair( + pool, + deployment_profile, + chain, + block_hashes, + true, + progress, ) .await } @@ -117,6 +157,7 @@ async fn sync_live_adapter_state_from_persisted_raw_payloads_with_reorg_repair( chain: &str, block_hashes: &[String], reconcile_discovery_full_sources: bool, + progress: &mut Option<&mut dyn bigname_adapters::StartupAdapterProgress>, ) -> Result { info!( service = "indexer", @@ -152,17 +193,19 @@ async fn sync_live_adapter_state_from_persisted_raw_payloads_with_reorg_repair( PersistedRawPayloadAdapterSyncMode::LivePoll, true, full_source_reconciliation, + progress, ) .await } -pub(crate) async fn sync_replay_normalized_events_from_persisted_raw_payloads( +pub(crate) async fn sync_replay_normalized_events_from_persisted_raw_payloads_with_progress( pool: &sqlx::PgPool, chain: &str, block_hashes: &[String], source_scope: Option<&[(String, String, i64, i64)]>, canonical_raw_log_count: usize, replay_contract_plan: RawFactReplayContractPlan, + progress: &mut Option<&mut dyn bigname_adapters::StartupAdapterProgress>, ) -> Result { sync_adapter_state_from_persisted_raw_payloads_with_mode( pool, @@ -176,6 +219,7 @@ pub(crate) async fn sync_replay_normalized_events_from_persisted_raw_payloads( }, false, FullSourceReconciliationScope::default(), + progress, ) .await } @@ -195,6 +239,7 @@ pub(crate) async fn sync_adapter_state_from_scoped_persisted_raw_payloads( PersistedRawPayloadAdapterSyncMode::LiveOrBackfill, false, FullSourceReconciliationScope::default(), + &mut None, ) .await .map(|_| ())?; diff --git a/apps/indexer/src/main/reconciliation/adapter_sync/full_closure.rs b/apps/indexer/src/main/reconciliation/adapter_sync/full_closure.rs index 622846b2..97cde23d 100644 --- a/apps/indexer/src/main/reconciliation/adapter_sync/full_closure.rs +++ b/apps/indexer/src/main/reconciliation/adapter_sync/full_closure.rs @@ -1,20 +1,17 @@ use std::{future::Future, time::Instant}; -use anyhow::{Context, Result, ensure}; -use bigname_storage::{ - RawLogStagingInputVersion, acquire_raw_log_staging_read_guard, - load_raw_log_staging_input_version, -}; -#[cfg(target_os = "linux")] -use tracing::info; - -use crate::resolver_profile_convergence::journal_resolver_profile_authority_if_epoch_changed; use crate::runtime::{ log_ens_v1_reverse_claim_sync_summary, log_ens_v1_subregistry_discovery_sync_summary, log_ens_v1_unwrapped_authority_sync_summary, log_ens_v2_permissions_sync_summary, log_ens_v2_registrar_sync_summary, log_ens_v2_registry_resource_surface_sync_summary, log_ens_v2_resolver_sync_summary, }; +use anyhow::{Context, Result, ensure}; +use bigname_adapters::StartupAdapterProgress; +use bigname_storage::{ + RawLogStagingInputVersion, acquire_raw_log_staging_read_guard, + load_raw_log_staging_input_version, +}; use super::sync_logging::log_adapter_call_timing; use crate::reconciliation::{ @@ -27,6 +24,8 @@ use crate::reconciliation::{ #[path = "full_closure/automatic.rs"] mod automatic; +#[path = "full_closure/heartbeat.rs"] +mod heartbeat; #[path = "full_closure/ownership.rs"] mod ownership; #[path = "full_closure/reverse_claim.rs"] @@ -38,16 +37,15 @@ pub(crate) use automatic::{ }; #[cfg(test)] pub(crate) use automatic::{install_after_stateless_failure, install_stateless_page_observer}; +use heartbeat::{ + journal_full_closure_authority_with_progress, record_full_closure_progress, + trim_allocator_after_full_closure_adapter, +}; #[cfg(test)] pub(crate) use ownership::install_ownership_release_test_hook; use ownership::with_full_closure_replay_lock; use reverse_claim::sync_ens_v1_reverse_claim_range_in_pages; -#[cfg(target_os = "linux")] -unsafe extern "C" { - fn malloc_trim(pad: usize) -> i32; -} - #[cfg(test)] #[expect(clippy::too_many_arguments)] pub(crate) async fn sync_full_closure_normalized_events_from_persisted_raw_payloads( @@ -70,6 +68,7 @@ pub(crate) async fn sync_full_closure_normalized_events_from_persisted_raw_paylo adapters, max_raw_logs_per_page, FullClosureCheckpointCompletion::Retain, + &mut None, ) .await } @@ -95,6 +94,7 @@ pub(crate) async fn sync_manual_full_closure_normalized_events_from_persisted_ra adapters, max_raw_logs_per_page, FullClosureCheckpointCompletion::ClearOnSuccess, + &mut None, ) .await? .summary) @@ -121,6 +121,7 @@ async fn sync_full_closure_with_checkpoint_completion( adapters: &[NormalizedEventReplayAdapter], max_raw_logs_per_page: usize, checkpoint_completion: FullClosureCheckpointCompletion, + progress: &mut Option<&mut dyn StartupAdapterProgress>, ) -> Result { let (result, ()) = sync_full_closure_with_checkpoint_completion_and_prelude( pool, @@ -132,6 +133,7 @@ async fn sync_full_closure_with_checkpoint_completion( adapters, max_raw_logs_per_page, checkpoint_completion, + progress, || async { Ok(()) }, ) .await?; @@ -149,6 +151,7 @@ async fn sync_full_closure_with_checkpoint_completion_and_prelude, prelude: Prelude, ) -> Result<(FullClosureSyncResult, T)> where @@ -193,6 +196,7 @@ where target_block_number, adapters, max_raw_logs_per_page, + progress, ) .await?; let raw_log_guard = if adapters.is_empty() { @@ -244,6 +248,7 @@ async fn sync_full_closure_normalized_events_without_lock( target_block_number: i64, adapters: &[NormalizedEventReplayAdapter], max_raw_logs_per_page: usize, + progress: &mut Option<&mut dyn StartupAdapterProgress>, ) -> Result { let mut aggregate = PersistedRawPayloadAdapterSyncSummary::default(); let checkpoint_context = bigname_adapters::ReplayAdapterCheckpointContext { @@ -262,6 +267,7 @@ async fn sync_full_closure_normalized_events_without_lock( target_block_number, replay_contract(NormalizedEventReplayAdapter::EnsV1ReverseClaim).source_families, max_raw_logs_per_page, + progress, ) .await?; log_adapter_call_timing( @@ -283,21 +289,36 @@ async fn sync_full_closure_normalized_events_without_lock( summary.total_synced_count, summary.total_inserted_count, ); + record_full_closure_progress(pool, progress).await?; } if adapters.contains(&NormalizedEventReplayAdapter::EnsV1SubregistryDiscovery) { - let epoch_guard = journal_resolver_profile_authority_if_epoch_changed(pool, chain).await?; + let epoch_guard = + journal_full_closure_authority_with_progress(pool, chain, progress).await?; aggregate.resolver_profile_authority_epoch_guard_count += epoch_guard.epoch_guard_count; aggregate.resolver_profile_authority_scan_count += epoch_guard.authority_scan_count; let adapter_started = Instant::now(); - let summary = - bigname_adapters::sync_ens_v1_subregistry_discovery_with_replay_checkpoint_and_log_limit( - pool, - chain, - &checkpoint_context, - max_raw_logs_per_page, - ) - .await?; + let summary = match progress.as_deref_mut() { + Some(progress) => { + bigname_adapters::sync_ens_v1_subregistry_discovery_with_replay_checkpoint_and_log_limit_and_progress( + pool, + chain, + &checkpoint_context, + max_raw_logs_per_page, + progress, + ) + .await? + } + None => { + bigname_adapters::sync_ens_v1_subregistry_discovery_with_replay_checkpoint_and_log_limit( + pool, + chain, + &checkpoint_context, + max_raw_logs_per_page, + ) + .await? + } + }; log_adapter_call_timing( chain, "ens_v1_subregistry_discovery", @@ -317,22 +338,37 @@ async fn sync_full_closure_normalized_events_without_lock( summary.total_normalized_event_count, summary.total_normalized_event_inserted_count, ); - let epoch_guard = journal_resolver_profile_authority_if_epoch_changed(pool, chain).await?; + let epoch_guard = + journal_full_closure_authority_with_progress(pool, chain, progress).await?; aggregate.resolver_profile_authority_epoch_guard_count += epoch_guard.epoch_guard_count; aggregate.resolver_profile_authority_scan_count += epoch_guard.authority_scan_count; trim_allocator_after_full_closure_adapter("ens_v1_subregistry_discovery"); + record_full_closure_progress(pool, progress).await?; } if adapters.contains(&NormalizedEventReplayAdapter::EnsV1UnwrappedAuthority) { let adapter_started = Instant::now(); - let summary = - bigname_adapters::sync_ens_v1_unwrapped_authority_with_replay_checkpoint_and_log_limit( - pool, - chain, - &checkpoint_context, - max_raw_logs_per_page, - ) - .await?; + let summary = match progress.as_deref_mut() { + Some(progress) => { + bigname_adapters::sync_ens_v1_unwrapped_authority_with_replay_checkpoint_and_log_limit_and_progress( + pool, + chain, + &checkpoint_context, + max_raw_logs_per_page, + progress, + ) + .await? + } + None => { + bigname_adapters::sync_ens_v1_unwrapped_authority_with_replay_checkpoint_and_log_limit( + pool, + chain, + &checkpoint_context, + max_raw_logs_per_page, + ) + .await? + } + }; log_adapter_call_timing( chain, "ens_v1_unwrapped_authority", @@ -353,16 +389,30 @@ async fn sync_full_closure_normalized_events_without_lock( summary.total_normalized_event_inserted_count, ); trim_allocator_after_full_closure_adapter("ens_v1_unwrapped_authority"); + record_full_closure_progress(pool, progress).await?; } if adapters.contains(&NormalizedEventReplayAdapter::EnsV2RegistryResourceSurface) { let adapter_started = Instant::now(); - let summary = bigname_adapters::sync_ens_v2_registry_resource_surface_through_block( - pool, - chain, - target_block_number, - ) - .await?; + let summary = match progress.as_deref_mut() { + Some(progress) => { + bigname_adapters::sync_ens_v2_registry_resource_surface_through_block_with_progress( + pool, + chain, + target_block_number, + progress, + ) + .await? + } + None => { + bigname_adapters::sync_ens_v2_registry_resource_surface_through_block( + pool, + chain, + target_block_number, + ) + .await? + } + }; log_adapter_call_timing( chain, "ens_v2_registry_resource_surface", @@ -383,13 +433,30 @@ async fn sync_full_closure_normalized_events_without_lock( summary.total_normalized_event_inserted_count, ); trim_allocator_after_full_closure_adapter("ens_v2_registry_resource_surface"); + record_full_closure_progress(pool, progress).await?; } if adapters.contains(&NormalizedEventReplayAdapter::EnsV2Registrar) { let adapter_started = Instant::now(); - let summary = - bigname_adapters::sync_ens_v2_registrar_through_block(pool, chain, target_block_number) - .await?; + let summary = match progress.as_deref_mut() { + Some(progress) => { + bigname_adapters::sync_ens_v2_registrar_through_block_with_progress( + pool, + chain, + target_block_number, + progress, + ) + .await? + } + None => { + bigname_adapters::sync_ens_v2_registrar_through_block( + pool, + chain, + target_block_number, + ) + .await? + } + }; log_adapter_call_timing( chain, "ens_v2_registrar", @@ -410,13 +477,30 @@ async fn sync_full_closure_normalized_events_without_lock( summary.total_inserted_count, ); trim_allocator_after_full_closure_adapter("ens_v2_registrar"); + record_full_closure_progress(pool, progress).await?; } if adapters.contains(&NormalizedEventReplayAdapter::EnsV2Resolver) { let adapter_started = Instant::now(); - let summary = - bigname_adapters::sync_ens_v2_resolver_through_block(pool, chain, target_block_number) - .await?; + let summary = match progress.as_deref_mut() { + Some(progress) => { + bigname_adapters::sync_ens_v2_resolver_through_block_with_progress( + pool, + chain, + target_block_number, + progress, + ) + .await? + } + None => { + bigname_adapters::sync_ens_v2_resolver_through_block( + pool, + chain, + target_block_number, + ) + .await? + } + }; log_adapter_call_timing( chain, "ens_v2_resolver", @@ -437,16 +521,30 @@ async fn sync_full_closure_normalized_events_without_lock( summary.total_inserted_count, ); trim_allocator_after_full_closure_adapter("ens_v2_resolver"); + record_full_closure_progress(pool, progress).await?; } if adapters.contains(&NormalizedEventReplayAdapter::EnsV2Permissions) { let adapter_started = Instant::now(); - let summary = bigname_adapters::sync_ens_v2_permissions_through_block( - pool, - chain, - target_block_number, - ) - .await?; + let summary = match progress.as_deref_mut() { + Some(progress) => { + bigname_adapters::sync_ens_v2_permissions_through_block_with_progress( + pool, + chain, + target_block_number, + progress, + ) + .await? + } + None => { + bigname_adapters::sync_ens_v2_permissions_through_block( + pool, + chain, + target_block_number, + ) + .await? + } + }; log_adapter_call_timing( chain, "ens_v2_permissions", @@ -467,23 +565,8 @@ async fn sync_full_closure_normalized_events_without_lock( summary.total_inserted_count, ); trim_allocator_after_full_closure_adapter("ens_v2_permissions"); + record_full_closure_progress(pool, progress).await?; } Ok(aggregate) } - -fn trim_allocator_after_full_closure_adapter(adapter: &'static str) { - #[cfg(target_os = "linux")] - { - let malloc_trim_result = unsafe { malloc_trim(0) }; - info!( - service = "indexer", - adapter, malloc_trim_result, "allocator trim requested after full closure adapter" - ); - } - - #[cfg(not(target_os = "linux"))] - { - let _ = adapter; - } -} diff --git a/apps/indexer/src/main/reconciliation/adapter_sync/full_closure/automatic.rs b/apps/indexer/src/main/reconciliation/adapter_sync/full_closure/automatic.rs index cc01960f..8589fead 100644 --- a/apps/indexer/src/main/reconciliation/adapter_sync/full_closure/automatic.rs +++ b/apps/indexer/src/main/reconciliation/adapter_sync/full_closure/automatic.rs @@ -7,11 +7,13 @@ use std::{ }; use anyhow::{Context, Result, ensure}; +use bigname_adapters::StartupAdapterProgress; use tracing::info; use crate::reconciliation::{ replay::{ - NormalizedEventReplayAdapter, replay_stateless_normalized_events_before_full_closure, + NormalizedEventReplayAdapter, + replay_stateless_normalized_events_before_full_closure_with_progress, select_log_bounded_replay_to_block, }, types::{ @@ -63,6 +65,8 @@ pub(crate) async fn sync_automatic_two_phase_full_closure_normalized_events( stateless_ranges: &[(i64, i64)], adapters: &[NormalizedEventReplayAdapter], max_raw_logs_per_page: usize, + stateless_progress: &mut dyn StartupAdapterProgress, + closure_progress: &mut dyn StartupAdapterProgress, ) -> Result { info!( service = "indexer", @@ -87,6 +91,7 @@ pub(crate) async fn sync_automatic_two_phase_full_closure_normalized_events( adapters, max_raw_logs_per_page, FullClosureCheckpointCompletion::Retain, + &mut Some(closure_progress), move || async move { let stateless = replay_stateless_normalized_events_for_ranges( pool, @@ -94,6 +99,7 @@ pub(crate) async fn sync_automatic_two_phase_full_closure_normalized_events( chain, stateless_ranges, max_raw_logs_per_page, + stateless_progress, ) .await?; stateless_replay_completed_in_prelude.store(true, Ordering::Release); @@ -123,6 +129,7 @@ async fn replay_stateless_normalized_events_for_ranges( chain: &str, stateless_ranges: &[(i64, i64)], max_raw_logs_per_page: usize, + progress: &mut dyn StartupAdapterProgress, ) -> Result { ensure!( !stateless_ranges.is_empty(), @@ -150,6 +157,7 @@ async fn replay_stateless_normalized_events_for_ranges( range_start_block_number, target_block_number, max_raw_logs_per_page, + progress, ) .await?; aggregate.selected_block_count += range.selected_block_count; @@ -172,6 +180,7 @@ async fn replay_stateless_normalized_events_in_pages( range_start_block_number: i64, target_block_number: i64, max_raw_logs_per_page: usize, + progress: &mut dyn StartupAdapterProgress, ) -> Result { ensure!( range_start_block_number <= target_block_number, @@ -208,7 +217,7 @@ async fn replay_stateless_normalized_events_in_pages( #[cfg(test)] test_hook::record_stateless_page(pool, deployment_profile, chain, from_block, to_block) .await?; - let page = replay_stateless_normalized_events_before_full_closure( + let page = replay_stateless_normalized_events_before_full_closure_with_progress( pool, RawFactNormalizedEventReplayRequest { deployment_profile: deployment_profile.to_owned(), @@ -218,6 +227,7 @@ async fn replay_stateless_normalized_events_in_pages( to_block, }, }, + progress, ) .await?; aggregate.selected_block_count += page.selected_block_count; @@ -229,6 +239,7 @@ async fn replay_stateless_normalized_events_in_pages( aggregate .stateless_replay_authority .add(&page.stateless_replay_authority); + progress.record(pool).await?; if to_block == target_block_number { break; diff --git a/apps/indexer/src/main/reconciliation/adapter_sync/full_closure/heartbeat.rs b/apps/indexer/src/main/reconciliation/adapter_sync/full_closure/heartbeat.rs new file mode 100644 index 00000000..22f33218 --- /dev/null +++ b/apps/indexer/src/main/reconciliation/adapter_sync/full_closure/heartbeat.rs @@ -0,0 +1,54 @@ +use anyhow::Result; +use bigname_adapters::StartupAdapterProgress; + +use crate::resolver_profile_convergence::{ + journal_resolver_profile_authority_if_epoch_changed, + journal_resolver_profile_authority_if_epoch_changed_with_progress, +}; + +#[cfg(target_os = "linux")] +unsafe extern "C" { + fn malloc_trim(pad: usize) -> i32; +} + +pub(super) async fn record_full_closure_progress( + pool: &sqlx::PgPool, + progress: &mut Option<&mut dyn StartupAdapterProgress>, +) -> Result<()> { + if let Some(progress) = progress.as_deref_mut() { + progress.record(pool).await?; + } + Ok(()) +} + +pub(super) async fn journal_full_closure_authority_with_progress( + pool: &sqlx::PgPool, + chain: &str, + progress: &mut Option<&mut dyn StartupAdapterProgress>, +) -> Result { + match progress.as_deref_mut() { + Some(progress) => { + journal_resolver_profile_authority_if_epoch_changed_with_progress(pool, chain, progress) + .await + } + None => journal_resolver_profile_authority_if_epoch_changed(pool, chain).await, + } +} + +pub(super) fn trim_allocator_after_full_closure_adapter(adapter: &'static str) { + #[cfg(target_os = "linux")] + { + let malloc_trim_result = unsafe { malloc_trim(0) }; + tracing::info!( + service = "indexer", + adapter, + malloc_trim_result, + "allocator trim requested after full closure adapter" + ); + } + + #[cfg(not(target_os = "linux"))] + { + let _ = adapter; + } +} diff --git a/apps/indexer/src/main/reconciliation/adapter_sync/full_closure/reverse_claim.rs b/apps/indexer/src/main/reconciliation/adapter_sync/full_closure/reverse_claim.rs index f7fb2927..7064d0c4 100644 --- a/apps/indexer/src/main/reconciliation/adapter_sync/full_closure/reverse_claim.rs +++ b/apps/indexer/src/main/reconciliation/adapter_sync/full_closure/reverse_claim.rs @@ -1,6 +1,7 @@ use std::collections::BTreeMap; use anyhow::{Context, Result, ensure}; +use bigname_adapters::StartupAdapterProgress; use tracing::info; use crate::{ @@ -15,6 +16,7 @@ pub(super) async fn sync_ens_v1_reverse_claim_range_in_pages( target_block_number: i64, source_families: &[&str], max_raw_logs_per_page: usize, + progress: &mut Option<&mut dyn StartupAdapterProgress>, ) -> Result { ensure!( max_raw_logs_per_page > 0, @@ -61,15 +63,31 @@ pub(super) async fn sync_ens_v1_reverse_claim_range_in_pages( &reverse_scope.targets, ) .await?; - let page_summary = - bigname_adapters::EnsV1ReverseClaimSyncSummary::sync_for_block_hashes_with_source_scope( - pool, - chain, - &page_selection.block_hashes, - &reverse_scope.adapter_sync_scope, - ) - .await?; + let page_summary = match progress.as_deref_mut() { + Some(progress) => { + bigname_adapters::EnsV1ReverseClaimSyncSummary::sync_for_block_hashes_with_source_scope_and_progress( + pool, + chain, + &page_selection.block_hashes, + &reverse_scope.adapter_sync_scope, + progress, + ) + .await? + } + None => { + bigname_adapters::EnsV1ReverseClaimSyncSummary::sync_for_block_hashes_with_source_scope( + pool, + chain, + &page_selection.block_hashes, + &reverse_scope.adapter_sync_scope, + ) + .await? + } + }; merge_reverse_claim_summary(&mut aggregate, page_summary); + if let Some(progress) = progress.as_deref_mut() { + progress.record(pool).await?; + } page_count += 1; info!( service = "indexer", diff --git a/apps/indexer/src/main/reconciliation/adapter_sync/progress.rs b/apps/indexer/src/main/reconciliation/adapter_sync/progress.rs new file mode 100644 index 00000000..77e36883 --- /dev/null +++ b/apps/indexer/src/main/reconciliation/adapter_sync/progress.rs @@ -0,0 +1,31 @@ +use anyhow::Result; +use bigname_adapters::StartupAdapterProgress; + +use crate::resolver_profile_convergence::{ + journal_resolver_profile_authority_if_epoch_changed, + journal_resolver_profile_authority_if_epoch_changed_with_progress, +}; + +pub(super) async fn record_adapter_progress( + pool: &sqlx::PgPool, + progress: &mut Option<&mut dyn StartupAdapterProgress>, +) -> Result<()> { + if let Some(progress) = progress.as_deref_mut() { + progress.record(pool).await?; + } + Ok(()) +} + +pub(super) async fn journal_authority_epoch_with_progress( + pool: &sqlx::PgPool, + chain: &str, + progress: &mut Option<&mut dyn StartupAdapterProgress>, +) -> Result { + match progress.as_deref_mut() { + Some(progress) => { + journal_resolver_profile_authority_if_epoch_changed_with_progress(pool, chain, progress) + .await + } + None => journal_resolver_profile_authority_if_epoch_changed(pool, chain).await, + } +} diff --git a/apps/indexer/src/main/reconciliation/adapter_sync/stateless.rs b/apps/indexer/src/main/reconciliation/adapter_sync/stateless.rs index 2aa3c259..8ff989e8 100644 --- a/apps/indexer/src/main/reconciliation/adapter_sync/stateless.rs +++ b/apps/indexer/src/main/reconciliation/adapter_sync/stateless.rs @@ -1,6 +1,7 @@ use std::time::Instant; use anyhow::Result; +use bigname_adapters::StartupAdapterProgress; use tracing::info; use crate::runtime::{ @@ -9,6 +10,7 @@ use crate::runtime::{ use super::{ mode::{PersistedRawPayloadAdapterSyncMode, ensure_raw_fact_adapter_allowed}, + progress::record_adapter_progress, sync_logging::log_adapter_call_timing, }; use crate::reconciliation::{ @@ -22,6 +24,7 @@ pub(super) async fn sync_block_derived_for_mode( source_scope: Option<&[(String, String, i64, i64)]>, mode: PersistedRawPayloadAdapterSyncMode, aggregate: &mut PersistedRawPayloadAdapterSyncSummary, + progress: &mut Option<&mut dyn StartupAdapterProgress>, ) -> Result<()> { ensure_raw_fact_adapter_allowed( mode, @@ -40,11 +43,31 @@ pub(super) async fn sync_block_derived_for_mode( "adapter sync call started" ); - let (summary, authority) = match mode { - PersistedRawPayloadAdapterSyncMode::RawFactReplay { - canonical_raw_log_count, - .. - } if mode.uses_stateless_replay_authority() => { + let (summary, authority) = match (mode, progress.as_deref_mut()) { + ( + PersistedRawPayloadAdapterSyncMode::RawFactReplay { + canonical_raw_log_count, + .. + }, + Some(progress), + ) if mode.uses_stateless_replay_authority() => { + bigname_adapters::sync_block_derived_normalized_events_with_stateless_replay_authority_and_progress( + pool, + chain, + block_hashes, + source_scope, + canonical_raw_log_count, + progress, + ) + .await? + } + ( + PersistedRawPayloadAdapterSyncMode::RawFactReplay { + canonical_raw_log_count, + .. + }, + None, + ) if mode.uses_stateless_replay_authority() => { bigname_adapters::sync_block_derived_normalized_events_with_stateless_replay_authority( pool, chain, @@ -54,10 +77,31 @@ pub(super) async fn sync_block_derived_for_mode( ) .await? } - PersistedRawPayloadAdapterSyncMode::RawFactReplay { - canonical_raw_log_count, - .. - } => ( + ( + PersistedRawPayloadAdapterSyncMode::RawFactReplay { + canonical_raw_log_count, + .. + }, + Some(progress), + ) => ( + bigname_adapters::sync_block_derived_normalized_events_with_scanned_log_count_and_progress( + pool, + chain, + block_hashes, + source_scope, + canonical_raw_log_count, + progress, + ) + .await?, + bigname_storage::NormalizedEventReplayAuthoritySummary::default(), + ), + ( + PersistedRawPayloadAdapterSyncMode::RawFactReplay { + canonical_raw_log_count, + .. + }, + None, + ) => ( bigname_adapters::sync_block_derived_normalized_events_with_scanned_log_count( pool, chain, @@ -68,8 +112,26 @@ pub(super) async fn sync_block_derived_for_mode( .await?, bigname_storage::NormalizedEventReplayAuthoritySummary::default(), ), - PersistedRawPayloadAdapterSyncMode::LivePoll - | PersistedRawPayloadAdapterSyncMode::LiveOrBackfill => ( + ( + PersistedRawPayloadAdapterSyncMode::LivePoll + | PersistedRawPayloadAdapterSyncMode::LiveOrBackfill, + Some(progress), + ) => ( + bigname_adapters::sync_block_derived_normalized_events_with_progress( + pool, + chain, + block_hashes, + source_scope, + progress, + ) + .await?, + bigname_storage::NormalizedEventReplayAuthoritySummary::default(), + ), + ( + PersistedRawPayloadAdapterSyncMode::LivePoll + | PersistedRawPayloadAdapterSyncMode::LiveOrBackfill, + None, + ) => ( bigname_adapters::sync_block_derived_normalized_events( pool, chain, @@ -100,6 +162,7 @@ pub(super) async fn sync_block_derived_for_mode( summary.total_inserted_count, ); aggregate.add_stateless_replay_authority(&authority); + record_adapter_progress(pool, progress).await?; Ok(()) } @@ -110,6 +173,7 @@ pub(super) async fn sync_reverse_claim_for_mode( source_scope: Option<&[(String, String, i64, i64)]>, mode: PersistedRawPayloadAdapterSyncMode, aggregate: &mut PersistedRawPayloadAdapterSyncSummary, + progress: &mut Option<&mut dyn StartupAdapterProgress>, ) -> Result<()> { if !mode.selects_adapter( source_scope, @@ -136,8 +200,22 @@ pub(super) async fn sync_reverse_claim_for_mode( "adapter sync call started" ); - let (summary, authority) = match (source_scope, mode.uses_stateless_replay_authority()) { - (Some(source_scope), true) => { + let (summary, authority) = match ( + source_scope, + mode.uses_stateless_replay_authority(), + progress.as_deref_mut(), + ) { + (Some(source_scope), true, Some(progress)) => { + bigname_adapters::EnsV1ReverseClaimSyncSummary::sync_for_block_hashes_with_source_scope_and_stateless_replay_authority_and_progress( + pool, + chain, + block_hashes, + source_scope, + progress, + ) + .await? + } + (Some(source_scope), true, None) => { bigname_adapters::EnsV1ReverseClaimSyncSummary::sync_for_block_hashes_with_source_scope_and_stateless_replay_authority( pool, chain, @@ -146,7 +224,16 @@ pub(super) async fn sync_reverse_claim_for_mode( ) .await? } - (None, true) => { + (None, true, Some(progress)) => { + bigname_adapters::EnsV1ReverseClaimSyncSummary::sync_for_block_hashes_with_stateless_replay_authority_and_progress( + pool, + chain, + block_hashes, + progress, + ) + .await? + } + (None, true, None) => { bigname_adapters::EnsV1ReverseClaimSyncSummary::sync_for_block_hashes_with_stateless_replay_authority( pool, chain, @@ -154,7 +241,18 @@ pub(super) async fn sync_reverse_claim_for_mode( ) .await? } - (Some(source_scope), false) => ( + (Some(source_scope), false, Some(progress)) => ( + bigname_adapters::EnsV1ReverseClaimSyncSummary::sync_for_block_hashes_with_source_scope_and_progress( + pool, + chain, + block_hashes, + source_scope, + progress, + ) + .await?, + bigname_storage::NormalizedEventReplayAuthoritySummary::default(), + ), + (Some(source_scope), false, None) => ( bigname_adapters::EnsV1ReverseClaimSyncSummary::sync_for_block_hashes_with_source_scope( pool, chain, @@ -164,7 +262,17 @@ pub(super) async fn sync_reverse_claim_for_mode( .await?, bigname_storage::NormalizedEventReplayAuthoritySummary::default(), ), - (None, false) => ( + (None, false, Some(progress)) => ( + bigname_adapters::EnsV1ReverseClaimSyncSummary::sync_for_block_hashes_with_progress( + pool, + chain, + block_hashes, + progress, + ) + .await?, + bigname_storage::NormalizedEventReplayAuthoritySummary::default(), + ), + (None, false, None) => ( bigname_adapters::EnsV1ReverseClaimSyncSummary::sync_for_block_hashes( pool, chain, @@ -194,5 +302,6 @@ pub(super) async fn sync_reverse_claim_for_mode( summary.total_inserted_count, ); aggregate.add_stateless_replay_authority(&authority); + record_adapter_progress(pool, progress).await?; Ok(()) } diff --git a/apps/indexer/src/main/reconciliation/canonical.rs b/apps/indexer/src/main/reconciliation/canonical.rs index b74a99e1..8629e5fa 100644 --- a/apps/indexer/src/main/reconciliation/canonical.rs +++ b/apps/indexer/src/main/reconciliation/canonical.rs @@ -1,7 +1,7 @@ use anyhow::{Result, bail}; +use bigname_adapters::StartupAdapterProgress; use bigname_storage::{ CanonicalityState, ChainCheckpoint, chain_lineage_contains_ancestor, load_chain_lineage_block, - mark_chain_lineage_range_orphaned, upsert_chain_lineage_blocks_recanonicalizing_orphaned as upsert_recanonicalized_lineage_blocks, upsert_chain_lineage_blocks_without_snapshots, upsert_chain_lineage_blocks_without_snapshots_recanonicalizing_orphaned as upsert_recanonicalized_lineage_blocks_without_snapshots, @@ -11,7 +11,9 @@ use tracing::info; use crate::{ backfill::DEFAULT_HASH_PINNED_BACKFILL_CHUNK_BLOCKS, provider::{ChainProviderOps, ProviderBlock, ProviderHeadSnapshot}, - resolver_profile_convergence::drain_resolver_profile_input_changes, + resolver_profile_convergence::{ + drain_resolver_profile_input_changes, drain_resolver_profile_input_changes_with_progress, + }, runtime::IntakeChainTask, }; @@ -20,7 +22,7 @@ use super::{ head_change_set, lineage_block_to_provider, provider_block_to_checkpoint_ref, provider_block_to_lineage_with_header_audit_mode, }, - persistence::persist_reconciled_raw_state, + persistence::persist_reconciled_raw_state_with_progress, types::{ CanonicalReconciliation, CanonicalReconciliationStatus, ChainReconciliationOutcome, HeaderAuditMode, @@ -39,6 +41,8 @@ mod ens_v2_coverage_recovery; mod orphaning; #[path = "canonical/poll.rs"] mod poll; +#[path = "canonical/progress.rs"] +mod progress; #[path = "canonical/stored_lineage.rs"] pub(crate) mod stored_lineage; @@ -47,9 +51,25 @@ use cold_start::ColdStartCheckpoint; use contiguous_gap::reconcile_contiguous_checkpoint_gap; pub(crate) use ens_v2_coverage_recovery::{ EnsV2LiveCoverageRecoveryStatus, recover_ens_v2_live_coverage_requirement, + recover_ens_v2_live_coverage_requirement_with_progress, +}; +pub(crate) use orphaning::{orphan_canonical_branch, orphan_reorg_losing_branch_payloads}; +use orphaning::{ + orphan_canonical_branch_with_progress, orphan_reorg_losing_branch_payloads_with_progress, +}; +pub(crate) use poll::{ + poll_provider_heads, poll_provider_heads_with_adapter_sync, + poll_provider_heads_with_adapter_sync_and_progress, +}; +#[cfg(test)] +pub(crate) use progress::reconcile_canonical_head_with_adapter_progress; +pub(super) use progress::reconcile_intake_chain_task_with_adapter_sync_and_progress; +use progress::record_live_progress; +#[allow(unused_imports)] +pub(crate) use progress::{ + reconcile_canonical_head, reconcile_intake_chain_task, + reconcile_intake_chain_task_with_adapter_sync, }; -pub(crate) use orphaning::orphan_reorg_losing_branch_payloads; -pub(crate) use poll::{poll_provider_heads, poll_provider_heads_with_adapter_sync}; pub(crate) use stored_lineage::{ChainCoverageFrontiers, RawCodeBaselineFrontier}; use stored_lineage::{ StoredLineagePromotion, reconcile_large_checkpoint_gap_from_stored_lineage, @@ -59,29 +79,8 @@ use stored_lineage::{ const MAX_PARENT_FETCH_DEPTH: usize = 131_072; // Live polling fails closed before a large catch-up; hash-pinned backfill owns larger gaps. const MAX_LIVE_CONTIGUOUS_GAP_FILL_BLOCKS: i64 = DEFAULT_HASH_PINNED_BACKFILL_CHUNK_BLOCKS; -#[allow(dead_code)] -pub(crate) async fn reconcile_intake_chain_task( - pool: &sqlx::PgPool, - task: &IntakeChainTask, - provider: &(impl ChainProviderOps + ?Sized), -) -> Result> { - reconcile_intake_chain_task_with_adapter_sync( - pool, - "test", - task, - provider, - 0, - true, - HeaderAuditMode::Minimal, - &[], - &ChainCoverageFrontiers::default(), - None, - ) - .await -} - #[expect(clippy::too_many_arguments)] -pub(crate) async fn reconcile_intake_chain_task_with_adapter_sync( +pub(super) async fn reconcile_intake_chain_task_with_adapter_sync_inner( pool: &sqlx::PgPool, deployment_profile: &str, task: &IntakeChainTask, @@ -92,6 +91,7 @@ pub(crate) async fn reconcile_intake_chain_task_with_adapter_sync( event_silent_reverse_resolver_addresses: &[String], coverage_frontiers: &ChainCoverageFrontiers, latched_bootstrap_finalized_head: Option<&ProviderBlock>, + progress: &mut Option<&mut dyn StartupAdapterProgress>, ) -> Result> { let heads = provider.fetch_chain_heads().await?; reconcile_fetched_heads_with_gap_policy( @@ -106,6 +106,7 @@ pub(crate) async fn reconcile_intake_chain_task_with_adapter_sync( event_silent_reverse_resolver_addresses, coverage_frontiers, latched_bootstrap_finalized_head, + progress, ) .await } @@ -129,6 +130,7 @@ pub(crate) async fn reconcile_fetched_heads( &[], &ChainCoverageFrontiers::default(), None, + &mut None, ) .await } @@ -157,6 +159,7 @@ pub(crate) async fn reconcile_fetched_heads_with_adapter_sync( event_silent_reverse_resolver_addresses, coverage_frontiers, None, + &mut None, ) .await } @@ -174,6 +177,7 @@ async fn reconcile_fetched_heads_with_gap_policy( event_silent_reverse_resolver_addresses: &[String], coverage_frontiers: &ChainCoverageFrontiers, latched_bootstrap_finalized_head: Option<&ProviderBlock>, + progress: &mut Option<&mut dyn StartupAdapterProgress>, ) -> Result> { let mut stored_promotion_anchors = stored_lineage_promotion_anchors(heads); if let Some(latched_head) = latched_bootstrap_finalized_head @@ -183,7 +187,7 @@ async fn reconcile_fetched_heads_with_gap_policy( { stored_promotion_anchors.push(latched_head.clone()); } - let canonical = reconcile_canonical_head( + let canonical = reconcile_canonical_head_with_progress( pool, provider, &task.chain, @@ -192,6 +196,7 @@ async fn reconcile_fetched_heads_with_gap_policy( header_audit_mode, &stored_promotion_anchors, coverage_frontiers, + progress, ) .await?; let promotion_epoch = coverage_frontiers.take_promotion_epoch( @@ -200,12 +205,13 @@ async fn reconcile_fetched_heads_with_gap_policy( )?; let provider_head_change_set = head_change_set(task, heads, &canonical); if canonical.status == CanonicalReconciliationStatus::ReorgReconciled { - orphan_reorg_losing_branch_payloads( + orphan_reorg_losing_branch_payloads_with_progress( pool, &task.chain, task.checkpoint.canonical_block_hash.as_deref(), canonical.raw_orphan_stop_before_hash.as_deref(), coverage_frontiers, + progress, ) .await?; } @@ -217,6 +223,7 @@ async fn reconcile_fetched_heads_with_gap_policy( heads, &canonical, header_audit_mode, + progress, ) .await?; if fetched_checkpoint_ancestor_count > 0 { @@ -266,7 +273,7 @@ async fn reconcile_fetched_heads_with_gap_policy( }; let head_change_set = head_change_set(task, &accepted_heads, &canonical); - persist_reconciled_raw_state( + persist_reconciled_raw_state_with_progress( pool, deployment_profile, task, @@ -279,10 +286,16 @@ async fn reconcile_fetched_heads_with_gap_policy( header_audit_mode, event_silent_reverse_resolver_addresses, coverage_frontiers, + progress, ) .await?; if adapter_sync_enabled { - let profile_convergence = drain_resolver_profile_input_changes(pool).await?; + let profile_convergence = match progress.as_deref_mut() { + Some(progress) => { + drain_resolver_profile_input_changes_with_progress(pool, progress).await? + } + None => drain_resolver_profile_input_changes(pool).await?, + }; profile_convergence .ensure_chain_completion_allowed(&task.chain, "chain checkpoint advancement")?; } @@ -304,8 +317,19 @@ async fn reconcile_fetched_heads_with_gap_policy( { return Ok(None); } - let mut next_task = task.clone(); - next_task.checkpoint = next_checkpoint.clone(); + let mut addresses = Vec::with_capacity(task.addresses.len()); + for address_chunk in task.addresses.chunks(10_000) { + addresses.extend_from_slice(address_chunk); + record_live_progress(pool, progress).await?; + } + let next_task = IntakeChainTask { + chain: task.chain.clone(), + addresses, + manifest_root_entry_count: task.manifest_root_entry_count, + manifest_contract_entry_count: task.manifest_contract_entry_count, + discovery_edge_entry_count: task.discovery_edge_entry_count, + checkpoint: next_checkpoint.clone(), + }; Ok(Some(( next_task, ChainReconciliationOutcome { @@ -324,7 +348,7 @@ async fn reconcile_fetched_heads_with_gap_policy( } #[expect(clippy::too_many_arguments)] -pub(crate) async fn reconcile_canonical_head( +pub(super) async fn reconcile_canonical_head_with_progress( pool: &sqlx::PgPool, provider: &(impl ChainProviderOps + ?Sized), chain: &str, @@ -333,6 +357,7 @@ pub(crate) async fn reconcile_canonical_head( header_audit_mode: HeaderAuditMode, stored_lineage_promotion_anchors: &[ProviderBlock], coverage_frontiers: &ChainCoverageFrontiers, + progress: &mut Option<&mut dyn StartupAdapterProgress>, ) -> Result { let latest_hash = latest_head.block_hash.as_str(); let cold_start = @@ -387,6 +412,7 @@ pub(crate) async fn reconcile_canonical_head( current_canonical_number, latest_head, header_audit_mode, + progress, ) .await? { @@ -402,6 +428,7 @@ pub(crate) async fn reconcile_canonical_head( latest_head, stored_lineage_promotion_anchors, coverage_frontiers, + progress, ) .await? { @@ -463,6 +490,7 @@ pub(crate) async fn reconcile_canonical_head( cursor = lineage_block_to_provider(&stored_parent); path.push(cursor.clone()); + record_live_progress(pool, progress).await?; continue; } @@ -483,6 +511,7 @@ pub(crate) async fn reconcile_canonical_head( cursor = fetched_parent.clone(); path.push(fetched_parent); + record_live_progress(pool, progress).await?; } if common_ancestor_hash.is_none() { @@ -502,7 +531,10 @@ pub(crate) async fn reconcile_canonical_head( ) }) .collect::>(); - upsert_chain_lineage_blocks_without_snapshots(pool, &lineage_blocks).await?; + for block_chunk in lineage_blocks.chunks(32) { + upsert_chain_lineage_blocks_without_snapshots(pool, block_chunk).await?; + record_live_progress(pool, progress).await?; + } return Ok(CanonicalReconciliation { status: CanonicalReconciliationStatus::AwaitingAncestor, @@ -525,11 +557,12 @@ pub(crate) async fn reconcile_canonical_head( CanonicalReconciliationStatus::GapBackfilled } } else { - orphaned_block_count = orphan_canonical_branch( + orphaned_block_count = orphan_canonical_branch_with_progress( pool, chain, current_canonical_hash.expect("current checkpoint must exist"), Some(common_ancestor_hash.as_str()), + progress, ) .await?; CanonicalReconciliationStatus::ReorgReconciled @@ -547,7 +580,10 @@ pub(crate) async fn reconcile_canonical_head( ) }) .collect::>(); - upsert_recanonicalized_lineage_blocks_without_snapshots(pool, &lineage_blocks).await?; + for block_chunk in lineage_blocks.chunks(32) { + upsert_recanonicalized_lineage_blocks_without_snapshots(pool, block_chunk).await?; + record_live_progress(pool, progress).await?; + } Ok(CanonicalReconciliation { status, @@ -559,26 +595,3 @@ pub(crate) async fn reconcile_canonical_head( .then_some(common_ancestor_hash), }) } - -pub(crate) async fn orphan_canonical_branch( - pool: &sqlx::PgPool, - chain: &str, - from_hash: &str, - stop_before_hash: Option<&str>, -) -> Result { - let mut orphaned_block_count = 0usize; - let mut cursor_hash = Some(from_hash.to_owned()); - - while let Some(block_hash) = cursor_hash { - if Some(block_hash.as_str()) == stop_before_hash { - break; - } - - let snapshots = - mark_chain_lineage_range_orphaned(pool, chain, &block_hash, stop_before_hash).await?; - orphaned_block_count += snapshots.len(); - cursor_hash = None; - } - - Ok(orphaned_block_count) -} diff --git a/apps/indexer/src/main/reconciliation/canonical/checkpoints.rs b/apps/indexer/src/main/reconciliation/canonical/checkpoints.rs index 05378818..07649bb7 100644 --- a/apps/indexer/src/main/reconciliation/canonical/checkpoints.rs +++ b/apps/indexer/src/main/reconciliation/canonical/checkpoints.rs @@ -1,4 +1,5 @@ use anyhow::Result; +use bigname_adapters::StartupAdapterProgress; use bigname_storage::{ CanonicalityState, CheckpointBlockRef, chain_lineage_contains_ancestor, load_chain_lineage_block, upsert_chain_lineage_blocks_recanonicalizing_orphaned, @@ -23,6 +24,7 @@ pub(super) async fn fill_checkpoint_ancestor_path( heads: &ProviderHeadSnapshot, canonical: &CanonicalReconciliation, header_audit_mode: HeaderAuditMode, + progress: &mut Option<&mut dyn StartupAdapterProgress>, ) -> Result { let Some(canonical_update) = &canonical.canonical else { return Ok(0); @@ -73,11 +75,22 @@ pub(super) async fn fill_checkpoint_ancestor_path( )], ) .await?; + record_progress(pool, progress).await?; } Ok(fetched_parent_count) } +async fn record_progress( + pool: &sqlx::PgPool, + progress: &mut Option<&mut dyn StartupAdapterProgress>, +) -> Result<()> { + if let Some(progress) = progress.as_deref_mut() { + progress.record(pool).await?; + } + Ok(()) +} + pub(super) async fn checkpoint_update_for_head( pool: &sqlx::PgPool, chain: &str, diff --git a/apps/indexer/src/main/reconciliation/canonical/contiguous_gap.rs b/apps/indexer/src/main/reconciliation/canonical/contiguous_gap.rs index 77cafb88..0d0441f9 100644 --- a/apps/indexer/src/main/reconciliation/canonical/contiguous_gap.rs +++ b/apps/indexer/src/main/reconciliation/canonical/contiguous_gap.rs @@ -1,4 +1,5 @@ use anyhow::Result; +use bigname_adapters::StartupAdapterProgress; use bigname_storage::{ CanonicalityState, upsert_chain_lineage_blocks_without_snapshots_recanonicalizing_orphaned as upsert_recanonicalized_lineage_blocks_without_snapshots, @@ -12,6 +13,9 @@ use super::super::{ }; use super::{MAX_LIVE_CONTIGUOUS_GAP_FILL_BLOCKS, MAX_PARENT_FETCH_DEPTH}; +const LIVE_GAP_PROGRESS_BLOCKS: usize = 32; + +#[expect(clippy::too_many_arguments)] pub(super) async fn reconcile_contiguous_checkpoint_gap( pool: &sqlx::PgPool, provider: &(impl ChainProviderOps + ?Sized), @@ -20,6 +24,7 @@ pub(super) async fn reconcile_contiguous_checkpoint_gap( current_canonical_number: i64, latest_head: &ProviderBlock, header_audit_mode: HeaderAuditMode, + progress: &mut Option<&mut dyn StartupAdapterProgress>, ) -> Result> { if latest_head.block_number <= current_canonical_number { return Ok(None); @@ -34,9 +39,15 @@ pub(super) async fn reconcile_contiguous_checkpoint_gap( let block_numbers = ((current_canonical_number + 1)..=latest_head.block_number).collect::>(); - let resolved_blocks = provider - .fetch_block_hashes_by_numbers(&block_numbers) - .await?; + let mut resolved_blocks = Vec::with_capacity(block_numbers.len()); + for block_number_chunk in block_numbers.chunks(LIVE_GAP_PROGRESS_BLOCKS) { + resolved_blocks.extend( + provider + .fetch_block_hashes_by_numbers(block_number_chunk) + .await?, + ); + record_progress(pool, progress).await?; + } if resolved_blocks .last() .map(|block| block.block_hash.as_str()) @@ -45,9 +56,15 @@ pub(super) async fn reconcile_contiguous_checkpoint_gap( return Ok(None); } - let mut path = provider - .fetch_block_headers_by_hashes(&resolved_blocks) - .await?; + let mut path = Vec::with_capacity(resolved_blocks.len()); + for resolved_block_chunk in resolved_blocks.chunks(LIVE_GAP_PROGRESS_BLOCKS) { + path.extend( + provider + .fetch_block_headers_by_hashes(resolved_block_chunk) + .await?, + ); + record_progress(pool, progress).await?; + } let first_parent_hash = path.first().and_then(|block| block.parent_hash.as_deref()); if first_parent_hash != Some(current_canonical_hash) { return Ok(None); @@ -70,7 +87,10 @@ pub(super) async fn reconcile_contiguous_checkpoint_gap( ) }) .collect::>(); - upsert_recanonicalized_lineage_blocks_without_snapshots(pool, &lineage_blocks).await?; + for lineage_chunk in lineage_blocks.chunks(LIVE_GAP_PROGRESS_BLOCKS) { + upsert_recanonicalized_lineage_blocks_without_snapshots(pool, lineage_chunk).await?; + record_progress(pool, progress).await?; + } let status = if path.len() == 1 { CanonicalReconciliationStatus::Appended @@ -89,3 +109,13 @@ pub(super) async fn reconcile_contiguous_checkpoint_gap( raw_orphan_stop_before_hash: None, })) } + +async fn record_progress( + pool: &sqlx::PgPool, + progress: &mut Option<&mut dyn StartupAdapterProgress>, +) -> Result<()> { + if let Some(progress) = progress.as_deref_mut() { + progress.record(pool).await?; + } + Ok(()) +} diff --git a/apps/indexer/src/main/reconciliation/canonical/ens_v2_coverage_recovery.rs b/apps/indexer/src/main/reconciliation/canonical/ens_v2_coverage_recovery.rs index cb4f147c..5add0d99 100644 --- a/apps/indexer/src/main/reconciliation/canonical/ens_v2_coverage_recovery.rs +++ b/apps/indexer/src/main/reconciliation/canonical/ens_v2_coverage_recovery.rs @@ -1,7 +1,7 @@ use anyhow::{Context, Result, ensure}; -use bigname_adapters::EnsV2MissingCoverage; +use bigname_adapters::{EnsV2MissingCoverage, StartupAdapterProgress}; use bigname_manifests::{ - WatchedSourceSelector, WatchedTargetIdentity, load_historical_watched_contracts_by_chain, + WatchedSourceSelector, WatchedTargetIdentity, load_historical_watched_contracts_for_target, resolve_watched_source_selector, }; @@ -9,6 +9,7 @@ use crate::{ backfill::{ BackfillAdapterSyncMode, BackfillBlockRange, BackfillJobRunConfig, DEFAULT_HASH_PINNED_BACKFILL_CHUNK_BLOCKS, run_resumable_hash_pinned_backfill_job, + run_resumable_hash_pinned_backfill_job_with_progress, }, backfill_lease_expires_at, default_backfill_lease_owner, generated_backfill_lease_token, provider::ChainProviderOps, @@ -39,6 +40,44 @@ pub(crate) async fn recover_ens_v2_live_coverage_requirement( provider: &(impl ChainProviderOps + ?Sized), header_audit_mode: HeaderAuditMode, requirement: &EnsV2MissingCoverage, +) -> Result { + recover_ens_v2_live_coverage_requirement_inner( + pool, + deployment_profile, + provider, + header_audit_mode, + requirement, + &mut None, + ) + .await +} + +pub(crate) async fn recover_ens_v2_live_coverage_requirement_with_progress( + pool: &sqlx::PgPool, + deployment_profile: &str, + provider: &(impl ChainProviderOps + ?Sized), + header_audit_mode: HeaderAuditMode, + requirement: &EnsV2MissingCoverage, + progress: &mut dyn StartupAdapterProgress, +) -> Result { + recover_ens_v2_live_coverage_requirement_inner( + pool, + deployment_profile, + provider, + header_audit_mode, + requirement, + &mut Some(progress), + ) + .await +} + +async fn recover_ens_v2_live_coverage_requirement_inner( + pool: &sqlx::PgPool, + deployment_profile: &str, + provider: &(impl ChainProviderOps + ?Sized), + header_audit_mode: HeaderAuditMode, + requirement: &EnsV2MissingCoverage, + progress: &mut Option<&mut dyn StartupAdapterProgress>, ) -> Result { let initial_authority = load_retention_authority(pool, &requirement.chain).await?; if initial_authority.retention_generation != requirement.retention_generation { @@ -49,20 +88,23 @@ pub(crate) async fn recover_ens_v2_live_coverage_requirement( requirement.required_from_block, requirement.required_to_block, )?; - let historical_contracts = load_historical_watched_contracts_by_chain(pool, &requirement.chain) - .await? - .into_iter() - .filter(|contract| { - contract.source_family == requirement.source_family - && contract.address.eq_ignore_ascii_case(&requirement.address) - && contract - .active_from_block_number - .is_none_or(|from| from <= requirement.required_to_block) - && contract - .active_to_block_number - .is_none_or(|to| to >= requirement.required_from_block) - }) - .collect::>(); + let historical_contracts = load_historical_watched_contracts_for_target( + pool, + &requirement.chain, + &requirement.source_family, + &requirement.address, + ) + .await? + .into_iter() + .filter(|contract| { + contract + .active_from_block_number + .is_none_or(|from| from <= requirement.required_to_block) + && contract + .active_to_block_number + .is_none_or(|to| to >= requirement.required_from_block) + }) + .collect::>(); ensure!( !historical_contracts.is_empty(), "ENSv2 live coverage recovery cannot resolve watched target {} {} on {} over {}..={}", @@ -109,30 +151,35 @@ pub(crate) async fn recover_ens_v2_live_coverage_requirement( "indexer-live-ens-v2-coverage-recovery:v1:deployment_profile={deployment_profile}:chain={}:generation={}:source_identity_hash={source_identity_hash}:from={}:to={}", requirement.chain, requirement.retention_generation, range.from_block, range.to_block, ); - run_resumable_hash_pinned_backfill_job( - pool, - &source_plan, - provider, - BackfillJobRunConfig { - deployment_profile: deployment_profile.to_owned(), - idempotency_key, - scope_idempotency_to_raw_log_retention_generation: true, - range, - lease_owner: format!( - "{}:live-ens-v2-coverage-recovery", - default_backfill_lease_owner() - ), - lease_token: generated_backfill_lease_token()?, - lease_expires_at: backfill_lease_expires_at( - LIVE_COVERAGE_RECOVERY_LEASE_DURATION_SECS, - )?, - hash_pinned_chunk_blocks: DEFAULT_HASH_PINNED_BACKFILL_CHUNK_BLOCKS, - adapter_sync_mode: BackfillAdapterSyncMode::RawOnly, - header_audit_mode, - }, - ) - .await - .with_context(|| { + let config = BackfillJobRunConfig { + deployment_profile: deployment_profile.to_owned(), + idempotency_key, + scope_idempotency_to_raw_log_retention_generation: true, + range, + lease_owner: format!( + "{}:live-ens-v2-coverage-recovery", + default_backfill_lease_owner() + ), + lease_token: generated_backfill_lease_token()?, + lease_expires_at: backfill_lease_expires_at(LIVE_COVERAGE_RECOVERY_LEASE_DURATION_SECS)?, + hash_pinned_chunk_blocks: DEFAULT_HASH_PINNED_BACKFILL_CHUNK_BLOCKS, + adapter_sync_mode: BackfillAdapterSyncMode::RawOnly, + header_audit_mode, + }; + let recovery_result = match progress.as_deref_mut() { + Some(progress) => { + run_resumable_hash_pinned_backfill_job_with_progress( + pool, + &source_plan, + provider, + config, + progress, + ) + .await + } + None => run_resumable_hash_pinned_backfill_job(pool, &source_plan, provider, config).await, + }; + recovery_result.with_context(|| { format!( "failed provider-backed ENSv2 live coverage recovery for {} {} {} over {}..={}", requirement.chain, diff --git a/apps/indexer/src/main/reconciliation/canonical/orphaning.rs b/apps/indexer/src/main/reconciliation/canonical/orphaning.rs index 18307fcb..e6847fc2 100644 --- a/apps/indexer/src/main/reconciliation/canonical/orphaning.rs +++ b/apps/indexer/src/main/reconciliation/canonical/orphaning.rs @@ -1,48 +1,166 @@ -use anyhow::Result; +use anyhow::{Context, Result}; +use bigname_adapters::StartupAdapterProgress; use bigname_storage::{ - invalidate_execution_outcomes_for_orphaned_blocks, - mark_block_derived_normalized_events_range_orphaned, mark_identity_rows_range_orphaned, + ExecutionOutcomeInvalidationProgress, ExecutionOutcomeInvalidationProgressFuture, + IdentityOrphanCounts, RawFactOrphanCounts, invalidate_execution_outcomes_for_orphaned_blocks, + invalidate_execution_outcomes_for_orphaned_blocks_with_progress, load_chain_lineage_block, + load_raw_block, mark_block_derived_normalized_events_range_orphaned, + mark_chain_lineage_range_orphaned, mark_identity_rows_range_orphaned, mark_raw_block_facts_range_orphaned, }; use tracing::info; use super::{ - super::persistence::ensure_losing_branch_raw_blocks_exist, - stored_lineage::ChainCoverageFrontiers, + super::persistence::ensure_losing_branch_raw_blocks_exist_with_progress, + progress::record_live_progress, stored_lineage::ChainCoverageFrontiers, }; +struct ExecutionInvalidationHeartbeat<'a>(&'a mut dyn StartupAdapterProgress); + +impl ExecutionOutcomeInvalidationProgress for ExecutionInvalidationHeartbeat<'_> { + fn record<'a>( + &'a mut self, + pool: &'a sqlx::PgPool, + ) -> ExecutionOutcomeInvalidationProgressFuture<'a> { + self.0.record(pool) + } +} + +#[allow(dead_code)] +pub(crate) async fn orphan_canonical_branch( + pool: &sqlx::PgPool, + chain: &str, + from_hash: &str, + stop_before_hash: Option<&str>, +) -> Result { + orphan_canonical_branch_with_progress(pool, chain, from_hash, stop_before_hash, &mut None).await +} + +pub(super) async fn orphan_canonical_branch_with_progress( + pool: &sqlx::PgPool, + chain: &str, + from_hash: &str, + stop_before_hash: Option<&str>, + progress: &mut Option<&mut dyn StartupAdapterProgress>, +) -> Result { + let mut orphaned_block_count = 0usize; + let mut cursor_hash = Some(from_hash.to_owned()); + + while let Some(block_hash) = cursor_hash { + if Some(block_hash.as_str()) == stop_before_hash { + break; + } + + let block = load_chain_lineage_block(pool, chain, &block_hash) + .await? + .with_context(|| { + format!( + "missing stored lineage row for chain {chain} block {block_hash} while orphaning the losing branch" + ) + })?; + let parent_hash = block.parent_hash.clone(); + let snapshots = + mark_chain_lineage_range_orphaned(pool, chain, &block_hash, parent_hash.as_deref()) + .await?; + orphaned_block_count += snapshots.len(); + cursor_hash = parent_hash; + record_live_progress(pool, progress).await?; + } + + Ok(orphaned_block_count) +} + +#[allow(dead_code)] pub(crate) async fn orphan_reorg_losing_branch_payloads( pool: &sqlx::PgPool, chain: &str, current_canonical_hash: Option<&str>, stop_before_hash: Option<&str>, coverage_frontiers: &ChainCoverageFrontiers, +) -> Result<()> { + orphan_reorg_losing_branch_payloads_inner( + pool, + chain, + current_canonical_hash, + stop_before_hash, + coverage_frontiers, + &mut None, + ) + .await +} + +pub(super) async fn orphan_reorg_losing_branch_payloads_with_progress( + pool: &sqlx::PgPool, + chain: &str, + current_canonical_hash: Option<&str>, + stop_before_hash: Option<&str>, + coverage_frontiers: &ChainCoverageFrontiers, + progress: &mut Option<&mut dyn StartupAdapterProgress>, +) -> Result<()> { + orphan_reorg_losing_branch_payloads_inner( + pool, + chain, + current_canonical_hash, + stop_before_hash, + coverage_frontiers, + progress, + ) + .await +} + +async fn orphan_reorg_losing_branch_payloads_inner( + pool: &sqlx::PgPool, + chain: &str, + current_canonical_hash: Option<&str>, + stop_before_hash: Option<&str>, + coverage_frontiers: &ChainCoverageFrontiers, + progress: &mut Option<&mut dyn StartupAdapterProgress>, ) -> Result<()> { if let Some(current_canonical_hash) = current_canonical_hash { - ensure_losing_branch_raw_blocks_exist( + ensure_losing_branch_raw_blocks_exist_with_progress( pool, chain, current_canonical_hash, stop_before_hash, + progress, ) .await?; - let orphaned_raw_facts = mark_raw_block_facts_range_orphaned( + let losing_blocks = load_losing_raw_block_path( pool, chain, current_canonical_hash, stop_before_hash, + progress, ) .await?; + + let mut orphaned_raw_facts = RawFactOrphanCounts::default(); + for (block_hash, parent_hash) in &losing_blocks { + let counts = mark_raw_block_facts_range_orphaned( + pool, + chain, + block_hash, + parent_hash.as_deref(), + ) + .await?; + add_raw_fact_counts(&mut orphaned_raw_facts, &counts); + record_live_progress(pool, progress).await?; + } if orphaned_raw_facts.code_hash_count > 0 { coverage_frontiers.invalidate_raw_code_baseline_frontier(chain); } - let orphaned_normalized_event_count = mark_block_derived_normalized_events_range_orphaned( - pool, - chain, - current_canonical_hash, - stop_before_hash, - ) - .await?; + + let mut orphaned_normalized_event_count = 0u64; + for (block_hash, parent_hash) in &losing_blocks { + orphaned_normalized_event_count += mark_block_derived_normalized_events_range_orphaned( + pool, + chain, + block_hash, + parent_hash.as_deref(), + ) + .await?; + record_live_progress(pool, progress).await?; + } if orphaned_normalized_event_count > 0 { info!( service = "indexer", @@ -51,13 +169,15 @@ pub(crate) async fn orphan_reorg_losing_branch_payloads( "block-derived normalized events orphaned for the losing branch" ); } - let orphaned_identity_counts = mark_identity_rows_range_orphaned( - pool, - chain, - current_canonical_hash, - stop_before_hash, - ) - .await?; + + let mut orphaned_identity_counts = IdentityOrphanCounts::default(); + for (block_hash, parent_hash) in &losing_blocks { + let counts = + mark_identity_rows_range_orphaned(pool, chain, block_hash, parent_hash.as_deref()) + .await?; + add_identity_counts(&mut orphaned_identity_counts, &counts); + record_live_progress(pool, progress).await?; + } if orphaned_identity_counts.token_lineage_count > 0 || orphaned_identity_counts.resource_count > 0 || orphaned_identity_counts.name_surface_count > 0 @@ -75,8 +195,14 @@ pub(crate) async fn orphan_reorg_losing_branch_payloads( } } - let execution_invalidation_summary = - invalidate_execution_outcomes_for_orphaned_blocks(pool).await?; + let execution_invalidation_summary = match progress.as_deref_mut() { + Some(progress) => { + let mut heartbeat = ExecutionInvalidationHeartbeat(progress); + invalidate_execution_outcomes_for_orphaned_blocks_with_progress(pool, &mut heartbeat) + .await? + } + None => invalidate_execution_outcomes_for_orphaned_blocks(pool).await?, + }; if execution_invalidation_summary.deleted_outcome_count > 0 { info!( service = "indexer", @@ -89,3 +215,47 @@ pub(crate) async fn orphan_reorg_losing_branch_payloads( Ok(()) } + +async fn load_losing_raw_block_path( + pool: &sqlx::PgPool, + chain: &str, + from_hash: &str, + stop_before_hash: Option<&str>, + progress: &mut Option<&mut dyn StartupAdapterProgress>, +) -> Result)>> { + let mut blocks = Vec::new(); + let mut cursor_hash = Some(from_hash.to_owned()); + while let Some(block_hash) = cursor_hash { + if Some(block_hash.as_str()) == stop_before_hash { + break; + } + let raw_block = load_raw_block(pool, chain, &block_hash) + .await? + .with_context(|| { + format!( + "missing stored raw block for chain {chain} block {block_hash} while orphaning losing-branch payloads" + ) + })?; + cursor_hash = raw_block.parent_hash.clone(); + blocks.push((raw_block.block_hash, raw_block.parent_hash)); + record_live_progress(pool, progress).await?; + } + Ok(blocks) +} + +fn add_raw_fact_counts(total: &mut RawFactOrphanCounts, next: &RawFactOrphanCounts) { + total.block_count += next.block_count; + total.code_hash_count += next.code_hash_count; + total.transaction_count += next.transaction_count; + total.receipt_count += next.receipt_count; + total.log_count += next.log_count; + total.call_snapshot_count += next.call_snapshot_count; + total.payload_cache_metadata_count += next.payload_cache_metadata_count; +} + +fn add_identity_counts(total: &mut IdentityOrphanCounts, next: &IdentityOrphanCounts) { + total.token_lineage_count += next.token_lineage_count; + total.resource_count += next.resource_count; + total.name_surface_count += next.name_surface_count; + total.surface_binding_count += next.surface_binding_count; +} diff --git a/apps/indexer/src/main/reconciliation/canonical/poll.rs b/apps/indexer/src/main/reconciliation/canonical/poll.rs index 5f55d3c7..1dfd10bf 100644 --- a/apps/indexer/src/main/reconciliation/canonical/poll.rs +++ b/apps/indexer/src/main/reconciliation/canonical/poll.rs @@ -2,11 +2,12 @@ use std::collections::BTreeMap; use crate::{provider::ProviderRegistry, runtime::IntakeChainTask}; use anyhow::Result; +use bigname_adapters::StartupAdapterProgress; use tracing::{info, warn}; use super::{ - EnsV2LiveCoverageRecoveryStatus, reconcile_intake_chain_task_with_adapter_sync, - recover_ens_v2_live_coverage_requirement, stored_lineage::ChainCoverageFrontiers, + EnsV2LiveCoverageRecoveryStatus, reconcile_intake_chain_task_with_adapter_sync_and_progress, + recover_ens_v2_live_coverage_requirement_with_progress, stored_lineage::ChainCoverageFrontiers, }; use crate::{ provider::ProviderBlock, @@ -22,7 +23,7 @@ const MAX_LIVE_COVERAGE_RECOVERY_ATTEMPTS: usize = 32; #[allow(dead_code)] pub(crate) async fn poll_provider_heads( pool: &sqlx::PgPool, - tasks: &mut Vec, + tasks: &mut [IntakeChainTask], provider_registry: &ProviderRegistry, ) -> Result<()> { poll_provider_heads_with_adapter_sync( @@ -43,7 +44,7 @@ pub(crate) async fn poll_provider_heads( #[expect(clippy::too_many_arguments)] pub(crate) async fn poll_provider_heads_with_adapter_sync( pool: &sqlx::PgPool, - tasks: &mut Vec, + tasks: &mut [IntakeChainTask], provider_registry: &ProviderRegistry, deployment_profile: &str, watched_plan_admission_epochs: &BTreeMap, @@ -53,8 +54,67 @@ pub(crate) async fn poll_provider_heads_with_adapter_sync( coverage_frontiers: &ChainCoverageFrontiers, latched_bootstrap_finalized_heads: &BTreeMap, ) -> Result<()> { - let mut next_tasks = tasks.clone(); - let mut any_change = false; + poll_provider_heads_with_adapter_sync_inner( + pool, + tasks, + provider_registry, + deployment_profile, + watched_plan_admission_epochs, + adapter_sync_enabled, + header_audit_mode, + event_silent_reverse_resolver_addresses, + coverage_frontiers, + latched_bootstrap_finalized_heads, + &mut None, + ) + .await +} + +#[expect(clippy::too_many_arguments)] +pub(crate) async fn poll_provider_heads_with_adapter_sync_and_progress( + pool: &sqlx::PgPool, + tasks: &mut [IntakeChainTask], + provider_registry: &ProviderRegistry, + deployment_profile: &str, + watched_plan_admission_epochs: &BTreeMap, + adapter_sync_enabled: bool, + header_audit_mode: HeaderAuditMode, + event_silent_reverse_resolver_addresses: &[String], + coverage_frontiers: &ChainCoverageFrontiers, + latched_bootstrap_finalized_heads: &BTreeMap, + progress: &mut dyn StartupAdapterProgress, +) -> Result<()> { + poll_provider_heads_with_adapter_sync_inner( + pool, + tasks, + provider_registry, + deployment_profile, + watched_plan_admission_epochs, + adapter_sync_enabled, + header_audit_mode, + event_silent_reverse_resolver_addresses, + coverage_frontiers, + latched_bootstrap_finalized_heads, + &mut Some(progress), + ) + .await +} + +#[expect(clippy::too_many_arguments)] +async fn poll_provider_heads_with_adapter_sync_inner( + pool: &sqlx::PgPool, + tasks: &mut [IntakeChainTask], + provider_registry: &ProviderRegistry, + deployment_profile: &str, + watched_plan_admission_epochs: &BTreeMap, + adapter_sync_enabled: bool, + header_audit_mode: HeaderAuditMode, + event_silent_reverse_resolver_addresses: &[String], + coverage_frontiers: &ChainCoverageFrontiers, + latched_bootstrap_finalized_heads: &BTreeMap, + progress: &mut Option<&mut dyn StartupAdapterProgress>, +) -> Result<()> { + let mut changed_tasks = Vec::<(usize, IntakeChainTask)>::new(); for (index, task) in tasks.iter().enumerate() { let Some(provider) = provider_registry.provider_for(&task.chain) else { @@ -62,7 +122,7 @@ pub(crate) async fn poll_provider_heads_with_adapter_sync( }; let mut coverage_recovery_attempt = 0_usize; loop { - match reconcile_intake_chain_task_with_adapter_sync( + match reconcile_intake_chain_task_with_adapter_sync_and_progress( pool, deployment_profile, task, @@ -76,13 +136,13 @@ pub(crate) async fn poll_provider_heads_with_adapter_sync( event_silent_reverse_resolver_addresses, coverage_frontiers, latched_bootstrap_finalized_heads.get(&task.chain), + progress, ) .await { Ok(Some((next_task, outcome))) => { log_chain_reconciliation_outcome(&outcome); - next_tasks[index] = next_task; - any_change = true; + changed_tasks.push((index, next_task)); break; } Ok(None) => break, @@ -110,16 +170,32 @@ pub(crate) async fn poll_provider_heads_with_adapter_sync( } coverage_recovery_attempt += 1; let recovery_requirement = requirement.as_recovery_requirement(); - match recover_ens_v2_live_coverage_requirement( - pool, - deployment_profile, - provider, - header_audit_mode, - &recovery_requirement, - ) - .await - { + let recovery_result = match progress.as_deref_mut() { + Some(progress) => { + recover_ens_v2_live_coverage_requirement_with_progress( + pool, + deployment_profile, + provider, + header_audit_mode, + &recovery_requirement, + progress, + ) + .await + } + None => { + super::recover_ens_v2_live_coverage_requirement( + pool, + deployment_profile, + provider, + header_audit_mode, + &recovery_requirement, + ) + .await + } + }; + match recovery_result { Ok(status) => { + record_progress(pool, progress).await?; info!( service = "indexer", command = "poll", @@ -160,8 +236,18 @@ pub(crate) async fn poll_provider_heads_with_adapter_sync( } } } - if any_change { - *tasks = next_tasks; + for (index, next_task) in changed_tasks { + tasks[index] = next_task; + } + Ok(()) +} + +async fn record_progress( + pool: &sqlx::PgPool, + progress: &mut Option<&mut dyn StartupAdapterProgress>, +) -> Result<()> { + if let Some(progress) = progress.as_deref_mut() { + progress.record(pool).await?; } Ok(()) } diff --git a/apps/indexer/src/main/reconciliation/canonical/progress.rs b/apps/indexer/src/main/reconciliation/canonical/progress.rs new file mode 100644 index 00000000..5806e483 --- /dev/null +++ b/apps/indexer/src/main/reconciliation/canonical/progress.rs @@ -0,0 +1,156 @@ +use anyhow::Result; +use bigname_adapters::StartupAdapterProgress; +use bigname_storage::ChainCheckpoint; + +use crate::{ + provider::{ChainProviderOps, ProviderBlock}, + runtime::IntakeChainTask, +}; + +use super::{ + CanonicalReconciliation, ChainCoverageFrontiers, ChainReconciliationOutcome, HeaderAuditMode, + reconcile_canonical_head_with_progress, reconcile_intake_chain_task_with_adapter_sync_inner, +}; + +#[allow(dead_code)] +pub(crate) async fn reconcile_intake_chain_task( + pool: &sqlx::PgPool, + task: &IntakeChainTask, + provider: &(impl ChainProviderOps + ?Sized), +) -> Result> { + reconcile_intake_chain_task_with_adapter_sync( + pool, + "test", + task, + provider, + 0, + true, + HeaderAuditMode::Minimal, + &[], + &ChainCoverageFrontiers::default(), + None, + ) + .await +} + +#[expect(clippy::too_many_arguments)] +pub(crate) async fn reconcile_intake_chain_task_with_adapter_sync( + pool: &sqlx::PgPool, + deployment_profile: &str, + task: &IntakeChainTask, + provider: &(impl ChainProviderOps + ?Sized), + loaded_plan_admission_epoch: i64, + adapter_sync_enabled: bool, + header_audit_mode: HeaderAuditMode, + event_silent_reverse_resolver_addresses: &[String], + coverage_frontiers: &ChainCoverageFrontiers, + latched_bootstrap_finalized_head: Option<&ProviderBlock>, +) -> Result> { + reconcile_intake_chain_task_with_adapter_sync_inner( + pool, + deployment_profile, + task, + provider, + loaded_plan_admission_epoch, + adapter_sync_enabled, + header_audit_mode, + event_silent_reverse_resolver_addresses, + coverage_frontiers, + latched_bootstrap_finalized_head, + &mut None, + ) + .await +} + +#[expect(clippy::too_many_arguments)] +pub(crate) async fn reconcile_intake_chain_task_with_adapter_sync_and_progress( + pool: &sqlx::PgPool, + deployment_profile: &str, + task: &IntakeChainTask, + provider: &(impl ChainProviderOps + ?Sized), + loaded_plan_admission_epoch: i64, + adapter_sync_enabled: bool, + header_audit_mode: HeaderAuditMode, + event_silent_reverse_resolver_addresses: &[String], + coverage_frontiers: &ChainCoverageFrontiers, + latched_bootstrap_finalized_head: Option<&ProviderBlock>, + progress: &mut Option<&mut dyn StartupAdapterProgress>, +) -> Result> { + reconcile_intake_chain_task_with_adapter_sync_inner( + pool, + deployment_profile, + task, + provider, + loaded_plan_admission_epoch, + adapter_sync_enabled, + header_audit_mode, + event_silent_reverse_resolver_addresses, + coverage_frontiers, + latched_bootstrap_finalized_head, + progress, + ) + .await +} + +#[expect(clippy::too_many_arguments)] +#[allow(dead_code)] +pub(crate) async fn reconcile_canonical_head( + pool: &sqlx::PgPool, + provider: &(impl ChainProviderOps + ?Sized), + chain: &str, + checkpoint: &ChainCheckpoint, + latest_head: &ProviderBlock, + header_audit_mode: HeaderAuditMode, + stored_lineage_promotion_anchors: &[ProviderBlock], + coverage_frontiers: &ChainCoverageFrontiers, +) -> Result { + reconcile_canonical_head_with_progress( + pool, + provider, + chain, + checkpoint, + latest_head, + header_audit_mode, + stored_lineage_promotion_anchors, + coverage_frontiers, + &mut None, + ) + .await +} + +#[cfg(test)] +#[expect(clippy::too_many_arguments)] +pub(crate) async fn reconcile_canonical_head_with_adapter_progress( + pool: &sqlx::PgPool, + provider: &(impl ChainProviderOps + ?Sized), + chain: &str, + checkpoint: &ChainCheckpoint, + latest_head: &ProviderBlock, + header_audit_mode: HeaderAuditMode, + stored_lineage_promotion_anchors: &[ProviderBlock], + coverage_frontiers: &ChainCoverageFrontiers, + progress: &mut dyn StartupAdapterProgress, +) -> Result { + reconcile_canonical_head_with_progress( + pool, + provider, + chain, + checkpoint, + latest_head, + header_audit_mode, + stored_lineage_promotion_anchors, + coverage_frontiers, + &mut Some(progress), + ) + .await +} + +pub(super) async fn record_live_progress( + pool: &sqlx::PgPool, + progress: &mut Option<&mut dyn StartupAdapterProgress>, +) -> Result<()> { + if let Some(progress) = progress.as_deref_mut() { + progress.record(pool).await?; + } + Ok(()) +} diff --git a/apps/indexer/src/main/reconciliation/canonical/stored_lineage.rs b/apps/indexer/src/main/reconciliation/canonical/stored_lineage.rs index 4889666d..8e7d463b 100644 --- a/apps/indexer/src/main/reconciliation/canonical/stored_lineage.rs +++ b/apps/indexer/src/main/reconciliation/canonical/stored_lineage.rs @@ -1,4 +1,5 @@ use anyhow::Result; +use bigname_adapters::StartupAdapterProgress; use bigname_storage::{ CanonicalityState, ChainLineageBlock, CheckpointBlockRef, chain_lineage_contains_ancestor, load_chain_lineage_block, load_chain_lineage_canonical_child_path, @@ -56,6 +57,7 @@ pub(super) async fn reconcile_large_checkpoint_gap_from_stored_lineage( latest_head: &ProviderBlock, stored_anchor_candidates: &[ProviderBlock], coverage_frontiers: &ChainCoverageFrontiers, + progress: &mut Option<&mut dyn StartupAdapterProgress>, ) -> Result { if latest_head.block_number <= current_canonical_number { return Ok(StoredLineagePromotion::NotApplicable); @@ -71,6 +73,7 @@ pub(super) async fn reconcile_large_checkpoint_gap_from_stored_lineage( chain, current_canonical_number, stored_anchor_candidates, + progress, ) .await? else { @@ -89,6 +92,7 @@ pub(super) async fn reconcile_large_checkpoint_gap_from_stored_lineage( batch_blocks, ) .await?; + record_progress(pool, progress).await?; if path.len() != batch_blocks { return Ok(StoredLineagePromotion::Refused(format!( "canonical gap of {live_gap_blocks} blocks for chain {chain} exceeds live gap fill limit {MAX_LIVE_CONTIGUOUS_GAP_FILL_BLOCKS}; stored lineage path from checkpoint {} is incomplete or has duplicate canonical children before the stored safe/finalized anchor {}; rerun hash-pinned backfill for the missing range and retry", @@ -103,6 +107,7 @@ pub(super) async fn reconcile_large_checkpoint_gap_from_stored_lineage( coverage_frontiers, stored_anchor.block_number, admission_epoch, + progress, ) .await { @@ -139,6 +144,7 @@ pub(super) async fn reconcile_large_checkpoint_gap_from_stored_lineage( let provider_targets = provider .fetch_block_hashes_by_numbers(&[target.block_number]) .await?; + record_progress(pool, progress).await?; let Some(provider_target) = provider_targets .iter() .find(|block| block.block_number == target.block_number) @@ -199,6 +205,7 @@ async fn select_stored_promotion_anchor( chain: &str, current_canonical_number: i64, candidates: &[ProviderBlock], + progress: &mut Option<&mut dyn StartupAdapterProgress>, ) -> Result> { let max_candidate_height = candidates .iter() @@ -213,6 +220,7 @@ async fn select_stored_promotion_anchor( let resolved = provider .fetch_block_hashes_by_numbers(&[stored_frontier.block_number]) .await?; + record_progress(pool, progress).await?; if let Some(provider_block) = resolved .iter() .find(|block| block.block_number == stored_frontier.block_number) @@ -255,12 +263,23 @@ async fn select_stored_promotion_anchor( break; } cursor = parent; + record_progress(pool, progress).await?; } } Ok(None) } +async fn record_progress( + pool: &sqlx::PgPool, + progress: &mut Option<&mut dyn StartupAdapterProgress>, +) -> Result<()> { + if let Some(progress) = progress.as_deref_mut() { + progress.record(pool).await?; + } + Ok(()) +} + fn stored_anchor_is_canonical(state: CanonicalityState) -> bool { matches!( state, diff --git a/apps/indexer/src/main/reconciliation/canonical/stored_lineage/coverage.rs b/apps/indexer/src/main/reconciliation/canonical/stored_lineage/coverage.rs index 2c13de84..5c5df797 100644 --- a/apps/indexer/src/main/reconciliation/canonical/stored_lineage/coverage.rs +++ b/apps/indexer/src/main/reconciliation/canonical/stored_lineage/coverage.rs @@ -2,19 +2,24 @@ use std::collections::{BTreeMap, BTreeSet}; use std::sync::Mutex; use anyhow::Result; +use bigname_adapters::StartupAdapterProgress; use bigname_manifests::{ - RequiredWatchedTuple, UncoveredWatchedTuple, - find_uncovered_required_watched_tuples_in_transaction, + ManifestRuntimeProgress, ManifestRuntimeProgressFuture, RequiredWatchedTuple, + UncoveredWatchedTuple, find_uncovered_required_watched_tuples_in_transaction, load_active_manifest_abi_events_by_chain_and_source_families, - load_earliest_known_watched_block, load_log_producing_source_families, - load_stored_lineage_coverage_candidate_delta_page, + load_earliest_known_watched_block, load_earliest_known_watched_block_with_progress, + load_log_producing_source_families, load_stored_lineage_coverage_candidate_delta_page, materialize_stored_lineage_coverage_candidate, + materialize_stored_lineage_coverage_candidate_delta_with_progress, + materialize_stored_lineage_coverage_candidate_with_progress, }; use bigname_storage::{ - ChainLineageBlock, StoredLineageCoverageFrontierPublication, - StoredLineageCoveragePublicationOutcome, begin_stored_lineage_coverage_frontier_publication, + ChainLineageBlock, StoredLineageCoverageFrontierPublication, StoredLineageCoverageProgress, + StoredLineageCoverageProgressFuture, StoredLineageCoveragePublicationOutcome, + begin_stored_lineage_coverage_frontier_publication, load_stored_lineage_coverage_frontier_header, stored_lineage_coverage_frontier_requirements_are_valid, + stored_lineage_coverage_frontier_requirements_are_valid_with_progress, }; #[path = "coverage/companions.rs"] mod companions; @@ -22,8 +27,30 @@ mod companions; mod frontier; #[path = "coverage/lineage_forks.rs"] mod lineage_forks; +#[path = "coverage/verification.rs"] +mod verification; use frontier::Topic0sByFamily; +use verification::*; + +struct AdapterManifestProgress<'a>(&'a mut dyn StartupAdapterProgress); + +impl ManifestRuntimeProgress for AdapterManifestProgress<'_> { + fn record<'a>(&'a mut self, pool: &'a sqlx::PgPool) -> ManifestRuntimeProgressFuture<'a> { + self.0.record(pool) + } +} + +struct AdapterStorageProgress<'a> { + pool: &'a sqlx::PgPool, + progress: &'a mut dyn StartupAdapterProgress, +} + +impl StoredLineageCoverageProgress for AdapterStorageProgress<'_> { + fn record<'a>(&'a mut self) -> StoredLineageCoverageProgressFuture<'a> { + self.progress.record(self.pool) + } +} /// Frontier extensions verify coverage in chunks of this many blocks, so a /// deep gap costs a handful of coverage queries once and every promotion @@ -174,6 +201,7 @@ pub(super) async fn stored_path_has_required_raw_fact_coverage( coverage_frontiers: &ChainCoverageFrontiers, verify_ahead_through_block: i64, discovery_admission_epoch: i64, + progress: &mut Option<&mut dyn StartupAdapterProgress>, ) -> std::result::Result<(), String> { if path.is_empty() { return Ok(()); @@ -211,6 +239,7 @@ pub(super) async fn stored_path_has_required_raw_fact_coverage( path_through, verify_ahead_through_block, discovery_admission_epoch, + progress, ) .await?; @@ -244,6 +273,7 @@ async fn ensure_verified_coverage_frontier( required_through: i64, verify_ahead_through_block: i64, discovery_admission_epoch: i64, + progress: &mut Option<&mut dyn StartupAdapterProgress>, ) -> std::result::Result<(), String> { let verify_ahead_through_block = verify_ahead_through_block.max(required_through); let optimistic = ensure_verified_coverage_frontier_through( @@ -256,6 +286,7 @@ async fn ensure_verified_coverage_frontier( required_through, verify_ahead_through_block, discovery_admission_epoch, + progress, ) .await; match optimistic { @@ -271,6 +302,7 @@ async fn ensure_verified_coverage_frontier( required_through, required_through, discovery_admission_epoch, + progress, ) .await } @@ -289,6 +321,7 @@ async fn ensure_verified_coverage_frontier_through( required_through: i64, verify_ahead_through_block: i64, discovery_admission_epoch: i64, + progress: &mut Option<&mut dyn StartupAdapterProgress>, ) -> std::result::Result<(), String> { let current_topics = frontier::canonical_topic_sets(current_topic0s_by_family); let mut cas_conflicts = 0; @@ -305,19 +338,45 @@ async fn ensure_verified_coverage_frontier_through( { true } - Some(header) => stored_lineage_coverage_frontier_requirements_are_valid(pool, header) - .await - .map_err(|error| error.to_string())?, + Some(header) => match progress.as_deref_mut() { + Some(progress) => { + let mut bridge = AdapterStorageProgress { pool, progress }; + stored_lineage_coverage_frontier_requirements_are_valid_with_progress( + pool, + header, + &mut bridge, + ) + .await + } + None => stored_lineage_coverage_frontier_requirements_are_valid(pool, header).await, + } + .map_err(|error| error.to_string())?, None => true, }; - let earliest_known_watched_block = load_earliest_known_watched_block( - pool, - chain, - verify_ahead_through_block, - log_producing_source_families, - ) - .await + let earliest_known_watched_block = match progress.as_deref_mut() { + Some(progress) => { + let mut bridge = AdapterManifestProgress(progress); + load_earliest_known_watched_block_with_progress( + pool, + chain, + verify_ahead_through_block, + log_producing_source_families, + &mut bridge, + ) + .await + } + None => { + load_earliest_known_watched_block( + pool, + chain, + verify_ahead_through_block, + log_producing_source_families, + ) + .await + } + } .map_err(|error| error.to_string())?; + record_progress(pool, progress).await?; let Some(plan) = frontier::plan_publication( header.as_ref(), persisted_requirements_valid, @@ -361,14 +420,31 @@ async fn ensure_verified_coverage_frontier_through( ) .await .map_err(|error| error.to_string())?; - materialize_stored_lineage_coverage_candidate( - guard.connection_mut(), - chain, - plan.verified_from_block, - plan.verified_through_block, - log_producing_source_families, - ) - .await + match progress.as_deref_mut() { + Some(progress) => { + let mut bridge = AdapterManifestProgress(progress); + materialize_stored_lineage_coverage_candidate_with_progress( + guard.connection_mut(), + pool, + chain, + plan.verified_from_block, + plan.verified_through_block, + log_producing_source_families, + &mut bridge, + ) + .await + } + None => { + materialize_stored_lineage_coverage_candidate( + guard.connection_mut(), + chain, + plan.verified_from_block, + plan.verified_through_block, + log_producing_source_families, + ) + .await + } + } .map_err(|error| error.to_string())?; super::topic_drift::materialize_topic_evidence_in_transaction( guard.connection_mut(), @@ -379,6 +455,21 @@ async fn ensure_verified_coverage_frontier_through( None, ) .await?; + record_progress(pool, progress).await?; + + if let Some(progress) = progress.as_deref_mut() { + let mut bridge = AdapterManifestProgress(progress); + materialize_stored_lineage_coverage_candidate_delta_with_progress( + guard.connection_mut(), + pool, + chain, + &plan.topic_changed_source_families, + plan.reverify_all, + &mut bridge, + ) + .await + .map_err(|error| error.to_string())?; + } let mut cursor = None; loop { @@ -393,11 +484,13 @@ async fn ensure_verified_coverage_frontier_through( .await .map_err(|error| error.to_string())?; verify_requirements(guard.connection_mut(), chain, &page.requirements).await?; + record_progress(pool, progress).await?; cursor = page.next_cursor; if cursor.is_none() { break; } } + record_progress(pool, progress).await?; let publication = StoredLineageCoverageFrontierPublication { discovery_admission_epoch, @@ -405,11 +498,15 @@ async fn ensure_verified_coverage_frontier_through( verified_through_block: plan.verified_through_block, topic0s_by_family: current_topics.clone(), }; - match guard - .publish(&publication) - .await - .map_err(|error| error.to_string())? - { + let publication_outcome = match progress.as_deref_mut() { + Some(progress) => { + let mut bridge = AdapterStorageProgress { pool, progress }; + guard.publish_with_progress(&publication, &mut bridge).await + } + None => guard.publish(&publication).await, + } + .map_err(|error| error.to_string())?; + match publication_outcome { StoredLineageCoveragePublicationOutcome::Published { snapshot_revision } => { cas_conflicts = 0; coverage_frontiers.record_validated_snapshot_revision(chain, snapshot_revision); @@ -430,116 +527,3 @@ async fn ensure_verified_coverage_frontier_through( "stored-lineage coverage frontier for chain {chain} could not reach required block {required_through} within {MAX_PUBLICATIONS_PER_PROMOTION} bounded publications" )) } - -async fn verify_requirements( - connection: &mut sqlx::PgConnection, - chain: &str, - requirements: &[RequiredWatchedTuple], -) -> std::result::Result<(), String> { - if requirements.is_empty() { - return Ok(()); - } - super::topic_drift::ensure_required_topic_sets_undrifted_in_transaction( - connection, - chain, - requirements, - ) - .await?; - let violations = find_uncovered_required_watched_tuples_in_transaction( - connection, - chain, - requirements, - MAX_REPORTED_UNCOVERED_TUPLES, - ) - .await - .map_err(|error| error.to_string())?; - if violations.is_empty() { - return Ok(()); - } - let from_block = requirements - .iter() - .map(|requirement| requirement.required_from_block) - .min() - .expect("non-empty requirements must have a lower bound"); - let through_block = requirements - .iter() - .map(|requirement| requirement.required_to_block) - .max() - .expect("non-empty requirements must have an upper bound"); - Err(uncovered_tuples_refusal( - from_block, - through_block, - &violations, - )) -} - -fn uncovered_tuples_refusal( - from_block: i64, - through_block: i64, - violations: &[UncoveredWatchedTuple], -) -> String { - let listed = violations - .iter() - .map(|tuple| { - format!( - "(source_family {}, address {}, blocks {}..={})", - tuple.source_family, - tuple.address, - tuple.required_from_block, - tuple.required_to_block - ) - }) - .collect::>() - .join(", "); - let suffix = if violations.len() as i64 >= MAX_REPORTED_UNCOVERED_TUPLES { - " (further violations elided)" - } else { - "" - }; - format!( - "watched tuples over blocks {from_block}..={through_block} do not form gap-free coverage from exact address- or family-scoped backfill_coverage_facts: {listed}{suffix}; run hash-pinned or Coinbase SQL backfill for the missing tuple intervals (or repair derive-backfill-coverage-facts for legacy full-payload jobs) and retry" - ) -} - -/// Current manifest topic selectors per log-producing family. The frontier -/// stores them per family so a semantic change invalidates only that family's -/// tuple proofs. -async fn load_current_topic0s_by_family( - pool: &sqlx::PgPool, - chain: &str, - log_producing_source_families: &[String], -) -> std::result::Result>, String> { - if log_producing_source_families.is_empty() { - return Ok(BTreeMap::new()); - } - let events = load_active_manifest_abi_events_by_chain_and_source_families( - pool, - chain, - log_producing_source_families, - ) - .await - .map_err(|error| error.to_string())?; - let mut current_topic0s_by_family = BTreeMap::>::new(); - for event in events { - let Some(topic0) = event.topic0 else { - continue; - }; - current_topic0s_by_family - .entry(event.source_family) - .or_default() - .insert(topic0.to_ascii_lowercase()); - } - Ok(current_topic0s_by_family) -} - -fn path_start_number(path: &[ChainLineageBlock]) -> i64 { - path.first() - .expect("stored lineage path must not be empty") - .block_number -} - -fn path_end_number(path: &[ChainLineageBlock]) -> i64 { - path.last() - .expect("stored lineage path must not be empty") - .block_number -} diff --git a/apps/indexer/src/main/reconciliation/canonical/stored_lineage/coverage/verification.rs b/apps/indexer/src/main/reconciliation/canonical/stored_lineage/coverage/verification.rs new file mode 100644 index 00000000..d0a15f32 --- /dev/null +++ b/apps/indexer/src/main/reconciliation/canonical/stored_lineage/coverage/verification.rs @@ -0,0 +1,127 @@ +use super::*; + +pub(super) async fn record_progress( + pool: &sqlx::PgPool, + progress: &mut Option<&mut dyn StartupAdapterProgress>, +) -> std::result::Result<(), String> { + if let Some(progress) = progress.as_deref_mut() { + progress + .record(pool) + .await + .map_err(|error| error.to_string())?; + } + Ok(()) +} + +pub(super) async fn verify_requirements( + connection: &mut sqlx::PgConnection, + chain: &str, + requirements: &[RequiredWatchedTuple], +) -> std::result::Result<(), String> { + if requirements.is_empty() { + return Ok(()); + } + super::super::topic_drift::ensure_required_topic_sets_undrifted_in_transaction( + connection, + chain, + requirements, + ) + .await?; + let violations = find_uncovered_required_watched_tuples_in_transaction( + connection, + chain, + requirements, + MAX_REPORTED_UNCOVERED_TUPLES, + ) + .await + .map_err(|error| error.to_string())?; + if violations.is_empty() { + return Ok(()); + } + let from_block = requirements + .iter() + .map(|requirement| requirement.required_from_block) + .min() + .expect("non-empty requirements must have a lower bound"); + let through_block = requirements + .iter() + .map(|requirement| requirement.required_to_block) + .max() + .expect("non-empty requirements must have an upper bound"); + Err(uncovered_tuples_refusal( + from_block, + through_block, + &violations, + )) +} + +pub(super) fn uncovered_tuples_refusal( + from_block: i64, + through_block: i64, + violations: &[UncoveredWatchedTuple], +) -> String { + let listed = violations + .iter() + .map(|tuple| { + format!( + "(source_family {}, address {}, blocks {}..={})", + tuple.source_family, + tuple.address, + tuple.required_from_block, + tuple.required_to_block + ) + }) + .collect::>() + .join(", "); + let suffix = if violations.len() as i64 >= MAX_REPORTED_UNCOVERED_TUPLES { + " (further violations elided)" + } else { + "" + }; + format!( + "watched tuples over blocks {from_block}..={through_block} do not form gap-free coverage from exact address- or family-scoped backfill_coverage_facts: {listed}{suffix}; run hash-pinned or Coinbase SQL backfill for the missing tuple intervals (or repair derive-backfill-coverage-facts for legacy full-payload jobs) and retry" + ) +} + +/// Current manifest topic selectors per log-producing family. The frontier +/// stores them per family so a semantic change invalidates only that family's +/// tuple proofs. +pub(super) async fn load_current_topic0s_by_family( + pool: &sqlx::PgPool, + chain: &str, + log_producing_source_families: &[String], +) -> std::result::Result>, String> { + if log_producing_source_families.is_empty() { + return Ok(BTreeMap::new()); + } + let events = load_active_manifest_abi_events_by_chain_and_source_families( + pool, + chain, + log_producing_source_families, + ) + .await + .map_err(|error| error.to_string())?; + let mut current_topic0s_by_family = BTreeMap::>::new(); + for event in events { + let Some(topic0) = event.topic0 else { + continue; + }; + current_topic0s_by_family + .entry(event.source_family) + .or_default() + .insert(topic0.to_ascii_lowercase()); + } + Ok(current_topic0s_by_family) +} + +pub(super) fn path_start_number(path: &[ChainLineageBlock]) -> i64 { + path.first() + .expect("stored lineage path must not be empty") + .block_number +} + +pub(super) fn path_end_number(path: &[ChainLineageBlock]) -> i64 { + path.last() + .expect("stored lineage path must not be empty") + .block_number +} diff --git a/apps/indexer/src/main/reconciliation/persistence.rs b/apps/indexer/src/main/reconciliation/persistence.rs index 8c4fdab9..a0711adb 100644 --- a/apps/indexer/src/main/reconciliation/persistence.rs +++ b/apps/indexer/src/main/reconciliation/persistence.rs @@ -1,6 +1,7 @@ use std::collections::{BTreeMap, BTreeSet}; use anyhow::{Context, Result, bail}; +use bigname_adapters::StartupAdapterProgress; use bigname_storage::{ CanonicalityState, RawBlock, RawLog, RawPayloadCacheMetadataUpsert, RawReceipt, RawTransaction, load_chain_lineage_block, load_raw_block, load_raw_blocks_by_hashes, upsert_raw_blocks, @@ -17,8 +18,8 @@ use crate::{ use super::{ adapter_sync::{ - sync_live_adapter_state_from_persisted_raw_payloads, - sync_live_adapter_state_from_persisted_raw_payloads_after_reorg, + sync_live_adapter_state_from_persisted_raw_payloads_after_reorg_with_progress, + sync_live_adapter_state_from_persisted_raw_payloads_with_progress, }, canonical::ChainCoverageFrontiers, payload::{ @@ -37,58 +38,87 @@ use super::{ #[path = "persistence/code_hashes.rs"] mod code_hashes; -pub(crate) use code_hashes::persist_reconciled_raw_code_hashes; +#[path = "persistence/raw_blocks.rs"] +mod raw_blocks; +pub(crate) use code_hashes::{ + persist_reconciled_raw_code_hashes, persist_reconciled_raw_code_hashes_with_progress, +}; +pub(crate) use raw_blocks::persist_reconciled_raw_blocks; +use raw_blocks::persist_reconciled_raw_blocks_inner; + +const LIVE_RAW_FACT_PROGRESS_ROWS: usize = 1_000; +const LIVE_ADAPTER_PROGRESS_BLOCKS: usize = 32; -pub(crate) async fn persist_reconciled_raw_blocks( +#[expect(clippy::too_many_arguments)] +#[allow(dead_code)] +pub(crate) async fn persist_reconciled_raw_state( pool: &sqlx::PgPool, - chain: &str, + deployment_profile: &str, + task: &IntakeChainTask, + provider: &(impl ChainProviderOps + ?Sized), heads: &ProviderHeadSnapshot, canonical: &CanonicalReconciliation, + head_change_set: HeadChangeSet, + loaded_plan_admission_epoch: i64, + adapter_sync_enabled: bool, header_audit_mode: HeaderAuditMode, + event_silent_resolver_addresses: &[String], + coverage_frontiers: &ChainCoverageFrontiers, ) -> Result<()> { - let mut blocks = BTreeMap::::new(); - - let canonical_state = canonical_raw_state(canonical.status); - for block in &canonical.reconciled_blocks { - insert_raw_block_candidate( - &mut blocks, - chain, - block, - canonical_state, - header_audit_mode, - ); - } - if let Some(safe) = &heads.safe { - insert_raw_block_candidate( - &mut blocks, - chain, - safe, - CanonicalityState::Safe, - header_audit_mode, - ); - } - if let Some(finalized) = &heads.finalized { - insert_raw_block_candidate( - &mut blocks, - chain, - finalized, - CanonicalityState::Finalized, - header_audit_mode, - ); - } + persist_reconciled_raw_state_inner( + pool, + deployment_profile, + task, + provider, + heads, + canonical, + head_change_set, + loaded_plan_admission_epoch, + adapter_sync_enabled, + header_audit_mode, + event_silent_resolver_addresses, + coverage_frontiers, + &mut None, + ) + .await +} - let blocks = blocks.into_values().collect::>(); - if canonical.status == CanonicalReconciliationStatus::AwaitingAncestor { - upsert_raw_blocks(pool, &blocks).await?; - } else { - upsert_raw_blocks_recanonicalizing_orphaned(pool, &blocks).await?; - } - Ok(()) +#[expect(clippy::too_many_arguments)] +pub(crate) async fn persist_reconciled_raw_state_with_progress( + pool: &sqlx::PgPool, + deployment_profile: &str, + task: &IntakeChainTask, + provider: &(impl ChainProviderOps + ?Sized), + heads: &ProviderHeadSnapshot, + canonical: &CanonicalReconciliation, + head_change_set: HeadChangeSet, + loaded_plan_admission_epoch: i64, + adapter_sync_enabled: bool, + header_audit_mode: HeaderAuditMode, + event_silent_resolver_addresses: &[String], + coverage_frontiers: &ChainCoverageFrontiers, + progress: &mut Option<&mut dyn StartupAdapterProgress>, +) -> Result<()> { + persist_reconciled_raw_state_inner( + pool, + deployment_profile, + task, + provider, + heads, + canonical, + head_change_set, + loaded_plan_admission_epoch, + adapter_sync_enabled, + header_audit_mode, + event_silent_resolver_addresses, + coverage_frontiers, + progress, + ) + .await } -// Raw-state persistence keeps deployment, heads, canonicality, and refresh inputs explicit. #[expect(clippy::too_many_arguments)] -pub(crate) async fn persist_reconciled_raw_state( +async fn persist_reconciled_raw_state_inner( pool: &sqlx::PgPool, deployment_profile: &str, task: &IntakeChainTask, @@ -101,10 +131,19 @@ pub(crate) async fn persist_reconciled_raw_state( header_audit_mode: HeaderAuditMode, event_silent_resolver_addresses: &[String], coverage_frontiers: &ChainCoverageFrontiers, + progress: &mut Option<&mut dyn StartupAdapterProgress>, ) -> Result<()> { - persist_reconciled_raw_blocks(pool, &task.chain, heads, canonical, header_audit_mode).await?; + persist_reconciled_raw_blocks_inner( + pool, + &task.chain, + heads, + canonical, + header_audit_mode, + progress, + ) + .await?; if head_change_set.requires_raw_payload_refresh(canonical.status) { - persist_reconciled_raw_payloads( + persist_reconciled_raw_payloads_inner( pool, deployment_profile, &task.chain, @@ -115,10 +154,11 @@ pub(crate) async fn persist_reconciled_raw_state( head_change_set, adapter_sync_enabled, event_silent_resolver_addresses, + progress, ) .await?; } - persist_reconciled_raw_code_hashes( + persist_reconciled_raw_code_hashes_with_progress( pool, task, provider, @@ -127,12 +167,15 @@ pub(crate) async fn persist_reconciled_raw_state( head_change_set, loaded_plan_admission_epoch, coverage_frontiers, + progress, ) - .await + .await?; + record_progress(pool, progress).await } // Raw-payload persistence keeps selection, heads, canonicality, and adapter inputs explicit. #[expect(clippy::too_many_arguments)] +#[allow(dead_code)] pub(crate) async fn persist_reconciled_raw_payloads( pool: &sqlx::PgPool, deployment_profile: &str, @@ -144,13 +187,53 @@ pub(crate) async fn persist_reconciled_raw_payloads( head_change_set: HeadChangeSet, adapter_sync_enabled: bool, event_silent_resolver_addresses: &[String], +) -> Result<()> { + persist_reconciled_raw_payloads_inner( + pool, + deployment_profile, + chain, + selected_addresses, + provider, + heads, + canonical, + head_change_set, + adapter_sync_enabled, + event_silent_resolver_addresses, + &mut None, + ) + .await +} + +#[expect(clippy::too_many_arguments)] +async fn persist_reconciled_raw_payloads_inner( + pool: &sqlx::PgPool, + deployment_profile: &str, + chain: &str, + selected_addresses: &[String], + provider: &(impl ChainProviderOps + ?Sized), + heads: &ProviderHeadSnapshot, + canonical: &CanonicalReconciliation, + head_change_set: HeadChangeSet, + adapter_sync_enabled: bool, + event_silent_resolver_addresses: &[String], + progress: &mut Option<&mut dyn StartupAdapterProgress>, ) -> Result<()> { let block_hashes = raw_payload_candidate_hashes(heads, canonical, head_change_set); if block_hashes.is_empty() { return Ok(()); } - let raw_blocks = load_raw_blocks_by_hashes(pool, chain, &block_hashes).await?; + let mut raw_blocks_by_order = BTreeMap::new(); + for block_hash_page in block_hashes.chunks(LIVE_RAW_FACT_PROGRESS_ROWS) { + for raw_block in load_raw_blocks_by_hashes(pool, chain, block_hash_page).await? { + raw_blocks_by_order.insert( + (raw_block.block_number, raw_block.block_hash.clone()), + raw_block, + ); + } + record_progress(pool, progress).await?; + } + let raw_blocks = raw_blocks_by_order.into_values().collect::>(); if raw_blocks.len() != block_hashes.len() { bail!( "stored raw block count {} does not match the raw payload fetch plan size {} for chain {}", @@ -217,41 +300,95 @@ pub(crate) async fn persist_reconciled_raw_payloads( &retained_transaction_keys, )?); logs.extend(selected_logs); + record_progress(pool, progress).await?; } - upsert_raw_payload_cache_metadata(pool, &cache_metadata).await?; - upsert_raw_transactions(pool, &transactions).await?; - upsert_raw_receipts(pool, &receipts).await?; - upsert_raw_logs(pool, &logs).await?; - upsert_event_silent_resolver_call_observations(pool, &event_silent_resolver_calls).await?; - bigname_adapters::record_ens_v2_live_selected_raw_log_coverage( - pool, - chain, - selected_addresses, - &block_hashes, - ) - .await - .with_context(|| { - format!("failed to record exact live selected-log coverage for ENSv2 on chain {chain}") - })?; - if adapter_sync_enabled { - if canonical.status == CanonicalReconciliationStatus::ReorgReconciled { - sync_live_adapter_state_from_persisted_raw_payloads_after_reorg( + for chunk in cache_metadata.chunks(LIVE_RAW_FACT_PROGRESS_ROWS) { + upsert_raw_payload_cache_metadata(pool, chunk).await?; + record_progress(pool, progress).await?; + } + for chunk in transactions.chunks(LIVE_RAW_FACT_PROGRESS_ROWS) { + upsert_raw_transactions(pool, chunk).await?; + record_progress(pool, progress).await?; + } + for chunk in receipts.chunks(LIVE_RAW_FACT_PROGRESS_ROWS) { + upsert_raw_receipts(pool, chunk).await?; + record_progress(pool, progress).await?; + } + for chunk in logs.chunks(LIVE_RAW_FACT_PROGRESS_ROWS) { + upsert_raw_logs(pool, chunk).await?; + record_progress(pool, progress).await?; + } + upsert_event_silent_resolver_call_observations(pool, &event_silent_resolver_calls, progress) + .await?; + match progress.as_deref_mut() { + Some(progress) => { + bigname_adapters::record_ens_v2_live_selected_raw_log_coverage_with_progress( pool, - deployment_profile, chain, + selected_addresses, &block_hashes, + progress, ) - .await?; - } else { - sync_live_adapter_state_from_persisted_raw_payloads( + .await + } + None => { + bigname_adapters::record_ens_v2_live_selected_raw_log_coverage( pool, - deployment_profile, chain, + selected_addresses, &block_hashes, ) - .await?; + .await + } + } + .with_context(|| { + format!("failed to record exact live selected-log coverage for ENSv2 on chain {chain}") + })?; + record_progress(pool, progress).await?; + if adapter_sync_enabled { + let ordered_block_hashes = raw_blocks + .iter() + .map(|block| block.block_hash.clone()) + .collect::>(); + if canonical.status == CanonicalReconciliationStatus::ReorgReconciled { + let mut chunks = ordered_block_hashes + .chunks(LIVE_ADAPTER_PROGRESS_BLOCKS) + .peekable(); + while let Some(block_chunk) = chunks.next() { + if chunks.peek().is_some() { + sync_live_adapter_state_from_persisted_raw_payloads_with_progress( + pool, + deployment_profile, + chain, + block_chunk, + progress, + ) + .await?; + } else { + sync_live_adapter_state_from_persisted_raw_payloads_after_reorg_with_progress( + pool, + deployment_profile, + chain, + block_chunk, + progress, + ) + .await?; + } + } + } else { + for block_chunk in ordered_block_hashes.chunks(LIVE_ADAPTER_PROGRESS_BLOCKS) { + sync_live_adapter_state_from_persisted_raw_payloads_with_progress( + pool, + deployment_profile, + chain, + block_chunk, + progress, + ) + .await?; + } } + record_progress(pool, progress).await?; } else { info!( service = "indexer", @@ -294,6 +431,7 @@ async fn load_live_generic_resolver_topic0s( async fn upsert_event_silent_resolver_call_observations( pool: &sqlx::PgPool, observations: &[EventSilentResolverCallObservation], + progress: &mut Option<&mut dyn StartupAdapterProgress>, ) -> Result<()> { for observation in observations { sqlx::query( @@ -351,16 +489,49 @@ async fn upsert_event_silent_resolver_call_observations( observation.chain_id, observation.block_hash, observation.transaction_index ) })?; + record_progress(pool, progress).await?; } Ok(()) } +async fn record_progress( + pool: &sqlx::PgPool, + progress: &mut Option<&mut dyn StartupAdapterProgress>, +) -> Result<()> { + if let Some(progress) = progress.as_deref_mut() { + progress.record(pool).await?; + } + Ok(()) +} + pub(crate) async fn ensure_losing_branch_raw_blocks_exist( pool: &sqlx::PgPool, chain: &str, from_hash: &str, stop_before_hash: Option<&str>, +) -> Result<()> { + ensure_losing_branch_raw_blocks_exist_inner(pool, chain, from_hash, stop_before_hash, &mut None) + .await +} + +pub(crate) async fn ensure_losing_branch_raw_blocks_exist_with_progress( + pool: &sqlx::PgPool, + chain: &str, + from_hash: &str, + stop_before_hash: Option<&str>, + progress: &mut Option<&mut dyn StartupAdapterProgress>, +) -> Result<()> { + ensure_losing_branch_raw_blocks_exist_inner(pool, chain, from_hash, stop_before_hash, progress) + .await +} + +async fn ensure_losing_branch_raw_blocks_exist_inner( + pool: &sqlx::PgPool, + chain: &str, + from_hash: &str, + stop_before_hash: Option<&str>, + progress: &mut Option<&mut dyn StartupAdapterProgress>, ) -> Result<()> { if stop_before_hash == Some(from_hash) { return Ok(()); @@ -376,6 +547,7 @@ pub(crate) async fn ensure_losing_branch_raw_blocks_exist( if let Some(raw_block) = load_raw_block(pool, chain, &block_hash).await? { cursor_hash = raw_block.parent_hash.clone(); + record_progress(pool, progress).await?; continue; } @@ -399,10 +571,16 @@ pub(crate) async fn ensure_losing_branch_raw_blocks_exist( state_root: lineage_block.state_root, canonicality_state: CanonicalityState::Orphaned, }); + if missing_raw_blocks.len() >= LIVE_RAW_FACT_PROGRESS_ROWS { + upsert_raw_blocks(pool, &missing_raw_blocks).await?; + missing_raw_blocks.clear(); + } + record_progress(pool, progress).await?; } if !missing_raw_blocks.is_empty() { upsert_raw_blocks(pool, &missing_raw_blocks).await?; + record_progress(pool, progress).await?; } Ok(()) diff --git a/apps/indexer/src/main/reconciliation/persistence/code_hashes.rs b/apps/indexer/src/main/reconciliation/persistence/code_hashes.rs index 7e4794e7..27e2e572 100644 --- a/apps/indexer/src/main/reconciliation/persistence/code_hashes.rs +++ b/apps/indexer/src/main/reconciliation/persistence/code_hashes.rs @@ -1,9 +1,7 @@ -use std::{ - collections::{BTreeMap, BTreeSet}, - sync::OnceLock, -}; +use std::collections::{BTreeMap, BTreeSet}; use anyhow::{Context, Result, bail}; +use bigname_adapters::StartupAdapterProgress; use bigname_storage::{RawBlock, RawCodeHash, load_raw_blocks_by_hashes, upsert_raw_code_hashes}; use sqlx::Row; @@ -22,36 +20,17 @@ use super::super::{ }; use super::load_live_generic_resolver_topic0s; -/// Per-chain watched-address cap for one baseline poll tick. The sweep walks -/// the sorted watch surface behind a process-lifetime -/// cursor, so a multi-million-address surface is baselined across ticks -/// instead of arming one whole-surface `eth_getCode` storm inside a tick. -const DEFAULT_RAW_CODE_BASELINE_MAX_ADDRESSES_PER_TICK: usize = 2_048; -const RAW_CODE_BASELINE_MAX_ADDRESSES_PER_TICK_ENV: &str = - "BIGNAME_INDEXER_RAW_CODE_BASELINE_MAX_ADDRESSES_PER_TICK"; -static RAW_CODE_BASELINE_MAX_ADDRESSES_PER_TICK: OnceLock = OnceLock::new(); - -fn raw_code_baseline_max_addresses_per_tick() -> usize { - *RAW_CODE_BASELINE_MAX_ADDRESSES_PER_TICK.get_or_init(|| { - parse_raw_code_baseline_max_addresses_per_tick( - std::env::var(RAW_CODE_BASELINE_MAX_ADDRESSES_PER_TICK_ENV) - .ok() - .as_deref(), - ) - }) -} - -fn parse_raw_code_baseline_max_addresses_per_tick(value: Option<&str>) -> usize { - value - .and_then(|value| value.trim().parse::().ok()) - .filter(|value| *value > 0) - .unwrap_or(DEFAULT_RAW_CODE_BASELINE_MAX_ADDRESSES_PER_TICK) -} +#[path = "code_hashes/config.rs"] +mod config; +use config::*; /// Provider-fetch chunk size; successful rounds persist before the sweep advances. const RAW_CODE_BASELINE_FETCH_CHUNK_ADDRESSES: usize = 256; +const LIVE_CODE_HASH_PROGRESS_ROWS: usize = 1_000; +const LIVE_CODE_OBSERVATION_PROGRESS_BLOCKS: usize = 32; #[expect(clippy::too_many_arguments)] +#[allow(dead_code)] pub(crate) async fn persist_reconciled_raw_code_hashes( pool: &sqlx::PgPool, task: &IntakeChainTask, @@ -61,6 +40,58 @@ pub(crate) async fn persist_reconciled_raw_code_hashes( head_change_set: HeadChangeSet, loaded_plan_admission_epoch: i64, coverage_frontiers: &ChainCoverageFrontiers, +) -> Result<()> { + persist_reconciled_raw_code_hashes_inner( + pool, + task, + provider, + heads, + canonical, + head_change_set, + loaded_plan_admission_epoch, + coverage_frontiers, + &mut None, + ) + .await +} + +#[expect(clippy::too_many_arguments)] +pub(crate) async fn persist_reconciled_raw_code_hashes_with_progress( + pool: &sqlx::PgPool, + task: &IntakeChainTask, + provider: &(impl ChainProviderOps + ?Sized), + heads: &ProviderHeadSnapshot, + canonical: &CanonicalReconciliation, + head_change_set: HeadChangeSet, + loaded_plan_admission_epoch: i64, + coverage_frontiers: &ChainCoverageFrontiers, + progress: &mut Option<&mut dyn StartupAdapterProgress>, +) -> Result<()> { + persist_reconciled_raw_code_hashes_inner( + pool, + task, + provider, + heads, + canonical, + head_change_set, + loaded_plan_admission_epoch, + coverage_frontiers, + progress, + ) + .await +} + +#[expect(clippy::too_many_arguments)] +async fn persist_reconciled_raw_code_hashes_inner( + pool: &sqlx::PgPool, + task: &IntakeChainTask, + provider: &(impl ChainProviderOps + ?Sized), + heads: &ProviderHeadSnapshot, + canonical: &CanonicalReconciliation, + head_change_set: HeadChangeSet, + loaded_plan_admission_epoch: i64, + coverage_frontiers: &ChainCoverageFrontiers, + progress: &mut Option<&mut dyn StartupAdapterProgress>, ) -> Result<()> { if canonical.status == CanonicalReconciliationStatus::StoredLineagePromoted { return Ok(()); @@ -74,7 +105,17 @@ pub(crate) async fn persist_reconciled_raw_code_hashes( return Ok(()); } - let raw_blocks = load_raw_blocks_by_hashes(pool, &task.chain, &candidate_hashes).await?; + let mut raw_blocks_by_order = BTreeMap::new(); + for block_page in candidate_hashes.chunks(LIVE_CODE_OBSERVATION_PROGRESS_BLOCKS) { + for raw_block in load_raw_blocks_by_hashes(pool, &task.chain, block_page).await? { + raw_blocks_by_order.insert( + (raw_block.block_number, raw_block.block_hash.clone()), + raw_block, + ); + } + record_progress(pool, progress).await?; + } + let raw_blocks = raw_blocks_by_order.into_values().collect::>(); if raw_blocks.len() != candidate_hashes.len() { bail!( "stored raw block count {} does not match the raw code-hash fetch plan size {} for chain {}", @@ -93,15 +134,22 @@ pub(crate) async fn persist_reconciled_raw_code_hashes( // watched-membership filter runs client-side against the already-sorted // plan surface instead of binding the multi-million-address watch set // back into Postgres on every head-changed tick. - let emitter_addresses_by_block_hash = load_raw_log_emitter_addresses_by_block_hashes( - pool, - &task.chain, - &candidate_hashes, - &generic_resolver_topic0s, - ) - .await? - .into_iter() - .map(|(block_hash, emitters)| { + let mut loaded_emitters_by_block_hash = BTreeMap::new(); + for block_page in candidate_hashes.chunks(LIVE_CODE_OBSERVATION_PROGRESS_BLOCKS) { + loaded_emitters_by_block_hash.extend( + load_raw_log_emitter_addresses_by_block_hashes( + pool, + &task.chain, + block_page, + &generic_resolver_topic0s, + ) + .await?, + ); + record_progress(pool, progress).await?; + } + let mut emitter_addresses_by_block_hash = BTreeMap::new(); + let loaded_emitter_count = loaded_emitters_by_block_hash.len(); + for (index, (block_hash, emitters)) in loaded_emitters_by_block_hash.into_iter().enumerate() { let selected = emitters .into_iter() .filter(|(address, topic0_selected)| { @@ -109,27 +157,39 @@ pub(crate) async fn persist_reconciled_raw_code_hashes( }) .map(|(address, _)| address) .collect::>(); - (block_hash, selected) - }) - .filter(|(_, emitters)| !emitters.is_empty()) - .collect::>>(); - let code_observation_addresses = emitter_addresses_by_block_hash - .values() - .flat_map(|addresses| addresses.iter().cloned()) - .collect::>() - .into_iter() - .collect::>(); - let stored_code_addresses_by_block_hash = load_raw_code_addresses_by_block_hashes( - pool, - &task.chain, - &candidate_hashes, - &code_observation_addresses, - ) - .await?; - let emitter_raw_blocks = raw_blocks - .iter() - .filter_map(|raw_block| { - let emitter_addresses = emitter_addresses_by_block_hash.get(&raw_block.block_hash)?; + if !selected.is_empty() { + emitter_addresses_by_block_hash.insert(block_hash, selected); + } + if index + 1 == loaded_emitter_count + || (index + 1).is_multiple_of(LIVE_CODE_HASH_PROGRESS_ROWS) + { + record_progress(pool, progress).await?; + } + } + let mut stored_code_addresses_by_block_hash = BTreeMap::new(); + for block_page in candidate_hashes.chunks(LIVE_CODE_OBSERVATION_PROGRESS_BLOCKS) { + let code_observation_addresses = block_page + .iter() + .filter_map(|block_hash| emitter_addresses_by_block_hash.get(block_hash)) + .flat_map(|addresses| addresses.iter().cloned()) + .collect::>() + .into_iter() + .collect::>(); + stored_code_addresses_by_block_hash.extend( + load_raw_code_addresses_by_block_hashes( + pool, + &task.chain, + block_page, + &code_observation_addresses, + ) + .await?, + ); + record_progress(pool, progress).await?; + } + let mut emitter_raw_blocks = Vec::new(); + for (index, raw_block) in raw_blocks.iter().enumerate() { + if let Some(emitter_addresses) = emitter_addresses_by_block_hash.get(&raw_block.block_hash) + { let stored_code_addresses = stored_code_addresses_by_block_hash .get(&raw_block.block_hash) .cloned() @@ -140,12 +200,15 @@ pub(crate) async fn persist_reconciled_raw_code_hashes( .filter(|address| block_refreshed || !stored_code_addresses.contains(*address)) .cloned() .collect::>(); - if addresses.is_empty() { - return None; + if !addresses.is_empty() { + emitter_raw_blocks.push((raw_block, addresses)); } - Some((raw_block, addresses)) - }) - .collect::>(); + } + if index + 1 == raw_blocks.len() || (index + 1).is_multiple_of(LIVE_CODE_HASH_PROGRESS_ROWS) + { + record_progress(pool, progress).await?; + } + } let emitter_raw_blocks_by_hash = emitter_raw_blocks .iter() @@ -161,9 +224,15 @@ pub(crate) async fn persist_reconciled_raw_code_hashes( }, ) .collect::>(); - let fetched_observations = provider - .fetch_code_observations_at_block_hashes(&code_observation_requests) - .await?; + let mut fetched_observations = Vec::with_capacity(code_observation_requests.len()); + for request_chunk in code_observation_requests.chunks(LIVE_CODE_OBSERVATION_PROGRESS_BLOCKS) { + fetched_observations.extend( + provider + .fetch_code_observations_at_block_hashes(request_chunk) + .await?, + ); + record_progress(pool, progress).await?; + } if fetched_observations.len() != code_observation_requests.len() { bail!( "provider returned {} code-observation block groups for {} requested blocks on chain {}", @@ -192,8 +261,9 @@ pub(crate) async fn persist_reconciled_raw_code_hashes( .collect::>>()?, ); } - if !code_hashes.is_empty() { - upsert_raw_code_hashes(pool, &code_hashes).await?; + for chunk in code_hashes.chunks(LIVE_CODE_HASH_PROGRESS_ROWS) { + upsert_raw_code_hashes(pool, chunk).await?; + record_progress(pool, progress).await?; } let canonical_baseline_block = canonical.canonical.as_ref().and_then(|canonical_head| { @@ -210,6 +280,7 @@ pub(crate) async fn persist_reconciled_raw_code_hashes( watched_addresses, loaded_plan_admission_epoch, coverage_frontiers, + progress, ) .await?; } @@ -228,6 +299,7 @@ pub(crate) async fn persist_reconciled_raw_code_hashes( /// with a newer epoch is applied, a fresh sweep re-verifies the surface (cheap /// membership probes; only genuinely missing addresses are fetched) so newly /// watched addresses are eventually baselined too. +#[expect(clippy::too_many_arguments)] async fn sweep_raw_code_baseline_chunk( pool: &sqlx::PgPool, task: &IntakeChainTask, @@ -236,6 +308,7 @@ async fn sweep_raw_code_baseline_chunk( watched_addresses: SelectedAddressSet<'_>, loaded_plan_admission_epoch: i64, coverage_frontiers: &ChainCoverageFrontiers, + progress: &mut Option<&mut dyn StartupAdapterProgress>, ) -> Result<()> { let sorted_watched_addresses = watched_addresses.as_sorted_slice(); if sorted_watched_addresses.is_empty() { @@ -304,6 +377,7 @@ async fn sweep_raw_code_baseline_chunk( // advance past a failure boundary without losing that progress. frontier.verified_through_address = chunk.last().cloned(); coverage_frontiers.store_raw_code_baseline_frontier(&task.chain, frontier.clone()); + record_progress(pool, progress).await?; } frontier.verified_through_address = batch.last().cloned(); @@ -316,6 +390,16 @@ async fn sweep_raw_code_baseline_chunk( Ok(()) } +async fn record_progress( + pool: &sqlx::PgPool, + progress: &mut Option<&mut dyn StartupAdapterProgress>, +) -> Result<()> { + if let Some(progress) = progress.as_deref_mut() { + progress.record(pool).await?; + } + Ok(()) +} + async fn load_raw_log_emitter_addresses_by_block_hashes( pool: &sqlx::PgPool, chain: &str, diff --git a/apps/indexer/src/main/reconciliation/persistence/code_hashes/config.rs b/apps/indexer/src/main/reconciliation/persistence/code_hashes/config.rs new file mode 100644 index 00000000..09703a14 --- /dev/null +++ b/apps/indexer/src/main/reconciliation/persistence/code_hashes/config.rs @@ -0,0 +1,26 @@ +use std::sync::OnceLock; + +/// Per-chain watched-address cap for one baseline poll tick. The sweep walks +/// the sorted watch surface behind a process-lifetime cursor, so a +/// multi-million-address surface is baselined across ticks. +pub(super) const DEFAULT_RAW_CODE_BASELINE_MAX_ADDRESSES_PER_TICK: usize = 2_048; +const RAW_CODE_BASELINE_MAX_ADDRESSES_PER_TICK_ENV: &str = + "BIGNAME_INDEXER_RAW_CODE_BASELINE_MAX_ADDRESSES_PER_TICK"; +static RAW_CODE_BASELINE_MAX_ADDRESSES_PER_TICK: OnceLock = OnceLock::new(); + +pub(super) fn raw_code_baseline_max_addresses_per_tick() -> usize { + *RAW_CODE_BASELINE_MAX_ADDRESSES_PER_TICK.get_or_init(|| { + parse_raw_code_baseline_max_addresses_per_tick( + std::env::var(RAW_CODE_BASELINE_MAX_ADDRESSES_PER_TICK_ENV) + .ok() + .as_deref(), + ) + }) +} + +pub(super) fn parse_raw_code_baseline_max_addresses_per_tick(value: Option<&str>) -> usize { + value + .and_then(|value| value.trim().parse::().ok()) + .filter(|value| *value > 0) + .unwrap_or(DEFAULT_RAW_CODE_BASELINE_MAX_ADDRESSES_PER_TICK) +} diff --git a/apps/indexer/src/main/reconciliation/persistence/raw_blocks.rs b/apps/indexer/src/main/reconciliation/persistence/raw_blocks.rs new file mode 100644 index 00000000..fcd0e83d --- /dev/null +++ b/apps/indexer/src/main/reconciliation/persistence/raw_blocks.rs @@ -0,0 +1,89 @@ +use super::*; + +#[allow(dead_code)] +pub(crate) async fn persist_reconciled_raw_blocks( + pool: &sqlx::PgPool, + chain: &str, + heads: &ProviderHeadSnapshot, + canonical: &CanonicalReconciliation, + header_audit_mode: HeaderAuditMode, +) -> Result<()> { + persist_reconciled_raw_blocks_inner(pool, chain, heads, canonical, header_audit_mode, &mut None) + .await +} + +pub(super) async fn persist_reconciled_raw_blocks_inner( + pool: &sqlx::PgPool, + chain: &str, + heads: &ProviderHeadSnapshot, + canonical: &CanonicalReconciliation, + header_audit_mode: HeaderAuditMode, + progress: &mut Option<&mut dyn StartupAdapterProgress>, +) -> Result<()> { + let mut blocks = BTreeMap::::new(); + + let canonical_state = canonical_raw_state(canonical.status); + for (index, block) in canonical.reconciled_blocks.iter().enumerate() { + insert_raw_block_candidate( + &mut blocks, + chain, + block, + canonical_state, + header_audit_mode, + ); + if index + 1 == canonical.reconciled_blocks.len() + || (index + 1).is_multiple_of(LIVE_RAW_FACT_PROGRESS_ROWS) + { + record_progress(pool, progress).await?; + } + } + if let Some(safe) = &heads.safe { + insert_raw_block_candidate( + &mut blocks, + chain, + safe, + CanonicalityState::Safe, + header_audit_mode, + ); + } + if let Some(finalized) = &heads.finalized { + insert_raw_block_candidate( + &mut blocks, + chain, + finalized, + CanonicalityState::Finalized, + header_audit_mode, + ); + } + + let recanonicalize = canonical.status != CanonicalReconciliationStatus::AwaitingAncestor; + let mut page = Vec::with_capacity(LIVE_RAW_FACT_PROGRESS_ROWS); + for block in blocks.into_values() { + page.push(block); + if page.len() == LIVE_RAW_FACT_PROGRESS_ROWS { + persist_raw_block_page(pool, &page, recanonicalize).await?; + page.clear(); + record_progress(pool, progress).await?; + } + } + if !page.is_empty() { + persist_raw_block_page(pool, &page, recanonicalize).await?; + record_progress(pool, progress).await?; + } + Ok(()) +} + +async fn persist_raw_block_page( + pool: &sqlx::PgPool, + blocks: &[RawBlock], + recanonicalize: bool, +) -> Result<()> { + if recanonicalize { + upsert_raw_blocks_recanonicalizing_orphaned(pool, blocks).await?; + } else { + upsert_raw_blocks(pool, blocks).await?; + } + Ok(()) +} + +// Raw-state persistence keeps deployment, heads, canonicality, and refresh inputs explicit. diff --git a/apps/indexer/src/main/reconciliation/replay.rs b/apps/indexer/src/main/reconciliation/replay.rs index 36ed8985..38b96123 100644 --- a/apps/indexer/src/main/reconciliation/replay.rs +++ b/apps/indexer/src/main/reconciliation/replay.rs @@ -1,6 +1,7 @@ use std::{collections::BTreeSet, time::Instant}; use anyhow::{Context, Result, bail}; +use bigname_adapters::StartupAdapterProgress; use bigname_storage::list_canonical_raw_log_replay_inputs_for_block_hashes; use sqlx::Row; use tracing::info; @@ -17,7 +18,7 @@ pub(crate) mod scoped; use super::{ adapter_sync::{ sync_manual_full_closure_normalized_events_from_persisted_raw_payloads, - sync_replay_normalized_events_from_persisted_raw_payloads, + sync_replay_normalized_events_from_persisted_raw_payloads_with_progress, }, types::{ PersistedRawPayloadAdapterSyncSummary, RawFactNormalizedEventReplayOutcome, @@ -74,18 +75,35 @@ pub(crate) async fn replay_raw_fact_normalized_events( pool, request, RawFactReplayExecution::Complete, + &mut None, ) .await } -pub(crate) async fn replay_stateless_normalized_events_before_full_closure( +pub(crate) async fn replay_raw_fact_normalized_events_with_progress( pool: &sqlx::PgPool, request: RawFactNormalizedEventReplayRequest, + progress: &mut dyn StartupAdapterProgress, +) -> Result { + replay_raw_fact_normalized_events_with_execution( + pool, + request, + RawFactReplayExecution::Complete, + &mut Some(progress), + ) + .await +} + +pub(crate) async fn replay_stateless_normalized_events_before_full_closure_with_progress( + pool: &sqlx::PgPool, + request: RawFactNormalizedEventReplayRequest, + progress: &mut dyn StartupAdapterProgress, ) -> Result { replay_raw_fact_normalized_events_with_execution( pool, request, RawFactReplayExecution::StatelessBeforeFullClosure, + &mut Some(progress), ) .await } @@ -98,6 +116,7 @@ pub(crate) async fn replay_stateless_only_raw_fact_normalized_events( pool, request, RawFactReplayExecution::StatelessOnlyAuthoritative, + &mut None, ) .await } @@ -106,6 +125,7 @@ async fn replay_raw_fact_normalized_events_with_execution( pool: &sqlx::PgPool, request: RawFactNormalizedEventReplayRequest, execution: RawFactReplayExecution, + progress: &mut Option<&mut dyn StartupAdapterProgress>, ) -> Result { if request.deployment_profile.trim().is_empty() { bail!("deployment_profile must not be empty"); @@ -176,13 +196,14 @@ async fn replay_raw_fact_normalized_events_with_execution( resolver_profile_authority_scan_count: 0, } } else { - sync_replay_normalized_events_from_persisted_raw_payloads( + sync_replay_normalized_events_from_persisted_raw_payloads_with_progress( pool, &request.chain, &raw_log_selection.block_hashes, Some(source_scope), raw_log_selection.canonical_raw_log_count, replay_contract_plan, + progress, ) .await? }; diff --git a/apps/indexer/src/main/resolver_profile_convergence.rs b/apps/indexer/src/main/resolver_profile_convergence.rs index d140900d..dc45dc0f 100644 --- a/apps/indexer/src/main/resolver_profile_convergence.rs +++ b/apps/indexer/src/main/resolver_profile_convergence.rs @@ -14,9 +14,14 @@ use authority::ResolverProfileAuthorityEntry; #[cfg(test)] pub(crate) use authority::ResolverProfileAuthoritySnapshot; pub(crate) use authority::{ - journal_resolver_profile_authority, journal_resolver_profile_authority_if_epoch_changed, + ResolverProfileAuthorityJournalSummary, journal_resolver_profile_authority, + journal_resolver_profile_authority_if_epoch_changed, + journal_resolver_profile_authority_if_epoch_changed_with_progress, + journal_resolver_profile_authority_with_progress, +}; +pub(crate) use drain::{ + drain_resolver_profile_input_changes, drain_resolver_profile_input_changes_with_progress, }; -pub(crate) use drain::drain_resolver_profile_input_changes; #[derive(Clone, Debug, Default, Eq, PartialEq)] pub(crate) struct ResolverProfileConvergenceSummary { diff --git a/apps/indexer/src/main/resolver_profile_convergence/authority.rs b/apps/indexer/src/main/resolver_profile_convergence/authority.rs index 8ae15e5d..4ed63728 100644 --- a/apps/indexer/src/main/resolver_profile_convergence/authority.rs +++ b/apps/indexer/src/main/resolver_profile_convergence/authority.rs @@ -15,7 +15,10 @@ mod journal; #[cfg(test)] use journal::journal_resolver_profile_authority_attempt; pub(crate) use journal::{ - journal_resolver_profile_authority, journal_resolver_profile_authority_if_epoch_changed, + ResolverProfileAuthorityJournalSummary, journal_resolver_profile_authority, + journal_resolver_profile_authority_if_epoch_changed, + journal_resolver_profile_authority_if_epoch_changed_with_progress, + journal_resolver_profile_authority_with_progress, }; #[derive(Clone, Debug, Deserialize, Eq, Ord, PartialEq, PartialOrd, Serialize)] diff --git a/apps/indexer/src/main/resolver_profile_convergence/authority/journal.rs b/apps/indexer/src/main/resolver_profile_convergence/authority/journal.rs index 34ae2d64..5ad29916 100644 --- a/apps/indexer/src/main/resolver_profile_convergence/authority/journal.rs +++ b/apps/indexer/src/main/resolver_profile_convergence/authority/journal.rs @@ -1,9 +1,11 @@ -use std::collections::BTreeMap; +use std::collections::{BTreeMap, BTreeSet}; use anyhow::{Context, Result, bail, ensure}; +use bigname_adapters::StartupAdapterProgress; use bigname_manifests::ResolverProfileAuthorityTargetPages; use bigname_storage::{ ResolverProfileAuthorityJournalAdvance, ResolverProfileAuthorityJournalEntry, + ResolverProfileAuthorityJournalProgress, ResolverProfileAuthorityJournalProgressFuture, begin_resolver_profile_authority_journal_advance, load_resolver_profile_authority_journal, }; use tracing::info; @@ -16,6 +18,7 @@ use bigname_storage::ResolverProfileAuthorityJournal; const MAX_AUTHORITY_JOURNAL_ATTEMPTS: usize = 32; const AUTHORITY_TARGET_PAGE_SIZE: usize = 250; +const AUTHORITY_DIFF_PAGE_SIZE: usize = 1_000; const MIN_AUTHORITY_JOURNAL_POOL_CONNECTIONS: u32 = 3; #[derive(Clone, Debug, Default, Eq, PartialEq)] @@ -35,6 +38,20 @@ pub(crate) struct ResolverProfileAuthorityJournalSummary { /// rolls both changes back. pub(crate) async fn journal_resolver_profile_authority( pool: &sqlx::PgPool, +) -> Result { + journal_resolver_profile_authority_inner(pool, None).await +} + +pub(crate) async fn journal_resolver_profile_authority_with_progress( + pool: &sqlx::PgPool, + progress: &mut dyn StartupAdapterProgress, +) -> Result { + journal_resolver_profile_authority_inner(pool, Some(progress)).await +} + +async fn journal_resolver_profile_authority_inner( + pool: &sqlx::PgPool, + mut progress: Option<&mut dyn StartupAdapterProgress>, ) -> Result { ensure!( pool.options().get_max_connections() >= MIN_AUTHORITY_JOURNAL_POOL_CONNECTIONS, @@ -55,7 +72,7 @@ pub(crate) async fn journal_resolver_profile_authority( let epochs_before = load_discovery_admission_epochs(pool).await?; let mut advance = begin_resolver_profile_authority_journal_advance(pool, persisted.revision).await?; - stage_current_authority(pool, &mut advance).await?; + stage_current_authority(pool, &mut advance, &mut progress).await?; summary.authority_scan_count += 1; let epochs_after = load_discovery_admission_epochs(pool).await?; if epochs_before != epochs_after { @@ -63,7 +80,11 @@ pub(crate) async fn journal_resolver_profile_authority( summary.unstable_epoch_count += 1; continue; } - let changed_entry_count = advance.staged_entry_diff_count().await?; + let changed_entry_count = if progress.is_some() { + materialize_authority_diff_with_progress(pool, &mut advance, &mut progress).await? + } else { + advance.staged_entry_diff_count().await? + }; if persisted.revision > 0 && changed_entry_count == 0 && persisted_epochs == epochs_after { advance.abort().await?; return Ok(summary); @@ -71,7 +92,15 @@ pub(crate) async fn journal_resolver_profile_authority( let serialized_epochs = serde_json::to_value(&epochs_after) .context("failed to encode current resolver-profile discovery-epoch snapshot")?; - let advanced = advance.publish(&serialized_epochs).await?; + let advanced = if let Some(progress) = progress.as_deref_mut() { + let mut publication_progress = AuthorityJournalPublicationProgress { pool, progress }; + advance + .publish_with_progress(&serialized_epochs, &mut publication_progress) + .await? + } else { + advance.publish(&serialized_epochs).await? + }; + record_progress(pool, &mut progress).await?; if let Some(advanced) = advanced { summary.enqueued_target_count += u64::try_from(advanced.enqueued_target_count)?; summary.journal_advanced = true; @@ -98,18 +127,65 @@ pub(crate) async fn journal_resolver_profile_authority( ) } +struct AuthorityJournalPublicationProgress<'a> { + pool: &'a sqlx::PgPool, + progress: &'a mut dyn StartupAdapterProgress, +} + +impl ResolverProfileAuthorityJournalProgress for AuthorityJournalPublicationProgress<'_> { + fn record(&mut self) -> ResolverProfileAuthorityJournalProgressFuture<'_> { + Box::pin(self.progress.record(self.pool)) + } +} + +async fn materialize_authority_diff_with_progress( + pool: &sqlx::PgPool, + advance: &mut ResolverProfileAuthorityJournalAdvance, + progress: &mut Option<&mut dyn StartupAdapterProgress>, +) -> Result { + advance.begin_staged_entry_diff().await?; + let mut after = None::; + let mut changed_entry_count = 0_i64; + loop { + let Some((last_key, inserted_count)) = advance + .materialize_staged_entry_diff_page(after.as_deref(), AUTHORITY_DIFF_PAGE_SIZE) + .await? + else { + break; + }; + after = Some(last_key); + changed_entry_count += inserted_count; + record_progress(pool, progress).await?; + } + let changed_entry_count = advance + .finish_staged_entry_diff(changed_entry_count) + .await?; + record_progress(pool, progress).await?; + Ok(changed_entry_count) +} + async fn stage_current_authority( pool: &sqlx::PgPool, advance: &mut ResolverProfileAuthorityJournalAdvance, + progress: &mut Option<&mut dyn StartupAdapterProgress>, ) -> Result<()> { let mut targets = ResolverProfileAuthorityTargetPages::begin(advance.connection_mut()?).await?; + let mut seen_targets = BTreeSet::new(); loop { - let page = targets + let streamed_page = targets .next_page(advance.connection_mut()?, AUTHORITY_TARGET_PAGE_SIZE) .await?; - if page.is_empty() { + if streamed_page.is_empty() { break; } + let page = streamed_page + .into_iter() + .filter(|target| seen_targets.insert(target.clone())) + .collect::>(); + if page.is_empty() { + record_progress(pool, progress).await?; + continue; + } let entries = capture_resolver_profile_authority_for_targets(pool, &page).await?; let entries = entries .into_iter() @@ -120,16 +196,43 @@ async fn stage_current_authority( }) .collect::>>()?; advance.stage_entries(&entries).await?; + record_progress(pool, progress).await?; } targets.finish(advance.connection_mut()?).await } +async fn record_progress( + pool: &sqlx::PgPool, + progress: &mut Option<&mut dyn StartupAdapterProgress>, +) -> Result<()> { + if let Some(progress) = progress.as_deref_mut() { + progress.record(pool).await?; + } + Ok(()) +} + /// Cheap ordinary-live guard. A chain epoch match performs no /// [resolver-profile](../../../../../../docs/glossary.md) authority scan; drift /// falls through to the full revision-fenced journal. pub(crate) async fn journal_resolver_profile_authority_if_epoch_changed( pool: &sqlx::PgPool, chain: &str, +) -> Result { + journal_resolver_profile_authority_if_epoch_changed_inner(pool, chain, None).await +} + +pub(crate) async fn journal_resolver_profile_authority_if_epoch_changed_with_progress( + pool: &sqlx::PgPool, + chain: &str, + progress: &mut dyn StartupAdapterProgress, +) -> Result { + journal_resolver_profile_authority_if_epoch_changed_inner(pool, chain, Some(progress)).await +} + +async fn journal_resolver_profile_authority_if_epoch_changed_inner( + pool: &sqlx::PgPool, + chain: &str, + progress: Option<&mut dyn StartupAdapterProgress>, ) -> Result { let (revision, persisted_epoch) = sqlx::query_as::<_, (i64, i64)>( r#" @@ -157,7 +260,7 @@ pub(crate) async fn journal_resolver_profile_authority_if_epoch_changed( }); } - let mut summary = journal_resolver_profile_authority(pool).await?; + let mut summary = journal_resolver_profile_authority_inner(pool, progress).await?; summary.epoch_guard_count += 1; Ok(summary) } diff --git a/apps/indexer/src/main/resolver_profile_convergence/drain.rs b/apps/indexer/src/main/resolver_profile_convergence/drain.rs index 5c38864e..dece732b 100644 --- a/apps/indexer/src/main/resolver_profile_convergence/drain.rs +++ b/apps/indexer/src/main/resolver_profile_convergence/drain.rs @@ -1,7 +1,7 @@ use std::collections::{BTreeMap, BTreeSet}; use anyhow::{Context, Result, bail, ensure}; -use bigname_adapters::begin_resolver_profile_event_reconciliation; +use bigname_adapters::{StartupAdapterProgress, begin_resolver_profile_event_reconciliation}; use bigname_storage::{ RESOLVER_PROFILE_AUTHORITY_JOURNAL_ENTRY_BATCH_SIZE, ResolverProfileInputChange, ResolverProfileReconciliationTarget, acknowledge_resolver_profile_input_changes, @@ -17,7 +17,9 @@ use super::{ input_requires_reconciliation, invalidations::{ publish_resolver_profile_projection_invalidations, + publish_resolver_profile_projection_invalidations_with_progress, stage_resolver_profile_projection_invalidations, + stage_resolver_profile_projection_invalidations_with_progress, }, reconciliation_scope, }; @@ -25,8 +27,10 @@ use super::{ // Preserve the previous durable input budget while making every authority and // reconciliation query inside that budget independently bounded. const MAX_DRAIN_INPUTS: usize = 128 * 1_024; +const DRAIN_INPUT_PAGE_SIZE: usize = 1_000; const RECONCILIATION_TARGET_PAGE_SIZE: usize = 250; const MIN_CONVERGENCE_POOL_CONNECTIONS: u32 = 3; +const MIN_PROGRESS_CONVERGENCE_POOL_CONNECTIONS: u32 = 4; /// Converge every currently eligible durable /// [resolver-profile](../../../../../docs/glossary.md) input change. @@ -35,13 +39,28 @@ const MIN_CONVERGENCE_POOL_CONNECTIONS: u32 = 3; /// authorize absence replay remain durably pending. pub(crate) async fn drain_resolver_profile_input_changes( pool: &sqlx::PgPool, +) -> Result { + drain_resolver_profile_input_changes_inner(pool, None).await +} + +pub(crate) async fn drain_resolver_profile_input_changes_with_progress( + pool: &sqlx::PgPool, + progress: &mut dyn StartupAdapterProgress, +) -> Result { + drain_resolver_profile_input_changes_inner(pool, Some(progress)).await +} + +async fn drain_resolver_profile_input_changes_inner( + pool: &sqlx::PgPool, + mut progress: Option<&mut dyn StartupAdapterProgress>, ) -> Result { let mut aggregate = ResolverProfileConvergenceSummary::default(); let mut deferred_inputs = Vec::::new(); let mut deferred_chains = BTreeMap::::new(); while aggregate.loaded_input_count < MAX_DRAIN_INPUTS { - let remaining_input_count = MAX_DRAIN_INPUTS - aggregate.loaded_input_count; + let remaining_input_count = + (MAX_DRAIN_INPUTS - aggregate.loaded_input_count).min(DRAIN_INPUT_PAGE_SIZE); let pending = load_pending_resolver_profile_input_changes_excluding( pool, i64::try_from(remaining_input_count) @@ -53,18 +72,30 @@ pub(crate) async fn drain_resolver_profile_input_changes( log_completed_drain(&aggregate); return Ok(aggregate); } + let (minimum_pool_connections, connection_roles) = if progress.is_some() { + ( + MIN_PROGRESS_CONVERGENCE_POOL_CONNECTIONS, + "runtime writer guard, reconciliation guard, bounded authority/event reads, and progress heartbeat writes", + ) + } else { + ( + MIN_CONVERGENCE_POOL_CONNECTIONS, + "runtime writer guard, reconciliation guard, and bounded authority/event reads", + ) + }; ensure!( - pool.options().get_max_connections() >= MIN_CONVERGENCE_POOL_CONNECTIONS, - "resolver-profile convergence requires at least {MIN_CONVERGENCE_POOL_CONNECTIONS} \ - database connections (runtime writer guard, reconciliation guard, and bounded \ - authority/event reads), but the pool allows only {}", + pool.options().get_max_connections() >= minimum_pool_connections, + "resolver-profile convergence requires at least {minimum_pool_connections} database \ + connections ({connection_roles}), but the pool allows only {}", pool.options().get_max_connections() ); + record_progress(pool, &mut progress).await?; aggregate.loaded_input_count += pending.len(); - let authority_index = load_scoped_authority_index(pool, &pending, &mut aggregate).await?; + let authority_index = + load_scoped_authority_index(pool, &pending, &mut aggregate, &mut progress).await?; let scope = reconciliation_scope(&pending, &authority_index); - classify_deferred_chains(pool, &scope, &mut deferred_chains).await?; + classify_deferred_chains(pool, &scope, &mut deferred_chains, &mut progress).await?; aggregate.deferred_chains = deferred_chains.keys().cloned().collect(); let eligible_direct_targets = scope @@ -82,6 +113,7 @@ pub(crate) async fn drain_resolver_profile_input_changes( &eligible_direct_targets, &eligible_seed_families, &mut aggregate, + &mut progress, ) .await?; @@ -92,6 +124,7 @@ pub(crate) async fn drain_resolver_profile_input_changes( &deferred_chains, &mut deferred_inputs, &mut aggregate, + &mut progress, ) .await?; } @@ -111,6 +144,7 @@ async fn load_scoped_authority_index( pool: &sqlx::PgPool, pending: &[ResolverProfileInputChange], aggregate: &mut ResolverProfileConvergenceSummary, + progress: &mut Option<&mut dyn StartupAdapterProgress>, ) -> Result { let targets = pending .iter() @@ -131,6 +165,7 @@ async fn load_scoped_authority_index( .context("failed to decode scoped resolver-profile authority entry")?, ); } + record_progress(pool, progress).await?; } Ok(authority_index) } @@ -139,6 +174,7 @@ async fn classify_deferred_chains( pool: &sqlx::PgPool, scope: &super::ResolverProfileReconciliationScope, deferred_chains: &mut BTreeMap, + progress: &mut Option<&mut dyn StartupAdapterProgress>, ) -> Result<()> { let chains = scope .direct_targets_by_chain @@ -153,6 +189,7 @@ async fn classify_deferred_chains( let retention_generation = load_raw_log_staging_input_version(pool, &chain) .await? .retention_generation; + record_progress(pool, progress).await?; if retention_generation == 0 { continue; } @@ -173,6 +210,7 @@ async fn reconcile_target_chains( direct_targets_by_chain: &BTreeMap>, seed_families: &[(String, String)], aggregate: &mut ResolverProfileConvergenceSummary, + progress: &mut Option<&mut dyn StartupAdapterProgress>, ) -> Result<()> { let mut families_by_chain = BTreeMap::>::new(); for (chain, source_family) in seed_families { @@ -199,6 +237,7 @@ async fn reconcile_target_chains( let addresses = direct_targets.iter().cloned().collect::>(); for page in addresses.chunks(RECONCILIATION_TARGET_PAGE_SIZE) { reconciliation.stage_addresses(page).await?; + record_progress(pool, progress).await?; } } @@ -239,9 +278,11 @@ async fn reconcile_target_chains( }) .collect::>(); if addresses.is_empty() { + record_progress(pool, progress).await?; continue; } reconciliation.stage_addresses(&addresses).await?; + record_progress(pool, progress).await?; } } @@ -249,20 +290,55 @@ async fn reconcile_target_chains( { aggregate.invalidation_capture_pass_count += 1; } - stage_resolver_profile_projection_invalidations(pool, reconciliation.run_id(), &chain) - .await?; + match progress.as_deref_mut() { + Some(progress) => { + stage_resolver_profile_projection_invalidations_with_progress( + pool, + reconciliation.run_id(), + &chain, + progress, + ) + .await? + } + None => { + stage_resolver_profile_projection_invalidations( + pool, + reconciliation.run_id(), + &chain, + ) + .await? + } + } #[cfg(test)] { aggregate.adapter_reconciliation_call_count += 1; } - let mut publication = reconciliation - .reconcile() - .await + let reconciliation_result = match progress.as_deref_mut() { + Some(progress) => reconciliation.reconcile_with_progress(progress).await, + None => reconciliation.reconcile().await, + }; + let mut publication = reconciliation_result .with_context(|| format!("failed to reconcile resolver-profile events on {chain}"))?; - aggregate.invalidated_projection_key_count += - publish_resolver_profile_projection_invalidations(publication.connection_mut(), &chain) - .await?; + aggregate.invalidated_projection_key_count += match progress.as_deref_mut() { + Some(progress) => { + publish_resolver_profile_projection_invalidations_with_progress( + pool, + publication.connection_mut(), + &chain, + progress, + ) + .await? + } + None => { + publish_resolver_profile_projection_invalidations( + publication.connection_mut(), + &chain, + ) + .await? + } + }; let summary = publication.finish().await?; + record_progress(pool, progress).await?; info!( service = "indexer", command = "resolver-profile-convergence", @@ -288,6 +364,7 @@ async fn acknowledge_inputs( deferred_chains: &BTreeMap, deferred_inputs: &mut Vec, aggregate: &mut ResolverProfileConvergenceSummary, + progress: &mut Option<&mut dyn StartupAdapterProgress>, ) -> Result<()> { let mut candidates = Vec::new(); for input in pending { @@ -301,11 +378,24 @@ async fn acknowledge_inputs( } } let acknowledged = acknowledge_resolver_profile_input_changes(pool, &candidates).await?; + if !candidates.is_empty() { + record_progress(pool, progress).await?; + } aggregate.acknowledged_input_count += acknowledged; aggregate.concurrent_input_count += candidates.len() - acknowledged; Ok(()) } +async fn record_progress( + pool: &sqlx::PgPool, + progress: &mut Option<&mut dyn StartupAdapterProgress>, +) -> Result<()> { + if let Some(progress) = progress.as_deref_mut() { + progress.record(pool).await?; + } + Ok(()) +} + fn log_completed_drain(aggregate: &ResolverProfileConvergenceSummary) { if aggregate.loaded_input_count == 0 { return; diff --git a/apps/indexer/src/main/resolver_profile_convergence/invalidations.rs b/apps/indexer/src/main/resolver_profile_convergence/invalidations.rs index 0b58fcce..a3c853fa 100644 --- a/apps/indexer/src/main/resolver_profile_convergence/invalidations.rs +++ b/apps/indexer/src/main/resolver_profile_convergence/invalidations.rs @@ -1,201 +1,44 @@ use anyhow::{Context, Result, ensure}; -use serde_json::Value; -use sqlx::{Connection, PgConnection, Postgres, QueryBuilder}; -use uuid::Uuid; +use bigname_adapters::StartupAdapterProgress; +use sqlx::PgConnection; -const INVALIDATION_PAGE_SIZE: usize = 1_000; - -/// Stream every projection key from the exact staged target set before adapter -/// publication can orphan normalized events used to derive record inventories. -pub(super) async fn stage_resolver_profile_projection_invalidations( - pool: &sqlx::PgPool, - run_id: Uuid, - chain: &str, -) -> Result<()> { - let cursor_name = format!("resolver_profile_invalidations_{}", run_id.simple()); - let declare_cursor = invalidation_cursor_sql(run_id, &cursor_name); - let mut connection = pool - .acquire() - .await - .context("failed to acquire resolver-profile invalidation capture connection")?; - let mut transaction = connection - .begin() - .await - .context("failed to begin resolver-profile invalidation capture")?; - sqlx::query(&declare_cursor) - .execute(transaction.as_mut()) - .await - .with_context(|| { - format!("failed to declare resolver-profile invalidation cursor for {chain}") - })?; - // WITH HOLD materializes one stable pre-adapter key stream while allowing - // each staging mutation below to commit as its own bounded statement. - transaction - .commit() - .await - .context("failed to materialize resolver-profile invalidation cursor")?; - - let capture_result = - stage_invalidation_cursor_pages(connection.as_mut(), &cursor_name, chain).await; - let close_result = sqlx::query(&format!("CLOSE {cursor_name}")) - .execute(connection.as_mut()) - .await - .context("failed to close resolver-profile invalidation cursor"); - if let Err(error) = capture_result { - let _ = close_result; - return Err(error); - } - close_result?; - Ok(()) -} - -fn invalidation_cursor_sql(run_id: Uuid, cursor_name: &str) -> String { - format!( - r#" - DECLARE {cursor_name} NO SCROLL CURSOR WITH HOLD FOR - WITH targets AS ( - SELECT run.chain_id, target.resolver_address - FROM resolver_profile_reconciliation_targets target - JOIN resolver_profile_reconciliation_runs run - ON run.run_id = target.run_id - WHERE target.run_id = '{run_id}'::UUID - ), - bound_names AS ( - SELECT DISTINCT event.logical_name_id - FROM normalized_events event - JOIN LATERAL ( - VALUES - (event.before_state ->> 'resolver'), - (event.after_state ->> 'resolver') - ) resolver(resolver_address) ON TRUE - JOIN targets target - ON target.chain_id = event.chain_id - AND target.resolver_address = lower(resolver.resolver_address) - WHERE event.event_kind = 'ResolverChanged' - AND event.logical_name_id IS NOT NULL - AND event.canonicality_state IN ( - 'canonical'::canonicality_state, - 'safe'::canonicality_state, - 'finalized'::canonicality_state - ) - ), - inventory_resources AS ( - SELECT DISTINCT event.resource_id - FROM normalized_events event - JOIN bound_names name - ON name.logical_name_id = event.logical_name_id - JOIN resources resource - ON resource.resource_id = event.resource_id - WHERE event.resource_id IS NOT NULL - AND event.canonicality_state IN ( - 'canonical'::canonicality_state, - 'safe'::canonicality_state, - 'finalized'::canonicality_state - ) - AND resource.canonicality_state IN ( - 'canonical'::canonicality_state, - 'safe'::canonicality_state, - 'finalized'::canonicality_state - ) +#[path = "invalidations/capture.rs"] +mod capture; - UNION - - SELECT DISTINCT binding.resource_id - FROM surface_bindings binding - JOIN bound_names name - ON name.logical_name_id = binding.logical_name_id - JOIN resources resource - ON resource.resource_id = binding.resource_id - WHERE binding.canonicality_state IN ( - 'canonical'::canonicality_state, - 'safe'::canonicality_state, - 'finalized'::canonicality_state - ) - AND resource.canonicality_state IN ( - 'canonical'::canonicality_state, - 'safe'::canonicality_state, - 'finalized'::canonicality_state - ) - ), - candidate_keys AS ( - SELECT - target.chain_id, - 'resolver_current'::TEXT AS projection, - target.chain_id || ':' || target.resolver_address AS projection_key, - jsonb_build_object( - 'chain_id', target.chain_id, - 'resolver_address', target.resolver_address - ) AS key_payload - FROM targets target +pub(super) use capture::{ + stage_resolver_profile_projection_invalidations, + stage_resolver_profile_projection_invalidations_with_progress, +}; - UNION +const INVALIDATION_PAGE_SIZE: usize = 1_000; - SELECT - target_chain.chain_id, - 'record_inventory_current'::TEXT AS projection, - resource_id::TEXT AS projection_key, - jsonb_build_object('resource_id', resource_id::TEXT) AS key_payload - FROM inventory_resources - CROSS JOIN ( - SELECT DISTINCT chain_id - FROM targets - ) target_chain - ) - SELECT chain_id, projection, projection_key, key_payload - FROM candidate_keys - ORDER BY projection, projection_key - "#, - ) +/// Publish and remove staged key pages in the same transaction that commits +/// the adapter's matching chain-context reconciliation. +pub(super) async fn publish_resolver_profile_projection_invalidations( + connection: &mut PgConnection, + chain: &str, +) -> Result { + publish_resolver_profile_projection_invalidations_inner(connection, chain, None).await } -async fn stage_invalidation_cursor_pages( +pub(super) async fn publish_resolver_profile_projection_invalidations_with_progress( + pool: &sqlx::PgPool, connection: &mut PgConnection, - cursor_name: &str, chain: &str, -) -> Result<()> { - loop { - let rows = sqlx::query_as::<_, (String, String, String, Value)>(&format!( - "FETCH FORWARD {INVALIDATION_PAGE_SIZE} FROM {cursor_name}" - )) - .fetch_all(&mut *connection) - .await - .context("failed to fetch resolver-profile invalidation key page")?; - if rows.is_empty() { - return Ok(()); - } - ensure!( - rows.iter().all(|(row_chain, _, _, _)| row_chain == chain), - "resolver-profile invalidation cursor crossed its requested chain boundary" - ); - let mut builder = QueryBuilder::::new( - "INSERT INTO resolver_profile_reconciliation_invalidation_keys \ - (chain_id, projection, projection_key, key_payload) ", - ); - builder.push_values(&rows, |mut row, (chain, projection, key, payload)| { - row.push_bind(chain) - .push_bind(projection) - .push_bind(key) - .push_bind(payload); - }); - builder.push( - " ON CONFLICT (chain_id, projection, projection_key) \ - DO UPDATE SET key_payload = EXCLUDED.key_payload", - ); - builder - .build() - .execute(&mut *connection) - .await - .with_context(|| { - format!("failed to stage resolver-profile projection invalidation page on {chain}") - })?; - } + progress: &mut dyn StartupAdapterProgress, +) -> Result { + publish_resolver_profile_projection_invalidations_inner( + connection, + chain, + Some((pool, progress)), + ) + .await } -/// Publish and remove staged key pages in the same transaction that commits -/// the adapter's matching chain-context reconciliation. -pub(super) async fn publish_resolver_profile_projection_invalidations( +async fn publish_resolver_profile_projection_invalidations_inner( connection: &mut PgConnection, chain: &str, + mut progress: Option<(&sqlx::PgPool, &mut dyn StartupAdapterProgress)>, ) -> Result { let mut published_count = 0u64; loop { @@ -258,5 +101,8 @@ pub(super) async fn publish_resolver_profile_projection_invalidations( published_count = published_count .checked_add(u64::try_from(page_count)?) .context("resolver-profile invalidation count overflowed u64")?; + if let Some((pool, progress)) = progress.as_mut() { + progress.record(pool).await?; + } } } diff --git a/apps/indexer/src/main/resolver_profile_convergence/invalidations/capture.rs b/apps/indexer/src/main/resolver_profile_convergence/invalidations/capture.rs new file mode 100644 index 00000000..a34d6fa5 --- /dev/null +++ b/apps/indexer/src/main/resolver_profile_convergence/invalidations/capture.rs @@ -0,0 +1,431 @@ +use anyhow::{Context, Result, ensure}; +use bigname_adapters::StartupAdapterProgress; +use serde_json::{Value, json}; +use sqlx::{Postgres, QueryBuilder, Transaction}; +use uuid::Uuid; + +const SOURCE_PAGE_SIZE: i64 = 1_000; +const TEMP_BOUND_NAMES: &str = "resolver_profile_invalidation_bound_names"; + +type InvalidationRow = (String, String, String, Value); + +/// Capture every projection key from a stable pre-publication snapshot. Raw +/// event and binding scans advance by primary-key pages so no query must first +/// materialize the whole historical target set before progress is observable. +pub(crate) async fn stage_resolver_profile_projection_invalidations( + pool: &sqlx::PgPool, + run_id: Uuid, + chain: &str, +) -> Result<()> { + stage_resolver_profile_projection_invalidations_inner(pool, run_id, chain, None).await +} + +pub(crate) async fn stage_resolver_profile_projection_invalidations_with_progress( + pool: &sqlx::PgPool, + run_id: Uuid, + chain: &str, + progress: &mut dyn StartupAdapterProgress, +) -> Result<()> { + stage_resolver_profile_projection_invalidations_inner(pool, run_id, chain, Some(progress)).await +} + +async fn stage_resolver_profile_projection_invalidations_inner( + pool: &sqlx::PgPool, + run_id: Uuid, + chain: &str, + mut progress: Option<&mut dyn StartupAdapterProgress>, +) -> Result<()> { + let mut transaction = pool + .begin() + .await + .context("failed to begin resolver-profile invalidation capture")?; + sqlx::query("SET TRANSACTION ISOLATION LEVEL REPEATABLE READ") + .execute(&mut *transaction) + .await + .context("failed to establish resolver-profile invalidation snapshot")?; + sqlx::query(&format!( + "CREATE TEMP TABLE {TEMP_BOUND_NAMES} (logical_name_id TEXT PRIMARY KEY) ON COMMIT DROP" + )) + .execute(&mut *transaction) + .await + .context("failed to create resolver-profile invalidation bound-name staging table")?; + + stage_resolver_targets(pool, &mut transaction, run_id, chain, &mut progress).await?; + materialize_bound_names(pool, &mut transaction, run_id, chain, &mut progress).await?; + stage_event_resources(pool, &mut transaction, chain, &mut progress).await?; + stage_binding_resources(pool, &mut transaction, chain, &mut progress).await?; + + transaction + .commit() + .await + .context("failed to commit resolver-profile invalidation capture")?; + Ok(()) +} + +async fn stage_resolver_targets( + pool: &sqlx::PgPool, + transaction: &mut Transaction<'_, Postgres>, + run_id: Uuid, + chain: &str, + progress: &mut Option<&mut dyn StartupAdapterProgress>, +) -> Result<()> { + let mut after = None::; + loop { + let targets = sqlx::query_as::<_, (String, String)>( + r#" + SELECT run.chain_id, target.resolver_address + FROM resolver_profile_reconciliation_targets target + JOIN resolver_profile_reconciliation_runs run + ON run.run_id = target.run_id + WHERE target.run_id = $1 + AND ($2::TEXT IS NULL OR target.resolver_address > $2) + ORDER BY target.resolver_address + LIMIT $3 + "#, + ) + .bind(run_id) + .bind(after.as_deref()) + .bind(SOURCE_PAGE_SIZE) + .fetch_all(&mut **transaction) + .await + .with_context(|| format!("failed to load resolver-profile target page on {chain}"))?; + let Some((_, last_address)) = targets.last() else { + return Ok(()); + }; + ensure!( + targets + .iter() + .all(|(target_chain, _)| target_chain == chain), + "resolver-profile invalidation target page crossed its requested chain boundary" + ); + after = Some(last_address.clone()); + let invalidations = targets + .into_iter() + .map(|(target_chain, resolver_address)| { + ( + target_chain.clone(), + "resolver_current".to_owned(), + format!("{target_chain}:{resolver_address}"), + json!({ + "chain_id": target_chain, + "resolver_address": resolver_address, + }), + ) + }) + .collect::>(); + insert_invalidations(transaction, &invalidations, chain).await?; + record_progress(pool, progress).await?; + } +} + +async fn materialize_bound_names( + pool: &sqlx::PgPool, + transaction: &mut Transaction<'_, Postgres>, + run_id: Uuid, + chain: &str, + progress: &mut Option<&mut dyn StartupAdapterProgress>, +) -> Result<()> { + let event_watermark = normalized_event_watermark(transaction).await?; + let mut after = 0i64; + while after < event_watermark { + let rows = sqlx::query_as::<_, (Option, Option)>( + r#" + WITH source_page AS MATERIALIZED ( + SELECT + normalized_event_id, + chain_id, + logical_name_id, + event_kind, + canonicality_state, + before_state, + after_state + FROM normalized_events + WHERE normalized_event_id > $1 + AND normalized_event_id <= $2 + ORDER BY normalized_event_id + LIMIT $3 + ), + page_end AS ( + SELECT MAX(normalized_event_id) AS last_id + FROM source_page + ), + bound_names AS ( + SELECT DISTINCT event.logical_name_id + FROM source_page event + JOIN LATERAL ( + VALUES + (event.before_state ->> 'resolver'), + (event.after_state ->> 'resolver') + ) resolver(resolver_address) ON TRUE + JOIN resolver_profile_reconciliation_targets target + ON target.run_id = $4 + AND target.resolver_address = lower(resolver.resolver_address) + WHERE event.chain_id = $5 + AND event.event_kind = 'ResolverChanged' + AND event.logical_name_id IS NOT NULL + AND event.canonicality_state IN ( + 'canonical'::canonicality_state, + 'safe'::canonicality_state, + 'finalized'::canonicality_state + ) + ) + SELECT page_end.last_id, bound.logical_name_id + FROM page_end + LEFT JOIN bound_names bound ON TRUE + "#, + ) + .bind(after) + .bind(event_watermark) + .bind(SOURCE_PAGE_SIZE) + .bind(run_id) + .bind(chain) + .fetch_all(&mut **transaction) + .await + .with_context(|| format!("failed to scan resolver-profile bound-name page on {chain}"))?; + let Some(last_id) = rows.first().and_then(|(last_id, _)| *last_id) else { + break; + }; + ensure!( + last_id > after, + "resolver-profile bound-name scan did not advance" + ); + after = last_id; + let names = rows + .into_iter() + .filter_map(|(_, logical_name_id)| logical_name_id) + .collect::>(); + insert_bound_names(transaction, &names).await?; + record_progress(pool, progress).await?; + } + Ok(()) +} + +async fn stage_event_resources( + pool: &sqlx::PgPool, + transaction: &mut Transaction<'_, Postgres>, + chain: &str, + progress: &mut Option<&mut dyn StartupAdapterProgress>, +) -> Result<()> { + let event_watermark = normalized_event_watermark(transaction).await?; + let mut after = 0i64; + while after < event_watermark { + let rows = sqlx::query_as::<_, (Option, Option)>(&format!( + r#" + WITH source_page AS MATERIALIZED ( + SELECT normalized_event_id, logical_name_id, resource_id, canonicality_state + FROM normalized_events + WHERE normalized_event_id > $1 + AND normalized_event_id <= $2 + ORDER BY normalized_event_id + LIMIT $3 + ), + page_end AS ( + SELECT MAX(normalized_event_id) AS last_id + FROM source_page + ), + resources AS ( + SELECT DISTINCT event.resource_id + FROM source_page event + JOIN {TEMP_BOUND_NAMES} name + ON name.logical_name_id = event.logical_name_id + JOIN resources resource + ON resource.resource_id = event.resource_id + WHERE event.resource_id IS NOT NULL + AND event.canonicality_state IN ( + 'canonical'::canonicality_state, + 'safe'::canonicality_state, + 'finalized'::canonicality_state + ) + AND resource.canonicality_state IN ( + 'canonical'::canonicality_state, + 'safe'::canonicality_state, + 'finalized'::canonicality_state + ) + ) + SELECT page_end.last_id, resource.resource_id + FROM page_end + LEFT JOIN resources resource ON TRUE + "#, + )) + .bind(after) + .bind(event_watermark) + .bind(SOURCE_PAGE_SIZE) + .fetch_all(&mut **transaction) + .await + .with_context(|| { + format!("failed to scan resolver-profile event-resource page on {chain}") + })?; + let Some(last_id) = rows.first().and_then(|(last_id, _)| *last_id) else { + break; + }; + ensure!( + last_id > after, + "resolver-profile event-resource scan did not advance" + ); + after = last_id; + let invalidations = record_inventory_invalidations(chain, rows); + insert_invalidations(transaction, &invalidations, chain).await?; + record_progress(pool, progress).await?; + } + Ok(()) +} + +async fn stage_binding_resources( + pool: &sqlx::PgPool, + transaction: &mut Transaction<'_, Postgres>, + chain: &str, + progress: &mut Option<&mut dyn StartupAdapterProgress>, +) -> Result<()> { + let mut after = None::; + loop { + let rows = sqlx::query_as::<_, (Option, Option)>(&format!( + r#" + WITH source_page AS MATERIALIZED ( + SELECT + surface_binding_id, + logical_name_id, + resource_id, + canonicality_state + FROM surface_bindings + WHERE ($1::UUID IS NULL OR surface_binding_id > $1) + ORDER BY surface_binding_id + LIMIT $2 + ), + page_end AS ( + SELECT surface_binding_id AS last_id + FROM source_page + ORDER BY surface_binding_id DESC + LIMIT 1 + ), + resources AS ( + SELECT DISTINCT binding.resource_id + FROM source_page binding + JOIN {TEMP_BOUND_NAMES} name + ON name.logical_name_id = binding.logical_name_id + JOIN resources resource + ON resource.resource_id = binding.resource_id + WHERE binding.canonicality_state IN ( + 'canonical'::canonicality_state, + 'safe'::canonicality_state, + 'finalized'::canonicality_state + ) + AND resource.canonicality_state IN ( + 'canonical'::canonicality_state, + 'safe'::canonicality_state, + 'finalized'::canonicality_state + ) + ) + SELECT page_end.last_id, resource.resource_id + FROM page_end + LEFT JOIN resources resource ON TRUE + "#, + )) + .bind(after) + .bind(SOURCE_PAGE_SIZE) + .fetch_all(&mut **transaction) + .await + .with_context(|| { + format!("failed to scan resolver-profile binding-resource page on {chain}") + })?; + let Some(last_id) = rows.first().and_then(|(last_id, _)| *last_id) else { + return Ok(()); + }; + ensure!( + after.is_none_or(|previous| last_id > previous), + "resolver-profile binding-resource scan did not advance" + ); + after = Some(last_id); + let invalidations = record_inventory_invalidations(chain, rows); + insert_invalidations(transaction, &invalidations, chain).await?; + record_progress(pool, progress).await?; + } +} + +async fn normalized_event_watermark(transaction: &mut Transaction<'_, Postgres>) -> Result { + sqlx::query_scalar::<_, i64>( + "SELECT COALESCE(MAX(normalized_event_id), 0)::BIGINT FROM normalized_events", + ) + .fetch_one(&mut **transaction) + .await + .context("failed to load resolver-profile invalidation event watermark") +} + +async fn insert_bound_names( + transaction: &mut Transaction<'_, Postgres>, + names: &[String], +) -> Result<()> { + if names.is_empty() { + return Ok(()); + } + let mut builder = + QueryBuilder::::new(format!("INSERT INTO {TEMP_BOUND_NAMES} (logical_name_id) ")); + builder.push_values(names, |mut row, name| { + row.push_bind(name); + }); + builder.push(" ON CONFLICT (logical_name_id) DO NOTHING"); + builder + .build() + .execute(&mut **transaction) + .await + .context("failed to stage resolver-profile invalidation bound-name page")?; + Ok(()) +} + +fn record_inventory_invalidations( + chain: &str, + rows: Vec<(Option, Option)>, +) -> Vec { + rows.into_iter() + .filter_map(|(_, resource_id)| resource_id) + .map(|resource_id| { + ( + chain.to_owned(), + "record_inventory_current".to_owned(), + resource_id.to_string(), + json!({ "resource_id": resource_id.to_string() }), + ) + }) + .collect() +} + +async fn insert_invalidations( + transaction: &mut Transaction<'_, Postgres>, + rows: &[InvalidationRow], + chain: &str, +) -> Result<()> { + if rows.is_empty() { + return Ok(()); + } + let mut builder = QueryBuilder::::new( + "INSERT INTO resolver_profile_reconciliation_invalidation_keys \ + (chain_id, projection, projection_key, key_payload) ", + ); + builder.push_values(rows, |mut row, (chain, projection, key, payload)| { + row.push_bind(chain) + .push_bind(projection) + .push_bind(key) + .push_bind(payload); + }); + builder.push( + " ON CONFLICT (chain_id, projection, projection_key) \ + DO UPDATE SET key_payload = EXCLUDED.key_payload", + ); + builder + .build() + .execute(&mut **transaction) + .await + .with_context(|| { + format!("failed to stage resolver-profile projection invalidation page on {chain}") + })?; + Ok(()) +} + +async fn record_progress( + pool: &sqlx::PgPool, + progress: &mut Option<&mut dyn StartupAdapterProgress>, +) -> Result<()> { + if let Some(progress) = progress.as_deref_mut() { + progress.record(pool).await?; + } + Ok(()) +} diff --git a/apps/indexer/src/main/resolver_profile_convergence/tests.rs b/apps/indexer/src/main/resolver_profile_convergence/tests.rs index d6289252..2ba1ddfe 100644 --- a/apps/indexer/src/main/resolver_profile_convergence/tests.rs +++ b/apps/indexer/src/main/resolver_profile_convergence/tests.rs @@ -2,12 +2,15 @@ use bigname_storage::ResolverProfileInputChange; use bigname_test_support::{TestDatabase, TestDatabaseConfig}; use uuid::Uuid; +use crate::tests::{BlockingHeartbeatProgress, install_stale_indexer_heartbeat}; + use super::{ ResolverProfileAuthorityIndex, ResolverProfileAuthoritySnapshot, ResolverProfileConvergenceSummary, authority::{ResolverProfileAdmissionSemantics, ResolverProfileAuthorityEntry}, - drain_resolver_profile_input_changes, expanded_reconciliation_targets, - expanded_reconciliation_targets_with_family_count, input_requires_reconciliation, + drain_resolver_profile_input_changes, drain_resolver_profile_input_changes_with_progress, + expanded_reconciliation_targets, expanded_reconciliation_targets_with_family_count, + input_requires_reconciliation, }; #[test] @@ -310,7 +313,8 @@ fn indexed_authority_matches_full_scans_for_load_shaped_inputs() { #[tokio::test] async fn pending_input_drain_never_loads_the_full_authority_snapshot() -> anyhow::Result<()> { let database = TestDatabase::create_migrated( - TestDatabaseConfig::new("indexer_resolver_profile_scoped_authority_drain"), + TestDatabaseConfig::new("indexer_resolver_profile_scoped_authority_drain") + .pool_max_connections(3), &bigname_storage::MIGRATOR, "failed to apply migrations for scoped resolver-profile drain test", ) @@ -344,11 +348,60 @@ async fn pending_input_drain_never_loads_the_full_authority_snapshot() -> anyhow } #[tokio::test] -async fn seed_change_reconciles_journal_family_in_bounded_pages() -> anyhow::Result<()> { +async fn convergence_rejects_a_pool_without_a_progress_heartbeat_connection() -> anyhow::Result<()> +{ let database = TestDatabase::create_migrated( - TestDatabaseConfig::new("indexer_resolver_profile_seed_family_pages") + TestDatabaseConfig::new("indexer_resolver_profile_three_connection_rejection") .pool_max_connections(3), &bigname_storage::MIGRATOR, + "failed to apply migrations for resolver-profile pool rejection test", + ) + .await?; + sqlx::query( + r#" + INSERT INTO resolver_profile_input_changes ( + chain_id, + contract_address, + previous_code_hash, + current_code_hash + ) VALUES ( + 'ethereum-mainnet', + '0x0000000000000000000000000000000000000001', + '0x01', + '0x02' + ) + "#, + ) + .execute(database.pool()) + .await?; + let instance_id = "resolver-profile-three-connection-rejection"; + install_stale_indexer_heartbeat(database.pool(), instance_id).await?; + let (mut progress, _) = BlockingHeartbeatProgress::new( + instance_id, + vec!["ethereum-mainnet".to_owned()], + usize::MAX, + ); + + let error = drain_resolver_profile_input_changes_with_progress(database.pool(), &mut progress) + .await + .expect_err("three connections cannot reserve a real convergence heartbeat writer"); + assert!( + error + .to_string() + .contains("requires at least 4 database connections") + && error.to_string().contains("progress heartbeat writes"), + "unexpected convergence pool error: {error:#}" + ); + + database.cleanup().await +} + +#[tokio::test] +async fn seed_change_reconciles_journal_family_with_real_heartbeats() -> anyhow::Result<()> { + let database = TestDatabase::create_migrated( + TestDatabaseConfig::new("indexer_resolver_profile_seed_family_pages") + .pool_max_connections(4), + &bigname_storage::MIGRATOR, "failed to apply migrations for resolver-profile seed-family paging test", ) .await?; @@ -393,12 +446,43 @@ async fn seed_change_reconciles_journal_family_in_bounded_pages() -> anyhow::Res "bigname-indexer", ) .await?; - let summary = tokio::time::timeout( - std::time::Duration::from_secs(30), - drain_resolver_profile_input_changes(database.pool()), - ) + let heartbeat_instance_id = "resolver-profile-seed-family-pages"; + install_stale_indexer_heartbeat(database.pool(), heartbeat_instance_id).await?; + let (mut progress, progress_handle) = BlockingHeartbeatProgress::new( + heartbeat_instance_id, + vec!["ethereum-mainnet".to_owned()], + 2, + ); + let mut operation = Box::pin(drain_resolver_profile_input_changes_with_progress( + database.pool(), + &mut progress, + )); + tokio::time::timeout(std::time::Duration::from_secs(10), async { + tokio::select! { + () = progress_handle.wait_until_blocked() => Ok(()), + result = operation.as_mut() => Err(anyhow::anyhow!( + "resolver-profile convergence completed before its later progress boundary blocked: {result:?}" + )), + } + }) .await - .expect("three-connection seed-family drain must not starve bounded journal/event reads")?; + .expect("resolver-profile convergence did not reach its later progress boundary")?; + let persisted_heartbeat = bigname_storage::load_service_loop_heartbeat( + database.pool(), + bigname_storage::INDEXER_SERVICE_NAME, + heartbeat_instance_id, + ) + .await? + .expect("resolver-profile progress heartbeat must remain registered"); + assert!( + persisted_heartbeat.age_seconds <= 1, + "an earlier convergence page must beat before later target work finishes" + ); + progress_handle.resume(); + let summary = tokio::time::timeout(std::time::Duration::from_secs(30), operation.as_mut()) + .await + .expect("four-connection seed-family drain must not starve progress heartbeat writes")?; + drop(operation); drop(runtime_guard); assert_eq!(summary.loaded_input_count, 1); assert_eq!(summary.authority_target_read_statement_count, 1); @@ -416,6 +500,10 @@ async fn seed_change_reconciles_journal_family_in_bounded_pages() -> anyhow::Res assert_eq!(summary.reconciled_target_count, 2_501); assert_eq!(summary.invalidated_projection_key_count, 2_501); assert_eq!(summary.acknowledged_input_count, 1); + assert!( + progress_handle.record_count() >= 20, + "the multi-page authority scan and reconciliation must report progress before completion" + ); database.cleanup().await } diff --git a/apps/indexer/src/main/run.rs b/apps/indexer/src/main/run.rs index 19bb9072..df7b47ea 100644 --- a/apps/indexer/src/main/run.rs +++ b/apps/indexer/src/main/run.rs @@ -1,6 +1,6 @@ use std::future::Future; -use anyhow::Result; +use anyhow::{Result, ensure}; use tokio::time::Duration; use tracing::info; @@ -18,25 +18,40 @@ use crate::{ provider_configuration::ProviderSourceArgs, reconciliation::HeaderAuditMode, replay::deployment_profile_from_manifest_root, - resolver_profile_convergence::drain_resolver_profile_input_changes, + resolver_profile_convergence::drain_resolver_profile_input_changes_with_progress, run_mode::IndexerRunMode, runtime::{ - IntakeChainTask, ManifestRuntimeState, build_manifest_runtime_state_with_watch_scope, - ensure_manifest_root_ready, intake_runtime_state, load_manifest_repository, - log_intake_chain_tasks, log_manifest_runtime_state, log_manifest_summary, - log_provider_registry, log_watched_chain_plan, manifest_normalized_event_kind_count, - run_poll_loop, sync_adapter_owned_raw_log_state_with_heartbeat, - sync_discovery_adapter_owned_raw_log_state_with_heartbeat, sync_intake_chain_tasks, - validate_provider_registry_for_intake_tasks, watched_chain_plan_state, - widen_runtime_state_to_live_watch_scope_with_admission_epochs, + IntakeChainTask, ManifestRuntimeState, + build_manifest_runtime_state_with_watch_scope_and_progress, ensure_manifest_root_ready, + intake_runtime_state, load_manifest_repository, log_intake_chain_tasks, + log_manifest_runtime_state, log_manifest_summary, log_provider_registry, + log_watched_chain_plan, manifest_normalized_event_kind_count, run_poll_loop, + sync_adapter_owned_raw_log_state_with_heartbeat, + sync_discovery_adapter_owned_raw_log_state_with_heartbeat, + sync_intake_chain_tasks_with_progress, validate_provider_registry_for_intake_tasks, + watched_chain_plan_state, + widen_runtime_state_to_live_watch_scope_with_admission_epochs_and_progress, }, }; -use startup_heartbeat::StartupHeartbeat; +use startup_heartbeat::{RequiredSubtaskActivity, StartupHeartbeat}; const NORMALIZED_REPLAY_CATCHUP_SUBTASK: &str = "normalized_replay_catchup"; +const MIN_INDEXER_RUN_POOL_CONNECTIONS: u32 = 4; + +fn ensure_indexer_run_pool_capacity(database: &bigname_storage::DatabaseConfig) -> Result<()> { + ensure!( + database.max_connections >= MIN_INDEXER_RUN_POOL_CONNECTIONS, + "indexer run progress heartbeats require at least {MIN_INDEXER_RUN_POOL_CONNECTIONS} \ + database connections (runtime writer guard, nested work guards, heartbeat writer); \ + set BIGNAME_DATABASE_MAX_CONNECTIONS or --database-max-connections to \ + {MIN_INDEXER_RUN_POOL_CONNECTIONS} or higher" + ); + Ok(()) +} pub(crate) async fn run(args: RunArgs) -> Result<()> { + ensure_indexer_run_pool_capacity(&args.database)?; let heartbeat_instance_id = bigname_storage::resolve_service_instance_id(args.heartbeat_instance_id.as_deref())?; let manifest_repository = load_manifest_repository(&args.manifests_root)?; @@ -57,6 +72,7 @@ pub(crate) async fn run(args: RunArgs) -> Result<()> { &heartbeat_instance_id, ) .await?; + let normalized_replay_activity = RequiredSubtaskActivity::default(); let mut startup_heartbeat = StartupHeartbeat::new( heartbeat_instance_id.clone(), Duration::from_secs(args.poll_interval_secs.max(1)), @@ -65,12 +81,17 @@ pub(crate) async fn run(args: RunArgs) -> Result<()> { let header_audit_mode = HeaderAuditMode::from_retain_audit_fields(args.retain_header_audit_fields); let run_mode = IndexerRunMode::new(adapter_sync_mode, args.normalized_replay_catchup_enabled); - let manifest_runtime_state = build_manifest_runtime_state_with_watch_scope( - &pool, - &manifest_repository, - run_mode.bootstrap_watch_scope, - ) - .await?; + let manifest_runtime_state = { + let mut progress = + startup_heartbeat::StartupAdapterHeartbeat::new(&mut startup_heartbeat, &[]); + build_manifest_runtime_state_with_watch_scope_and_progress( + &pool, + &manifest_repository, + run_mode.bootstrap_watch_scope, + &mut progress, + ) + .await? + }; let bootstrap_chain_ids = manifest_runtime_state .watched_chain_plan .iter() @@ -99,8 +120,16 @@ pub(crate) async fn run(args: RunArgs) -> Result<()> { } log_manifest_runtime_state(&manifest_runtime_state); log_watched_chain_plan("startup", &manifest_runtime_state.watched_chain_plan); - let intake_chain_tasks = - sync_intake_chain_tasks(&pool, &manifest_runtime_state.watched_chain_plan).await?; + let intake_chain_tasks = { + let mut progress = + startup_heartbeat::StartupAdapterHeartbeat::new(&mut startup_heartbeat, &[]); + sync_intake_chain_tasks_with_progress( + &pool, + &manifest_runtime_state.watched_chain_plan, + &mut progress, + ) + .await? + }; let startup_chain_ids = intake_chain_tasks .iter() .map(|task| task.chain.clone()) @@ -147,6 +176,8 @@ pub(crate) async fn run(args: RunArgs) -> Result<()> { &run_mode, &manifest_runtime_state, &provider_registry, + &mut startup_heartbeat, + &startup_chain_ids, ) .await?; let live_chain_ids = intake_chain_tasks @@ -157,7 +188,11 @@ pub(crate) async fn run(args: RunArgs) -> Result<()> { if adapter_sync_mode != BackfillAdapterSyncMode::RawOnly && !run_mode.normalized_replay_catchup_enabled { - drain_resolver_profile_input_changes(&pool).await?; + let mut progress = startup_heartbeat::StartupAdapterHeartbeat::new( + &mut startup_heartbeat, + &live_chain_ids, + ); + drain_resolver_profile_input_changes_with_progress(&pool, &mut progress).await?; } let (subtasks, subtask_monitor) = subtask_supervision::channel("indexer"); if run_mode.normalized_replay_catchup_enabled { @@ -174,6 +209,11 @@ pub(crate) async fn run(args: RunArgs) -> Result<()> { .with_defer_projection_indexes(args.normalized_replay_defer_projection_indexes); let catchup_pool = pool.clone(); let catchup_provider_registry = provider_registry.clone(); + let catchup_heartbeat = startup_heartbeat::NormalizedReplayHeartbeat::new( + heartbeat_instance_id.clone(), + Duration::from_secs(args.poll_interval_secs.max(1)), + live_chain_ids.clone(), + ); spawn_normalized_replay_catchup( &subtasks, run_normalized_replay_catchup( @@ -181,6 +221,8 @@ pub(crate) async fn run(args: RunArgs) -> Result<()> { catchup_config, catchup_provider_registry, header_audit_mode, + catchup_heartbeat, + normalized_replay_activity.clone(), ), )?; } @@ -311,6 +353,7 @@ pub(crate) async fn run(args: RunArgs) -> Result<()> { // verifies fact coverage in large chunks once, then every poll cycle // is an O(1) in-memory check. &crate::reconciliation::ChainCoverageFrontiers::default(), + &normalized_replay_activity, )) .await } @@ -377,6 +420,8 @@ async fn widen_to_live_watch_scope( run_mode: &IndexerRunMode, manifest_runtime_state: &ManifestRuntimeState, provider_registry: &ProviderRegistry, + heartbeat: &mut StartupHeartbeat, + heartbeat_chain_ids: &[String], ) -> Result<( ManifestRuntimeState, Vec, @@ -386,11 +431,26 @@ async fn widen_to_live_watch_scope( // the post-bootstrap adapter-owned sync may have materialized new discovery edges, and this // reload is what carries them into the live plan. let previous_watch_state = watched_chain_plan_state(&manifest_runtime_state.watched_chain_plan); - let (live_manifest_runtime_state, watched_plan_admission_epochs) = - widen_runtime_state_to_live_watch_scope_with_admission_epochs(pool, manifest_runtime_state) - .await?; - let live_intake_chain_tasks = - sync_intake_chain_tasks(pool, &live_manifest_runtime_state.watched_chain_plan).await?; + let (live_manifest_runtime_state, watched_plan_admission_epochs) = { + let mut progress = + startup_heartbeat::StartupAdapterHeartbeat::new(heartbeat, heartbeat_chain_ids); + widen_runtime_state_to_live_watch_scope_with_admission_epochs_and_progress( + pool, + manifest_runtime_state, + &mut progress, + ) + .await? + }; + let live_intake_chain_tasks = { + let mut progress = + startup_heartbeat::StartupAdapterHeartbeat::new(heartbeat, heartbeat_chain_ids); + sync_intake_chain_tasks_with_progress( + pool, + &live_manifest_runtime_state.watched_chain_plan, + &mut progress, + ) + .await? + }; validate_provider_registry_for_intake_tasks(&live_intake_chain_tasks, provider_registry)?; let live_watch_state = diff --git a/apps/indexer/src/main/run/subtask_tests.rs b/apps/indexer/src/main/run/subtask_tests.rs index d8e9e671..b4190af2 100644 --- a/apps/indexer/src/main/run/subtask_tests.rs +++ b/apps/indexer/src/main/run/subtask_tests.rs @@ -5,7 +5,7 @@ use bigname_storage::DatabaseConfig; use bigname_test_support::{TestDatabase, TestDatabaseConfig}; use sqlx::{ConnectOptions, PgPool, postgres::PgConnectOptions}; use tokio::{ - sync::Notify, + sync::{Notify, oneshot}, time::{Duration, sleep, timeout}, }; @@ -29,6 +29,39 @@ fn healthcheck_args(database: &TestDatabase, instance_id: &str) -> Result Result<()> { let chains = vec!["ethereum-mainnet".to_owned()]; loop { @@ -111,3 +144,174 @@ async fn normalized_replay_catchup_panic_stops_indexer_liveness() -> Result<()> database.cleanup().await } + +#[tokio::test] +async fn normalized_replay_catchup_wedge_is_not_masked_by_parent_heartbeat() -> Result<()> { + let database = TestDatabase::create_migrated( + TestDatabaseConfig::new("bigname_indexer_catchup_wedge_test"), + &bigname_storage::MIGRATOR, + "failed to migrate catch-up wedge test database", + ) + .await?; + let instance_id = "catchup-wedge-test"; + bigname_storage::register_service_loop( + database.pool(), + bigname_storage::INDEXER_SERVICE_NAME, + instance_id, + ) + .await?; + sqlx::query( + r#" + UPDATE service_loop_heartbeats + SET started_at = clock_timestamp() - INTERVAL '2 minutes', + heartbeat_at = clock_timestamp() - INTERVAL '1 minute' + WHERE service_name = 'indexer' + AND instance_id = $1 + AND scope_kind = 'process' + "#, + ) + .bind(instance_id) + .execute(database.pool()) + .await?; + + let activity = RequiredSubtaskActivity::default(); + let wedged_catchup = activity.begin().await; + let parent_activity = activity.clone(); + let parent_pool = database.pool().clone(); + let parent = tokio::spawn(async move { + let _required_subtask_exclusion = parent_activity.exclude_required_subtask().await; + let mut parent_heartbeat = StartupHeartbeat::new(instance_id.to_owned(), Duration::ZERO); + parent_heartbeat + .record(&parent_pool, &["ethereum-mainnet".to_owned()]) + .await + }); + sleep(Duration::from_millis(100)).await; + assert!( + !parent.is_finished(), + "the parent heartbeat must wait behind active required catch-up work" + ); + let heartbeat_age = bigname_storage::load_service_loop_heartbeat( + database.pool(), + bigname_storage::INDEXER_SERVICE_NAME, + instance_id, + ) + .await? + .context("indexer heartbeat must remain registered")? + .age_seconds; + assert!( + heartbeat_age >= 30, + "the parent poll must not refresh liveness while required catch-up work is wedged" + ); + drop(wedged_catchup); + timeout(Duration::from_secs(2), parent) + .await + .context("parent heartbeat did not resume after catch-up released ownership")? + .context("parent heartbeat task panicked")??; + database.cleanup().await?; + Ok(()) +} + +#[tokio::test] +async fn indexer_shutdown_does_not_wait_for_required_subtask_activity() -> Result<()> { + let activity = RequiredSubtaskActivity::default(); + let wedged_catchup = activity.begin().await; + let waiting_activity = activity.clone(); + let (shutdown, shutdown_received) = oneshot::channel::<()>(); + let parent = tokio::spawn(async move { + waiting_activity + .exclude_required_subtask_or_shutdown(async { + let _ = shutdown_received.await; + }) + .await + }); + + sleep(Duration::from_millis(100)).await; + assert!( + !parent.is_finished(), + "the parent must wait while required catch-up work owns the activity gate" + ); + assert!( + shutdown.send(()).is_ok(), + "shutdown receiver must remain live" + ); + let exclusion = timeout(Duration::from_secs(2), parent) + .await + .context("indexer parent did not observe shutdown while catch-up held the activity gate")? + .context("indexer parent shutdown test task panicked")?; + assert!( + exclusion.is_none(), + "shutdown must win without acquiring the parent activity exclusion" + ); + + drop(wedged_catchup); + Ok(()) +} + +#[tokio::test] +async fn normalized_replay_progress_does_not_mask_a_parent_wedge() -> Result<()> { + let database = TestDatabase::create_migrated( + TestDatabaseConfig::new("bigname_indexer_parent_wedge_test"), + &bigname_storage::MIGRATOR, + "failed to migrate parent-wedge test database", + ) + .await?; + let instance_id = "parent-wedge-test"; + bigname_storage::register_service_loop( + database.pool(), + bigname_storage::INDEXER_SERVICE_NAME, + instance_id, + ) + .await?; + sqlx::query( + r#" + UPDATE service_loop_heartbeats + SET started_at = clock_timestamp() - INTERVAL '2 minutes', + heartbeat_at = clock_timestamp() - INTERVAL '1 minute' + WHERE service_name = 'indexer' + AND instance_id = $1 + AND scope_kind = 'process' + "#, + ) + .bind(instance_id) + .execute(database.pool()) + .await?; + + let activity = RequiredSubtaskActivity::default(); + let parent_operation = activity.exclude_required_subtask().await; + let child_activity = activity.clone(); + let child_pool = database.pool().clone(); + let child = tokio::spawn(async move { + let _active_catchup = child_activity.begin().await; + let mut child_heartbeat = startup_heartbeat::NormalizedReplayHeartbeat::new( + instance_id.to_owned(), + Duration::ZERO, + vec!["ethereum-mainnet".to_owned()], + ); + bigname_adapters::StartupAdapterProgress::record(&mut child_heartbeat, &child_pool).await + }); + sleep(Duration::from_millis(100)).await; + assert!( + !child.is_finished(), + "required catch-up must wait while the parent poll owns liveness" + ); + + let heartbeat_age = bigname_storage::load_service_loop_heartbeat( + database.pool(), + bigname_storage::INDEXER_SERVICE_NAME, + instance_id, + ) + .await? + .context("indexer heartbeat must remain registered")? + .age_seconds; + assert!( + heartbeat_age >= 30, + "required catch-up progress must not refresh liveness while the parent poll is wedged" + ); + drop(parent_operation); + timeout(Duration::from_secs(2), child) + .await + .context("catch-up heartbeat did not resume after the parent released ownership")? + .context("catch-up heartbeat task panicked")??; + database.cleanup().await?; + Ok(()) +} diff --git a/apps/indexer/src/main/runtime.rs b/apps/indexer/src/main/runtime.rs index 91e8237a..02365eb3 100644 --- a/apps/indexer/src/main/runtime.rs +++ b/apps/indexer/src/main/runtime.rs @@ -30,8 +30,8 @@ pub(crate) use adapter_sync::{ #[allow(unused_imports)] pub(crate) use intake::{ IntakeChainTask, IntakeRuntimeState, WatchedChainPlanState, checkpoint_mode, - intake_runtime_state, sync_intake_chain_tasks, validate_provider_registry_for_intake_tasks, - watched_chain_plan_state, + intake_runtime_state, sync_intake_chain_tasks, sync_intake_chain_tasks_with_progress, + validate_provider_registry_for_intake_tasks, watched_chain_plan_state, }; #[allow(unused_imports)] pub(crate) use logging::{ @@ -47,8 +47,10 @@ pub(crate) use logging::{ #[allow(unused_imports)] pub(crate) use manifest::{ DiscoveryAdmissionSnapshot, ManifestRuntimeState, RuntimeWatchScope, - build_manifest_runtime_state, build_manifest_runtime_state_for_repository_refresh, - build_manifest_runtime_state_with_watch_scope, discovery_admission_snapshot, + build_manifest_runtime_state, + build_manifest_runtime_state_for_repository_refresh_with_progress, + build_manifest_runtime_state_with_watch_scope, + build_manifest_runtime_state_with_watch_scope_and_progress, discovery_admission_snapshot, ensure_manifest_root_ready, load_manifest_repository, manifest_normalized_event_kind_count, verify_stored_manifest_state, }; @@ -59,13 +61,20 @@ pub(crate) use poll_loop::{ ReplayHandoffLatchStatus, install_replay_handoff_before_latch_test_hook, latch_replay_handoff_if_stable, refresh_discovery_watch_state, }; -#[allow(unused_imports)] +#[cfg(test)] pub(crate) use refresh::{ refresh_intake_chain_tasks, refresh_manifest_normalized_events_from_storage, +}; +#[allow(unused_imports)] +pub(crate) use refresh::{ + refresh_intake_chain_tasks_with_progress, + refresh_manifest_normalized_events_from_storage_with_progress, refresh_runtime_state_from_storage_discovery, refresh_runtime_state_from_stored_discovery, - refresh_runtime_state_from_stored_discovery_when_epochs_move, refresh_watched_chain_plan, - widen_runtime_state_to_live_watch_scope, + refresh_runtime_state_from_stored_discovery_when_epochs_move, + refresh_runtime_state_from_stored_discovery_when_epochs_move_with_progress, + refresh_watched_chain_plan, widen_runtime_state_to_live_watch_scope, widen_runtime_state_to_live_watch_scope_with_admission_epochs, + widen_runtime_state_to_live_watch_scope_with_admission_epochs_and_progress, }; #[allow(unused_imports)] pub(crate) use tracing_init::init_tracing; diff --git a/apps/indexer/src/main/runtime/adapter_sync.rs b/apps/indexer/src/main/runtime/adapter_sync.rs index 63fbf39e..9f831776 100644 --- a/apps/indexer/src/main/runtime/adapter_sync.rs +++ b/apps/indexer/src/main/runtime/adapter_sync.rs @@ -2,7 +2,9 @@ use anyhow::{Context, Result}; use bigname_manifests::WatchedChainPlan; use crate::{ - resolver_profile_convergence::journal_resolver_profile_authority, + resolver_profile_convergence::{ + journal_resolver_profile_authority, journal_resolver_profile_authority_with_progress, + }, run::startup_heartbeat::{StartupAdapterHeartbeat, StartupHeartbeat}, }; @@ -17,7 +19,7 @@ use super::logging::{ // about 10x smaller than normalized replay's resident set and 400x smaller than // the #218 OOM class. Ownership/control pages preserve whole blocks, so an // unusually dense block can exceed this target. -pub(crate) const DEFAULT_STARTUP_DISCOVERY_PAGE_LOGS: usize = 100_000; +pub(crate) const DEFAULT_STARTUP_DISCOVERY_PAGE_LOGS: usize = 1_000; pub(crate) async fn sync_adapter_owned_raw_log_state( pool: &sqlx::PgPool, @@ -54,7 +56,7 @@ async fn sync_adapter_owned_raw_log_state_with_startup_context( record_startup_sync_progress(pool, &mut startup_heartbeat).await?; // Broad startup/timer passes also recover any prior discovery transaction // that committed before its caller could journal the epoch change. - journal_resolver_profile_authority(pool).await?; + journal_authority_with_startup_progress(pool, &mut startup_heartbeat).await?; let mut completed_startup_checkpoints = Vec::new(); for chain in watched_chain_plan { let startup_checkpoint = match startup_context { @@ -245,7 +247,7 @@ async fn sync_adapter_owned_raw_log_state_with_startup_context( } } - journal_resolver_profile_authority(pool).await?; + journal_authority_with_startup_progress(pool, &mut startup_heartbeat).await?; clear_completed_startup_adapter_checkpoints(pool, &completed_startup_checkpoints).await?; record_startup_sync_progress(pool, &mut startup_heartbeat).await?; Ok(()) @@ -298,7 +300,7 @@ async fn sync_discovery_adapter_owned_raw_log_state_inner( mut startup_heartbeat: Option<(&mut StartupHeartbeat, &[String])>, ) -> Result<()> { record_startup_sync_progress(pool, &mut startup_heartbeat).await?; - journal_resolver_profile_authority(pool).await?; + journal_authority_with_startup_progress(pool, &mut startup_heartbeat).await?; let mut completed_startup_checkpoints = Vec::new(); for chain in watched_chain_plan { let startup_checkpoint = @@ -358,7 +360,7 @@ async fn sync_discovery_adapter_owned_raw_log_state_inner( record_startup_sync_progress(pool, &mut startup_heartbeat).await?; completed_startup_checkpoints.push((chain.chain.clone(), startup_checkpoint)); } - journal_resolver_profile_authority(pool).await?; + journal_authority_with_startup_progress(pool, &mut startup_heartbeat).await?; clear_completed_startup_adapter_checkpoints(pool, &completed_startup_checkpoints).await?; record_startup_sync_progress(pool, &mut startup_heartbeat).await?; Ok(()) @@ -374,6 +376,22 @@ async fn record_startup_sync_progress( Ok(()) } +async fn journal_authority_with_startup_progress( + pool: &sqlx::PgPool, + startup_heartbeat: &mut Option<(&mut StartupHeartbeat, &[String])>, +) -> Result<()> { + match startup_heartbeat.as_mut() { + Some((heartbeat, chain_ids)) => { + let mut progress = StartupAdapterHeartbeat::new(heartbeat, chain_ids); + journal_resolver_profile_authority_with_progress(pool, &mut progress).await?; + } + None => { + journal_resolver_profile_authority(pool).await?; + } + } + Ok(()) +} + async fn load_startup_adapter_checkpoint_context( pool: &sqlx::PgPool, deployment_profile: &str, diff --git a/apps/indexer/src/main/runtime/intake.rs b/apps/indexer/src/main/runtime/intake.rs index 279b12ef..0195dfbd 100644 --- a/apps/indexer/src/main/runtime/intake.rs +++ b/apps/indexer/src/main/runtime/intake.rs @@ -1,9 +1,11 @@ use anyhow::{Context, Result}; -use bigname_manifests::WatchedChainPlan; +use bigname_manifests::{ManifestRuntimeProgress, WatchedChainPlan}; use bigname_storage::{ChainCheckpoint, sync_chain_checkpoints}; use crate::provider::ProviderRegistry; +const INTAKE_ADDRESS_CLONE_PROGRESS_SIZE: usize = 10_000; + #[derive(Clone, Debug, Eq, PartialEq)] pub(crate) struct WatchedChainPlanState { pub(crate) chain_count: usize, @@ -194,6 +196,132 @@ pub(crate) fn validate_provider_registry_for_intake_tasks( pub(crate) async fn sync_intake_chain_tasks( pool: &sqlx::PgPool, watched_chain_plan: &[WatchedChainPlan], +) -> Result> { + sync_intake_chain_tasks_inner(pool, watched_chain_plan, &mut None).await +} + +pub(crate) async fn sync_intake_chain_tasks_with_progress( + pool: &sqlx::PgPool, + watched_chain_plan: &[WatchedChainPlan], + progress: &mut dyn ManifestRuntimeProgress, +) -> Result> { + sync_intake_chain_tasks_inner(pool, watched_chain_plan, &mut Some(progress)).await +} + +pub(crate) async fn watched_chain_plans_equal_with_progress( + pool: &sqlx::PgPool, + left: &[WatchedChainPlan], + right: &[WatchedChainPlan], + progress: &mut dyn ManifestRuntimeProgress, +) -> Result { + if left.len() != right.len() { + return Ok(false); + } + for (left, right) in left.iter().zip(right) { + if left.chain != right.chain + || left.manifest_root_entry_count != right.manifest_root_entry_count + || left.manifest_contract_entry_count != right.manifest_contract_entry_count + || left.discovery_edge_entry_count != right.discovery_edge_entry_count + || left.addresses.len() != right.addresses.len() + { + return Ok(false); + } + for (left, right) in left + .addresses + .chunks(INTAKE_ADDRESS_CLONE_PROGRESS_SIZE) + .zip(right.addresses.chunks(INTAKE_ADDRESS_CLONE_PROGRESS_SIZE)) + { + let equal = left == right; + progress.record(pool).await?; + if !equal { + return Ok(false); + } + } + } + Ok(true) +} + +pub(crate) async fn clone_watched_chain_plan_with_progress( + pool: &sqlx::PgPool, + plan: &[WatchedChainPlan], + progress: &mut dyn ManifestRuntimeProgress, +) -> Result> { + let mut cloned = Vec::with_capacity(plan.len()); + for chain in plan { + let mut addresses = Vec::with_capacity(chain.addresses.len()); + for address_chunk in chain.addresses.chunks(INTAKE_ADDRESS_CLONE_PROGRESS_SIZE) { + addresses.extend_from_slice(address_chunk); + progress.record(pool).await?; + } + cloned.push(WatchedChainPlan { + chain: chain.chain.clone(), + addresses, + manifest_root_entry_count: chain.manifest_root_entry_count, + manifest_contract_entry_count: chain.manifest_contract_entry_count, + discovery_edge_entry_count: chain.discovery_edge_entry_count, + }); + } + Ok(cloned) +} + +pub(crate) async fn intake_chain_tasks_equal_with_progress( + pool: &sqlx::PgPool, + left: &[IntakeChainTask], + right: &[IntakeChainTask], + progress: &mut dyn ManifestRuntimeProgress, +) -> Result { + if left.len() != right.len() { + return Ok(false); + } + for (left, right) in left.iter().zip(right) { + if left.chain != right.chain + || left.manifest_root_entry_count != right.manifest_root_entry_count + || left.manifest_contract_entry_count != right.manifest_contract_entry_count + || left.discovery_edge_entry_count != right.discovery_edge_entry_count + || left.checkpoint != right.checkpoint + || left.addresses.len() != right.addresses.len() + { + return Ok(false); + } + for (left, right) in left + .addresses + .chunks(INTAKE_ADDRESS_CLONE_PROGRESS_SIZE) + .zip(right.addresses.chunks(INTAKE_ADDRESS_CLONE_PROGRESS_SIZE)) + { + let equal = left == right; + progress.record(pool).await?; + if !equal { + return Ok(false); + } + } + } + Ok(true) +} + +pub(crate) async fn clone_intake_chain_task_with_progress( + pool: &sqlx::PgPool, + task: &IntakeChainTask, + progress: &mut dyn ManifestRuntimeProgress, +) -> Result { + let mut addresses = Vec::with_capacity(task.addresses.len()); + for address_chunk in task.addresses.chunks(INTAKE_ADDRESS_CLONE_PROGRESS_SIZE) { + addresses.extend_from_slice(address_chunk); + progress.record(pool).await?; + } + Ok(IntakeChainTask { + chain: task.chain.clone(), + addresses, + manifest_root_entry_count: task.manifest_root_entry_count, + manifest_contract_entry_count: task.manifest_contract_entry_count, + discovery_edge_entry_count: task.discovery_edge_entry_count, + checkpoint: task.checkpoint.clone(), + }) +} + +async fn sync_intake_chain_tasks_inner( + pool: &sqlx::PgPool, + watched_chain_plan: &[WatchedChainPlan], + progress: &mut Option<&mut dyn ManifestRuntimeProgress>, ) -> Result> { let chain_ids = watched_chain_plan .iter() @@ -213,9 +341,16 @@ pub(crate) async fn sync_intake_chain_tasks( chain.chain ) })?; + let mut addresses = Vec::with_capacity(chain.addresses.len()); + for address_chunk in chain.addresses.chunks(INTAKE_ADDRESS_CLONE_PROGRESS_SIZE) { + addresses.extend_from_slice(address_chunk); + if let Some(progress) = progress.as_deref_mut() { + progress.record(pool).await?; + } + } tasks.push(IntakeChainTask { chain: chain.chain.clone(), - addresses: chain.addresses.clone(), + addresses, manifest_root_entry_count: chain.manifest_root_entry_count, manifest_contract_entry_count: chain.manifest_contract_entry_count, discovery_edge_entry_count: chain.discovery_edge_entry_count, diff --git a/apps/indexer/src/main/runtime/manifest.rs b/apps/indexer/src/main/runtime/manifest.rs index 731a02f6..42bf3d85 100644 --- a/apps/indexer/src/main/runtime/manifest.rs +++ b/apps/indexer/src/main/runtime/manifest.rs @@ -3,15 +3,20 @@ use std::path::Path; use anyhow::{Context, Result, bail}; use bigname_adapters::{ ManifestNormalizedEventKindSyncSummary, ManifestNormalizedEventSyncSummary, + StartupAdapterProgress, }; use bigname_manifests::{ DiscoveryAdmissionState, ManifestLoadStatus, ManifestLoadSummary, ManifestRepository, - ManifestSyncStatus, ManifestSyncSummary, WatchedChainPlan, WatchedContractSummary, - load_manifest_declared_watched_chain_plan, load_manifest_declared_watched_contract_summary, - load_watched_contract_summary_and_chain_plan, + ManifestRuntimeProgress, ManifestSyncStatus, ManifestSyncSummary, WatchedChainPlan, + WatchedContractSummary, load_manifest_declared_watched_chain_plan, + load_manifest_declared_watched_contract_summary, load_watched_contract_summary_and_chain_plan, + load_watched_contract_summary_and_chain_plan_with_progress, }; -use crate::resolver_profile_convergence::journal_resolver_profile_authority; +use crate::resolver_profile_convergence::{ + journal_resolver_profile_authority, journal_resolver_profile_authority_with_progress, +}; +use crate::run::startup_heartbeat::StartupAdapterHeartbeat; pub(crate) fn load_manifest_repository(manifests_root: &Path) -> Result { bigname_manifests::load_repository(manifests_root).with_context(|| { @@ -83,12 +88,57 @@ pub(crate) async fn build_manifest_runtime_state_with_watch_scope( manifest_repository: &ManifestRepository, watch_scope: RuntimeWatchScope, ) -> Result { + let mut progress: Option<&mut StartupAdapterHeartbeat<'_>> = None; + build_manifest_runtime_state_with_watch_scope_inner( + pool, + manifest_repository, + watch_scope, + &mut progress, + ) + .await +} + +pub(crate) async fn build_manifest_runtime_state_with_watch_scope_and_progress

( + pool: &sqlx::PgPool, + manifest_repository: &ManifestRepository, + watch_scope: RuntimeWatchScope, + progress: &mut P, +) -> Result +where + P: ManifestRuntimeProgress + StartupAdapterProgress, +{ + build_manifest_runtime_state_with_watch_scope_inner( + pool, + manifest_repository, + watch_scope, + &mut Some(progress), + ) + .await +} + +async fn build_manifest_runtime_state_with_watch_scope_inner

( + pool: &sqlx::PgPool, + manifest_repository: &ManifestRepository, + watch_scope: RuntimeWatchScope, + progress: &mut Option<&mut P>, +) -> Result +where + P: ManifestRuntimeProgress + StartupAdapterProgress, +{ let manifest_summary = manifest_repository.summary().clone(); let sync_summary = - sync_repository_or_load_stored_for_pending_rederive(pool, manifest_repository).await?; + sync_repository_or_load_stored_for_pending_rederive(pool, manifest_repository, progress) + .await?; // Repository sync is the manifest-authority mutation boundary. Journal // its resolver-profile effects before any later bootstrap work can fail. - journal_resolver_profile_authority(pool).await?; + match progress.as_deref_mut() { + Some(progress) => { + journal_resolver_profile_authority_with_progress(pool, progress).await?; + } + None => { + journal_resolver_profile_authority(pool).await?; + } + } let ( sync_summary, discovery_admission, @@ -97,14 +147,27 @@ pub(crate) async fn build_manifest_runtime_state_with_watch_scope( watched_chain_plan, ) = match watch_scope { RuntimeWatchScope::ActiveWatchedChain => { - let admission_state = bigname_manifests::load_discovery_admission_state(pool).await?; + let admission_state = match progress.as_deref_mut() { + Some(progress) => { + bigname_manifests::load_discovery_admission_state_with_progress(pool, progress) + .await? + } + None => bigname_manifests::load_discovery_admission_state(pool).await?, + }; verify_stored_manifest_state(&sync_summary, &admission_state)?; let (watched_contract_summary, watched_chain_plan) = - load_watched_contract_summary_and_chain_plan(pool).await?; + load_active_watched_plan(pool, progress).await?; + let manifest_normalized_event_summary = match progress.as_deref_mut() { + Some(progress) => { + bigname_adapters::sync_manifest_normalized_events_with_progress(pool, progress) + .await? + } + None => bigname_adapters::sync_manifest_normalized_events(pool).await?, + }; ( sync_summary, discovery_admission_snapshot(&admission_state), - bigname_adapters::sync_manifest_normalized_events(pool).await?, + manifest_normalized_event_summary, watched_contract_summary, watched_chain_plan, ) @@ -137,38 +200,91 @@ pub(crate) async fn build_manifest_runtime_state_with_watch_scope( /// (`inline`) runtime refresh may also emit manifest-derived normalized /// events. Non-inline modes build through the declared-only read/write scope, /// then widen the in-memory plan from stored discovery without adapter writes. -pub(crate) async fn build_manifest_runtime_state_for_repository_refresh( +pub(crate) async fn build_manifest_runtime_state_for_repository_refresh_with_progress

( pool: &sqlx::PgPool, manifest_repository: &ManifestRepository, runtime_watch_scope: RuntimeWatchScope, broad_runtime_refresh_enabled: bool, -) -> Result { + progress: &mut P, +) -> Result +where + P: ManifestRuntimeProgress + StartupAdapterProgress, +{ + build_manifest_runtime_state_for_repository_refresh_inner( + pool, + manifest_repository, + runtime_watch_scope, + broad_runtime_refresh_enabled, + &mut Some(progress), + ) + .await +} + +async fn build_manifest_runtime_state_for_repository_refresh_inner

( + pool: &sqlx::PgPool, + manifest_repository: &ManifestRepository, + runtime_watch_scope: RuntimeWatchScope, + broad_runtime_refresh_enabled: bool, + progress: &mut Option<&mut P>, +) -> Result +where + P: ManifestRuntimeProgress + StartupAdapterProgress, +{ let build_scope = if broad_runtime_refresh_enabled { runtime_watch_scope } else { RuntimeWatchScope::ManifestDeclaredOnly }; - let mut state = - build_manifest_runtime_state_with_watch_scope(pool, manifest_repository, build_scope) - .await?; + let mut state = build_manifest_runtime_state_with_watch_scope_inner( + pool, + manifest_repository, + build_scope, + progress, + ) + .await?; if build_scope != runtime_watch_scope { let (watched_contract_summary, watched_chain_plan) = - load_watched_contract_summary_and_chain_plan(pool).await?; + load_active_watched_plan(pool, progress).await?; state.watched_contract_summary = watched_contract_summary; state.watched_chain_plan = watched_chain_plan; } Ok(state) } -async fn sync_repository_or_load_stored_for_pending_rederive( +async fn load_active_watched_plan

( + pool: &sqlx::PgPool, + progress: &mut Option<&mut P>, +) -> Result<(WatchedContractSummary, Vec)> +where + P: ManifestRuntimeProgress + StartupAdapterProgress, +{ + match progress.as_deref_mut() { + Some(progress) => { + load_watched_contract_summary_and_chain_plan_with_progress(pool, progress).await + } + None => load_watched_contract_summary_and_chain_plan(pool).await, + } +} + +async fn sync_repository_or_load_stored_for_pending_rederive

( pool: &sqlx::PgPool, manifest_repository: &ManifestRepository, -) -> Result { + progress: &mut Option<&mut P>, +) -> Result +where + P: ManifestRuntimeProgress + StartupAdapterProgress, +{ if bigname_storage::base_normalized_rederive_manifest_sync_pending_replay(pool).await? { return load_stored_manifest_sync_summary_for_pending_rederive(pool).await; } - bigname_manifests::sync_repository(pool, manifest_repository).await + match progress.as_deref_mut() { + Some(progress) => { + bigname_manifests::sync_repository_with_progress(pool, manifest_repository, progress) + .await + } + None => bigname_manifests::sync_repository(pool, manifest_repository).await, + } } async fn load_stored_manifest_sync_summary_for_pending_rederive( diff --git a/apps/indexer/src/main/runtime/poll_loop.rs b/apps/indexer/src/main/runtime/poll_loop.rs index 510ae7e0..221555b7 100644 --- a/apps/indexer/src/main/runtime/poll_loop.rs +++ b/apps/indexer/src/main/runtime/poll_loop.rs @@ -5,26 +5,33 @@ use tracing::{info, warn}; use crate::provider::{ProviderBlock, ProviderRegistry}; use crate::reconciliation::{ - ChainCoverageFrontiers, HeaderAuditMode, poll_provider_heads_with_adapter_sync, + ChainCoverageFrontiers, HeaderAuditMode, poll_provider_heads_with_adapter_sync_and_progress, }; use crate::replay::deployment_profile_from_manifest_root; -use crate::resolver_profile_convergence::drain_resolver_profile_input_changes; -use crate::run::startup_heartbeat::StartupHeartbeat; +use crate::resolver_profile_convergence::drain_resolver_profile_input_changes_with_progress; +use crate::run::startup_heartbeat::{ + RequiredSubtaskActivity, StartupAdapterHeartbeat, StartupHeartbeat, +}; use super::adapter_sync::sync_adapter_owned_raw_log_state_with_heartbeat; use super::intake::{ - IntakeChainTask, intake_runtime_state, sync_intake_chain_tasks, + IntakeChainTask, intake_runtime_state, sync_intake_chain_tasks_with_progress, validate_provider_registry_for_intake_tasks, watched_chain_plan_state, + watched_chain_plans_equal_with_progress, }; use super::logging::{ log_intake_chain_tasks, log_manifest_normalized_event_summary, log_manifest_runtime_state, log_manifest_summary, log_provider_registry, log_watched_chain_plan, }; use super::manifest::{ - ManifestRuntimeState, RuntimeWatchScope, build_manifest_runtime_state_for_repository_refresh, - ensure_manifest_root_ready, load_manifest_repository, + ManifestRuntimeState, RuntimeWatchScope, + build_manifest_runtime_state_for_repository_refresh_with_progress, ensure_manifest_root_ready, + load_manifest_repository, +}; +use super::refresh::{ + refresh_intake_chain_tasks_with_progress, + refresh_manifest_normalized_events_from_storage_with_progress, }; -use super::refresh::{refresh_intake_chain_tasks, refresh_manifest_normalized_events_from_storage}; #[path = "poll_loop/discovery_refresh.rs"] mod discovery_refresh; @@ -65,6 +72,7 @@ pub(crate) async fn run_poll_loop( event_silent_reverse_resolver_addresses: Vec, latched_bootstrap_finalized_heads: BTreeMap, coverage_frontiers: &ChainCoverageFrontiers, + required_subtask_activity: &RequiredSubtaskActivity, ) -> Result<()> { let deployment_profile = deployment_profile_from_manifest_root(&manifests_root); let mut live_poll_adapter_sync_restored_after_replay = false; @@ -82,6 +90,13 @@ pub(crate) async fn run_poll_loop( return Ok(()); } _ = interval.tick() => { + let Some(_required_subtask_exclusion) = required_subtask_activity + .exclude_required_subtask_or_shutdown(tokio::signal::ctrl_c()) + .await + else { + info!(service = "indexer", "shutdown signal received"); + return Ok(()); + }; let heartbeat_chains = intake_chain_tasks .iter() .map(|task| task.chain.clone()) @@ -116,20 +131,38 @@ pub(crate) async fn run_poll_loop( "failed to reload repository manifests; keeping last successful runtime state" ); } else { - match build_manifest_runtime_state_for_repository_refresh( - pool, - &manifest_repository, - runtime_watch_scope, - adapter_sync_on_manifest_refresh, - ) - .await - { + let refresh_result = { + let mut progress = + StartupAdapterHeartbeat::new(heartbeat, &heartbeat_chains); + build_manifest_runtime_state_for_repository_refresh_with_progress( + pool, + &manifest_repository, + runtime_watch_scope, + adapter_sync_on_manifest_refresh, + &mut progress, + ) + .await + }; + match refresh_result { Ok(next_manifest_runtime_state) => { - let manifest_state_changed = - next_manifest_runtime_state != manifest_runtime_state; - let watched_plan_changed = next_manifest_runtime_state - .watched_chain_plan - != manifest_runtime_state.watched_chain_plan; + let watched_plan_changed = { + let mut progress = StartupAdapterHeartbeat::new( + heartbeat, + &heartbeat_chains, + ); + !watched_chain_plans_equal_with_progress( + pool, + &next_manifest_runtime_state.watched_chain_plan, + &manifest_runtime_state.watched_chain_plan, + &mut progress, + ) + .await? + }; + let manifest_state_changed = watched_plan_changed + || manifest_runtime_metadata_changed( + &next_manifest_runtime_state, + &manifest_runtime_state, + ); if adapter_sync_on_manifest_refresh && (manifest_state_changed || watched_plan_changed) @@ -168,11 +201,21 @@ pub(crate) async fn run_poll_loop( adapter_sync_on_manifest_refresh, live_poll_adapter_sync_restored_after_replay, ) - && !discovery_refresh::resolver_profile_drain_succeeded( - drain_resolver_profile_input_changes(pool).await, - "timer", - "repository_manifest_reload", - ) + && !{ + let mut progress = StartupAdapterHeartbeat::new( + heartbeat, + &heartbeat_chains, + ); + discovery_refresh::resolver_profile_drain_succeeded( + drain_resolver_profile_input_changes_with_progress( + pool, + &mut progress, + ) + .await, + "timer", + "repository_manifest_reload", + ) + } { continue; } @@ -206,12 +249,19 @@ pub(crate) async fn run_poll_loop( } if watched_plan_changed { - match sync_intake_chain_tasks( - pool, - &next_manifest_runtime_state.watched_chain_plan, - ) - .await - { + let next_tasks_result = { + let mut progress = StartupAdapterHeartbeat::new( + heartbeat, + &heartbeat_chains, + ); + sync_intake_chain_tasks_with_progress( + pool, + &next_manifest_runtime_state.watched_chain_plan, + &mut progress, + ) + .await + }; + match next_tasks_result { Ok(next_tasks) => { validate_provider_registry_for_intake_tasks( &next_tasks, @@ -333,13 +383,18 @@ pub(crate) async fn run_poll_loop( } } - match refresh_intake_chain_tasks( - pool, - &intake_chain_tasks, - &manifest_runtime_state.watched_chain_plan, - ) - .await - { + let intake_refresh_result = { + let mut progress = + StartupAdapterHeartbeat::new(heartbeat, &heartbeat_chains); + refresh_intake_chain_tasks_with_progress( + pool, + &intake_chain_tasks, + &manifest_runtime_state.watched_chain_plan, + &mut progress, + ) + .await + }; + match intake_refresh_result { Ok(Some(next_tasks)) => { let previous_state = intake_runtime_state(&intake_chain_tasks); let next_state = intake_runtime_state(&next_tasks); @@ -417,6 +472,9 @@ pub(crate) async fn run_poll_loop( &event_silent_reverse_resolver_addresses, coverage_frontiers, &latched_bootstrap_finalized_heads, + adapter_sync_page_logs, + heartbeat, + &heartbeat_chains, ) .await?; if !effective_adapter_sync_on_live_poll { @@ -427,27 +485,37 @@ pub(crate) async fn run_poll_loop( let loaded_plan_admission_epochs = watched_plan_admission_epochs .as_ref() .context("live watch plan is missing its loaded admission-epoch snapshot")?; - poll_provider_heads_with_adapter_sync( - pool, - &mut intake_chain_tasks, - provider_registry, - &deployment_profile, - loaded_plan_admission_epochs, - effective_adapter_sync_on_live_poll, - header_audit_mode, - &event_silent_reverse_resolver_addresses, - coverage_frontiers, - &latched_bootstrap_finalized_heads, - ) - .await?; - - if manifest_observation_refresh_enabled { - match refresh_manifest_normalized_events_from_storage( + { + let mut progress = + StartupAdapterHeartbeat::new(heartbeat, &heartbeat_chains); + poll_provider_heads_with_adapter_sync_and_progress( pool, - &manifest_runtime_state, + &mut intake_chain_tasks, + provider_registry, + &deployment_profile, + loaded_plan_admission_epochs, + effective_adapter_sync_on_live_poll, + header_audit_mode, + &event_silent_reverse_resolver_addresses, + coverage_frontiers, + &latched_bootstrap_finalized_heads, + &mut progress, ) - .await - { + .await?; + } + + if manifest_observation_refresh_enabled { + let refresh_result = { + let mut progress = + StartupAdapterHeartbeat::new(heartbeat, &heartbeat_chains); + refresh_manifest_normalized_events_from_storage_with_progress( + pool, + &manifest_runtime_state, + &mut progress, + ) + .await + }; + match refresh_result { Ok(Some(next_manifest_runtime_state)) => { info!( service = "indexer", @@ -516,3 +584,15 @@ pub(crate) async fn run_poll_loop( } } } + +fn manifest_runtime_metadata_changed( + left: &ManifestRuntimeState, + right: &ManifestRuntimeState, +) -> bool { + left.manifest_repository != right.manifest_repository + || left.manifest_summary != right.manifest_summary + || left.sync_summary != right.sync_summary + || left.discovery_admission != right.discovery_admission + || left.manifest_normalized_event_summary != right.manifest_normalized_event_summary + || left.watched_contract_summary != right.watched_contract_summary +} diff --git a/apps/indexer/src/main/runtime/poll_loop/discovery_refresh.rs b/apps/indexer/src/main/runtime/poll_loop/discovery_refresh.rs index 5f373673..50b9fbdd 100644 --- a/apps/indexer/src/main/runtime/poll_loop/discovery_refresh.rs +++ b/apps/indexer/src/main/runtime/poll_loop/discovery_refresh.rs @@ -6,20 +6,24 @@ use tracing::{info, warn}; use crate::provider::ProviderRegistry; use crate::resolver_profile_convergence::{ ResolverProfileConvergenceSummary, drain_resolver_profile_input_changes, + drain_resolver_profile_input_changes_with_progress, }; -use crate::run::startup_heartbeat::StartupHeartbeat; +use crate::run::startup_heartbeat::{StartupAdapterHeartbeat, StartupHeartbeat}; use super::super::adapter_sync::sync_adapter_owned_raw_log_state_with_heartbeat; use super::super::intake::{ IntakeChainTask, intake_runtime_state, validate_provider_registry_for_intake_tasks, - watched_chain_plan_state, + watched_chain_plan_state, watched_chain_plans_equal_with_progress, }; use super::super::logging::{ log_intake_chain_tasks, log_provider_registry, log_watched_chain_plan, log_watched_contract_summary, }; use super::super::manifest::ManifestRuntimeState; -use super::super::refresh::refresh_runtime_state_from_stored_discovery_when_epochs_move; +use super::super::refresh::{ + refresh_runtime_state_from_stored_discovery_when_epochs_move, + refresh_runtime_state_from_stored_discovery_when_epochs_move_with_progress, +}; /// Timer-driven stored-discovery refresh of the runtime watch state. A /// successful refresh replaces the manifest runtime state and intake tasks in @@ -40,6 +44,7 @@ use super::super::refresh::refresh_runtime_state_from_stored_discovery_when_epoc /// `resolver_profile_convergence_enabled` is false for raw-only operation, so /// reloading a stored plan cannot drain adapter-owned resolver-profile work in /// a mode that explicitly defers those writes. +#[cfg(test)] pub(crate) async fn refresh_discovery_watch_state( pool: &sqlx::PgPool, provider_registry: &ProviderRegistry, @@ -105,18 +110,19 @@ async fn refresh_discovery_watch_state_inner( sync_adapter_state_before_refresh: bool, resolver_profile_convergence_enabled: bool, last_admission_epochs: &mut Option>, - adapter_sync_heartbeat: Option>, + mut adapter_sync_heartbeat: Option>, ) -> Result { // The whole-corpus re-derivation must run before the sentinel read: it is // what materializes new edges (and bumps epochs) on the broad-refresh path. let adapter_sync_result: Result<()> = if sync_adapter_state_before_refresh { let (deployment_profile, page_logs, heartbeat, chain_ids) = adapter_sync_heartbeat + .as_mut() .context("discovery adapter refresh requires a live loop heartbeat")?; sync_adapter_owned_raw_log_state_with_heartbeat( pool, deployment_profile, &manifest_runtime_state.watched_chain_plan, - page_logs, + *page_logs, heartbeat, chain_ids, ) @@ -125,25 +131,39 @@ async fn refresh_discovery_watch_state_inner( Ok(()) }; let refreshed_state = match adapter_sync_result { - Ok(()) => { - refresh_runtime_state_from_stored_discovery_when_epochs_move( - pool, - manifest_runtime_state, - last_admission_epochs.as_ref(), - ) - .await - } + Ok(()) => match adapter_sync_heartbeat.as_mut() { + Some((_, _, heartbeat, chain_ids)) => { + let mut progress = StartupAdapterHeartbeat::new(heartbeat, chain_ids); + refresh_runtime_state_from_stored_discovery_when_epochs_move_with_progress( + pool, + manifest_runtime_state, + last_admission_epochs.as_ref(), + &mut progress, + ) + .await + } + None => { + refresh_runtime_state_from_stored_discovery_when_epochs_move( + pool, + manifest_runtime_state, + last_admission_epochs.as_ref(), + ) + .await + } + }, Err(error) => Err(error), }; - if refreshed_state.is_ok() - && resolver_profile_convergence_enabled - && !resolver_profile_drain_succeeded( - drain_resolver_profile_input_changes(pool).await, - "timer", - "stored_discovery_state", - ) - { - return Ok(false); + if refreshed_state.is_ok() && resolver_profile_convergence_enabled { + let drain_result = match adapter_sync_heartbeat.as_mut() { + Some((_, _, heartbeat, chain_ids)) => { + let mut progress = StartupAdapterHeartbeat::new(heartbeat, chain_ids); + drain_resolver_profile_input_changes_with_progress(pool, &mut progress).await + } + None => drain_resolver_profile_input_changes(pool).await, + }; + if !resolver_profile_drain_succeeded(drain_result, "timer", "stored_discovery_state") { + return Ok(false); + } } match refreshed_state { Ok(Some(refresh)) => { @@ -158,8 +178,22 @@ async fn refresh_discovery_watch_state_inner( watched_chain_plan_state(&manifest_runtime_state.watched_chain_plan); let next_watch_state = watched_chain_plan_state(&next_manifest_runtime_state.watched_chain_plan); - let watched_plan_changed = manifest_runtime_state.watched_chain_plan - != next_manifest_runtime_state.watched_chain_plan; + let watched_plan_changed = match adapter_sync_heartbeat.as_mut() { + Some((_, _, heartbeat, chain_ids)) => { + let mut progress = StartupAdapterHeartbeat::new(heartbeat, chain_ids); + !watched_chain_plans_equal_with_progress( + pool, + &manifest_runtime_state.watched_chain_plan, + &next_manifest_runtime_state.watched_chain_plan, + &mut progress, + ) + .await? + } + None => { + manifest_runtime_state.watched_chain_plan + != next_manifest_runtime_state.watched_chain_plan + } + }; let previous_intake_state = intake_runtime_state(intake_chain_tasks); let next_intake_state = intake_runtime_state(&next_tasks); diff --git a/apps/indexer/src/main/runtime/poll_loop/replay_handoff.rs b/apps/indexer/src/main/runtime/poll_loop/replay_handoff.rs index c66c904f..f1e72c07 100644 --- a/apps/indexer/src/main/runtime/poll_loop/replay_handoff.rs +++ b/apps/indexer/src/main/runtime/poll_loop/replay_handoff.rs @@ -8,13 +8,15 @@ use crate::normalized_replay_catchup::normalized_replay_cursors_complete; use crate::provider::{ProviderBlock, ProviderRegistry}; use crate::reconciliation::{ BacklogHandoffStatus, ChainCoverageFrontiers, HeaderAuditMode, - poll_provider_heads_with_adapter_sync, sync_live_adapter_backlog_after_normalized_replay, + poll_provider_heads_with_adapter_sync_and_progress, + sync_live_adapter_backlog_after_normalized_replay_with_progress, validate_chain_handoff_while_guarded, }; +use crate::run::startup_heartbeat::{StartupAdapterHeartbeat, StartupHeartbeat}; -use super::super::intake::IntakeChainTask; +use super::super::intake::{IntakeChainTask, clone_intake_chain_task_with_progress}; use super::super::manifest::ManifestRuntimeState; -use super::discovery_refresh::refresh_discovery_watch_state; +use super::discovery_refresh::refresh_discovery_watch_state_with_heartbeat; fn resolver_profile_convergence_before_handoff() -> bool { false @@ -122,12 +124,15 @@ pub(super) async fn poll_replay_ready_chains_raw_only( event_silent_reverse_resolver_addresses: &[String], coverage_frontiers: &ChainCoverageFrontiers, latched_bootstrap_finalized_heads: &BTreeMap, + adapter_sync_page_logs: usize, + heartbeat: &mut StartupHeartbeat, + heartbeat_chain_ids: &[String], ) -> Result<()> { if raw_poll_chains.is_empty() { return Ok(()); } - if !refresh_discovery_watch_state( + if !refresh_discovery_watch_state_with_heartbeat( pool, provider_registry, manifest_runtime_state, @@ -135,33 +140,47 @@ pub(super) async fn poll_replay_ready_chains_raw_only( false, resolver_profile_convergence_before_handoff(), watched_plan_admission_epochs, + deployment_profile, + adapter_sync_page_logs, + heartbeat, + heartbeat_chain_ids, ) .await? { return Ok(()); } - let mut scoped_tasks = intake_chain_tasks - .iter() - .filter(|task| raw_poll_chains.contains(&task.chain)) - .cloned() - .collect::>(); + let mut scoped_tasks = Vec::with_capacity(raw_poll_chains.len()); + { + let mut progress = StartupAdapterHeartbeat::new(heartbeat, heartbeat_chain_ids); + for task in intake_chain_tasks + .iter() + .filter(|task| raw_poll_chains.contains(&task.chain)) + { + scoped_tasks + .push(clone_intake_chain_task_with_progress(pool, task, &mut progress).await?); + } + } let loaded_plan_admission_epochs = watched_plan_admission_epochs .as_ref() .context("replay-ready watch plan is missing its loaded admission-epoch snapshot")?; - poll_provider_heads_with_adapter_sync( - pool, - &mut scoped_tasks, - provider_registry, - deployment_profile, - loaded_plan_admission_epochs, - false, - header_audit_mode, - event_silent_reverse_resolver_addresses, - coverage_frontiers, - latched_bootstrap_finalized_heads, - ) - .await?; + { + let mut progress = StartupAdapterHeartbeat::new(heartbeat, heartbeat_chain_ids); + poll_provider_heads_with_adapter_sync_and_progress( + pool, + &mut scoped_tasks, + provider_registry, + deployment_profile, + loaded_plan_admission_epochs, + false, + header_audit_mode, + event_silent_reverse_resolver_addresses, + coverage_frontiers, + latched_bootstrap_finalized_heads, + &mut progress, + ) + .await?; + } let mut updated_tasks = scoped_tasks .into_iter() @@ -190,6 +209,9 @@ pub(super) async fn renew_live_poll_adapter_sync_permit( event_silent_reverse_resolver_addresses: &[String], coverage_frontiers: &ChainCoverageFrontiers, latched_bootstrap_finalized_heads: &BTreeMap, + adapter_sync_page_logs: usize, + heartbeat: &mut StartupHeartbeat, + heartbeat_chain_ids: &[String], ) -> Result { let handoff_was_previously_latched = *live_adapter_sync_latched; *live_adapter_sync_latched = false; @@ -208,18 +230,25 @@ pub(super) async fn renew_live_poll_adapter_sync_permit( event_silent_reverse_resolver_addresses, coverage_frontiers, latched_bootstrap_finalized_heads, + adapter_sync_page_logs, + heartbeat, + heartbeat_chain_ids, ) .await?; return Ok(false); } - let backlog_summary = match sync_live_adapter_backlog_after_normalized_replay( - pool, - deployment_profile, - provider_configured_chains, - ) - .await - { + let backlog_result = { + let mut progress = StartupAdapterHeartbeat::new(heartbeat, heartbeat_chain_ids); + sync_live_adapter_backlog_after_normalized_replay_with_progress( + pool, + deployment_profile, + provider_configured_chains, + &mut progress, + ) + .await + }; + let backlog_summary = match backlog_result { Ok(summary) if summary.awaiting_replay_chain_count == 0 => summary, Ok(summary) => { warn!( @@ -249,7 +278,7 @@ pub(super) async fn renew_live_poll_adapter_sync_permit( *forced_handoff_plan_reload_complete, watched_plan_admission_epochs, ); - if !refresh_discovery_watch_state( + if !refresh_discovery_watch_state_with_heartbeat( pool, provider_registry, manifest_runtime_state, @@ -257,6 +286,10 @@ pub(super) async fn renew_live_poll_adapter_sync_permit( false, resolver_profile_convergence_before_handoff(), watched_plan_admission_epochs, + deployment_profile, + adapter_sync_page_logs, + heartbeat, + heartbeat_chain_ids, ) .await? { diff --git a/apps/indexer/src/main/runtime/refresh.rs b/apps/indexer/src/main/runtime/refresh.rs index b06ac51f..dede8750 100644 --- a/apps/indexer/src/main/runtime/refresh.rs +++ b/apps/indexer/src/main/runtime/refresh.rs @@ -2,12 +2,17 @@ use std::collections::BTreeMap; use anyhow::Result; use bigname_manifests::{ - WatchedChainPlan, load_discovery_admission_epochs, load_watched_chain_plan, - load_watched_contract_summary_and_chain_plan, + ManifestRuntimeProgress, WatchedChainPlan, load_discovery_admission_epochs, + load_watched_chain_plan, load_watched_contract_summary_and_chain_plan, + load_watched_contract_summary_and_chain_plan_with_progress, }; use super::adapter_sync::sync_adapter_owned_raw_log_state; -use super::intake::{IntakeChainTask, sync_intake_chain_tasks}; +use super::intake::{ + IntakeChainTask, clone_watched_chain_plan_with_progress, + intake_chain_tasks_equal_with_progress, sync_intake_chain_tasks, + sync_intake_chain_tasks_with_progress, watched_chain_plans_equal_with_progress, +}; use super::manifest::ManifestRuntimeState; pub(crate) struct AdmissionEpochGatedRefresh { @@ -15,6 +20,7 @@ pub(crate) struct AdmissionEpochGatedRefresh { pub(crate) refreshed_state: Option<(ManifestRuntimeState, Vec)>, } +#[cfg(test)] pub(crate) async fn refresh_manifest_normalized_events_from_storage( pool: &sqlx::PgPool, manifest_runtime_state: &ManifestRuntimeState, @@ -29,6 +35,30 @@ pub(crate) async fn refresh_manifest_normalized_events_from_storage( Ok(Some(next_manifest_runtime_state)) } +pub(crate) async fn refresh_manifest_normalized_events_from_storage_with_progress( + pool: &sqlx::PgPool, + manifest_runtime_state: &ManifestRuntimeState, + progress: &mut dyn ManifestRuntimeProgress, +) -> Result> { + let next_summary = + bigname_adapters::sync_manifest_normalized_events_with_progress(pool, progress).await?; + if next_summary.total_inserted_count == 0 { + return Ok(None); + } + + let watched_chain_plan = clone_watched_chain_plan_with_progress( + pool, + &manifest_runtime_state.watched_chain_plan, + progress, + ) + .await?; + Ok(Some(ManifestRuntimeState { + manifest_normalized_event_summary: next_summary, + watched_chain_plan, + ..manifest_runtime_state_without_watch_plan(manifest_runtime_state) + })) +} + #[allow(dead_code)] pub(crate) async fn refresh_watched_chain_plan( pool: &sqlx::PgPool, @@ -42,6 +72,7 @@ pub(crate) async fn refresh_watched_chain_plan( } } +#[cfg(test)] pub(crate) async fn refresh_intake_chain_tasks( pool: &sqlx::PgPool, current_tasks: &[IntakeChainTask], @@ -55,6 +86,21 @@ pub(crate) async fn refresh_intake_chain_tasks( } } +pub(crate) async fn refresh_intake_chain_tasks_with_progress( + pool: &sqlx::PgPool, + current_tasks: &[IntakeChainTask], + watched_chain_plan: &[WatchedChainPlan], + progress: &mut dyn ManifestRuntimeProgress, +) -> Result>> { + let next_tasks = + sync_intake_chain_tasks_with_progress(pool, watched_chain_plan, progress).await?; + if intake_chain_tasks_equal_with_progress(pool, &next_tasks, current_tasks, progress).await? { + Ok(None) + } else { + Ok(Some(next_tasks)) + } +} + /// Re-derives discovery edges from the whole stored raw-log corpus before the plan reload. Live /// poll already writes discovery edges per block, so the tailer only needs /// [`refresh_runtime_state_from_stored_discovery`]; the full re-derivation stays opt-in for broad @@ -85,9 +131,41 @@ pub(crate) async fn refresh_runtime_state_from_stored_discovery( return Ok(None); } let next_intake_chain_tasks = sync_intake_chain_tasks(pool, &next_watched_chain_plan).await?; - let mut next_manifest_runtime_state = manifest_runtime_state.clone(); - next_manifest_runtime_state.watched_contract_summary = next_watched_contract_summary; - next_manifest_runtime_state.watched_chain_plan = next_watched_chain_plan; + let next_manifest_runtime_state = ManifestRuntimeState { + watched_contract_summary: next_watched_contract_summary, + watched_chain_plan: next_watched_chain_plan, + ..manifest_runtime_state_without_watch_plan(manifest_runtime_state) + }; + + Ok(Some((next_manifest_runtime_state, next_intake_chain_tasks))) +} + +pub(crate) async fn refresh_runtime_state_from_stored_discovery_with_progress( + pool: &sqlx::PgPool, + manifest_runtime_state: &ManifestRuntimeState, + progress: &mut dyn ManifestRuntimeProgress, +) -> Result)>> { + let (next_watched_contract_summary, next_watched_chain_plan) = + load_watched_contract_summary_and_chain_plan_with_progress(pool, progress).await?; + let plan_unchanged = watched_chain_plans_equal_with_progress( + pool, + &next_watched_chain_plan, + &manifest_runtime_state.watched_chain_plan, + progress, + ) + .await?; + if next_watched_contract_summary == manifest_runtime_state.watched_contract_summary + && plan_unchanged + { + return Ok(None); + } + let next_intake_chain_tasks = + sync_intake_chain_tasks_with_progress(pool, &next_watched_chain_plan, progress).await?; + let next_manifest_runtime_state = ManifestRuntimeState { + watched_contract_summary: next_watched_contract_summary, + watched_chain_plan: next_watched_chain_plan, + ..manifest_runtime_state_without_watch_plan(manifest_runtime_state) + }; Ok(Some((next_manifest_runtime_state, next_intake_chain_tasks))) } @@ -122,6 +200,28 @@ pub(crate) async fn refresh_runtime_state_from_stored_discovery_when_epochs_move })) } +pub(crate) async fn refresh_runtime_state_from_stored_discovery_when_epochs_move_with_progress( + pool: &sqlx::PgPool, + manifest_runtime_state: &ManifestRuntimeState, + last_admission_epochs: Option<&BTreeMap>, + progress: &mut dyn ManifestRuntimeProgress, +) -> Result> { + let current_admission_epochs = load_discovery_admission_epochs(pool).await?; + if last_admission_epochs == Some(¤t_admission_epochs) { + return Ok(None); + } + let refreshed_state = refresh_runtime_state_from_stored_discovery_with_progress( + pool, + manifest_runtime_state, + progress, + ) + .await?; + Ok(Some(AdmissionEpochGatedRefresh { + admission_epochs: current_admission_epochs, + refreshed_state, + })) +} + /// Widens a bootstrap-scoped runtime state to the live watch scope by reloading the stored plan. /// This deliberately avoids `build_manifest_runtime_state_with_watch_scope`: the widen needs no /// manifest re-sync, and re-running one here would race the normalized-replay catch-up task, which @@ -146,10 +246,40 @@ pub(crate) async fn widen_runtime_state_to_live_watch_scope_with_admission_epoch manifest_runtime_state: &ManifestRuntimeState, ) -> Result<(ManifestRuntimeState, BTreeMap)> { let admission_epochs = load_discovery_admission_epochs(pool).await?; - let mut live_manifest_runtime_state = manifest_runtime_state.clone(); let (watched_contract_summary, watched_chain_plan) = load_watched_contract_summary_and_chain_plan(pool).await?; - live_manifest_runtime_state.watched_contract_summary = watched_contract_summary; - live_manifest_runtime_state.watched_chain_plan = watched_chain_plan; + let live_manifest_runtime_state = ManifestRuntimeState { + watched_contract_summary, + watched_chain_plan, + ..manifest_runtime_state_without_watch_plan(manifest_runtime_state) + }; Ok((live_manifest_runtime_state, admission_epochs)) } + +pub(crate) async fn widen_runtime_state_to_live_watch_scope_with_admission_epochs_and_progress( + pool: &sqlx::PgPool, + manifest_runtime_state: &ManifestRuntimeState, + progress: &mut dyn ManifestRuntimeProgress, +) -> Result<(ManifestRuntimeState, BTreeMap)> { + let admission_epochs = load_discovery_admission_epochs(pool).await?; + let (watched_contract_summary, watched_chain_plan) = + load_watched_contract_summary_and_chain_plan_with_progress(pool, progress).await?; + let live_manifest_runtime_state = ManifestRuntimeState { + watched_contract_summary, + watched_chain_plan, + ..manifest_runtime_state_without_watch_plan(manifest_runtime_state) + }; + Ok((live_manifest_runtime_state, admission_epochs)) +} + +fn manifest_runtime_state_without_watch_plan(state: &ManifestRuntimeState) -> ManifestRuntimeState { + ManifestRuntimeState { + manifest_repository: state.manifest_repository.clone(), + manifest_summary: state.manifest_summary.clone(), + sync_summary: state.sync_summary.clone(), + discovery_admission: state.discovery_admission.clone(), + manifest_normalized_event_summary: state.manifest_normalized_event_summary.clone(), + watched_contract_summary: state.watched_contract_summary.clone(), + watched_chain_plan: Vec::new(), + } +} diff --git a/apps/indexer/src/main/startup_heartbeat.rs b/apps/indexer/src/main/startup_heartbeat.rs index e77bd0c1..4f2035f0 100644 --- a/apps/indexer/src/main/startup_heartbeat.rs +++ b/apps/indexer/src/main/startup_heartbeat.rs @@ -1,6 +1,48 @@ +use std::{future::Future, sync::Arc}; + use anyhow::Result; use sqlx::PgPool; -use tokio::time::{Duration, Instant}; +use tokio::{ + sync::{Mutex, OwnedMutexGuard}, + time::{Duration, Instant}, +}; + +const MAX_PROGRESS_HEARTBEAT_INTERVAL: Duration = Duration::from_secs(5); + +#[derive(Clone, Default)] +pub(crate) struct RequiredSubtaskActivity { + exclusion: Arc>, +} + +pub(crate) struct RequiredSubtaskActivityGuard { + _exclusion: OwnedMutexGuard<()>, +} + +impl RequiredSubtaskActivity { + pub(crate) async fn begin(&self) -> RequiredSubtaskActivityGuard { + RequiredSubtaskActivityGuard { + _exclusion: Arc::clone(&self.exclusion).lock_owned().await, + } + } + + pub(crate) async fn exclude_required_subtask(&self) -> OwnedMutexGuard<()> { + Arc::clone(&self.exclusion).lock_owned().await + } + + pub(crate) async fn exclude_required_subtask_or_shutdown( + &self, + shutdown: F, + ) -> Option> + where + F: Future, + { + tokio::select! { + biased; + _ = shutdown => None, + exclusion = self.exclude_required_subtask() => Some(exclusion), + } + } +} pub(crate) struct StartupHeartbeat { instance_id: String, @@ -15,6 +57,21 @@ pub(crate) struct StartupAdapterHeartbeat<'a> { chain_ids: &'a [String], } +#[derive(Clone)] +pub(crate) struct NormalizedReplayHeartbeat { + heartbeat: Arc>, + chain_ids: Arc>, +} + +impl NormalizedReplayHeartbeat { + pub(crate) fn new(instance_id: String, interval: Duration, chain_ids: Vec) -> Self { + Self { + heartbeat: Arc::new(Mutex::new(StartupHeartbeat::new(instance_id, interval))), + chain_ids: Arc::new(chain_ids), + } + } +} + impl<'a> StartupAdapterHeartbeat<'a> { pub(crate) fn new(heartbeat: &'a mut StartupHeartbeat, chain_ids: &'a [String]) -> Self { Self { @@ -39,11 +96,41 @@ impl bigname_adapters::StartupAdapterProgress for StartupAdapterHeartbeat<'_> { } } +impl bigname_manifests::ManifestRuntimeProgress for StartupAdapterHeartbeat<'_> { + fn record<'a>( + &'a mut self, + pool: &'a PgPool, + ) -> bigname_manifests::ManifestRuntimeProgressFuture<'a> { + Box::pin(async move { + #[cfg(test)] + { + self.heartbeat.adapter_progress_count += 1; + } + self.heartbeat.record_if_due(pool, self.chain_ids).await + }) + } +} + +impl bigname_adapters::StartupAdapterProgress for NormalizedReplayHeartbeat { + fn record<'a>( + &'a mut self, + pool: &'a PgPool, + ) -> bigname_adapters::StartupAdapterProgressFuture<'a> { + Box::pin(async move { + self.heartbeat + .lock() + .await + .record_if_due(pool, &self.chain_ids) + .await + }) + } +} + impl StartupHeartbeat { pub(crate) fn new(instance_id: String, interval: Duration) -> Self { Self { instance_id, - interval, + interval: interval.min(MAX_PROGRESS_HEARTBEAT_INTERVAL), last_recorded_at: Instant::now(), #[cfg(test)] adapter_progress_count: 0, @@ -83,6 +170,13 @@ impl StartupHeartbeat { mod tests { use super::*; + #[test] + fn progress_heartbeat_throttle_never_inherits_a_stale_poll_interval() { + let heartbeat = StartupHeartbeat::new("long-poll-test".to_owned(), Duration::from_secs(60)); + + assert_eq!(heartbeat.interval, Duration::from_secs(5)); + } + #[tokio::test] async fn bootstrap_progress_refreshes_the_registered_indexer_loop() -> Result<()> { let database = bigname_test_support::TestDatabase::create_migrated( diff --git a/apps/indexer/src/main/tests/adapter_sync.rs b/apps/indexer/src/main/tests/adapter_sync.rs index e8ef1e35..98b421cd 100644 --- a/apps/indexer/src/main/tests/adapter_sync.rs +++ b/apps/indexer/src/main/tests/adapter_sync.rs @@ -319,13 +319,52 @@ async fn live_adapter_sync_continues_after_block_derived_events() -> Result<()> ) .await?; - let summary = sync_live_adapter_state_from_persisted_raw_payloads( + let heartbeat_instance_id = "live-adapter-in-flight-progress-test"; + install_stale_indexer_heartbeat(database.pool(), heartbeat_instance_id).await?; + let (mut progress, progress_handle) = BlockingHeartbeatProgress::new( + heartbeat_instance_id, + vec!["ethereum-mainnet".to_owned()], + 2, + ); + let mut progress_ref = Some(&mut progress as &mut dyn bigname_adapters::StartupAdapterProgress); + let mut operation = Box::pin(sync_live_adapter_state_from_persisted_raw_payloads_with_progress( database.pool(), "test", "ethereum-mainnet", std::slice::from_ref(&stored_block.block_hash), + &mut progress_ref, + )); + tokio::time::timeout(tokio::time::Duration::from_secs(10), async { + tokio::select! { + () = progress_handle.wait_until_blocked() => Ok(()), + result = operation.as_mut() => Err(anyhow::anyhow!( + "live adapter sync completed before its later progress boundary blocked: {result:?}" + )), + } + }) + .await + .context("live adapter sync did not reach its later progress boundary")??; + let heartbeat = bigname_storage::load_service_loop_heartbeat( + database.pool(), + bigname_storage::INDEXER_SERVICE_NAME, + heartbeat_instance_id, ) - .await?; + .await? + .context("live adapter progress heartbeat must remain registered")?; + assert!( + heartbeat.age_seconds <= 1, + "an earlier adapter page must beat before later family work finishes" + ); + progress_handle.resume(); + let summary = tokio::time::timeout(tokio::time::Duration::from_secs(10), operation.as_mut()) + .await + .context("live adapter sync did not finish after progress resumed")??; + drop(operation); + + assert!( + progress_handle.record_count() >= 6, + "block-derived loading plus reverse-claim loading, decoding, persistence, and family completion must each report live progress" + ); assert_eq!(summary.total_synced_count, 1); assert_eq!( @@ -1056,6 +1095,21 @@ async fn replay_handoff_multi_chain_fence_uses_one_connection_and_orders_writers #[tokio::test] async fn post_replay_handoff_fetches_provider_gap_after_backlog() -> Result<()> { + #[derive(Default)] + struct CountingBacklogProgress(usize); + + impl bigname_adapters::StartupAdapterProgress for CountingBacklogProgress { + fn record<'a>( + &'a mut self, + _pool: &'a PgPool, + ) -> bigname_adapters::StartupAdapterProgressFuture<'a> { + Box::pin(async move { + self.0 += 1; + Ok(()) + }) + } + } + let database = TestDatabase::new().await?; create_normalized_replay_cursor_table(database.pool()).await?; let chain = "ethereum-mainnet"; @@ -1183,13 +1237,19 @@ async fn post_replay_handoff_fetches_provider_gap_after_backlog() -> Result<()> ) .await?; - let summary = sync_live_adapter_backlog_after_normalized_replay( + let mut progress = CountingBacklogProgress::default(); + let summary = sync_live_adapter_backlog_after_normalized_replay_with_progress( database.pool(), "mainnet", &[chain.to_owned()], + &mut progress, ) .await?; assert_eq!(summary.selected_block_count, 1); + assert!( + progress.0 > 1, + "backlog adapter work and durable cursor publication must each report progress" + ); assert_eq!( sqlx::query_scalar::<_, i64>( "SELECT COUNT(*) FROM normalized_events WHERE event_kind = 'ReverseChanged'" diff --git a/apps/indexer/src/main/tests/backfill.rs b/apps/indexer/src/main/tests/backfill.rs index 073c130e..aa08c08a 100644 --- a/apps/indexer/src/main/tests/backfill.rs +++ b/apps/indexer/src/main/tests/backfill.rs @@ -948,13 +948,52 @@ async fn hash_pinned_backfill_persists_range_and_is_idempotent_without_advancing let range = BackfillBlockRange::new(42, 43)?; let config = backfill_job_config(range, "indexer-backfill-hash-pinned", "lease-first")?; - let outcome = run_resumable_hash_pinned_backfill_job( + let heartbeat_instance_id = "backfill-in-flight-progress-test"; + install_stale_indexer_heartbeat(database.pool(), heartbeat_instance_id).await?; + let (mut progress, progress_handle) = BlockingHeartbeatProgress::new( + heartbeat_instance_id, + vec!["ethereum-mainnet".to_owned()], + 2, + ); + let mut operation = Box::pin( + backfill::run_resumable_hash_pinned_backfill_job_with_progress( + database.pool(), + &source_plan, + &provider, + config.clone(), + &mut progress, + ), + ); + tokio::time::timeout(StdDuration::from_secs(10), async { + tokio::select! { + () = progress_handle.wait_until_blocked() => Ok(()), + result = operation.as_mut() => Err(anyhow::anyhow!( + "backfill completed before its later progress boundary blocked: {result:?}" + )), + } + }) + .await + .context("backfill did not reach its later progress boundary")??; + let heartbeat = bigname_storage::load_service_loop_heartbeat( database.pool(), - &source_plan, - &provider, - config.clone(), + bigname_storage::INDEXER_SERVICE_NAME, + heartbeat_instance_id, ) - .await?; + .await? + .context("backfill progress heartbeat must remain registered")?; + assert!( + heartbeat.age_seconds <= 1, + "an earlier completed backfill unit must beat before later work finishes" + ); + assert!( + progress_handle.record_count() >= 2, + "backfill must reach a later progress boundary while still in flight" + ); + progress_handle.resume(); + let outcome = tokio::time::timeout(StdDuration::from_secs(10), operation.as_mut()) + .await + .context("backfill did not finish after its progress blocker was released")??; + drop(operation); assert_eq!( outcome, backfill::BackfillJobRunOutcome { diff --git a/apps/indexer/src/main/tests/cli.rs b/apps/indexer/src/main/tests/cli.rs index f15b3bb5..25d50de1 100644 --- a/apps/indexer/src/main/tests/cli.rs +++ b/apps/indexer/src/main/tests/cli.rs @@ -82,7 +82,7 @@ fn run_cli_parses_startup_discovery_page_logs() { ); assert_eq!( page_logs_arg.get_default_values(), - &[std::ffi::OsStr::new("100000")] + &[std::ffi::OsStr::new("1000")] ); let cli = Cli::try_parse_from([ diff --git a/apps/indexer/src/main/tests/normalized_replay_catchup.rs b/apps/indexer/src/main/tests/normalized_replay_catchup.rs index f5bbbdd6..49cd9a0d 100644 --- a/apps/indexer/src/main/tests/normalized_replay_catchup.rs +++ b/apps/indexer/src/main/tests/normalized_replay_catchup.rs @@ -148,6 +148,271 @@ async fn normalized_replay_catchup_rewinds_for_later_older_raw_backfill() -> Res Ok(()) } +#[tokio::test] +async fn spawned_normalized_replay_beats_on_progress_and_exposes_a_later_wedge() -> Result<()> { + let database = bigname_test_support::TestDatabase::create_migrated( + bigname_test_support::TestDatabaseConfig::new( + "bigname_indexer_normalized_replay_heartbeat_test", + ) + .pool_max_connections(5), + &bigname_storage::MIGRATOR, + "failed to migrate normalized replay heartbeat test database", + ) + .await?; + let chain = "ethereum-mainnet"; + let instance_id = "normalized-replay-progress-test"; + let reverse_address = "0x00000000000000000000000000000000000000be"; + let block = provider_block( + "0xbebebebebebebebebebebebebebebebebebebebebebebebebebebebebebebebe", + Some("0xadadadadadadadadadadadadadadadadadadadadadadadadadadadadadadad"), + 100, + ); + insert_active_replay_watched_contract_with_source_family( + database.pool(), + 389, + chain, + "ens_v1_reverse_l1", + Uuid::from_u128(0x389), + reverse_address, + "reverse_registrar", + ) + .await?; + insert_chain_lineage_for_block( + database.pool(), + chain, + &block, + CanonicalityState::Canonical, + ) + .await?; + insert_raw_reverse_claimed_log( + database.pool(), + chain, + &block, + reverse_address, + "0x2222222222222222222222222222222222222222", + CanonicalityState::Canonical, + ) + .await?; + install_stale_indexer_heartbeat(database.pool(), instance_id).await?; + + let config = normalized_replay_catchup::NormalizedReplayCatchupConfig::new( + "mainnet".to_owned(), + vec![chain.to_owned()], + 1_000, + 1_000, + 1, + )? + .with_defer_projection_indexes(false); + let hook = normalized_replay_catchup::install_after_progress_test_hook( + database.pool(), + "mainnet", + chain, + ) + .await; + let activity = crate::run::startup_heartbeat::RequiredSubtaskActivity::default(); + let child_activity = activity.clone(); + let catchup_pool = database.pool().clone(); + let catchup = tokio::spawn(async move { + let mut heartbeat = crate::run::startup_heartbeat::NormalizedReplayHeartbeat::new( + instance_id.to_owned(), + tokio::time::Duration::ZERO, + vec![chain.to_owned()], + ); + normalized_replay_catchup::run_required_normalized_replay_catchup_iteration_for_test( + &catchup_pool, + &config, + chain, + &mut heartbeat, + &child_activity, + ) + .await + }); + tokio::time::timeout( + tokio::time::Duration::from_secs(10), + hook.wait_until_after_progress(), + ) + .await + .context("normalized replay did not reach its post-progress barrier")?; + let heartbeat = bigname_storage::load_service_loop_heartbeat( + database.pool(), + bigname_storage::INDEXER_SERVICE_NAME, + instance_id, + ) + .await? + .context("normalized replay must retain its registered heartbeat")?; + assert!( + heartbeat.age_seconds <= 1, + "completed replay work must refresh the process heartbeat" + ); + assert!( + !catchup.is_finished(), + "the replay iteration must remain blocked after its progress beat" + ); + + sqlx::query( + r#" + UPDATE service_loop_heartbeats + SET started_at = clock_timestamp() - INTERVAL '2 minutes', + heartbeat_at = clock_timestamp() - INTERVAL '1 minute' + WHERE service_name = 'indexer' + AND instance_id = $1 + AND scope_kind = 'process' + "#, + ) + .bind(instance_id) + .execute(database.pool()) + .await?; + let parent_activity = activity.clone(); + let parent_pool = database.pool().clone(); + let parent = tokio::spawn(async move { + let _required_subtask_exclusion = parent_activity.exclude_required_subtask().await; + let mut parent_heartbeat = crate::run::startup_heartbeat::StartupHeartbeat::new( + instance_id.to_owned(), + tokio::time::Duration::ZERO, + ); + parent_heartbeat + .record(&parent_pool, &[chain.to_owned()]) + .await + }); + tokio::time::sleep(tokio::time::Duration::from_millis(100)).await; + assert!( + !parent.is_finished(), + "the parent heartbeat must wait while replay owns required-operation liveness" + ); + let heartbeat_age = bigname_storage::load_service_loop_heartbeat( + database.pool(), + bigname_storage::INDEXER_SERVICE_NAME, + instance_id, + ) + .await? + .context("normalized replay must retain its registered heartbeat")? + .age_seconds; + assert!( + heartbeat_age >= 30, + "parent polling must not hide a wedged required replay iteration" + ); + + hook.resume(); + let status = tokio::time::timeout(tokio::time::Duration::from_secs(10), catchup) + .await + .context("normalized replay did not finish after its barrier was released")? + .context("normalized replay task failed")??; + assert_eq!( + status, + normalized_replay_catchup::CatchupIterationStatus::Progressed + ); + tokio::time::timeout(tokio::time::Duration::from_secs(10), parent) + .await + .context("parent heartbeat did not resume after replay completed")? + .context("parent heartbeat task failed")??; + database.cleanup().await +} + +#[tokio::test] +async fn normalized_replay_failure_journal_keeps_child_heartbeat_ownership() -> Result<()> { + let database = TestDatabase::new().await?; + create_normalized_replay_cursor_table(database.pool()).await?; + let chain = "ethereum-mainnet"; + let instance_id = "normalized-replay-failure-journal-test"; + normalized_replay_catchup::ensure_cursor_for_test( + database.pool(), + "mainnet", + chain, + 1, + 1, + true, + ) + .await?; + install_stale_indexer_heartbeat(database.pool(), instance_id).await?; + sqlx::query("DROP TABLE raw_logs CASCADE") + .execute(database.pool()) + .await?; + + let config = normalized_replay_catchup::NormalizedReplayCatchupConfig::new( + "mainnet".to_owned(), + vec![chain.to_owned()], + 1_000, + 1_000, + 1, + )?; + let hook = normalized_replay_catchup::install_before_cursor_failure_record_test_hook( + database.pool(), + "mainnet", + chain, + ) + .await; + let activity = crate::run::startup_heartbeat::RequiredSubtaskActivity::default(); + let child_activity = activity.clone(); + let catchup_pool = database.pool().clone(); + let mut catchup = tokio::spawn(async move { + let mut heartbeat = crate::run::startup_heartbeat::NormalizedReplayHeartbeat::new( + instance_id.to_owned(), + tokio::time::Duration::ZERO, + vec![chain.to_owned()], + ); + normalized_replay_catchup::run_required_normalized_replay_catchup_iteration_for_test( + &catchup_pool, + &config, + chain, + &mut heartbeat, + &child_activity, + ) + .await + }); + tokio::select! { + () = hook.wait_until_before_record() => {} + result = &mut catchup => { + panic!("normalized replay returned before failure journaling was protected: {result:?}"); + } + } + + let parent_activity = activity.clone(); + let parent_pool = database.pool().clone(); + let parent = tokio::spawn(async move { + let _required_subtask_exclusion = parent_activity.exclude_required_subtask().await; + let mut parent_heartbeat = crate::run::startup_heartbeat::StartupHeartbeat::new( + instance_id.to_owned(), + tokio::time::Duration::ZERO, + ); + parent_heartbeat + .record(&parent_pool, &[chain.to_owned()]) + .await + }); + tokio::time::sleep(tokio::time::Duration::from_millis(100)).await; + assert!( + !parent.is_finished(), + "the parent heartbeat must wait while the child journals its failed work unit" + ); + let heartbeat_age = bigname_storage::load_service_loop_heartbeat( + database.pool(), + bigname_storage::INDEXER_SERVICE_NAME, + instance_id, + ) + .await? + .context("normalized replay must retain its registered heartbeat")? + .age_seconds; + assert!( + heartbeat_age >= 30, + "parent polling must not hide a failure-journal wedge" + ); + + hook.resume(); + let catchup_error = tokio::time::timeout(tokio::time::Duration::from_secs(10), catchup) + .await + .context("normalized replay did not finish after failure journaling resumed")? + .context("normalized replay task panicked")? + .expect_err("the injected missing raw-log table must fail the replay iteration"); + assert!( + catchup_error.to_string().contains("raw-log bounds"), + "unexpected normalized replay error: {catchup_error:#}" + ); + tokio::time::timeout(tokio::time::Duration::from_secs(10), parent) + .await + .context("parent heartbeat did not resume after failure journaling completed")? + .context("parent heartbeat task panicked")??; + database.cleanup().await +} + #[tokio::test] async fn normalized_replay_catchup_log_bound_preserves_whole_blocks() -> Result<()> { let database = TestDatabase::new().await?; diff --git a/apps/indexer/src/main/tests/reconcile.rs b/apps/indexer/src/main/tests/reconcile.rs index 8237c8fd..844555b8 100644 --- a/apps/indexer/src/main/tests/reconcile.rs +++ b/apps/indexer/src/main/tests/reconcile.rs @@ -2833,7 +2833,15 @@ async fn reconcile_fetched_heads_promotes_stored_anchor_at_parent_fetch_depth_li finalized_block_number: None, }; - let outcome = reconcile_canonical_head( + let heartbeat_instance_id = "canonical-reconciliation-in-flight-progress-test"; + install_stale_indexer_heartbeat(database.pool(), heartbeat_instance_id).await?; + let (mut progress, progress_handle) = BlockingHeartbeatProgress::new( + heartbeat_instance_id, + vec![chain.to_owned()], + 2, + ); + let coverage_frontiers = ChainCoverageFrontiers::default(); + let mut operation = Box::pin(reconcile_canonical_head_with_adapter_progress( database.pool(), &provider, chain, @@ -2841,9 +2849,44 @@ async fn reconcile_fetched_heads_promotes_stored_anchor_at_parent_fetch_depth_li &safe, HeaderAuditMode::Minimal, std::slice::from_ref(&safe), - &ChainCoverageFrontiers::default(), + &coverage_frontiers, + &mut progress, + )); + tokio::time::timeout(tokio::time::Duration::from_secs(10), async { + tokio::select! { + () = progress_handle.wait_until_blocked() => Ok(()), + result = operation.as_mut() => Err(anyhow::anyhow!( + "canonical reconciliation completed before its later progress boundary blocked: {result:?}" + )), + } + }) + .await + .context("canonical reconciliation did not reach its later progress boundary")??; + let heartbeat = bigname_storage::load_service_loop_heartbeat( + database.pool(), + bigname_storage::INDEXER_SERVICE_NAME, + heartbeat_instance_id, ) - .await?; + .await? + .context("canonical reconciliation progress heartbeat must remain registered")?; + assert!( + heartbeat.age_seconds <= 1, + "an earlier fetched ancestor must beat before the later parent walk finishes" + ); + + assert!( + progress_handle.record_count() >= 2, + "the depth-limit parent walk must expose a later in-flight progress boundary" + ); + progress_handle.resume(); + let outcome = tokio::time::timeout(tokio::time::Duration::from_secs(30), operation.as_mut()) + .await + .context("canonical reconciliation did not finish after progress resumed")??; + drop(operation); + assert!( + progress_handle.record_count() >= usize::try_from(parent_fetch_depth)?, + "the depth-limit parent walk must beat after each fetched ancestor" + ); assert_eq!( outcome.status, diff --git a/apps/indexer/src/main/tests/reorg.rs b/apps/indexer/src/main/tests/reorg.rs index 5e898144..5ff040f3 100644 --- a/apps/indexer/src/main/tests/reorg.rs +++ b/apps/indexer/src/main/tests/reorg.rs @@ -1971,6 +1971,21 @@ async fn silent_winning_reorg_removes_losing_legacy_registry_discovery_authority async fn assert_silent_winning_reorg_removes_losing_legacy_registry_discovery_authority( fixture: LegacyRegistrySilentReorgFixture, ) -> Result<()> { + #[derive(Default)] + struct CountingProgress(usize); + + impl bigname_adapters::StartupAdapterProgress for CountingProgress { + fn record<'a>( + &'a mut self, + _pool: &'a PgPool, + ) -> bigname_adapters::StartupAdapterProgressFuture<'a> { + Box::pin(async move { + self.0 += 1; + Ok(()) + }) + } + } + let database = TestDatabase::new().await?; create_ops_catchup_backfill_job_tables(database.pool()).await?; @@ -2134,11 +2149,13 @@ async fn assert_silent_winning_reorg_removes_losing_legacy_registry_discovery_au .bind(stale_epoch) .execute(database.pool()) .await?; - let epoch_drift_error = bigname_adapters::sync_ens_v1_subregistry_discovery_through_block_with_expected_admission_epoch( + let mut progress = CountingProgress::default(); + let epoch_drift_error = bigname_adapters::sync_ens_v1_subregistry_discovery_through_block_with_expected_admission_epoch_and_progress( database.pool(), fixture.chain, 20, stale_epoch, + &mut progress, ) .await .expect_err("legacy registry reconciliation must reject a stale admission epoch"); @@ -2146,6 +2163,10 @@ async fn assert_silent_winning_reorg_removes_losing_legacy_registry_discovery_au format!("{epoch_drift_error:#}").contains("discovery admission epoch changed"), "epoch-drift refusal must be explicit: {epoch_drift_error:#}" ); + assert!( + progress.0 > 0, + "target-bounded registry replay must report completed work before its writer fence" + ); let watched_plan = load_watched_chain_plan(database.pool()).await?; let mut tasks = sync_intake_chain_tasks(database.pool(), &watched_plan).await?; diff --git a/apps/indexer/src/main/tests/replay.rs b/apps/indexer/src/main/tests/replay.rs index 1d5cb2ae..ddda1ae7 100644 --- a/apps/indexer/src/main/tests/replay.rs +++ b/apps/indexer/src/main/tests/replay.rs @@ -3377,6 +3377,7 @@ async fn insert_active_replay_watched_contract_with_source_family( file_path, manifest_payload ) + OVERRIDING SYSTEM VALUE VALUES ( $1, 1, diff --git a/apps/indexer/src/main/tests/runtime_refresh.rs b/apps/indexer/src/main/tests/runtime_refresh.rs index 6dbab05e..286f3bd7 100644 --- a/apps/indexer/src/main/tests/runtime_refresh.rs +++ b/apps/indexer/src/main/tests/runtime_refresh.rs @@ -1502,13 +1502,46 @@ async fn non_broad_repository_refresh_widens_plan_without_manifest_event_writes( manifests.write_manifest_for_source_family("ens_v1_registry_l1", &ens_v1_manifest_contents())?; let repository = load_manifest_repository(&manifests.path)?; - let state = build_manifest_runtime_state_for_repository_refresh( + let heartbeat_instance_id = "manifest-refresh-in-flight-progress-test"; + install_stale_indexer_heartbeat(database.pool(), heartbeat_instance_id).await?; + let (mut progress, progress_handle) = BlockingHeartbeatProgress::new( + heartbeat_instance_id, + vec!["ethereum-mainnet".to_owned()], + 2, + ); + let mut operation = Box::pin(build_manifest_runtime_state_for_repository_refresh_with_progress( database.pool(), &repository, RuntimeWatchScope::ActiveWatchedChain, false, + &mut progress, + )); + tokio::time::timeout(tokio::time::Duration::from_secs(10), async { + tokio::select! { + () = progress_handle.wait_until_blocked() => Ok(()), + result = operation.as_mut() => Err(anyhow::anyhow!( + "manifest refresh completed before its later progress boundary blocked: {result:?}" + )), + } + }) + .await + .context("manifest refresh did not reach its later progress boundary")??; + let heartbeat = bigname_storage::load_service_loop_heartbeat( + database.pool(), + bigname_storage::INDEXER_SERVICE_NAME, + heartbeat_instance_id, ) - .await?; + .await? + .context("manifest refresh progress heartbeat must remain registered")?; + assert!( + heartbeat.age_seconds <= 1, + "an earlier manifest page must beat before later refresh work finishes" + ); + progress_handle.resume(); + let state = tokio::time::timeout(tokio::time::Duration::from_secs(10), operation.as_mut()) + .await + .context("manifest refresh did not finish after progress resumed")??; + drop(operation); assert_eq!(state.watched_chain_plan.len(), 1); assert_eq!(state.manifest_normalized_event_summary.total_synced_count, 0); diff --git a/apps/indexer/src/main/tests/support.rs b/apps/indexer/src/main/tests/support.rs index b42296c6..0427e5a1 100644 --- a/apps/indexer/src/main/tests/support.rs +++ b/apps/indexer/src/main/tests/support.rs @@ -3,7 +3,7 @@ use std::{ str::FromStr, sync::{ Arc, - atomic::{AtomicU64, Ordering}, + atomic::{AtomicU64, AtomicUsize as HeartbeatRecordCount, Ordering}, }, time::{SystemTime, UNIX_EPOCH}, }; @@ -26,6 +26,7 @@ use sqlx::{ use tokio::{ io::{AsyncReadExt, AsyncWriteExt}, net::TcpListener, + sync::Notify, task::JoinHandle, }; @@ -34,6 +35,131 @@ use crate::run_mode::IndexerRunMode; static NEXT_TEST_ID: AtomicU64 = AtomicU64::new(0); +#[derive(Clone)] +pub(crate) struct BlockingHeartbeatHandle { + blocked: Arc, + resume: Arc, + record_count: Arc, +} + +pub(crate) struct BlockingHeartbeatProgress { + instance_id: String, + chain_ids: Vec, + block_before_record: usize, + handle: BlockingHeartbeatHandle, +} + +impl BlockingHeartbeatProgress { + pub(crate) fn new( + instance_id: impl Into, + chain_ids: Vec, + block_before_record: usize, + ) -> (Self, BlockingHeartbeatHandle) { + let handle = BlockingHeartbeatHandle { + blocked: Arc::new(Notify::new()), + resume: Arc::new(Notify::new()), + record_count: Arc::new(HeartbeatRecordCount::new(0)), + }; + ( + Self { + instance_id: instance_id.into(), + chain_ids, + block_before_record, + handle: handle.clone(), + }, + handle, + ) + } + + async fn record(&mut self, pool: &PgPool) -> Result<()> { + let record_number = self.handle.record_count.fetch_add(1, Ordering::AcqRel) + 1; + if record_number == self.block_before_record { + self.handle.blocked.notify_one(); + self.handle.resume.notified().await; + } + bigname_storage::record_service_loop_heartbeat( + pool, + bigname_storage::INDEXER_SERVICE_NAME, + &self.instance_id, + &self.chain_ids, + ) + .await + } +} + +impl BlockingHeartbeatHandle { + pub(crate) async fn wait_until_blocked(&self) { + self.blocked.notified().await; + } + + pub(crate) fn resume(&self) { + self.resume.notify_one(); + } + + pub(crate) fn record_count(&self) -> usize { + self.record_count.load(Ordering::Acquire) + } +} + +impl bigname_adapters::StartupAdapterProgress for BlockingHeartbeatProgress { + fn record<'a>( + &'a mut self, + pool: &'a PgPool, + ) -> bigname_adapters::StartupAdapterProgressFuture<'a> { + Box::pin(BlockingHeartbeatProgress::record(self, pool)) + } +} + +impl bigname_manifests::ManifestRuntimeProgress for BlockingHeartbeatProgress { + fn record<'a>( + &'a mut self, + pool: &'a PgPool, + ) -> bigname_manifests::ManifestRuntimeProgressFuture<'a> { + Box::pin(BlockingHeartbeatProgress::record(self, pool)) + } +} + +pub(crate) async fn install_stale_indexer_heartbeat( + pool: &PgPool, + instance_id: &str, +) -> Result<()> { + sqlx::query( + r#" + CREATE TABLE IF NOT EXISTS service_loop_heartbeats ( + service_name TEXT NOT NULL, + instance_id TEXT NOT NULL, + scope_kind TEXT NOT NULL, + scope_id TEXT NOT NULL, + started_at TIMESTAMPTZ NOT NULL, + heartbeat_at TIMESTAMPTZ NOT NULL, + PRIMARY KEY (service_name, instance_id, scope_kind, scope_id) + ) + "#, + ) + .execute(pool) + .await?; + bigname_storage::register_service_loop( + pool, + bigname_storage::INDEXER_SERVICE_NAME, + instance_id, + ) + .await?; + sqlx::query( + r#" + UPDATE service_loop_heartbeats + SET started_at = clock_timestamp() - INTERVAL '2 minutes', + heartbeat_at = clock_timestamp() - INTERVAL '1 minute' + WHERE service_name = 'indexer' + AND instance_id = $1 + AND scope_kind = 'process' + "#, + ) + .bind(instance_id) + .execute(pool) + .await?; + Ok(()) +} + struct TestManifestDir { path: PathBuf, } @@ -232,11 +358,11 @@ impl TestDatabase { .await .with_context(|| format!("failed to create test database {database_name}"))?; - // The full-closure replay holds the raw-log staging guard and the - // streamed reconcile transaction while paging staged checkpoint - // assignments over a third pooled connection. + // Progress-enabled resolver convergence can hold the runtime writer + // guard, reconciliation guard, and a bounded read transaction while + // recording a heartbeat over a fourth pooled connection. let pool = PgPoolOptions::new() - .max_connections(3) + .max_connections(4) .connect_with(base_options.database(&database_name)) .await .context("failed to connect indexer test pool")?; @@ -2198,22 +2324,27 @@ async fn insert_manifest_contract_instance( "failed to insert manifest contract contract_instance_id {contract_instance_id} for manifest_id {manifest_id}" ) })?; - sqlx::query( - r#" - INSERT INTO manifest_contracts (manifest_id, role, address, proxy_kind, implementation) - VALUES ($1, $2, $3, $4, $5) - "#, - ) - .bind(manifest_id) - .bind(role) - .bind(address) - .bind(proxy_kind) - .bind(declared_implementation_address) - .execute(pool) - .await - .with_context(|| { - format!("failed to mirror manifest_contracts row for manifest_id {manifest_id}") - })?; + if sqlx::query_scalar::<_, bool>("SELECT to_regclass('manifest_contracts') IS NOT NULL") + .fetch_one(pool) + .await? + { + sqlx::query( + r#" + INSERT INTO manifest_contracts (manifest_id, role, address, proxy_kind, implementation) + VALUES ($1, $2, $3, $4, $5) + "#, + ) + .bind(manifest_id) + .bind(role) + .bind(address) + .bind(proxy_kind) + .bind(declared_implementation_address) + .execute(pool) + .await + .with_context(|| { + format!("failed to mirror manifest_contracts row for manifest_id {manifest_id}") + })?; + } Ok(()) } diff --git a/apps/worker/src/address_names.rs b/apps/worker/src/address_names.rs index c50f00aa..2d8a72aa 100644 --- a/apps/worker/src/address_names.rs +++ b/apps/worker/src/address_names.rs @@ -8,11 +8,14 @@ mod relations; mod source_policy; mod util; -pub(crate) use rebuild::rebuild_address_names_current_with_heartbeat; pub use rebuild::{ rebuild_address_names_current, rebuild_address_names_current_logical_name, rebuild_address_names_current_logical_names, }; +pub(crate) use rebuild::{ + rebuild_address_names_current_logical_names_with_heartbeat, + rebuild_address_names_current_with_heartbeat, +}; #[derive(Clone, Debug, Default, Eq, PartialEq)] pub struct AddressNamesCurrentRebuildSummary { diff --git a/apps/worker/src/address_names/rebuild.rs b/apps/worker/src/address_names/rebuild.rs index 65d7cf87..c5b6e614 100644 --- a/apps/worker/src/address_names/rebuild.rs +++ b/apps/worker/src/address_names/rebuild.rs @@ -51,7 +51,7 @@ async fn rebuild_address_names_current_inner( loop_heartbeat: Option<&mut LoopHeartbeat>, ) -> Result { match address { - Some(address) => rebuild_one_address(pool, address).await, + Some(address) => rebuild_one_address(pool, address, loop_heartbeat).await, None => rebuild_all_addresses(pool, loop_heartbeat).await, } } @@ -68,14 +68,40 @@ pub async fn rebuild_address_names_current_logical_names( pool: &PgPool, address: &str, logical_name_ids: &[String], +) -> Result { + rebuild_address_names_current_logical_names_inner(pool, address, logical_name_ids, None).await +} + +pub(crate) async fn rebuild_address_names_current_logical_names_with_heartbeat( + pool: &PgPool, + address: &str, + logical_name_ids: &[String], + loop_heartbeat: &mut LoopHeartbeat, +) -> Result { + rebuild_address_names_current_logical_names_inner( + pool, + address, + logical_name_ids, + Some(loop_heartbeat), + ) + .await +} + +async fn rebuild_address_names_current_logical_names_inner( + pool: &PgPool, + address: &str, + logical_name_ids: &[String], + mut loop_heartbeat: Option<&mut LoopHeartbeat>, ) -> Result { let normalized_address = normalize_address(address); let mut rows = Vec::new(); for logical_name_id in logical_name_ids { let bindings = load_current_bindings_for_logical_name(pool, logical_name_id).await?; + record_rebuild_progress(pool, &mut loop_heartbeat).await; for binding in &bindings { rows.extend(build_rows_for_binding(pool, binding, Some(&normalized_address)).await?); + record_rebuild_progress(pool, &mut loop_heartbeat).await; } } @@ -86,6 +112,7 @@ pub async fn rebuild_address_names_current_logical_names( &rows, ) .await?; + record_rebuild_progress(pool, &mut loop_heartbeat).await; tracing::info!( projection = "address_names_current", @@ -273,17 +300,21 @@ fn spawn_address_names_rebuild_task( async fn rebuild_one_address( pool: &PgPool, address: &str, + mut loop_heartbeat: Option<&mut LoopHeartbeat>, ) -> Result { let normalized_address = normalize_address(address); let bindings = load_current_bindings_for_address(pool, &normalized_address).await?; + record_rebuild_progress(pool, &mut loop_heartbeat).await; let replacement = begin_address_names_current_address_replacement(pool, &normalized_address).await?; + record_rebuild_progress(pool, &mut loop_heartbeat).await; let staged = match stage_one_address_rows( pool, &replacement, &normalized_address, bindings.as_slice(), + &mut loop_heartbeat, ) .await { @@ -320,6 +351,7 @@ async fn rebuild_one_address( return Err(error); } }; + record_rebuild_progress(pool, &mut loop_heartbeat).await; if let Err(error) = drop_address_names_current_address_replacement(pool, &replacement).await { tracing::warn!( @@ -351,6 +383,7 @@ async fn stage_one_address_rows( replacement: &AddressNamesCurrentAddressReplacement, normalized_address: &str, bindings: &[CurrentBindingSeed], + loop_heartbeat: &mut Option<&mut LoopHeartbeat>, ) -> Result { let mut queued_binding_count = 0usize; let mut completed_binding_count = 0usize; @@ -375,6 +408,7 @@ async fn stage_one_address_rows( while let Some(result) = tasks.join_next().await { completed_binding_count += 1; let binding_rows = result??; + record_rebuild_progress(pool, loop_heartbeat).await; rows.extend(binding_rows); if rows.len() >= ADDRESS_NAMES_CURRENT_REBUILD_BATCH_SIZE { @@ -383,6 +417,7 @@ async fn stage_one_address_rows( .await? .len(); rows.clear(); + record_rebuild_progress(pool, loop_heartbeat).await; } if completed_binding_count.is_multiple_of(5_000) { @@ -415,6 +450,7 @@ async fn stage_one_address_rows( insert_address_names_current_address_replacement_rows(pool, replacement, &rows) .await? .len(); + record_rebuild_progress(pool, loop_heartbeat).await; } Ok(AddressNamesCurrentStagingSummary { upserted_row_count }) diff --git a/apps/worker/src/automatic_projection_replay.rs b/apps/worker/src/automatic_projection_replay.rs index c4839e97..e8130bc1 100644 --- a/apps/worker/src/automatic_projection_replay.rs +++ b/apps/worker/src/automatic_projection_replay.rs @@ -163,13 +163,17 @@ async fn run_automatic_current_projection_replay_loop( let mut projection_derivation_started = false; let projection_apply_generation = Arc::new(AtomicU64::new(0)); let projection_apply_hydration_lock = Arc::new(Mutex::new(())); + let invalidation_derive_activity = heartbeat::RequiredSubtaskActivity::default(); + let derive_heartbeat = + heartbeat::LoopHeartbeat::new(heartbeat_instance_id.clone(), poll_interval); let loop_heartbeat = Arc::new(Mutex::new(heartbeat::LoopHeartbeat::new( heartbeat_instance_id, poll_interval, ))); + let mut derive_heartbeat = Some(derive_heartbeat); loop { - record_loop_heartbeat_if_due(&pool, &loop_heartbeat).await; + record_loop_heartbeat_if_due(&pool, &loop_heartbeat, &invalidation_derive_activity).await; let mut progressed = false; if !bootstrap_completed { @@ -221,12 +225,21 @@ async fn run_automatic_current_projection_replay_loop( ); } } - record_loop_heartbeat_if_due(&pool, &loop_heartbeat).await; + record_loop_heartbeat_if_due(&pool, &loop_heartbeat, &invalidation_derive_activity) + .await; } if bootstrap_completed { if !projection_derivation_started { - invalidation_derive_loop::spawn(&subtasks, pool.clone(), poll_interval_secs)?; + invalidation_derive_loop::spawn( + &subtasks, + pool.clone(), + poll_interval_secs, + derive_heartbeat + .take() + .expect("invalidation derive heartbeat is consumed exactly once"), + invalidation_derive_activity.clone(), + )?; projection_derivation_started = true; } @@ -245,6 +258,7 @@ async fn run_automatic_current_projection_replay_loop( config, Arc::clone(&projection_apply_generation), Arc::clone(&projection_apply_hydration_lock), + invalidation_derive_activity.clone(), )?; } primary_hydration_started = true; @@ -252,6 +266,9 @@ async fn run_automatic_current_projection_replay_loop( if hydration_schedule.run_text_hydration { let hydration_result = { + let _required_subtask_exclusion = invalidation_derive_activity + .exclude_required_subtask() + .await; let mut loop_heartbeat = loop_heartbeat.lock().await; hydrate_record_inventory_text_values_after_bootstrap( &pool, @@ -274,9 +291,13 @@ async fn run_automatic_current_projection_replay_loop( ); } } - record_loop_heartbeat_if_due(&pool, &loop_heartbeat).await; + record_loop_heartbeat_if_due(&pool, &loop_heartbeat, &invalidation_derive_activity) + .await; } + let _required_subtask_exclusion = invalidation_derive_activity + .exclude_required_subtask() + .await; let _apply_hydration_guard = projection_apply_hydration_lock.lock().await; let apply_result = { let mut loop_heartbeat = loop_heartbeat.lock().await; @@ -304,7 +325,10 @@ async fn run_automatic_current_projection_replay_loop( ); } } - record_loop_heartbeat_if_due(&pool, &loop_heartbeat).await; + drop(_apply_hydration_guard); + drop(_required_subtask_exclusion); + record_loop_heartbeat_if_due(&pool, &loop_heartbeat, &invalidation_derive_activity) + .await; } if !progressed { @@ -313,7 +337,12 @@ async fn run_automatic_current_projection_replay_loop( } } -async fn record_loop_heartbeat_if_due(pool: &PgPool, loop_heartbeat: &SharedLoopHeartbeat) { +async fn record_loop_heartbeat_if_due( + pool: &PgPool, + loop_heartbeat: &SharedLoopHeartbeat, + required_subtask_activity: &heartbeat::RequiredSubtaskActivity, +) { + let _required_subtask_exclusion = required_subtask_activity.exclude_required_subtask().await; loop_heartbeat.lock().await.record_if_due(pool).await; } diff --git a/apps/worker/src/automatic_projection_replay/invalidation_derive_loop.rs b/apps/worker/src/automatic_projection_replay/invalidation_derive_loop.rs index 3117c8f1..45299c29 100644 --- a/apps/worker/src/automatic_projection_replay/invalidation_derive_loop.rs +++ b/apps/worker/src/automatic_projection_replay/invalidation_derive_loop.rs @@ -6,7 +6,10 @@ use tokio::time::{Duration, sleep}; use tracing::{info, warn}; use super::subtask_supervision::SubtaskSpawner; -use crate::projection_apply; +use crate::{ + primary_name::rebuild_heartbeat::{LoopHeartbeat, RequiredSubtaskActivity}, + projection_apply, +}; const SUBTASK_NAME: &str = "projection_invalidation_derivation"; @@ -14,10 +17,17 @@ pub(super) fn spawn( subtasks: &SubtaskSpawner, pool: PgPool, poll_interval_secs: u64, + loop_heartbeat: LoopHeartbeat, + activity: RequiredSubtaskActivity, ) -> Result<()> { spawn_future( subtasks, - run_continuous_projection_invalidation_derivation(pool, poll_interval_secs), + run_continuous_projection_invalidation_derivation( + pool, + poll_interval_secs, + loop_heartbeat, + activity, + ), ) } @@ -31,6 +41,8 @@ where async fn run_continuous_projection_invalidation_derivation( pool: PgPool, poll_interval_secs: u64, + mut loop_heartbeat: LoopHeartbeat, + activity: RequiredSubtaskActivity, ) -> Result<()> { let poll_interval = Duration::from_secs(poll_interval_secs.max(1)); info!( @@ -41,7 +53,7 @@ async fn run_continuous_projection_invalidation_derivation( loop { let mut progressed = false; - match projection_apply::derive_once(&pool).await { + match run_invalidation_derivation_iteration(&pool, &mut loop_heartbeat, &activity).await { Ok(summary) => { progressed = summary.scanned_event_count > 0; if progressed { @@ -70,6 +82,15 @@ async fn run_continuous_projection_invalidation_derivation( } } +async fn run_invalidation_derivation_iteration( + pool: &PgPool, + loop_heartbeat: &mut LoopHeartbeat, + activity: &RequiredSubtaskActivity, +) -> Result { + let _activity = activity.begin().await; + projection_apply::derive_once_with_heartbeat(pool, loop_heartbeat).await +} + #[cfg(test)] mod tests { use std::{str::FromStr, sync::Arc}; @@ -180,4 +201,186 @@ mod tests { database.cleanup().await } + + #[tokio::test] + async fn invalidation_derive_wedge_is_not_masked_by_parent_heartbeat() -> Result<()> { + let database = TestDatabase::create_migrated( + TestDatabaseConfig::new("bigname_worker_invalidation_wedge_test"), + &bigname_storage::MIGRATOR, + "failed to migrate invalidation wedge test database", + ) + .await?; + let instance_id = "invalidation-wedge-test"; + bigname_storage::register_service_loop( + database.pool(), + bigname_storage::WORKER_SERVICE_NAME, + instance_id, + ) + .await?; + sqlx::query( + r#" + UPDATE service_loop_heartbeats + SET started_at = clock_timestamp() - INTERVAL '2 minutes', + heartbeat_at = clock_timestamp() - INTERVAL '1 minute' + WHERE service_name = 'worker' + AND instance_id = $1 + AND scope_kind = 'process' + "#, + ) + .bind(instance_id) + .execute(database.pool()) + .await?; + + let (first_progress_change_id, later_unit_blocker) = + crate::projection_apply::heartbeat_tests::seed_blocked_later_progress_unit(&database) + .await?; + let activity = RequiredSubtaskActivity::default(); + let parent_activity = activity.clone(); + let derive_pool = database.pool().clone(); + let derive = tokio::spawn(async move { + let mut heartbeat = LoopHeartbeat::new(instance_id.to_owned(), Duration::ZERO); + run_invalidation_derivation_iteration(&derive_pool, &mut heartbeat, &activity).await + }); + timeout( + Duration::from_secs(10), + crate::projection_apply::heartbeat_tests::wait_for_derive_cursor( + &database, + first_progress_change_id, + ), + ) + .await + .context("spawned invalidation derive did not commit a bounded progress unit")??; + timeout( + Duration::from_secs(10), + crate::projection_apply::heartbeat_tests::wait_for_fresh_worker_heartbeat( + &database, + instance_id, + ), + ) + .await + .context("spawned invalidation derive did not record progress")??; + assert!( + !derive.is_finished(), + "the later invalidation derive unit must remain blocked" + ); + sqlx::query( + r#" + UPDATE service_loop_heartbeats + SET started_at = clock_timestamp() - INTERVAL '2 minutes', + heartbeat_at = clock_timestamp() - INTERVAL '1 minute' + WHERE service_name = 'worker' + AND instance_id = $1 + AND scope_kind = 'process' + "#, + ) + .bind(instance_id) + .execute(database.pool()) + .await?; + let parent_pool = database.pool().clone(); + let parent = tokio::spawn(async move { + let _required_subtask_exclusion = parent_activity.exclude_required_subtask().await; + let mut parent_heartbeat = crate::primary_name::rebuild_heartbeat::LoopHeartbeat::new( + instance_id.to_owned(), + Duration::ZERO, + ); + parent_heartbeat.record_if_due(&parent_pool).await; + }); + sleep(Duration::from_millis(100)).await; + assert!( + !parent.is_finished(), + "the parent heartbeat must wait while derive owns required-operation liveness" + ); + let heartbeat_age = bigname_storage::load_service_loop_heartbeat( + database.pool(), + bigname_storage::WORKER_SERVICE_NAME, + instance_id, + ) + .await? + .context("worker heartbeat must remain registered")? + .age_seconds; + later_unit_blocker.commit().await?; + let summary = timeout(Duration::from_secs(10), derive) + .await + .context("spawned invalidation derive did not finish after release")? + .context("spawned invalidation derive task failed")??; + assert_eq!( + summary.scanned_event_count, + crate::projection_apply::NORMALIZED_EVENT_DERIVE_PROGRESS_LIMIT + 1 + ); + timeout(Duration::from_secs(10), parent) + .await + .context("parent heartbeat did not resume after derive completed")? + .context("parent heartbeat task panicked")?; + database.cleanup().await?; + + assert!( + heartbeat_age >= 30, + "the parent loop must not refresh liveness while required invalidation derivation is wedged" + ); + Ok(()) + } + + #[tokio::test] + async fn invalidation_derive_progress_does_not_mask_a_parent_wedge() -> Result<()> { + let database = TestDatabase::create_migrated( + TestDatabaseConfig::new("bigname_worker_parent_wedge_test"), + &bigname_storage::MIGRATOR, + "failed to migrate worker parent-wedge test database", + ) + .await?; + let instance_id = "worker-parent-wedge-test"; + bigname_storage::register_service_loop( + database.pool(), + bigname_storage::WORKER_SERVICE_NAME, + instance_id, + ) + .await?; + sqlx::query( + r#" + UPDATE service_loop_heartbeats + SET started_at = clock_timestamp() - INTERVAL '2 minutes', + heartbeat_at = clock_timestamp() - INTERVAL '1 minute' + WHERE service_name = 'worker' + AND instance_id = $1 + AND scope_kind = 'process' + "#, + ) + .bind(instance_id) + .execute(database.pool()) + .await?; + + let activity = RequiredSubtaskActivity::default(); + let parent_operation = activity.exclude_required_subtask().await; + let child_activity = activity.clone(); + let child_pool = database.pool().clone(); + let child = tokio::spawn(async move { + let _active_derive = child_activity.begin().await; + let mut child_heartbeat = LoopHeartbeat::new(instance_id.to_owned(), Duration::ZERO); + child_heartbeat.record_if_due(&child_pool).await; + }); + sleep(Duration::from_millis(100)).await; + assert!( + !child.is_finished(), + "required derive must wait while the parent apply loop owns liveness" + ); + let heartbeat_age = bigname_storage::load_service_loop_heartbeat( + database.pool(), + bigname_storage::WORKER_SERVICE_NAME, + instance_id, + ) + .await? + .context("worker heartbeat must remain registered")? + .age_seconds; + assert!( + heartbeat_age >= 30, + "required derive progress must not refresh liveness while the parent apply loop is wedged" + ); + drop(parent_operation); + timeout(Duration::from_secs(2), child) + .await + .context("derive heartbeat did not resume after the parent released ownership")? + .context("derive heartbeat task panicked")?; + database.cleanup().await?; + Ok(()) + } } diff --git a/apps/worker/src/automatic_projection_replay/primary_hydration_loop.rs b/apps/worker/src/automatic_projection_replay/primary_hydration_loop.rs index edb291c6..307c1fad 100644 --- a/apps/worker/src/automatic_projection_replay/primary_hydration_loop.rs +++ b/apps/worker/src/automatic_projection_replay/primary_hydration_loop.rs @@ -9,8 +9,12 @@ use tokio::time::{Duration, sleep}; use tracing::{info, warn}; use super::{SharedLoopHeartbeat, primary_hydration, subtask_supervision::SubtaskSpawner}; -use crate::{primary_name, projection_apply}; +use crate::{ + primary_name::{self, rebuild_heartbeat::RequiredSubtaskActivity}, + projection_apply, +}; +#[expect(clippy::too_many_arguments)] pub(super) fn spawn( subtasks: &SubtaskSpawner, pool: PgPool, @@ -19,6 +23,7 @@ pub(super) fn spawn( config: primary_name::PrimaryNameLegacyReverseHydrationConfig, projection_apply_generation: Arc, projection_apply_hydration_lock: Arc>, + required_subtask_activity: RequiredSubtaskActivity, ) -> anyhow::Result<()> { subtasks.spawn("primary_name_legacy_reverse_hydration", async move { run( @@ -28,6 +33,7 @@ pub(super) fn spawn( config, projection_apply_generation, projection_apply_hydration_lock, + required_subtask_activity, ) .await; Ok(()) @@ -41,6 +47,7 @@ async fn run( config: primary_name::PrimaryNameLegacyReverseHydrationConfig, projection_apply_generation: Arc, projection_apply_hydration_lock: Arc>, + required_subtask_activity: RequiredSubtaskActivity, ) { let poll_interval = Duration::from_secs(poll_interval_secs.max(1)); let mut bootstrap_completed = false; @@ -55,10 +62,12 @@ async fn run( loop { let mut progressed = false; + let required_subtask_exclusion = required_subtask_activity.exclude_required_subtask().await; let apply_hydration_guard = projection_apply_hydration_lock.lock().await; match projection_apply::has_primary_hydration_blocking_work(&pool).await { Ok(true) => { drop(apply_hydration_guard); + drop(required_subtask_exclusion); sleep(poll_interval).await; continue; } @@ -71,6 +80,7 @@ async fn run( "failed to inspect projection apply work before primary-name hydration" ); drop(apply_hydration_guard); + drop(required_subtask_exclusion); sleep(poll_interval).await; continue; } @@ -137,6 +147,7 @@ async fn run( } } drop(apply_hydration_guard); + drop(required_subtask_exclusion); if !progressed { sleep(poll_interval).await; @@ -188,6 +199,7 @@ mod tests { config, Arc::new(AtomicU64::new(0)), Arc::new(Mutex::new(())), + RequiredSubtaskActivity::default(), )); sleep(Duration::from_millis(250)).await; @@ -274,6 +286,7 @@ mod tests { config, Arc::new(AtomicU64::new(0)), Arc::clone(&projection_apply_hydration_lock), + RequiredSubtaskActivity::default(), )); tokio::time::timeout(Duration::from_secs(1), async { loop { diff --git a/apps/worker/src/children.rs b/apps/worker/src/children.rs index f70dc75b..cd559d86 100644 --- a/apps/worker/src/children.rs +++ b/apps/worker/src/children.rs @@ -1,10 +1,9 @@ -use std::collections::BTreeMap; +use std::collections::{BTreeMap, HashSet}; use anyhow::{Context, Result}; use bigname_storage::{ - CanonicalityState, ChildrenCurrentRow, delete_children_current, - load_canonical_declared_child_sources, load_raw_block, stream_canonical_declared_child_sources, - upsert_children_current_rows, + CanonicalityState, ChildrenCurrentRow, load_canonical_declared_child_sources, load_raw_block, + stream_canonical_declared_child_sources, upsert_children_current_rows, }; use futures_util::{TryStreamExt, pin_mut}; use serde_json::{Value, json}; @@ -27,6 +26,8 @@ use staged_rebuild::{ const DECLARED_SURFACE_CLASS: &str = "declared"; const CHILDREN_CURRENT_DERIVATION_KIND: &str = "children_current_rebuild"; const CHILDREN_CURRENT_REBUILD_BATCH_SIZE: usize = 2_000; +const CHILDREN_CURRENT_TARGETED_BATCH_SIZE: usize = 500; +const CHILDREN_CURRENT_TARGETED_CLEANUP_PAGE_SIZE: i64 = 1_000; const CHILDREN_CURRENT_BLOCK_CACHE_LIMIT: usize = 4_096; const CHILDREN_CURRENT_REBUILD_CONCURRENCY: usize = 8; @@ -58,7 +59,9 @@ async fn rebuild_children_current_inner( loop_heartbeat: Option<&mut LoopHeartbeat>, ) -> Result { match parent_logical_name_id { - Some(parent_logical_name_id) => rebuild_one_parent(pool, parent_logical_name_id).await, + Some(parent_logical_name_id) => { + rebuild_one_parent(pool, parent_logical_name_id, loop_heartbeat).await + } None => rebuild_all_parents(pool, loop_heartbeat).await, } } @@ -179,12 +182,29 @@ fn spawn_children_rebuild_task( async fn rebuild_one_parent( pool: &PgPool, parent_logical_name_id: &str, + mut loop_heartbeat: Option<&mut LoopHeartbeat>, ) -> Result { - let sources = load_canonical_declared_child_sources(pool, Some(parent_logical_name_id)).await?; - let rows = build_children_rows(pool, &sources).await?; - let upserted_row_count = upsert_children_current_rows(pool, &rows).await?.len(); - let deleted_row_count = - delete_stale_children_current_rows_for_parent(pool, parent_logical_name_id, &rows).await?; + let sources = run_rebuild_phase( + pool, + &mut loop_heartbeat, + "children_current.target_load", + load_canonical_declared_child_sources(pool, Some(parent_logical_name_id)), + ) + .await?; + record_rebuild_progress(pool, &mut loop_heartbeat).await; + let rows = build_children_rows(pool, &sources, &mut loop_heartbeat).await?; + let mut upserted_row_count = 0usize; + for chunk in rows.chunks(CHILDREN_CURRENT_TARGETED_BATCH_SIZE) { + upserted_row_count += upsert_children_current_rows(pool, chunk).await?.len(); + record_rebuild_progress(pool, &mut loop_heartbeat).await; + } + let deleted_row_count = delete_stale_children_current_rows_for_parent( + pool, + parent_logical_name_id, + &rows, + &mut loop_heartbeat, + ) + .await?; Ok(ChildrenCurrentRebuildSummary { requested_parent_count: 1, @@ -196,12 +216,14 @@ async fn rebuild_one_parent( async fn build_children_rows( pool: &PgPool, sources: &[bigname_storage::DeclaredChildEventSource], + loop_heartbeat: &mut Option<&mut LoopHeartbeat>, ) -> Result> { let mut block_cache = BTreeMap::new(); let mut rows = Vec::with_capacity(sources.len()); for source in sources { rows.push(build_children_row(pool, source, &mut block_cache).await?); + record_rebuild_progress(pool, loop_heartbeat).await; } Ok(rows) @@ -211,39 +233,72 @@ async fn delete_stale_children_current_rows_for_parent( pool: &PgPool, parent_logical_name_id: &str, rows: &[ChildrenCurrentRow], + loop_heartbeat: &mut Option<&mut LoopHeartbeat>, ) -> Result { - if rows.is_empty() { - return delete_children_current(pool, parent_logical_name_id).await; - } - - let child_logical_name_ids = rows + let replacement_ids = rows .iter() - .map(|row| row.child_logical_name_id.clone()) - .collect::>(); - - sqlx::query( - r#" - DELETE FROM children_current current - WHERE current.parent_logical_name_id = $1 - AND current.surface_class = $2 - AND NOT EXISTS ( - SELECT 1 - FROM UNNEST($3::TEXT[]) AS replacement(child_logical_name_id) - WHERE replacement.child_logical_name_id = current.child_logical_name_id - ) - "#, - ) - .bind(parent_logical_name_id) - .bind(DECLARED_SURFACE_CLASS) - .bind(&child_logical_name_ids) - .execute(pool) - .await - .with_context(|| { - format!( - "failed to delete stale children_current rows for parent_logical_name_id {parent_logical_name_id}" + .map(|row| row.child_logical_name_id.as_str()) + .collect::>(); + let mut cursor = None::; + let mut deleted_row_count = 0u64; + + loop { + let page = sqlx::query_scalar::<_, String>( + r#" + SELECT child_logical_name_id + FROM children_current + WHERE parent_logical_name_id = $1 + AND surface_class = $2 + AND ($3::TEXT IS NULL OR child_logical_name_id > $3) + ORDER BY child_logical_name_id + LIMIT $4 + "#, ) - }) - .map(|result| result.rows_affected()) + .bind(parent_logical_name_id) + .bind(DECLARED_SURFACE_CLASS) + .bind(cursor.as_deref()) + .bind(CHILDREN_CURRENT_TARGETED_CLEANUP_PAGE_SIZE) + .fetch_all(pool) + .await + .with_context(|| { + format!( + "failed to scan stale children_current rows for parent_logical_name_id {parent_logical_name_id}" + ) + })?; + let Some(last_id) = page.last().cloned() else { + break; + }; + let stale_ids = page + .iter() + .filter(|child_id| !replacement_ids.contains(child_id.as_str())) + .cloned() + .collect::>(); + if !stale_ids.is_empty() { + deleted_row_count += sqlx::query( + r#" + DELETE FROM children_current + WHERE parent_logical_name_id = $1 + AND surface_class = $2 + AND child_logical_name_id = ANY($3::TEXT[]) + "#, + ) + .bind(parent_logical_name_id) + .bind(DECLARED_SURFACE_CLASS) + .bind(&stale_ids) + .execute(pool) + .await + .with_context(|| { + format!( + "failed to delete stale children_current page for parent_logical_name_id {parent_logical_name_id}" + ) + })? + .rows_affected(); + } + cursor = Some(last_id); + record_rebuild_progress(pool, loop_heartbeat).await; + } + + Ok(deleted_row_count) } async fn build_children_row( diff --git a/apps/worker/src/name_current.rs b/apps/worker/src/name_current.rs index 8eff7b88..62000f01 100644 --- a/apps/worker/src/name_current.rs +++ b/apps/worker/src/name_current.rs @@ -136,7 +136,9 @@ async fn rebuild_name_current_inner( loop_heartbeat: Option<&mut LoopHeartbeat>, ) -> Result { match logical_name_id { - Some(logical_name_id) => rebuild_one_name_current(pool, logical_name_id).await, + Some(logical_name_id) => { + rebuild_one_name_current(pool, logical_name_id, loop_heartbeat).await + } None => rebuild_all_name_current(pool, loop_heartbeat).await, } } @@ -228,18 +230,22 @@ fn spawn_name_current_rebuild_task( async fn rebuild_one_name_current( pool: &PgPool, logical_name_id: &str, + mut loop_heartbeat: Option<&mut LoopHeartbeat>, ) -> Result { let Some(name) = load_canonical_name_surface(pool, logical_name_id).await? else { let deleted_row_count = delete_name_current(pool, logical_name_id).await?; + record_rebuild_progress(pool, &mut loop_heartbeat).await; return Ok(NameCurrentRebuildSummary { requested_name_count: 1, upserted_row_count: 0, deleted_row_count, }); }; + record_rebuild_progress(pool, &mut loop_heartbeat).await; - let row = build_name_current_row(pool, &name).await?; + let row = build_name_current_row_inner(pool, &name, &mut loop_heartbeat).await?; let upserted_row_count = upsert_name_current_rows(pool, &[row]).await?.len(); + record_rebuild_progress(pool, &mut loop_heartbeat).await; Ok(NameCurrentRebuildSummary { requested_name_count: 1, upserted_row_count, @@ -248,13 +254,26 @@ async fn rebuild_one_name_current( } async fn build_name_current_row(pool: &PgPool, name: &NameSurfaceSeed) -> Result { + build_name_current_row_inner(pool, name, &mut None).await +} + +async fn build_name_current_row_inner( + pool: &PgPool, + name: &NameSurfaceSeed, + loop_heartbeat: &mut Option<&mut LoopHeartbeat>, +) -> Result { let current_binding = load_current_binding_context(pool, &name.logical_name_id).await?; + record_rebuild_progress(pool, loop_heartbeat).await; let events = load_relevant_events(pool, name, current_binding.as_ref()).await?; + record_rebuild_progress(pool, loop_heartbeat).await; let history_heads = load_history_heads(pool, &name.logical_name_id).await?; + record_rebuild_progress(pool, loop_heartbeat).await; let basenames_execution_manifest = load_active_basenames_execution_manifest(pool, &name.namespace).await?; + record_rebuild_progress(pool, loop_heartbeat).await; let wildcard_source_context = load_wildcard_source_context(pool, name, current_binding.as_ref()).await?; + record_rebuild_progress(pool, loop_heartbeat).await; let supplemental_chain_observations = load_supplemental_chain_observations( pool, name, @@ -265,6 +284,7 @@ async fn build_name_current_row(pool: &PgPool, name: &NameSurfaceSeed) -> Result basenames_execution_manifest.as_ref(), ) .await?; + record_rebuild_progress(pool, loop_heartbeat).await; let mut facts = project_facts(&events, current_binding.as_ref(), &history_heads)?; // created_at is the first observation of this name. Supplemental observations // can come from parent wildcard names or Basenames transport lineage. diff --git a/apps/worker/src/permissions.rs b/apps/worker/src/permissions.rs index 93702332..c8d6222a 100644 --- a/apps/worker/src/permissions.rs +++ b/apps/worker/src/permissions.rs @@ -77,7 +77,7 @@ async fn rebuild_permissions_current_inner( loop_heartbeat: Option<&mut LoopHeartbeat>, ) -> Result { match resource_id { - Some(resource_id) => rebuild_one_resource(pool, resource_id).await, + Some(resource_id) => rebuild_one_resource(pool, resource_id, loop_heartbeat).await, None => rebuild_all_resources(pool, loop_heartbeat).await, } } @@ -204,10 +204,12 @@ fn spawn_permissions_rebuild_task( async fn rebuild_one_resource( pool: &PgPool, resource_id: &str, + mut loop_heartbeat: Option<&mut LoopHeartbeat>, ) -> Result { let resource_id = Uuid::parse_str(resource_id) .with_context(|| format!("resource_id must be a UUID: {resource_id}"))?; let projection = build_resource_projection(pool, resource_id).await?; + record_rebuild_progress(pool, &mut loop_heartbeat).await; let (upserted_row_count, deleted_row_count) = replace_permissions_current_resource_projection( pool, resource_id, @@ -215,6 +217,7 @@ async fn rebuild_one_resource( projection.summary.as_ref(), ) .await?; + record_rebuild_progress(pool, &mut loop_heartbeat).await; Ok(PermissionsCurrentRebuildSummary { requested_resource_count: 1, diff --git a/apps/worker/src/primary_name/projection.rs b/apps/worker/src/primary_name/projection.rs index da9eb4aa..e81ec538 100644 --- a/apps/worker/src/primary_name/projection.rs +++ b/apps/worker/src/primary_name/projection.rs @@ -68,7 +68,7 @@ async fn rebuild_primary_names_current_inner( ) -> Result { match (address, namespace, coin_type) { (Some(address), Some(namespace), Some(coin_type)) => { - rebuild_one_primary_name(pool, address, namespace, coin_type).await + rebuild_one_primary_name(pool, address, namespace, coin_type, loop_heartbeat).await } (None, None, None) => rebuild_all_primary_names(pool, loop_heartbeat).await, _ => bail!( @@ -195,6 +195,7 @@ async fn rebuild_one_primary_name( address: &str, namespace: &str, coin_type: &str, + mut loop_heartbeat: Option<&mut LoopHeartbeat>, ) -> Result { #[cfg(test)] let test_database = test_hooks::current_database(pool).await?; @@ -210,6 +211,7 @@ async fn rebuild_one_primary_name( } None => None, }; + record_rebuild_progress(pool, &mut loop_heartbeat).await; let mut transaction = pool .begin() .await @@ -304,6 +306,7 @@ async fn rebuild_one_primary_name( target.address, target.namespace, target.coin_type ) })?; + record_rebuild_progress(pool, &mut loop_heartbeat).await; let projected_rows = projected_row .iter() .map(|projection| projection.row.clone()) diff --git a/apps/worker/src/projection_apply.rs b/apps/worker/src/projection_apply.rs index 7f185698..f6b8f13e 100644 --- a/apps/worker/src/projection_apply.rs +++ b/apps/worker/src/projection_apply.rs @@ -16,13 +16,14 @@ use crate::primary_name::rebuild_heartbeat::LoopHeartbeat; use crate::record_inventory; pub(crate) use derive::{ + ProjectionInvalidationDeriveSummary, capture_normalized_event_change_watermark as load_normalized_event_change_watermark, normalized_event_cursor_exists, seed_normalized_event_cursor_if_absent, }; const NORMALIZED_EVENT_CURSOR: &str = "normalized_events_to_projection_invalidations"; const NORMALIZED_EVENT_DERIVE_BATCH_LIMIT: i64 = 5_000; -const NORMALIZED_EVENT_DERIVE_PROGRESS_LIMIT: i64 = 250; +pub(crate) const NORMALIZED_EVENT_DERIVE_PROGRESS_LIMIT: i64 = 250; const PROJECTION_APPLY_BATCH_LIMIT: i64 = 25; const FAILURE_RETRY_DELAY: &str = "60 seconds"; const CLAIM_RETRY_DELAY: &str = "5 minutes"; @@ -89,13 +90,14 @@ pub(crate) async fn run_once( Ok(summary) } +#[cfg(test)] pub(crate) async fn derive_once( pool: &PgPool, ) -> Result { derive_once_inner(pool, None).await } -async fn derive_once_with_heartbeat( +pub(crate) async fn derive_once_with_heartbeat( pool: &PgPool, loop_heartbeat: &mut LoopHeartbeat, ) -> Result { @@ -203,7 +205,7 @@ pub(crate) async fn load_chain_checkpoint_max_block(pool: &PgPool) -> Result Result<(i64, Transaction<'static, Postgres>)> { sqlx::query( @@ -294,7 +296,10 @@ mod heartbeat_tests { Ok((first_progress_change_id, later_unit_blocker)) } - async fn wait_for_derive_cursor(database: &TestDatabase, expected: i64) -> Result<()> { + pub(crate) async fn wait_for_derive_cursor( + database: &TestDatabase, + expected: i64, + ) -> Result<()> { loop { let cursor = sqlx::query_scalar::<_, i64>( r#" @@ -313,7 +318,7 @@ mod heartbeat_tests { } } - async fn wait_for_fresh_worker_heartbeat( + pub(crate) async fn wait_for_fresh_worker_heartbeat( database: &TestDatabase, instance_id: &str, ) -> Result<()> { diff --git a/apps/worker/src/projection_apply/apply.rs b/apps/worker/src/projection_apply/apply.rs index 30e4eb9d..914bd8f4 100644 --- a/apps/worker/src/projection_apply/apply.rs +++ b/apps/worker/src/projection_apply/apply.rs @@ -11,18 +11,17 @@ use crate::{ primary_name::rebuild_heartbeat::LoopHeartbeat, record_inventory, resolver, }; -use super::{ - apply_locks::{ - acquire_invalidation_apply_locks, ensure_invalidation_apply_locks_alive, - release_invalidation_apply_locks, - }, - dead_letters::dead_letter_invalidation, +use super::apply_locks::{ + acquire_invalidation_apply_locks, ensure_invalidation_apply_locks_alive, + release_invalidation_apply_locks, }; mod claim; +mod completion; #[cfg(test)] use claim::refresh_claimed_invalidation_claim; use claim::{claim_pending_invalidations, spawn_claim_heartbeats, stop_claim_heartbeats}; +use completion::{complete_invalidation, fail_invalidation}; const NAME_CURRENT_SINGLE_APPLY_TIMEOUT: Duration = Duration::from_secs(120); const MAX_PROJECTION_INVALIDATION_ATTEMPTS: i64 = 5; @@ -92,7 +91,7 @@ async fn apply_pending_invalidations_inner( let group = drain_address_names_group(&mut invalidations); let group_len = group.len(); let mut locks = acquire_invalidation_apply_locks(pool, &group).await?; - let result = apply_address_names_group(pool, &group).await; + let result = apply_address_names_group(pool, &group, &mut loop_heartbeat).await; let finish = async { ensure_invalidation_apply_locks_alive(&mut locks).await?; match result { @@ -123,9 +122,15 @@ async fn apply_pending_invalidations_inner( let mut locks = acquire_invalidation_apply_locks(pool, std::slice::from_ref(&invalidation)).await?; let result = if invalidation.projection == "name_current" { - apply_name_current_single(pool, &invalidation).await + apply_name_current_single(pool, &invalidation, &mut loop_heartbeat).await } else { - apply_one(pool, &invalidation, text_hydration_config).await + apply_one( + pool, + &invalidation, + text_hydration_config, + &mut loop_heartbeat, + ) + .await }; let finish = async { ensure_invalidation_apply_locks_alive(&mut locks).await?; @@ -184,6 +189,7 @@ fn drain_address_names_group( async fn apply_address_names_group( pool: &PgPool, invalidations: &[ClaimedInvalidation], + loop_heartbeat: &mut Option<&mut LoopHeartbeat>, ) -> Result<()> { let Some(first) = invalidations.first() else { return Ok(()); @@ -192,7 +198,19 @@ async fn apply_address_names_group( if invalidations.iter().any(|invalidation| { optional_payload_str(&invalidation.key_payload, "logical_name_id").is_none() }) { - address_names::rebuild_address_names_current(pool, Some(address)).await?; + match loop_heartbeat.as_deref_mut() { + Some(loop_heartbeat) => { + address_names::rebuild_address_names_current_with_heartbeat( + pool, + Some(address), + loop_heartbeat, + ) + .await?; + } + None => { + address_names::rebuild_address_names_current(pool, Some(address)).await?; + } + } return Ok(()); } @@ -203,8 +221,25 @@ async fn apply_address_names_group( .into_iter() .map(str::to_owned) .collect::>(); - address_names::rebuild_address_names_current_logical_names(pool, address, &logical_name_ids) - .await?; + match loop_heartbeat.as_deref_mut() { + Some(loop_heartbeat) => { + address_names::rebuild_address_names_current_logical_names_with_heartbeat( + pool, + address, + &logical_name_ids, + loop_heartbeat, + ) + .await?; + } + None => { + address_names::rebuild_address_names_current_logical_names( + pool, + address, + &logical_name_ids, + ) + .await?; + } + } Ok(()) } @@ -217,13 +252,29 @@ fn address_names_invalidation_address(invalidation: &ClaimedInvalidation) -> Res async fn apply_name_current_single( pool: &PgPool, invalidation: &ClaimedInvalidation, + loop_heartbeat: &mut Option<&mut LoopHeartbeat>, ) -> Result<()> { - timeout( - NAME_CURRENT_SINGLE_APPLY_TIMEOUT, - name_current::rebuild_name_current(pool, Some(&invalidation.projection_key)), - ) - .await - .with_context(|| { + let result = match loop_heartbeat.as_deref_mut() { + Some(loop_heartbeat) => { + timeout( + NAME_CURRENT_SINGLE_APPLY_TIMEOUT, + name_current::rebuild_name_current_with_heartbeat( + pool, + Some(&invalidation.projection_key), + loop_heartbeat, + ), + ) + .await + } + None => { + timeout( + NAME_CURRENT_SINGLE_APPLY_TIMEOUT, + name_current::rebuild_name_current(pool, Some(&invalidation.projection_key)), + ) + .await + } + }; + result.with_context(|| { format!( "timed out applying name_current invalidation {}", invalidation.projection_key @@ -268,31 +319,89 @@ async fn apply_one( pool: &PgPool, invalidation: &ClaimedInvalidation, text_hydration_config: Option<&record_inventory::RecordInventoryTextHydrationConfig>, + loop_heartbeat: &mut Option<&mut LoopHeartbeat>, ) -> Result<()> { match invalidation.projection.as_str() { - "name_current" => { - name_current::rebuild_name_current(pool, Some(&invalidation.projection_key)).await?; - } - "children_current" => { - children::rebuild_children_current(pool, Some(&invalidation.projection_key)).await?; - } - "permissions_current" => { - permissions::rebuild_permissions_current(pool, Some(&invalidation.projection_key)) + "name_current" => match loop_heartbeat.as_deref_mut() { + Some(loop_heartbeat) => { + name_current::rebuild_name_current_with_heartbeat( + pool, + Some(&invalidation.projection_key), + loop_heartbeat, + ) .await?; - } - "record_inventory_current" => { - record_inventory::rebuild_record_inventory_current( - pool, - Some(&invalidation.projection_key), - ) - .await?; - if let Some(config) = text_hydration_config { - let hydration_summary = record_inventory::hydrate_record_inventory_text_values( + } + None => { + name_current::rebuild_name_current(pool, Some(&invalidation.projection_key)) + .await?; + } + }, + "children_current" => match loop_heartbeat.as_deref_mut() { + Some(loop_heartbeat) => { + children::rebuild_children_current_with_heartbeat( pool, Some(&invalidation.projection_key), - config.clone(), + loop_heartbeat, ) .await?; + } + None => { + children::rebuild_children_current(pool, Some(&invalidation.projection_key)) + .await?; + } + }, + "permissions_current" => match loop_heartbeat.as_deref_mut() { + Some(loop_heartbeat) => { + permissions::rebuild_permissions_current_with_heartbeat( + pool, + Some(&invalidation.projection_key), + loop_heartbeat, + ) + .await?; + } + None => { + permissions::rebuild_permissions_current(pool, Some(&invalidation.projection_key)) + .await?; + } + }, + "record_inventory_current" => { + match loop_heartbeat.as_deref_mut() { + Some(loop_heartbeat) => { + record_inventory::rebuild_record_inventory_current_with_heartbeat( + pool, + Some(&invalidation.projection_key), + loop_heartbeat, + ) + .await?; + } + None => { + record_inventory::rebuild_record_inventory_current( + pool, + Some(&invalidation.projection_key), + ) + .await?; + } + } + if let Some(config) = text_hydration_config { + let hydration_summary = match loop_heartbeat.as_deref_mut() { + Some(loop_heartbeat) => { + record_inventory::hydrate_record_inventory_text_values_with_heartbeat( + pool, + Some(&invalidation.projection_key), + config.clone(), + loop_heartbeat, + ) + .await? + } + None => { + record_inventory::hydrate_record_inventory_text_values( + pool, + Some(&invalidation.projection_key), + config.clone(), + ) + .await? + } + }; record_inventory::log_text_hydration_summary( Some(&invalidation.projection_key), &hydration_summary, @@ -302,39 +411,95 @@ async fn apply_one( "resolver_current" => { let chain_id = payload_str(&invalidation.key_payload, "chain_id")?; let resolver_address = payload_str(&invalidation.key_payload, "resolver_address")?; - resolver::rebuild_resolver_current(pool, Some(chain_id), Some(resolver_address)) - .await?; + match loop_heartbeat.as_deref_mut() { + Some(loop_heartbeat) => { + resolver::rebuild_resolver_current_with_heartbeat( + pool, + Some(chain_id), + Some(resolver_address), + loop_heartbeat, + ) + .await?; + } + None => { + resolver::rebuild_resolver_current( + pool, + Some(chain_id), + Some(resolver_address), + ) + .await?; + } + } } "address_names_current" => { if let Some(logical_name_id) = optional_payload_str(&invalidation.key_payload, "logical_name_id") { let address = payload_str(&invalidation.key_payload, "address")?; - address_names::rebuild_address_names_current_logical_name( - pool, - address, - logical_name_id, - ) - .await?; + match loop_heartbeat.as_deref_mut() { + Some(loop_heartbeat) => { + address_names::rebuild_address_names_current_logical_names_with_heartbeat( + pool, + address, + &[logical_name_id.to_owned()], + loop_heartbeat, + ) + .await?; + } + None => { + address_names::rebuild_address_names_current_logical_name( + pool, + address, + logical_name_id, + ) + .await?; + } + } } else { - address_names::rebuild_address_names_current( - pool, - Some(&invalidation.projection_key), - ) - .await?; + match loop_heartbeat.as_deref_mut() { + Some(loop_heartbeat) => { + address_names::rebuild_address_names_current_with_heartbeat( + pool, + Some(&invalidation.projection_key), + loop_heartbeat, + ) + .await?; + } + None => { + address_names::rebuild_address_names_current( + pool, + Some(&invalidation.projection_key), + ) + .await?; + } + } } } "primary_names_current" => { let address = payload_str(&invalidation.key_payload, "address")?; let namespace = payload_str(&invalidation.key_payload, "namespace")?; let coin_type = payload_str(&invalidation.key_payload, "coin_type")?; - primary_name::rebuild_primary_names_current( - pool, - Some(address), - Some(namespace), - Some(coin_type), - ) - .await?; + match loop_heartbeat.as_deref_mut() { + Some(loop_heartbeat) => { + primary_name::rebuild_primary_names_current_with_heartbeat( + pool, + Some(address), + Some(namespace), + Some(coin_type), + loop_heartbeat, + ) + .await?; + } + None => { + primary_name::rebuild_primary_names_current( + pool, + Some(address), + Some(namespace), + Some(coin_type), + ) + .await?; + } + } } projection => bail!("unsupported projection invalidation family {projection}"), } @@ -365,127 +530,6 @@ fn nonblank_str(value: &str) -> Option<&str> { } } -async fn complete_invalidation(pool: &PgPool, invalidation: &ClaimedInvalidation) -> Result<()> { - let result = sqlx::query( - r#" - DELETE FROM projection_invalidations - WHERE projection = $1 - AND projection_key = $2 - AND generation = $3 - AND claim_token = $4 - "#, - ) - .bind(&invalidation.projection) - .bind(&invalidation.projection_key) - .bind(invalidation.generation) - .bind(invalidation.claim_token) - .execute(pool) - .await - .with_context(|| { - format!( - "failed to complete projection invalidation {}:{}", - invalidation.projection, invalidation.projection_key - ) - })?; - if result.rows_affected() == 0 { - release_superseded_claim(pool, invalidation).await?; - } - - Ok(()) -} - -async fn fail_invalidation( - pool: &PgPool, - invalidation: &ClaimedInvalidation, - error: &anyhow::Error, -) -> Result<()> { - let failure_reason = postgres_text_safe(&format!("{error:#}")); - let failed_attempt_count = invalidation.attempt_count + 1; - let should_dead_letter = failed_attempt_count >= MAX_PROJECTION_INVALIDATION_ATTEMPTS; - let rows_affected = if should_dead_letter { - dead_letter_invalidation(pool, invalidation, &failure_reason, failed_attempt_count).await? - } else { - sqlx::query( - r#" - UPDATE projection_invalidations - SET - claim_token = NULL, - claimed_at = NULL, - attempt_count = $6, - last_failure_reason = $5, - last_failure_at = now() - WHERE projection = $1 - AND projection_key = $2 - AND generation = $3 - AND claim_token = $4 - "#, - ) - .bind(&invalidation.projection) - .bind(&invalidation.projection_key) - .bind(invalidation.generation) - .bind(invalidation.claim_token) - .bind(&failure_reason) - .bind(failed_attempt_count) - .execute(pool) - .await - .with_context(|| { - format!( - "failed to record projection invalidation failure {}:{}", - invalidation.projection, invalidation.projection_key - ) - })? - .rows_affected() - }; - if should_dead_letter && rows_affected > 0 { - tracing::warn!( - projection = %invalidation.projection, - projection_key = %invalidation.projection_key, - generation = invalidation.generation, - failed_attempt_count = failed_attempt_count, - max_attempts = MAX_PROJECTION_INVALIDATION_ATTEMPTS, - failure_reason = %failure_reason, - "moved projection invalidation to dead letter" - ); - } - if rows_affected == 0 { - release_superseded_claim(pool, invalidation).await?; - } - - Ok(()) -} - -async fn release_superseded_claim(pool: &PgPool, invalidation: &ClaimedInvalidation) -> Result<()> { - sqlx::query( - r#" - UPDATE projection_invalidations - SET claim_token = NULL, - claimed_at = NULL - WHERE projection = $1 - AND projection_key = $2 - AND generation > $3 - AND claim_token = $4 - "#, - ) - .bind(&invalidation.projection) - .bind(&invalidation.projection_key) - .bind(invalidation.generation) - .bind(invalidation.claim_token) - .execute(pool) - .await - .with_context(|| { - format!( - "failed to release superseded projection invalidation claim {}:{}", - invalidation.projection, invalidation.projection_key - ) - })?; - - Ok(()) -} - -fn postgres_text_safe(text: &str) -> String { - text.replace('\0', "\\u0000") -} - #[cfg(test)] #[path = "apply/tests.rs"] mod tests; diff --git a/apps/worker/src/projection_apply/apply/completion.rs b/apps/worker/src/projection_apply/apply/completion.rs new file mode 100644 index 00000000..294eda98 --- /dev/null +++ b/apps/worker/src/projection_apply/apply/completion.rs @@ -0,0 +1,130 @@ +use anyhow::{Context, Result}; +use sqlx::PgPool; + +use crate::projection_apply::dead_letters::dead_letter_invalidation; + +use super::{ClaimedInvalidation, MAX_PROJECTION_INVALIDATION_ATTEMPTS}; + +pub(super) async fn complete_invalidation( + pool: &PgPool, + invalidation: &ClaimedInvalidation, +) -> Result<()> { + let result = sqlx::query( + r#" + DELETE FROM projection_invalidations + WHERE projection = $1 + AND projection_key = $2 + AND generation = $3 + AND claim_token = $4 + "#, + ) + .bind(&invalidation.projection) + .bind(&invalidation.projection_key) + .bind(invalidation.generation) + .bind(invalidation.claim_token) + .execute(pool) + .await + .with_context(|| { + format!( + "failed to complete projection invalidation {}:{}", + invalidation.projection, invalidation.projection_key + ) + })?; + if result.rows_affected() == 0 { + release_superseded_claim(pool, invalidation).await?; + } + + Ok(()) +} + +pub(super) async fn fail_invalidation( + pool: &PgPool, + invalidation: &ClaimedInvalidation, + error: &anyhow::Error, +) -> Result<()> { + let failure_reason = postgres_text_safe(&format!("{error:#}")); + let failed_attempt_count = invalidation.attempt_count + 1; + let should_dead_letter = failed_attempt_count >= MAX_PROJECTION_INVALIDATION_ATTEMPTS; + let rows_affected = if should_dead_letter { + dead_letter_invalidation(pool, invalidation, &failure_reason, failed_attempt_count).await? + } else { + sqlx::query( + r#" + UPDATE projection_invalidations + SET + claim_token = NULL, + claimed_at = NULL, + attempt_count = $6, + last_failure_reason = $5, + last_failure_at = now() + WHERE projection = $1 + AND projection_key = $2 + AND generation = $3 + AND claim_token = $4 + "#, + ) + .bind(&invalidation.projection) + .bind(&invalidation.projection_key) + .bind(invalidation.generation) + .bind(invalidation.claim_token) + .bind(&failure_reason) + .bind(failed_attempt_count) + .execute(pool) + .await + .with_context(|| { + format!( + "failed to record projection invalidation failure {}:{}", + invalidation.projection, invalidation.projection_key + ) + })? + .rows_affected() + }; + if should_dead_letter && rows_affected > 0 { + tracing::warn!( + projection = %invalidation.projection, + projection_key = %invalidation.projection_key, + generation = invalidation.generation, + failed_attempt_count, + max_attempts = MAX_PROJECTION_INVALIDATION_ATTEMPTS, + failure_reason = %failure_reason, + "moved projection invalidation to dead letter" + ); + } + if rows_affected == 0 { + release_superseded_claim(pool, invalidation).await?; + } + + Ok(()) +} + +async fn release_superseded_claim(pool: &PgPool, invalidation: &ClaimedInvalidation) -> Result<()> { + sqlx::query( + r#" + UPDATE projection_invalidations + SET claim_token = NULL, + claimed_at = NULL + WHERE projection = $1 + AND projection_key = $2 + AND generation > $3 + AND claim_token = $4 + "#, + ) + .bind(&invalidation.projection) + .bind(&invalidation.projection_key) + .bind(invalidation.generation) + .bind(invalidation.claim_token) + .execute(pool) + .await + .with_context(|| { + format!( + "failed to release superseded projection invalidation claim {}:{}", + invalidation.projection, invalidation.projection_key + ) + })?; + + Ok(()) +} + +fn postgres_text_safe(text: &str) -> String { + text.replace('\0', "\\u0000") +} diff --git a/apps/worker/src/projection_apply/apply/tests.rs b/apps/worker/src/projection_apply/apply/tests.rs index e8d3282c..ede0ecb7 100644 --- a/apps/worker/src/projection_apply/apply/tests.rs +++ b/apps/worker/src/projection_apply/apply/tests.rs @@ -572,6 +572,161 @@ fn address_name_invalidations_are_grouped_by_address() { ); } +#[tokio::test] +async fn every_targeted_apply_family_records_internal_progress() -> Result<()> { + let database = test_database().await?; + let instance_id = "projection-apply-family-progress-test"; + bigname_storage::register_service_loop( + database.pool(), + bigname_storage::WORKER_SERVICE_NAME, + instance_id, + ) + .await?; + let mut heartbeat = LoopHeartbeat::new(instance_id.to_owned(), Duration::ZERO); + + let cases = [ + claimed_invalidation("name_current", "ens:missing-name.eth"), + claimed_invalidation("children_current", "ens:missing-parent.eth"), + claimed_invalidation("permissions_current", &Uuid::new_v4().to_string()), + claimed_invalidation("record_inventory_current", &Uuid::new_v4().to_string()), + ClaimedInvalidation { + projection: "resolver_current".to_owned(), + projection_key: "ethereum-mainnet:0x1111111111111111111111111111111111111111" + .to_owned(), + key_payload: serde_json::json!({ + "chain_id": "ethereum-mainnet", + "resolver_address": "0x1111111111111111111111111111111111111111" + }), + generation: 0, + claim_token: Uuid::nil(), + attempt_count: 0, + }, + ClaimedInvalidation { + projection: "primary_names_current".to_owned(), + projection_key: "0x2222222222222222222222222222222222222222:ens:60".to_owned(), + key_payload: serde_json::json!({ + "address": "0x2222222222222222222222222222222222222222", + "namespace": "ens", + "coin_type": "60" + }), + generation: 0, + claim_token: Uuid::nil(), + attempt_count: 0, + }, + ]; + + for invalidation in &cases { + let before = heartbeat.progress_record_count(); + { + let mut progress = Some(&mut heartbeat); + apply_one(database.pool(), invalidation, None, &mut progress).await?; + } + assert!( + heartbeat.progress_record_count() > before, + "{} targeted apply must report progress inside its projection work", + invalidation.projection + ); + } + + let address = "0x3333333333333333333333333333333333333333"; + let group = (0..40) + .map(|index| { + address_names_claimed_invalidation(address, Some(&format!("ens:missing-{index}.eth"))) + }) + .collect::>(); + let before = heartbeat.progress_record_count(); + { + let mut progress = Some(&mut heartbeat); + apply_address_names_group(database.pool(), &group, &mut progress).await?; + } + assert!( + heartbeat.progress_record_count() >= before + group.len(), + "address_names_current targeted apply must report each completed logical-name unit" + ); + + database.cleanup().await +} + +#[tokio::test] +async fn targeted_apply_beats_before_a_blocked_publish_finishes() -> Result<()> { + let database = test_database().await?; + let instance_id = "projection-apply-in-flight-progress-test"; + bigname_storage::register_service_loop( + database.pool(), + bigname_storage::WORKER_SERVICE_NAME, + instance_id, + ) + .await?; + sqlx::query( + r#" + UPDATE service_loop_heartbeats + SET started_at = clock_timestamp() - INTERVAL '2 minutes', + heartbeat_at = clock_timestamp() - INTERVAL '1 minute' + WHERE service_name = 'worker' + AND instance_id = $1 + "#, + ) + .bind(instance_id) + .execute(database.pool()) + .await?; + + let address = "0x4444444444444444444444444444444444444444"; + let mut publish_blocker = database.pool().begin().await?; + sqlx::query("SELECT address_names_current_identity_counts_lock_address($1)") + .bind(address) + .execute(&mut *publish_blocker) + .await?; + + let group = (0..40) + .map(|index| { + address_names_claimed_invalidation(address, Some(&format!("ens:blocked-{index}.eth"))) + }) + .collect::>(); + let apply_pool = database.pool().clone(); + let apply = tokio::spawn(async move { + let mut heartbeat = LoopHeartbeat::new(instance_id.to_owned(), Duration::ZERO); + let mut progress = Some(&mut heartbeat); + apply_address_names_group(&apply_pool, &group, &mut progress).await + }); + + let heartbeat_observed = timeout(Duration::from_secs(10), async { + loop { + let heartbeat = bigname_storage::load_service_loop_heartbeat( + database.pool(), + bigname_storage::WORKER_SERVICE_NAME, + instance_id, + ) + .await? + .context("targeted apply must retain its registered heartbeat")?; + if heartbeat.age_seconds <= 1 { + return Ok::<(), anyhow::Error>(()); + } + tokio::time::sleep(Duration::from_millis(10)).await; + } + }) + .await; + if let Err(error) = heartbeat_observed { + apply.abort(); + publish_blocker.rollback().await?; + let _ = apply.await; + database.cleanup().await?; + return Err(error).context("targeted apply did not beat before blocked publication"); + } + heartbeat_observed??; + assert!( + !apply.is_finished(), + "publication must still be blocked when internal progress is observed" + ); + + publish_blocker.rollback().await?; + timeout(Duration::from_secs(10), apply) + .await + .context("targeted apply did not finish after publication was unblocked")? + .context("targeted apply task failed")??; + + database.cleanup().await +} + #[tokio::test] async fn address_name_default_payload_invalidation_rebuilds_by_projection_key() -> Result<()> { let database = test_database().await?; diff --git a/apps/worker/src/rebuild_heartbeat.rs b/apps/worker/src/rebuild_heartbeat.rs index f9968974..4cf0aa10 100644 --- a/apps/worker/src/rebuild_heartbeat.rs +++ b/apps/worker/src/rebuild_heartbeat.rs @@ -1,22 +1,53 @@ -use std::future::Future; +use std::{future::Future, sync::Arc}; use anyhow::{Context, Result}; use sqlx::PgPool; -use tokio::time::{Duration, Instant}; +use tokio::{ + sync::{Mutex, OwnedMutexGuard}, + time::{Duration, Instant}, +}; use tracing::warn; +const MAX_PROGRESS_HEARTBEAT_INTERVAL: Duration = Duration::from_secs(5); + +#[derive(Clone, Default)] +pub(crate) struct RequiredSubtaskActivity { + phase_exclusion: Arc>, +} + +pub(crate) struct RequiredSubtaskActivityGuard { + _phase_exclusion: OwnedMutexGuard<()>, +} + +impl RequiredSubtaskActivity { + pub(crate) async fn begin(&self) -> RequiredSubtaskActivityGuard { + let phase_exclusion = Arc::clone(&self.phase_exclusion).lock_owned().await; + RequiredSubtaskActivityGuard { + _phase_exclusion: phase_exclusion, + } + } + + pub(crate) async fn exclude_required_subtask(&self) -> OwnedMutexGuard<()> { + Arc::clone(&self.phase_exclusion).lock_owned().await + } +} + pub(crate) struct LoopHeartbeat { instance_id: String, interval: Duration, last_recorded_at: Option, + #[cfg(test)] + progress_record_count: usize, } impl LoopHeartbeat { pub(crate) fn new(instance_id: String, interval: Duration) -> Self { Self { instance_id, - interval, + interval: interval.min(MAX_PROGRESS_HEARTBEAT_INTERVAL), last_recorded_at: None, + #[cfg(test)] + progress_record_count: 0, } } @@ -33,7 +64,13 @@ impl LoopHeartbeat { ) .await; match result { - Ok(()) => self.last_recorded_at = Some(Instant::now()), + Ok(()) => { + self.last_recorded_at = Some(Instant::now()); + #[cfg(test)] + { + self.progress_record_count += 1; + } + } Err(error) => warn!( service = "worker", heartbeat_instance_id = %self.instance_id, @@ -123,6 +160,11 @@ impl LoopHeartbeat { .map(|recorded_at| recorded_at.elapsed() >= self.interval) .unwrap_or(true) } + + #[cfg(test)] + pub(crate) const fn progress_record_count(&self) -> usize { + self.progress_record_count + } } pub(crate) async fn record_rebuild_progress( @@ -184,6 +226,13 @@ mod tests { assert!(!heartbeat.is_due()); } + #[test] + fn progress_heartbeat_throttle_never_inherits_a_stale_poll_interval() { + let heartbeat = LoopHeartbeat::new("long-poll-test".to_owned(), Duration::from_secs(60)); + + assert_eq!(heartbeat.interval, Duration::from_secs(5)); + } + #[tokio::test] async fn heartbeat_write_failure_is_warn_and_continue() { let pool = PgPool::connect_lazy("postgres://bigname:bigname@127.0.0.1:5432/bigname") diff --git a/apps/worker/src/record_inventory/projection.rs b/apps/worker/src/record_inventory/projection.rs index 66d4ffcc..7d4ceb54 100644 --- a/apps/worker/src/record_inventory/projection.rs +++ b/apps/worker/src/record_inventory/projection.rs @@ -101,7 +101,7 @@ async fn rebuild_record_inventory_current_inner( loop_heartbeat: Option<&mut LoopHeartbeat>, ) -> Result { match resource_id { - Some(resource_id) => rebuild_one_resource(pool, resource_id).await, + Some(resource_id) => rebuild_one_resource(pool, resource_id, loop_heartbeat).await, None => rebuild_all_resources(pool, loop_heartbeat).await, } } @@ -220,13 +220,16 @@ async fn build_row_for_resource( async fn rebuild_one_resource( pool: &PgPool, resource_id: &str, + mut loop_heartbeat: Option<&mut LoopHeartbeat>, ) -> Result { let resource_id = Uuid::parse_str(resource_id) .with_context(|| format!("resource_id must be a UUID: {resource_id}"))?; let events = load_relevant_events(pool, resource_id).await?; + record_rebuild_progress(pool, &mut loop_heartbeat).await; if events.is_empty() { let deleted_row_count = delete_record_inventory_rows_for_resource(pool, resource_id).await?; + record_rebuild_progress(pool, &mut loop_heartbeat).await; return Ok(RecordInventoryCurrentRebuildSummary { requested_resource_count: 1, upserted_row_count: 0, @@ -235,21 +238,26 @@ async fn rebuild_one_resource( }; let profile_gate = ResolverProfileGate::load_for_events(pool, &events).await?; + record_rebuild_progress(pool, &mut loop_heartbeat).await; let Some(row) = build_row_from_events(pool, &profile_gate, resource_id, &events).await? else { let deleted_row_count = delete_record_inventory_rows_for_resource(pool, resource_id).await?; + record_rebuild_progress(pool, &mut loop_heartbeat).await; return Ok(RecordInventoryCurrentRebuildSummary { requested_resource_count: 1, upserted_row_count: 0, deleted_row_count, }); }; + record_rebuild_progress(pool, &mut loop_heartbeat).await; let upserted_row_count = upsert_record_inventory_current_rows(pool, std::slice::from_ref(&row)) .await? .len(); + record_rebuild_progress(pool, &mut loop_heartbeat).await; let deleted_row_count = delete_stale_record_inventory_current_rows_for_resource(pool, resource_id, &row).await?; + record_rebuild_progress(pool, &mut loop_heartbeat).await; Ok(RecordInventoryCurrentRebuildSummary { requested_resource_count: 1, upserted_row_count, diff --git a/apps/worker/src/resolver.rs b/apps/worker/src/resolver.rs index e2c58da5..e1ea78c9 100644 --- a/apps/worker/src/resolver.rs +++ b/apps/worker/src/resolver.rs @@ -22,7 +22,7 @@ mod summary_json; mod target_loading; use profile::ResolverProfileGate; -use summary_json::build_resolver_current_row; +use summary_json::{build_resolver_current_row, build_resolver_current_row_with_progress}; use target_loading::{ ResolverTarget, count_current_binding_candidate_pairs, load_target_resolvers, normalize_resolver_address, @@ -103,7 +103,7 @@ async fn rebuild_resolver_current_inner( ) -> Result { match (chain_id, resolver_address) { (Some(chain_id), Some(resolver_address)) => { - rebuild_one_resolver(pool, chain_id, resolver_address).await + rebuild_one_resolver(pool, chain_id, resolver_address, loop_heartbeat).await } (None, None) => rebuild_all_resolvers(pool, loop_heartbeat).await, _ => bail!( @@ -230,6 +230,7 @@ async fn rebuild_one_resolver( pool: &PgPool, chain_id: &str, resolver_address: &str, + mut loop_heartbeat: Option<&mut LoopHeartbeat>, ) -> Result { let resolver_address = normalize_resolver_address(resolver_address); let target = ResolverTarget { @@ -242,10 +243,16 @@ async fn rebuild_one_resolver( enumerate_bindings: should_enumerate_targeted_resolver_bindings(pool, &target).await?, ..target }; + record_rebuild_progress(pool, &mut loop_heartbeat).await; let profile_gate = ResolverProfileGate::load_for_target(pool, &target).await?; - let Some(row) = build_resolver_current_row(pool, &profile_gate, &target).await? else { + record_rebuild_progress(pool, &mut loop_heartbeat).await; + let Some(row) = + build_resolver_current_row_with_progress(pool, &profile_gate, &target, &mut loop_heartbeat) + .await? + else { let deleted_row_count = delete_resolver_current(pool, &target.chain_id, &target.resolver_address).await?; + record_rebuild_progress(pool, &mut loop_heartbeat).await; return Ok(ResolverCurrentRebuildSummary { requested_resolver_count: 1, upserted_row_count: 0, @@ -254,6 +261,7 @@ async fn rebuild_one_resolver( }; let upserted_row_count = upsert_resolver_current_rows(pool, &[row]).await?.len(); + record_rebuild_progress(pool, &mut loop_heartbeat).await; Ok(ResolverCurrentRebuildSummary { requested_resolver_count: 1, upserted_row_count, diff --git a/apps/worker/src/resolver/summary_json.rs b/apps/worker/src/resolver/summary_json.rs index 97a3ae7c..1e94cc7f 100644 --- a/apps/worker/src/resolver/summary_json.rs +++ b/apps/worker/src/resolver/summary_json.rs @@ -6,6 +6,7 @@ use serde_json::{Value, json}; use sqlx::PgPool; use sqlx::types::time::OffsetDateTime; +use crate::primary_name::rebuild_heartbeat::{LoopHeartbeat, record_rebuild_progress}; use crate::projection_json::{ dedupe_json_values, json_array_field, json_string_array_field, projection_coverage, }; @@ -34,6 +35,15 @@ pub(super) async fn build_resolver_current_row( pool: &PgPool, profile_gate: &ResolverProfileGate, target: &ResolverTarget, +) -> Result> { + build_resolver_current_row_with_progress(pool, profile_gate, target, &mut None).await +} + +pub(super) async fn build_resolver_current_row_with_progress( + pool: &PgPool, + profile_gate: &ResolverProfileGate, + target: &ResolverTarget, + loop_heartbeat: &mut Option<&mut LoopHeartbeat>, ) -> Result> { let skip_known_binding_enumeration = profile_gate.skips_binding_enumeration(target); let hinted_target_status = target @@ -57,16 +67,19 @@ pub(super) async fn build_resolver_current_row( } else { load_current_bindings(pool, target).await? }; + record_rebuild_progress(pool, loop_heartbeat).await; let aliases = if skip_binding_enumeration { Vec::new() } else { load_alias_events(pool, target).await? }; + record_rebuild_progress(pool, loop_heartbeat).await; let permissions = if skip_permission_enumeration { Vec::new() } else { load_resolver_permissions(pool, target).await? }; + record_rebuild_progress(pool, loop_heartbeat).await; if bindings.is_empty() && aliases.is_empty() && permissions.is_empty() diff --git a/crates/adapters/src/block_derived_normalized_events.rs b/crates/adapters/src/block_derived_normalized_events.rs index e968f2ad..c5836ec1 100644 --- a/crates/adapters/src/block_derived_normalized_events.rs +++ b/crates/adapters/src/block_derived_normalized_events.rs @@ -20,9 +20,16 @@ mod preimage_observation; mod source_selection; mod types; -use crate::normalized_event_support::{ - NormalizedEventSyncCounts, count_events_by_kind, upsert_normalized_events_with_counts, - upsert_normalized_events_with_stateless_replay_authority_counts, +use crate::{ + checkpoint_context::{StartupAdapterProgress, record_startup_adapter_progress}, + normalized_event_support::{ + NormalizedEventSyncCounts, count_events_by_kind, + upsert_normalized_events_in_chunks_with_counts_and_progress, + upsert_normalized_events_in_chunks_with_stateless_replay_authority_counts_and_progress, + upsert_normalized_events_with_counts, + upsert_normalized_events_with_stateless_replay_authority_counts, + }, + startup_progress::{STARTUP_ADAPTER_PROGRESS_PAGE_ROWS, record_processed_row_progress}, }; use event_builders::build_preimage_observed_events; use loading::{RawLogCanonicalityFilter, load_scanned_log_count, load_watched_raw_logs}; @@ -61,6 +68,28 @@ pub async fn sync_block_derived_normalized_events( RawLogCanonicalityFilter::IncludeObserved, None, false, + None, + ) + .await + .map(|(summary, _)| summary) +} + +pub async fn sync_block_derived_normalized_events_with_progress( + pool: &PgPool, + chain: &str, + block_hashes: &[String], + source_scope: Option<&[(String, String, i64, i64)]>, + progress: &mut dyn StartupAdapterProgress, +) -> Result { + sync_block_derived_normalized_events_inner( + pool, + chain, + block_hashes, + source_scope, + RawLogCanonicalityFilter::IncludeObserved, + None, + false, + Some(progress), ) .await .map(|(summary, _)| summary) @@ -83,6 +112,29 @@ pub async fn sync_block_derived_normalized_events_with_scanned_log_count( RawLogCanonicalityFilter::CanonicalOnly, Some(scanned_log_count), false, + None, + ) + .await + .map(|(summary, _)| summary) +} + +pub async fn sync_block_derived_normalized_events_with_scanned_log_count_and_progress( + pool: &PgPool, + chain: &str, + block_hashes: &[String], + source_scope: Option<&[(String, String, i64, i64)]>, + scanned_log_count: usize, + progress: &mut dyn StartupAdapterProgress, +) -> Result { + sync_block_derived_normalized_events_inner( + pool, + chain, + block_hashes, + source_scope, + RawLogCanonicalityFilter::CanonicalOnly, + Some(scanned_log_count), + false, + Some(progress), ) .await .map(|(summary, _)| summary) @@ -107,6 +159,31 @@ pub async fn sync_block_derived_normalized_events_with_stateless_replay_authorit RawLogCanonicalityFilter::CanonicalOnly, Some(scanned_log_count), true, + None, + ) + .await +} + +pub async fn sync_block_derived_normalized_events_with_stateless_replay_authority_and_progress( + pool: &PgPool, + chain: &str, + block_hashes: &[String], + source_scope: Option<&[(String, String, i64, i64)]>, + scanned_log_count: usize, + progress: &mut dyn StartupAdapterProgress, +) -> Result<( + BlockDerivedNormalizedEventSyncSummary, + NormalizedEventReplayAuthoritySummary, +)> { + sync_block_derived_normalized_events_inner( + pool, + chain, + block_hashes, + source_scope, + RawLogCanonicalityFilter::CanonicalOnly, + Some(scanned_log_count), + true, + Some(progress), ) .await } @@ -119,6 +196,7 @@ async fn sync_block_derived_normalized_events_inner( canonicality_filter: RawLogCanonicalityFilter, known_scanned_log_count: Option, stateless_replay_authority: bool, + mut progress: Option<&mut dyn StartupAdapterProgress>, ) -> Result<( BlockDerivedNormalizedEventSyncSummary, NormalizedEventReplayAuthoritySummary, @@ -142,6 +220,7 @@ async fn sync_block_derived_normalized_events_inner( let watched_raw_logs_started = Instant::now(); let raw_log_load = load_watched_raw_logs(pool, chain, block_hashes, source_scope, canonicality_filter).await?; + record_startup_adapter_progress(pool, &mut progress).await?; let load_watched_raw_logs_ms = watched_raw_logs_started.elapsed().as_millis(); let raw_logs = raw_log_load.raw_logs; if raw_logs.is_empty() { @@ -167,7 +246,7 @@ async fn sync_block_derived_normalized_events_inner( let mut matched_log_refs = HashSet::new(); let mut events = Vec::new(); let mut build_by_source_family = BTreeMap::::new(); - for raw_log in &raw_logs { + for (index, raw_log) in raw_logs.iter().enumerate() { let row_started = Instant::now(); let observed_events = build_preimage_observed_events(raw_log, &raw_log_load.event_topics)?; let elapsed_us = row_started.elapsed().as_micros(); @@ -177,6 +256,7 @@ async fn sync_block_derived_normalized_events_inner( source_family_timing.raw_log_count += 1; source_family_timing.elapsed_us += elapsed_us; if observed_events.is_empty() { + record_processed_row_progress(pool, &mut progress, index + 1, raw_logs.len()).await?; continue; } source_family_timing.matched_log_count += 1; @@ -188,6 +268,7 @@ async fn sync_block_derived_normalized_events_inner( raw_log.log_index, )); events.extend(observed_events); + record_processed_row_progress(pool, &mut progress, index + 1, raw_logs.len()).await?; } let build_events_ms = build_started.elapsed().as_millis(); @@ -213,18 +294,48 @@ async fn sync_block_derived_normalized_events_inner( let event_kind_counts = count_events_by_kind(&events); let persistence_started = Instant::now(); let (counts, authority) = if stateless_replay_authority { - upsert_normalized_events_with_stateless_replay_authority_counts( - pool, - &events, - "block-derived normalized-event", - ) - .await? + match progress { + Some(progress) => { + upsert_normalized_events_in_chunks_with_stateless_replay_authority_counts_and_progress( + pool, + &events, + "block-derived normalized-event", + STARTUP_ADAPTER_PROGRESS_PAGE_ROWS, + Some(progress), + ) + .await? + } + None => { + upsert_normalized_events_with_stateless_replay_authority_counts( + pool, + &events, + "block-derived normalized-event", + ) + .await? + } + } } else { - ( - upsert_normalized_events_with_counts(pool, &events, "block-derived normalized-event") - .await?, - NormalizedEventReplayAuthoritySummary::default(), - ) + let counts = match progress { + Some(progress) => { + upsert_normalized_events_in_chunks_with_counts_and_progress( + pool, + &events, + "block-derived normalized-event", + STARTUP_ADAPTER_PROGRESS_PAGE_ROWS, + Some(progress), + ) + .await? + } + None => { + upsert_normalized_events_with_counts( + pool, + &events, + "block-derived normalized-event", + ) + .await? + } + }; + (counts, NormalizedEventReplayAuthoritySummary::default()) }; let persistence_ms = persistence_started.elapsed().as_millis(); diff --git a/crates/adapters/src/checkpoint_context.rs b/crates/adapters/src/checkpoint_context.rs index 6d6ad6ed..3353eacb 100644 --- a/crates/adapters/src/checkpoint_context.rs +++ b/crates/adapters/src/checkpoint_context.rs @@ -27,6 +27,15 @@ pub(crate) async fn record_startup_adapter_progress( Ok(()) } +pub(crate) fn reborrow_startup_adapter_progress<'a>( + progress: &'a mut Option<&mut dyn StartupAdapterProgress>, +) -> Option<&'a mut dyn StartupAdapterProgress> { + match progress.as_mut() { + Some(progress) => Some(&mut **progress), + None => None, + } +} + #[derive(Clone, Debug, Eq, PartialEq)] pub struct ReplayAdapterCheckpointContext { pub deployment_profile: String, diff --git a/crates/adapters/src/ens_v1_reverse_claim.rs b/crates/adapters/src/ens_v1_reverse_claim.rs index 39ed45a9..991f47f1 100644 --- a/crates/adapters/src/ens_v1_reverse_claim.rs +++ b/crates/adapters/src/ens_v1_reverse_claim.rs @@ -1,6 +1,6 @@ use std::collections::{BTreeMap, HashSet}; -use anyhow::{Result, ensure}; +use anyhow::Result; use bigname_storage::NormalizedEventReplayAuthoritySummary; use sqlx::PgPool; @@ -10,6 +10,7 @@ use crate::{ upsert_normalized_events_in_chunks_with_counts, upsert_normalized_events_in_chunks_with_counts_and_progress, upsert_normalized_events_in_chunks_with_stateless_replay_authority_counts, + upsert_normalized_events_in_chunks_with_stateless_replay_authority_counts_and_progress, }, startup_progress::{STARTUP_ADAPTER_PROGRESS_PAGE_ROWS, record_processed_row_progress}, }; @@ -115,6 +116,82 @@ impl EnsV1ReverseClaimSyncSummary { ) .await } + + pub async fn sync_for_block_hashes_with_stateless_replay_authority_and_progress( + pool: &PgPool, + chain: &str, + block_hashes: &[String], + progress: &mut dyn StartupAdapterProgress, + ) -> Result<(Self, NormalizedEventReplayAuthoritySummary)> { + sync_ens_v1_reverse_claim_with_scope( + pool, + chain, + true, + block_hashes, + None, + true, + Some(progress), + ) + .await + } + + pub async fn sync_for_block_hashes_with_source_scope_and_stateless_replay_authority_and_progress( + pool: &PgPool, + chain: &str, + block_hashes: &[String], + source_scope: &[(String, String, i64, i64)], + progress: &mut dyn StartupAdapterProgress, + ) -> Result<(Self, NormalizedEventReplayAuthoritySummary)> { + sync_ens_v1_reverse_claim_with_scope( + pool, + chain, + true, + block_hashes, + Some(source_scope), + true, + Some(progress), + ) + .await + } + + pub async fn sync_for_block_hashes_with_progress( + pool: &PgPool, + chain: &str, + block_hashes: &[String], + progress: &mut dyn StartupAdapterProgress, + ) -> Result { + sync_ens_v1_reverse_claim_with_scope( + pool, + chain, + true, + block_hashes, + None, + false, + Some(progress), + ) + .await + .map(|(summary, _)| summary) + } + + pub async fn sync_for_block_hashes_with_source_scope_and_progress( + pool: &PgPool, + chain: &str, + block_hashes: &[String], + source_scope: &[(String, String, i64, i64)], + progress: &mut dyn StartupAdapterProgress, + ) -> Result { + sync_ens_v1_reverse_claim_with_scope( + pool, + chain, + true, + block_hashes, + Some(source_scope), + false, + Some(progress), + ) + .await + .map(|(summary, _)| summary) + } } pub async fn sync_ens_v1_reverse_claim( @@ -217,17 +294,27 @@ async fn sync_ens_v1_reverse_claim_with_scope( } let (counts, authority) = if stateless_replay_authority { - ensure!( - progress.is_none(), - "stateless replay authority is unavailable to checkpointed reverse-claim sync" - ); - upsert_normalized_events_in_chunks_with_stateless_replay_authority_counts( - pool, - &events, - "ENSv1 reverse normalized-event", - 10_000, - ) - .await? + match progress { + Some(progress) => { + upsert_normalized_events_in_chunks_with_stateless_replay_authority_counts_and_progress( + pool, + &events, + "ENSv1 reverse normalized-event", + STARTUP_ADAPTER_PROGRESS_PAGE_ROWS, + Some(progress), + ) + .await? + } + None => { + upsert_normalized_events_in_chunks_with_stateless_replay_authority_counts( + pool, + &events, + "ENSv1 reverse normalized-event", + 10_000, + ) + .await? + } + } } else { let counts = match progress { Some(progress) => { diff --git a/crates/adapters/src/ens_v1_subregistry_discovery.rs b/crates/adapters/src/ens_v1_subregistry_discovery.rs index 9ef907c5..fe708246 100644 --- a/crates/adapters/src/ens_v1_subregistry_discovery.rs +++ b/crates/adapters/src/ens_v1_subregistry_discovery.rs @@ -1,6 +1,6 @@ use std::collections::BTreeMap; -use anyhow::{Context, Result, ensure}; +use anyhow::{Result, ensure}; use bigname_manifests::{FullDiscoveryReconciliationOptions, reconcile_discovery_observations}; use sqlx::PgPool; @@ -9,6 +9,7 @@ use crate::checkpoint_context::{ }; use crate::registry_migration_cache::MigratedRegistryNodes; +mod application; mod assignment; mod checkpoint; mod constants; @@ -25,6 +26,7 @@ mod scope; mod startup; mod types; +use application::apply_registry_raw_logs; use assignment::{ ObservedRegistryAssignment, build_registry_assignment, ens_v1_resolver_discovery_source, ens_v1_subregistry_discovery_source, @@ -32,7 +34,7 @@ use assignment::{ use checkpoint::SubregistryReplayCheckpoint; use constants::*; use emitter::{emit_registry_changed_events, emit_registry_changed_events_from_checkpoint}; -use hex_topic::{ZERO_ADDRESS, normalize_address}; +use hex_topic::ZERO_ADDRESS; use loader::{ load_active_emitters, load_registry_raw_log_checkpoint_page, load_registry_raw_logs, stream_registry_raw_logs, stream_registry_raw_logs_through_block, @@ -42,8 +44,9 @@ use migration_guard::{ }; use mode::{DiscoveryEdgeMutation, EnsV1SubregistryDiscoverySyncOutcome}; use reconciliation::{ + count_active_assignments_with_progress, + reconcile_subregistry_discovery_from_assignments_through_block, reconcile_subregistry_discovery_from_checkpoint, - reconcile_subregistry_discovery_source_through_block, }; use scope::{load_migrated_registry_nodes_before_block, normalized_registry_source_scope_targets}; pub use types::EnsV1SubregistryDiscoverySyncSummary; @@ -55,10 +58,12 @@ pub use checkpoint::clear_replay_adapter_checkpoints; pub use entrypoints::{ sync_ens_v1_subregistry_discovery, sync_ens_v1_subregistry_discovery_through_block, sync_ens_v1_subregistry_discovery_through_block_with_expected_admission_epoch, + sync_ens_v1_subregistry_discovery_through_block_with_expected_admission_epoch_and_progress, }; pub use replay::{ sync_ens_v1_subregistry_discovery_with_replay_checkpoint, sync_ens_v1_subregistry_discovery_with_replay_checkpoint_and_log_limit, + sync_ens_v1_subregistry_discovery_with_replay_checkpoint_and_log_limit_and_progress, }; pub use startup::{ sync_ens_v1_subregistry_discovery_with_startup_checkpoint_and_log_limit, @@ -115,8 +120,10 @@ async fn sync_ens_v1_subregistry_discovery_with_scope( chain, source_scope.as_deref(), full_source_through_block.is_some(), + startup_progress, ) .await?; + record_startup_adapter_progress(pool, startup_progress).await?; let current_registry = current_registry_emitter(&emitters, full_source_through_block).cloned(); let discovery_sources = [ ens_v1_subregistry_discovery_source(chain), @@ -179,6 +186,7 @@ async fn sync_ens_v1_subregistry_discovery_with_scope( &emitters, through_block, checkpoint_page_limit, + startup_progress, |raw_log| { let applied = apply_registry_raw_log( &raw_log, @@ -245,12 +253,15 @@ async fn sync_ens_v1_subregistry_discovery_with_scope( } } matched_log_count += apply_registry_raw_logs( + pool, &raw_logs, chain, current_registry.as_ref(), &mut latest_assignments, &mut migrated_registry_nodes, - )?; + startup_progress, + ) + .await?; } let finalize_from_checkpoint = active_checkpoint @@ -284,10 +295,7 @@ async fn sync_ens_v1_subregistry_discovery_with_scope( .active_assignment_count(pool, &discovery_sources) .await? } else { - latest_assignments - .values() - .filter(|assignment| normalize_address(&assignment.to_address) != ZERO_ADDRESS) - .count() + count_active_assignments_with_progress(pool, &latest_assignments, startup_progress).await? }; let mut reconciliation = EnsV1SubregistryDiscoverySyncSummary { @@ -348,45 +356,37 @@ async fn sync_ens_v1_subregistry_discovery_with_scope( reconciliation.deactivated_edge_count += source_reconciliation.deactivated_edge_count; } + } else if let Some(through_block) = full_source_through_block { + reconcile_subregistry_discovery_from_assignments_through_block( + pool, + chain, + &latest_assignments, + &discovery_sources, + through_block, + full_source_expected_admission_epoch, + &mut reconciliation, + startup_progress, + ) + .await?; } else { - let mut expected_admission_epoch = full_source_expected_admission_epoch; for discovery_source in &discovery_sources { let source_observations = latest_assignments .values() .filter(|assignment| assignment.discovery_source == discovery_source.as_str()) .map(ObservedRegistryAssignment::discovery_observation) .collect::>>()?; - let source_reconciliation = if let Some(through_block) = full_source_through_block { - reconcile_subregistry_discovery_source_through_block( - pool, - chain, - discovery_source, - &source_observations, - through_block, - expected_admission_epoch, - ) - .await? - } else { - reconcile_discovery_observations( - pool, - discovery_source, - &source_observations, - FullDiscoveryReconciliationOptions::default(), - ) - .await? - }; + let source_reconciliation = reconcile_discovery_observations( + pool, + discovery_source, + &source_observations, + FullDiscoveryReconciliationOptions::default(), + ) + .await?; reconciliation.active_edge_count += source_reconciliation.active_edge_count; reconciliation.admitted_edge_count += source_reconciliation.admitted_edge_count; reconciliation.inserted_edge_count += source_reconciliation.inserted_edge_count; reconciliation.deactivated_edge_count += source_reconciliation.deactivated_edge_count; - if let Some(expected_epoch) = expected_admission_epoch.as_mut() { - *expected_epoch = expected_epoch - .checked_add(i64::try_from( - source_reconciliation.admission_epoch_bump_count, - )?) - .context("legacy registry reconciliation admission epoch overflowed")?; - } } } } @@ -402,7 +402,13 @@ async fn sync_ens_v1_subregistry_discovery_with_scope( ) .await? } else { - emit_registry_changed_events(pool, &latest_assignments, &discovery_sources).await? + emit_registry_changed_events( + pool, + &latest_assignments, + &discovery_sources, + startup_progress, + ) + .await? }; reconciliation.total_normalized_event_count = event_summary.synced_count; reconciliation.total_normalized_event_inserted_count = event_summary.inserted_count; @@ -512,30 +518,6 @@ async fn sync_checkpointed_registry_raw_logs( Ok((scanned_log_count, matched_log_count)) } -fn apply_registry_raw_logs( - raw_logs: &[loader::RegistryRawLogRow], - chain: &str, - current_registry: Option<&loader::ActiveEmitter>, - latest_assignments: &mut BTreeMap, - migrated_registry_nodes: &mut MigratedRegistryNodes, -) -> Result { - let mut matched_log_count = 0; - for raw_log in raw_logs { - if apply_registry_raw_log( - raw_log, - chain, - current_registry, - latest_assignments, - migrated_registry_nodes, - )? - .matched - { - matched_log_count += 1; - } - } - Ok(matched_log_count) -} - #[derive(Clone, Debug, Default, Eq, PartialEq)] struct AppliedRegistryRawLog { matched: bool, diff --git a/crates/adapters/src/ens_v1_subregistry_discovery/application.rs b/crates/adapters/src/ens_v1_subregistry_discovery/application.rs new file mode 100644 index 00000000..84882058 --- /dev/null +++ b/crates/adapters/src/ens_v1_subregistry_discovery/application.rs @@ -0,0 +1,34 @@ +use super::*; + +pub(super) async fn apply_registry_raw_logs( + pool: &PgPool, + raw_logs: &[loader::RegistryRawLogRow], + chain: &str, + current_registry: Option<&loader::ActiveEmitter>, + latest_assignments: &mut BTreeMap, + migrated_registry_nodes: &mut MigratedRegistryNodes, + startup_progress: &mut Option<&mut dyn StartupAdapterProgress>, +) -> Result { + let mut matched_log_count = 0; + for (index, raw_log) in raw_logs.iter().enumerate() { + if apply_registry_raw_log( + raw_log, + chain, + current_registry, + latest_assignments, + migrated_registry_nodes, + )? + .matched + { + matched_log_count += 1; + } + crate::startup_progress::record_processed_row_progress( + pool, + startup_progress, + index + 1, + raw_logs.len(), + ) + .await?; + } + Ok(matched_log_count) +} diff --git a/crates/adapters/src/ens_v1_subregistry_discovery/emitter.rs b/crates/adapters/src/ens_v1_subregistry_discovery/emitter.rs index e358c664..d7487e54 100644 --- a/crates/adapters/src/ens_v1_subregistry_discovery/emitter.rs +++ b/crates/adapters/src/ens_v1_subregistry_discovery/emitter.rs @@ -26,8 +26,8 @@ pub(super) async fn emit_registry_changed_events( pool: &PgPool, latest_assignments: &BTreeMap, discovery_sources: &[String], + startup_progress: &mut Option<&mut dyn StartupAdapterProgress>, ) -> Result { - let mut startup_progress = None; let discovery_sources = discovery_sources .iter() .map(String::as_str) @@ -46,7 +46,7 @@ pub(super) async fn emit_registry_changed_events( &assignments, &mut events, &mut summary, - &mut startup_progress, + startup_progress, ) .await?; assignments.clear(); @@ -57,10 +57,10 @@ pub(super) async fn emit_registry_changed_events( &assignments, &mut events, &mut summary, - &mut startup_progress, + startup_progress, ) .await?; - flush_registry_changed_events(pool, &mut events, &mut summary, &mut startup_progress).await?; + flush_registry_changed_events(pool, &mut events, &mut summary, startup_progress).await?; Ok(summary) } diff --git a/crates/adapters/src/ens_v1_subregistry_discovery/entrypoints.rs b/crates/adapters/src/ens_v1_subregistry_discovery/entrypoints.rs index ace8577f..4bf28aef 100644 --- a/crates/adapters/src/ens_v1_subregistry_discovery/entrypoints.rs +++ b/crates/adapters/src/ens_v1_subregistry_discovery/entrypoints.rs @@ -5,6 +5,7 @@ use super::{ DiscoveryEdgeMutation, EnsV1SubregistryDiscoverySyncSummary, checkpoint, sync_ens_v1_subregistry_discovery_with_scope, }; +use crate::checkpoint_context::StartupAdapterProgress; pub async fn sync_ens_v1_subregistry_discovery( pool: &PgPool, @@ -83,6 +84,31 @@ pub async fn sync_ens_v1_subregistry_discovery_through_block_with_expected_admis .map(|(summary, _, _)| summary) } +pub async fn sync_ens_v1_subregistry_discovery_through_block_with_expected_admission_epoch_and_progress( + pool: &PgPool, + chain: &str, + through_block_number: i64, + expected_admission_epoch: i64, + progress: &mut dyn StartupAdapterProgress, +) -> Result { + sync_ens_v1_subregistry_discovery_with_scope( + pool, + chain, + false, + &[], + None, + DiscoveryEdgeMutation::Reconcile, + Some(through_block_number), + Some(expected_admission_epoch), + None, + checkpoint::PAGE_LIMIT, + None, + &mut Some(progress), + ) + .await + .map(|(summary, _, _)| summary) +} + impl EnsV1SubregistryDiscoverySyncSummary { pub async fn sync_for_block_hashes_with_source_scope( pool: &PgPool, @@ -108,6 +134,31 @@ impl EnsV1SubregistryDiscoverySyncSummary { .map(|(summary, _, _)| summary) } + pub async fn sync_for_block_hashes_with_source_scope_and_progress( + pool: &PgPool, + chain: &str, + block_hashes: &[String], + source_scope: &[(String, String, i64, i64)], + progress: &mut dyn StartupAdapterProgress, + ) -> Result { + sync_ens_v1_subregistry_discovery_with_scope( + pool, + chain, + true, + block_hashes, + Some(source_scope), + DiscoveryEdgeMutation::Reconcile, + None, + None, + None, + checkpoint::PAGE_LIMIT, + None, + &mut Some(progress), + ) + .await + .map(|(summary, _, _)| summary) + } + pub async fn sync_for_block_hashes_with_source_scope_without_discovery_reconciliation( pool: &PgPool, chain: &str, @@ -132,6 +183,31 @@ impl EnsV1SubregistryDiscoverySyncSummary { .map(|(summary, _, _)| summary) } + pub async fn sync_for_block_hashes_with_source_scope_without_discovery_reconciliation_and_progress( + pool: &PgPool, + chain: &str, + block_hashes: &[String], + source_scope: &[(String, String, i64, i64)], + progress: &mut dyn StartupAdapterProgress, + ) -> Result { + sync_ens_v1_subregistry_discovery_with_scope( + pool, + chain, + true, + block_hashes, + Some(source_scope), + DiscoveryEdgeMutation::Skip, + None, + None, + None, + checkpoint::PAGE_LIMIT, + None, + &mut Some(progress), + ) + .await + .map(|(summary, _, _)| summary) + } + pub async fn sync_for_block_hashes_without_discovery_reconciliation( pool: &PgPool, chain: &str, @@ -154,4 +230,28 @@ impl EnsV1SubregistryDiscoverySyncSummary { .await .map(|(summary, _, _)| summary) } + + pub async fn sync_for_block_hashes_without_discovery_reconciliation_and_progress( + pool: &PgPool, + chain: &str, + block_hashes: &[String], + progress: &mut dyn StartupAdapterProgress, + ) -> Result { + sync_ens_v1_subregistry_discovery_with_scope( + pool, + chain, + true, + block_hashes, + None, + DiscoveryEdgeMutation::Skip, + None, + None, + None, + checkpoint::PAGE_LIMIT, + None, + &mut Some(progress), + ) + .await + .map(|(summary, _, _)| summary) + } } diff --git a/crates/adapters/src/ens_v1_subregistry_discovery/loader/active_emitters.rs b/crates/adapters/src/ens_v1_subregistry_discovery/loader/active_emitters.rs index 400f4876..f121c120 100644 --- a/crates/adapters/src/ens_v1_subregistry_discovery/loader/active_emitters.rs +++ b/crates/adapters/src/ens_v1_subregistry_discovery/loader/active_emitters.rs @@ -4,17 +4,27 @@ use super::super::{ }; use super::ActiveEmitter; use anyhow::{Context, Result}; +use futures_util::TryStreamExt; use sqlx::PgPool; +use crate::{ + checkpoint_context::{StartupAdapterProgress, record_startup_adapter_progress}, + startup_progress::STARTUP_ADAPTER_PROGRESS_PAGE_ROWS, +}; + mod rows; -use rows::{active_emitters_from_rows, sort_active_emitters, source_scope_covered_by_emitters}; +use rows::{ + active_emitters_from_rows, active_emitters_from_rows_with_progress, sort_active_emitters, + source_scope_covered_by_emitters, +}; pub(in crate::ens_v1_subregistry_discovery) async fn load_active_emitters( pool: &PgPool, chain: &str, source_scope: Option<&[RegistryRawLogSourceScopeTarget]>, include_historical: bool, + progress: &mut Option<&mut dyn StartupAdapterProgress>, ) -> Result> { let has_source_scope = source_scope.is_some(); let source_scope = source_scope.unwrap_or(&[]); @@ -39,7 +49,7 @@ pub(in crate::ens_v1_subregistry_discovery) async fn load_active_emitters( return Ok(manifest_emitters); } - let rows = sqlx::query( + let mut rows = sqlx::query( r#" SELECT chain, @@ -119,7 +129,7 @@ pub(in crate::ens_v1_subregistry_discovery) async fn load_active_emitters( ) ) - UNION + UNION ALL SELECT de.chain_id AS chain, @@ -177,7 +187,6 @@ pub(in crate::ens_v1_subregistry_discovery) async fn load_active_emitters( OR cia.active_from_block_number <= de.active_to_block_number ) ) registry_emitters - ORDER BY lower(address), source_rank, source_manifest_id, contract_instance_id "#, ) .bind(chain) @@ -188,11 +197,34 @@ pub(in crate::ens_v1_subregistry_discovery) async fn load_active_emitters( .bind(&scoped_source_families) .bind(&scoped_addresses) .bind(include_historical) - .fetch_all(pool) - .await - .with_context(|| format!("failed to load active ENSv1 registry emitters for {chain}"))?; - - active_emitters_from_rows(rows) + .fetch(pool); + let mut loaded_rows = Vec::new(); + while let Some(row) = rows + .try_next() + .await + .with_context(|| format!("failed to stream active ENSv1 registry emitters for {chain}"))? + { + loaded_rows.push(row); + if loaded_rows + .len() + .is_multiple_of(STARTUP_ADAPTER_PROGRESS_PAGE_ROWS) + { + record_startup_adapter_progress(pool, progress).await?; + } + } + if !loaded_rows.is_empty() + && !loaded_rows + .len() + .is_multiple_of(STARTUP_ADAPTER_PROGRESS_PAGE_ROWS) + { + record_startup_adapter_progress(pool, progress).await?; + } + match progress.as_deref_mut() { + Some(progress) => { + active_emitters_from_rows_with_progress(pool, loaded_rows, progress).await + } + None => active_emitters_from_rows(loaded_rows), + } } async fn load_scoped_discovery_active_emitters( diff --git a/crates/adapters/src/ens_v1_subregistry_discovery/loader/active_emitters/rows.rs b/crates/adapters/src/ens_v1_subregistry_discovery/loader/active_emitters/rows.rs index a243de6d..dc12d692 100644 --- a/crates/adapters/src/ens_v1_subregistry_discovery/loader/active_emitters/rows.rs +++ b/crates/adapters/src/ens_v1_subregistry_discovery/loader/active_emitters/rows.rs @@ -1,4 +1,4 @@ -use std::collections::HashMap; +use std::collections::{BTreeMap, HashMap}; use anyhow::{Context, Result}; use sqlx::{Row, postgres::PgRow}; @@ -8,42 +8,16 @@ use super::super::super::{ hex_topic::normalize_address, scope::RegistryRawLogSourceScopeTarget, }; use super::super::ActiveEmitter; +use crate::{ + checkpoint_context::StartupAdapterProgress, + startup_progress::STARTUP_ADAPTER_PROGRESS_PAGE_ROWS, +}; pub(super) fn active_emitters_from_rows(rows: Vec) -> Result> { let mut emitters_by_scope = HashMap::<(String, String, Option, Option), ActiveEmitter>::new(); for row in rows { - let address = normalize_address(&row.try_get::("address")?); - let candidate = ActiveEmitter { - address, - contract_instance_id: row - .try_get("contract_instance_id") - .context("missing registry emitter contract_instance_id")?, - source_manifest_id: row - .try_get("source_manifest_id") - .context("missing registry emitter source_manifest_id")?, - namespace: row - .try_get("namespace") - .context("missing registry emitter namespace")?, - source_family: row - .try_get("source_family") - .context("missing registry emitter source_family")?, - manifest_version: row - .try_get("manifest_version") - .context("missing registry emitter manifest_version")?, - contract_role: row - .try_get("contract_role") - .context("missing registry emitter contract_role")?, - active_from_block_number: row - .try_get("active_from_block_number") - .context("missing registry emitter active_from_block_number")?, - active_to_block_number: row - .try_get("active_to_block_number") - .context("missing registry emitter active_to_block_number")?, - source_rank: row - .try_get("source_rank") - .context("missing registry emitter source_rank")?, - }; + let candidate = active_emitter_from_row(&row)?; let scope_key = ( candidate.source_family.clone(), @@ -66,6 +40,135 @@ pub(super) fn active_emitters_from_rows(rows: Vec) -> Result, + progress: &mut dyn StartupAdapterProgress, +) -> Result> { + let row_count = rows.len(); + let mut emitters_by_scope = BTreeMap::new(); + for (index, row) in rows.into_iter().enumerate() { + let candidate = active_emitter_from_row(&row)?; + let scope_key = ( + candidate.source_family.clone(), + candidate.address.clone(), + candidate.active_from_block_number, + candidate.active_to_block_number, + ); + match emitters_by_scope.get(&scope_key) { + Some(current) if !candidate_precedes(&candidate, current) => {} + _ => { + emitters_by_scope.insert(scope_key, candidate); + } + } + record_progress(pool, progress, index + 1, row_count).await?; + } + remove_shadowed_emitters_with_progress(pool, emitters_by_scope.into_values(), progress).await +} + +fn active_emitter_from_row(row: &PgRow) -> Result { + let address = normalize_address(&row.try_get::("address")?); + Ok(ActiveEmitter { + address, + contract_instance_id: row + .try_get("contract_instance_id") + .context("missing registry emitter contract_instance_id")?, + source_manifest_id: row + .try_get("source_manifest_id") + .context("missing registry emitter source_manifest_id")?, + namespace: row + .try_get("namespace") + .context("missing registry emitter namespace")?, + source_family: row + .try_get("source_family") + .context("missing registry emitter source_family")?, + manifest_version: row + .try_get("manifest_version") + .context("missing registry emitter manifest_version")?, + contract_role: row + .try_get("contract_role") + .context("missing registry emitter contract_role")?, + active_from_block_number: row + .try_get("active_from_block_number") + .context("missing registry emitter active_from_block_number")?, + active_to_block_number: row + .try_get("active_to_block_number") + .context("missing registry emitter active_to_block_number")?, + source_rank: row + .try_get("source_rank") + .context("missing registry emitter source_rank")?, + }) +} + +async fn remove_shadowed_emitters_with_progress( + pool: &sqlx::PgPool, + emitters: impl IntoIterator, + progress: &mut dyn StartupAdapterProgress, +) -> Result> { + let mut emitters_by_address = BTreeMap::< + (String, String), + BTreeMap<(Option, Option, i32, i64, sqlx::types::Uuid), ActiveEmitter>, + >::new(); + let mut grouped = 0usize; + for emitter in emitters { + emitters_by_address + .entry((emitter.address.clone(), emitter.source_family.clone())) + .or_default() + .insert( + ( + emitter.active_from_block_number, + emitter.active_to_block_number, + emitter.source_rank, + emitter.source_manifest_id, + emitter.contract_instance_id, + ), + emitter, + ); + grouped += 1; + if grouped.is_multiple_of(STARTUP_ADAPTER_PROGRESS_PAGE_ROWS) { + progress.record(pool).await?; + } + } + if grouped > 0 && !grouped.is_multiple_of(STARTUP_ADAPTER_PROGRESS_PAGE_ROWS) { + progress.record(pool).await?; + } + + let mut compacted = Vec::new(); + let mut compared = 0usize; + for address_emitters in emitters_by_address.into_values() { + let mut retained = Vec::::new(); + 'candidate: for candidate in address_emitters.into_values() { + for retained_emitter in &retained { + compared += 1; + if compared.is_multiple_of(STARTUP_ADAPTER_PROGRESS_PAGE_ROWS) { + progress.record(pool).await?; + } + if emitter_shadows(retained_emitter, &candidate) { + continue 'candidate; + } + } + retained.push(candidate); + } + compacted.extend(retained); + } + if compared > 0 && !compared.is_multiple_of(STARTUP_ADAPTER_PROGRESS_PAGE_ROWS) { + progress.record(pool).await?; + } + Ok(compacted) +} + +async fn record_progress( + pool: &sqlx::PgPool, + progress: &mut dyn StartupAdapterProgress, + completed: usize, + total: usize, +) -> Result<()> { + if completed == total || completed.is_multiple_of(STARTUP_ADAPTER_PROGRESS_PAGE_ROWS) { + progress.record(pool).await?; + } + Ok(()) +} + pub(super) fn sort_active_emitters(emitters: &mut [ActiveEmitter]) { emitters.sort_by(|left, right| { left.address diff --git a/crates/adapters/src/ens_v1_subregistry_discovery/loader/raw_logs.rs b/crates/adapters/src/ens_v1_subregistry_discovery/loader/raw_logs.rs index b88529f6..01cae175 100644 --- a/crates/adapters/src/ens_v1_subregistry_discovery/loader/raw_logs.rs +++ b/crates/adapters/src/ens_v1_subregistry_discovery/loader/raw_logs.rs @@ -1,6 +1,3 @@ -use bigname_storage::sql_row; -use std::collections::HashMap; - use super::super::{ CONTRACT_ROLE_REGISTRY_OLD, hex_topic::{ @@ -14,8 +11,10 @@ use super::super::{ }; use super::{ActiveEmitter, RegistryRawLogRow}; use anyhow::{Context, Result}; +use bigname_storage::sql_row; use futures_util::TryStreamExt; use sqlx::PgPool; +use std::collections::HashMap; #[derive(Clone, Debug, Eq, PartialEq)] pub(in crate::ens_v1_subregistry_discovery) struct RegistryRawLogPosition { @@ -165,6 +164,7 @@ pub(in crate::ens_v1_subregistry_discovery) async fn stream_registry_raw_logs_th emitters: &[ActiveEmitter], through_block_number: i64, page_limit: i64, + startup_progress: &mut Option<&mut dyn crate::StartupAdapterProgress>, mut handle_raw_log: impl FnMut(RegistryRawLogRow) -> Result<()>, ) -> Result { let mut start_after = None; @@ -188,6 +188,7 @@ pub(in crate::ens_v1_subregistry_discovery) async fn stream_registry_raw_logs_th scanned_log_count += 1; } start_after = Some(last_position); + crate::checkpoint_context::record_startup_adapter_progress(pool, startup_progress).await?; } Ok(scanned_log_count) } diff --git a/crates/adapters/src/ens_v1_subregistry_discovery/reconciliation.rs b/crates/adapters/src/ens_v1_subregistry_discovery/reconciliation.rs index 7e37e1f6..b842c48f 100644 --- a/crates/adapters/src/ens_v1_subregistry_discovery/reconciliation.rs +++ b/crates/adapters/src/ens_v1_subregistry_discovery/reconciliation.rs @@ -1,17 +1,22 @@ -use anyhow::Result; +use std::{collections::BTreeMap, ops::Bound}; + +use anyhow::{Context, Result}; use bigname_manifests::{ - DiscoveryObservation, DiscoveryObservationPageSource, DiscoveryReconciliationSummary, - ExpectedDiscoveryAdmissionEpoch, FullDiscoveryReconciliationOptions, - reconcile_discovery_observations, reconcile_discovery_observations_streamed, + DiscoveryObservation, DiscoveryObservationPageSource, ExpectedDiscoveryAdmissionEpoch, + FullDiscoveryReconciliationOptions, reconcile_discovery_observations_streamed, + reconcile_discovery_observations_streamed_with_full_options, }; use sqlx::PgPool; use super::{ EnsV1SubregistryDiscoverySyncSummary, + assignment::ObservedRegistryAssignment, checkpoint::{RECONCILIATION_PAGE_LIMIT, SubregistryReplayCheckpoint}, }; use crate::checkpoint_context::{StartupAdapterProgress, record_startup_adapter_progress}; +use super::hex_topic::{ZERO_ADDRESS, normalize_address}; + /// Pages one discovery source's staged latest-per-key assignments straight /// from the checkpoint items, so the finalize reconcile never materializes a /// source's observations in memory (#168). @@ -43,6 +48,60 @@ impl DiscoveryObservationPageSource for CheckpointAssignmentPageSource<'_, '_> { } } +struct AssignmentMapPageSource<'a, 'progress> { + pool: &'a PgPool, + assignments: &'a BTreeMap, + discovery_source: &'a str, + startup_progress: &'a tokio::sync::Mutex>, +} + +impl DiscoveryObservationPageSource for AssignmentMapPageSource<'_, '_> { + async fn load_page( + &self, + after_key: Option<&str>, + limit: i64, + ) -> Result> { + let prefix = format!("{}:", self.discovery_source); + let start = match after_key { + Some(after_key) => Bound::Excluded(format!("{prefix}{after_key}")), + None => Bound::Included(prefix.clone()), + }; + self.assignments + .range((start, Bound::Unbounded)) + .take_while(|(key, _)| key.starts_with(&prefix)) + .take(usize::try_from(limit.clamp(1, RECONCILIATION_PAGE_LIMIT))?) + .map(|(_, assignment)| { + Ok(( + assignment.observation_key.clone(), + assignment.discovery_observation()?, + )) + }) + .collect() + } + + async fn record_progress(&self) -> Result<()> { + let mut startup_progress = self.startup_progress.lock().await; + record_startup_adapter_progress(self.pool, &mut startup_progress).await + } +} + +pub(super) async fn count_active_assignments_with_progress( + pool: &PgPool, + assignments: &BTreeMap, + startup_progress: &mut Option<&mut dyn StartupAdapterProgress>, +) -> Result { + let mut active_count = 0usize; + for (index, assignment) in assignments.values().enumerate() { + if normalize_address(&assignment.to_address) != ZERO_ADDRESS { + active_count += 1; + } + if (index + 1).is_multiple_of(super::checkpoint::PAGE_LIMIT as usize) { + record_startup_adapter_progress(pool, startup_progress).await?; + } + } + Ok(active_count) +} + pub(super) async fn reconcile_subregistry_discovery_from_checkpoint( pool: &PgPool, checkpoint: &SubregistryReplayCheckpoint, @@ -74,23 +133,53 @@ pub(super) async fn reconcile_subregistry_discovery_from_checkpoint( result } -pub(super) async fn reconcile_subregistry_discovery_source_through_block( +#[expect(clippy::too_many_arguments)] +pub(super) async fn reconcile_subregistry_discovery_from_assignments_through_block( pool: &PgPool, chain: &str, - discovery_source: &str, - observations: &[DiscoveryObservation], + assignments: &BTreeMap, + discovery_sources: &[String], through_block: i64, - expected_admission_epoch: Option, -) -> Result { - reconcile_discovery_observations( - pool, - discovery_source, - observations, - FullDiscoveryReconciliationOptions { - through_block_number: Some(through_block), - expected_admission_epoch: expected_admission_epoch - .map(|epoch| ExpectedDiscoveryAdmissionEpoch { chain, epoch }), - }, - ) - .await + mut expected_admission_epoch: Option, + reconciliation: &mut EnsV1SubregistryDiscoverySyncSummary, + startup_progress: &mut Option<&mut dyn StartupAdapterProgress>, +) -> Result<()> { + let progress = tokio::sync::Mutex::new(startup_progress.take()); + let result = async { + for discovery_source in discovery_sources { + let page_source = AssignmentMapPageSource { + pool, + assignments, + discovery_source, + startup_progress: &progress, + }; + let source_reconciliation = + reconcile_discovery_observations_streamed_with_full_options( + pool, + discovery_source, + &page_source, + FullDiscoveryReconciliationOptions { + through_block_number: Some(through_block), + expected_admission_epoch: expected_admission_epoch + .map(|epoch| ExpectedDiscoveryAdmissionEpoch { chain, epoch }), + }, + ) + .await?; + reconciliation.active_edge_count += source_reconciliation.active_edge_count; + reconciliation.admitted_edge_count += source_reconciliation.admitted_edge_count; + reconciliation.inserted_edge_count += source_reconciliation.inserted_edge_count; + reconciliation.deactivated_edge_count += source_reconciliation.deactivated_edge_count; + if let Some(expected_epoch) = expected_admission_epoch.as_mut() { + *expected_epoch = expected_epoch + .checked_add(i64::try_from( + source_reconciliation.admission_epoch_bump_count, + )?) + .context("legacy registry reconciliation admission epoch overflowed")?; + } + } + Ok(()) + } + .await; + *startup_progress = progress.into_inner(); + result } diff --git a/crates/adapters/src/ens_v1_subregistry_discovery/replay.rs b/crates/adapters/src/ens_v1_subregistry_discovery/replay.rs index aa81e4d1..2358f398 100644 --- a/crates/adapters/src/ens_v1_subregistry_discovery/replay.rs +++ b/crates/adapters/src/ens_v1_subregistry_discovery/replay.rs @@ -50,6 +50,25 @@ pub async fn sync_ens_v1_subregistry_discovery_with_replay_checkpoint_and_log_li .await } +pub async fn sync_ens_v1_subregistry_discovery_with_replay_checkpoint_and_log_limit_and_progress( + pool: &PgPool, + chain: &str, + checkpoint: &ReplayAdapterCheckpointContext, + max_raw_logs_per_page: usize, + progress: &mut dyn StartupAdapterProgress, +) -> Result { + let checkpoint = AdapterCheckpointContext::for_replay(checkpoint); + sync_ens_v1_subregistry_discovery_with_checkpoint_context( + pool, + chain, + &checkpoint, + max_raw_logs_per_page, + None, + Some(progress), + ) + .await +} + pub(super) async fn sync_ens_v1_subregistry_discovery_with_checkpoint_context( pool: &PgPool, chain: &str, diff --git a/crates/adapters/src/ens_v1_unwrapped_authority.rs b/crates/adapters/src/ens_v1_unwrapped_authority.rs index 9603559f..68bcc3a8 100644 --- a/crates/adapters/src/ens_v1_unwrapped_authority.rs +++ b/crates/adapters/src/ens_v1_unwrapped_authority.rs @@ -477,6 +477,7 @@ mod transition; pub use self::pipeline::{ sync_ens_v1_unwrapped_authority, sync_ens_v1_unwrapped_authority_with_replay_checkpoint_and_log_limit, + sync_ens_v1_unwrapped_authority_with_replay_checkpoint_and_log_limit_and_progress, sync_ens_v1_unwrapped_authority_with_startup_checkpoint_and_log_limit, sync_ens_v1_unwrapped_authority_with_startup_checkpoint_and_log_limit_and_progress, }; diff --git a/crates/adapters/src/ens_v1_unwrapped_authority/checkpoint.rs b/crates/adapters/src/ens_v1_unwrapped_authority/checkpoint.rs index d13a6956..70ce2e4e 100644 --- a/crates/adapters/src/ens_v1_unwrapped_authority/checkpoint.rs +++ b/crates/adapters/src/ens_v1_unwrapped_authority/checkpoint.rs @@ -1,5 +1,7 @@ use super::*; -use crate::checkpoint_context::AdapterCheckpointContext; +use crate::checkpoint_context::{ + AdapterCheckpointContext, StartupAdapterProgress, record_startup_adapter_progress, +}; use anyhow::{Context, Result, ensure}; use bigname_storage::{ RawLogStagingInputVersion, load_raw_log_staging_input_version, @@ -14,9 +16,12 @@ mod payload; mod persistence; mod startup_events; use crate::checkpoint_codec::JsonbCheckpointCodec; +#[cfg(test)] +use items::{checkpoint_item_rows, checkpoint_pending_observation_delete_keys}; use items::{ - checkpoint_item_rows, checkpoint_pending_observation_delete_keys, delete_checkpoint_items, - insert_checkpoint_items, update_checkpoint_progress, + checkpoint_item_rows_with_progress, checkpoint_pending_observation_delete_keys_with_progress, + delete_checkpoint_items, delete_checkpoint_items_with_progress, + insert_checkpoint_items_with_progress, update_checkpoint_progress, }; pub(super) use payload::{decode_item, encode_item}; use payload::{flushed_events_from_payload, summary_from_payload, summary_payload}; @@ -50,46 +55,6 @@ pub(super) struct UnwrappedAuthorityReplayCheckpointDelta { pub(super) migrated_nodes: BTreeSet, } -impl UnwrappedAuthorityReplayCheckpointDelta { - pub(super) fn mark_history(&mut self, key: impl Into) { - self.history_keys.insert(key.into()); - } - - pub(super) fn mark_reverse_history(&mut self, key: impl Into) { - self.reverse_history_keys.insert(key.into()); - } - - pub(super) fn mark_known_name(&mut self, key: impl Into) { - self.known_name_keys.insert(key.into()); - } - - pub(super) fn mark_known_name_ref(&mut self, key: impl Into) { - self.known_name_ref_keys.insert(key.into()); - } - - pub(super) fn mark_namehash_labelhash(&mut self, key: impl Into) { - self.namehash_labelhash_keys.insert(key.into()); - } - - pub(super) fn mark_pending_observations(&mut self, key: impl Into) { - self.pending_observation_keys.insert(key.into()); - } - - pub(super) fn mark_migrated_node(&mut self, node: impl Into) { - self.migrated_nodes.insert(node.into()); - } - - pub(super) fn clear(&mut self) { - self.history_keys.clear(); - self.reverse_history_keys.clear(); - self.known_name_keys.clear(); - self.known_name_ref_keys.clear(); - self.namehash_labelhash_keys.clear(); - self.pending_observation_keys.clear(); - self.migrated_nodes.clear(); - } -} - pub(super) struct UnwrappedAuthorityReplayCheckpointState { pub(super) histories: BTreeMap, pub(super) reverse_histories: BTreeMap, @@ -322,6 +287,7 @@ impl UnwrappedAuthorityReplayCheckpoint { &self, pool: &PgPool, include_replay_auxiliary_state: bool, + progress: &mut Option<&mut dyn StartupAdapterProgress>, ) -> Result> { if self.last_block_number.is_none() { return Ok(None); @@ -348,7 +314,6 @@ impl UnwrappedAuthorityReplayCheckpoint { $6::BOOLEAN OR item_kind = ANY($7::TEXT[]) ) - ORDER BY item_kind, item_key "#, ) .bind(&self.context.deployment_profile) @@ -368,6 +333,7 @@ impl UnwrappedAuthorityReplayCheckpoint { let mut pending_namehash_observations = HashMap::new(); let mut migrated_nodes = HashSet::new(); + let mut loaded_item_count = 0usize; while let Some(row) = rows.try_next().await.with_context(|| { format!( "failed to load staged {ADAPTER} replay checkpoint items for {}/{}", @@ -416,6 +382,15 @@ impl UnwrappedAuthorityReplayCheckpoint { } _ => {} } + loaded_item_count += 1; + if loaded_item_count.is_multiple_of(CHECKPOINT_ITEM_INSERT_BATCH_SIZE) { + record_startup_adapter_progress(pool, progress).await?; + } + } + if loaded_item_count > 0 + && !loaded_item_count.is_multiple_of(CHECKPOINT_ITEM_INSERT_BATCH_SIZE) + { + record_startup_adapter_progress(pool, progress).await?; } Ok(Some(UnwrappedAuthorityReplayCheckpointState { @@ -438,10 +413,12 @@ impl UnwrappedAuthorityReplayCheckpoint { state: UnwrappedAuthorityReplayCheckpointStateRef<'_>, delta: &UnwrappedAuthorityReplayCheckpointDelta, flushed_events: &UnwrappedAuthorityReplayFlushedEvents, + progress: &mut Option<&mut dyn StartupAdapterProgress>, ) -> Result<()> { - let item_rows = checkpoint_item_rows(&state, delta)?; + let item_rows = checkpoint_item_rows_with_progress(pool, &state, delta, progress).await?; let pending_observation_delete_keys = - checkpoint_pending_observation_delete_keys(&state, delta); + checkpoint_pending_observation_delete_keys_with_progress(pool, &state, delta, progress) + .await?; let staged_item_count = state.histories.len() + state.reverse_histories.len(); let staged_aux_item_count = state.known_names_by_namehash.len() + state.known_name_refs_by_namehash.len() @@ -460,14 +437,17 @@ impl UnwrappedAuthorityReplayCheckpoint { .begin() .await .context("failed to start unwrapped-authority replay checkpoint transaction")?; - delete_checkpoint_items( + delete_checkpoint_items_with_progress( + pool, &mut transaction, self, ITEM_KIND_PENDING_OBSERVATIONS, &pending_observation_delete_keys, + progress, ) .await?; - insert_checkpoint_items(&mut transaction, self, &item_rows).await?; + insert_checkpoint_items_with_progress(pool, &mut transaction, self, &item_rows, progress) + .await?; update_checkpoint_progress( &mut transaction, self, @@ -484,6 +464,7 @@ impl UnwrappedAuthorityReplayCheckpoint { .commit() .await .context("failed to commit unwrapped-authority replay checkpoint progress")?; + record_startup_adapter_progress(pool, progress).await?; self.last_block_number = Some(boundary_block_number); self.scanned_log_count = scanned_log_count; diff --git a/crates/adapters/src/ens_v1_unwrapped_authority/checkpoint/items.rs b/crates/adapters/src/ens_v1_unwrapped_authority/checkpoint/items.rs index 295b447c..cbc89ef4 100644 --- a/crates/adapters/src/ens_v1_unwrapped_authority/checkpoint/items.rs +++ b/crates/adapters/src/ens_v1_unwrapped_authority/checkpoint/items.rs @@ -2,6 +2,8 @@ use anyhow::{Context, Result}; use serde_json::{Value, json}; use sqlx::{Postgres, QueryBuilder}; +use crate::checkpoint_context::{StartupAdapterProgress, record_startup_adapter_progress}; + use super::{ ADAPTER, CHECKPOINT_CODEC, CHECKPOINT_ITEM_DELETE_BATCH_SIZE, CHECKPOINT_ITEM_INSERT_BATCH_SIZE, ITEM_KIND_HISTORY, ITEM_KIND_KNOWN_NAME, @@ -11,6 +13,65 @@ use super::{ encode_item, }; +impl UnwrappedAuthorityReplayCheckpointDelta { + pub(in crate::ens_v1_unwrapped_authority) fn mark_history(&mut self, key: impl Into) { + self.history_keys.insert(key.into()); + } + + pub(in crate::ens_v1_unwrapped_authority) fn mark_reverse_history( + &mut self, + key: impl Into, + ) { + self.reverse_history_keys.insert(key.into()); + } + + pub(in crate::ens_v1_unwrapped_authority) fn mark_known_name( + &mut self, + key: impl Into, + ) { + self.known_name_keys.insert(key.into()); + } + + pub(in crate::ens_v1_unwrapped_authority) fn mark_known_name_ref( + &mut self, + key: impl Into, + ) { + self.known_name_ref_keys.insert(key.into()); + } + + pub(in crate::ens_v1_unwrapped_authority) fn mark_namehash_labelhash( + &mut self, + key: impl Into, + ) { + self.namehash_labelhash_keys.insert(key.into()); + } + + pub(in crate::ens_v1_unwrapped_authority) fn mark_pending_observations( + &mut self, + key: impl Into, + ) { + self.pending_observation_keys.insert(key.into()); + } + + pub(in crate::ens_v1_unwrapped_authority) fn mark_migrated_node( + &mut self, + node: impl Into, + ) { + self.migrated_nodes.insert(node.into()); + } + + pub(in crate::ens_v1_unwrapped_authority) fn clear(&mut self) { + self.history_keys.clear(); + self.reverse_history_keys.clear(); + self.known_name_keys.clear(); + self.known_name_ref_keys.clear(); + self.namehash_labelhash_keys.clear(); + self.pending_observation_keys.clear(); + self.migrated_nodes.clear(); + } +} + +#[cfg(test)] pub(super) fn checkpoint_item_rows( state: &UnwrappedAuthorityReplayCheckpointStateRef<'_>, delta: &UnwrappedAuthorityReplayCheckpointDelta, @@ -68,6 +129,7 @@ pub(super) fn checkpoint_item_rows( Ok(rows) } +#[cfg(test)] pub(super) fn checkpoint_pending_observation_delete_keys( state: &UnwrappedAuthorityReplayCheckpointStateRef<'_>, delta: &UnwrappedAuthorityReplayCheckpointDelta, @@ -85,6 +147,147 @@ pub(super) fn checkpoint_pending_observation_delete_keys( .collect() } +pub(super) async fn checkpoint_item_rows_with_progress( + pool: &sqlx::PgPool, + state: &UnwrappedAuthorityReplayCheckpointStateRef<'_>, + delta: &UnwrappedAuthorityReplayCheckpointDelta, + progress: &mut Option<&mut dyn StartupAdapterProgress>, +) -> Result> { + let mut rows = Vec::new(); + let mut examined = 0usize; + macro_rules! push_encoded { + ($keys:expr, $values:expr, $kind:expr) => { + for key in $keys { + if let Some(value) = $values.get(key) { + rows.push(($kind, key.clone(), encode_item(value)?)); + } + examined += 1; + if examined.is_multiple_of(CHECKPOINT_ITEM_INSERT_BATCH_SIZE) { + record_startup_adapter_progress(pool, progress).await?; + } + } + }; + } + push_encoded!( + delta.history_keys.iter(), + state.histories, + ITEM_KIND_HISTORY + ); + push_encoded!( + delta.reverse_history_keys.iter(), + state.reverse_histories, + ITEM_KIND_REVERSE_HISTORY + ); + push_encoded!( + delta.known_name_keys.iter(), + state.known_names_by_namehash, + ITEM_KIND_KNOWN_NAME + ); + push_encoded!( + delta.known_name_ref_keys.iter(), + state.known_name_refs_by_namehash, + ITEM_KIND_KNOWN_NAME_REF + ); + for key in &delta.namehash_labelhash_keys { + if let Some(labelhash) = state.namehash_to_labelhash.get(key) { + rows.push(( + ITEM_KIND_NAMEHASH_LABELHASH, + key.clone(), + CHECKPOINT_CODEC.encode(json!({ "labelhash": labelhash })), + )); + } + examined += 1; + if examined.is_multiple_of(CHECKPOINT_ITEM_INSERT_BATCH_SIZE) { + record_startup_adapter_progress(pool, progress).await?; + } + } + for key in &delta.pending_observation_keys { + if let Some(observations) = state + .pending_namehash_observations + .get(key) + .filter(|observations| !observations.is_empty()) + { + rows.push(( + ITEM_KIND_PENDING_OBSERVATIONS, + key.clone(), + encode_item(observations.as_slice())?, + )); + } + examined += 1; + if examined.is_multiple_of(CHECKPOINT_ITEM_INSERT_BATCH_SIZE) { + record_startup_adapter_progress(pool, progress).await?; + } + } + for node in &delta.migrated_nodes { + rows.push(( + ITEM_KIND_MIGRATED_NODE, + node.clone(), + CHECKPOINT_CODEC.encode(json!({ "node": node })), + )); + examined += 1; + if examined.is_multiple_of(CHECKPOINT_ITEM_INSERT_BATCH_SIZE) { + record_startup_adapter_progress(pool, progress).await?; + } + } + if examined > 0 && !examined.is_multiple_of(CHECKPOINT_ITEM_INSERT_BATCH_SIZE) { + record_startup_adapter_progress(pool, progress).await?; + } + Ok(rows) +} + +pub(super) async fn checkpoint_pending_observation_delete_keys_with_progress( + pool: &sqlx::PgPool, + state: &UnwrappedAuthorityReplayCheckpointStateRef<'_>, + delta: &UnwrappedAuthorityReplayCheckpointDelta, + progress: &mut Option<&mut dyn StartupAdapterProgress>, +) -> Result> { + let mut keys = Vec::new(); + for (index, key) in delta.pending_observation_keys.iter().enumerate() { + if state + .pending_namehash_observations + .get(key) + .is_none_or(Vec::is_empty) + { + keys.push(key.clone()); + } + if index + 1 == delta.pending_observation_keys.len() + || (index + 1).is_multiple_of(CHECKPOINT_ITEM_DELETE_BATCH_SIZE) + { + record_startup_adapter_progress(pool, progress).await?; + } + } + Ok(keys) +} + +pub(super) async fn delete_checkpoint_items_with_progress( + pool: &sqlx::PgPool, + transaction: &mut sqlx::Transaction<'_, Postgres>, + checkpoint: &UnwrappedAuthorityReplayCheckpoint, + item_kind: &str, + item_keys: &[String], + progress: &mut Option<&mut dyn StartupAdapterProgress>, +) -> Result<()> { + for chunk in item_keys.chunks(CHECKPOINT_ITEM_DELETE_BATCH_SIZE) { + delete_checkpoint_items(transaction, checkpoint, item_kind, chunk).await?; + record_startup_adapter_progress(pool, progress).await?; + } + Ok(()) +} + +pub(super) async fn insert_checkpoint_items_with_progress( + pool: &sqlx::PgPool, + transaction: &mut sqlx::Transaction<'_, Postgres>, + checkpoint: &UnwrappedAuthorityReplayCheckpoint, + item_rows: &[(&'static str, String, Value)], + progress: &mut Option<&mut dyn StartupAdapterProgress>, +) -> Result<()> { + for chunk in item_rows.chunks(CHECKPOINT_ITEM_INSERT_BATCH_SIZE) { + insert_checkpoint_items(transaction, checkpoint, chunk).await?; + record_startup_adapter_progress(pool, progress).await?; + } + Ok(()) +} + pub(super) async fn delete_checkpoint_items( transaction: &mut sqlx::Transaction<'_, Postgres>, checkpoint: &UnwrappedAuthorityReplayCheckpoint, diff --git a/crates/adapters/src/ens_v1_unwrapped_authority/checkpoint/startup_events.rs b/crates/adapters/src/ens_v1_unwrapped_authority/checkpoint/startup_events.rs index bc4548f6..b9d58b77 100644 --- a/crates/adapters/src/ens_v1_unwrapped_authority/checkpoint/startup_events.rs +++ b/crates/adapters/src/ens_v1_unwrapped_authority/checkpoint/startup_events.rs @@ -15,6 +15,7 @@ impl UnwrappedAuthorityReplayCheckpoint { pool: &PgPool, events: &[NormalizedEvent], staged_events: &mut UnwrappedAuthorityReplayFlushedEvents, + startup_progress: &mut Option<&mut dyn StartupAdapterProgress>, ) -> Result<()> { ensure!( self.is_startup(), @@ -24,32 +25,47 @@ impl UnwrappedAuthorityReplayCheckpoint { return Ok(()); } - let item_rows = events - .iter() - .map(|event| { - Ok(( - ITEM_KIND_STARTUP_PENDING_EVENT, - event.event_identity.clone(), - encode_item(event)?, - )) - }) - .collect::>>()?; + let mut item_rows = Vec::with_capacity(events.len()); + for (index, event) in events.iter().enumerate() { + item_rows.push(( + ITEM_KIND_STARTUP_PENDING_EVENT, + event.event_identity.clone(), + encode_item(event)?, + )); + if index + 1 == events.len() + || (index + 1).is_multiple_of(CHECKPOINT_ITEM_INSERT_BATCH_SIZE) + { + record_startup_adapter_progress(pool, startup_progress).await?; + } + } let mut transaction = pool .begin() .await .context("failed to start startup authority event-staging transaction")?; - insert_checkpoint_items(&mut transaction, self, &item_rows).await?; + insert_checkpoint_items_with_progress( + pool, + &mut transaction, + self, + &item_rows, + startup_progress, + ) + .await?; transaction .commit() .await .context("failed to commit startup authority event staging")?; staged_events.total_count += events.len(); - for event in events { + for (index, event) in events.iter().enumerate() { *staged_events .by_kind .entry(event.event_kind.clone()) .or_insert(0) += 1; + if index + 1 == events.len() + || (index + 1).is_multiple_of(CHECKPOINT_ITEM_INSERT_BATCH_SIZE) + { + record_startup_adapter_progress(pool, startup_progress).await?; + } } Ok(()) } diff --git a/crates/adapters/src/ens_v1_unwrapped_authority/loading.rs b/crates/adapters/src/ens_v1_unwrapped_authority/loading.rs index a6defcfb..488b84e1 100644 --- a/crates/adapters/src/ens_v1_unwrapped_authority/loading.rs +++ b/crates/adapters/src/ens_v1_unwrapped_authority/loading.rs @@ -8,7 +8,7 @@ pub(super) use active_emitters::{load_active_emitters, load_generic_resolver_eve pub(super) use blocks::{ load_canonical_block_at_number, load_canonical_blocks, load_canonical_blocks_for_authority_logs_through_head, - load_canonical_blocks_for_restricted_authority_sync, + load_canonical_blocks_for_restricted_authority_sync, load_canonical_blocks_with_progress, }; pub(super) use raw_logs::{ AuthorityRawLogStreamSourceRouter, load_authority_raw_logs, diff --git a/crates/adapters/src/ens_v1_unwrapped_authority/loading/active_emitters.rs b/crates/adapters/src/ens_v1_unwrapped_authority/loading/active_emitters.rs index f5fdcf45..ce2a4c1e 100644 --- a/crates/adapters/src/ens_v1_unwrapped_authority/loading/active_emitters.rs +++ b/crates/adapters/src/ens_v1_unwrapped_authority/loading/active_emitters.rs @@ -1,5 +1,5 @@ use bigname_storage::sql_row; -use std::collections::{HashMap, HashSet}; +use std::collections::{BTreeMap, HashMap, HashSet}; use super::super::scope::{ AuthorityRawLogSourceScopeTarget, is_generic_resolver_event_source_scope_target, @@ -8,27 +8,44 @@ use super::super::*; use anyhow::{Context, Result}; use bigname_manifests::{ WatchedContract, WatchedContractSource, load_manifest_declared_watched_contracts, - load_watched_contracts_by_chain, + load_watched_contracts_by_chain, load_watched_contracts_scoped_with_progress, }; use sqlx::{PgPool, types::Uuid}; -use crate::adapter_manifest::{required_source_manifest_id, watched_contract_manifest_ids}; +use crate::{ + adapter_manifest::required_source_manifest_id, + checkpoint_context::{StartupAdapterProgress, record_startup_adapter_progress}, + startup_progress::{STARTUP_ADAPTER_PROGRESS_PAGE_ROWS, StartupManifestProgress}, +}; + +#[path = "active_emitters/manifest.rs"] +mod manifest; +use manifest::*; pub(in crate::ens_v1_unwrapped_authority) async fn load_active_emitters( pool: &PgPool, chain: &str, source_scope: Option<&[AuthorityRawLogSourceScopeTarget]>, + progress: &mut Option<&mut dyn StartupAdapterProgress>, ) -> Result> { - let watched_contracts = load_scoped_watched_contracts(pool, chain, source_scope).await?; + let watched_contracts = + load_scoped_watched_contracts(pool, chain, source_scope, progress).await?; if watched_contracts.is_empty() { return Ok(Vec::new()); } - let contract_roles = load_manifest_contract_roles(pool, &watched_contracts).await?; + let contract_roles = load_manifest_contract_roles(pool, &watched_contracts, progress).await?; - let manifest_ids = watched_contract_manifest_ids(&watched_contracts)?; + let mut manifest_ids = HashSet::new(); + for (index, contract) in watched_contracts.iter().enumerate() { + manifest_ids.insert(required_source_manifest_id(contract)?); + record_emitter_progress(pool, progress, index + 1, watched_contracts.len()).await?; + } + let manifest_ids = manifest_ids.into_iter().collect::>(); let active_manifests = load_active_manifest_metadata(pool, &manifest_ids).await?; - let mut emitters = Vec::new(); - for watched_contract in watched_contracts { + record_startup_adapter_progress(pool, progress).await?; + let watched_contract_count = watched_contracts.len(); + let mut emitters = BTreeMap::new(); + for (index, watched_contract) in watched_contracts.into_iter().enumerate() { let source_manifest_id = required_source_manifest_id(&watched_contract)?; let Some(manifest) = active_manifests.get(&source_manifest_id) else { continue; @@ -60,16 +77,21 @@ pub(in crate::ens_v1_unwrapped_authority) async fn load_active_emitters( source_rank: source_rank(watched_contract.source), }; - emitters.push(candidate); + emitters.insert( + ( + candidate.address.clone(), + candidate.source_rank, + candidate.source_manifest_id, + candidate.contract_instance_id, + candidate.active_from_block_number, + candidate.active_to_block_number, + index, + ), + candidate, + ); + record_emitter_progress(pool, progress, index + 1, watched_contract_count).await?; } - - emitters.sort_by(|left, right| { - left.address - .cmp(&right.address) - .then(left.source_rank.cmp(&right.source_rank)) - .then(left.source_manifest_id.cmp(&right.source_manifest_id)) - }); - Ok(emitters) + Ok(emitters.into_values().collect()) } pub(in crate::ens_v1_unwrapped_authority) async fn load_generic_resolver_event_sources( @@ -130,8 +152,23 @@ async fn load_scoped_watched_contracts( pool: &PgPool, chain: &str, source_scope: Option<&[AuthorityRawLogSourceScopeTarget]>, + progress: &mut Option<&mut dyn StartupAdapterProgress>, ) -> Result> { let Some(source_scope) = source_scope else { + if let Some(progress) = progress.as_deref_mut() { + let source_families = unwrapped_authority_source_families(); + let mut manifest_progress = StartupManifestProgress::new(progress); + return load_watched_contracts_scoped_with_progress( + pool, + Some(chain), + &source_families, + &mut manifest_progress, + ) + .await + .context( + "failed to stream watched contracts for ENSv1 unwrapped authority attribution", + ); + } return load_watched_contracts_by_chain(pool, chain) .await .context("failed to load watched contracts for ENSv1 unwrapped authority attribution"); @@ -769,124 +806,3 @@ fn watched_contract_intersects_source_scope( let contract_to = contract.active_to_block_number.unwrap_or(i64::MAX); target.effective_from_block <= contract_to && contract_from <= target.effective_to_block } - -async fn load_active_manifest_metadata( - pool: &PgPool, - manifest_ids: &[i64], -) -> Result> { - let rows = sqlx::query( - r#" - SELECT manifest_id, chain, namespace, source_family, manifest_version, normalizer_version - FROM manifest_versions - WHERE rollout_status = 'active' - AND manifest_id = ANY($1::BIGINT[]) - "#, - ) - .bind(manifest_ids) - .fetch_all(pool) - .await - .context("failed to load active manifest metadata for ENSv1 unwrapped authority")?; - - rows.into_iter() - .map(|row| { - let manifest = ActiveManifestMetadata { - manifest_id: sql_row::get(&row, "manifest_id")?, - chain: sql_row::get(&row, "chain")?, - namespace: sql_row::get(&row, "namespace")?, - source_family: sql_row::get(&row, "source_family")?, - manifest_version: sql_row::get(&row, "manifest_version")?, - normalizer_version: sql_row::get(&row, "normalizer_version")?, - }; - Ok((manifest.manifest_id, manifest)) - }) - .collect() -} - -async fn load_manifest_contract_roles( - pool: &PgPool, - watched_contracts: &[WatchedContract], -) -> Result> { - let manifest_ids = watched_contracts - .iter() - .filter_map(|contract| contract.source_manifest_id) - .collect::>() - .into_iter() - .collect::>(); - let contract_instance_ids = watched_contracts - .iter() - .map(|contract| contract.contract_instance_id) - .collect::>() - .into_iter() - .collect::>(); - if manifest_ids.is_empty() || contract_instance_ids.is_empty() { - return Ok(HashMap::new()); - } - - let rows = sqlx::query( - r#" - SELECT manifest_id, contract_instance_id, role - FROM manifest_contract_instances - WHERE declaration_kind = 'contract' - AND manifest_id = ANY($1::BIGINT[]) - AND contract_instance_id = ANY($2::UUID[]) - "#, - ) - .bind(&manifest_ids) - .bind(&contract_instance_ids) - .fetch_all(pool) - .await - .context("failed to load manifest contract roles for ENSv1 unwrapped authority")?; - - rows.into_iter() - .map(|row| { - Ok(( - ( - sql_row::get(&row, "manifest_id")?, - sql_row::get(&row, "contract_instance_id")?, - ), - sql_row::get(&row, "role")?, - )) - }) - .collect() -} - -fn source_rank(source: WatchedContractSource) -> i32 { - crate::adapter_manifest::source_rank(source) -} - -async fn load_active_manifest_metadata_for_source_family( - pool: &PgPool, - chain: &str, - source_family: &str, -) -> Result> { - let rows = sqlx::query( - r#" - SELECT manifest_id, chain, namespace, source_family, manifest_version, normalizer_version - FROM manifest_versions - WHERE rollout_status = 'active' - AND chain = $1 - AND source_family = $2 - ORDER BY manifest_id - "#, - ) - .bind(chain) - .bind(source_family) - .fetch_all(pool) - .await - .with_context(|| { - format!("failed to load active {source_family} manifest metadata for {chain}") - })?; - - rows.into_iter() - .map(|row| { - Ok(ActiveManifestMetadata { - manifest_id: sql_row::get(&row, "manifest_id")?, - chain: sql_row::get(&row, "chain")?, - namespace: sql_row::get(&row, "namespace")?, - source_family: sql_row::get(&row, "source_family")?, - manifest_version: sql_row::get(&row, "manifest_version")?, - normalizer_version: sql_row::get(&row, "normalizer_version")?, - }) - }) - .collect() -} diff --git a/crates/adapters/src/ens_v1_unwrapped_authority/loading/active_emitters/manifest.rs b/crates/adapters/src/ens_v1_unwrapped_authority/loading/active_emitters/manifest.rs new file mode 100644 index 00000000..29d8a684 --- /dev/null +++ b/crates/adapters/src/ens_v1_unwrapped_authority/loading/active_emitters/manifest.rs @@ -0,0 +1,142 @@ +use super::*; + +pub(super) async fn load_active_manifest_metadata( + pool: &PgPool, + manifest_ids: &[i64], +) -> Result> { + let rows = sqlx::query( + r#" + SELECT manifest_id, chain, namespace, source_family, manifest_version, normalizer_version + FROM manifest_versions + WHERE rollout_status = 'active' + AND manifest_id = ANY($1::BIGINT[]) + "#, + ) + .bind(manifest_ids) + .fetch_all(pool) + .await + .context("failed to load active manifest metadata for ENSv1 unwrapped authority")?; + + rows.into_iter() + .map(|row| { + let manifest = ActiveManifestMetadata { + manifest_id: sql_row::get(&row, "manifest_id")?, + chain: sql_row::get(&row, "chain")?, + namespace: sql_row::get(&row, "namespace")?, + source_family: sql_row::get(&row, "source_family")?, + manifest_version: sql_row::get(&row, "manifest_version")?, + normalizer_version: sql_row::get(&row, "normalizer_version")?, + }; + Ok((manifest.manifest_id, manifest)) + }) + .collect() +} + +pub(super) async fn load_manifest_contract_roles( + pool: &PgPool, + watched_contracts: &[WatchedContract], + progress: &mut Option<&mut dyn StartupAdapterProgress>, +) -> Result> { + let mut manifest_ids = HashSet::new(); + for (index, contract) in watched_contracts.iter().enumerate() { + manifest_ids.extend(contract.source_manifest_id); + record_emitter_progress(pool, progress, index + 1, watched_contracts.len()).await?; + } + let manifest_ids = manifest_ids.into_iter().collect::>(); + if manifest_ids.is_empty() { + return Ok(HashMap::new()); + } + + let rows = sqlx::query( + r#" + SELECT manifest_id, contract_instance_id, role + FROM manifest_contract_instances + WHERE declaration_kind = 'contract' + AND manifest_id = ANY($1::BIGINT[]) + "#, + ) + .bind(&manifest_ids) + .fetch_all(pool) + .await + .context("failed to load manifest contract roles for ENSv1 unwrapped authority")?; + + rows.into_iter() + .map(|row| { + Ok(( + ( + sql_row::get(&row, "manifest_id")?, + sql_row::get(&row, "contract_instance_id")?, + ), + sql_row::get(&row, "role")?, + )) + }) + .collect() +} + +pub(super) fn unwrapped_authority_source_families() -> Vec { + [ + SOURCE_FAMILY_ENS_V1_REGISTRAR_L1, + SOURCE_FAMILY_ENS_V1_REGISTRY_L1, + SOURCE_FAMILY_ENS_V1_RESOLVER_L1, + SOURCE_FAMILY_ENS_V1_WRAPPER_L1, + SOURCE_FAMILY_BASENAMES_BASE_REGISTRAR, + SOURCE_FAMILY_BASENAMES_BASE_REGISTRY, + SOURCE_FAMILY_BASENAMES_BASE_RESOLVER, + ] + .into_iter() + .map(str::to_owned) + .collect() +} + +pub(super) async fn record_emitter_progress( + pool: &PgPool, + progress: &mut Option<&mut dyn StartupAdapterProgress>, + completed: usize, + total: usize, +) -> Result<()> { + if completed == total || completed.is_multiple_of(STARTUP_ADAPTER_PROGRESS_PAGE_ROWS) { + record_startup_adapter_progress(pool, progress).await?; + } + Ok(()) +} + +pub(super) fn source_rank(source: WatchedContractSource) -> i32 { + crate::adapter_manifest::source_rank(source) +} + +pub(super) async fn load_active_manifest_metadata_for_source_family( + pool: &PgPool, + chain: &str, + source_family: &str, +) -> Result> { + let rows = sqlx::query( + r#" + SELECT manifest_id, chain, namespace, source_family, manifest_version, normalizer_version + FROM manifest_versions + WHERE rollout_status = 'active' + AND chain = $1 + AND source_family = $2 + ORDER BY manifest_id + "#, + ) + .bind(chain) + .bind(source_family) + .fetch_all(pool) + .await + .with_context(|| { + format!("failed to load active {source_family} manifest metadata for {chain}") + })?; + + rows.into_iter() + .map(|row| { + Ok(ActiveManifestMetadata { + manifest_id: sql_row::get(&row, "manifest_id")?, + chain: sql_row::get(&row, "chain")?, + namespace: sql_row::get(&row, "namespace")?, + source_family: sql_row::get(&row, "source_family")?, + manifest_version: sql_row::get(&row, "manifest_version")?, + normalizer_version: sql_row::get(&row, "normalizer_version")?, + }) + }) + .collect() +} diff --git a/crates/adapters/src/ens_v1_unwrapped_authority/loading/blocks.rs b/crates/adapters/src/ens_v1_unwrapped_authority/loading/blocks.rs index a2e53cc1..d9bd54ed 100644 --- a/crates/adapters/src/ens_v1_unwrapped_authority/loading/blocks.rs +++ b/crates/adapters/src/ens_v1_unwrapped_authority/loading/blocks.rs @@ -3,6 +3,10 @@ use anyhow::{Context, Result}; use bigname_storage::sql_row; use sqlx::{PgPool, postgres::PgRow, types::time::OffsetDateTime}; +use crate::checkpoint_context::StartupAdapterProgress; + +const CANONICAL_BLOCK_PROGRESS_ROWS: i64 = 10_000; + pub(in crate::ens_v1_unwrapped_authority) async fn load_canonical_blocks( pool: &PgPool, chain: &str, @@ -46,6 +50,65 @@ pub(in crate::ens_v1_unwrapped_authority) async fn load_canonical_blocks( .collect() } +pub(in crate::ens_v1_unwrapped_authority) async fn load_canonical_blocks_with_progress( + pool: &PgPool, + chain: &str, + target_block_number: Option, + progress: &mut dyn StartupAdapterProgress, +) -> Result> { + let mut blocks = Vec::new(); + let mut after = None::<(i64, String)>; + loop { + let rows = sqlx::query( + r#" + SELECT + chain_id, + block_hash, + block_number, + block_timestamp, + canonicality_state::TEXT AS canonicality_state + FROM chain_lineage + WHERE chain_id = $1 + AND ($2::BIGINT IS NULL OR block_number <= $2::BIGINT) + AND canonicality_state IN ( + 'canonical'::canonicality_state, + 'safe'::canonicality_state, + 'finalized'::canonicality_state + ) + AND ( + $3::BIGINT IS NULL + OR (block_number, block_hash) > ($3::BIGINT, $4::TEXT) + ) + ORDER BY block_number, block_hash + LIMIT $5 + "#, + ) + .bind(chain) + .bind(target_block_number) + .bind(after.as_ref().map(|(number, _)| *number)) + .bind(after.as_ref().map(|(_, hash)| hash.as_str())) + .bind(CANONICAL_BLOCK_PROGRESS_ROWS) + .fetch_all(pool) + .await + .with_context(|| format!("failed to page canonical raw blocks for chain {chain}"))?; + if rows.is_empty() { + break; + } + let page_len = rows.len(); + for row in rows { + let block = raw_block_snapshot_from_row(row)?; + after = Some((block.block_number, block.block_hash.clone())); + blocks.push(block); + } + progress.record(pool).await?; + if page_len < usize::try_from(CANONICAL_BLOCK_PROGRESS_ROWS).expect("page limit fits usize") + { + break; + } + } + Ok(blocks) +} + pub(in crate::ens_v1_unwrapped_authority) async fn load_canonical_block_at_number( pool: &PgPool, chain: &str, diff --git a/crates/adapters/src/ens_v1_unwrapped_authority/pipeline.rs b/crates/adapters/src/ens_v1_unwrapped_authority/pipeline.rs index e1c2b9fb..9e771e8a 100644 --- a/crates/adapters/src/ens_v1_unwrapped_authority/pipeline.rs +++ b/crates/adapters/src/ens_v1_unwrapped_authority/pipeline.rs @@ -22,7 +22,7 @@ use profile_stream::{ResolverProfileStreamInput, sync_resolver_profile_stream}; use summary::empty_summary; const FULL_REPLAY_RAW_LOG_STREAM_MAX_BLOCK_SCAN_SPAN: i64 = 262_144; -const FULL_REPLAY_RAW_LOG_STREAM_DEFAULT_MAX_LOGS_PER_PAGE: usize = 100_000; +const FULL_REPLAY_RAW_LOG_STREAM_DEFAULT_MAX_LOGS_PER_PAGE: usize = 1_000; pub(super) async fn sync_ens_v1_unwrapped_authority_with_scope( pool: &PgPool, @@ -44,15 +44,22 @@ pub(super) async fn sync_ens_v1_unwrapped_authority_with_scope( let active_emitters_started = Instant::now(); let generic_resolver_event_sources = load_generic_resolver_event_sources(pool, chain, source_scope.as_deref()).await?; - let active_emitters = load_active_emitters(pool, chain, source_scope.as_deref()).await?; - let raw_log_active_emitters = active_emitters - .iter() - .filter(|emitter| { - generic_resolver_event_sources.is_empty() - || emitter.source_family != SOURCE_FAMILY_ENS_V1_RESOLVER_L1 - }) - .cloned() - .collect::>(); + let active_emitters = + load_active_emitters(pool, chain, source_scope.as_deref(), &mut startup_progress).await?; + let mut raw_log_active_emitters = Vec::new(); + for (index, emitter) in active_emitters.iter().enumerate() { + if generic_resolver_event_sources.is_empty() + || emitter.source_family != SOURCE_FAMILY_ENS_V1_RESOLVER_L1 + { + raw_log_active_emitters.push(emitter.clone()); + } + if index + 1 == active_emitters.len() + || (index + 1) + .is_multiple_of(crate::startup_progress::STARTUP_ADAPTER_PROGRESS_PAGE_ROWS) + { + record_startup_adapter_progress(pool, &mut startup_progress).await?; + } + } let active_emitters_ms = active_emitters_started.elapsed().as_millis(); if active_emitters.is_empty() && generic_resolver_event_sources.is_empty() { let summary = empty_summary(0); @@ -81,14 +88,17 @@ pub(super) async fn sync_ens_v1_unwrapped_authority_with_scope( && replay_checkpoint.is_none(), "resolver-profile replay cannot be combined with another restricted or checkpointed authority scope" ); - return sync_resolver_profile_stream(ResolverProfileStreamInput { - pool, - chain, - raw_log_active_emitters: &raw_log_active_emitters, - generic_resolver_event_sources: &generic_resolver_event_sources, - event_topics: &event_topics, - replay, - }) + return sync_resolver_profile_stream( + ResolverProfileStreamInput { + pool, + chain, + raw_log_active_emitters: &raw_log_active_emitters, + generic_resolver_event_sources: &generic_resolver_event_sources, + event_topics: &event_topics, + replay, + }, + &mut startup_progress, + ) .await; } let max_raw_logs_per_page = replay_max_raw_logs_per_page @@ -127,14 +137,16 @@ pub(super) async fn sync_ens_v1_unwrapped_authority_with_scope( active_replay_checkpoint = Some(checkpoint); } let canonical_blocks_started = Instant::now(); - let canonical_blocks = load_canonical_blocks( - pool, - chain, - active_replay_checkpoint - .as_ref() - .map(UnwrappedAuthorityReplayCheckpoint::target_block_number), - ) - .await?; + let target_block_number = active_replay_checkpoint + .as_ref() + .map(UnwrappedAuthorityReplayCheckpoint::target_block_number); + let canonical_blocks = match startup_progress.as_deref_mut() { + Some(progress) => { + load_canonical_blocks_with_progress(pool, chain, target_block_number, progress) + .await? + } + None => load_canonical_blocks(pool, chain, target_block_number).await?, + }; canonical_blocks_ms = canonical_blocks_started.elapsed().as_millis(); if canonical_blocks.is_empty() { let summary = empty_summary(0); @@ -160,12 +172,17 @@ pub(super) async fn sync_ens_v1_unwrapped_authority_with_scope( .cloned() .context("canonical block index must contain a head block")?; let reverse_claim_sources_started = Instant::now(); - let reverse_claim_sources = load_reverse_claim_sources(pool, chain).await?; + let reverse_claim_sources = match startup_progress.as_deref_mut() { + Some(progress) => { + load_reverse_claim_sources_with_progress(pool, chain, progress).await? + } + None => load_reverse_claim_sources(pool, chain).await?, + }; reverse_claim_sources_ms = reverse_claim_sources_started.elapsed().as_millis(); if let Some(checkpoint) = active_replay_checkpoint.as_ref() { let include_replay_auxiliary_state = checkpoint.needs_replay_auxiliary_state(); if let Some(state) = checkpoint - .load_state(pool, include_replay_auxiliary_state) + .load_state(pool, include_replay_auxiliary_state, &mut startup_progress) .await? { histories = state.histories; @@ -293,6 +310,7 @@ pub(super) async fn sync_ens_v1_unwrapped_authority_with_scope( &mut reverse_histories, &mut checkpoint_delta, &mut flushed_events, + &mut startup_progress, ) .await? } else { @@ -325,6 +343,7 @@ pub(super) async fn sync_ens_v1_unwrapped_authority_with_scope( }, &checkpoint_delta, &flushed_events, + &mut startup_progress, ) .await?; record_startup_adapter_progress(pool, &mut startup_progress).await?; @@ -393,6 +412,7 @@ pub(super) async fn sync_ens_v1_unwrapped_authority_with_scope( .await?; raw_log_load_ms = raw_log_load_started.elapsed().as_millis(); scanned_log_count = raw_logs.len(); + record_startup_adapter_progress(pool, &mut startup_progress).await?; if raw_logs.is_empty() { return Ok(empty_summary(scanned_log_count)); } @@ -406,6 +426,7 @@ pub(super) async fn sync_ens_v1_unwrapped_authority_with_scope( ) .await?; canonical_blocks_ms = canonical_blocks_started.elapsed().as_millis(); + record_startup_adapter_progress(pool, &mut startup_progress).await?; if canonical_blocks.is_empty() { return Ok(empty_summary(scanned_log_count)); } @@ -421,6 +442,7 @@ pub(super) async fn sync_ens_v1_unwrapped_authority_with_scope( HashMap::new() }; reverse_claim_sources_ms = reverse_claim_sources_started.elapsed().as_millis(); + record_startup_adapter_progress(pool, &mut startup_progress).await?; let resolver_profile_gate_started = Instant::now(); let resolver_profile_gate = if !resolver_profile_fact_nodes.is_empty() { ResolverProfileGate::load_for_raw_logs(pool, &raw_logs, &event_topics).await? @@ -428,10 +450,12 @@ pub(super) async fn sync_ens_v1_unwrapped_authority_with_scope( ResolverProfileGate::default() }; resolver_profile_gate_ms += resolver_profile_gate_started.elapsed().as_millis(); + record_startup_adapter_progress(pool, &mut startup_progress).await?; let same_tx_name_intro_started = Instant::now(); let same_tx_name_intro_positions = name_intro_positions_for_raw_logs(&raw_logs, &event_topics)?; same_tx_name_intro_ms = same_tx_name_intro_started.elapsed().as_millis(); + record_startup_adapter_progress(pool, &mut startup_progress).await?; let preload_name_metadata_started = Instant::now(); preload_name_metadata_for_raw_logs( pool, @@ -441,6 +465,7 @@ pub(super) async fn sync_ens_v1_unwrapped_authority_with_scope( ) .await?; preload_name_metadata_ms = preload_name_metadata_started.elapsed().as_millis(); + record_startup_adapter_progress(pool, &mut startup_progress).await?; for name in known_names_by_namehash.values() { if let Some(labelhash) = name.labelhashes.first() { namehash_to_labelhash.insert(name.namehash.clone(), labelhash.clone()); @@ -461,6 +486,7 @@ pub(super) async fn sync_ens_v1_unwrapped_authority_with_scope( .await?; preload_restricted_histories_ms = preload_restricted_histories_started.elapsed().as_millis(); + record_startup_adapter_progress(pool, &mut startup_progress).await?; let preload_migrated_registry_nodes = raw_logs .iter() @@ -481,10 +507,11 @@ pub(super) async fn sync_ens_v1_unwrapped_authority_with_scope( ) .await?; migrated_registry_nodes_ms = migrated_registry_nodes_started.elapsed().as_millis(); + record_startup_adapter_progress(pool, &mut startup_progress).await?; } let apply_started = Instant::now(); - for raw_log in &raw_logs { + for (index, raw_log) in raw_logs.iter().enumerate() { if apply_authority_raw_log( raw_log, &mut histories, @@ -503,6 +530,13 @@ pub(super) async fn sync_ens_v1_unwrapped_authority_with_scope( )? { matched_log_count += 1; } + crate::startup_progress::record_processed_row_progress( + pool, + &mut startup_progress, + index + 1, + raw_logs.len(), + ) + .await?; } apply_ms = apply_started.elapsed().as_millis(); } diff --git a/crates/adapters/src/ens_v1_unwrapped_authority/pipeline/entrypoints.rs b/crates/adapters/src/ens_v1_unwrapped_authority/pipeline/entrypoints.rs index e1b9b162..99ea8547 100644 --- a/crates/adapters/src/ens_v1_unwrapped_authority/pipeline/entrypoints.rs +++ b/crates/adapters/src/ens_v1_unwrapped_authority/pipeline/entrypoints.rs @@ -39,6 +39,23 @@ pub async fn sync_ens_v1_unwrapped_authority_with_replay_checkpoint_and_log_limi .await } +pub async fn sync_ens_v1_unwrapped_authority_with_replay_checkpoint_and_log_limit_and_progress( + pool: &PgPool, + chain: &str, + checkpoint: &ReplayAdapterCheckpointContext, + max_raw_logs_per_page: usize, + progress: &mut dyn StartupAdapterProgress, +) -> Result { + sync_ens_v1_unwrapped_authority_with_checkpoint_context( + pool, + chain, + &AdapterCheckpointContext::for_replay(checkpoint), + max_raw_logs_per_page, + Some(progress), + ) + .await +} + pub async fn sync_ens_v1_unwrapped_authority_with_startup_checkpoint_and_log_limit( pool: &PgPool, chain: &str, @@ -141,6 +158,49 @@ impl EnsV1UnwrappedAuthoritySyncSummary { .await } + pub async fn sync_for_block_hashes_with_progress( + pool: &PgPool, + chain: &str, + block_hashes: &[String], + progress: &mut dyn StartupAdapterProgress, + ) -> Result { + sync_ens_v1_unwrapped_authority_with_scope( + pool, + chain, + true, + block_hashes, + None, + None, + None, + None, + None, + Some(progress), + ) + .await + } + + pub async fn sync_for_block_hashes_with_source_scope_and_progress( + pool: &PgPool, + chain: &str, + block_hashes: &[String], + source_scope: &[(String, String, i64, i64)], + progress: &mut dyn StartupAdapterProgress, + ) -> Result { + sync_ens_v1_unwrapped_authority_with_scope( + pool, + chain, + true, + block_hashes, + None, + Some(source_scope), + None, + None, + None, + Some(progress), + ) + .await + } + pub async fn sync_for_block_hashes_with_source_scope_and_transactions( pool: &PgPool, chain: &str, diff --git a/crates/adapters/src/ens_v1_unwrapped_authority/pipeline/flush.rs b/crates/adapters/src/ens_v1_unwrapped_authority/pipeline/flush.rs index 90d51bf3..b2e142ff 100644 --- a/crates/adapters/src/ens_v1_unwrapped_authority/pipeline/flush.rs +++ b/crates/adapters/src/ens_v1_unwrapped_authority/pipeline/flush.rs @@ -47,6 +47,7 @@ pub(super) async fn stage_startup_checkpoint_events( reverse_histories: &mut BTreeMap, checkpoint_delta: &mut UnwrappedAuthorityReplayCheckpointDelta, staged_events: &mut UnwrappedAuthorityReplayFlushedEvents, + startup_progress: &mut Option<&mut dyn StartupAdapterProgress>, ) -> Result { let mut staged_count = 0usize; let mut buffer = Vec::with_capacity(REPLAY_EVENT_FLUSH_BATCH_SIZE); @@ -56,19 +57,40 @@ pub(super) async fn stage_startup_checkpoint_events( checkpoint_delta.mark_history(key.clone()); buffer.append(&mut history.events); } - staged_count += - stage_startup_event_buffer(pool, checkpoint, &mut buffer, staged_events, false).await?; + staged_count += stage_startup_event_buffer( + pool, + checkpoint, + &mut buffer, + staged_events, + false, + startup_progress, + ) + .await?; } for (key, history) in reverse_histories.iter_mut() { if !history.events.is_empty() { checkpoint_delta.mark_reverse_history(key.clone()); buffer.append(&mut history.events); } - staged_count += - stage_startup_event_buffer(pool, checkpoint, &mut buffer, staged_events, false).await?; + staged_count += stage_startup_event_buffer( + pool, + checkpoint, + &mut buffer, + staged_events, + false, + startup_progress, + ) + .await?; } - staged_count += - stage_startup_event_buffer(pool, checkpoint, &mut buffer, staged_events, true).await?; + staged_count += stage_startup_event_buffer( + pool, + checkpoint, + &mut buffer, + staged_events, + true, + startup_progress, + ) + .await?; Ok(staged_count) } @@ -78,13 +100,14 @@ async fn stage_startup_event_buffer( buffer: &mut Vec, staged_events: &mut UnwrappedAuthorityReplayFlushedEvents, flush_partial: bool, + startup_progress: &mut Option<&mut dyn StartupAdapterProgress>, ) -> Result { if buffer.is_empty() || (!flush_partial && buffer.len() < REPLAY_EVENT_FLUSH_BATCH_SIZE) { return Ok(0); } let event_count = buffer.len(); checkpoint - .stage_startup_events(pool, buffer, staged_events) + .stage_startup_events(pool, buffer, staged_events, startup_progress) .await?; buffer.clear(); if buffer.capacity() > REPLAY_EVENT_FLUSH_BATCH_SIZE * 4 { diff --git a/crates/adapters/src/ens_v1_unwrapped_authority/pipeline/profile_stream.rs b/crates/adapters/src/ens_v1_unwrapped_authority/pipeline/profile_stream.rs index c992af38..c10729f4 100644 --- a/crates/adapters/src/ens_v1_unwrapped_authority/pipeline/profile_stream.rs +++ b/crates/adapters/src/ens_v1_unwrapped_authority/pipeline/profile_stream.rs @@ -18,6 +18,7 @@ pub(super) struct ResolverProfileStreamInput<'a> { pub(super) async fn sync_resolver_profile_stream( input: ResolverProfileStreamInput<'_>, + progress: &mut Option<&mut dyn StartupAdapterProgress>, ) -> Result { let router = AuthorityRawLogStreamSourceRouter::new( input.raw_log_active_emitters, @@ -179,6 +180,7 @@ pub(super) async fn sync_resolver_profile_stream( input .replay .record_page(raw_logs.len(), state.live_item_count(), payload_bytes); + record_startup_adapter_progress(input.pool, progress).await?; from_block = to_block .checked_add(1) .context("resolver-profile page boundary overflowed")?; diff --git a/crates/adapters/src/ens_v1_unwrapped_authority/resolver_profile_reconciliation.rs b/crates/adapters/src/ens_v1_unwrapped_authority/resolver_profile_reconciliation.rs index a1c41580..5ea62737 100644 --- a/crates/adapters/src/ens_v1_unwrapped_authority/resolver_profile_reconciliation.rs +++ b/crates/adapters/src/ens_v1_unwrapped_authority/resolver_profile_reconciliation.rs @@ -1,4 +1,5 @@ use super::*; +use crate::checkpoint_context::StartupAdapterProgress; use anyhow::ensure; use sqlx::Postgres; @@ -8,7 +9,7 @@ pub use targets::{ ResolverProfileEventReconciliation, ResolverProfileEventReconciliationPublication, }; -const DEFAULT_REPLAY_MAX_RAW_LOGS_PER_PAGE: usize = 100_000; +const DEFAULT_REPLAY_MAX_RAW_LOGS_PER_PAGE: usize = 1_000; #[derive(Clone, Debug, Default, Eq, PartialEq)] pub struct ResolverProfileEventReconciliationSummary { @@ -96,7 +97,7 @@ pub(super) async fn reconcile_resolver_profile_events_with_log_limit( let mut reconciliation = begin_resolver_profile_event_reconciliation(pool, chain).await?; reconciliation.stage_addresses(resolver_addresses).await?; reconciliation - .reconcile_with_log_limit(max_raw_logs_per_page) + .reconcile_with_log_limit(max_raw_logs_per_page, None) .await? .finish() .await @@ -114,19 +115,28 @@ impl ResolverProfileEventReconciliation { /// staged. The returned publication retains the exact target set until the /// indexer durably publishes its projection invalidations. pub async fn reconcile(self) -> Result { - self.reconcile_with_log_limit(DEFAULT_REPLAY_MAX_RAW_LOGS_PER_PAGE) + self.reconcile_with_log_limit(DEFAULT_REPLAY_MAX_RAW_LOGS_PER_PAGE, None) + .await + } + + pub async fn reconcile_with_progress( + self, + progress: &mut dyn StartupAdapterProgress, + ) -> Result { + self.reconcile_with_log_limit(DEFAULT_REPLAY_MAX_RAW_LOGS_PER_PAGE, Some(progress)) .await } async fn reconcile_with_log_limit( mut self, max_raw_logs_per_page: usize, + mut progress: Option<&mut dyn StartupAdapterProgress>, ) -> Result { ensure!( max_raw_logs_per_page > 0, "resolver profile reconciliation max logs per page must be positive" ); - let prepared = self.prepare().await?; + let prepared = self.prepare(&mut progress).await?; let targets::ResolverProfileEventReconciliation { pool, chain, @@ -161,19 +171,38 @@ impl ResolverProfileEventReconciliation { // Derive one chronological history under the current resolver profile. // Resolver facts define the inclusive repair range; registry, registrar, // and wrapper facts inside that range remain required authority context. - let replay = pipeline::sync_ens_v1_unwrapped_authority_with_scope( - &pool, - &chain, - false, - &[], - None, - None, - None, - None, - Some(&mut replay_context), - None, - ) - .await?; + let replay = match progress.as_mut() { + Some(progress) => { + pipeline::sync_ens_v1_unwrapped_authority_with_scope( + &pool, + &chain, + false, + &[], + None, + None, + None, + None, + Some(&mut replay_context), + Some(&mut **progress), + ) + .await? + } + None => { + pipeline::sync_ens_v1_unwrapped_authority_with_scope( + &pool, + &chain, + false, + &[], + None, + None, + None, + None, + Some(&mut replay_context), + None, + ) + .await? + } + }; summary.scanned_log_count = replay.scanned_log_count; summary.matched_log_count = replay.matched_log_count; summary.normalized_event_count = replay.total_normalized_event_count; @@ -185,12 +214,14 @@ impl ResolverProfileEventReconciliation { mark_reconciliation_replay_complete(&pool, &chain, run_id).await?; let (inserted_count, orphaned_count) = publish_resolver_profile_events( + &pool, raw_log_guard.transaction_mut(), &chain, replay_range, run_id, &prepared.resolver_address_set_digest, prepared.resolver_address_count, + &mut progress, ) .await?; summary.normalized_event_inserted_count = inserted_count; @@ -234,12 +265,14 @@ async fn mark_reconciliation_replay_complete( } async fn publish_resolver_profile_events( + pool: &PgPool, transaction: &mut sqlx::Transaction<'_, Postgres>, chain: &str, replay_range: ResolverEmitterReplayRange, run_id: Uuid, resolver_address_set_digest: &str, resolver_address_count: usize, + progress: &mut Option<&mut dyn StartupAdapterProgress>, ) -> Result<(usize, u64)> { let run = sqlx::query_as::<_, (i64, i64, i64, String, String)>( r#" @@ -304,13 +337,35 @@ async fn publish_resolver_profile_events( &events, ) .await?; + record_progress(pool, progress).await?; } - let orphaned = sqlx::query( - r#" - WITH stale AS ( + let event_watermark = sqlx::query_scalar::<_, i64>( + "SELECT COALESCE(MAX(normalized_event_id), 0)::BIGINT FROM normalized_events", + ) + .fetch_one(transaction.as_mut()) + .await + .context("failed to load resolver-profile orphan scan watermark")?; + let mut after_event_id = 0i64; + let mut orphaned = 0u64; + while after_event_id < event_watermark { + let (last_event_id, page_orphaned_count) = sqlx::query_as::<_, (Option, i64)>( + r#" + WITH source_page AS MATERIALIZED ( + SELECT * + FROM normalized_events + WHERE normalized_event_id > $5 + AND normalized_event_id <= $6 + ORDER BY normalized_event_id + LIMIT 1000 + ), + page_end AS ( + SELECT MAX(normalized_event_id) AS last_event_id + FROM source_page + ), + stale AS ( SELECT DISTINCT event.normalized_event_id - FROM normalized_events event + FROM source_page event JOIN raw_logs raw_log ON raw_log.chain_id = event.chain_id AND raw_log.block_hash = event.block_hash @@ -337,20 +392,53 @@ async fn publish_resolver_profile_events( AND event.canonicality_state IN ('canonical', 'safe', 'finalized') AND raw_log.canonicality_state IN ('canonical', 'safe', 'finalized') AND lineage.canonicality_state IN ('canonical', 'safe', 'finalized') + ), + updated AS ( + UPDATE normalized_events event + SET canonicality_state = 'orphaned'::canonicality_state, observed_at = now() + FROM stale + WHERE event.normalized_event_id = stale.normalized_event_id + RETURNING 1 ) - UPDATE normalized_events event - SET canonicality_state = 'orphaned'::canonicality_state, observed_at = now() - FROM stale - WHERE event.normalized_event_id = stale.normalized_event_id + SELECT + page_end.last_event_id, + (SELECT COUNT(*)::BIGINT FROM updated) + FROM page_end "#, - ) - .bind(chain) - .bind(replay_range.first_block_number) - .bind(replay_range.last_block_number) - .bind(run_id) - .execute(transaction.as_mut()) - .await - .with_context(|| format!("failed to orphan stale resolver-profile events for {chain}"))? - .rows_affected(); + ) + .bind(chain) + .bind(replay_range.first_block_number) + .bind(replay_range.last_block_number) + .bind(run_id) + .bind(after_event_id) + .bind(event_watermark) + .fetch_one(transaction.as_mut()) + .await + .with_context(|| { + format!("failed to orphan stale resolver-profile event page for {chain}") + })?; + let Some(last_event_id) = last_event_id else { + break; + }; + ensure!( + last_event_id > after_event_id, + "resolver-profile orphan scan did not advance" + ); + after_event_id = last_event_id; + orphaned = orphaned + .checked_add(u64::try_from(page_orphaned_count)?) + .context("resolver-profile orphan count overflowed u64")?; + record_progress(pool, progress).await?; + } Ok((inserted_count, orphaned)) } + +async fn record_progress( + pool: &PgPool, + progress: &mut Option<&mut dyn StartupAdapterProgress>, +) -> Result<()> { + if let Some(progress) = progress.as_deref_mut() { + progress.record(pool).await?; + } + Ok(()) +} diff --git a/crates/adapters/src/ens_v1_unwrapped_authority/resolver_profile_reconciliation/targets.rs b/crates/adapters/src/ens_v1_unwrapped_authority/resolver_profile_reconciliation/targets.rs index 3f3cf783..3f0095e5 100644 --- a/crates/adapters/src/ens_v1_unwrapped_authority/resolver_profile_reconciliation/targets.rs +++ b/crates/adapters/src/ens_v1_unwrapped_authority/resolver_profile_reconciliation/targets.rs @@ -6,6 +6,7 @@ use bigname_storage::{RawLogStagingReadGuard, acquire_raw_log_staging_read_guard use sqlx::{PgConnection, PgPool, Postgres, QueryBuilder, types::Uuid}; use super::{ResolverEmitterReplayRange, ResolverProfileEventReconciliationSummary}; +use crate::checkpoint_context::{StartupAdapterProgress, record_startup_adapter_progress}; const TARGET_BATCH_SIZE: usize = 1_000; @@ -115,15 +116,19 @@ impl ResolverProfileEventReconciliation { Ok(()) } - pub(super) async fn prepare(&mut self) -> Result { + pub(super) async fn prepare( + &mut self, + progress: &mut Option<&mut dyn StartupAdapterProgress>, + ) -> Result { let (resolver_address_count, resolver_address_set_digest) = - load_target_metadata(&self.pool, self.run_id).await?; + load_target_metadata(&self.pool, self.run_id, progress).await?; ensure!( resolver_address_count > 0, "resolver-profile reconciliation must stage at least one target" ); let replay_range = - load_resolver_emitter_replay_range(&self.pool, &self.chain, self.run_id).await?; + load_resolver_emitter_replay_range(&self.pool, &self.chain, self.run_id, progress) + .await?; let (first_block_number, last_block_number, status) = replay_range .map_or((0, 0, "replay_complete"), |range| { (range.first_block_number, range.last_block_number, "running") @@ -233,7 +238,11 @@ fn normalized_resolver_addresses(resolver_addresses: &[String]) -> Result Result<(usize, String)> { +async fn load_target_metadata( + pool: &PgPool, + run_id: Uuid, + progress: &mut Option<&mut dyn StartupAdapterProgress>, +) -> Result<(usize, String)> { let mut after = None::; let mut count = 0usize; let mut digest = Keccak256::new(); @@ -265,6 +274,7 @@ async fn load_target_metadata(pool: &PgPool, run_id: Uuid) -> Result<(usize, Str digest.update(address.as_bytes()); count += 1; } + record_startup_adapter_progress(pool, progress).await?; } Ok((count, format!("{:#x}", digest.finalize()))) } @@ -273,23 +283,61 @@ async fn load_resolver_emitter_replay_range( pool: &PgPool, chain: &str, run_id: Uuid, + progress: &mut Option<&mut dyn StartupAdapterProgress>, ) -> Result> { - let (first_block_number, last_block_number, resolver_block_count, invalid_lineage_count) = - sqlx::query_as::<_, (Option, Option, i64, i64)>( + let mut transaction = pool + .begin() + .await + .context("failed to begin resolver-emitter replay-range scan")?; + sqlx::query("SET TRANSACTION ISOLATION LEVEL REPEATABLE READ") + .execute(transaction.as_mut()) + .await + .context("failed to pin resolver-emitter replay-range scan")?; + sqlx::query( + r#" + CREATE TEMP TABLE resolver_profile_replay_block_hashes ( + block_hash TEXT PRIMARY KEY + ) ON COMMIT DROP + "#, + ) + .execute(transaction.as_mut()) + .await + .context("failed to create resolver-emitter replay block set")?; + + let mut after_raw_log_id = 0i64; + let mut first_block_number = None::; + let mut last_block_number = None::; + let mut resolver_block_count = 0usize; + let mut invalid_lineage_count = 0usize; + loop { + let raw_log_ids = sqlx::query_scalar::<_, i64>( r#" - SELECT - MIN(raw_log.block_number), - MAX(raw_log.block_number), - COUNT(DISTINCT raw_log.block_hash)::BIGINT, - COUNT(*) FILTER ( - WHERE lineage.block_hash IS NULL - OR lineage.block_number <> raw_log.block_number - OR lineage.canonicality_state NOT IN ( - 'canonical'::canonicality_state, - 'safe'::canonicality_state, - 'finalized'::canonicality_state - ) - )::BIGINT AS invalid_lineage_count + SELECT raw_log_id + FROM raw_logs + WHERE raw_log_id > $1 + ORDER BY raw_log_id + LIMIT $2 + "#, + ) + .bind(after_raw_log_id) + .bind(i64::try_from(TARGET_BATCH_SIZE)?) + .fetch_all(transaction.as_mut()) + .await + .context("failed to page raw-log identities for resolver replay range")?; + let Some(last_raw_log_id) = raw_log_ids.last().copied() else { + break; + }; + after_raw_log_id = last_raw_log_id; + let rows = sqlx::query( + r#" + SELECT raw_log.block_number, raw_log.block_hash, + lineage.block_hash IS NOT NULL + AND lineage.block_number = raw_log.block_number + AND lineage.canonicality_state IN ( + 'canonical'::canonicality_state, + 'safe'::canonicality_state, + 'finalized'::canonicality_state + ) AS valid_lineage FROM raw_logs raw_log JOIN resolver_profile_reconciliation_targets target ON target.run_id = $2 @@ -298,6 +346,7 @@ async fn load_resolver_emitter_replay_range( ON lineage.chain_id = raw_log.chain_id AND lineage.block_hash = raw_log.block_hash WHERE raw_log.chain_id = $1 + AND raw_log.raw_log_id = ANY($3::BIGINT[]) AND raw_log.canonicality_state IN ( 'canonical'::canonicality_state, 'safe'::canonicality_state, @@ -307,17 +356,50 @@ async fn load_resolver_emitter_replay_range( ) .bind(chain) .bind(run_id) - .fetch_one(pool) + .bind(&raw_log_ids) + .fetch_all(transaction.as_mut()) .await .with_context(|| { - format!("failed to load retained resolver-emitter replay range for chain {chain}") + format!("failed to load a retained resolver-emitter replay page for chain {chain}") })?; - + let mut page_hashes = BTreeSet::new(); + for row in rows { + let block_number: i64 = sqlx::Row::try_get(&row, "block_number")?; + first_block_number = + Some(first_block_number.map_or(block_number, |value| value.min(block_number))); + last_block_number = + Some(last_block_number.map_or(block_number, |value| value.max(block_number))); + page_hashes.insert(sqlx::Row::try_get::(&row, "block_hash")?); + if !sqlx::Row::try_get::(&row, "valid_lineage")? { + invalid_lineage_count += 1; + } + } + if !page_hashes.is_empty() { + let hashes = page_hashes.into_iter().collect::>(); + resolver_block_count += sqlx::query( + r#" + INSERT INTO resolver_profile_replay_block_hashes (block_hash) + SELECT * FROM UNNEST($1::TEXT[]) + ON CONFLICT (block_hash) DO NOTHING + "#, + ) + .bind(&hashes) + .execute(transaction.as_mut()) + .await + .context("failed to accumulate resolver-emitter replay block hashes")? + .rows_affected() as usize; + } + record_startup_adapter_progress(pool, progress).await?; + } if invalid_lineage_count > 0 { bail!( "{invalid_lineage_count} canonical resolver-emitter raw logs lack matching canonical lineage on chain {chain}" ); } + transaction + .commit() + .await + .context("failed to close resolver-emitter replay-range scan")?; let Some(first_block_number) = first_block_number else { ensure!( last_block_number.is_none() && resolver_block_count == 0, @@ -330,7 +412,6 @@ async fn load_resolver_emitter_replay_range( Ok(Some(ResolverEmitterReplayRange { first_block_number, last_block_number, - resolver_block_count: usize::try_from(resolver_block_count) - .context("resolver-emitter block count does not fit usize")?, + resolver_block_count, })) } diff --git a/crates/adapters/src/ens_v1_unwrapped_authority/reverse_claims.rs b/crates/adapters/src/ens_v1_unwrapped_authority/reverse_claims.rs index 02265479..0c75917a 100644 --- a/crates/adapters/src/ens_v1_unwrapped_authority/reverse_claims.rs +++ b/crates/adapters/src/ens_v1_unwrapped_authority/reverse_claims.rs @@ -1,5 +1,10 @@ use super::*; use bigname_storage::sql_row; +use sqlx::postgres::PgRow; + +use crate::checkpoint_context::StartupAdapterProgress; + +const REVERSE_CLAIM_PROGRESS_ROWS: i64 = 1_000; pub(super) async fn load_reverse_claim_sources( pool: &PgPool, @@ -8,6 +13,93 @@ pub(super) async fn load_reverse_claim_sources( load_reverse_claim_sources_internal(pool, chain, None).await } +pub(super) async fn load_reverse_claim_sources_with_progress( + pool: &PgPool, + chain: &str, + progress: &mut dyn StartupAdapterProgress, +) -> Result> { + let mut sources = HashMap::new(); + let mut after_reverse_node = None::; + loop { + let rows = sqlx::query( + r#" + SELECT DISTINCT ON (LOWER(ne.after_state->>'reverse_node')) + LOWER(ne.after_state->>'reverse_node') AS reverse_node, + LOWER(ne.after_state->>'address') AS address, + COALESCE(ne.after_state->>'namespace', ne.namespace) AS namespace, + ne.after_state->>'coin_type' AS coin_type, + ne.after_state->>'reverse_name' AS reverse_name, + COALESCE( + ne.after_state->'claim_provenance'->>'source_family', + ne.source_family + ) AS claim_source_family, + COALESCE( + ne.after_state->'claim_provenance'->>'contract_role', + $4 + ) AS claim_contract_role, + ne.after_state->'claim_provenance'->>'contract_instance_id' AS claim_contract_instance_id, + COALESCE( + ne.after_state->'claim_provenance'->>'emitting_address', + ne.raw_fact_ref->>'emitting_address' + ) AS claim_emitting_address + FROM normalized_events ne + WHERE ne.chain_id = $1 + AND COALESCE(ne.after_state->>'namespace', ne.namespace) IN ($2, $3) + AND ne.event_kind = $5 + AND ne.derivation_kind = $6 + AND ne.canonicality_state IN ( + 'canonical'::canonicality_state, + 'safe'::canonicality_state, + 'finalized'::canonicality_state + ) + AND ne.after_state->>'reverse_node' IS NOT NULL + AND ne.after_state->>'reverse_node' <> '' + AND ne.after_state->>'address' IS NOT NULL + AND ne.after_state->>'address' <> '' + AND ne.after_state->>'coin_type' IS NOT NULL + AND ne.after_state->>'coin_type' <> '' + AND ne.after_state->>'reverse_name' IS NOT NULL + AND ne.after_state->>'reverse_name' <> '' + AND ( + $7::TEXT IS NULL + OR LOWER(ne.after_state->>'reverse_node') > $7::TEXT + ) + ORDER BY + LOWER(ne.after_state->>'reverse_node'), + ne.block_number DESC NULLS LAST, + ne.log_index DESC NULLS LAST, + ne.normalized_event_id DESC + LIMIT $8 + "#, + ) + .bind(chain) + .bind("ens") + .bind("basenames") + .bind(CONTRACT_ROLE_REVERSE_REGISTRAR) + .bind(EVENT_KIND_REVERSE_CHANGED) + .bind(DERIVATION_KIND_ENS_V1_REVERSE_CLAIM) + .bind(after_reverse_node.as_deref()) + .bind(REVERSE_CLAIM_PROGRESS_ROWS) + .fetch_all(pool) + .await + .with_context(|| format!("failed to page reverse claim sources for chain {chain}"))?; + if rows.is_empty() { + break; + } + let page_len = rows.len(); + for row in rows { + let (reverse_node, source) = reverse_claim_source_from_row(&row)?; + after_reverse_node = Some(reverse_node.clone()); + sources.insert(reverse_node, source); + } + progress.record(pool).await?; + if page_len < usize::try_from(REVERSE_CLAIM_PROGRESS_ROWS).expect("page limit fits usize") { + break; + } + } + Ok(sources) +} + pub(super) async fn load_reverse_claim_sources_for_nodes( pool: &PgPool, chain: &str, @@ -143,48 +235,48 @@ async fn load_reverse_claim_sources_internal( .with_context(|| format!("failed to load reverse claim sources for chain {chain}"))? }; - rows.into_iter() - .map(|row| { - let reverse_node = sql_row::get::(&row, "reverse_node")?; - let address = row - .try_get::("address") - .context("missing reverse claim address")?; - let namespace = row - .try_get::("namespace") - .context("missing reverse claim namespace")?; - let coin_type = row - .try_get::("coin_type") - .context("missing reverse claim coin_type")?; - let reverse_name = row - .try_get::("reverse_name") - .context("missing reverse claim reverse_name")?; + rows.iter().map(reverse_claim_source_from_row).collect() +} + +fn reverse_claim_source_from_row(row: &PgRow) -> Result<(String, ReverseClaimSource)> { + let reverse_node = sql_row::get::(row, "reverse_node")?; + let address = row + .try_get::("address") + .context("missing reverse claim address")?; + let namespace = row + .try_get::("namespace") + .context("missing reverse claim namespace")?; + let coin_type = row + .try_get::("coin_type") + .context("missing reverse claim coin_type")?; + let reverse_name = row + .try_get::("reverse_name") + .context("missing reverse claim reverse_name")?; - Ok(( - reverse_node.clone(), - ReverseClaimSource { - address, - namespace, - coin_type, - reverse_name, - reverse_node, - claim_provenance: ReverseClaimProvenance { - source_family: row - .try_get::("claim_source_family") - .context("missing reverse claim source_family")?, - contract_role: row - .try_get::("claim_contract_role") - .context("missing reverse claim contract_role")?, - contract_instance_id: row - .try_get("claim_contract_instance_id") - .context("missing reverse claim contract_instance_id column")?, - emitting_address: row - .try_get("claim_emitting_address") - .context("missing reverse claim emitting_address column")?, - }, - }, - )) - }) - .collect() + Ok(( + reverse_node.clone(), + ReverseClaimSource { + address, + namespace, + coin_type, + reverse_name, + reverse_node, + claim_provenance: ReverseClaimProvenance { + source_family: row + .try_get::("claim_source_family") + .context("missing reverse claim source_family")?, + contract_role: row + .try_get::("claim_contract_role") + .context("missing reverse claim contract_role")?, + contract_instance_id: row + .try_get("claim_contract_instance_id") + .context("missing reverse claim contract_instance_id column")?, + emitting_address: row + .try_get("claim_emitting_address") + .context("missing reverse claim emitting_address column")?, + }, + }, + )) } pub(super) fn apply_reverse_claim_source_observation( diff --git a/crates/adapters/src/ens_v2_common.rs b/crates/adapters/src/ens_v2_common.rs index 2fa4fed0..126b43de 100644 --- a/crates/adapters/src/ens_v2_common.rs +++ b/crates/adapters/src/ens_v2_common.rs @@ -1,16 +1,21 @@ use bigname_storage::sql_row; -use std::collections::HashMap; +use std::collections::{BTreeMap, HashMap, HashSet}; use anyhow::{Context, Result}; -use bigname_manifests::load_watched_contracts; +use bigname_manifests::{load_watched_contracts, load_watched_contracts_scoped_with_progress}; +use futures_util::TryStreamExt; use sqlx::{PgPool, Row, types::Uuid}; use crate::adapter_manifest::{ ActiveManifestMetadata, active_manifest_for_watched_contract, ensure_watched_contract_manifest_chain, load_active_manifest_metadata, - load_latest_active_manifest_metadata_for_source_family, watched_contract_manifest_ids, + load_latest_active_manifest_metadata_for_source_family, required_source_manifest_id, }; pub(crate) use crate::evm_abi::{keccak256_bytes, keccak256_hex}; +use crate::{ + checkpoint_context::{StartupAdapterProgress, record_startup_adapter_progress}, + startup_progress::{STARTUP_ADAPTER_PROGRESS_PAGE_ROWS, StartupManifestProgress}, +}; #[derive(Clone, Debug, Eq, PartialEq)] pub(crate) struct ActiveEmitter { @@ -67,10 +72,10 @@ pub(crate) fn source_scope_bindings( /// discovery-edge `active_to`) and the dedup tie-break is plain id ordering — `ActiveEmitter` here /// carries no `source_rank`, and a collision only happens on an exact identical interval. fn insert_distinct_emitter( - emitters_by_scope: &mut HashMap, + emitters_by_scope: &mut BTreeMap, emitter: ActiveEmitter, ) { - use std::collections::hash_map::Entry; + use std::collections::btree_map::Entry; let scope_key = ( emitter.address.clone(), emitter.active_from_block_number, @@ -130,10 +135,23 @@ pub(crate) async fn load_active_emitters( source_family: &str, resolver_edge_kind: &str, adapter_label: &str, + progress: &mut Option<&mut dyn StartupAdapterProgress>, ) -> Result> { - let watched_contracts = load_watched_contracts(pool) + let watched_contracts = if let Some(progress) = progress.as_deref_mut() { + let mut manifest_progress = StartupManifestProgress::new(progress); + load_watched_contracts_scoped_with_progress( + pool, + Some(chain), + &[source_family.to_owned()], + &mut manifest_progress, + ) .await - .with_context(|| format!("failed to load watched contracts for {adapter_label} adapter"))?; + .with_context(|| format!("failed to load watched contracts for {adapter_label} adapter"))? + } else { + load_watched_contracts(pool).await.with_context(|| { + format!("failed to load watched contracts for {adapter_label} adapter") + })? + }; let watched_contracts = watched_contracts .into_iter() .filter(|contract| contract.chain == chain) @@ -142,13 +160,20 @@ pub(crate) async fn load_active_emitters( return Ok(Vec::new()); } - let manifest_ids = watched_contract_manifest_ids(&watched_contracts)?; + let mut manifest_ids = HashSet::new(); + for (index, watched_contract) in watched_contracts.iter().enumerate() { + manifest_ids.insert(required_source_manifest_id(watched_contract)?); + record_common_progress(pool, progress, index + 1, watched_contracts.len()).await?; + } + let manifest_ids = manifest_ids.into_iter().collect::>(); let context_label = format!("{adapter_label} emitters"); let active_manifests = load_active_manifest_metadata(pool, &manifest_ids, &context_label).await?; - let mut emitters_by_scope = HashMap::::new(); - for watched_contract in watched_contracts { + record_startup_adapter_progress(pool, progress).await?; + let watched_contract_count = watched_contracts.len(); + let mut emitters_by_scope = BTreeMap::::new(); + for (index, watched_contract) in watched_contracts.into_iter().enumerate() { let (source_manifest_id, manifest) = active_manifest_for_watched_contract(&active_manifests, &watched_contract)?; if manifest.source_family != source_family { @@ -169,33 +194,20 @@ pub(crate) async fn load_active_emitters( active_to_block_number: watched_contract.active_to_block_number, }, ); + record_common_progress(pool, progress, index + 1, watched_contract_count).await?; } if let Some(manifest) = load_active_source_family_manifest_metadata(pool, chain, source_family).await? { for emitter in - load_discovered_resolver_emitters(pool, chain, resolver_edge_kind, &manifest).await? + load_discovered_resolver_emitters(pool, chain, resolver_edge_kind, &manifest, progress) + .await? { insert_distinct_emitter(&mut emitters_by_scope, emitter); } } - let mut emitters = emitters_by_scope.into_values().collect::>(); - emitters.sort_by(|left, right| { - left.address - .cmp(&right.address) - .then( - left.active_from_block_number - .cmp(&right.active_from_block_number), - ) - .then( - left.active_to_block_number - .cmp(&right.active_to_block_number), - ) - .then(left.source_manifest_id.cmp(&right.source_manifest_id)) - .then(left.contract_instance_id.cmp(&right.contract_instance_id)) - }); - Ok(emitters) + Ok(emitters_by_scope.into_values().collect()) } async fn load_discovered_resolver_emitters( @@ -203,8 +215,9 @@ async fn load_discovered_resolver_emitters( chain: &str, resolver_edge_kind: &str, manifest: &ActiveManifestMetadata, + progress: &mut Option<&mut dyn StartupAdapterProgress>, ) -> Result> { - let rows = sqlx::query( + let mut rows = sqlx::query( r#" SELECT cia.address, @@ -219,35 +232,58 @@ async fn load_discovered_resolver_emitters( ON cia.contract_instance_id = de.to_contract_instance_id WHERE de.chain_id = $1 AND de.edge_kind = $2 - ORDER BY lower(cia.address), de.active_from_block_number NULLS FIRST, de.discovery_edge_id "#, ) .bind(chain) .bind(resolver_edge_kind) - .fetch_all(pool) - .await - .with_context(|| format!("failed to load ENSv2 discovered resolver emitters for {chain}"))?; - - rows.into_iter() - .map(|row| { - let address = normalize_address( - &row.try_get::("address") - .context("missing discovered resolver address")?, - ); - Ok(ActiveEmitter { - address, - contract_instance_id: row - .try_get("to_contract_instance_id") - .context("missing discovered resolver contract_instance_id")?, - source_manifest_id: manifest.manifest_id, - namespace: manifest.namespace.clone(), - source_family: manifest.source_family.clone(), - manifest_version: manifest.manifest_version, - active_from_block_number: sql_row::get(&row, "active_from_block_number")?, - active_to_block_number: sql_row::get(&row, "active_to_block_number")?, - }) - }) - .collect() + .fetch(pool); + let mut emitters = Vec::new(); + while let Some(row) = rows.try_next().await.with_context(|| { + format!("failed to stream ENSv2 discovered resolver emitters for {chain}") + })? { + let address = normalize_address( + &row.try_get::("address") + .context("missing discovered resolver address")?, + ); + emitters.push(ActiveEmitter { + address, + contract_instance_id: row + .try_get("to_contract_instance_id") + .context("missing discovered resolver contract_instance_id")?, + source_manifest_id: manifest.manifest_id, + namespace: manifest.namespace.clone(), + source_family: manifest.source_family.clone(), + manifest_version: manifest.manifest_version, + active_from_block_number: sql_row::get(&row, "active_from_block_number")?, + active_to_block_number: sql_row::get(&row, "active_to_block_number")?, + }); + if emitters + .len() + .is_multiple_of(STARTUP_ADAPTER_PROGRESS_PAGE_ROWS) + { + record_startup_adapter_progress(pool, progress).await?; + } + } + if !emitters.is_empty() + && !emitters + .len() + .is_multiple_of(STARTUP_ADAPTER_PROGRESS_PAGE_ROWS) + { + record_startup_adapter_progress(pool, progress).await?; + } + Ok(emitters) +} + +async fn record_common_progress( + pool: &PgPool, + progress: &mut Option<&mut dyn StartupAdapterProgress>, + completed: usize, + total: usize, +) -> Result<()> { + if completed == total || completed.is_multiple_of(STARTUP_ADAPTER_PROGRESS_PAGE_ROWS) { + record_startup_adapter_progress(pool, progress).await?; + } + Ok(()) } async fn load_active_source_family_manifest_metadata( @@ -315,7 +351,7 @@ mod tests { #[test] fn insert_distinct_emitter_preserves_disjoint_intervals_per_address() { - let mut by_scope = HashMap::new(); + let mut by_scope = BTreeMap::new(); // Two genuinely disjoint discovery edges for the same resolver address, discovered in // arbitrary order. They must NOT collapse into one envelope [100, 600): the gap [200, 500) @@ -422,7 +458,7 @@ mod tests { // The same interval reported by two different sources (a watched-contract row and a // discovered-resolver row carry different source_manifest_id) dedups to one emitter, // keeping the lower (source_manifest_id, contract_instance_id). - let mut by_scope = HashMap::new(); + let mut by_scope = BTreeMap::new(); insert_distinct_emitter( &mut by_scope, emitter_with("0xresolver", 7, 5, Some(100), Some(200)), diff --git a/crates/adapters/src/ens_v2_permissions.rs b/crates/adapters/src/ens_v2_permissions.rs index 40135670..ebfaa9f8 100644 --- a/crates/adapters/src/ens_v2_permissions.rs +++ b/crates/adapters/src/ens_v2_permissions.rs @@ -73,6 +73,43 @@ impl EnsV2PermissionsSyncSummary { ) .await } + + pub async fn sync_for_block_hashes_with_progress( + pool: &PgPool, + chain: &str, + block_hashes: &[String], + progress: &mut dyn StartupAdapterProgress, + ) -> Result { + sync_ens_v2_permissions_with_scope( + pool, + chain, + true, + block_hashes, + None, + None, + Some(progress), + ) + .await + } + + pub async fn sync_for_block_hashes_with_source_scope_and_progress( + pool: &PgPool, + chain: &str, + block_hashes: &[String], + source_scope: &[(String, String, i64, i64)], + progress: &mut dyn StartupAdapterProgress, + ) -> Result { + sync_ens_v2_permissions_with_scope( + pool, + chain, + true, + block_hashes, + Some(source_scope), + None, + Some(progress), + ) + .await + } } pub async fn sync_ens_v2_permissions( @@ -107,6 +144,24 @@ pub async fn sync_ens_v2_permissions_through_block( .await } +pub async fn sync_ens_v2_permissions_through_block_with_progress( + pool: &PgPool, + chain: &str, + target_block_number: i64, + progress: &mut dyn StartupAdapterProgress, +) -> Result { + sync_ens_v2_permissions_with_scope( + pool, + chain, + false, + &[], + None, + Some(target_block_number), + Some(progress), + ) + .await +} + async fn sync_ens_v2_permissions_with_scope( pool: &PgPool, chain: &str, @@ -116,7 +171,7 @@ async fn sync_ens_v2_permissions_with_scope( max_block_number: Option, mut progress: Option<&mut dyn StartupAdapterProgress>, ) -> Result { - let mut active_emitters = load_active_emitters(pool, chain).await?; + let mut active_emitters = load_active_emitters(pool, chain, &mut progress).await?; if let Some(source_scope) = source_scope { active_emitters.retain(|emitter| permissions_scope_includes_emitter(source_scope, emitter)); } diff --git a/crates/adapters/src/ens_v2_permissions/load.rs b/crates/adapters/src/ens_v2_permissions/load.rs index 80d9a459..959dfae1 100644 --- a/crates/adapters/src/ens_v2_permissions/load.rs +++ b/crates/adapters/src/ens_v2_permissions/load.rs @@ -1,11 +1,13 @@ +use std::collections::{BTreeMap, HashSet}; + use anyhow::{Context, Result}; -use bigname_manifests::load_watched_contracts; +use bigname_manifests::{load_watched_contracts, load_watched_contracts_scoped_with_progress}; use bigname_storage::sql_row; use sqlx::PgPool; use crate::adapter_manifest::{ active_manifest_for_watched_contract, ensure_watched_contract_manifest_chain, - load_active_manifest_metadata, watched_contract_manifest_ids, + load_active_manifest_metadata, required_source_manifest_id, }; use crate::ens_v2_common::{ ActiveEmitter, active_emitter_for_block, emitters_by_address, normalize_address, @@ -14,7 +16,7 @@ use crate::{ checkpoint_context::{StartupAdapterProgress, record_startup_adapter_progress}, startup_progress::{ RawLogPagePosition, STARTUP_ADAPTER_PROGRESS_PAGE_ROWS, - STARTUP_ADAPTER_PROGRESS_PAGE_ROWS_I64, + STARTUP_ADAPTER_PROGRESS_PAGE_ROWS_I64, StartupManifestProgress, }, }; @@ -187,44 +189,71 @@ pub(super) async fn load_permissions_raw_logs( Ok(output) } -pub(super) async fn load_active_emitters(pool: &PgPool, chain: &str) -> Result> { +pub(super) async fn load_active_emitters( + pool: &PgPool, + chain: &str, + progress: &mut Option<&mut dyn StartupAdapterProgress>, +) -> Result> { let mut emitters = crate::ens_v2_common::load_active_emitters( pool, chain, SOURCE_FAMILY_ENS_V2_RESOLVER_L1, RESOLVER_EDGE_KIND, "ENSv2 permissions", + progress, ) .await?; - emitters.extend(load_registry_active_emitters(pool, chain).await?); + emitters.extend(load_registry_active_emitters(pool, chain, progress).await?); // Order intervals by activation within each (address, source_family) group, then by identity. // Including active_from/active_to keeps this sort TOTAL: one address can now carry several // distinct intervals that share a (source_manifest_id, contract_instance_id) — without the // interval bounds in the key, their order would fall back to HashMap-iteration order and make // `active_emitter_for_block`'s first-match selection nondeterministic for overlapping windows. // This matches the resolver path's earliest-activation-first ordering (ens_v2_common). - emitters.sort_by(|left, right| { - left.address - .cmp(&right.address) - .then(left.source_family.cmp(&right.source_family)) - .then( - left.active_from_block_number - .cmp(&right.active_from_block_number), - ) - .then( - left.active_to_block_number - .cmp(&right.active_to_block_number), - ) - .then(left.source_manifest_id.cmp(&right.source_manifest_id)) - .then(left.contract_instance_id.cmp(&right.contract_instance_id)) - }); - Ok(emitters) + let emitter_count = emitters.len(); + let mut ordered = BTreeMap::new(); + for (index, emitter) in emitters.into_iter().enumerate() { + ordered.insert( + ( + emitter.address.clone(), + emitter.source_family.clone(), + emitter.active_from_block_number, + emitter.active_to_block_number, + emitter.source_manifest_id, + emitter.contract_instance_id, + index, + ), + emitter, + ); + record_emitter_progress(pool, progress, index + 1, emitter_count).await?; + } + Ok(ordered.into_values().collect()) } -async fn load_registry_active_emitters(pool: &PgPool, chain: &str) -> Result> { - let watched_contracts = load_watched_contracts(pool) +async fn load_registry_active_emitters( + pool: &PgPool, + chain: &str, + progress: &mut Option<&mut dyn StartupAdapterProgress>, +) -> Result> { + let source_families = vec![ + SOURCE_FAMILY_ENS_V2_ROOT_L1.to_owned(), + SOURCE_FAMILY_ENS_V2_REGISTRY_L1.to_owned(), + ]; + let watched_contracts = if let Some(progress) = progress.as_deref_mut() { + let mut manifest_progress = StartupManifestProgress::new(progress); + load_watched_contracts_scoped_with_progress( + pool, + Some(chain), + &source_families, + &mut manifest_progress, + ) .await - .context("failed to load watched contracts for ENSv2 registry permissions")?; + .context("failed to load watched contracts for ENSv2 registry permissions")? + } else { + load_watched_contracts(pool) + .await + .context("failed to load watched contracts for ENSv2 registry permissions")? + }; let watched_contracts = watched_contracts .into_iter() .filter(|contract| contract.chain == chain) @@ -233,11 +262,18 @@ async fn load_registry_active_emitters(pool: &PgPool, chain: &str) -> Result>(); let active_manifests = load_active_manifest_metadata(pool, &manifest_ids, "ENSv2 registry permissions").await?; let mut emitters = Vec::new(); - for watched_contract in watched_contracts { + record_startup_adapter_progress(pool, progress).await?; + let watched_contract_count = watched_contracts.len(); + for (index, watched_contract) in watched_contracts.into_iter().enumerate() { let (source_manifest_id, manifest) = active_manifest_for_watched_contract(&active_manifests, &watched_contract)?; if manifest.source_family != SOURCE_FAMILY_ENS_V2_ROOT_L1 @@ -256,10 +292,23 @@ async fn load_registry_active_emitters(pool: &PgPool, chain: &str) -> Result, + completed: usize, + total: usize, +) -> Result<()> { + if completed == total || completed.is_multiple_of(STARTUP_ADAPTER_PROGRESS_PAGE_ROWS) { + record_startup_adapter_progress(pool, progress).await?; + } + Ok(()) +} + fn source_scope_bindings( source_scope: Option<&[(String, String, i64, i64)]>, ) -> (Vec, Vec, Vec) { diff --git a/crates/adapters/src/ens_v2_registrar.rs b/crates/adapters/src/ens_v2_registrar.rs index 01b901f5..23e290dc 100644 --- a/crates/adapters/src/ens_v2_registrar.rs +++ b/crates/adapters/src/ens_v2_registrar.rs @@ -66,6 +66,43 @@ impl EnsV2RegistrarSyncSummary { ) .await } + + pub async fn sync_for_block_hashes_with_progress( + pool: &PgPool, + chain: &str, + block_hashes: &[String], + progress: &mut dyn StartupAdapterProgress, + ) -> Result { + sync_ens_v2_registrar_with_scope( + pool, + chain, + true, + block_hashes, + None, + None, + Some(progress), + ) + .await + } + + pub async fn sync_for_block_hashes_with_source_scope_and_progress( + pool: &PgPool, + chain: &str, + block_hashes: &[String], + source_scope: &[(String, String, i64, i64)], + progress: &mut dyn StartupAdapterProgress, + ) -> Result { + sync_ens_v2_registrar_with_scope( + pool, + chain, + true, + block_hashes, + Some(source_scope), + None, + Some(progress), + ) + .await + } } pub async fn sync_ens_v2_registrar( @@ -100,6 +137,24 @@ pub async fn sync_ens_v2_registrar_through_block( .await } +pub async fn sync_ens_v2_registrar_through_block_with_progress( + pool: &PgPool, + chain: &str, + target_block_number: i64, + progress: &mut dyn StartupAdapterProgress, +) -> Result { + sync_ens_v2_registrar_with_scope( + pool, + chain, + false, + &[], + None, + Some(target_block_number), + Some(progress), + ) + .await +} + async fn sync_ens_v2_registrar_with_scope( pool: &PgPool, chain: &str, @@ -109,7 +164,7 @@ async fn sync_ens_v2_registrar_with_scope( max_block_number: Option, mut progress: Option<&mut dyn StartupAdapterProgress>, ) -> Result { - let mut active_emitters = load_active_emitters(pool, chain).await?; + let mut active_emitters = load_active_emitters(pool, chain, &mut progress).await?; if let Some(source_scope) = source_scope { active_emitters.retain(|emitter| registrar_scope_includes_emitter(source_scope, emitter)); } diff --git a/crates/adapters/src/ens_v2_registrar/active_emitters.rs b/crates/adapters/src/ens_v2_registrar/active_emitters.rs index 878e0996..33239fc9 100644 --- a/crates/adapters/src/ens_v2_registrar/active_emitters.rs +++ b/crates/adapters/src/ens_v2_registrar/active_emitters.rs @@ -1,12 +1,16 @@ -use std::collections::HashMap; +use std::collections::{BTreeMap, HashSet}; use anyhow::{Context, Result}; -use bigname_manifests::load_watched_contracts; +use bigname_manifests::{load_watched_contracts, load_watched_contracts_scoped_with_progress}; use sqlx::PgPool; use crate::adapter_manifest::{ active_manifest_for_watched_contract, ensure_watched_contract_manifest_chain, - load_active_manifest_metadata, source_rank, watched_contract_manifest_ids, + load_active_manifest_metadata, required_source_manifest_id, source_rank, +}; +use crate::{ + checkpoint_context::{StartupAdapterProgress, record_startup_adapter_progress}, + startup_progress::{STARTUP_ADAPTER_PROGRESS_PAGE_ROWS, StartupManifestProgress}, }; use super::SOURCE_FAMILY_ENS_V2_REGISTRAR_L1; @@ -21,10 +25,26 @@ pub(super) struct ActiveEmitter { pub(super) source_rank: i32, } -pub(super) async fn load_active_emitters(pool: &PgPool, chain: &str) -> Result> { - let watched_contracts = load_watched_contracts(pool) +pub(super) async fn load_active_emitters( + pool: &PgPool, + chain: &str, + progress: &mut Option<&mut dyn StartupAdapterProgress>, +) -> Result> { + let watched_contracts = if let Some(progress) = progress.as_deref_mut() { + let mut manifest_progress = StartupManifestProgress::new(progress); + load_watched_contracts_scoped_with_progress( + pool, + Some(chain), + &[SOURCE_FAMILY_ENS_V2_REGISTRAR_L1.to_owned()], + &mut manifest_progress, + ) .await - .context("failed to load watched contracts for ENSv2 registrar adapter")?; + .context("failed to load watched contracts for ENSv2 registrar adapter")? + } else { + load_watched_contracts(pool) + .await + .context("failed to load watched contracts for ENSv2 registrar adapter")? + }; let watched_contracts = watched_contracts .into_iter() .filter(|contract| contract.chain == chain) @@ -33,12 +53,19 @@ pub(super) async fn load_active_emitters(pool: &PgPool, chain: &str) -> Result>(); let active_manifests = load_active_manifest_metadata(pool, &manifest_ids, "ENSv2 registrar emitters").await?; - let mut emitters_by_address = HashMap::::new(); - for watched_contract in watched_contracts { + record_startup_adapter_progress(pool, progress).await?; + let watched_contract_count = watched_contracts.len(); + let mut emitters_by_address = BTreeMap::::new(); + for (index, watched_contract) in watched_contracts.into_iter().enumerate() { let (source_manifest_id, manifest) = active_manifest_for_watched_contract(&active_manifests, &watched_contract)?; if manifest.source_family != SOURCE_FAMILY_ENS_V2_REGISTRAR_L1 { @@ -61,16 +88,22 @@ pub(super) async fn load_active_emitters(pool: &PgPool, chain: &str) -> Result>(); - emitters.sort_by(|left, right| { - left.address - .cmp(&right.address) - .then(left.source_rank.cmp(&right.source_rank)) - .then(left.source_manifest_id.cmp(&right.source_manifest_id)) - }); - Ok(emitters) + Ok(emitters_by_address.into_values().collect()) +} + +async fn record_emitter_progress( + pool: &PgPool, + progress: &mut Option<&mut dyn StartupAdapterProgress>, + completed: usize, + total: usize, +) -> Result<()> { + if completed == total || completed.is_multiple_of(STARTUP_ADAPTER_PROGRESS_PAGE_ROWS) { + record_startup_adapter_progress(pool, progress).await?; + } + Ok(()) } fn candidate_precedes(candidate: &ActiveEmitter, current: &ActiveEmitter) -> bool { diff --git a/crates/adapters/src/ens_v2_registry.rs b/crates/adapters/src/ens_v2_registry.rs index bc2e4f90..b2d4580f 100644 --- a/crates/adapters/src/ens_v2_registry.rs +++ b/crates/adapters/src/ens_v2_registry.rs @@ -13,6 +13,7 @@ mod constants; mod decode; mod discovery; mod emitters; +mod entrypoints; mod events; mod identity; mod live; @@ -26,7 +27,9 @@ mod util; use crate::{ adapter_manifest::load_required_active_manifest_event_topic0s_by_signature, - checkpoint_context::{StartupAdapterProgress, record_startup_adapter_progress}, + checkpoint_context::{ + StartupAdapterProgress, reborrow_startup_adapter_progress, record_startup_adapter_progress, + }, normalized_event_support::count_events_by_kind, startup_progress::record_processed_row_progress, }; @@ -45,7 +48,8 @@ use identity::{ use live::{ FullSourceRawLogHistoryGuard, RegistryReplayState, acquire_registry_sync_fence, clear_live_registry_replay_checkpoints_for_chain, has_authoritative_ens_v2_closure_through, - invalidate_live_registry_replay_state, release_registry_sync_fence, + has_authoritative_ens_v2_closure_through_with_progress, invalidate_live_registry_replay_state, + release_registry_sync_fence, }; use load::{RawLogCanonicalityFilter, load_registry_raw_logs_with_progress}; use names::initial_registry_suffixes; @@ -53,9 +57,16 @@ use startup::persist_registry_outputs_with_progress; use types::*; use util::normalize_address; +pub use entrypoints::{ + sync_ens_v2_registry_resource_surface, sync_ens_v2_registry_resource_surface_through_block, + sync_ens_v2_registry_resource_surface_through_block_with_progress, + sync_ens_v2_registry_resource_surface_with_progress, +}; pub use live::{ ensure_ens_v2_retained_history_proof_through, record_ens_v2_live_selected_raw_log_coverage, + record_ens_v2_live_selected_raw_log_coverage_with_progress, sync_ens_v2_registry_resource_surface_live_poll, + sync_ens_v2_registry_resource_surface_live_poll_with_progress, }; pub use recovery::{EnsV2MissingCoverage, ens_v2_missing_coverage, is_ens_v2_missing_coverage}; @@ -93,154 +104,6 @@ pub struct EnsV2RegistryResourceSurfaceSyncSummary { pub by_kind: BTreeMap, } -impl EnsV2RegistryResourceSurfaceSyncSummary { - pub fn empty(scanned_log_count: usize) -> Self { - Self { - scanned_log_count, - matched_log_count: 0, - total_name_surface_count: 0, - total_resource_count: 0, - total_surface_binding_count: 0, - total_normalized_event_count: 0, - total_normalized_event_inserted_count: 0, - active_discovery_observation_count: 0, - active_edge_count: 0, - admitted_edge_count: 0, - inserted_edge_count: 0, - deactivated_edge_count: 0, - discovery_admission_epoch_bump_count: 0, - by_kind: BTreeMap::new(), - } - } - - pub async fn sync_for_block_hashes( - pool: &PgPool, - chain: &str, - block_hashes: &[String], - ) -> Result { - sync_ens_v2_registry_resource_surface_with_scope( - pool, - chain, - true, - block_hashes, - None, - RawLogCanonicalityFilter::IncludeObserved, - None, - None, - ) - .await - } - - pub async fn sync_for_block_hashes_canonical_only( - pool: &PgPool, - chain: &str, - block_hashes: &[String], - ) -> Result { - sync_ens_v2_registry_resource_surface_with_scope( - pool, - chain, - true, - block_hashes, - None, - RawLogCanonicalityFilter::CanonicalOnly, - None, - None, - ) - .await - } - - pub async fn sync_for_block_hashes_with_source_scope( - pool: &PgPool, - chain: &str, - block_hashes: &[String], - source_scope: &[(String, String, i64, i64)], - ) -> Result { - sync_ens_v2_registry_resource_surface_with_scope( - pool, - chain, - true, - block_hashes, - Some(source_scope), - RawLogCanonicalityFilter::IncludeObserved, - None, - None, - ) - .await - } - - pub async fn sync_for_block_hashes_with_source_scope_canonical_only( - pool: &PgPool, - chain: &str, - block_hashes: &[String], - source_scope: &[(String, String, i64, i64)], - ) -> Result { - sync_ens_v2_registry_resource_surface_with_scope( - pool, - chain, - true, - block_hashes, - Some(source_scope), - RawLogCanonicalityFilter::CanonicalOnly, - None, - None, - ) - .await - } -} - -pub async fn sync_ens_v2_registry_resource_surface( - pool: &PgPool, - chain: &str, -) -> Result { - sync_ens_v2_registry_resource_surface_with_scope( - pool, - chain, - false, - &[], - None, - RawLogCanonicalityFilter::IncludeObserved, - None, - None, - ) - .await -} - -pub async fn sync_ens_v2_registry_resource_surface_with_progress( - pool: &PgPool, - chain: &str, - progress: &mut dyn StartupAdapterProgress, -) -> Result { - sync_ens_v2_registry_resource_surface_with_scope( - pool, - chain, - false, - &[], - None, - RawLogCanonicalityFilter::IncludeObserved, - None, - Some(progress), - ) - .await -} - -pub async fn sync_ens_v2_registry_resource_surface_through_block( - pool: &PgPool, - chain: &str, - target_block_number: i64, -) -> Result { - sync_ens_v2_registry_resource_surface_with_scope( - pool, - chain, - false, - &[], - None, - RawLogCanonicalityFilter::CanonicalOnly, - Some(target_block_number), - None, - ) - .await -} - async fn sync_ens_v2_registry_resource_surface_with_scope( pool: &PgPool, chain: &str, @@ -249,7 +112,7 @@ async fn sync_ens_v2_registry_resource_surface_with_scope( source_scope: Option<&[(String, String, i64, i64)]>, canonicality_filter: RawLogCanonicalityFilter, max_block_number: Option, - progress: Option<&mut dyn StartupAdapterProgress>, + mut progress: Option<&mut dyn StartupAdapterProgress>, ) -> Result { let mut registry_sync_fence = Some(acquire_registry_sync_fence(pool, chain).await?); // Non-live entrypoints may rewrite persisted state behind the process-local live cache. @@ -273,7 +136,21 @@ async fn sync_ens_v2_registry_resource_surface_with_scope( .with_context(|| format!("failed to load ENSv2 full-source target for {chain}"))? .unwrap_or(0) }; - if !has_authoritative_ens_v2_closure_through(pool, chain, full_source_target).await? { + let has_authoritative_closure = match progress.as_deref_mut() { + Some(progress) => { + has_authoritative_ens_v2_closure_through_with_progress( + pool, + chain, + full_source_target, + progress, + ) + .await? + } + None => { + has_authoritative_ens_v2_closure_through(pool, chain, full_source_target).await? + } + }; + if !has_authoritative_closure { let fence = registry_sync_fence .take() .context("ENSv2 registry sync fence is absent before empty-source release")?; @@ -284,10 +161,31 @@ async fn sync_ens_v2_registry_resource_surface_with_scope( .take() .context("ENSv2 registry sync fence is absent before full-source upgrade")?; let guard = FullSourceRawLogHistoryGuard::acquire(fence, chain).await?; - let proof = guard.ensure_proof_through(pool, full_source_target).await?; - let pre_sync_requirements = guard - .load_requirements_through(pool, proof, full_source_target) - .await?; + let proof = match progress.as_deref_mut() { + Some(progress) => { + guard + .ensure_proof_through_with_progress(pool, full_source_target, progress) + .await? + } + None => guard.ensure_proof_through(pool, full_source_target).await?, + }; + let pre_sync_requirements = match progress.as_deref_mut() { + Some(progress) => { + guard + .load_requirements_through_with_progress( + pool, + proof, + full_source_target, + progress, + ) + .await? + } + None => { + guard + .load_requirements_through(pool, proof, full_source_target) + .await? + } + }; Some((guard, proof, full_source_target, pre_sync_requirements)) } else { None @@ -307,7 +205,7 @@ async fn sync_ens_v2_registry_resource_surface_with_scope( !restrict_to_block_hashes, !restrict_to_block_hashes, expected_discovery_admission_epoch, - progress, + reborrow_startup_adapter_progress(&mut progress), ) .await; let result = match (sync_result, full_source_guard) { @@ -315,16 +213,33 @@ async fn sync_ens_v2_registry_resource_surface_with_scope( Ok((summary, replay_state)), Some((guard, proof, full_source_target, pre_sync_requirements)), ) => { - guard - .finish( - pool, - proof, - full_source_target, - summary.discovery_admission_epoch_bump_count, - &pre_sync_requirements, - None, - ) - .await?; + match progress.as_deref_mut() { + Some(progress) => { + guard + .finish_with_progress( + pool, + proof, + full_source_target, + summary.discovery_admission_epoch_bump_count, + &pre_sync_requirements, + None, + progress, + ) + .await?; + } + None => { + guard + .finish( + pool, + proof, + full_source_target, + summary.discovery_admission_epoch_bump_count, + &pre_sync_requirements, + None, + ) + .await?; + } + } Ok((summary, replay_state)) } (Ok(result), None) => Ok(result), @@ -386,6 +301,7 @@ async fn sync_ens_v2_registry_resource_surface_with_scope_and_state( chain, scoped_emitter_identities.as_ref(), include_historical_emitters, + &mut progress, ) .await?; if active_emitters.is_empty() { diff --git a/crates/adapters/src/ens_v2_registry/emitters.rs b/crates/adapters/src/ens_v2_registry/emitters.rs index d14c80b7..05a6292a 100644 --- a/crates/adapters/src/ens_v2_registry/emitters.rs +++ b/crates/adapters/src/ens_v2_registry/emitters.rs @@ -1,13 +1,21 @@ use bigname_storage::sql_row; -use std::collections::{HashMap, HashSet}; +use std::collections::{BTreeMap, HashMap, HashSet}; use anyhow::{Context, Result, bail}; use bigname_manifests::{ - WatchedContractSource, load_historical_watched_contracts_by_chain, load_watched_contracts, + WatchedContractSource, load_historical_watched_contracts_by_chain, + load_historical_watched_contracts_scoped_with_progress, load_watched_contracts, + load_watched_contracts_scoped_with_progress, }; -use sqlx::{PgPool, types::Uuid}; +use sqlx::PgPool; +#[cfg(test)] +use sqlx::types::Uuid; -use crate::adapter_manifest::watched_contract_manifest_ids; +use crate::{ + adapter_manifest::required_source_manifest_id, + checkpoint_context::{StartupAdapterProgress, record_startup_adapter_progress}, + startup_progress::{STARTUP_ADAPTER_PROGRESS_PAGE_ROWS, StartupManifestProgress}, +}; use super::{ constants::*, @@ -20,8 +28,34 @@ pub(super) async fn load_active_emitters( chain: &str, scoped_emitter_identities: Option<&HashSet<(String, String)>>, include_historical: bool, + progress: &mut Option<&mut dyn StartupAdapterProgress>, ) -> Result> { - let watched_contracts = if include_historical { + let source_families = vec![ + SOURCE_FAMILY_ENS_V2_ROOT_L1.to_owned(), + SOURCE_FAMILY_ENS_V2_REGISTRY_L1.to_owned(), + ]; + let watched_contracts = if let Some(progress) = progress.as_deref_mut() { + let mut manifest_progress = StartupManifestProgress::new(progress); + if include_historical { + load_historical_watched_contracts_scoped_with_progress( + pool, + chain, + &source_families, + &mut manifest_progress, + ) + .await + .context("failed to load historical watched contracts for ENSv2 registry adapter")? + } else { + load_watched_contracts_scoped_with_progress( + pool, + Some(chain), + &source_families, + &mut manifest_progress, + ) + .await + .context("failed to load watched contracts for ENSv2 registry adapter")? + } + } else if include_historical { load_historical_watched_contracts_by_chain(pool, chain) .await .context("failed to load historical watched contracts for ENSv2 registry adapter")? @@ -38,11 +72,18 @@ pub(super) async fn load_active_emitters( return Ok(Vec::new()); } - let manifest_ids = watched_contract_manifest_ids(&watched_contracts)?; + let mut manifest_ids = HashSet::new(); + for (index, watched_contract) in watched_contracts.iter().enumerate() { + manifest_ids.insert(required_source_manifest_id(watched_contract)?); + record_processed_progress(pool, progress, index + 1, watched_contracts.len()).await?; + } + let manifest_ids = manifest_ids.into_iter().collect::>(); let active_manifests = load_active_manifest_metadata(pool, &manifest_ids).await?; + record_startup_adapter_progress(pool, progress).await?; - let mut emitter_candidates = Vec::new(); - for watched_contract in watched_contracts { + let watched_contract_count = watched_contracts.len(); + let mut emitters_by_scope = BTreeMap::new(); + for (index, watched_contract) in watched_contracts.into_iter().enumerate() { let source_manifest_id = watched_contract .source_manifest_id .context("watched contract missing source_manifest_id after validation")?; @@ -86,10 +127,40 @@ pub(super) async fn load_active_emitters( active_to_block_number: watched_contract.active_to_block_number, }; - emitter_candidates.push(candidate); + let scope_key = ( + candidate.source_family.clone(), + candidate.address.clone(), + candidate.active_from_block_number, + candidate.active_to_block_number, + ); + match emitters_by_scope.get(&scope_key) { + Some(current) if !candidate_precedes(&candidate, current) => {} + _ => { + emitters_by_scope.insert(scope_key, candidate); + } + } + record_processed_progress(pool, progress, index + 1, watched_contract_count).await?; + } + + let emitter_count = emitters_by_scope.len(); + let mut emitters = Vec::with_capacity(emitter_count); + for (index, emitter) in emitters_by_scope.into_values().enumerate() { + emitters.push(emitter); + record_processed_progress(pool, progress, index + 1, emitter_count).await?; } + Ok(emitters) +} - Ok(preferred_emitters_by_scope(emitter_candidates)) +async fn record_processed_progress( + pool: &PgPool, + progress: &mut Option<&mut dyn StartupAdapterProgress>, + completed: usize, + total: usize, +) -> Result<()> { + if completed == total || completed.is_multiple_of(STARTUP_ADAPTER_PROGRESS_PAGE_ROWS) { + record_startup_adapter_progress(pool, progress).await?; + } + Ok(()) } pub(super) fn source_scope_target_intersects_active_emitter( @@ -106,6 +177,7 @@ pub(super) fn source_scope_target_intersects_active_emitter( target.effective_from_block <= emitter_to && emitter_from <= target.effective_to_block } +#[cfg(test)] pub(super) fn emitter_sort_key( emitter: &ActiveEmitter, ) -> (&str, &str, Option, Option, i32, i64, Uuid) { @@ -120,10 +192,12 @@ pub(super) fn emitter_sort_key( ) } +#[cfg(test)] pub(super) fn sort_emitters_by_scope(emitters: &mut [ActiveEmitter]) { emitters.sort_by(|left, right| emitter_sort_key(left).cmp(&emitter_sort_key(right))); } +#[cfg(test)] pub(super) fn preferred_emitters_by_scope( candidates: impl IntoIterator, ) -> Vec { diff --git a/crates/adapters/src/ens_v2_registry/entrypoints.rs b/crates/adapters/src/ens_v2_registry/entrypoints.rs new file mode 100644 index 00000000..8ffe724d --- /dev/null +++ b/crates/adapters/src/ens_v2_registry/entrypoints.rs @@ -0,0 +1,168 @@ +use super::*; + +impl EnsV2RegistryResourceSurfaceSyncSummary { + pub fn empty(scanned_log_count: usize) -> Self { + Self { + scanned_log_count, + matched_log_count: 0, + total_name_surface_count: 0, + total_resource_count: 0, + total_surface_binding_count: 0, + total_normalized_event_count: 0, + total_normalized_event_inserted_count: 0, + active_discovery_observation_count: 0, + active_edge_count: 0, + admitted_edge_count: 0, + inserted_edge_count: 0, + deactivated_edge_count: 0, + discovery_admission_epoch_bump_count: 0, + by_kind: BTreeMap::new(), + } + } + + pub async fn sync_for_block_hashes( + pool: &PgPool, + chain: &str, + block_hashes: &[String], + ) -> Result { + sync_ens_v2_registry_resource_surface_with_scope( + pool, + chain, + true, + block_hashes, + None, + RawLogCanonicalityFilter::IncludeObserved, + None, + None, + ) + .await + } + + pub async fn sync_for_block_hashes_canonical_only( + pool: &PgPool, + chain: &str, + block_hashes: &[String], + ) -> Result { + sync_ens_v2_registry_resource_surface_with_scope( + pool, + chain, + true, + block_hashes, + None, + RawLogCanonicalityFilter::CanonicalOnly, + None, + None, + ) + .await + } + + pub async fn sync_for_block_hashes_with_source_scope( + pool: &PgPool, + chain: &str, + block_hashes: &[String], + source_scope: &[(String, String, i64, i64)], + ) -> Result { + sync_ens_v2_registry_resource_surface_with_scope( + pool, + chain, + true, + block_hashes, + Some(source_scope), + RawLogCanonicalityFilter::IncludeObserved, + None, + None, + ) + .await + } + + pub async fn sync_for_block_hashes_with_source_scope_canonical_only( + pool: &PgPool, + chain: &str, + block_hashes: &[String], + source_scope: &[(String, String, i64, i64)], + ) -> Result { + sync_ens_v2_registry_resource_surface_with_scope( + pool, + chain, + true, + block_hashes, + Some(source_scope), + RawLogCanonicalityFilter::CanonicalOnly, + None, + None, + ) + .await + } +} + +pub async fn sync_ens_v2_registry_resource_surface( + pool: &PgPool, + chain: &str, +) -> Result { + sync_ens_v2_registry_resource_surface_with_scope( + pool, + chain, + false, + &[], + None, + RawLogCanonicalityFilter::IncludeObserved, + None, + None, + ) + .await +} + +pub async fn sync_ens_v2_registry_resource_surface_with_progress( + pool: &PgPool, + chain: &str, + progress: &mut dyn StartupAdapterProgress, +) -> Result { + sync_ens_v2_registry_resource_surface_with_scope( + pool, + chain, + false, + &[], + None, + RawLogCanonicalityFilter::IncludeObserved, + None, + Some(progress), + ) + .await +} + +pub async fn sync_ens_v2_registry_resource_surface_through_block( + pool: &PgPool, + chain: &str, + target_block_number: i64, +) -> Result { + sync_ens_v2_registry_resource_surface_with_scope( + pool, + chain, + false, + &[], + None, + RawLogCanonicalityFilter::CanonicalOnly, + Some(target_block_number), + None, + ) + .await +} + +pub async fn sync_ens_v2_registry_resource_surface_through_block_with_progress( + pool: &PgPool, + chain: &str, + target_block_number: i64, + progress: &mut dyn StartupAdapterProgress, +) -> Result { + sync_ens_v2_registry_resource_surface_with_scope( + pool, + chain, + false, + &[], + None, + RawLogCanonicalityFilter::CanonicalOnly, + Some(target_block_number), + Some(progress), + ) + .await +} diff --git a/crates/adapters/src/ens_v2_registry/live.rs b/crates/adapters/src/ens_v2_registry/live.rs index e70d6140..bdcf252a 100644 --- a/crates/adapters/src/ens_v2_registry/live.rs +++ b/crates/adapters/src/ens_v2_registry/live.rs @@ -8,12 +8,16 @@ use super::{ load::RawLogCanonicalityFilter, sync_ens_v2_registry_resource_surface_with_scope_and_state, types::RegistryNameState, }; +use crate::checkpoint_context::{ + StartupAdapterProgress, reborrow_startup_adapter_progress, record_startup_adapter_progress, +}; mod cache; mod checkpoint; mod completeness; mod fence; mod path; +mod reuse; use cache::{ CachedLiveRegistryReplayState, MAX_LIVE_REGISTRY_REPLAY_STATE_WEIGHT, @@ -27,6 +31,7 @@ use checkpoint::{ }; pub(super) use completeness::{ FullSourceRawLogHistoryGuard, RawLogClosureProof, has_authoritative_ens_v2_closure_through, + has_authoritative_ens_v2_closure_through_with_progress, }; pub(super) use fence::{acquire_registry_sync_fence, release_registry_sync_fence}; use path::{ @@ -34,6 +39,7 @@ use path::{ load_registry_cache_metadata, load_selected_registry_target, raw_log_mutations_leave_cached_path_unchanged, }; +use reuse::*; #[derive(Debug, Default, Eq, PartialEq)] pub(super) struct RegistryReplayState { @@ -64,6 +70,40 @@ pub async fn record_ens_v2_live_selected_raw_log_coverage( chain: &str, selected_addresses: &[String], selected_block_hashes: &[String], +) -> Result<()> { + record_ens_v2_live_selected_raw_log_coverage_inner( + pool, + chain, + selected_addresses, + selected_block_hashes, + None, + ) + .await +} + +pub async fn record_ens_v2_live_selected_raw_log_coverage_with_progress( + pool: &PgPool, + chain: &str, + selected_addresses: &[String], + selected_block_hashes: &[String], + progress: &mut dyn StartupAdapterProgress, +) -> Result<()> { + record_ens_v2_live_selected_raw_log_coverage_inner( + pool, + chain, + selected_addresses, + selected_block_hashes, + Some(progress), + ) + .await +} + +async fn record_ens_v2_live_selected_raw_log_coverage_inner( + pool: &PgPool, + chain: &str, + selected_addresses: &[String], + selected_block_hashes: &[String], + mut progress: Option<&mut dyn StartupAdapterProgress>, ) -> Result<()> { if selected_block_hashes.is_empty() { return Ok(()); @@ -83,7 +123,19 @@ pub async fn record_ens_v2_live_selected_raw_log_coverage( .await .with_context(|| format!("failed to load ENSv2 live coverage target for {chain}"))? .with_context(|| format!("ENSv2 live coverage selection is absent for {chain}"))?; - if !has_authoritative_ens_v2_closure_through(pool, chain, target_block_number).await? { + let has_closure = match progress.as_deref_mut() { + Some(progress) => { + has_authoritative_ens_v2_closure_through_with_progress( + pool, + chain, + target_block_number, + progress, + ) + .await? + } + None => has_authoritative_ens_v2_closure_through(pool, chain, target_block_number).await?, + }; + if !has_closure { return Ok(()); } @@ -92,14 +144,29 @@ pub async fn record_ens_v2_live_selected_raw_log_coverage( chain, ) .await?; - let result = guard - .ensure_proof_through_live_selection( - pool, - target_block_number, - selected_addresses, - selected_block_hashes, - ) - .await; + let result = match progress.as_deref_mut() { + Some(progress) => { + guard + .ensure_proof_through_live_selection_with_progress( + pool, + target_block_number, + selected_addresses, + selected_block_hashes, + progress, + ) + .await + } + None => { + guard + .ensure_proof_through_live_selection( + pool, + target_block_number, + selected_addresses, + selected_block_hashes, + ) + .await + } + }; match result { Ok(_) => guard.release().await, Err(error) => { @@ -152,6 +219,27 @@ pub async fn sync_ens_v2_registry_resource_surface_live_poll( target_block_number, block_hashes, MAX_LIVE_REGISTRY_REPLAY_STATE_WEIGHT, + None, + ) + .await +} + +pub async fn sync_ens_v2_registry_resource_surface_live_poll_with_progress( + pool: &PgPool, + deployment_profile: &str, + chain: &str, + target_block_number: i64, + block_hashes: &[String], + progress: &mut dyn StartupAdapterProgress, +) -> Result { + sync_ens_v2_registry_resource_surface_live_poll_with_cache_budget( + pool, + deployment_profile, + chain, + target_block_number, + block_hashes, + MAX_LIVE_REGISTRY_REPLAY_STATE_WEIGHT, + Some(progress), ) .await } @@ -171,6 +259,7 @@ pub(in crate::ens_v2_registry) async fn sync_ens_v2_registry_resource_surface_li target_block_number, block_hashes, 1, + None, ) .await } @@ -182,6 +271,7 @@ async fn sync_ens_v2_registry_resource_surface_live_poll_with_cache_budget( target_block_number: i64, block_hashes: &[String], max_process_cache_weight: usize, + progress: Option<&mut dyn StartupAdapterProgress>, ) -> Result { ensure!( !deployment_profile.trim().is_empty(), @@ -203,6 +293,7 @@ async fn sync_ens_v2_registry_resource_surface_live_poll_with_cache_budget( block_hashes, raw_log_guard, max_process_cache_weight, + progress, ) .await } @@ -215,6 +306,7 @@ async fn sync_ens_v2_registry_resource_surface_live_poll_locked( block_hashes: &[String], raw_log_guard: FullSourceRawLogHistoryGuard, max_process_cache_weight: usize, + mut progress: Option<&mut dyn StartupAdapterProgress>, ) -> Result { let current_closure_proof = raw_log_guard.load_current_proof(pool).await?; let target_block_hash = @@ -275,12 +367,22 @@ async fn sync_ens_v2_registry_resource_surface_live_poll_locked( let proof = if let Some(proof) = current_closure_proof { proof } else { - raw_log_guard - .ensure_proof_through(pool, target_block_number) - .await? + match progress.as_deref_mut() { + Some(progress) => { + raw_log_guard + .ensure_proof_through_with_progress(pool, target_block_number, progress) + .await? + } + None => { + raw_log_guard + .ensure_proof_through(pool, target_block_number) + .await? + } + } }; metadata_before = load_registry_cache_metadata(pool, chain).await?; - let registry_emitters = load_active_emitters(pool, chain, None, true).await?; + let registry_emitters = + load_active_emitters(pool, chain, None, true, &mut progress).await?; let retained_log_floor = load_raw_log_closure_floor(pool, chain, target_block_number, ®istry_emitters) .await?; @@ -296,9 +398,23 @@ async fn sync_ens_v2_registry_resource_surface_live_poll_locked( .await?; (None, proof, path) }; - let pre_sync_requirements = raw_log_guard - .load_requirements_through(pool, closure_proof, target_block_number) - .await?; + let pre_sync_requirements = match progress.as_deref_mut() { + Some(progress) => { + raw_log_guard + .load_requirements_through_with_progress( + pool, + closure_proof, + target_block_number, + progress, + ) + .await? + } + None => { + raw_log_guard + .load_requirements_through(pool, closure_proof, target_block_number) + .await? + } + }; let sync_result = if let Some(cached) = cached { let incremental_block_hashes = selected_path.hashes_after(cached.through_block_number); @@ -320,7 +436,7 @@ async fn sync_ens_v2_registry_resource_surface_live_poll_locked( true, false, Some(closure_proof.discovery_admission_epoch), - None, + reborrow_startup_adapter_progress(&mut progress), ) .await } @@ -338,7 +454,7 @@ async fn sync_ens_v2_registry_resource_surface_live_poll_locked( true, true, Some(closure_proof.discovery_admission_epoch), - None, + reborrow_startup_adapter_progress(&mut progress), ) .await }; @@ -351,6 +467,7 @@ async fn sync_ens_v2_registry_resource_surface_live_poll_locked( }; let metadata_after = load_registry_cache_metadata(pool, chain).await?; + record_startup_adapter_progress(pool, &mut progress).await?; ensure!( metadata_after.raw_log_input_revision == metadata_before.raw_log_input_revision, "ENSv2 raw-log input changed during live sync on {chain}; refusing to publish a stale replay cache" @@ -392,120 +509,36 @@ async fn sync_ens_v2_registry_resource_surface_live_poll_locked( } }; - raw_log_guard - .finish( - pool, - closure_proof, - target_block_number, - summary.discovery_admission_epoch_bump_count, - &pre_sync_requirements, - staged_checkpoint.as_ref(), - ) - .await?; + match progress.as_deref_mut() { + Some(progress) => { + raw_log_guard + .finish_with_progress( + pool, + closure_proof, + target_block_number, + summary.discovery_admission_epoch_bump_count, + &pre_sync_requirements, + staged_checkpoint.as_ref(), + progress, + ) + .await?; + } + None => { + raw_log_guard + .finish( + pool, + closure_proof, + target_block_number, + summary.discovery_admission_epoch_bump_count, + &pre_sync_requirements, + staged_checkpoint.as_ref(), + ) + .await?; + } + } if fits_process_cache { store_live_registry_replay_state(pool, deployment_profile, chain, snapshot); } Ok(summary) } - -async fn load_reusable_durable_checkpoint( - pool: &PgPool, - chain: &str, - target_block_number: i64, - target_block_hash: &str, - current_closure_proof: Option, - metadata: &RegistryCacheMetadata, - header: LiveRegistryReplayCheckpointHeader, -) -> Result< - Option<( - CachedLiveRegistryReplayState, - RawLogClosureProof, - SelectedRegistryPath, - )>, -> { - let Some((proof, path)) = reusable_checkpoint_path( - pool, - chain, - target_block_number, - target_block_hash, - current_closure_proof, - metadata, - header.through_block_number, - &header.through_block_hash, - header.raw_log_input_revision, - header.raw_log_retention_generation, - header.discovery_admission_epoch, - ) - .await? - else { - tracing::warn!( - deployment_profile = header.deployment_profile, - chain, - "discarding stale ENSv2 live checkpoint" - ); - return Ok(None); - }; - match load_live_registry_replay_checkpoint(pool, &header).await? { - LiveRegistryReplayCheckpointLoad::Ready(snapshot) => Ok(Some((snapshot, proof, path))), - LiveRegistryReplayCheckpointLoad::Missing => Ok(None), - LiveRegistryReplayCheckpointLoad::Invalid(reason) => { - tracing::warn!( - deployment_profile = header.deployment_profile, - chain, - reason, - "discarding invalid ENSv2 live checkpoint payload" - ); - Ok(None) - } - } -} - -#[allow(clippy::too_many_arguments)] -async fn reusable_checkpoint_path( - pool: &PgPool, - chain: &str, - target_block_number: i64, - target_block_hash: &str, - current_closure_proof: Option, - metadata: &RegistryCacheMetadata, - through_block_number: i64, - through_block_hash: &str, - raw_log_input_revision: i64, - raw_log_retention_generation: i64, - discovery_admission_epoch: i64, -) -> Result> { - let Some(proof) = current_closure_proof.filter(|proof| { - proof.proven_through_block >= through_block_number - && target_block_number >= through_block_number - }) else { - return Ok(None); - }; - if discovery_admission_epoch != proof.discovery_admission_epoch - || raw_log_retention_generation != proof.retention_generation - || !metadata.retained_raw_log_history_complete - { - return Ok(None); - } - let path = load_selected_registry_path_to_floor( - pool, - chain, - target_block_number, - target_block_hash, - through_block_number, - ) - .await?; - if !path.contains_anchor(through_block_number, through_block_hash) - || !raw_log_mutations_leave_cached_path_unchanged( - pool, - chain, - raw_log_input_revision, - through_block_number, - through_block_hash, - ) - .await? - { - return Ok(None); - } - Ok(Some((proof, path))) -} diff --git a/crates/adapters/src/ens_v2_registry/live/completeness.rs b/crates/adapters/src/ens_v2_registry/live/completeness.rs index 8761c404..fbffb59a 100644 --- a/crates/adapters/src/ens_v2_registry/live/completeness.rs +++ b/crates/adapters/src/ens_v2_registry/live/completeness.rs @@ -1,17 +1,33 @@ use anyhow::{Context, Result, ensure}; -use bigname_manifests::{RequiredWatchedTuple, load_required_watched_tuples_in_transaction}; +use bigname_manifests::{ + RequiredWatchedTuple, load_required_watched_tuples_in_transaction, + load_required_watched_tuples_in_transaction_with_progress, +}; use sqlx::{PgPool, Postgres, Transaction}; +use crate::{ + checkpoint_context::{StartupAdapterProgress, record_startup_adapter_progress}, + startup_progress::StartupManifestProgress, +}; + +const RETAINED_REQUIREMENT_PROGRESS_ROWS: usize = 1_000; + mod finish; +mod guard; mod requirements; mod state; -pub(in crate::ens_v2_registry) use requirements::has_authoritative_ens_v2_closure_through; use requirements::{ ens_v2_closure_source_families, ens_v2_discovery_history_source_families, ensure_generation_bound_coverage, ensure_generation_bound_coverage_with_live_selection, + ensure_generation_bound_coverage_with_live_selection_with_progress, ensure_newly_required_generation_bound_coverage, ensure_retained_semantic_witnesses, - requirement_intervals_not_covered_by, + ensure_retained_semantic_witnesses_with_progress, requirement_intervals_not_covered_by, + requirement_intervals_not_covered_by_with_progress, +}; +pub(in crate::ens_v2_registry) use requirements::{ + has_authoritative_ens_v2_closure_through, + has_authoritative_ens_v2_closure_through_with_progress, }; use state::{ ensure_discovery_epoch_row, ensure_retained_history_state_row, @@ -35,432 +51,51 @@ pub(in crate::ens_v2_registry) struct FullSourceRawLogHistoryGuard { chain: String, } -impl FullSourceRawLogHistoryGuard { - pub(in crate::ens_v2_registry) async fn acquire( - mut registry_sync_fence: Transaction<'static, Postgres>, - chain: &str, - ) -> Result { - // Semantic raw-log mutation triggers take this same transaction-scoped - // chain lock before advancing the revision/proof state. Other chains - // remain writable during a full ENSv2 reconciliation. - sqlx::query("SELECT pg_advisory_xact_lock(hashtextextended($1, 0))") - .bind(format!("raw_log_staging:{chain}")) - .execute(registry_sync_fence.as_mut()) - .await - .with_context(|| format!("failed to fence raw-log mutation for {chain}"))?; - // ACCESS SHARE permits ordinary INSERT/UPDATE/DELETE RowExclusive - // locks but blocks a global TRUNCATE until this proof user finishes. - sqlx::query("LOCK TABLE raw_logs IN ACCESS SHARE MODE") - .execute(registry_sync_fence.as_mut()) - .await - .with_context(|| format!("failed to fence raw-log truncation for {chain}"))?; - Ok(Self { - transaction: registry_sync_fence, - chain: chain.to_owned(), - }) - } - - /// Load the currently valid proof without requiring it to reach a newer - /// live target. A complete live fetch may extend this boundary after it - /// proves a contiguous selected path back to the stored boundary. - pub(in crate::ens_v2_registry) async fn load_current_proof( - &self, - pool: &PgPool, - ) -> Result> { - ensure_discovery_epoch_row(pool, &self.chain).await?; - let mut proof_transaction = pool - .begin() - .await - .context("failed to begin ENSv2 retained-history proof read")?; - let discovery_epoch = sqlx::query_scalar::<_, i64>( - r#" - SELECT epoch - FROM discovery_admission_epochs - WHERE chain_id = $1 - FOR SHARE - "#, +async fn ensure_generation_bound_coverage_with_optional_progress( + pool: &PgPool, + connection: &mut sqlx::PgConnection, + chain: &str, + requirements: &[RequiredWatchedTuple], + retention_generation: i64, + progress: &mut Option<&mut dyn StartupAdapterProgress>, +) -> Result<()> { + if progress.is_none() { + return ensure_generation_bound_coverage( + connection, + chain, + requirements, + retention_generation, ) - .bind(&self.chain) - .fetch_one(proof_transaction.as_mut()) - .await - .with_context(|| { - format!( - "failed to lock ENSv2 discovery-admission epoch for {}", - self.chain - ) - })?; - let state = - load_locked_retained_history_state(proof_transaction.as_mut(), &self.chain).await?; - let proof = (state.retained_history_complete - && state.proven_retention_generation == Some(state.retention_generation) - && state.proven_discovery_admission_epoch == Some(discovery_epoch)) - .then_some(RawLogClosureProof { - retention_generation: state.retention_generation, - discovery_admission_epoch: discovery_epoch, - proven_through_block: state.proven_through_block.unwrap_or(0), - }); - proof_transaction - .commit() - .await - .context("failed to release retained-history proof read locks")?; - Ok(proof) + .await; } - - /// Load a matching proof or rebuild it from generation-bound, gap-free - /// coverage while the long-lived raw-log fence is held. - pub(in crate::ens_v2_registry) async fn ensure_proof_through( - &self, - pool: &PgPool, - through_block: i64, - ) -> Result { - ensure!( - through_block >= 0, - "ENSv2 retained-history proof boundary cannot be negative" - ); - ensure_discovery_epoch_row(pool, &self.chain).await?; - - let mut proof_transaction = pool - .begin() - .await - .context("failed to begin ENSv2 retained-history proof")?; - let discovery_epoch = sqlx::query_scalar::<_, i64>( - r#" - SELECT epoch - FROM discovery_admission_epochs - WHERE chain_id = $1 - FOR SHARE - "#, - ) - .bind(&self.chain) - .fetch_one(proof_transaction.as_mut()) - .await - .with_context(|| { - format!( - "failed to lock ENSv2 discovery-admission epoch for {}", - self.chain - ) - })?; - let state = - load_locked_retained_history_state(proof_transaction.as_mut(), &self.chain).await?; - - if state.retained_history_complete - && state.proven_retention_generation == Some(state.retention_generation) - && state.proven_discovery_admission_epoch == Some(discovery_epoch) - && state - .proven_through_block - .is_some_and(|proven| proven >= through_block) - { - let proof = RawLogClosureProof { - retention_generation: state.retention_generation, - discovery_admission_epoch: discovery_epoch, - proven_through_block: state - .proven_through_block - .context("complete retained-history state is missing its proof boundary")?, - }; - proof_transaction - .commit() - .await - .context("failed to release retained-history proof locks")?; - return Ok(proof); - } - - let source_families = ens_v2_closure_source_families(); - let requirements = load_required_watched_tuples_in_transaction( - proof_transaction.as_mut(), - &self.chain, - 0, - through_block, - &source_families, - ) - .await?; - ensure!( - !requirements.is_empty(), - "ENSv2 retained history on {} has no authoritative watched closure through block {through_block}", - self.chain - ); - ensure_generation_bound_coverage( - proof_transaction.as_mut(), - &self.chain, - &requirements, - state.retention_generation, - ) - .await?; - ensure_retained_semantic_witnesses( - proof_transaction.as_mut(), - &self.chain, - &requirements, - through_block, - ) - .await?; - - sqlx::query( - r#" - UPDATE raw_log_staging_input_revisions - SET retained_history_complete = true, - incomplete_since = NULL, - proven_retention_generation = retention_generation, - proven_discovery_admission_epoch = $2, - proven_through_block = $3 - WHERE chain_id = $1 - AND retention_generation = $4 - "#, - ) - .bind(&self.chain) - .bind(discovery_epoch) - .bind(through_block) - .bind(state.retention_generation) - .execute(proof_transaction.as_mut()) - .await - .with_context(|| { - format!( - "failed to store ENSv2 retained-history proof for {}", - self.chain - ) - })?; - proof_transaction - .commit() - .await - .context("failed to commit ENSv2 retained-history proof")?; - Ok(RawLogClosureProof { - retention_generation: state.retention_generation, - discovery_admission_epoch: discovery_epoch, - proven_through_block: through_block, - }) - } - - /// Establish or extend the retained-history proof from exact provider-live - /// block bundles. Only watched tuples whose addresses were in the live - /// fetch selection receive live coverage; every remaining interval still - /// requires ordinary generation-bound backfill facts. - pub(in crate::ens_v2_registry) async fn ensure_proof_through_live_selection( - &self, - pool: &PgPool, - through_block: i64, - selected_addresses: &[String], - selected_block_hashes: &[String], - ) -> Result { - ensure!( - through_block >= 0, - "ENSv2 retained-history proof boundary cannot be negative" - ); - ensure_discovery_epoch_row(pool, &self.chain).await?; - ensure_retained_history_state_row(pool, &self.chain).await?; - let selected_block_intervals = load_selected_live_block_intervals( - pool, - &self.chain, - through_block, - selected_block_hashes, - ) - .await?; - - let mut proof_transaction = pool - .begin() - .await - .context("failed to begin ENSv2 live retained-history proof")?; - let discovery_epoch = sqlx::query_scalar::<_, i64>( - r#" - SELECT epoch - FROM discovery_admission_epochs - WHERE chain_id = $1 - FOR SHARE - "#, - ) - .bind(&self.chain) - .fetch_one(proof_transaction.as_mut()) - .await - .with_context(|| { - format!( - "failed to lock ENSv2 discovery-admission epoch for live coverage on {}", - self.chain - ) - })?; - let state = - load_locked_retained_history_state(proof_transaction.as_mut(), &self.chain).await?; - let current_proof_is_valid = state.retained_history_complete - && state.proven_retention_generation == Some(state.retention_generation) - && state.proven_discovery_admission_epoch == Some(discovery_epoch); - let required_from_block = if current_proof_is_valid { - let proven_through = state - .proven_through_block - .context("complete retained-history state is missing its proof boundary")?; - if proven_through >= through_block { - selected_block_intervals - .iter() - .map(|(from, _)| *from) - .min() - .unwrap_or(through_block) - } else { - proven_through - .checked_add(1) - .context("ENSv2 retained-history proof boundary overflow")? - } - } else { - 0 - }; - let discovery_history_source_families = ens_v2_discovery_history_source_families(); - let discovery_history_requirements = if required_from_block <= through_block { - load_required_watched_tuples_in_transaction( - proof_transaction.as_mut(), - &self.chain, - required_from_block, - through_block, - &discovery_history_source_families, - ) - .await? - } else { - Vec::new() - }; - ensure_generation_bound_coverage_with_live_selection( - proof_transaction.as_mut(), - &self.chain, - &discovery_history_requirements, - state.retention_generation, - selected_addresses, - &selected_block_intervals, - ) - .await?; - if current_proof_is_valid - && state - .proven_through_block - .is_some_and(|proven| proven >= through_block) - { - let proof = RawLogClosureProof { - retention_generation: state.retention_generation, - discovery_admission_epoch: discovery_epoch, - proven_through_block: state - .proven_through_block - .context("complete retained-history state is missing its proof boundary")?, - }; - proof_transaction - .commit() - .await - .context("failed to release live retained-history proof locks")?; - return Ok(proof); - } - - let closure_source_families = ens_v2_closure_source_families(); - let all_requirements = load_required_watched_tuples_in_transaction( - proof_transaction.as_mut(), - &self.chain, - 0, - through_block, - &closure_source_families, - ) - .await?; - ensure!( - !all_requirements.is_empty(), - "ENSv2 retained history on {} has no authoritative watched closure through block {through_block}", - self.chain - ); - ensure_retained_semantic_witnesses( - proof_transaction.as_mut(), - &self.chain, - &all_requirements, - through_block, - ) - .await?; - - sqlx::query( - r#" - UPDATE raw_log_staging_input_revisions - SET retained_history_complete = true, - incomplete_since = NULL, - proven_retention_generation = retention_generation, - proven_discovery_admission_epoch = $2, - proven_through_block = $3 - WHERE chain_id = $1 - AND retention_generation = $4 - "#, - ) - .bind(&self.chain) - .bind(discovery_epoch) - .bind(through_block) - .bind(state.retention_generation) - .execute(proof_transaction.as_mut()) - .await - .with_context(|| { - format!( - "failed to store ENSv2 live retained-history proof for {}", - self.chain - ) - })?; - proof_transaction - .commit() - .await - .context("failed to commit ENSv2 live retained-history proof")?; - Ok(RawLogClosureProof { - retention_generation: state.retention_generation, - discovery_admission_epoch: discovery_epoch, - proven_through_block: through_block, - }) - } - - /// Capture the root/registry/resolver discovery-history requirement union - /// which the caller's complete fetch represents at its new target. A - /// post-sync epoch may add authority; [`Self::finish`] requires durable - /// coverage for only the portions not in this pre-sync union. Resolver - /// requirements remain outside the retained-history proof and semantic - /// witness set. - pub(in crate::ens_v2_registry) async fn load_requirements_through( - &self, - pool: &PgPool, - proof: RawLogClosureProof, - through_block: i64, - ) -> Result> { - ensure_discovery_epoch_row(pool, &self.chain).await?; - let mut transaction = pool - .begin() - .await - .context("failed to begin ENSv2 pre-sync requirement capture")?; - let current_epoch = sqlx::query_scalar::<_, i64>( - r#" - SELECT epoch - FROM discovery_admission_epochs - WHERE chain_id = $1 - FOR SHARE - "#, - ) - .bind(&self.chain) - .fetch_one(transaction.as_mut()) - .await - .with_context(|| { - format!( - "failed to lock ENSv2 discovery-admission epoch while capturing requirements for {}", - self.chain - ) - })?; - ensure!( - current_epoch == proof.discovery_admission_epoch, - "ENSv2 discovery admission changed before requirement capture on {}: expected epoch {}, observed {current_epoch}", - self.chain, - proof.discovery_admission_epoch - ); - let requirements = load_required_watched_tuples_in_transaction( - transaction.as_mut(), - &self.chain, - 0, - through_block, - &ens_v2_discovery_history_source_families(), - ) - .await?; - transaction - .commit() - .await - .context("failed to release ENSv2 pre-sync requirement locks")?; - Ok(requirements) - } - - pub(in crate::ens_v2_registry) async fn abort(self) -> Result<()> { - self.transaction - .rollback() - .await - .context("failed to release ENSv2 raw-log read fence after reconciliation failure") + for page in requirements.chunks(RETAINED_REQUIREMENT_PROGRESS_ROWS) { + ensure_generation_bound_coverage(connection, chain, page, retention_generation).await?; + record_startup_adapter_progress(pool, progress).await?; } + Ok(()) +} - pub(in crate::ens_v2_registry) async fn release(self) -> Result<()> { - self.transaction - .commit() - .await - .context("failed to release ENSv2 raw-log read fence") +async fn ensure_retained_semantic_witnesses_with_optional_progress( + pool: &PgPool, + connection: &mut sqlx::PgConnection, + chain: &str, + requirements: &[RequiredWatchedTuple], + through_block: i64, + progress: &mut Option<&mut dyn StartupAdapterProgress>, +) -> Result<()> { + if progress.is_none() { + return ensure_retained_semantic_witnesses(connection, chain, requirements, through_block) + .await; } + ensure_retained_semantic_witnesses_with_progress( + pool, + connection, + chain, + requirements, + through_block, + progress + .as_deref_mut() + .expect("ENSv2 witness progress was checked above"), + ) + .await } diff --git a/crates/adapters/src/ens_v2_registry/live/completeness/finish.rs b/crates/adapters/src/ens_v2_registry/live/completeness/finish.rs index 3065a8cb..75082205 100644 --- a/crates/adapters/src/ens_v2_registry/live/completeness/finish.rs +++ b/crates/adapters/src/ens_v2_registry/live/completeness/finish.rs @@ -1,16 +1,23 @@ use anyhow::{Context, Result, ensure}; -use bigname_manifests::{RequiredWatchedTuple, load_required_watched_tuples_in_transaction}; +use bigname_manifests::{ + RequiredWatchedTuple, load_required_watched_tuples_in_transaction, + load_required_watched_tuples_in_transaction_with_progress, +}; use sqlx::PgPool; use super::{ FullSourceRawLogHistoryGuard, RawLogClosureProof, ens_v2_closure_source_families, ens_v2_discovery_history_source_families, ensure_newly_required_generation_bound_coverage, - ensure_retained_semantic_witnesses, load_locked_retained_history_state, - requirement_intervals_not_covered_by, + ensure_retained_semantic_witnesses_with_optional_progress, load_locked_retained_history_state, + requirement_intervals_not_covered_by, requirement_intervals_not_covered_by_with_progress, }; use crate::ens_v2_registry::live::checkpoint::{ StagedLiveRegistryReplayCheckpoint, finalize_live_registry_replay_checkpoint, }; +use crate::{ + checkpoint_context::{StartupAdapterProgress, record_startup_adapter_progress}, + startup_progress::StartupManifestProgress, +}; impl FullSourceRawLogHistoryGuard { /// Advance the proof after a successful full-source reconciliation or a @@ -24,6 +31,52 @@ impl FullSourceRawLogHistoryGuard { own_discovery_epoch_bumps: usize, pre_sync_requirements: &[RequiredWatchedTuple], staged_checkpoint: Option<&StagedLiveRegistryReplayCheckpoint>, + ) -> Result<()> { + self.finish_inner( + pool, + proof, + through_block, + own_discovery_epoch_bumps, + pre_sync_requirements, + staged_checkpoint, + None, + ) + .await + } + + #[expect(clippy::too_many_arguments)] + pub(in crate::ens_v2_registry) async fn finish_with_progress( + self, + pool: &PgPool, + proof: RawLogClosureProof, + through_block: i64, + own_discovery_epoch_bumps: usize, + pre_sync_requirements: &[RequiredWatchedTuple], + staged_checkpoint: Option<&StagedLiveRegistryReplayCheckpoint>, + progress: &mut dyn StartupAdapterProgress, + ) -> Result<()> { + self.finish_inner( + pool, + proof, + through_block, + own_discovery_epoch_bumps, + pre_sync_requirements, + staged_checkpoint, + Some(progress), + ) + .await + } + + #[expect(clippy::too_many_arguments)] + async fn finish_inner( + self, + pool: &PgPool, + proof: RawLogClosureProof, + through_block: i64, + own_discovery_epoch_bumps: usize, + pre_sync_requirements: &[RequiredWatchedTuple], + staged_checkpoint: Option<&StagedLiveRegistryReplayCheckpoint>, + mut progress: Option<&mut dyn StartupAdapterProgress>, ) -> Result<()> { let expected_bumps = i64::try_from(own_discovery_epoch_bumps) .context("ENSv2 discovery admission-epoch bump count exceeds i64")?; @@ -71,38 +124,94 @@ impl FullSourceRawLogHistoryGuard { self.chain ); - let post_sync_discovery_history_requirements = load_required_watched_tuples_in_transaction( - finish_transaction.as_mut(), - &self.chain, - 0, - through_block, - &ens_v2_discovery_history_source_families(), - ) - .await?; - let newly_required_intervals = requirement_intervals_not_covered_by( - &post_sync_discovery_history_requirements, - pre_sync_requirements, - ); - ensure_newly_required_generation_bound_coverage( - finish_transaction.as_mut(), - &self.chain, - &newly_required_intervals, - proof.retention_generation, - ) - .await?; - let post_sync_closure_requirements = load_required_watched_tuples_in_transaction( - finish_transaction.as_mut(), - &self.chain, - 0, - through_block, - &ens_v2_closure_source_families(), - ) - .await?; - ensure_retained_semantic_witnesses( + let discovery_families = ens_v2_discovery_history_source_families(); + let post_sync_discovery_history_requirements = + if let Some(progress) = progress.as_deref_mut() { + let mut manifest_progress = StartupManifestProgress::new(progress); + load_required_watched_tuples_in_transaction_with_progress( + finish_transaction.as_mut(), + pool, + &self.chain, + 0, + through_block, + &discovery_families, + &mut manifest_progress, + ) + .await? + } else { + load_required_watched_tuples_in_transaction( + finish_transaction.as_mut(), + &self.chain, + 0, + through_block, + &discovery_families, + ) + .await? + }; + let newly_required_intervals = if let Some(progress) = progress.as_deref_mut() { + requirement_intervals_not_covered_by_with_progress( + pool, + &post_sync_discovery_history_requirements, + pre_sync_requirements, + progress, + ) + .await? + } else { + requirement_intervals_not_covered_by( + &post_sync_discovery_history_requirements, + pre_sync_requirements, + ) + }; + if progress.is_some() { + for page in newly_required_intervals.chunks(super::RETAINED_REQUIREMENT_PROGRESS_ROWS) { + ensure_newly_required_generation_bound_coverage( + finish_transaction.as_mut(), + &self.chain, + page, + proof.retention_generation, + ) + .await?; + record_startup_adapter_progress(pool, &mut progress).await?; + } + } else { + ensure_newly_required_generation_bound_coverage( + finish_transaction.as_mut(), + &self.chain, + &newly_required_intervals, + proof.retention_generation, + ) + .await?; + } + let closure_families = ens_v2_closure_source_families(); + let post_sync_closure_requirements = if let Some(progress) = progress.as_deref_mut() { + let mut manifest_progress = StartupManifestProgress::new(progress); + load_required_watched_tuples_in_transaction_with_progress( + finish_transaction.as_mut(), + pool, + &self.chain, + 0, + through_block, + &closure_families, + &mut manifest_progress, + ) + .await? + } else { + load_required_watched_tuples_in_transaction( + finish_transaction.as_mut(), + &self.chain, + 0, + through_block, + &closure_families, + ) + .await? + }; + ensure_retained_semantic_witnesses_with_optional_progress( + pool, finish_transaction.as_mut(), &self.chain, &post_sync_closure_requirements, through_block, + &mut progress, ) .await?; @@ -140,6 +249,7 @@ impl FullSourceRawLogHistoryGuard { .commit() .await .context("failed to release ENSv2 raw-log read fence")?; + record_startup_adapter_progress(pool, &mut progress).await?; Ok(()) } } diff --git a/crates/adapters/src/ens_v2_registry/live/completeness/guard.rs b/crates/adapters/src/ens_v2_registry/live/completeness/guard.rs new file mode 100644 index 00000000..04b5e5d9 --- /dev/null +++ b/crates/adapters/src/ens_v2_registry/live/completeness/guard.rs @@ -0,0 +1,591 @@ +use super::*; + +impl FullSourceRawLogHistoryGuard { + pub(in crate::ens_v2_registry) async fn acquire( + mut registry_sync_fence: Transaction<'static, Postgres>, + chain: &str, + ) -> Result { + // Semantic raw-log mutation triggers take this same transaction-scoped + // chain lock before advancing the revision/proof state. Other chains + // remain writable during a full ENSv2 reconciliation. + sqlx::query("SELECT pg_advisory_xact_lock(hashtextextended($1, 0))") + .bind(format!("raw_log_staging:{chain}")) + .execute(registry_sync_fence.as_mut()) + .await + .with_context(|| format!("failed to fence raw-log mutation for {chain}"))?; + // ACCESS SHARE permits ordinary INSERT/UPDATE/DELETE RowExclusive + // locks but blocks a global TRUNCATE until this proof user finishes. + sqlx::query("LOCK TABLE raw_logs IN ACCESS SHARE MODE") + .execute(registry_sync_fence.as_mut()) + .await + .with_context(|| format!("failed to fence raw-log truncation for {chain}"))?; + Ok(Self { + transaction: registry_sync_fence, + chain: chain.to_owned(), + }) + } + + /// Load the currently valid proof without requiring it to reach a newer + /// live target. A complete live fetch may extend this boundary after it + /// proves a contiguous selected path back to the stored boundary. + pub(in crate::ens_v2_registry) async fn load_current_proof( + &self, + pool: &PgPool, + ) -> Result> { + ensure_discovery_epoch_row(pool, &self.chain).await?; + let mut proof_transaction = pool + .begin() + .await + .context("failed to begin ENSv2 retained-history proof read")?; + let discovery_epoch = sqlx::query_scalar::<_, i64>( + r#" + SELECT epoch + FROM discovery_admission_epochs + WHERE chain_id = $1 + FOR SHARE + "#, + ) + .bind(&self.chain) + .fetch_one(proof_transaction.as_mut()) + .await + .with_context(|| { + format!( + "failed to lock ENSv2 discovery-admission epoch for {}", + self.chain + ) + })?; + let state = + load_locked_retained_history_state(proof_transaction.as_mut(), &self.chain).await?; + let proof = (state.retained_history_complete + && state.proven_retention_generation == Some(state.retention_generation) + && state.proven_discovery_admission_epoch == Some(discovery_epoch)) + .then_some(RawLogClosureProof { + retention_generation: state.retention_generation, + discovery_admission_epoch: discovery_epoch, + proven_through_block: state.proven_through_block.unwrap_or(0), + }); + proof_transaction + .commit() + .await + .context("failed to release retained-history proof read locks")?; + Ok(proof) + } + + /// Load a matching proof or rebuild it from generation-bound, gap-free + /// coverage while the long-lived raw-log fence is held. + pub(in crate::ens_v2_registry) async fn ensure_proof_through( + &self, + pool: &PgPool, + through_block: i64, + ) -> Result { + self.ensure_proof_through_inner(pool, through_block, None) + .await + } + + pub(in crate::ens_v2_registry) async fn ensure_proof_through_with_progress( + &self, + pool: &PgPool, + through_block: i64, + progress: &mut dyn StartupAdapterProgress, + ) -> Result { + self.ensure_proof_through_inner(pool, through_block, Some(progress)) + .await + } + + async fn ensure_proof_through_inner( + &self, + pool: &PgPool, + through_block: i64, + mut progress: Option<&mut dyn StartupAdapterProgress>, + ) -> Result { + ensure!( + through_block >= 0, + "ENSv2 retained-history proof boundary cannot be negative" + ); + ensure_discovery_epoch_row(pool, &self.chain).await?; + + let mut proof_transaction = pool + .begin() + .await + .context("failed to begin ENSv2 retained-history proof")?; + let discovery_epoch = sqlx::query_scalar::<_, i64>( + r#" + SELECT epoch + FROM discovery_admission_epochs + WHERE chain_id = $1 + FOR SHARE + "#, + ) + .bind(&self.chain) + .fetch_one(proof_transaction.as_mut()) + .await + .with_context(|| { + format!( + "failed to lock ENSv2 discovery-admission epoch for {}", + self.chain + ) + })?; + let state = + load_locked_retained_history_state(proof_transaction.as_mut(), &self.chain).await?; + + if state.retained_history_complete + && state.proven_retention_generation == Some(state.retention_generation) + && state.proven_discovery_admission_epoch == Some(discovery_epoch) + && state + .proven_through_block + .is_some_and(|proven| proven >= through_block) + { + let proof = RawLogClosureProof { + retention_generation: state.retention_generation, + discovery_admission_epoch: discovery_epoch, + proven_through_block: state + .proven_through_block + .context("complete retained-history state is missing its proof boundary")?, + }; + proof_transaction + .commit() + .await + .context("failed to release retained-history proof locks")?; + return Ok(proof); + } + + let source_families = ens_v2_closure_source_families(); + let requirements = if let Some(progress) = progress.as_deref_mut() { + let mut manifest_progress = StartupManifestProgress::new(progress); + load_required_watched_tuples_in_transaction_with_progress( + proof_transaction.as_mut(), + pool, + &self.chain, + 0, + through_block, + &source_families, + &mut manifest_progress, + ) + .await? + } else { + load_required_watched_tuples_in_transaction( + proof_transaction.as_mut(), + &self.chain, + 0, + through_block, + &source_families, + ) + .await? + }; + ensure!( + !requirements.is_empty(), + "ENSv2 retained history on {} has no authoritative watched closure through block {through_block}", + self.chain + ); + ensure_generation_bound_coverage_with_optional_progress( + pool, + proof_transaction.as_mut(), + &self.chain, + &requirements, + state.retention_generation, + &mut progress, + ) + .await?; + ensure_retained_semantic_witnesses_with_optional_progress( + pool, + proof_transaction.as_mut(), + &self.chain, + &requirements, + through_block, + &mut progress, + ) + .await?; + + sqlx::query( + r#" + UPDATE raw_log_staging_input_revisions + SET retained_history_complete = true, + incomplete_since = NULL, + proven_retention_generation = retention_generation, + proven_discovery_admission_epoch = $2, + proven_through_block = $3 + WHERE chain_id = $1 + AND retention_generation = $4 + "#, + ) + .bind(&self.chain) + .bind(discovery_epoch) + .bind(through_block) + .bind(state.retention_generation) + .execute(proof_transaction.as_mut()) + .await + .with_context(|| { + format!( + "failed to store ENSv2 retained-history proof for {}", + self.chain + ) + })?; + proof_transaction + .commit() + .await + .context("failed to commit ENSv2 retained-history proof")?; + Ok(RawLogClosureProof { + retention_generation: state.retention_generation, + discovery_admission_epoch: discovery_epoch, + proven_through_block: through_block, + }) + } + + /// Establish or extend the retained-history proof from exact provider-live + /// block bundles. Only watched tuples whose addresses were in the live + /// fetch selection receive live coverage; every remaining interval still + /// requires ordinary generation-bound backfill facts. + pub(in crate::ens_v2_registry) async fn ensure_proof_through_live_selection( + &self, + pool: &PgPool, + through_block: i64, + selected_addresses: &[String], + selected_block_hashes: &[String], + ) -> Result { + self.ensure_proof_through_live_selection_inner( + pool, + through_block, + selected_addresses, + selected_block_hashes, + None, + ) + .await + } + + pub(in crate::ens_v2_registry) async fn ensure_proof_through_live_selection_with_progress( + &self, + pool: &PgPool, + through_block: i64, + selected_addresses: &[String], + selected_block_hashes: &[String], + progress: &mut dyn StartupAdapterProgress, + ) -> Result { + self.ensure_proof_through_live_selection_inner( + pool, + through_block, + selected_addresses, + selected_block_hashes, + Some(progress), + ) + .await + } + + async fn ensure_proof_through_live_selection_inner( + &self, + pool: &PgPool, + through_block: i64, + selected_addresses: &[String], + selected_block_hashes: &[String], + mut progress: Option<&mut dyn StartupAdapterProgress>, + ) -> Result { + ensure!( + through_block >= 0, + "ENSv2 retained-history proof boundary cannot be negative" + ); + ensure_discovery_epoch_row(pool, &self.chain).await?; + ensure_retained_history_state_row(pool, &self.chain).await?; + let selected_block_intervals = load_selected_live_block_intervals( + pool, + &self.chain, + through_block, + selected_block_hashes, + ) + .await?; + + let mut proof_transaction = pool + .begin() + .await + .context("failed to begin ENSv2 live retained-history proof")?; + let discovery_epoch = sqlx::query_scalar::<_, i64>( + r#" + SELECT epoch + FROM discovery_admission_epochs + WHERE chain_id = $1 + FOR SHARE + "#, + ) + .bind(&self.chain) + .fetch_one(proof_transaction.as_mut()) + .await + .with_context(|| { + format!( + "failed to lock ENSv2 discovery-admission epoch for live coverage on {}", + self.chain + ) + })?; + let state = + load_locked_retained_history_state(proof_transaction.as_mut(), &self.chain).await?; + let current_proof_is_valid = state.retained_history_complete + && state.proven_retention_generation == Some(state.retention_generation) + && state.proven_discovery_admission_epoch == Some(discovery_epoch); + let required_from_block = if current_proof_is_valid { + let proven_through = state + .proven_through_block + .context("complete retained-history state is missing its proof boundary")?; + if proven_through >= through_block { + selected_block_intervals + .iter() + .map(|(from, _)| *from) + .min() + .unwrap_or(through_block) + } else { + proven_through + .checked_add(1) + .context("ENSv2 retained-history proof boundary overflow")? + } + } else { + 0 + }; + let discovery_history_source_families = ens_v2_discovery_history_source_families(); + let discovery_history_requirements = if required_from_block <= through_block { + if let Some(progress) = progress.as_deref_mut() { + let mut manifest_progress = StartupManifestProgress::new(progress); + load_required_watched_tuples_in_transaction_with_progress( + proof_transaction.as_mut(), + pool, + &self.chain, + required_from_block, + through_block, + &discovery_history_source_families, + &mut manifest_progress, + ) + .await? + } else { + load_required_watched_tuples_in_transaction( + proof_transaction.as_mut(), + &self.chain, + required_from_block, + through_block, + &discovery_history_source_families, + ) + .await? + } + } else { + Vec::new() + }; + match progress.as_deref_mut() { + Some(progress) => { + ensure_generation_bound_coverage_with_live_selection_with_progress( + pool, + proof_transaction.as_mut(), + &self.chain, + &discovery_history_requirements, + state.retention_generation, + selected_addresses, + &selected_block_intervals, + progress, + ) + .await?; + } + None => { + ensure_generation_bound_coverage_with_live_selection( + proof_transaction.as_mut(), + &self.chain, + &discovery_history_requirements, + state.retention_generation, + selected_addresses, + &selected_block_intervals, + ) + .await?; + } + } + if current_proof_is_valid + && state + .proven_through_block + .is_some_and(|proven| proven >= through_block) + { + let proof = RawLogClosureProof { + retention_generation: state.retention_generation, + discovery_admission_epoch: discovery_epoch, + proven_through_block: state + .proven_through_block + .context("complete retained-history state is missing its proof boundary")?, + }; + proof_transaction + .commit() + .await + .context("failed to release live retained-history proof locks")?; + record_startup_adapter_progress(pool, &mut progress).await?; + return Ok(proof); + } + + let closure_source_families = ens_v2_closure_source_families(); + let all_requirements = if let Some(progress) = progress.as_deref_mut() { + let mut manifest_progress = StartupManifestProgress::new(progress); + load_required_watched_tuples_in_transaction_with_progress( + proof_transaction.as_mut(), + pool, + &self.chain, + 0, + through_block, + &closure_source_families, + &mut manifest_progress, + ) + .await? + } else { + load_required_watched_tuples_in_transaction( + proof_transaction.as_mut(), + &self.chain, + 0, + through_block, + &closure_source_families, + ) + .await? + }; + ensure!( + !all_requirements.is_empty(), + "ENSv2 retained history on {} has no authoritative watched closure through block {through_block}", + self.chain + ); + ensure_retained_semantic_witnesses_with_optional_progress( + pool, + proof_transaction.as_mut(), + &self.chain, + &all_requirements, + through_block, + &mut progress, + ) + .await?; + + sqlx::query( + r#" + UPDATE raw_log_staging_input_revisions + SET retained_history_complete = true, + incomplete_since = NULL, + proven_retention_generation = retention_generation, + proven_discovery_admission_epoch = $2, + proven_through_block = $3 + WHERE chain_id = $1 + AND retention_generation = $4 + "#, + ) + .bind(&self.chain) + .bind(discovery_epoch) + .bind(through_block) + .bind(state.retention_generation) + .execute(proof_transaction.as_mut()) + .await + .with_context(|| { + format!( + "failed to store ENSv2 live retained-history proof for {}", + self.chain + ) + })?; + proof_transaction + .commit() + .await + .context("failed to commit ENSv2 live retained-history proof")?; + record_startup_adapter_progress(pool, &mut progress).await?; + Ok(RawLogClosureProof { + retention_generation: state.retention_generation, + discovery_admission_epoch: discovery_epoch, + proven_through_block: through_block, + }) + } + + /// Capture the root/registry/resolver discovery-history requirement union + /// which the caller's complete fetch represents at its new target. A + /// post-sync epoch may add authority; [`Self::finish`] requires durable + /// coverage for only the portions not in this pre-sync union. Resolver + /// requirements remain outside the retained-history proof and semantic + /// witness set. + pub(in crate::ens_v2_registry) async fn load_requirements_through( + &self, + pool: &PgPool, + proof: RawLogClosureProof, + through_block: i64, + ) -> Result> { + self.load_requirements_through_inner(pool, proof, through_block, None) + .await + } + + pub(in crate::ens_v2_registry) async fn load_requirements_through_with_progress( + &self, + pool: &PgPool, + proof: RawLogClosureProof, + through_block: i64, + progress: &mut dyn StartupAdapterProgress, + ) -> Result> { + self.load_requirements_through_inner(pool, proof, through_block, Some(progress)) + .await + } + + async fn load_requirements_through_inner( + &self, + pool: &PgPool, + proof: RawLogClosureProof, + through_block: i64, + mut progress: Option<&mut dyn StartupAdapterProgress>, + ) -> Result> { + ensure_discovery_epoch_row(pool, &self.chain).await?; + let mut transaction = pool + .begin() + .await + .context("failed to begin ENSv2 pre-sync requirement capture")?; + let current_epoch = sqlx::query_scalar::<_, i64>( + r#" + SELECT epoch + FROM discovery_admission_epochs + WHERE chain_id = $1 + FOR SHARE + "#, + ) + .bind(&self.chain) + .fetch_one(transaction.as_mut()) + .await + .with_context(|| { + format!( + "failed to lock ENSv2 discovery-admission epoch while capturing requirements for {}", + self.chain + ) + })?; + ensure!( + current_epoch == proof.discovery_admission_epoch, + "ENSv2 discovery admission changed before requirement capture on {}: expected epoch {}, observed {current_epoch}", + self.chain, + proof.discovery_admission_epoch + ); + let source_families = ens_v2_discovery_history_source_families(); + let requirements = if let Some(progress) = progress.as_deref_mut() { + let mut manifest_progress = StartupManifestProgress::new(progress); + load_required_watched_tuples_in_transaction_with_progress( + transaction.as_mut(), + pool, + &self.chain, + 0, + through_block, + &source_families, + &mut manifest_progress, + ) + .await? + } else { + load_required_watched_tuples_in_transaction( + transaction.as_mut(), + &self.chain, + 0, + through_block, + &source_families, + ) + .await? + }; + transaction + .commit() + .await + .context("failed to release ENSv2 pre-sync requirement locks")?; + Ok(requirements) + } + + pub(in crate::ens_v2_registry) async fn abort(self) -> Result<()> { + self.transaction + .rollback() + .await + .context("failed to release ENSv2 raw-log read fence after reconciliation failure") + } + + pub(in crate::ens_v2_registry) async fn release(self) -> Result<()> { + self.transaction + .commit() + .await + .context("failed to release ENSv2 raw-log read fence") + } +} diff --git a/crates/adapters/src/ens_v2_registry/live/completeness/requirements.rs b/crates/adapters/src/ens_v2_registry/live/completeness/requirements.rs index f17e7ce5..3507a25c 100644 --- a/crates/adapters/src/ens_v2_registry/live/completeness/requirements.rs +++ b/crates/adapters/src/ens_v2_registry/live/completeness/requirements.rs @@ -1,20 +1,31 @@ use std::collections::BTreeMap; -use anyhow::{Context, Result, ensure}; +use anyhow::{Result, ensure}; use bigname_domain::block_interval::InclusiveBlockInterval; -use bigname_manifests::{RequiredWatchedTuple, load_required_watched_tuples}; +use bigname_manifests::{ + RequiredWatchedTuple, load_required_watched_tuples, load_required_watched_tuples_with_progress, +}; use sqlx::PgPool; +use crate::{ + checkpoint_context::StartupAdapterProgress, startup_progress::StartupManifestProgress, +}; + use crate::ens_v2_registry::constants::{ - DERIVATION_KIND_ENS_V2_REGISTRY_RESOURCE_SURFACE, SOURCE_FAMILY_ENS_V2_REGISTRY_L1, - SOURCE_FAMILY_ENS_V2_RESOLVER_L1, SOURCE_FAMILY_ENS_V2_ROOT_L1, + SOURCE_FAMILY_ENS_V2_REGISTRY_L1, SOURCE_FAMILY_ENS_V2_RESOLVER_L1, + SOURCE_FAMILY_ENS_V2_ROOT_L1, }; mod coverage; +mod witnesses; pub(super) use coverage::{ ensure_generation_bound_coverage, ensure_generation_bound_coverage_with_live_selection, + ensure_generation_bound_coverage_with_live_selection_with_progress, ensure_newly_required_generation_bound_coverage, }; +pub(super) use witnesses::{ + ensure_retained_semantic_witnesses, ensure_retained_semantic_witnesses_with_progress, +}; pub(in crate::ens_v2_registry) async fn has_authoritative_ens_v2_closure_through( pool: &PgPool, @@ -36,6 +47,29 @@ pub(in crate::ens_v2_registry) async fn has_authoritative_ens_v2_closure_through .is_empty()) } +pub(in crate::ens_v2_registry) async fn has_authoritative_ens_v2_closure_through_with_progress( + pool: &PgPool, + chain: &str, + through_block: i64, + progress: &mut dyn StartupAdapterProgress, +) -> Result { + ensure!( + through_block >= 0, + "ENSv2 authoritative-closure boundary cannot be negative" + ); + let mut manifest_progress = StartupManifestProgress::new(progress); + Ok(!load_required_watched_tuples_with_progress( + pool, + chain, + 0, + through_block, + &ens_v2_closure_source_families(), + &mut manifest_progress, + ) + .await? + .is_empty()) +} + pub(super) fn ens_v2_closure_source_families() -> Vec { vec![ SOURCE_FAMILY_ENS_V2_ROOT_L1.to_owned(), @@ -103,152 +137,72 @@ pub(super) fn requirement_intervals_not_covered_by( gaps } -pub(super) async fn ensure_retained_semantic_witnesses( - connection: &mut sqlx::PgConnection, - chain: &str, - requirements: &[RequiredWatchedTuple], - through_block: i64, -) -> Result<()> { - let source_families = requirements - .iter() - .map(|requirement| requirement.source_family.clone()) - .collect::>(); - let addresses = requirements - .iter() - .map(|requirement| requirement.address.to_ascii_lowercase()) - .collect::>(); - let from_blocks = requirements - .iter() - .map(|requirement| requirement.required_from_block) - .collect::>(); - let to_blocks = requirements - .iter() - .map(|requirement| requirement.required_to_block) - .collect::>(); - let complete = sqlx::query_scalar::<_, bool>( - r#" - WITH required_tuples AS ( - SELECT * - FROM UNNEST( - $3::TEXT[], - $4::TEXT[], - $5::BIGINT[], - $6::BIGINT[] - ) AS watched( - source_family, - address, - required_from_block, - required_to_block - ) - ), - readable_lineage AS ( - SELECT chain_id, block_hash, block_number - FROM chain_lineage - WHERE canonicality_state IN ( - 'canonical'::canonicality_state, - 'safe'::canonicality_state, - 'finalized'::canonicality_state - ) - ), - readable_raw_logs AS ( - SELECT raw.chain_id, raw.block_hash, raw.block_number, raw.log_index - FROM raw_logs raw - JOIN readable_lineage lineage - ON lineage.chain_id = raw.chain_id - AND lineage.block_hash = raw.block_hash - AND lineage.block_number = raw.block_number - WHERE raw.canonicality_state IN ( - 'canonical'::canonicality_state, - 'safe'::canonicality_state, - 'finalized'::canonicality_state - ) - ) - SELECT - NOT EXISTS ( - SELECT 1 - FROM normalized_events event - WHERE event.chain_id = $1 - AND event.derivation_kind = $2 - AND event.raw_fact_ref ->> 'kind' = 'raw_log' - AND event.block_number <= $7 - AND event.canonicality_state IN ( - 'canonical'::canonicality_state, - 'safe'::canonicality_state, - 'finalized'::canonicality_state - ) - AND EXISTS ( - SELECT 1 - FROM readable_lineage lineage - WHERE lineage.chain_id = event.chain_id - AND lineage.block_hash = event.block_hash - AND lineage.block_number = event.block_number - ) - AND EXISTS ( - SELECT 1 - FROM required_tuples watched - WHERE watched.source_family = event.source_family - AND watched.address = lower(event.raw_fact_ref ->> 'emitting_address') - AND event.block_number BETWEEN watched.required_from_block - AND watched.required_to_block - ) - AND NOT EXISTS ( - SELECT 1 - FROM readable_raw_logs raw - WHERE raw.chain_id = event.chain_id - AND raw.block_hash = event.block_hash - AND raw.block_number = event.block_number - AND raw.log_index = event.log_index - ) - ) - AND NOT EXISTS ( - SELECT 1 - FROM discovery_edges edge - WHERE edge.chain_id = $1 - AND edge.discovery_source LIKE 'ens_v2_registry_%' - AND edge.provenance ->> 'source' = 'raw_log' - AND edge.active_from_block_number <= $7 - AND EXISTS ( - SELECT 1 - FROM readable_lineage lineage - WHERE lineage.chain_id = edge.chain_id - AND lineage.block_hash = edge.provenance ->> 'block_hash' - AND lineage.block_number = edge.active_from_block_number - ) - AND EXISTS ( - SELECT 1 - FROM required_tuples watched - WHERE watched.address = lower(edge.provenance ->> 'from_address') - AND edge.active_from_block_number - BETWEEN watched.required_from_block AND watched.required_to_block - ) - AND NOT EXISTS ( - SELECT 1 - FROM readable_raw_logs raw - WHERE raw.chain_id = edge.chain_id - AND raw.block_hash = edge.provenance ->> 'block_hash' - AND raw.block_number = edge.active_from_block_number - AND raw.log_index::TEXT = edge.provenance ->> 'log_index' - ) - ) - "#, - ) - .bind(chain) - .bind(DERIVATION_KIND_ENS_V2_REGISTRY_RESOURCE_SURFACE) - .bind(&source_families) - .bind(&addresses) - .bind(&from_blocks) - .bind(&to_blocks) - .bind(through_block) - .fetch_one(connection) - .await - .with_context(|| { - format!("failed to verify retained ENSv2 semantic raw-log witnesses for {chain}") - })?; - ensure!( - complete, - "ENSv2 retained history on {chain} is missing raw-log witnesses for materialized ENSv2 events or discovery through block {through_block}" - ); - Ok(()) +pub(super) async fn requirement_intervals_not_covered_by_with_progress( + pool: &PgPool, + required: &[RequiredWatchedTuple], + covered: &[RequiredWatchedTuple], + progress: &mut dyn StartupAdapterProgress, +) -> Result> { + let mut covered_by_tuple = BTreeMap::<(String, String), Vec>::new(); + for (index, requirement) in covered.iter().enumerate() { + if let Some(interval) = InclusiveBlockInterval::new( + requirement.required_from_block, + requirement.required_to_block, + ) { + covered_by_tuple + .entry(( + requirement.source_family.clone(), + requirement.address.clone(), + )) + .or_default() + .push(interval); + } + if (index + 1).is_multiple_of(super::RETAINED_REQUIREMENT_PROGRESS_ROWS) { + progress.record(pool).await?; + } + } + if !covered.is_empty() + && !covered + .len() + .is_multiple_of(super::RETAINED_REQUIREMENT_PROGRESS_ROWS) + { + progress.record(pool).await?; + } + + let mut gaps = Vec::new(); + for (index, requirement) in required.iter().enumerate() { + if let Some(required_interval) = InclusiveBlockInterval::new( + requirement.required_from_block, + requirement.required_to_block, + ) { + let key = ( + requirement.source_family.clone(), + requirement.address.clone(), + ); + gaps.extend( + required_interval + .uncovered_by(covered_by_tuple.get(&key).into_iter().flatten().copied()) + .into_iter() + .map(|gap| RequiredWatchedTuple { + source_family: requirement.source_family.clone(), + address: requirement.address.clone(), + required_from_block: gap.from_block(), + required_to_block: gap.through_block(), + }), + ); + } + if (index + 1).is_multiple_of(super::RETAINED_REQUIREMENT_PROGRESS_ROWS) { + progress.record(pool).await?; + } + } + if !required.is_empty() + && !required + .len() + .is_multiple_of(super::RETAINED_REQUIREMENT_PROGRESS_ROWS) + { + progress.record(pool).await?; + } + Ok(gaps) } #[cfg(test)] diff --git a/crates/adapters/src/ens_v2_registry/live/completeness/requirements/coverage.rs b/crates/adapters/src/ens_v2_registry/live/completeness/requirements/coverage.rs index 53869e85..a11390bc 100644 --- a/crates/adapters/src/ens_v2_registry/live/completeness/requirements/coverage.rs +++ b/crates/adapters/src/ens_v2_registry/live/completeness/requirements/coverage.rs @@ -3,6 +3,8 @@ use bigname_manifests::RequiredWatchedTuple; use sqlx::Row; use super::requirement_intervals_not_covered_by; +use super::requirement_intervals_not_covered_by_with_progress; +use crate::checkpoint_context::StartupAdapterProgress; use crate::ens_v2_registry::EnsV2MissingCoverage; pub(in crate::ens_v2_registry::live::completeness) async fn ensure_generation_bound_coverage( @@ -62,6 +64,71 @@ pub(in crate::ens_v2_registry::live::completeness) async fn ensure_generation_bo .await } +#[expect(clippy::too_many_arguments)] +pub(in crate::ens_v2_registry::live::completeness) async fn ensure_generation_bound_coverage_with_live_selection_with_progress( + pool: &sqlx::PgPool, + connection: &mut sqlx::PgConnection, + chain: &str, + requirements: &[RequiredWatchedTuple], + retention_generation: i64, + selected_addresses: &[String], + selected_block_intervals: &[(i64, i64)], + progress: &mut dyn StartupAdapterProgress, +) -> Result<()> { + let selected_addresses = selected_addresses + .iter() + .map(|address| address.to_ascii_lowercase()) + .collect::>(); + let mut live_coverage = Vec::new(); + let mut examined = 0usize; + for requirement in requirements { + if selected_addresses.contains(&requirement.address.to_ascii_lowercase()) { + for &(selected_from, selected_to) in selected_block_intervals { + let covered_from = requirement.required_from_block.max(selected_from); + let covered_to = requirement.required_to_block.min(selected_to); + if covered_from <= covered_to { + live_coverage.push(RequiredWatchedTuple { + source_family: requirement.source_family.clone(), + address: requirement.address.clone(), + required_from_block: covered_from, + required_to_block: covered_to, + }); + } + examined += 1; + if examined.is_multiple_of(super::super::RETAINED_REQUIREMENT_PROGRESS_ROWS) { + progress.record(pool).await?; + } + } + } else { + examined += 1; + if examined.is_multiple_of(super::super::RETAINED_REQUIREMENT_PROGRESS_ROWS) { + progress.record(pool).await?; + } + } + } + if examined > 0 && !examined.is_multiple_of(super::super::RETAINED_REQUIREMENT_PROGRESS_ROWS) { + progress.record(pool).await?; + } + let remaining_requirements = requirement_intervals_not_covered_by_with_progress( + pool, + requirements, + &live_coverage, + progress, + ) + .await?; + for page in remaining_requirements.chunks(super::super::RETAINED_REQUIREMENT_PROGRESS_ROWS) { + ensure_newly_required_generation_bound_coverage( + connection, + chain, + page, + retention_generation, + ) + .await?; + progress.record(pool).await?; + } + Ok(()) +} + pub(in crate::ens_v2_registry::live::completeness) async fn ensure_newly_required_generation_bound_coverage( connection: &mut sqlx::PgConnection, chain: &str, diff --git a/crates/adapters/src/ens_v2_registry/live/completeness/requirements/witnesses.rs b/crates/adapters/src/ens_v2_registry/live/completeness/requirements/witnesses.rs new file mode 100644 index 00000000..141b1700 --- /dev/null +++ b/crates/adapters/src/ens_v2_registry/live/completeness/requirements/witnesses.rs @@ -0,0 +1,297 @@ +use std::collections::BTreeMap; + +use anyhow::{Context, Result, ensure}; +use bigname_domain::block_interval::{InclusiveBlockInterval, coalesce_inclusive_block_intervals}; +use bigname_manifests::RequiredWatchedTuple; +use sqlx::{PgPool, Row}; + +use crate::{ + checkpoint_context::StartupAdapterProgress, + ens_v2_registry::constants::DERIVATION_KIND_ENS_V2_REGISTRY_RESOURCE_SURFACE, +}; + +const WITNESS_PAGE_ROWS: i64 = 1_000; + +type EventRequirements = BTreeMap<(String, String), Vec>; +type AddressRequirements = BTreeMap>; + +pub(in crate::ens_v2_registry) async fn ensure_retained_semantic_witnesses( + connection: &mut sqlx::PgConnection, + chain: &str, + requirements: &[RequiredWatchedTuple], + through_block: i64, +) -> Result<()> { + ensure_retained_semantic_witnesses_inner( + None, + connection, + chain, + requirements, + through_block, + None, + ) + .await +} + +pub(in crate::ens_v2_registry) async fn ensure_retained_semantic_witnesses_with_progress( + pool: &PgPool, + connection: &mut sqlx::PgConnection, + chain: &str, + requirements: &[RequiredWatchedTuple], + through_block: i64, + progress: &mut dyn StartupAdapterProgress, +) -> Result<()> { + ensure_retained_semantic_witnesses_inner( + Some(pool), + connection, + chain, + requirements, + through_block, + Some(progress), + ) + .await +} + +async fn ensure_retained_semantic_witnesses_inner( + pool: Option<&PgPool>, + connection: &mut sqlx::PgConnection, + chain: &str, + requirements: &[RequiredWatchedTuple], + through_block: i64, + mut progress: Option<&mut dyn StartupAdapterProgress>, +) -> Result<()> { + let (event_requirements, address_requirements) = + requirement_indexes(pool, requirements, &mut progress).await?; + verify_event_witnesses( + pool, + connection, + chain, + through_block, + &event_requirements, + &mut progress, + ) + .await?; + verify_discovery_witnesses( + pool, + connection, + chain, + through_block, + &address_requirements, + &mut progress, + ) + .await +} + +async fn requirement_indexes( + pool: Option<&PgPool>, + requirements: &[RequiredWatchedTuple], + progress: &mut Option<&mut dyn StartupAdapterProgress>, +) -> Result<(EventRequirements, AddressRequirements)> { + let mut events = EventRequirements::new(); + let mut addresses = AddressRequirements::new(); + for (index, requirement) in requirements.iter().enumerate() { + if let Some(interval) = InclusiveBlockInterval::new( + requirement.required_from_block, + requirement.required_to_block, + ) { + let address = requirement.address.to_ascii_lowercase(); + events + .entry((requirement.source_family.clone(), address.clone())) + .or_default() + .push(interval); + addresses.entry(address).or_default().push(interval); + } + if (index + 1).is_multiple_of(WITNESS_PAGE_ROWS as usize) { + record_progress(pool, progress).await?; + } + } + for intervals in events.values_mut() { + *intervals = coalesce_inclusive_block_intervals(intervals.iter().copied()); + } + for intervals in addresses.values_mut() { + *intervals = coalesce_inclusive_block_intervals(intervals.iter().copied()); + } + if !requirements.is_empty() { + record_progress(pool, progress).await?; + } + Ok((events, addresses)) +} + +async fn verify_event_witnesses( + pool: Option<&PgPool>, + connection: &mut sqlx::PgConnection, + chain: &str, + through_block: i64, + requirements: &EventRequirements, + progress: &mut Option<&mut dyn StartupAdapterProgress>, +) -> Result<()> { + let mut after_id = 0i64; + loop { + let ids = sqlx::query_scalar::<_, i64>( + "SELECT normalized_event_id FROM normalized_events WHERE normalized_event_id > $1 ORDER BY normalized_event_id LIMIT $2", + ) + .bind(after_id) + .bind(WITNESS_PAGE_ROWS) + .fetch_all(&mut *connection) + .await + .context("failed to page normalized-event identities for ENSv2 witness verification")?; + let Some(last_id) = ids.last().copied() else { + break; + }; + let rows = sqlx::query( + r#" + SELECT + event.source_family, + lower(event.raw_fact_ref ->> 'emitting_address') AS emitting_address, + event.block_number, + EXISTS ( + SELECT 1 FROM chain_lineage lineage + WHERE lineage.chain_id = event.chain_id + AND lineage.block_hash = event.block_hash + AND lineage.block_number = event.block_number + AND lineage.canonicality_state IN ('canonical', 'safe', 'finalized') + ) AS readable_lineage, + EXISTS ( + SELECT 1 + FROM raw_logs raw + JOIN chain_lineage lineage + ON lineage.chain_id = raw.chain_id + AND lineage.block_hash = raw.block_hash + AND lineage.block_number = raw.block_number + AND lineage.canonicality_state IN ('canonical', 'safe', 'finalized') + WHERE raw.chain_id = event.chain_id + AND raw.block_hash = event.block_hash + AND raw.block_number = event.block_number + AND raw.log_index = event.log_index + AND raw.canonicality_state IN ('canonical', 'safe', 'finalized') + ) AS has_raw_witness + FROM normalized_events event + WHERE event.normalized_event_id = ANY($1::BIGINT[]) + AND event.chain_id = $2 + AND event.derivation_kind = $3 + AND event.raw_fact_ref ->> 'kind' = 'raw_log' + AND event.block_number <= $4 + AND event.canonicality_state IN ('canonical', 'safe', 'finalized') + "#, + ) + .bind(&ids) + .bind(chain) + .bind(DERIVATION_KIND_ENS_V2_REGISTRY_RESOURCE_SURFACE) + .bind(through_block) + .fetch_all(&mut *connection) + .await + .context("failed to verify an ENSv2 normalized-event witness page")?; + for row in rows { + let source_family: String = row.try_get("source_family")?; + let address: String = row.try_get("emitting_address")?; + let block_number: i64 = row.try_get("block_number")?; + let required = requirements + .get(&(source_family, address)) + .is_some_and(|intervals| contains_block(intervals, block_number)); + let readable_lineage: bool = row.try_get("readable_lineage")?; + let has_raw_witness: bool = row.try_get("has_raw_witness")?; + ensure!( + !required || !readable_lineage || has_raw_witness, + "ENSv2 retained history on {chain} is missing raw-log witnesses for materialized ENSv2 events or discovery through block {through_block}" + ); + } + after_id = last_id; + record_progress(pool, progress).await?; + } + Ok(()) +} + +async fn verify_discovery_witnesses( + pool: Option<&PgPool>, + connection: &mut sqlx::PgConnection, + chain: &str, + through_block: i64, + requirements: &AddressRequirements, + progress: &mut Option<&mut dyn StartupAdapterProgress>, +) -> Result<()> { + let mut after_id = 0i64; + loop { + let ids = sqlx::query_scalar::<_, i64>( + "SELECT discovery_edge_id FROM discovery_edges WHERE discovery_edge_id > $1 ORDER BY discovery_edge_id LIMIT $2", + ) + .bind(after_id) + .bind(WITNESS_PAGE_ROWS) + .fetch_all(&mut *connection) + .await + .context("failed to page discovery-edge identities for ENSv2 witness verification")?; + let Some(last_id) = ids.last().copied() else { + break; + }; + let rows = sqlx::query( + r#" + SELECT + lower(edge.provenance ->> 'from_address') AS from_address, + edge.active_from_block_number, + EXISTS ( + SELECT 1 FROM chain_lineage lineage + WHERE lineage.chain_id = edge.chain_id + AND lineage.block_hash = edge.provenance ->> 'block_hash' + AND lineage.block_number = edge.active_from_block_number + AND lineage.canonicality_state IN ('canonical', 'safe', 'finalized') + ) AS readable_lineage, + EXISTS ( + SELECT 1 + FROM raw_logs raw + JOIN chain_lineage lineage + ON lineage.chain_id = raw.chain_id + AND lineage.block_hash = raw.block_hash + AND lineage.block_number = raw.block_number + AND lineage.canonicality_state IN ('canonical', 'safe', 'finalized') + WHERE raw.chain_id = edge.chain_id + AND raw.block_hash = edge.provenance ->> 'block_hash' + AND raw.block_number = edge.active_from_block_number + AND raw.log_index::TEXT = edge.provenance ->> 'log_index' + AND raw.canonicality_state IN ('canonical', 'safe', 'finalized') + ) AS has_raw_witness + FROM discovery_edges edge + WHERE edge.discovery_edge_id = ANY($1::BIGINT[]) + AND edge.chain_id = $2 + AND edge.discovery_source LIKE 'ens_v2_registry_%' + AND edge.provenance ->> 'source' = 'raw_log' + AND edge.active_from_block_number <= $3 + "#, + ) + .bind(&ids) + .bind(chain) + .bind(through_block) + .fetch_all(&mut *connection) + .await + .context("failed to verify an ENSv2 discovery-edge witness page")?; + for row in rows { + let address: String = row.try_get("from_address")?; + let block_number: i64 = row.try_get("active_from_block_number")?; + let required = requirements + .get(&address) + .is_some_and(|intervals| contains_block(intervals, block_number)); + let readable_lineage: bool = row.try_get("readable_lineage")?; + let has_raw_witness: bool = row.try_get("has_raw_witness")?; + ensure!( + !required || !readable_lineage || has_raw_witness, + "ENSv2 retained history on {chain} is missing raw-log witnesses for materialized ENSv2 events or discovery through block {through_block}" + ); + } + after_id = last_id; + record_progress(pool, progress).await?; + } + Ok(()) +} + +fn contains_block(intervals: &[InclusiveBlockInterval], block_number: i64) -> bool { + intervals.iter().any(|interval| { + block_number >= interval.from_block() && block_number <= interval.through_block() + }) +} + +async fn record_progress( + pool: Option<&PgPool>, + progress: &mut Option<&mut dyn StartupAdapterProgress>, +) -> Result<()> { + if let (Some(pool), Some(progress)) = (pool, progress.as_deref_mut()) { + progress.record(pool).await?; + } + Ok(()) +} diff --git a/crates/adapters/src/ens_v2_registry/live/reuse.rs b/crates/adapters/src/ens_v2_registry/live/reuse.rs new file mode 100644 index 00000000..6c0f2910 --- /dev/null +++ b/crates/adapters/src/ens_v2_registry/live/reuse.rs @@ -0,0 +1,102 @@ +use super::*; + +pub(super) async fn load_reusable_durable_checkpoint( + pool: &PgPool, + chain: &str, + target_block_number: i64, + target_block_hash: &str, + current_closure_proof: Option, + metadata: &RegistryCacheMetadata, + header: LiveRegistryReplayCheckpointHeader, +) -> Result< + Option<( + CachedLiveRegistryReplayState, + RawLogClosureProof, + SelectedRegistryPath, + )>, +> { + let Some((proof, path)) = reusable_checkpoint_path( + pool, + chain, + target_block_number, + target_block_hash, + current_closure_proof, + metadata, + header.through_block_number, + &header.through_block_hash, + header.raw_log_input_revision, + header.raw_log_retention_generation, + header.discovery_admission_epoch, + ) + .await? + else { + tracing::warn!( + deployment_profile = header.deployment_profile, + chain, + "discarding stale ENSv2 live checkpoint" + ); + return Ok(None); + }; + match load_live_registry_replay_checkpoint(pool, &header).await? { + LiveRegistryReplayCheckpointLoad::Ready(snapshot) => Ok(Some((snapshot, proof, path))), + LiveRegistryReplayCheckpointLoad::Missing => Ok(None), + LiveRegistryReplayCheckpointLoad::Invalid(reason) => { + tracing::warn!( + deployment_profile = header.deployment_profile, + chain, + reason, + "discarding invalid ENSv2 live checkpoint payload" + ); + Ok(None) + } + } +} + +#[allow(clippy::too_many_arguments)] +pub(super) async fn reusable_checkpoint_path( + pool: &PgPool, + chain: &str, + target_block_number: i64, + target_block_hash: &str, + current_closure_proof: Option, + metadata: &RegistryCacheMetadata, + through_block_number: i64, + through_block_hash: &str, + raw_log_input_revision: i64, + raw_log_retention_generation: i64, + discovery_admission_epoch: i64, +) -> Result> { + let Some(proof) = current_closure_proof.filter(|proof| { + proof.proven_through_block >= through_block_number + && target_block_number >= through_block_number + }) else { + return Ok(None); + }; + if discovery_admission_epoch != proof.discovery_admission_epoch + || raw_log_retention_generation != proof.retention_generation + || !metadata.retained_raw_log_history_complete + { + return Ok(None); + } + let path = load_selected_registry_path_to_floor( + pool, + chain, + target_block_number, + target_block_hash, + through_block_number, + ) + .await?; + if !path.contains_anchor(through_block_number, through_block_hash) + || !raw_log_mutations_leave_cached_path_unchanged( + pool, + chain, + raw_log_input_revision, + through_block_number, + through_block_hash, + ) + .await? + { + return Ok(None); + } + Ok(Some((proof, path))) +} diff --git a/crates/adapters/src/ens_v2_resolver.rs b/crates/adapters/src/ens_v2_resolver.rs index cdc42e74..1d7391cb 100644 --- a/crates/adapters/src/ens_v2_resolver.rs +++ b/crates/adapters/src/ens_v2_resolver.rs @@ -70,6 +70,35 @@ impl EnsV2ResolverSyncSummary { ) .await } + + pub async fn sync_for_block_hashes_with_progress( + pool: &PgPool, + chain: &str, + block_hashes: &[String], + progress: &mut dyn StartupAdapterProgress, + ) -> Result { + sync_ens_v2_resolver_with_scope(pool, chain, true, block_hashes, None, None, Some(progress)) + .await + } + + pub async fn sync_for_block_hashes_with_source_scope_and_progress( + pool: &PgPool, + chain: &str, + block_hashes: &[String], + source_scope: &[(String, String, i64, i64)], + progress: &mut dyn StartupAdapterProgress, + ) -> Result { + sync_ens_v2_resolver_with_scope( + pool, + chain, + true, + block_hashes, + Some(source_scope), + None, + Some(progress), + ) + .await + } } pub async fn sync_ens_v2_resolver(pool: &PgPool, chain: &str) -> Result { @@ -101,6 +130,24 @@ pub async fn sync_ens_v2_resolver_through_block( .await } +pub async fn sync_ens_v2_resolver_through_block_with_progress( + pool: &PgPool, + chain: &str, + target_block_number: i64, + progress: &mut dyn StartupAdapterProgress, +) -> Result { + sync_ens_v2_resolver_with_scope( + pool, + chain, + false, + &[], + None, + Some(target_block_number), + Some(progress), + ) + .await +} + async fn sync_ens_v2_resolver_with_scope( pool: &PgPool, chain: &str, @@ -110,7 +157,7 @@ async fn sync_ens_v2_resolver_with_scope( max_block_number: Option, mut progress: Option<&mut dyn StartupAdapterProgress>, ) -> Result { - let mut active_emitters = load_active_emitters(pool, chain).await?; + let mut active_emitters = load_active_emitters(pool, chain, &mut progress).await?; if let Some(source_scope) = source_scope { active_emitters.retain(|emitter| resolver_scope_includes_emitter(source_scope, emitter)); } diff --git a/crates/adapters/src/ens_v2_resolver/queries.rs b/crates/adapters/src/ens_v2_resolver/queries.rs index b49d9d63..2340fecf 100644 --- a/crates/adapters/src/ens_v2_resolver/queries.rs +++ b/crates/adapters/src/ens_v2_resolver/queries.rs @@ -314,13 +314,18 @@ pub(super) async fn load_resolver_raw_logs( Ok(output) } -pub(super) async fn load_active_emitters(pool: &PgPool, chain: &str) -> Result> { +pub(super) async fn load_active_emitters( + pool: &PgPool, + chain: &str, + progress: &mut Option<&mut dyn StartupAdapterProgress>, +) -> Result> { crate::ens_v2_common::load_active_emitters( pool, chain, SOURCE_FAMILY_ENS_V2_RESOLVER_L1, RESOLVER_EDGE_KIND, "ENSv2 resolver", + progress, ) .await } diff --git a/crates/adapters/src/lib.rs b/crates/adapters/src/lib.rs index d6927636..c78aa8eb 100644 --- a/crates/adapters/src/lib.rs +++ b/crates/adapters/src/lib.rs @@ -23,9 +23,11 @@ mod startup_progress; pub use block_derived_normalized_events::{ BlockDerivedNormalizedEventKindSyncSummary, BlockDerivedNormalizedEventSyncSummary, - sync_block_derived_normalized_events, + sync_block_derived_normalized_events, sync_block_derived_normalized_events_with_progress, sync_block_derived_normalized_events_with_scanned_log_count, + sync_block_derived_normalized_events_with_scanned_log_count_and_progress, sync_block_derived_normalized_events_with_stateless_replay_authority, + sync_block_derived_normalized_events_with_stateless_replay_authority_and_progress, }; pub use checkpoint_context::{ ReplayAdapterCheckpointContext, StartupAdapterCheckpointContext, StartupAdapterProgress, @@ -39,8 +41,10 @@ pub use ens_v1_subregistry_discovery::{ EnsV1SubregistryDiscoverySyncSummary, sync_ens_v1_subregistry_discovery, sync_ens_v1_subregistry_discovery_through_block, sync_ens_v1_subregistry_discovery_through_block_with_expected_admission_epoch, + sync_ens_v1_subregistry_discovery_through_block_with_expected_admission_epoch_and_progress, sync_ens_v1_subregistry_discovery_with_replay_checkpoint, sync_ens_v1_subregistry_discovery_with_replay_checkpoint_and_log_limit, + sync_ens_v1_subregistry_discovery_with_replay_checkpoint_and_log_limit_and_progress, sync_ens_v1_subregistry_discovery_with_startup_checkpoint_and_log_limit, sync_ens_v1_subregistry_discovery_with_startup_checkpoint_and_log_limit_and_progress, }; @@ -50,32 +54,39 @@ pub use ens_v1_unwrapped_authority::{ begin_resolver_profile_event_reconciliation, decode_ens_v1_text_record_change, reconcile_resolver_profile_events, sync_ens_v1_unwrapped_authority, sync_ens_v1_unwrapped_authority_with_replay_checkpoint_and_log_limit, + sync_ens_v1_unwrapped_authority_with_replay_checkpoint_and_log_limit_and_progress, sync_ens_v1_unwrapped_authority_with_startup_checkpoint_and_log_limit, sync_ens_v1_unwrapped_authority_with_startup_checkpoint_and_log_limit_and_progress, }; pub use ens_v2_permissions::{ EnsV2PermissionsKindSyncSummary, EnsV2PermissionsSyncSummary, sync_ens_v2_permissions, - sync_ens_v2_permissions_through_block, sync_ens_v2_permissions_with_progress, + sync_ens_v2_permissions_through_block, sync_ens_v2_permissions_through_block_with_progress, + sync_ens_v2_permissions_with_progress, }; pub use ens_v2_registrar::{ EnsV2RegistrarKindSyncSummary, EnsV2RegistrarSyncSummary, sync_ens_v2_registrar, - sync_ens_v2_registrar_through_block, sync_ens_v2_registrar_with_progress, + sync_ens_v2_registrar_through_block, sync_ens_v2_registrar_through_block_with_progress, + sync_ens_v2_registrar_with_progress, }; pub use ens_v2_registry::{ EnsV2MissingCoverage, EnsV2RegistryResourceSurfaceSyncSummary, ens_v2_missing_coverage, ensure_ens_v2_retained_history_proof_through, is_ens_v2_missing_coverage, - record_ens_v2_live_selected_raw_log_coverage, sync_ens_v2_registry_resource_surface, - sync_ens_v2_registry_resource_surface_live_poll, + record_ens_v2_live_selected_raw_log_coverage, + record_ens_v2_live_selected_raw_log_coverage_with_progress, + sync_ens_v2_registry_resource_surface, sync_ens_v2_registry_resource_surface_live_poll, + sync_ens_v2_registry_resource_surface_live_poll_with_progress, sync_ens_v2_registry_resource_surface_through_block, + sync_ens_v2_registry_resource_surface_through_block_with_progress, sync_ens_v2_registry_resource_surface_with_progress, }; pub use ens_v2_resolver::{ EnsV2ResolverKindSyncSummary, EnsV2ResolverSyncSummary, sync_ens_v2_resolver, - sync_ens_v2_resolver_through_block, sync_ens_v2_resolver_with_progress, + sync_ens_v2_resolver_through_block, sync_ens_v2_resolver_through_block_with_progress, + sync_ens_v2_resolver_with_progress, }; pub use manifest_normalized_events::{ ManifestNormalizedEventKindSyncSummary, ManifestNormalizedEventSyncSummary, - sync_manifest_normalized_events, + sync_manifest_normalized_events, sync_manifest_normalized_events_with_progress, }; pub async fn clear_replay_adapter_checkpoints( diff --git a/crates/adapters/src/manifest_normalized_events.rs b/crates/adapters/src/manifest_normalized_events.rs index dd67161d..b3b0beda 100644 --- a/crates/adapters/src/manifest_normalized_events.rs +++ b/crates/adapters/src/manifest_normalized_events.rs @@ -1,8 +1,10 @@ use std::collections::BTreeMap; use anyhow::Result; -use bigname_manifests::load_manifest_drift_inputs; -use bigname_storage::upsert_normalized_events; +use bigname_manifests::{ + ManifestRuntimeProgress, load_manifest_drift_inputs, load_manifest_drift_inputs_with_progress, +}; +use bigname_storage::upsert_normalized_events_with_summary; use sqlx::PgPool; mod builders; @@ -14,15 +16,34 @@ mod utils; use crate::normalized_event_support::count_events_by_kind; use builders::build_normalized_events; -use loading::{load_active_capabilities, load_normalized_event_counts_by_kind}; +use loading::load_active_capabilities; pub use types::{ManifestNormalizedEventKindSyncSummary, ManifestNormalizedEventSyncSummary}; use utils::active_proxy_contracts_by_manifest; +const MANIFEST_EVENT_UPSERT_PROGRESS_ROWS: usize = 1_000; + /// Sync manifest-derived normalized events from stored active manifest state. pub async fn sync_manifest_normalized_events( pool: &PgPool, ) -> Result { - let drift_inputs = load_manifest_drift_inputs(pool).await?; + sync_manifest_normalized_events_inner(pool, None).await +} + +pub async fn sync_manifest_normalized_events_with_progress( + pool: &PgPool, + progress: &mut dyn ManifestRuntimeProgress, +) -> Result { + sync_manifest_normalized_events_inner(pool, Some(progress)).await +} + +async fn sync_manifest_normalized_events_inner( + pool: &PgPool, + mut progress: Option<&mut dyn ManifestRuntimeProgress>, +) -> Result { + let drift_inputs = match progress.as_deref_mut() { + Some(progress) => load_manifest_drift_inputs_with_progress(pool, progress).await?, + None => load_manifest_drift_inputs(pool).await?, + }; if drift_inputs.active_manifests.is_empty() { return Ok(ManifestNormalizedEventSyncSummary { total_synced_count: 0, @@ -32,9 +53,18 @@ pub async fn sync_manifest_normalized_events( } let capabilities = load_active_capabilities(pool).await?; + if let Some(progress) = progress.as_deref_mut() { + progress.record(pool).await?; + } let contracts = active_proxy_contracts_by_manifest(&drift_inputs); - let before_counts = load_normalized_event_counts_by_kind(pool).await?; - let events = build_normalized_events(&drift_inputs, &capabilities, &contracts)?; + let events = build_normalized_events( + pool, + &drift_inputs, + &capabilities, + &contracts, + &mut progress, + ) + .await?; if events.is_empty() { return Ok(ManifestNormalizedEventSyncSummary { @@ -45,29 +75,46 @@ pub async fn sync_manifest_normalized_events( } let synced_by_kind = count_events_by_kind(&events); - upsert_normalized_events(pool, &events).await?; - let after_counts = load_normalized_event_counts_by_kind(pool).await?; - - let mut by_kind = BTreeMap::new(); + let mut by_kind = synced_by_kind + .iter() + .map(|(kind, synced_count)| { + ( + kind.clone(), + ManifestNormalizedEventKindSyncSummary { + synced_count: *synced_count, + inserted_count: 0, + }, + ) + }) + .collect::>(); let mut total_inserted_count = 0; - for (kind, synced_count) in synced_by_kind { - let inserted_count = after_counts - .get(&kind) - .copied() - .unwrap_or(0) - .saturating_sub(before_counts.get(&kind).copied().unwrap_or(0)); + let mut chunk_start = 0usize; + while chunk_start < events.len() { + let kind = events[chunk_start].event_kind.clone(); + let mut chunk_end = chunk_start + 1; + while chunk_end < events.len() + && chunk_end - chunk_start < MANIFEST_EVENT_UPSERT_PROGRESS_ROWS + && events[chunk_end].event_kind == kind + { + chunk_end += 1; + } + let inserted_count = + upsert_normalized_events_with_summary(pool, &events[chunk_start..chunk_end]) + .await? + .inserted_count; total_inserted_count += inserted_count; - by_kind.insert( - kind, - ManifestNormalizedEventKindSyncSummary { - synced_count, - inserted_count, - }, - ); + by_kind + .get_mut(&kind) + .expect("synced event kind must have a summary") + .inserted_count += inserted_count; + if let Some(progress) = progress.as_deref_mut() { + progress.record(pool).await?; + } + chunk_start = chunk_end; } Ok(ManifestNormalizedEventSyncSummary { - total_synced_count: events.len(), + total_synced_count: synced_by_kind.values().sum(), total_inserted_count, by_kind, }) diff --git a/crates/adapters/src/manifest_normalized_events/builders.rs b/crates/adapters/src/manifest_normalized_events/builders.rs index f78b0ba8..37e93875 100644 --- a/crates/adapters/src/manifest_normalized_events/builders.rs +++ b/crates/adapters/src/manifest_normalized_events/builders.rs @@ -3,24 +3,30 @@ use std::collections::HashMap; use anyhow::Result; use bigname_manifests::{ ManifestDeclaredContractDriftInput, ManifestDriftActiveManifest, ManifestDriftInputs, + ManifestRuntimeProgress, }; use bigname_storage::{CanonicalityState, NormalizedEvent}; use serde_json::json; +use sqlx::PgPool; use super::constants::{ DERIVATION_KIND_MANIFEST_SYNC, EVENT_KIND_CAPABILITY_CHANGED, EVENT_KIND_PROXY_IMPLEMENTATION_CHANGED, EVENT_KIND_SOURCE_MANIFEST_UPDATED, }; use super::drift_alerts::{ - build_code_hash_drift_alert_events, build_proxy_implementation_alert_event, + build_code_hash_drift_alert_events_with_progress, build_proxy_implementation_alert_event, }; use super::types::ActiveCapabilityRow; use super::utils::{event_identity, manifest_version_i64}; -pub(super) fn build_normalized_events( +const MANIFEST_EVENT_BUILD_PROGRESS_ROWS: usize = 1_000; + +pub(super) async fn build_normalized_events( + pool: &PgPool, drift_inputs: &ManifestDriftInputs, capabilities: &HashMap>, contracts: &HashMap>, + progress: &mut Option<&mut dyn ManifestRuntimeProgress>, ) -> Result> { let mut events = Vec::new(); @@ -42,9 +48,16 @@ pub(super) fn build_normalized_events( } } - events.extend(build_code_hash_drift_alert_events(drift_inputs)?); - for edge in &drift_inputs.proxy_implementation_edges { + events.extend( + build_code_hash_drift_alert_events_with_progress(pool, drift_inputs, progress).await?, + ); + for (index, edge) in drift_inputs.proxy_implementation_edges.iter().enumerate() { events.push(build_proxy_implementation_alert_event(edge)?); + if (index + 1).is_multiple_of(MANIFEST_EVENT_BUILD_PROGRESS_ROWS) + && let Some(progress) = progress.as_deref_mut() + { + progress.record(pool).await?; + } } Ok(events) diff --git a/crates/adapters/src/manifest_normalized_events/drift_alerts.rs b/crates/adapters/src/manifest_normalized_events/drift_alerts.rs index 3091cc6a..aa675e43 100644 --- a/crates/adapters/src/manifest_normalized_events/drift_alerts.rs +++ b/crates/adapters/src/manifest_normalized_events/drift_alerts.rs @@ -3,10 +3,11 @@ use std::collections::HashMap; use anyhow::Result; use bigname_manifests::{ ManifestCodeHashObservation, ManifestDeclaredContractDriftInput, ManifestDriftInputs, - ManifestProxyImplementationDriftEdge, + ManifestProxyImplementationDriftEdge, ManifestRuntimeProgress, }; use bigname_storage::NormalizedEvent; use serde_json::json; +use sqlx::PgPool; use super::constants::{ DERIVATION_KIND_MANIFEST_ALERT, EVENT_KIND_MANIFEST_CODE_HASH_DRIFT_ALERT, @@ -17,23 +18,29 @@ use super::utils::{ watched_contract_source_name, }; -pub(super) fn build_code_hash_drift_alert_events( +const CODE_HASH_DRIFT_BUILD_PROGRESS_ROWS: usize = 1_000; + +pub(super) async fn build_code_hash_drift_alert_events_with_progress( + pool: &PgPool, drift_inputs: &ManifestDriftInputs, + progress: &mut Option<&mut dyn ManifestRuntimeProgress>, ) -> Result> { - let observations = drift_inputs - .code_hash_observations - .iter() - .map(|observation| { - ( - code_hash_observation_key( - &observation.chain, - observation.contract_instance_id, - &observation.address, - ), - observation, - ) - }) - .collect::>(); + let mut observations = HashMap::new(); + for (index, observation) in drift_inputs.code_hash_observations.iter().enumerate() { + observations.insert( + code_hash_observation_key( + &observation.chain, + observation.contract_instance_id, + &observation.address, + ), + observation, + ); + if (index + 1).is_multiple_of(CODE_HASH_DRIFT_BUILD_PROGRESS_ROWS) + && let Some(progress) = progress.as_deref_mut() + { + progress.record(pool).await?; + } + } let mut events = Vec::new(); for declared_contract in &drift_inputs.declared_contracts { diff --git a/crates/adapters/src/manifest_normalized_events/loading.rs b/crates/adapters/src/manifest_normalized_events/loading.rs index e7cc3bc9..70759df8 100644 --- a/crates/adapters/src/manifest_normalized_events/loading.rs +++ b/crates/adapters/src/manifest_normalized_events/loading.rs @@ -1,5 +1,5 @@ use bigname_storage::sql_row; -use std::collections::{BTreeMap, HashMap}; +use std::collections::HashMap; use anyhow::{Context, Result}; use sqlx::{PgPool, Row}; @@ -43,35 +43,3 @@ pub(super) async fn load_active_capabilities( Ok(grouped) } - -pub(super) async fn load_normalized_event_counts_by_kind( - pool: &PgPool, -) -> Result> { - let rows = sqlx::query( - r#" - SELECT event_kind, COUNT(*)::BIGINT AS event_count - FROM normalized_events - GROUP BY event_kind - ORDER BY event_kind - "#, - ) - .fetch_all(pool) - .await - .context("failed to load normalized-event counts by kind")?; - - let mut counts = BTreeMap::new(); - for row in rows { - let event_kind = row - .try_get::("event_kind") - .context("missing event_kind from normalized-event count row")?; - let event_count = row - .try_get::("event_count") - .context("missing event_count from normalized-event count row")?; - counts.insert( - event_kind, - usize::try_from(event_count).context("normalized-event count does not fit in usize")?, - ); - } - - Ok(counts) -} diff --git a/crates/adapters/src/manifest_normalized_events/tests.rs b/crates/adapters/src/manifest_normalized_events/tests.rs index e950572f..4278f348 100644 --- a/crates/adapters/src/manifest_normalized_events/tests.rs +++ b/crates/adapters/src/manifest_normalized_events/tests.rs @@ -22,11 +22,26 @@ use super::{ DERIVATION_KIND_MANIFEST_SYNC, EVENT_KIND_CAPABILITY_CHANGED, EVENT_KIND_PROXY_IMPLEMENTATION_CHANGED, EVENT_KIND_SOURCE_MANIFEST_UPDATED, }, - sync_manifest_normalized_events, + sync_manifest_normalized_events, sync_manifest_normalized_events_with_progress, }; static NEXT_TEST_ID: AtomicU64 = AtomicU64::new(0); +#[derive(Default)] +struct CountingManifestEventProgress(usize); + +impl bigname_manifests::ManifestRuntimeProgress for CountingManifestEventProgress { + fn record<'a>( + &'a mut self, + _pool: &'a PgPool, + ) -> bigname_manifests::ManifestRuntimeProgressFuture<'a> { + Box::pin(async move { + self.0 += 1; + Ok(()) + }) + } +} + struct TestDatabase { admin_pool: PgPool, pool: PgPool, @@ -362,7 +377,13 @@ async fn sync_manifest_normalized_events_is_idempotent() -> Result<()> { ) .await?; - let first_summary = sync_manifest_normalized_events(database.pool()).await?; + let mut progress = CountingManifestEventProgress::default(); + let first_summary = + sync_manifest_normalized_events_with_progress(database.pool(), &mut progress).await?; + assert!( + progress.0 >= 4, + "manifest observation loading, event construction, and publication must report progress" + ); assert_eq!(first_summary.total_synced_count, 4); assert_eq!(first_summary.total_inserted_count, 4); assert_eq!( diff --git a/crates/adapters/src/normalized_event_support.rs b/crates/adapters/src/normalized_event_support.rs index 46e80529..e636524c 100644 --- a/crates/adapters/src/normalized_event_support.rs +++ b/crates/adapters/src/normalized_event_support.rs @@ -154,6 +154,26 @@ pub(crate) async fn upsert_normalized_events_in_chunks_with_stateless_replay_aut ) -> Result<( NormalizedEventSyncCounts, NormalizedEventReplayAuthoritySummary, +)> { + upsert_normalized_events_in_chunks_with_stateless_replay_authority_counts_and_progress( + pool, + events, + adapter_label, + chunk_size, + None, + ) + .await +} + +pub(crate) async fn upsert_normalized_events_in_chunks_with_stateless_replay_authority_counts_and_progress( + pool: &PgPool, + events: &[NormalizedEvent], + adapter_label: &str, + chunk_size: usize, + mut progress: Option<&mut dyn StartupAdapterProgress>, +) -> Result<( + NormalizedEventSyncCounts, + NormalizedEventReplayAuthoritySummary, )> { if chunk_size == 0 { bail!("normalized event upsert chunk size must be positive"); @@ -163,6 +183,7 @@ pub(crate) async fn upsert_normalized_events_in_chunks_with_stateless_replay_aut for chunk in events.chunks(chunk_size) { authority .add(&upsert_normalized_events_with_stateless_replay_authority(pool, chunk).await?); + record_startup_adapter_progress(pool, &mut progress).await?; } let counts = stateless_replay_counts(events, &authority); info!( diff --git a/crates/adapters/src/startup_progress.rs b/crates/adapters/src/startup_progress.rs index 54f9022a..b2cc95da 100644 --- a/crates/adapters/src/startup_progress.rs +++ b/crates/adapters/src/startup_progress.rs @@ -4,6 +4,25 @@ use sqlx::{PgPool, postgres::PgRow}; use crate::checkpoint_context::{StartupAdapterProgress, record_startup_adapter_progress}; +pub(crate) struct StartupManifestProgress<'a> { + progress: &'a mut dyn StartupAdapterProgress, +} + +impl<'a> StartupManifestProgress<'a> { + pub(crate) fn new(progress: &'a mut dyn StartupAdapterProgress) -> Self { + Self { progress } + } +} + +impl bigname_manifests::ManifestRuntimeProgress for StartupManifestProgress<'_> { + fn record<'a>( + &'a mut self, + pool: &'a PgPool, + ) -> bigname_manifests::ManifestRuntimeProgressFuture<'a> { + self.progress.record(pool) + } +} + pub(crate) const STARTUP_ADAPTER_PROGRESS_PAGE_ROWS: usize = 1_000; pub(crate) const STARTUP_ADAPTER_PROGRESS_PAGE_ROWS_I64: i64 = 1_000; diff --git a/crates/manifests/Cargo.toml b/crates/manifests/Cargo.toml index 96d7bc2f..caebef02 100644 --- a/crates/manifests/Cargo.toml +++ b/crates/manifests/Cargo.toml @@ -11,6 +11,7 @@ alloy-primitives.workspace = true anyhow.workspace = true bigname-domain.workspace = true bigname-storage.workspace = true +futures-util.workspace = true serde.workspace = true serde_json = "1.0.145" sqlx.workspace = true diff --git a/crates/manifests/src/lib.rs b/crates/manifests/src/lib.rs index 49691783..48ff80b5 100644 --- a/crates/manifests/src/lib.rs +++ b/crates/manifests/src/lib.rs @@ -38,7 +38,7 @@ pub use attribution::is_block_derived_preimage_source_family; pub use discovery::*; pub use model::*; pub use repository::load_repository; -pub use sync::sync_repository; +pub use sync::{sync_repository, sync_repository_with_progress}; pub use views::*; pub(crate) use managed_edges::{ diff --git a/crates/manifests/src/lib/discovery.rs b/crates/manifests/src/lib/discovery.rs index 3fab6519..28d617f1 100644 --- a/crates/manifests/src/lib/discovery.rs +++ b/crates/manifests/src/lib/discovery.rs @@ -18,14 +18,15 @@ pub(crate) use admission_epoch::{ bump_discovery_admission_epochs, fence_discovery_admission_epoch_writes, }; pub use admission_epoch::{load_discovery_admission_epoch, load_discovery_admission_epochs}; -pub use loading::load_discovery_admission_state; +pub use loading::{load_discovery_admission_state, load_discovery_admission_state_with_progress}; pub use persistence::persist_discovery_observation; pub use provenance::discovery_observation_evm_event_position; pub use reconciliation::{ DiscoveryObservationPageSource, ExpectedDiscoveryAdmissionEpoch, FullDiscoveryReconciliationOptions, reconcile_discovery_observations, - reconcile_discovery_observations_streamed, reconcile_scoped_discovery_observation_transitions, - reconcile_scoped_discovery_observations, + reconcile_discovery_observations_streamed, + reconcile_discovery_observations_streamed_with_full_options, + reconcile_scoped_discovery_observation_transitions, reconcile_scoped_discovery_observations, }; #[cfg(test)] pub(crate) use reconciliation::{ diff --git a/crates/manifests/src/lib/discovery/loading.rs b/crates/manifests/src/lib/discovery/loading.rs index 9680fd0a..bd1c6104 100644 --- a/crates/manifests/src/lib/discovery/loading.rs +++ b/crates/manifests/src/lib/discovery/loading.rs @@ -10,10 +10,31 @@ use super::types::{ DiscoveryObservation, StoredActiveContract, StoredActiveRoot, StoredDiscoveryRule, }; use crate::{ - PROPAGATED_ROLE_PROVENANCE_FIELD, REACHABLE_FROM_ROOT_ADMISSION, ZERO_ADDRESS, - normalize_address, + ManifestRuntimeProgress, PROPAGATED_ROLE_PROVENANCE_FIELD, REACHABLE_FROM_ROOT_ADMISSION, + ZERO_ADDRESS, normalize_address, }; +#[path = "loading/progress.rs"] +mod progress; + +use super::reconciliation::DiscoveryObservationPageSource; +use progress::{ + AdmissionLoadProgress, AdmissionStateProgress, AdmissionStateProgressFuture, + load_active_discovered_parent_rows_with_progress, + load_known_contract_instance_addresses_with_progress, +}; + +struct PageSourceAdmissionProgress<'a, Source>(&'a Source); + +impl AdmissionStateProgress for PageSourceAdmissionProgress<'_, Source> +where + Source: DiscoveryObservationPageSource + Sync, +{ + fn record(&mut self) -> AdmissionStateProgressFuture<'_> { + Box::pin(async move { self.0.record_progress().await }) + } +} + struct DiscoveryAdmissionScope { active_contract_chains: Vec, active_contract_addresses: Vec, @@ -40,7 +61,27 @@ pub async fn load_discovery_admission_state(pool: &PgPool) -> Result Result { + let mut connection = pool + .acquire() + .await + .context("failed to acquire connection for discovery admission state loading")?; + let mut progress = AdmissionLoadProgress::new(pool, progress); + load_discovery_admission_state_inner( + &mut connection, + None, + None, + KnownAddressLoad::All, + Some(&mut progress), + ) + .await } pub(super) async fn load_discovery_admission_state_with_excluded_source( @@ -52,6 +93,7 @@ pub(super) async fn load_discovery_admission_state_with_excluded_source( excluded_discovery_source, None, KnownAddressLoad::All, + None, ) .await } @@ -71,12 +113,15 @@ pub(super) async fn load_discovery_admission_state_with_excluded_source( pub(super) async fn load_streamed_discovery_admission_state_with_excluded_source( executor: &mut PgConnection, excluded_discovery_source: Option<&str>, + source: &(impl DiscoveryObservationPageSource + Sync), ) -> Result { + let mut progress = PageSourceAdmissionProgress(source); load_discovery_admission_state_inner( executor, excluded_discovery_source, None, KnownAddressLoad::Skip, + Some(&mut progress), ) .await } @@ -113,6 +158,7 @@ pub(super) async fn load_scoped_discovery_admission_state_with_excluded_source( known_address_addresses, }), KnownAddressLoad::Scoped, + None, ) .await } @@ -122,6 +168,7 @@ async fn load_discovery_admission_state_inner( excluded_discovery_source: Option<&str>, scope: Option, known_address_load: KnownAddressLoad, + mut progress: Option<&mut dyn AdmissionStateProgress>, ) -> Result { let ( scoped, @@ -146,6 +193,9 @@ async fn load_discovery_admission_state_inner( .fetch_one(&mut *executor) .await .context("failed to count active manifest versions")? as usize; + if let Some(progress) = progress.as_deref_mut() { + progress.record().await?; + } let active_root_rows = sqlx::query( r#" @@ -162,6 +212,9 @@ async fn load_discovery_admission_state_inner( .fetch_all(&mut *executor) .await .context("failed to load active manifest roots")?; + if let Some(progress) = progress.as_deref_mut() { + progress.record().await?; + } let active_contract_rows = sqlx::query( r#" @@ -178,6 +231,9 @@ async fn load_discovery_admission_state_inner( .fetch_all(&mut *executor) .await .context("failed to load active manifest contracts")?; + if let Some(progress) = progress.as_deref_mut() { + progress.record().await?; + } // Scoped ordered replay may use earlier edges from the same source as // ancestry, including edges inserted earlier in the current transaction. @@ -227,6 +283,15 @@ async fn load_discovery_admission_state_inner( .bind(&active_contract_scope_addresses) .fetch_all(&mut *executor) .await + .context("failed to load active transitive discovery parents")? + } else if let Some(progress) = progress.as_deref_mut() { + load_active_discovered_parent_rows_with_progress( + executor, + excluded_discovery_source, + progress, + ) + .await + .context("failed to load active transitive discovery parents")? } else { sqlx::query( r#" @@ -262,8 +327,8 @@ async fn load_discovery_admission_state_inner( .bind(TRANSITIVE_DISCOVERY_EDGE_KIND) .fetch_all(&mut *executor) .await - } - .context("failed to load active transitive discovery parents")?; + .context("failed to load active transitive discovery parents")? + }; let active_rule_rows = sqlx::query( r#" @@ -276,8 +341,18 @@ async fn load_discovery_admission_state_inner( .fetch_all(&mut *executor) .await .context("failed to load active discovery rules")?; + if let Some(progress) = progress.as_deref_mut() { + progress.record().await?; + } let known_contract_instances_by_address = match known_address_load { + KnownAddressLoad::All if progress.is_some() => { + load_known_contract_instance_addresses_with_progress( + executor, + progress.as_deref_mut().expect("progress checked above"), + ) + .await? + } KnownAddressLoad::All => { // The contract_instance_id tiebreak keeps the per-key winner // deterministic when an address has only deactivated rows with @@ -307,56 +382,78 @@ async fn load_discovery_admission_state_inner( KnownAddressLoad::Skip => HashMap::new(), }; - let active_roots = active_root_rows - .into_iter() - .map(|row| { - Ok(StoredActiveRoot { - manifest_id: row - .try_get("manifest_id") - .context("failed to read active root manifest_id")?, - chain: row - .try_get("chain") - .context("failed to read active root chain")?, - _contract_instance_id: row - .try_get("contract_instance_id") - .context("failed to read active root contract_instance_id")?, - address: normalize_address( - &row.try_get::("address") - .context("failed to read active root address")?, - ), - }) - }) - .collect::>>()?; + let mut active_roots = Vec::with_capacity(active_root_rows.len()); + for row in active_root_rows { + active_roots.push(StoredActiveRoot { + manifest_id: row + .try_get("manifest_id") + .context("failed to read active root manifest_id")?, + chain: row + .try_get("chain") + .context("failed to read active root chain")?, + _contract_instance_id: row + .try_get("contract_instance_id") + .context("failed to read active root contract_instance_id")?, + address: normalize_address( + &row.try_get::("address") + .context("failed to read active root address")?, + ), + }); + if active_roots + .len() + .is_multiple_of(progress::ADMISSION_LOAD_ROWS) + && let Some(progress) = progress.as_deref_mut() + { + progress.record().await?; + } + } let active_root_manifest_ids = active_roots.iter().map(|root| root.manifest_id).collect(); - let active_contracts = active_contract_rows + let mut active_contract_set = HashSet::new(); + let mut examined_contract_rows = 0usize; + for row in active_contract_rows .into_iter() .chain(active_discovered_parent_rows) - .map(|row| { - Ok(StoredActiveContract { - manifest_id: row - .try_get("manifest_id") - .context("failed to read active contract manifest_id")?, - chain: row - .try_get("chain") - .context("failed to read active contract chain")?, - role: row - .try_get("role") - .context("failed to read active contract role")?, - contract_instance_id: row - .try_get("contract_instance_id") - .context("failed to read active contract contract_instance_id")?, - address: normalize_address( - &row.try_get::("address") - .context("failed to read active contract address")?, - ), - }) - }) - .collect::>>()? - .into_iter() - .collect::>(); + { + active_contract_set.insert(StoredActiveContract { + manifest_id: row + .try_get("manifest_id") + .context("failed to read active contract manifest_id")?, + chain: row + .try_get("chain") + .context("failed to read active contract chain")?, + role: row + .try_get("role") + .context("failed to read active contract role")?, + contract_instance_id: row + .try_get("contract_instance_id") + .context("failed to read active contract contract_instance_id")?, + address: normalize_address( + &row.try_get::("address") + .context("failed to read active contract address")?, + ), + }); + examined_contract_rows += 1; + if examined_contract_rows.is_multiple_of(progress::ADMISSION_LOAD_ROWS) + && let Some(progress) = progress.as_deref_mut() + { + progress.record().await?; + } + } + let mut active_contracts = Vec::with_capacity(active_contract_set.len()); + for contract in active_contract_set { + active_contracts.push(contract); + if active_contracts + .len() + .is_multiple_of(progress::ADMISSION_LOAD_ROWS) + && let Some(progress) = progress.as_deref_mut() + { + progress.record().await?; + } + } let mut rules_by_manifest_id: HashMap> = HashMap::new(); + let mut loaded_rule_count = 0usize; for row in active_rule_rows { let manifest_id = row .try_get("manifest_id") @@ -376,6 +473,12 @@ async fn load_discovery_admission_state_inner( .entry(manifest_id) .or_default() .push(rule); + loaded_rule_count += 1; + if loaded_rule_count.is_multiple_of(progress::ADMISSION_LOAD_ROWS) + && let Some(progress) = progress.as_deref_mut() + { + progress.record().await?; + } } let active_rule_count = rules_by_manifest_id.values().map(Vec::len).sum(); diff --git a/crates/manifests/src/lib/discovery/loading/progress.rs b/crates/manifests/src/lib/discovery/loading/progress.rs new file mode 100644 index 00000000..bde394cf --- /dev/null +++ b/crates/manifests/src/lib/discovery/loading/progress.rs @@ -0,0 +1,204 @@ +use std::{collections::HashMap, future::Future, pin::Pin}; + +use anyhow::{Context, Result}; +use sqlx::{PgPool, Row, postgres::PgConnection, postgres::PgRow, types::time::OffsetDateTime}; +use uuid::Uuid; + +use crate::{ + ManifestRuntimeProgress, PROPAGATED_ROLE_PROVENANCE_FIELD, REACHABLE_FROM_ROOT_ADMISSION, + normalize_address, +}; + +use super::super::provenance::TRANSITIVE_DISCOVERY_EDGE_KIND; + +const ADMISSION_LOAD_PAGE_SIZE: i64 = 1_000; +pub(super) const ADMISSION_LOAD_ROWS: usize = ADMISSION_LOAD_PAGE_SIZE as usize; + +pub(super) type AdmissionStateProgressFuture<'a> = + Pin> + Send + 'a>>; + +pub(super) trait AdmissionStateProgress: Send { + fn record(&mut self) -> AdmissionStateProgressFuture<'_>; +} + +pub(super) struct AdmissionLoadProgress<'a> { + pool: &'a PgPool, + callback: &'a mut dyn ManifestRuntimeProgress, +} + +impl<'a> AdmissionLoadProgress<'a> { + pub(super) fn new(pool: &'a PgPool, callback: &'a mut dyn ManifestRuntimeProgress) -> Self { + Self { pool, callback } + } +} + +impl AdmissionStateProgress for AdmissionLoadProgress<'_> { + fn record(&mut self) -> AdmissionStateProgressFuture<'_> { + Box::pin(self.callback.record(self.pool)) + } +} + +pub(super) async fn load_active_discovered_parent_rows_with_progress( + executor: &mut PgConnection, + excluded_discovery_source: Option<&str>, + progress: &mut dyn AdmissionStateProgress, +) -> Result> { + let mut rows = Vec::new(); + let mut after_id = 0i64; + loop { + let edge_ids = sqlx::query_scalar::<_, i64>( + r#" + SELECT discovery_edge_id + FROM discovery_edges + WHERE discovery_edge_id > $1 + ORDER BY discovery_edge_id + LIMIT $2 + "#, + ) + .bind(after_id) + .bind(ADMISSION_LOAD_PAGE_SIZE) + .fetch_all(&mut *executor) + .await + .context("failed to page discovery edges for active transitive parents")?; + let Some(last_id) = edge_ids.last().copied() else { + break; + }; + after_id = last_id; + + let mut page = sqlx::query( + r#" + SELECT + mv.manifest_id, + mv.chain, + de.provenance ->> 'propagated_role' AS role, + de.to_contract_instance_id AS contract_instance_id, + cia.address AS address + FROM discovery_edges de + JOIN manifest_versions mv ON mv.manifest_id = de.source_manifest_id + JOIN contract_instance_addresses cia + ON cia.contract_instance_id = de.to_contract_instance_id + AND cia.deactivated_at IS NULL + WHERE de.discovery_edge_id = ANY($5::BIGINT[]) + AND mv.rollout_status = 'active' + AND de.deactivated_at IS NULL + AND de.edge_kind = $4 + AND de.admission = $1 + AND de.provenance ? $2 + AND ($3::TEXT IS NULL OR de.discovery_source <> $3) + AND NOT EXISTS ( + SELECT 1 + FROM chain_lineage start_block + WHERE start_block.chain_id = de.chain_id + AND start_block.block_hash = de.active_from_block_hash + AND start_block.canonicality_state = 'orphaned'::canonicality_state + ) + ORDER BY de.discovery_edge_id + "#, + ) + .bind(REACHABLE_FROM_ROOT_ADMISSION) + .bind(PROPAGATED_ROLE_PROVENANCE_FIELD) + .bind(excluded_discovery_source) + .bind(TRANSITIVE_DISCOVERY_EDGE_KIND) + .bind(&edge_ids) + .fetch_all(&mut *executor) + .await + .context("failed to load a page of active transitive discovery parents")?; + rows.append(&mut page); + progress.record().await?; + } + Ok(rows) +} + +#[derive(Clone, Copy)] +struct KnownAddressCandidate { + contract_instance_id: Uuid, + active: bool, + admitted_at: OffsetDateTime, +} + +pub(super) async fn load_known_contract_instance_addresses_with_progress( + executor: &mut PgConnection, + progress: &mut dyn AdmissionStateProgress, +) -> Result> { + let mut winners = HashMap::<(String, String), KnownAddressCandidate>::new(); + let mut after_id = 0i64; + loop { + let rows = sqlx::query( + r#" + SELECT + contract_instance_address_id, + chain_id, + address, + contract_instance_id, + deactivated_at IS NULL AS active, + admitted_at + FROM contract_instance_addresses + WHERE contract_instance_address_id > $1 + ORDER BY contract_instance_address_id + LIMIT $2 + "#, + ) + .bind(after_id) + .bind(ADMISSION_LOAD_PAGE_SIZE) + .fetch_all(&mut *executor) + .await + .context("failed to page known contract-instance addresses")?; + let Some(last_row) = rows.last() else { + break; + }; + after_id = last_row + .try_get("contract_instance_address_id") + .context("failed to read known address page cursor")?; + + for row in rows { + let key = ( + row.try_get("chain_id") + .context("failed to read known address chain_id")?, + normalize_address( + &row.try_get::("address") + .context("failed to read known address")?, + ), + ); + let candidate = KnownAddressCandidate { + contract_instance_id: row + .try_get("contract_instance_id") + .context("failed to read known address contract_instance_id")?, + active: row + .try_get("active") + .context("failed to read known address active state")?, + admitted_at: row + .try_get("admitted_at") + .context("failed to read known address admitted_at")?, + }; + match winners.get_mut(&key) { + Some(winner) if candidate_precedes(candidate, *winner) => *winner = candidate, + None => { + winners.insert(key, candidate); + } + Some(_) => {} + } + } + progress.record().await?; + } + + let mut result = HashMap::with_capacity(winners.len()); + for (key, winner) in winners { + result.insert(key, winner.contract_instance_id); + if result.len().is_multiple_of(ADMISSION_LOAD_ROWS) { + progress.record().await?; + } + } + if !result.is_empty() && !result.len().is_multiple_of(ADMISSION_LOAD_ROWS) { + progress.record().await?; + } + Ok(result) +} + +fn candidate_precedes(left: KnownAddressCandidate, right: KnownAddressCandidate) -> bool { + if left.active != right.active { + return left.active; + } + left.admitted_at > right.admitted_at + || (left.admitted_at == right.admitted_at + && left.contract_instance_id < right.contract_instance_id) +} diff --git a/crates/manifests/src/lib/discovery/reconciliation.rs b/crates/manifests/src/lib/discovery/reconciliation.rs index eb77f572..a3647b2e 100644 --- a/crates/manifests/src/lib/discovery/reconciliation.rs +++ b/crates/manifests/src/lib/discovery/reconciliation.rs @@ -24,7 +24,10 @@ pub use full::{ pub use scoped::{ reconcile_scoped_discovery_observation_transitions, reconcile_scoped_discovery_observations, }; -pub use streamed::{DiscoveryObservationPageSource, reconcile_discovery_observations_streamed}; +pub use streamed::{ + DiscoveryObservationPageSource, reconcile_discovery_observations_streamed, + reconcile_discovery_observations_streamed_with_full_options, +}; #[cfg(test)] pub(crate) use streamed::{ StreamedDiscoveryReconciliationOptions, reconcile_discovery_observations_streamed_with_options, diff --git a/crates/manifests/src/lib/discovery/reconciliation/bulk.rs b/crates/manifests/src/lib/discovery/reconciliation/bulk.rs index 0f3672d1..ef7622d9 100644 --- a/crates/manifests/src/lib/discovery/reconciliation/bulk.rs +++ b/crates/manifests/src/lib/discovery/reconciliation/bulk.rs @@ -8,7 +8,9 @@ use super::super::types::{ObservationTerminalState, ReconciledDiscoveryEdgeSpec} #[path = "bulk/historical.rs"] mod historical; -pub(super) use historical::reconcile_historical_discovery_edges; +pub(super) use historical::{ + reconcile_historical_discovery_edges, reconcile_historical_discovery_edges_with_progress, +}; const CONTRACT_INSTANCE_SEED_BATCH_SIZE: usize = 1000; const DISCOVERY_EDGE_INSERT_BATCH_SIZE: usize = 1000; diff --git a/crates/manifests/src/lib/discovery/reconciliation/bulk/historical.rs b/crates/manifests/src/lib/discovery/reconciliation/bulk/historical.rs index 5d0b595e..a6f3aa76 100644 --- a/crates/manifests/src/lib/discovery/reconciliation/bulk/historical.rs +++ b/crates/manifests/src/lib/discovery/reconciliation/bulk/historical.rs @@ -1,7 +1,8 @@ use anyhow::{Context, Result}; -use sqlx::{Postgres, QueryBuilder}; +use sqlx::{PgPool, Postgres, QueryBuilder}; use super::super::super::types::{ObservationTerminalState, ReconciledDiscoveryEdgeSpec}; +use crate::ManifestRuntimeProgress; const HISTORICAL_DISCOVERY_EDGE_BATCH_SIZE: usize = 1000; @@ -13,6 +14,24 @@ pub(in crate::discovery::reconciliation) struct HistoricalDiscoveryEdgeSummary { pub(in crate::discovery::reconciliation) async fn reconcile_historical_discovery_edges( executor: &mut sqlx::postgres::PgConnection, edges: &[(&ReconciledDiscoveryEdgeSpec, ObservationTerminalState)], +) -> Result { + reconcile_historical_discovery_edges_inner(executor, edges, None).await +} + +pub(in crate::discovery::reconciliation) async fn reconcile_historical_discovery_edges_with_progress( + executor: &mut sqlx::postgres::PgConnection, + edges: &[(&ReconciledDiscoveryEdgeSpec, ObservationTerminalState)], + progress_pool: &PgPool, + progress: &mut dyn ManifestRuntimeProgress, +) -> Result { + reconcile_historical_discovery_edges_inner(executor, edges, Some((progress_pool, progress))) + .await +} + +async fn reconcile_historical_discovery_edges_inner( + executor: &mut sqlx::postgres::PgConnection, + edges: &[(&ReconciledDiscoveryEdgeSpec, ObservationTerminalState)], + mut progress: Option<(&PgPool, &mut dyn ManifestRuntimeProgress)>, ) -> Result { let mut inserted_count = 0; let mut updated_count = 0; @@ -316,6 +335,9 @@ pub(in crate::discovery::reconciliation) async fn reconcile_historical_discovery .rows_affected(); updated_count += usize::try_from(predecessor_updates) .context("historical discovery predecessor update count exceeds usize")?; + if let Some((progress_pool, progress)) = progress.as_mut() { + progress.record(progress_pool).await?; + } } Ok(HistoricalDiscoveryEdgeSummary { diff --git a/crates/manifests/src/lib/discovery/reconciliation/streamed.rs b/crates/manifests/src/lib/discovery/reconciliation/streamed.rs index a2f66d0b..4ebc8518 100644 --- a/crates/manifests/src/lib/discovery/reconciliation/streamed.rs +++ b/crates/manifests/src/lib/discovery/reconciliation/streamed.rs @@ -14,6 +14,12 @@ #[path = "streamed/diff.rs"] mod diff; +#[path = "streamed/guard.rs"] +mod guard; +#[path = "streamed/options.rs"] +mod options; +#[path = "streamed/progress.rs"] +mod progress; #[path = "streamed/staging.rs"] mod staging; #[path = "streamed/walk_pages.rs"] @@ -22,7 +28,7 @@ mod walk_pages; use std::collections::{BTreeSet, HashMap, HashSet}; use std::future::Future; -use anyhow::{Context, Result, bail}; +use anyhow::{Context, Result, bail, ensure}; use sqlx::PgPool; use super::super::admission_epoch::{ @@ -32,25 +38,32 @@ use super::super::loading::load_streamed_discovery_admission_state_with_excluded use super::super::types::{DiscoveryObservation, DiscoveryReconciliationSummary}; use super::bulk::{ deactivate_reconciled_discovery_edge, insert_reconciled_discovery_edges, - reconcile_historical_discovery_edges, + reconcile_historical_discovery_edges_with_progress, }; use super::cascade::cascade_deactivation_terminal_states; use super::chronology::edge_starts_after_terminal; -use super::existing::{ - load_active_reconciled_discovery_edge_chains, load_active_reconciled_discovery_edge_count, -}; use super::full::protects_non_orphaned_newer_edge; use super::support::{lock_discovery_reconciliation, observation_terminal_states}; use super::{compare_reconciled_discovery_edge_specs, safe_deactivation_terminal}; -use crate::reconcile_active_contract_instance_addresses; +use crate::{ + FullDiscoveryReconciliationOptions, ManifestRuntimeProgress, + managed_edges::reconcile_active_contract_instance_addresses_with_mutations_and_progress, +}; use self::diff::{ collect_same_assignment_retained_edges, collect_streamed_historical_edges, - count_streamed_deactivation_candidates, load_streamed_deactivation_candidates, - load_streamed_insert_candidate_page, materialize_streamed_insert_candidates, + create_streamed_insert_candidate_table, finish_streamed_insert_candidate_table, + load_streamed_deactivation_source_page, load_streamed_insert_candidate_page, + materialize_streamed_insert_candidate_page, }; +use self::guard::{ + default_max_deactivation_candidates, default_max_deactivations, + max_deactivations_override_from_env, +}; +pub(crate) use self::options::StreamedDiscoveryReconciliationOptions; +use self::progress::{PageSourceManifestProgress, load_active_edge_summary_with_progress}; use self::staging::{ - count_temp_rows, create_streamed_reconcile_temp_tables, load_streamed_observations_for_keys, + create_streamed_reconcile_temp_tables, load_streamed_observations_for_keys, stage_streamed_observations, }; use self::walk_pages::run_streamed_admission_walk; @@ -58,13 +71,7 @@ use self::walk_pages::run_streamed_admission_walk; /// Environment override for the streamed reconcile's deactivation guard. /// Holds the maximum number of deactivation candidates permitted before the /// reconcile aborts (replacing the default `max(10_000, 1%)` bound). -pub const DISCOVERY_FULL_RECONCILE_MAX_DEACTIVATIONS_ENV: &str = - "BIGNAME_INDEXER_DISCOVERY_FULL_RECONCILE_MAX_DEACTIVATIONS"; - -const DEACTIVATION_GUARD_FLOOR: usize = 10_000; -const CANDIDATE_LOAD_CAP_FLOOR: usize = 100_000; -const DEFAULT_OBSERVATION_PAGE_LIMIT: i64 = 10_000; -const DEFAULT_MUTATION_BATCH_SIZE: usize = 1_000; +pub use guard::DISCOVERY_FULL_RECONCILE_MAX_DEACTIVATIONS_ENV; /// Ascending-key page source over a complete, latest-per-key retained /// observation set for one discovery source. @@ -91,33 +98,6 @@ pub trait DiscoveryObservationPageSource { } } -#[derive(Clone, Copy, Debug)] -pub(crate) struct StreamedDiscoveryReconciliationOptions { - /// Replaces the default `max(10_000, 1% of active edges)` precise - /// deactivation guard bound when set, and raises the coarse candidate - /// load cap to at least the same value so an operator override stays - /// effective end to end. - pub(crate) max_deactivations_override: Option, - /// Replaces the default `max(100_000, 10% of active edges)` coarse cap - /// on how many deactivation candidates may be materialized in memory. - /// Test hook; production overrides go through the env-driven precise - /// bound, which raises this cap alongside it. - pub(crate) coarse_deactivation_cap_override: Option, - pub(crate) observation_page_limit: i64, - pub(crate) mutation_batch_size: usize, -} - -impl Default for StreamedDiscoveryReconciliationOptions { - fn default() -> Self { - Self { - max_deactivations_override: None, - coarse_deactivation_cap_override: None, - observation_page_limit: DEFAULT_OBSERVATION_PAGE_LIMIT, - mutation_batch_size: DEFAULT_MUTATION_BATCH_SIZE, - } - } -} - /// Reconcile a complete latest-per-key retained observation set for one /// discovery source with memory bounded by pages instead of the observation /// count, producing the same `discovery_edges` outcome as @@ -140,22 +120,70 @@ impl Default for StreamedDiscoveryReconciliationOptions { pub async fn reconcile_discovery_observations_streamed( pool: &PgPool, discovery_source: &str, - source: &impl DiscoveryObservationPageSource, + source: &(impl DiscoveryObservationPageSource + Sync), ) -> Result { let options = StreamedDiscoveryReconciliationOptions { max_deactivations_override: max_deactivations_override_from_env()?, ..StreamedDiscoveryReconciliationOptions::default() }; - reconcile_discovery_observations_streamed_with_options(pool, discovery_source, source, options) - .await + reconcile_discovery_observations_streamed_inner( + pool, + discovery_source, + source, + options, + FullDiscoveryReconciliationOptions::default(), + ) + .await } +pub async fn reconcile_discovery_observations_streamed_with_full_options( + pool: &PgPool, + discovery_source: &str, + source: &(impl DiscoveryObservationPageSource + Sync), + full_options: FullDiscoveryReconciliationOptions<'_>, +) -> Result { + let options = StreamedDiscoveryReconciliationOptions { + max_deactivations_override: max_deactivations_override_from_env()?, + ..StreamedDiscoveryReconciliationOptions::default() + }; + reconcile_discovery_observations_streamed_inner( + pool, + discovery_source, + source, + options, + full_options, + ) + .await +} + +#[cfg(test)] pub(crate) async fn reconcile_discovery_observations_streamed_with_options( pool: &PgPool, discovery_source: &str, - source: &impl DiscoveryObservationPageSource, + source: &(impl DiscoveryObservationPageSource + Sync), + options: StreamedDiscoveryReconciliationOptions, +) -> Result { + reconcile_discovery_observations_streamed_inner( + pool, + discovery_source, + source, + options, + FullDiscoveryReconciliationOptions::default(), + ) + .await +} + +async fn reconcile_discovery_observations_streamed_inner( + pool: &PgPool, + discovery_source: &str, + source: &(impl DiscoveryObservationPageSource + Sync), options: StreamedDiscoveryReconciliationOptions, + full_options: FullDiscoveryReconciliationOptions<'_>, ) -> Result { + let through_block_number = full_options.through_block_number; + let expected_admission_epoch = full_options + .expected_admission_epoch + .map(|expected| (expected.chain, expected.epoch)); let mut transaction = pool .begin() .await @@ -180,34 +208,128 @@ pub(crate) async fn reconcile_discovery_observations_streamed_with_options( let staged = stage_streamed_observations(transaction.as_mut(), source, &options).await?; + let (active_edge_count_before, active_edge_chains) = load_active_edge_summary_with_progress( + transaction.as_mut(), + discovery_source, + options.observation_page_limit, + source, + ) + .await?; let mut candidate_chains = staged.observation_chains.clone(); - candidate_chains.extend( - load_active_reconciled_discovery_edge_chains(transaction.as_mut(), discovery_source) - .await?, - ); + candidate_chains.extend(active_edge_chains); + if let Some((chain, _)) = expected_admission_epoch { + candidate_chains.insert(chain.to_owned()); + } fence_discovery_admission_epoch_writes(transaction.as_mut(), &candidate_chains).await?; + if let Some((chain, expected_epoch)) = expected_admission_epoch { + ensure!( + candidate_chains.iter().all(|candidate| candidate == chain), + "discovery source {discovery_source} expected epoch fence for {chain} cannot reconcile observations from another chain" + ); + let current_epoch = sqlx::query_scalar::<_, i64>( + "SELECT epoch FROM discovery_admission_epochs WHERE chain_id = $1", + ) + .bind(chain) + .fetch_optional(transaction.as_mut()) + .await + .with_context(|| { + format!("failed to read discovery admission epoch for {chain} under the writer fence") + })? + .unwrap_or(0); + ensure!( + current_epoch == expected_epoch, + "discovery admission epoch changed before full-source reconciliation for {chain}: expected {expected_epoch}, observed {current_epoch}" + ); + } let admission_state = load_streamed_discovery_admission_state_with_excluded_source( transaction.as_mut(), Some(discovery_source), + source, ) .await?; source.record_progress().await?; - run_streamed_admission_walk(transaction.as_mut(), &admission_state, &options, source).await?; + let admitted_edge_count = + run_streamed_admission_walk(transaction.as_mut(), &admission_state, &options, source) + .await?; // Everything below diffs against the pre-mutation edge snapshot, exactly // like the in-memory reconcile computes its whole plan from one // `load_active_reconciled_discovery_edges` read before mutating. - materialize_streamed_insert_candidates(transaction.as_mut(), discovery_source).await?; - source.record_progress().await?; - let insert_candidate_count = - count_temp_rows(transaction.as_mut(), "reconcile_insert_candidates").await?; - let deactivation_candidate_count = - count_streamed_deactivation_candidates(transaction.as_mut(), discovery_source).await?; - let active_edge_count_before = - load_active_reconciled_discovery_edge_count(transaction.as_mut(), discovery_source).await?; - let desired_edge_count = - count_temp_rows(transaction.as_mut(), "reconcile_desired_edges").await?; + create_streamed_insert_candidate_table(transaction.as_mut()).await?; + let diff_page_limit = i64::try_from(options.mutation_batch_size.max(1)) + .context("streamed reconcile mutation batch size overflowed i64")?; + let mut insert_candidate_count = 0usize; + let mut desired_edge_count = 0usize; + let mut after_desired_row_id = 0i64; + loop { + let (last_row_id, source_rows, inserted_rows) = materialize_streamed_insert_candidate_page( + transaction.as_mut(), + discovery_source, + after_desired_row_id, + diff_page_limit, + ) + .await?; + let Some(last_row_id) = last_row_id else { + break; + }; + after_desired_row_id = last_row_id; + desired_edge_count += source_rows; + insert_candidate_count += inserted_rows; + source.record_progress().await?; + } + finish_streamed_insert_candidate_table(transaction.as_mut()).await?; + + let max_deactivation_candidates = + options.coarse_deactivation_cap_override.unwrap_or_else(|| { + default_max_deactivation_candidates(active_edge_count_before) + .max(options.max_deactivations_override.unwrap_or(0)) + }); + let mut deactivation_candidates = Vec::new(); + let mut after_edge_id = 0i64; + #[cfg(test)] + let mut deactivation_source_page_count = 0usize; + loop { + let page = load_streamed_deactivation_source_page( + transaction.as_mut(), + discovery_source, + after_edge_id, + diff_page_limit, + ) + .await?; + let Some(last_edge_id) = page.last_edge_id else { + break; + }; + after_edge_id = last_edge_id; + let candidate_count_after_page = deactivation_candidates + .len() + .checked_add(page.candidates.len()) + .context("streamed deactivation candidate count overflowed usize")?; + if candidate_count_after_page > max_deactivation_candidates { + bail!( + "streamed discovery reconciliation for {discovery_source} loaded at least \ + {candidate_count_after_page} deactivation candidates against \ + {active_edge_count_before} active edges, over the \ + {max_deactivation_candidates} candidate load cap; refusing to materialize \ + more of the diff — this indicates spec drift, raise \ + {DISCOVERY_FULL_RECONCILE_MAX_DEACTIVATIONS_ENV} only after confirming the \ + diff is intended" + ); + } + deactivation_candidates.extend(page.candidates); + source.record_progress().await?; + #[cfg(test)] + { + deactivation_source_page_count += 1; + if options + .fail_after_deactivation_source_pages + .is_some_and(|limit| deactivation_source_page_count > limit) + { + bail!("injected deactivation source-page scan limit exceeded"); + } + } + } + let deactivation_candidate_count = deactivation_candidates.len(); tracing::info!( discovery_source, staged_observation_count = staged.staged_observation_count, @@ -217,32 +339,13 @@ pub(crate) async fn reconcile_discovery_observations_streamed_with_options( insert_candidate_count, "streamed discovery reconciliation diff computed" ); - // Coarse memory cap: candidates are materialized in memory below, so a - // spec-drift flood must abort on the SQL count before anything is - // loaded. Chronology-retained candidates are excluded from deactivation - // only after loading, so the precise fail-closed threshold is applied - // separately, right before mutating. - let max_deactivation_candidates = - options.coarse_deactivation_cap_override.unwrap_or_else(|| { - default_max_deactivation_candidates(active_edge_count_before) - .max(options.max_deactivations_override.unwrap_or(0)) - }); - if deactivation_candidate_count > max_deactivation_candidates { - bail!( - "streamed discovery reconciliation for {discovery_source} computed \ - {deactivation_candidate_count} deactivation candidates against \ - {active_edge_count_before} active edges, over the {max_deactivation_candidates} \ - candidate load cap; refusing to materialize the diff — this indicates spec drift, \ - raise {DISCOVERY_FULL_RECONCILE_MAX_DEACTIVATIONS_ENV} only after confirming the \ - diff is intended" - ); - } - - let deactivation_candidates = - load_streamed_deactivation_candidates(transaction.as_mut(), discovery_source).await?; - let candidate_observations = - load_streamed_observations_for_keys(transaction.as_mut(), &deactivation_candidates).await?; - source.record_progress().await?; + let candidate_observations = load_streamed_observations_for_keys( + transaction.as_mut(), + &deactivation_candidates, + options.mutation_batch_size, + source, + ) + .await?; let direct_terminal_states_by_key = observation_terminal_states(&candidate_observations)?; let observations_by_key = candidate_observations .iter() @@ -278,9 +381,9 @@ pub(crate) async fn reconcile_discovery_observations_streamed_with_options( &deactivation_candidates, options.mutation_batch_size, &mut retained_newer_edge_ids, + source, ) .await?; - source.record_progress().await?; // Chronology rule 2: a desired edge with a newer non-orphaned successor // for the same assignment start is materialized as a closed historical // epoch and the successor is retained. @@ -290,6 +393,7 @@ pub(crate) async fn reconcile_discovery_observations_streamed_with_options( i64::try_from(options.mutation_batch_size) .context("streamed reconcile mutation batch size overflowed i64")?, &mut retained_newer_edge_ids, + source, ) .await?; source.record_progress().await?; @@ -324,7 +428,7 @@ pub(crate) async fn reconcile_discovery_observations_streamed_with_options( && !protects_non_orphaned_newer_edge( candidate, deactivation_terminal_states_by_key.get(&candidate.spec.observation_key), - None, + through_block_number, ) }) .count(); @@ -369,7 +473,7 @@ pub(crate) async fn reconcile_discovery_observations_streamed_with_options( } let terminal_state = deactivation_terminal_states_by_key.get(&candidate.spec.observation_key); - if protects_non_orphaned_newer_edge(candidate, terminal_state, None) { + if protects_non_orphaned_newer_edge(candidate, terminal_state, through_block_number) { continue; } let terminal_state = terminal_state @@ -408,6 +512,7 @@ pub(crate) async fn reconcile_discovery_observations_streamed_with_options( .map(|(_, spec)| spec) .collect::>(); if batch.is_empty() { + source.record_progress().await?; continue; } let edge_insert = insert_reconciled_discovery_edges(transaction.as_mut(), &batch).await?; @@ -426,9 +531,14 @@ pub(crate) async fn reconcile_discovery_observations_streamed_with_options( .iter() .map(|(spec, terminal_state)| (spec, terminal_state.clone())) .collect::>(); - let historical_edge_reconciliation = - reconcile_historical_discovery_edges(transaction.as_mut(), &historical_edge_refs).await?; - source.record_progress().await?; + let mut historical_progress = PageSourceManifestProgress::new(source); + let historical_edge_reconciliation = reconcile_historical_discovery_edges_with_progress( + transaction.as_mut(), + &historical_edge_refs, + pool, + &mut historical_progress, + ) + .await?; inserted_edge_count += historical_edge_reconciliation.inserted_count; if historical_edge_reconciliation.inserted_count > 0 || historical_edge_reconciliation.updated_count > 0 @@ -440,14 +550,22 @@ pub(crate) async fn reconcile_discovery_observations_streamed_with_options( || historical_edge_reconciliation.updated_count > 0 || deactivated_edge_count > 0 { - reconcile_active_contract_instance_addresses(transaction.as_mut()).await?; - source.record_progress().await?; + let mut callback = PageSourceManifestProgress::new(source); + let mut progress: Option<&mut dyn ManifestRuntimeProgress> = Some(&mut callback); + reconcile_active_contract_instance_addresses_with_mutations_and_progress( + transaction.as_mut(), + pool, + &mut progress, + ) + .await?; } - let active_edge_count = - load_active_reconciled_discovery_edge_count(transaction.as_mut(), discovery_source).await?; - let admitted_edge_count = count_temp_rows(transaction.as_mut(), "reconcile_admitted_edges") - .await - .context("failed to count streamed admitted discovery edges")?; + let (active_edge_count, _) = load_active_edge_summary_with_progress( + transaction.as_mut(), + discovery_source, + options.observation_page_limit, + source, + ) + .await?; let admission_epoch_bump_count = mutated_chains.len(); bump_discovery_admission_epochs(transaction.as_mut(), &mutated_chains).await?; source.record_progress().await?; @@ -468,83 +586,3 @@ pub(crate) async fn reconcile_discovery_observations_streamed_with_options( admitted_edges: Vec::new(), }) } - -/// Default precise deactivation guard bound: a verified full-closure -/// finalize must be a near-no-op, so anything beyond `max(10_000, 1%)` of -/// the source's active edges aborts instead of silently rewriting the -/// source. -fn default_max_deactivations(active_edge_count: usize) -> usize { - DEACTIVATION_GUARD_FLOOR.max(active_edge_count / 100) -} - -/// Default coarse cap on how many deactivation candidates may be loaded -/// into memory for chronology and cascade resolution: `max(100_000, 10%)` -/// of the source's active edges bounds the only diff-sized allocation the -/// streamed reconcile makes. -fn default_max_deactivation_candidates(active_edge_count: usize) -> usize { - CANDIDATE_LOAD_CAP_FLOOR.max(active_edge_count / 10) -} - -/// Thin, untested-by-design environment read; every sibling test that -/// exercises guard behavior goes through the explicit options override, so -/// nothing in this binary mutates the process-global variable. The parsing -/// itself is the pure function below. -fn max_deactivations_override_from_env() -> Result> { - match std::env::var(DISCOVERY_FULL_RECONCILE_MAX_DEACTIVATIONS_ENV) { - Ok(value) => parse_max_deactivations_override(Some(&value)), - Err(std::env::VarError::NotPresent) => parse_max_deactivations_override(None), - Err(error) => Err(error).context(format!( - "failed to read {DISCOVERY_FULL_RECONCILE_MAX_DEACTIVATIONS_ENV}" - )), - } -} - -fn parse_max_deactivations_override(value: Option<&str>) -> Result> { - match value { - None => Ok(None), - Some(value) => value - .trim() - .parse::() - .map(Some) - .with_context(|| format!( - "failed to parse {DISCOVERY_FULL_RECONCILE_MAX_DEACTIVATIONS_ENV} as a deactivation count: {value:?}" - )), - } -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn default_deactivation_guard_uses_floor_and_percentage() { - assert_eq!(default_max_deactivations(0), 10_000); - assert_eq!(default_max_deactivations(500_000), 10_000); - assert_eq!(default_max_deactivations(7_620_084), 76_200); - } - - #[test] - fn default_candidate_load_cap_uses_floor_and_percentage() { - assert_eq!(default_max_deactivation_candidates(0), 100_000); - assert_eq!(default_max_deactivation_candidates(500_000), 100_000); - assert_eq!(default_max_deactivation_candidates(7_620_084), 762_008); - } - - #[test] - fn deactivation_guard_override_parses_or_rejects() { - assert_eq!( - parse_max_deactivations_override(None).expect("no value must read as no override"), - None - ); - assert_eq!( - parse_max_deactivations_override(Some("123456")).expect("numeric override must parse"), - Some(123_456) - ); - assert_eq!( - parse_max_deactivations_override(Some(" 42 ")) - .expect("surrounding whitespace is trimmed"), - Some(42) - ); - assert!(parse_max_deactivations_override(Some("not-a-count")).is_err()); - } -} diff --git a/crates/manifests/src/lib/discovery/reconciliation/streamed/diff.rs b/crates/manifests/src/lib/discovery/reconciliation/streamed/diff.rs index 86dfa498..51fbb8d4 100644 --- a/crates/manifests/src/lib/discovery/reconciliation/streamed/diff.rs +++ b/crates/manifests/src/lib/discovery/reconciliation/streamed/diff.rs @@ -3,7 +3,7 @@ //! reconcile, including the SQL translations of the stored-spec equality //! and chronology comparisons. -use std::collections::HashSet; +use std::collections::{HashMap, HashSet}; use anyhow::{Context, Result, bail}; use sqlx::{Row, postgres::PgConnection}; @@ -12,8 +12,8 @@ use super::super::super::types::{ EvmEventPosition, ExistingReconciledDiscoveryEdge, ObservationTerminalState, ReconciledDiscoveryEdgeSpec, }; -use super::super::chronology::{compare_edge_starts, edge_starts_after_spec}; use super::super::existing::edge_from_row; +use super::DiscoveryObservationPageSource; use super::staging::analyze_temp_table; #[path = "diff/retention.rs"] @@ -119,77 +119,69 @@ const STREAMED_STARTS_NO_LATER_SQL: &str = r#" ) "#; -/// `starts_after(existing = de, desired)` in SQL: both block numbers must be -/// present; equal blocks only compare when both sides carry a full event -/// position. -const STREAMED_STARTS_AFTER_SQL: &str = r#" - ( - de.active_from_block_number IS NOT NULL - AND desired.active_from_block_number IS NOT NULL - AND ( - de.active_from_block_number > desired.active_from_block_number - OR ( - de.active_from_block_number = desired.active_from_block_number - AND (de.provenance ->> 'transaction_index') IS NOT NULL - AND (de.provenance ->> 'log_index') IS NOT NULL - AND desired.active_from_transaction_index IS NOT NULL - AND desired.active_from_log_index IS NOT NULL - AND ( - (de.provenance ->> 'transaction_index')::BIGINT, - (de.provenance ->> 'log_index')::BIGINT - ) > ( - desired.active_from_transaction_index, - desired.active_from_log_index - ) - ) - ) - ) -"#; +pub(super) struct StreamedDeactivationSourcePage { + pub(super) last_edge_id: Option, + pub(super) candidates: Vec, +} -pub(super) async fn count_streamed_deactivation_candidates( +pub(super) async fn load_streamed_deactivation_source_page( executor: &mut PgConnection, discovery_source: &str, -) -> Result { - let count = sqlx::query_scalar::<_, i64>(&format!( + after_edge_id: i64, + limit: i64, +) -> Result { + let edge_ids = sqlx::query_scalar::<_, i64>( r#" - SELECT COUNT(*)::BIGINT - {STREAMED_ACTIVE_EDGE_FROM_SQL} - AND NOT EXISTS ( - SELECT 1 - FROM pg_temp.reconcile_desired_edges desired - WHERE {STREAMED_EXACT_SPEC_MATCH_SQL} - ) - "# - )) - .bind(discovery_source) - .fetch_one(executor) + SELECT discovery_edge_id + FROM discovery_edges + WHERE discovery_edge_id > $1 + ORDER BY discovery_edge_id + LIMIT $2 + "#, + ) + .bind(after_edge_id) + .bind(limit) + .fetch_all(&mut *executor) .await - .context("failed to count streamed discovery-edge deactivation candidates")?; - usize::try_from(count).context("streamed deactivation candidate count overflowed usize") -} - -pub(super) async fn load_streamed_deactivation_candidates( - executor: &mut PgConnection, - discovery_source: &str, -) -> Result> { + .context("failed to page discovery-edge identities for streamed deactivation diff")?; + let Some(last_edge_id) = edge_ids.last().copied() else { + return Ok(StreamedDeactivationSourcePage { + last_edge_id: None, + candidates: Vec::new(), + }); + }; let rows = sqlx::query(&format!( r#" - SELECT {STREAMED_EXISTING_EDGE_COLUMNS_QUALIFIED} + SELECT {STREAMED_EXISTING_EDGE_COLUMNS_QUALIFIED}, + NOT EXISTS ( + SELECT 1 + FROM pg_temp.reconcile_desired_edges desired + WHERE {STREAMED_EXACT_SPEC_MATCH_SQL} + ) AS deactivation_candidate {STREAMED_ACTIVE_EDGE_FROM_SQL} - AND NOT EXISTS ( - SELECT 1 - FROM pg_temp.reconcile_desired_edges desired - WHERE {STREAMED_EXACT_SPEC_MATCH_SQL} - ) + AND de.discovery_edge_id = ANY($2::BIGINT[]) ORDER BY de.discovery_edge_id "# )) .bind(discovery_source) - .fetch_all(executor) + .bind(&edge_ids) + .fetch_all(&mut *executor) .await - .context("failed to load streamed discovery-edge deactivation candidates")?; + .context("failed to load a streamed discovery-edge deactivation source page")?; - rows.into_iter().map(edge_from_row).collect() + let mut candidates = Vec::new(); + for row in rows { + if row + .try_get::("deactivation_candidate") + .context("failed to read streamed deactivation-candidate flag")? + { + candidates.push(edge_from_row(row)?); + } + } + Ok(StreamedDeactivationSourcePage { + last_edge_id: Some(last_edge_id), + candidates, + }) } fn desired_edge_spec_from_row( @@ -275,9 +267,8 @@ const STREAMED_DESIRED_EDGE_COLUMNS: &str = r#" /// materializing the same assignment at a no-later start (`current_new_ /// edges` in the in-memory chronology, before its historical exclusion, /// which the caller applies from `collect_streamed_historical_edges`). -pub(super) async fn materialize_streamed_insert_candidates( +pub(super) async fn create_streamed_insert_candidate_table( executor: &mut PgConnection, - discovery_source: &str, ) -> Result<()> { sqlx::query( r#" @@ -290,12 +281,39 @@ pub(super) async fn materialize_streamed_insert_candidates( .await .context("failed to create the streamed reconcile insert-candidate temp table")?; - sqlx::query(&format!( + Ok(()) +} + +pub(super) async fn materialize_streamed_insert_candidate_page( + executor: &mut PgConnection, + discovery_source: &str, + after_row_id: i64, + limit: i64, +) -> Result<(Option, usize, usize)> { + let desired_row_ids = sqlx::query_scalar::<_, i64>( + r#" + SELECT desired_row_id + FROM pg_temp.reconcile_desired_edges + WHERE desired_row_id > $1 + ORDER BY desired_row_id + LIMIT $2 + "#, + ) + .bind(after_row_id) + .bind(limit) + .fetch_all(&mut *executor) + .await + .context("failed to page desired discovery edges for insert-candidate diff")?; + let Some(last_row_id) = desired_row_ids.last().copied() else { + return Ok((None, 0, 0)); + }; + let inserted = sqlx::query(&format!( r#" INSERT INTO pg_temp.reconcile_insert_candidates (desired_row_id) SELECT desired.desired_row_id FROM pg_temp.reconcile_desired_edges desired - WHERE NOT EXISTS ( + WHERE desired.desired_row_id = ANY($2::BIGINT[]) + AND NOT EXISTS ( SELECT 1 FROM discovery_edges de JOIN contract_instance_addresses cia @@ -327,12 +345,19 @@ pub(super) async fn materialize_streamed_insert_candidates( "# )) .bind(discovery_source) + .bind(&desired_row_ids) .execute(&mut *executor) .await - .context("failed to materialize streamed discovery-edge insert candidates")?; - analyze_temp_table(&mut *executor, "reconcile_insert_candidates").await?; + .context("failed to materialize a streamed discovery-edge insert-candidate page")? + .rows_affected() as usize; - Ok(()) + Ok((Some(last_row_id), desired_row_ids.len(), inserted)) +} + +pub(super) async fn finish_streamed_insert_candidate_table( + executor: &mut PgConnection, +) -> Result<()> { + analyze_temp_table(executor, "reconcile_insert_candidates").await } pub(super) async fn load_streamed_insert_candidate_page( @@ -371,12 +396,16 @@ pub(super) async fn load_streamed_insert_candidate_page( /// still held in memory: it is the input to the historical materialization /// batch and is bounded by the successor diff, which for a full-closure /// finalize means retained edges newer than the replay target. -pub(super) async fn collect_streamed_historical_edges( +pub(super) async fn collect_streamed_historical_edges( executor: &mut PgConnection, discovery_source: &str, page_limit: i64, retained_newer_edge_ids: &mut HashSet, -) -> Result> { + source: &Source, +) -> Result> +where + Source: DiscoveryObservationPageSource + Sync, +{ let mut historical_edges = Vec::new(); let mut after_row_id = 0i64; loop { @@ -386,21 +415,6 @@ pub(super) async fn collect_streamed_historical_edges( FROM pg_temp.reconcile_desired_edges desired WHERE desired.desired_row_id > $2 AND desired.active_from_block_number IS NOT NULL - AND EXISTS ( - SELECT 1 - FROM discovery_edges de - JOIN contract_instance_addresses cia - ON cia.contract_instance_id = de.to_contract_instance_id - AND cia.deactivated_at IS NULL - WHERE de.discovery_source = $1 - AND de.deactivated_at IS NULL - AND de.provenance ->> 'observation_key' = desired.observation_key - AND de.chain_id = desired.chain_id - AND de.edge_kind = desired.edge_kind - AND de.from_contract_instance_id = desired.from_contract_instance_id - AND NOT {STREAMED_EDGE_IS_ORPHANED_SQL} - AND {STREAMED_STARTS_AFTER_SQL} - ) ORDER BY desired.desired_row_id LIMIT $3 "# @@ -423,47 +437,119 @@ pub(super) async fn collect_streamed_historical_edges( .map(|(desired_row_id, _)| *desired_row_id) .expect("a non-empty historical page has a last row"); - for (desired_row_id, desired) in page { - let rows = sqlx::query(&format!( - r#" - SELECT {STREAMED_EXISTING_EDGE_COLUMNS_QUALIFIED} + source.record_progress().await?; + let desired_row_ids = page + .iter() + .map(|(desired_row_id, _)| *desired_row_id) + .collect::>(); + type HistoricalSuccessorRow = ( + i64, + i64, + String, + Option, + Option, + Option, + Option, + ); + let successor_rows = sqlx::query_as::<_, HistoricalSuccessorRow>(&format!( + r#" + SELECT + desired.desired_row_id, + successor.discovery_edge_id, + successor.chain_id, + successor.active_from_block_number, + successor.active_from_block_hash, + successor.transaction_index, + successor.log_index + FROM pg_temp.reconcile_desired_edges desired + JOIN LATERAL ( + SELECT + de.discovery_edge_id, + de.chain_id, + de.active_from_block_number, + de.active_from_block_hash, + (de.provenance ->> 'transaction_index')::BIGINT AS transaction_index, + (de.provenance ->> 'log_index')::BIGINT AS log_index {STREAMED_ACTIVE_EDGE_FROM_SQL} - AND de.provenance ->> 'observation_key' = $2 - AND de.chain_id = $3 - AND de.edge_kind = $4 - AND de.from_contract_instance_id = $5 - "# - )) - .bind(discovery_source) - .bind(&desired.observation_key) - .bind(&desired.chain) - .bind(&desired.edge_kind) - .bind(desired.from_contract_instance_id) - .fetch_all(&mut *executor) - .await - .context("failed to load successor candidates for a historical desired edge")?; - let successor_candidates = rows - .into_iter() - .map(edge_from_row) - .collect::>>()?; - let Some(successor) = successor_candidates - .iter() - .filter(|edge| { - !edge.active_from_block_is_orphaned && edge_starts_after_spec(edge, &desired) - }) - .min_by(compare_edge_starts) + AND de.provenance ->> 'observation_key' = desired.observation_key + AND de.chain_id = desired.chain_id + AND de.edge_kind = desired.edge_kind + AND de.from_contract_instance_id = desired.from_contract_instance_id + AND NOT {STREAMED_EDGE_IS_ORPHANED_SQL} + AND ( + de.active_from_block_number > desired.active_from_block_number + OR ( + de.active_from_block_number = desired.active_from_block_number + AND desired.active_from_transaction_index IS NOT NULL + AND desired.active_from_log_index IS NOT NULL + AND (de.provenance ->> 'transaction_index')::BIGINT IS NOT NULL + AND (de.provenance ->> 'log_index')::BIGINT IS NOT NULL + AND ( + (de.provenance ->> 'transaction_index')::BIGINT, + (de.provenance ->> 'log_index')::BIGINT + ) > ( + desired.active_from_transaction_index, + desired.active_from_log_index + ) + ) + ) + ORDER BY + de.active_from_block_number, + (de.provenance ->> 'transaction_index')::BIGINT NULLS FIRST, + (de.provenance ->> 'log_index')::BIGINT NULLS FIRST, + de.discovery_edge_id + LIMIT 1 + ) successor ON TRUE + WHERE desired.desired_row_id = ANY($2::BIGINT[]) + ORDER BY desired.desired_row_id + "# + )) + .bind(discovery_source) + .bind(&desired_row_ids) + .fetch_all(&mut *executor) + .await + .context("failed to load a page of historical desired-edge successors")?; + let mut successors = HashMap::with_capacity(successor_rows.len()); + for ( + desired_row_id, + discovery_edge_id, + chain, + block_number, + block_hash, + transaction_index, + log_index, + ) in successor_rows + { + let event_position = match (transaction_index, log_index) { + (Some(transaction_index), Some(log_index)) => Some(EvmEventPosition { + transaction_index, + log_index, + }), + (None, None) => None, + _ => bail!("historical successor carries a partial event position"), + }; + successors.insert( + desired_row_id, + ( + discovery_edge_id, + ObservationTerminalState { + chain, + block_number, + block_hash, + event_position, + }, + ), + ); + } + for (desired_row_id, desired) in page { + let Some((discovery_edge_id, terminal_state)) = successors.remove(&desired_row_id) else { continue; }; - retained_newer_edge_ids.insert(successor.discovery_edge_id); - let terminal_state = ObservationTerminalState { - chain: successor.spec.chain.clone(), - block_number: successor.spec.active_from_block_number, - block_hash: successor.spec.active_from_block_hash.clone(), - event_position: successor.spec.active_from_event_position, - }; + retained_newer_edge_ids.insert(discovery_edge_id); historical_edges.push((desired_row_id, desired, terminal_state)); } + source.record_progress().await?; } Ok(historical_edges) } diff --git a/crates/manifests/src/lib/discovery/reconciliation/streamed/diff/retention.rs b/crates/manifests/src/lib/discovery/reconciliation/streamed/diff/retention.rs index 1e6f9ba3..9b93db97 100644 --- a/crates/manifests/src/lib/discovery/reconciliation/streamed/diff/retention.rs +++ b/crates/manifests/src/lib/discovery/reconciliation/streamed/diff/retention.rs @@ -13,6 +13,7 @@ use super::super::super::super::types::{ use super::super::super::chronology::{assignment_starts_no_later, compare_edge_starts}; use super::super::super::compare_reconciled_discovery_edge_specs; use super::super::super::existing::edge_from_row; +use super::super::DiscoveryObservationPageSource; use super::{ STREAMED_DESIRED_EDGE_COLUMNS, STREAMED_EXISTING_EDGE_COLUMNS_QUALIFIED, desired_edge_spec_from_row, @@ -95,13 +96,17 @@ impl SameAssignmentKeyArrays { /// ordinality column re-associates each matched row with the candidate or /// desired identity it matched, so the per-pair filters and the per-desired /// min-epoch resolution are unchanged. -pub(in super::super) async fn collect_same_assignment_retained_edges( +pub(in super::super) async fn collect_same_assignment_retained_edges( executor: &mut PgConnection, discovery_source: &str, candidates: &[ExistingReconciledDiscoveryEdge], batch_size: usize, retained_newer_edge_ids: &mut HashSet, -) -> Result<()> { + source: &Source, +) -> Result<()> +where + Source: DiscoveryObservationPageSource + Sync, +{ // Orphaned candidates never satisfy the rule-3 filter, so they are // excluded from the batch up front (equivalent to the per-pair check). let live_candidates = candidates @@ -150,6 +155,7 @@ pub(in super::super) async fn collect_same_assignment_retained_edges( matched_desired.insert(desired); } } + source.record_progress().await?; } // Deterministic batching; the retained-id union is order-independent. @@ -215,6 +221,7 @@ pub(in super::super) async fn collect_same_assignment_retained_edges( retained_newer_edge_ids.insert(retained.discovery_edge_id); } } + source.record_progress().await?; } Ok(()) } diff --git a/crates/manifests/src/lib/discovery/reconciliation/streamed/guard.rs b/crates/manifests/src/lib/discovery/reconciliation/streamed/guard.rs new file mode 100644 index 00000000..d99b816d --- /dev/null +++ b/crates/manifests/src/lib/discovery/reconciliation/streamed/guard.rs @@ -0,0 +1,77 @@ +use anyhow::{Context, Result}; + +pub const DISCOVERY_FULL_RECONCILE_MAX_DEACTIVATIONS_ENV: &str = + "BIGNAME_INDEXER_DISCOVERY_FULL_RECONCILE_MAX_DEACTIVATIONS"; + +const DEACTIVATION_GUARD_FLOOR: usize = 10_000; +const CANDIDATE_LOAD_CAP_FLOOR: usize = 100_000; + +pub(super) fn default_max_deactivations(active_edge_count: usize) -> usize { + DEACTIVATION_GUARD_FLOOR.max(active_edge_count / 100) +} + +pub(super) fn default_max_deactivation_candidates(active_edge_count: usize) -> usize { + CANDIDATE_LOAD_CAP_FLOOR.max(active_edge_count / 10) +} + +pub(super) fn max_deactivations_override_from_env() -> Result> { + match std::env::var(DISCOVERY_FULL_RECONCILE_MAX_DEACTIVATIONS_ENV) { + Ok(value) => parse_max_deactivations_override(Some(&value)), + Err(std::env::VarError::NotPresent) => parse_max_deactivations_override(None), + Err(error) => Err(error).context(format!( + "failed to read {DISCOVERY_FULL_RECONCILE_MAX_DEACTIVATIONS_ENV}" + )), + } +} + +fn parse_max_deactivations_override(value: Option<&str>) -> Result> { + match value { + None => Ok(None), + Some(value) => value + .trim() + .parse::() + .map(Some) + .with_context(|| { + format!( + "failed to parse {DISCOVERY_FULL_RECONCILE_MAX_DEACTIVATIONS_ENV} as a deactivation count: {value:?}" + ) + }), + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn default_deactivation_guard_uses_floor_and_percentage() { + assert_eq!(default_max_deactivations(0), 10_000); + assert_eq!(default_max_deactivations(500_000), 10_000); + assert_eq!(default_max_deactivations(7_620_084), 76_200); + } + + #[test] + fn default_candidate_load_cap_uses_floor_and_percentage() { + assert_eq!(default_max_deactivation_candidates(0), 100_000); + assert_eq!(default_max_deactivation_candidates(500_000), 100_000); + assert_eq!(default_max_deactivation_candidates(7_620_084), 762_008); + } + + #[test] + fn deactivation_guard_override_parses_or_rejects() { + assert_eq!( + parse_max_deactivations_override(None).expect("no value must read as no override"), + None + ); + assert_eq!( + parse_max_deactivations_override(Some("123456")).expect("numeric override must parse"), + Some(123_456) + ); + assert_eq!( + parse_max_deactivations_override(Some(" 42 ")) + .expect("surrounding whitespace is trimmed"), + Some(42) + ); + assert!(parse_max_deactivations_override(Some("not-a-count")).is_err()); + } +} diff --git a/crates/manifests/src/lib/discovery/reconciliation/streamed/options.rs b/crates/manifests/src/lib/discovery/reconciliation/streamed/options.rs new file mode 100644 index 00000000..ecc9ebc1 --- /dev/null +++ b/crates/manifests/src/lib/discovery/reconciliation/streamed/options.rs @@ -0,0 +1,35 @@ +const DEFAULT_OBSERVATION_PAGE_LIMIT: i64 = 10_000; +const DEFAULT_MUTATION_BATCH_SIZE: usize = 1_000; + +#[derive(Clone, Copy, Debug)] +pub(crate) struct StreamedDiscoveryReconciliationOptions { + /// Replaces the default `max(10_000, 1% of active edges)` precise + /// deactivation guard bound when set, and raises the coarse candidate + /// load cap to at least the same value so an operator override stays + /// effective end to end. + pub(crate) max_deactivations_override: Option, + /// Replaces the default `max(100_000, 10% of active edges)` coarse cap + /// on how many deactivation candidates may be materialized in memory. + /// Test hook; production overrides go through the env-driven precise + /// bound, which raises this cap alongside it. + pub(crate) coarse_deactivation_cap_override: Option, + pub(crate) observation_page_limit: i64, + pub(crate) mutation_batch_size: usize, + /// Inject a failure if a test scans more than this many stored-edge + /// pages while computing deactivation candidates. + #[cfg(test)] + pub(crate) fail_after_deactivation_source_pages: Option, +} + +impl Default for StreamedDiscoveryReconciliationOptions { + fn default() -> Self { + Self { + max_deactivations_override: None, + coarse_deactivation_cap_override: None, + observation_page_limit: DEFAULT_OBSERVATION_PAGE_LIMIT, + mutation_batch_size: DEFAULT_MUTATION_BATCH_SIZE, + #[cfg(test)] + fail_after_deactivation_source_pages: None, + } + } +} diff --git a/crates/manifests/src/lib/discovery/reconciliation/streamed/progress.rs b/crates/manifests/src/lib/discovery/reconciliation/streamed/progress.rs new file mode 100644 index 00000000..fd80b948 --- /dev/null +++ b/crates/manifests/src/lib/discovery/reconciliation/streamed/progress.rs @@ -0,0 +1,79 @@ +use std::collections::BTreeSet; + +use anyhow::{Context, Result}; +use sqlx::{PgPool, postgres::PgConnection}; + +use super::DiscoveryObservationPageSource; +use crate::{ManifestRuntimeProgress, ManifestRuntimeProgressFuture}; + +pub(super) struct PageSourceManifestProgress<'a, Source>(&'a Source); + +impl<'a, Source> PageSourceManifestProgress<'a, Source> { + pub(super) fn new(source: &'a Source) -> Self { + Self(source) + } +} + +impl ManifestRuntimeProgress for PageSourceManifestProgress<'_, Source> +where + Source: DiscoveryObservationPageSource + Sync, +{ + fn record<'a>(&'a mut self, _pool: &'a PgPool) -> ManifestRuntimeProgressFuture<'a> { + Box::pin(async move { self.0.record_progress().await }) + } +} + +pub(super) async fn load_active_edge_summary_with_progress( + executor: &mut PgConnection, + discovery_source: &str, + page_limit: i64, + source: &Source, +) -> Result<(usize, BTreeSet)> +where + Source: DiscoveryObservationPageSource + Sync, +{ + let mut active_edge_count = 0usize; + let mut chains = BTreeSet::new(); + let mut after_edge_id = 0i64; + loop { + let edge_ids = sqlx::query_scalar::<_, i64>( + r#" + SELECT discovery_edge_id + FROM discovery_edges + WHERE discovery_edge_id > $1 + ORDER BY discovery_edge_id + LIMIT $2 + "#, + ) + .bind(after_edge_id) + .bind(page_limit.max(1)) + .fetch_all(&mut *executor) + .await + .context("failed to page discovery-edge identities for the active-edge summary")?; + let Some(last_edge_id) = edge_ids.last().copied() else { + break; + }; + after_edge_id = last_edge_id; + let rows = sqlx::query_scalar::<_, String>( + r#" + SELECT chain_id + FROM discovery_edges + WHERE discovery_edge_id = ANY($1::BIGINT[]) + AND discovery_source = $2 + AND deactivated_at IS NULL + ORDER BY discovery_edge_id + "#, + ) + .bind(&edge_ids) + .bind(discovery_source) + .fetch_all(&mut *executor) + .await + .context("failed to load an active discovery-edge summary page")?; + active_edge_count = active_edge_count + .checked_add(rows.len()) + .context("active discovery-edge count overflowed usize")?; + chains.extend(rows); + source.record_progress().await?; + } + Ok((active_edge_count, chains)) +} diff --git a/crates/manifests/src/lib/discovery/reconciliation/streamed/staging.rs b/crates/manifests/src/lib/discovery/reconciliation/streamed/staging.rs index 0ebd5f69..cb7acf1b 100644 --- a/crates/manifests/src/lib/discovery/reconciliation/streamed/staging.rs +++ b/crates/manifests/src/lib/discovery/reconciliation/streamed/staging.rs @@ -344,10 +344,15 @@ pub(super) const STREAMED_OBSERVATION_COLUMNS_QUALIFIED: &str = r#" obs.provenance "#; -pub(super) async fn load_streamed_observations_for_keys( +pub(super) async fn load_streamed_observations_for_keys( executor: &mut PgConnection, candidates: &[ExistingReconciledDiscoveryEdge], -) -> Result> { + page_size: usize, + source: &Source, +) -> Result> +where + Source: DiscoveryObservationPageSource + Sync, +{ if candidates.is_empty() { return Ok(Vec::new()); } @@ -357,22 +362,28 @@ pub(super) async fn load_streamed_observations_for_keys( .collect::>() .into_iter() .collect::>(); - let rows = sqlx::query(&format!( - r#" - SELECT {STREAMED_OBSERVATION_COLUMNS} - FROM pg_temp.reconcile_observations - WHERE observation_key = ANY($1::TEXT[]) - ORDER BY observation_key - "# - )) - .bind(&observation_keys) - .fetch_all(executor) - .await - .context("failed to load staged observations for deactivation candidates")?; - - rows.into_iter() - .map(|row| Ok(streamed_observation_from_row(row)?.observation)) - .collect() + let mut observations = Vec::new(); + for keys in observation_keys.chunks(page_size.max(1)) { + let rows = sqlx::query(&format!( + r#" + SELECT {STREAMED_OBSERVATION_COLUMNS} + FROM pg_temp.reconcile_observations + WHERE observation_key = ANY($1::TEXT[]) + ORDER BY observation_key + "# + )) + .bind(keys) + .fetch_all(&mut *executor) + .await + .context("failed to load a staged-observation page for deactivation candidates")?; + observations.extend( + rows.into_iter() + .map(|row| Ok(streamed_observation_from_row(row)?.observation)) + .collect::>>()?, + ); + source.record_progress().await?; + } + Ok(observations) } /// Temp tables are never autoanalyzed; without stats the diff's correlated @@ -385,12 +396,3 @@ pub(super) async fn analyze_temp_table(executor: &mut PgConnection, table: &str) .with_context(|| format!("failed to analyze streamed reconcile temp table {table}"))?; Ok(()) } - -pub(super) async fn count_temp_rows(executor: &mut PgConnection, table: &str) -> Result { - let count = - sqlx::query_scalar::<_, i64>(&format!("SELECT COUNT(*)::BIGINT FROM pg_temp.{table}")) - .fetch_one(executor) - .await - .with_context(|| format!("failed to count streamed reconcile rows in {table}"))?; - usize::try_from(count).with_context(|| format!("streamed {table} count overflowed usize")) -} diff --git a/crates/manifests/src/lib/discovery/reconciliation/streamed/walk_pages.rs b/crates/manifests/src/lib/discovery/reconciliation/streamed/walk_pages.rs index 890b618f..f64044b1 100644 --- a/crates/manifests/src/lib/discovery/reconciliation/streamed/walk_pages.rs +++ b/crates/manifests/src/lib/discovery/reconciliation/streamed/walk_pages.rs @@ -40,10 +40,11 @@ pub(super) async fn run_streamed_admission_walk( admission_state: &DiscoveryAdmissionState, options: &StreamedDiscoveryReconciliationOptions, progress: &impl DiscoveryObservationPageSource, -) -> Result<()> { +) -> Result { let mut walk = DiscoveryAdmissionWalk::new(admission_state); let mut desired_buffer = Vec::::new(); let mut admitted_buffer = Vec::::new(); + let mut admitted_edge_count = 0usize; let mut pending_derived_keys = BTreeSet::<(String, String)>::new(); let mut after_key = None::; @@ -77,6 +78,7 @@ pub(super) async fn run_streamed_admission_walk( &rows, &mut desired_buffer, &mut admitted_buffer, + &mut admitted_edge_count, &mut pending_derived_keys, options, progress, @@ -112,6 +114,7 @@ pub(super) async fn run_streamed_admission_walk( &rows, &mut desired_buffer, &mut admitted_buffer, + &mut admitted_edge_count, &mut pending_derived_keys, options, progress, @@ -123,7 +126,7 @@ pub(super) async fn run_streamed_admission_walk( flush_desired_edge_buffer(&mut *executor, &mut desired_buffer).await?; progress.record_progress().await?; - flush_admitted_edge_buffer(&mut *executor, &mut admitted_buffer).await?; + admitted_edge_count += flush_admitted_edge_buffer(&mut *executor, &mut admitted_buffer).await?; progress.record_progress().await?; insert_pending_contract_instance_seeds( &mut *executor, @@ -134,7 +137,7 @@ pub(super) async fn run_streamed_admission_walk( analyze_temp_table(&mut *executor, "reconcile_desired_edges").await?; analyze_temp_table(&mut *executor, "reconcile_admitted_edges").await?; progress.record_progress().await?; - Ok(()) + Ok(admitted_edge_count) } fn derived_observation_page_query(observation_page_limit: i64) -> String { @@ -160,6 +163,7 @@ async fn admit_streamed_observation_page( rows: &[StreamedObservationRow], desired_buffer: &mut Vec, admitted_buffer: &mut Vec, + admitted_edge_count: &mut usize, pending_derived_keys: &mut BTreeSet<(String, String)>, options: &StreamedDiscoveryReconciliationOptions, progress: &impl DiscoveryObservationPageSource, @@ -206,7 +210,8 @@ async fn admit_streamed_observation_page( progress.record_progress().await?; } if admitted_buffer.len() >= options.mutation_batch_size { - flush_admitted_edge_buffer(&mut *executor, admitted_buffer).await?; + *admitted_edge_count += + flush_admitted_edge_buffer(&mut *executor, admitted_buffer).await?; progress.record_progress().await?; } } @@ -291,9 +296,9 @@ async fn flush_desired_edge_buffer( async fn flush_admitted_edge_buffer( executor: &mut PgConnection, buffer: &mut Vec, -) -> Result<()> { +) -> Result { if buffer.is_empty() { - return Ok(()); + return Ok(0); } let mut builder = QueryBuilder::::new( r#" @@ -327,13 +332,14 @@ async fn flush_admitted_edge_buffer( .push_bind(&edge.from_role); }); builder.push(" ON CONFLICT DO NOTHING "); - builder + let inserted_count = builder .build() .execute(&mut *executor) .await - .context("failed to stage streamed admitted discovery edges")?; + .context("failed to stage streamed admitted discovery edges")? + .rows_affected(); buffer.clear(); - Ok(()) + usize::try_from(inserted_count).context("staged admitted discovery-edge count exceeds usize") } #[cfg(test)] diff --git a/crates/manifests/src/lib/managed_edges.rs b/crates/manifests/src/lib/managed_edges.rs index 7571ceee..4ff146db 100644 --- a/crates/manifests/src/lib/managed_edges.rs +++ b/crates/manifests/src/lib/managed_edges.rs @@ -8,6 +8,7 @@ mod source_graph; pub(crate) use active_addresses::{ reconcile_active_contract_instance_addresses, reconcile_active_contract_instance_addresses_for_ids, + reconcile_active_contract_instance_addresses_with_mutations_and_progress, }; pub(crate) use children::replace_manifest_children; -pub(crate) use source_graph::reconcile_manifest_source_graph; +pub(crate) use source_graph::reconcile_manifest_source_graph_with_progress; diff --git a/crates/manifests/src/lib/managed_edges/active_addresses.rs b/crates/manifests/src/lib/managed_edges/active_addresses.rs index 6098e42a..8dc7967f 100644 --- a/crates/manifests/src/lib/managed_edges/active_addresses.rs +++ b/crates/manifests/src/lib/managed_edges/active_addresses.rs @@ -1,26 +1,46 @@ use std::collections::{BTreeSet, HashMap, HashSet}; use anyhow::{Context, Result, bail}; +use futures_util::TryStreamExt; use serde_json::Value; -use sqlx::Row; +use sqlx::{PgPool, Row, postgres::PgRow}; use uuid::Uuid; -use crate::support::{ActiveAddressSpec, CurrentActiveAddressRow}; +use crate::{ + ManifestRuntimeProgress, + support::{ActiveAddressSpec, CurrentActiveAddressRow}, +}; + +const MANIFEST_GRAPH_PROGRESS_ROWS: usize = 1_000; pub(crate) async fn reconcile_active_contract_instance_addresses( executor: &mut sqlx::postgres::PgConnection, ) -> Result<()> { - reconcile_active_contract_instance_addresses_with_mutations(executor) + let mut progress = None; + reconcile_active_contract_instance_addresses_inner(executor, None, &mut progress) .await .map(|_| ()) } -pub(super) async fn reconcile_active_contract_instance_addresses_with_mutations( +pub(crate) async fn reconcile_active_contract_instance_addresses_with_mutations_and_progress( executor: &mut sqlx::postgres::PgConnection, + pool: &PgPool, + progress: &mut Option<&mut dyn ManifestRuntimeProgress>, ) -> Result> { - let desired_specs = load_desired_active_address_specs(executor, false, &[]).await?; - let existing_active = load_existing_active_address_rows(executor, false, &[]).await?; - apply_active_address_reconciliation(executor, desired_specs, existing_active).await + reconcile_active_contract_instance_addresses_inner(executor, Some(pool), progress).await +} + +async fn reconcile_active_contract_instance_addresses_inner( + executor: &mut sqlx::postgres::PgConnection, + pool: Option<&PgPool>, + progress: &mut Option<&mut dyn ManifestRuntimeProgress>, +) -> Result> { + let desired_specs = + load_desired_active_address_specs(executor, false, &[], pool, progress).await?; + let existing_active = + load_existing_active_address_rows(executor, false, &[], pool, progress).await?; + apply_active_address_reconciliation(executor, desired_specs, existing_active, pool, progress) + .await } pub(crate) async fn reconcile_active_contract_instance_addresses_for_ids( @@ -33,26 +53,39 @@ pub(crate) async fn reconcile_active_contract_instance_addresses_for_ids( let mut scoped_ids = contract_instance_ids.iter().copied().collect::>(); scoped_ids.sort(); - let desired_specs = load_desired_active_address_specs(executor, true, &scoped_ids).await?; - let existing_active = load_existing_active_address_rows(executor, true, &scoped_ids).await?; - apply_active_address_reconciliation(executor, desired_specs, existing_active) - .await - .map(|_| ()) + let mut progress = None; + let desired_specs = + load_desired_active_address_specs(executor, true, &scoped_ids, None, &mut progress).await?; + let existing_active = + load_existing_active_address_rows(executor, true, &scoped_ids, None, &mut progress).await?; + apply_active_address_reconciliation( + executor, + desired_specs, + existing_active, + None, + &mut progress, + ) + .await + .map(|_| ()) } async fn apply_active_address_reconciliation( executor: &mut sqlx::postgres::PgConnection, desired_specs: Vec, existing_active: Vec, + pool: Option<&PgPool>, + progress: &mut Option<&mut dyn ManifestRuntimeProgress>, ) -> Result> { let mut mutated_chains = BTreeSet::new(); - let desired_ids = desired_specs - .iter() - .map(|spec| spec.contract_instance_id) - .collect::>(); + let mut desired_ids = HashSet::with_capacity(desired_specs.len()); + for (index, spec) in desired_specs.iter().enumerate() { + desired_ids.insert(spec.contract_instance_id); + record_progress_after_row(pool, progress, index).await?; + } - for existing_row in &existing_active { + for (index, existing_row) in existing_active.iter().enumerate() { if desired_ids.contains(&existing_row.contract_instance_id) { + record_progress_after_row(pool, progress, index).await?; continue; } @@ -76,14 +109,16 @@ async fn apply_active_address_reconciliation( if result.rows_affected() > 0 { mutated_chains.insert(existing_row.chain.clone()); } + record_progress_after_row(pool, progress, index).await?; } - let existing_active_map = existing_active - .into_iter() - .map(|row| (row.contract_instance_id, row)) - .collect::>(); + let mut existing_active_map = HashMap::with_capacity(existing_active.len()); + for (index, row) in existing_active.into_iter().enumerate() { + existing_active_map.insert(row.contract_instance_id, row); + record_progress_after_row(pool, progress, index).await?; + } - for desired_spec in desired_specs { + for (index, desired_spec) in desired_specs.into_iter().enumerate() { let manifest_declared_active_from_block = manifest_declared_active_from_block(&desired_spec.provenance_json)?; if let Some(existing_row) = existing_active_map.get(&desired_spec.contract_instance_id) { @@ -127,6 +162,7 @@ async fn apply_active_address_reconciliation( mutated_chains.insert(desired_spec.chain.clone()); } } + record_progress_after_row(pool, progress, index).await?; continue; } @@ -159,6 +195,7 @@ async fn apply_active_address_reconciliation( ) })?; mutated_chains.insert(desired_spec.chain); + record_progress_after_row(pool, progress, index).await?; } Ok(mutated_chains) @@ -168,8 +205,10 @@ async fn load_existing_active_address_rows( executor: &mut sqlx::postgres::PgConnection, scoped: bool, scope_ids: &[Uuid], + pool: Option<&PgPool>, + progress: &mut Option<&mut dyn ManifestRuntimeProgress>, ) -> Result> { - let existing_active_rows = sqlx::query( + let query = sqlx::query( r#" SELECT contract_instance_id, chain_id, address FROM contract_instance_addresses @@ -178,33 +217,41 @@ async fn load_existing_active_address_rows( "#, ) .bind(scoped) - .bind(scope_ids) - .fetch_all(&mut *executor) - .await - .context("failed to load active contract-instance addresses")?; + .bind(scope_ids); + let mut rows = query.fetch(&mut *executor); + let mut existing = Vec::new(); + while let Some(row) = rows + .try_next() + .await + .context("failed to stream active contract-instance addresses")? + { + existing.push(current_active_address_from_row(row)?); + record_progress_after_row(pool, progress, existing.len() - 1).await?; + } + record_final_progress(pool, progress, existing.len()).await?; + Ok(existing) +} - existing_active_rows - .into_iter() - .map(|row| { - Ok(CurrentActiveAddressRow { - contract_instance_id: row - .try_get("contract_instance_id") - .context("failed to read active contract_instance_id")?, - chain: row - .try_get("chain_id") - .context("failed to read active chain_id")?, - address: row - .try_get("address") - .context("failed to read active address")?, - }) - }) - .collect::>>() +fn current_active_address_from_row(row: PgRow) -> Result { + Ok(CurrentActiveAddressRow { + contract_instance_id: row + .try_get("contract_instance_id") + .context("failed to read active contract_instance_id")?, + chain: row + .try_get("chain_id") + .context("failed to read active chain_id")?, + address: row + .try_get("address") + .context("failed to read active address")?, + }) } async fn load_desired_active_address_specs( executor: &mut sqlx::postgres::PgConnection, scoped: bool, scope_ids: &[Uuid], + pool: Option<&PgPool>, + progress: &mut Option<&mut dyn ManifestRuntimeProgress>, ) -> Result> { let manifest_rows = sqlx::query( r#" @@ -253,9 +300,15 @@ async fn load_desired_active_address_specs( .fetch_all(&mut *executor) .await .context("failed to load active manifest address specs")?; + record_progress(pool, progress).await?; - let discovery_endpoint_rows = if scoped { - sqlx::query( + let mut specs = HashMap::::new(); + for row in manifest_rows { + merge_manifest_address_spec(&mut specs, row, scoped, scope_ids)?; + } + + if scoped { + let discovery_endpoint_rows = sqlx::query( r#" WITH scoped_ids AS ( SELECT DISTINCT contract_instance_id @@ -290,144 +343,199 @@ async fn load_desired_active_address_specs( .bind(scope_ids) .fetch_all(&mut *executor) .await + .context("failed to load scoped discovery-edge endpoint address specs")?; + for row in discovery_endpoint_rows { + let spec = discovery_endpoint_spec_from_row(row)?; + specs.entry(spec.contract_instance_id).or_insert(spec); + } + record_progress(pool, progress).await?; } else { - sqlx::query( + let mut rows = sqlx::query( r#" - WITH active_discovery_endpoints AS ( - SELECT DISTINCT ON (de.to_contract_instance_id) - de.source_manifest_id, - de.to_contract_instance_id AS contract_instance_id - FROM discovery_edges de - JOIN manifest_versions mv ON mv.manifest_id = de.source_manifest_id - WHERE mv.rollout_status = 'active' - AND de.deactivated_at IS NULL - AND de.edge_kind <> 'migration' - ORDER BY de.to_contract_instance_id, de.source_manifest_id - ) SELECT - endpoints.source_manifest_id, + de.source_manifest_id, cia.contract_instance_id, cia.chain_id, cia.address - FROM active_discovery_endpoints endpoints + FROM discovery_edges de + JOIN manifest_versions mv ON mv.manifest_id = de.source_manifest_id JOIN LATERAL ( SELECT contract_instance_id, chain_id, address FROM contract_instance_addresses - WHERE contract_instance_id = endpoints.contract_instance_id + WHERE contract_instance_id = de.to_contract_instance_id ORDER BY (deactivated_at IS NULL) DESC, admitted_at DESC LIMIT 1 ) cia ON TRUE + WHERE mv.rollout_status = 'active' + AND de.deactivated_at IS NULL + AND de.edge_kind <> 'migration' "#, ) - .fetch_all(&mut *executor) - .await + .fetch(&mut *executor); + let mut discovery_specs = HashMap::::new(); + let mut row_count = 0usize; + while let Some(row) = rows + .try_next() + .await + .context("failed to stream discovery-edge endpoint address specs")? + { + let spec = discovery_endpoint_spec_from_row(row)?; + discovery_specs + .entry(spec.contract_instance_id) + .and_modify(|existing| { + if spec.source_manifest_id < existing.source_manifest_id { + *existing = spec.clone(); + } + }) + .or_insert(spec); + row_count += 1; + record_progress_after_row(pool, progress, row_count - 1).await?; + } + record_final_progress(pool, progress, row_count).await?; + for (index, spec) in discovery_specs.into_values().enumerate() { + specs.entry(spec.contract_instance_id).or_insert(spec); + record_progress_after_row(pool, progress, index).await?; + } } - .context("failed to load discovery-edge endpoint address specs")?; - let mut specs = HashMap::::new(); + Ok(specs.into_values().collect()) +} - for row in manifest_rows { - let contract_instance_id = row - .try_get::("contract_instance_id") - .context("failed to read manifest contract_instance_id")?; - let declaration_kind = row - .try_get::("declaration_kind") - .context("failed to read declaration_kind")?; - let declaration_name = row - .try_get::("declaration_name") - .context("failed to read declaration_name")?; - let chain: String = row.try_get("chain").context("failed to read chain")?; - let manifest_id = row - .try_get::("manifest_id") - .context("failed to read manifest_id")?; - let declared_address = row - .try_get::("declared_address") - .context("failed to read declared_address")?; - let manifest_start_block = row - .try_get::, _>("manifest_start_block") - .context("failed to read manifest_start_block")?; +fn merge_manifest_address_spec( + specs: &mut HashMap, + row: PgRow, + scoped: bool, + scope_ids: &[Uuid], +) -> Result<()> { + let contract_instance_id = row + .try_get::("contract_instance_id") + .context("failed to read manifest contract_instance_id")?; + let declaration_kind = row + .try_get::("declaration_kind") + .context("failed to read declaration_kind")?; + let declaration_name = row + .try_get::("declaration_name") + .context("failed to read declaration_name")?; + let chain: String = row.try_get("chain").context("failed to read chain")?; + let manifest_id = row + .try_get::("manifest_id") + .context("failed to read manifest_id")?; + let declared_address = row + .try_get::("declared_address") + .context("failed to read declared_address")?; + let manifest_start_block = row + .try_get::, _>("manifest_start_block") + .context("failed to read manifest_start_block")?; - if !scoped || scope_ids.contains(&contract_instance_id) { - let provenance_json = manifest_declared_address_provenance( - &declaration_kind, - &declaration_name, - manifest_start_block, - )?; - if let Some(existing_spec) = specs.get_mut(&contract_instance_id) { - if manifest_start_block.is_some() - && manifest_declared_active_from_block(&existing_spec.provenance_json)? - .flatten() - .is_none() - { - existing_spec.provenance_json = provenance_json; - } - } else { - specs.insert( - contract_instance_id, - ActiveAddressSpec { - contract_instance_id, - chain: chain.clone(), - address: declared_address.clone(), - source_manifest_id: Some(manifest_id), - provenance_json, - }, - ); + if !scoped || scope_ids.contains(&contract_instance_id) { + let provenance_json = manifest_declared_address_provenance( + &declaration_kind, + &declaration_name, + manifest_start_block, + )?; + if let Some(existing_spec) = specs.get_mut(&contract_instance_id) { + if manifest_start_block.is_some() + && manifest_declared_active_from_block(&existing_spec.provenance_json)? + .flatten() + .is_none() + { + existing_spec.provenance_json = provenance_json; } - } - - let implementation_contract_instance_id = row - .try_get::, _>("implementation_contract_instance_id") - .context("failed to read implementation_contract_instance_id")?; - let declared_implementation_address = row - .try_get::, _>("declared_implementation_address") - .context("failed to read declared_implementation_address")?; - if let (Some(implementation_contract_instance_id), Some(implementation_address)) = ( - implementation_contract_instance_id, - declared_implementation_address, - ) && (!scoped || scope_ids.contains(&implementation_contract_instance_id)) - { - specs - .entry(implementation_contract_instance_id) - .or_insert(ActiveAddressSpec { - contract_instance_id: implementation_contract_instance_id, + } else { + specs.insert( + contract_instance_id, + ActiveAddressSpec { + contract_instance_id, chain: chain.clone(), - address: implementation_address.clone(), + address: declared_address.clone(), source_manifest_id: Some(manifest_id), - provenance_json: serde_json::json!({ - "source": "manifest_proxy_implementation", - "proxy_contract_instance_id": contract_instance_id, - "proxy_address": declared_address, - }) - .to_string(), - }); + provenance_json, + }, + ); } } - for row in discovery_endpoint_rows { - let contract_instance_id = row - .try_get::("contract_instance_id") - .context("failed to read discovery endpoint contract_instance_id")?; + let implementation_contract_instance_id = row + .try_get::, _>("implementation_contract_instance_id") + .context("failed to read implementation_contract_instance_id")?; + let declared_implementation_address = row + .try_get::, _>("declared_implementation_address") + .context("failed to read declared_implementation_address")?; + if let (Some(implementation_contract_instance_id), Some(implementation_address)) = ( + implementation_contract_instance_id, + declared_implementation_address, + ) && (!scoped || scope_ids.contains(&implementation_contract_instance_id)) + { specs - .entry(contract_instance_id) + .entry(implementation_contract_instance_id) .or_insert(ActiveAddressSpec { - contract_instance_id, - chain: row - .try_get("chain_id") - .context("failed to read discovery endpoint chain_id")?, - address: row - .try_get("address") - .context("failed to read discovery endpoint address")?, - source_manifest_id: row - .try_get("source_manifest_id") - .context("failed to read discovery endpoint source_manifest_id")?, + contract_instance_id: implementation_contract_instance_id, + chain, + address: implementation_address, + source_manifest_id: Some(manifest_id), provenance_json: serde_json::json!({ - "source": "discovery_edge_endpoint", + "source": "manifest_proxy_implementation", + "proxy_contract_instance_id": contract_instance_id, + "proxy_address": declared_address, }) .to_string(), }); } + Ok(()) +} - Ok(specs.into_values().collect()) +fn discovery_endpoint_spec_from_row(row: PgRow) -> Result { + let contract_instance_id = row + .try_get::("contract_instance_id") + .context("failed to read discovery endpoint contract_instance_id")?; + Ok(ActiveAddressSpec { + contract_instance_id, + chain: row + .try_get("chain_id") + .context("failed to read discovery endpoint chain_id")?, + address: row + .try_get("address") + .context("failed to read discovery endpoint address")?, + source_manifest_id: row + .try_get("source_manifest_id") + .context("failed to read discovery endpoint source_manifest_id")?, + provenance_json: serde_json::json!({ + "source": "discovery_edge_endpoint", + }) + .to_string(), + }) +} + +async fn record_progress_after_row( + pool: Option<&PgPool>, + progress: &mut Option<&mut dyn ManifestRuntimeProgress>, + index: usize, +) -> Result<()> { + if (index + 1).is_multiple_of(MANIFEST_GRAPH_PROGRESS_ROWS) { + record_progress(pool, progress).await?; + } + Ok(()) +} + +async fn record_final_progress( + pool: Option<&PgPool>, + progress: &mut Option<&mut dyn ManifestRuntimeProgress>, + row_count: usize, +) -> Result<()> { + if row_count > 0 && !row_count.is_multiple_of(MANIFEST_GRAPH_PROGRESS_ROWS) { + record_progress(pool, progress).await?; + } + Ok(()) +} + +async fn record_progress( + pool: Option<&PgPool>, + progress: &mut Option<&mut dyn ManifestRuntimeProgress>, +) -> Result<()> { + if let (Some(pool), Some(progress)) = (pool, progress.as_deref_mut()) { + progress.record(pool).await?; + } + Ok(()) } fn manifest_declared_address_provenance( diff --git a/crates/manifests/src/lib/managed_edges/source_graph.rs b/crates/manifests/src/lib/managed_edges/source_graph.rs index 131a79d8..65b4dca2 100644 --- a/crates/manifests/src/lib/managed_edges/source_graph.rs +++ b/crates/manifests/src/lib/managed_edges/source_graph.rs @@ -3,7 +3,8 @@ use std::collections::{HashMap, HashSet}; use anyhow::{Context, Result}; use std::collections::BTreeSet; -use crate::discovery::bump_discovery_admission_epochs; +use crate::{ManifestRuntimeProgress, discovery::bump_discovery_admission_epochs}; +use sqlx::PgPool; use sqlx::Row; use uuid::Uuid; @@ -17,37 +18,48 @@ use crate::{ }, }; -use super::active_addresses::reconcile_active_contract_instance_addresses_with_mutations; +use super::active_addresses::reconcile_active_contract_instance_addresses_with_mutations_and_progress; -pub(crate) async fn reconcile_manifest_source_graph( +pub(crate) async fn reconcile_manifest_source_graph_with_progress( executor: &mut sqlx::postgres::PgConnection, + pool: &PgPool, in_place_transitions: &[ManifestTransition], + progress: &mut Option<&mut dyn ManifestRuntimeProgress>, ) -> Result<(usize, BTreeSet)> { let desired_proxy_edges = load_desired_proxy_edges(executor).await?; + record_progress(pool, progress).await?; let desired_successor_edges = load_desired_manifest_successor_edges(executor, in_place_transitions).await?; + record_progress(pool, progress).await?; let mut cleared_edge_count = 0; cleared_edge_count += reconcile_managed_edges( executor, + pool, &desired_proxy_edges, MANIFEST_PROXY_IMPLEMENTATION_DISCOVERY_SOURCE, + progress, ) .await?; cleared_edge_count += reconcile_managed_edges( executor, + pool, &desired_successor_edges, MANIFEST_SUCCESSOR_DISCOVERY_SOURCE, + progress, ) .await?; let stale_source_edge_count = - deactivate_discovery_edges_without_active_source_manifest(executor).await?; + deactivate_discovery_edges_without_active_source_manifest(executor, pool, progress).await?; cleared_edge_count += stale_source_edge_count; let mutated_address_chains = - reconcile_active_contract_instance_addresses_with_mutations(executor).await?; + reconcile_active_contract_instance_addresses_with_mutations_and_progress( + executor, pool, progress, + ) + .await?; Ok((cleared_edge_count, mutated_address_chains)) } @@ -248,8 +260,10 @@ async fn load_desired_manifest_successor_edges( async fn reconcile_managed_edges( executor: &mut sqlx::postgres::PgConnection, + pool: &PgPool, desired_edges: &[ManagedEdgeSpec], discovery_source: &str, + progress: &mut Option<&mut dyn ManifestRuntimeProgress>, ) -> Result { let existing_rows = sqlx::query( r#" @@ -274,6 +288,7 @@ async fn reconcile_managed_edges( .with_context(|| { format!("failed to load active managed edges for discovery_source {discovery_source}") })?; + record_progress(pool, progress).await?; let existing_edges = existing_rows .into_iter() @@ -344,6 +359,7 @@ async fn reconcile_managed_edges( ) })?; cleared_edge_count += 1; + record_progress(pool, progress).await?; } for desired_edge in desired_edges { @@ -385,6 +401,7 @@ async fn reconcile_managed_edges( desired_edge.to_contract_instance_id ) })?; + record_progress(pool, progress).await?; } bump_discovery_admission_epochs(executor, &mutated_chains).await?; @@ -394,41 +411,79 @@ async fn reconcile_managed_edges( async fn deactivate_discovery_edges_without_active_source_manifest( executor: &mut sqlx::postgres::PgConnection, + pool: &PgPool, + progress: &mut Option<&mut dyn ManifestRuntimeProgress>, ) -> Result { - // Aggregate server-side: a stale manifest can own millions of edges, and - // materializing one returned row per edge would buffer them all in the - // sync transaction. - let deactivated_counts_by_chain = sqlx::query_as::<_, (String, i64)>( - r#" - WITH deactivated AS ( - UPDATE discovery_edges de - SET deactivated_at = now() - WHERE de.deactivated_at IS NULL - AND NOT EXISTS ( - SELECT 1 - FROM manifest_versions mv - WHERE mv.manifest_id = de.source_manifest_id - AND mv.rollout_status = 'active' - ) - RETURNING de.chain_id + const PAGE_ROWS: i64 = 10_000; + + let mut cursor = 0_i64; + let mut deactivated_edge_count = 0usize; + let mut mutated_chains = BTreeSet::new(); + loop { + let page = sqlx::query_as::<_, (i64, String, bool)>( + r#" + SELECT + de.discovery_edge_id, + de.chain_id, + de.deactivated_at IS NULL + AND NOT EXISTS ( + SELECT 1 + FROM manifest_versions mv + WHERE mv.manifest_id = de.source_manifest_id + AND mv.rollout_status = 'active' + ) AS stale_source + FROM discovery_edges de + WHERE de.discovery_edge_id > $1 + ORDER BY de.discovery_edge_id + LIMIT $2 + "#, ) - SELECT chain_id, COUNT(*)::BIGINT AS deactivated_count - FROM deactivated - GROUP BY chain_id - "#, - ) - .fetch_all(&mut *executor) - .await - .context("failed to deactivate discovery edges without an active source manifest")?; - let deactivated_edge_count = deactivated_counts_by_chain - .iter() - .map(|(_, count)| *count as usize) - .sum(); - let mutated_chains = deactivated_counts_by_chain - .into_iter() - .map(|(chain, _)| chain) - .collect::>(); + .bind(cursor) + .bind(PAGE_ROWS) + .fetch_all(&mut *executor) + .await + .context("failed to scan discovery edges for inactive source manifests")?; + let Some((last_id, _, _)) = page.last() else { + break; + }; + cursor = *last_id; + let stale_ids = page + .iter() + .filter_map(|(edge_id, _, stale)| stale.then_some(*edge_id)) + .collect::>(); + if !stale_ids.is_empty() { + let result = sqlx::query( + r#" + UPDATE discovery_edges + SET deactivated_at = now() + WHERE discovery_edge_id = ANY($1::BIGINT[]) + AND deactivated_at IS NULL + "#, + ) + .bind(&stale_ids) + .execute(&mut *executor) + .await + .context("failed to deactivate discovery edges without an active source manifest")?; + deactivated_edge_count += result.rows_affected() as usize; + mutated_chains.extend( + page.iter() + .filter(|(_, _, stale)| *stale) + .map(|(_, chain, _)| chain.clone()), + ); + } + record_progress(pool, progress).await?; + } bump_discovery_admission_epochs(executor, &mutated_chains).await?; Ok(deactivated_edge_count) } + +async fn record_progress( + pool: &PgPool, + progress: &mut Option<&mut dyn ManifestRuntimeProgress>, +) -> Result<()> { + if let Some(progress) = progress.as_deref_mut() { + progress.record(pool).await?; + } + Ok(()) +} diff --git a/crates/manifests/src/lib/model/watched.rs b/crates/manifests/src/lib/model/watched.rs index cd5a7e34..fc6c46f3 100644 --- a/crates/manifests/src/lib/model/watched.rs +++ b/crates/manifests/src/lib/model/watched.rs @@ -7,7 +7,7 @@ use uuid::Uuid; const COMPACT_SOURCE_IDENTITY_SELECTED_TARGET_THRESHOLD: usize = 10_000; -#[derive(Clone, Debug)] +#[derive(Clone, Debug, Eq, Ord, PartialEq, PartialOrd)] pub struct WatchedContract { pub chain: String, pub source_family: String, diff --git a/crates/manifests/src/lib/sync.rs b/crates/manifests/src/lib/sync.rs index 2eb5872f..beb6f42c 100644 --- a/crates/manifests/src/lib/sync.rs +++ b/crates/manifests/src/lib/sync.rs @@ -27,14 +27,31 @@ use self::{ }; use crate::discovery::{bump_discovery_admission_epochs, fence_discovery_admission_epoch_writes}; use crate::{ - ManifestLoadStatus, ManifestRepository, ManifestSyncStatus, ManifestSyncSummary, - managed_edges::{reconcile_manifest_source_graph, replace_manifest_children}, + ManifestLoadStatus, ManifestRepository, ManifestRuntimeProgress, ManifestSyncStatus, + ManifestSyncSummary, + managed_edges::{reconcile_manifest_source_graph_with_progress, replace_manifest_children}, support::{ManifestStorageKey, ManifestTransition}, }; pub async fn sync_repository( pool: &PgPool, repository: &ManifestRepository, +) -> Result { + sync_repository_inner(pool, repository, None).await +} + +pub async fn sync_repository_with_progress( + pool: &PgPool, + repository: &ManifestRepository, + progress: &mut dyn ManifestRuntimeProgress, +) -> Result { + sync_repository_inner(pool, repository, Some(progress)).await +} + +async fn sync_repository_inner( + pool: &PgPool, + repository: &ManifestRepository, + mut progress: Option<&mut dyn ManifestRuntimeProgress>, ) -> Result { match repository.summary().status { ManifestLoadStatus::MissingRoot => { @@ -55,6 +72,7 @@ pub async fn sync_repository( .await .context("failed to start manifest sync transaction")?; let existing_manifests = load_existing_manifest_versions(transaction.as_mut()).await?; + record_manifest_progress(pool, &mut progress).await?; let admission_fence_chains = repository .manifests() .iter() @@ -189,6 +207,7 @@ pub async fn sync_repository( sync_summary.contract_count += loaded_manifest.manifest.contracts.len(); sync_summary.capability_count += loaded_manifest.manifest.capability_flags.len(); sync_summary.discovery_rule_count += loaded_manifest.manifest.discovery_rules.len(); + record_manifest_progress(pool, &mut progress).await?; } for existing_manifest in existing_manifests { @@ -199,10 +218,17 @@ pub async fn sync_repository( delete_stale_manifest_version(transaction.as_mut(), existing_manifest.manifest_id).await?; mutated_chains.insert(existing_manifest.storage_key.chain.clone()); sync_summary.removed_manifest_count += 1; + record_manifest_progress(pool, &mut progress).await?; } let (cleared_discovery_edge_count, mutated_address_chains) = - reconcile_manifest_source_graph(transaction.as_mut(), &in_place_transitions).await?; + reconcile_manifest_source_graph_with_progress( + transaction.as_mut(), + pool, + &in_place_transitions, + &mut progress, + ) + .await?; sync_summary.cleared_discovery_edge_count = cleared_discovery_edge_count; mutated_chains.extend(mutated_address_chains); @@ -220,3 +246,13 @@ pub async fn sync_repository( Ok(sync_summary) } + +async fn record_manifest_progress( + pool: &PgPool, + progress: &mut Option<&mut dyn ManifestRuntimeProgress>, +) -> Result<()> { + if let Some(progress) = progress.as_deref_mut() { + progress.record(pool).await?; + } + Ok(()) +} diff --git a/crates/manifests/src/lib/tests.rs b/crates/manifests/src/lib/tests.rs index a77c7d7e..88b0fad4 100644 --- a/crates/manifests/src/lib/tests.rs +++ b/crates/manifests/src/lib/tests.rs @@ -21,6 +21,20 @@ use super::*; static NEXT_TEST_ID: AtomicU64 = AtomicU64::new(0); const TEST_MIGRATOR: sqlx::migrate::Migrator = sqlx::migrate!("../../migrations"); +#[derive(Default)] +struct CountingManifestProgress { + count: usize, +} + +impl ManifestRuntimeProgress for CountingManifestProgress { + fn record<'a>(&'a mut self, _pool: &'a PgPool) -> ManifestRuntimeProgressFuture<'a> { + Box::pin(async move { + self.count += 1; + Ok(()) + }) + } +} + struct TestDir { path: PathBuf, } @@ -2398,6 +2412,35 @@ fn checked_in_ens_v2_public_resolver_boundary_separates_watch_from_profile_admis Ok(()) } +#[tokio::test] +async fn progress_aware_manifest_sync_and_watch_plan_preserve_results() -> Result<()> { + let database = TestDatabase::new().await?; + let repository = load_repository(checked_in_manifest_root("manifests/mainnet"))?; + let mut sync_progress = CountingManifestProgress::default(); + let progress_summary = + sync_repository_with_progress(database.pool(), &repository, &mut sync_progress).await?; + assert_eq!(progress_summary.status, ManifestSyncStatus::Synced); + assert!( + sync_progress.count > 0, + "manifest graph synchronization must expose completed progress units" + ); + + let expected = load_watched_contract_summary_and_chain_plan(database.pool()).await?; + let mut plan_progress = CountingManifestProgress::default(); + let observed = load_watched_contract_summary_and_chain_plan_with_progress( + database.pool(), + &mut plan_progress, + ) + .await?; + assert_eq!(observed, expected); + assert!( + plan_progress.count >= 2, + "watch-plan streaming and in-memory construction must each expose progress" + ); + + database.cleanup().await +} + #[tokio::test] async fn ens_v2_public_resolver_discovery_is_watch_only_without_profile_capabilities() -> Result<()> { @@ -2875,7 +2918,14 @@ async fn syncing_sepolia_profile_replaces_main_profile_without_mixing() -> Resul assert_eq!(watched_summary.manifest_contract_count, 3); assert_eq!(watched_summary.discovery_edge_count, 0); - let admission_state = load_discovery_admission_state(database.pool()).await?; + let mut admission_progress = CountingManifestProgress::default(); + let admission_state = + load_discovery_admission_state_with_progress(database.pool(), &mut admission_progress) + .await?; + assert!( + admission_progress.count >= 5, + "admission loading must report completed manifest and address-loading units" + ); assert_eq!(admission_state.active_manifest_count, 4); assert_eq!(admission_state.active_root_count, 3); assert_eq!(admission_state.active_contract_count, 3); @@ -9217,6 +9267,7 @@ async fn assert_streamed_reconciliation_parity_with_seqscans_disabled( coarse_deactivation_cap_override: None, observation_page_limit: 2, mutation_batch_size: 2, + fail_after_deactivation_source_pages: None, }, ) .await?; @@ -9926,6 +9977,7 @@ async fn streamed_reconcile_guard_aborts_deactivation_floods_unless_overridden() coarse_deactivation_cap_override: None, observation_page_limit: 2, mutation_batch_size: 2, + fail_after_deactivation_source_pages: None, }, ) .await @@ -9955,6 +10007,7 @@ async fn streamed_reconcile_guard_aborts_deactivation_floods_unless_overridden() coarse_deactivation_cap_override: None, observation_page_limit: 2, mutation_batch_size: 2, + fail_after_deactivation_source_pages: None, }, ) .await?; @@ -10002,6 +10055,7 @@ async fn streamed_reconcile_guard_ignores_chronology_retained_candidates() -> Re coarse_deactivation_cap_override: None, observation_page_limit: 2, mutation_batch_size: 2, + fail_after_deactivation_source_pages: None, }, ) .await?; @@ -10086,6 +10140,7 @@ async fn streamed_reconcile_guard_ignores_cascade_protected_descendants() -> Res coarse_deactivation_cap_override: None, observation_page_limit: 2, mutation_batch_size: 2, + fail_after_deactivation_source_pages: None, }, ) .await?; @@ -10149,9 +10204,10 @@ async fn streamed_reconcile_coarse_cap_aborts_before_loading_candidates() -> Res &empty_source, StreamedDiscoveryReconciliationOptions { max_deactivations_override: None, - coarse_deactivation_cap_override: Some(2), + coarse_deactivation_cap_override: Some(1), observation_page_limit: 2, - mutation_batch_size: 2, + mutation_batch_size: 1, + fail_after_deactivation_source_pages: Some(1), }, ) .await diff --git a/crates/manifests/src/lib/views/drift.rs b/crates/manifests/src/lib/views/drift.rs index 7c85e362..29921adc 100644 --- a/crates/manifests/src/lib/views/drift.rs +++ b/crates/manifests/src/lib/views/drift.rs @@ -2,11 +2,12 @@ mod code_hashes; use anyhow::{Context, Result}; +use futures_util::TryStreamExt; use sqlx::{PgPool, Row}; use crate::{ MANIFEST_PROXY_IMPLEMENTATION_DISCOVERY_SOURCE, MANIFEST_PROXY_IMPLEMENTATION_EDGE_KIND, - normalize_address, + ManifestRuntimeProgress, normalize_address, }; use super::types::{ @@ -17,8 +18,11 @@ use super::types::{ pub use code_hashes::{ load_manifest_code_hash_observations, load_manifest_code_hash_observations_for_watched_contracts, + load_manifest_code_hash_observations_with_progress, }; +const MANIFEST_DRIFT_PROGRESS_ROWS: usize = 1_000; + pub async fn load_manifest_drift_inputs(pool: &PgPool) -> Result { Ok(ManifestDriftInputs { active_manifests: load_manifest_drift_active_manifests(pool).await?, @@ -29,6 +33,29 @@ pub async fn load_manifest_drift_inputs(pool: &PgPool) -> Result Result { + let active_manifests = load_manifest_drift_active_manifests(pool).await?; + progress.record(pool).await?; + let declared_contracts = load_manifest_declared_contract_drift_inputs(pool).await?; + progress.record(pool).await?; + let proxy_implementation_edges = load_manifest_proxy_implementation_drift_edges(pool).await?; + progress.record(pool).await?; + let code_hash_observations = + load_manifest_code_hash_observations_with_progress(pool, progress).await?; + let normalized_manifest_events = + load_manifest_normalized_event_inputs_with_progress(pool, progress).await?; + Ok(ManifestDriftInputs { + active_manifests, + declared_contracts, + proxy_implementation_edges, + code_hash_observations, + normalized_manifest_events, + }) +} + pub async fn load_manifest_drift_active_manifests( pool: &PgPool, ) -> Result> { @@ -302,7 +329,21 @@ pub async fn load_manifest_proxy_implementation_drift_edges( pub async fn load_manifest_normalized_event_inputs( pool: &PgPool, ) -> Result> { - let rows = sqlx::query( + load_manifest_normalized_event_inputs_inner(pool, None).await +} + +async fn load_manifest_normalized_event_inputs_with_progress( + pool: &PgPool, + progress: &mut dyn ManifestRuntimeProgress, +) -> Result> { + load_manifest_normalized_event_inputs_inner(pool, Some(progress)).await +} + +async fn load_manifest_normalized_event_inputs_inner( + pool: &PgPool, + mut progress: Option<&mut dyn ManifestRuntimeProgress>, +) -> Result> { + let mut rows = sqlx::query( r#" SELECT normalized_event_id, @@ -334,71 +375,84 @@ pub async fn load_manifest_normalized_event_inputs( ORDER BY namespace, source_family, manifest_version, event_kind, normalized_event_id "#, ) - .fetch_all(pool) - .await - .context("failed to load manifest normalized-event inputs")?; + .fetch(pool); - rows.into_iter() - .map(|row| { - let manifest_version = row - .try_get::("manifest_version") - .context("failed to read manifest event manifest_version")?; - Ok(ManifestNormalizedEventInput { - normalized_event_id: row - .try_get("normalized_event_id") - .context("failed to read normalized_event_id")?, - event_identity: row - .try_get("event_identity") - .context("failed to read event_identity")?, - namespace: row - .try_get("namespace") - .context("failed to read manifest event namespace")?, - logical_name_id: row - .try_get("logical_name_id") - .context("failed to read logical_name_id")?, - resource_id: row - .try_get("resource_id") - .context("failed to read resource_id")?, - event_kind: row - .try_get("event_kind") - .context("failed to read event_kind")?, - source_family: row - .try_get("source_family") - .context("failed to read manifest event source_family")?, - manifest_version: u64::try_from(manifest_version) - .context("manifest_version must be non-negative")?, - source_manifest_id: row - .try_get("source_manifest_id") - .context("failed to read source_manifest_id")?, - chain_id: row.try_get("chain_id").context("failed to read chain_id")?, - block_number: row - .try_get("block_number") - .context("failed to read block_number")?, - block_hash: row - .try_get("block_hash") - .context("failed to read block_hash")?, - transaction_hash: row - .try_get("transaction_hash") - .context("failed to read transaction_hash")?, - log_index: row - .try_get("log_index") - .context("failed to read log_index")?, - raw_fact_ref: row - .try_get("raw_fact_ref") - .context("failed to read raw_fact_ref")?, - derivation_kind: row - .try_get("derivation_kind") - .context("failed to read derivation_kind")?, - canonicality_state: row - .try_get("canonicality_state") - .context("failed to read manifest event canonicality_state")?, - before_state: row - .try_get("before_state") - .context("failed to read before_state")?, - after_state: row - .try_get("after_state") - .context("failed to read after_state")?, - }) - }) - .collect() + let mut inputs = Vec::new(); + while let Some(row) = rows + .try_next() + .await + .context("failed to stream manifest normalized-event inputs")? + { + let manifest_version = row + .try_get::("manifest_version") + .context("failed to read manifest event manifest_version")?; + inputs.push(ManifestNormalizedEventInput { + normalized_event_id: row + .try_get("normalized_event_id") + .context("failed to read normalized_event_id")?, + event_identity: row + .try_get("event_identity") + .context("failed to read event_identity")?, + namespace: row + .try_get("namespace") + .context("failed to read manifest event namespace")?, + logical_name_id: row + .try_get("logical_name_id") + .context("failed to read logical_name_id")?, + resource_id: row + .try_get("resource_id") + .context("failed to read resource_id")?, + event_kind: row + .try_get("event_kind") + .context("failed to read event_kind")?, + source_family: row + .try_get("source_family") + .context("failed to read manifest event source_family")?, + manifest_version: u64::try_from(manifest_version) + .context("manifest_version must be non-negative")?, + source_manifest_id: row + .try_get("source_manifest_id") + .context("failed to read source_manifest_id")?, + chain_id: row.try_get("chain_id").context("failed to read chain_id")?, + block_number: row + .try_get("block_number") + .context("failed to read block_number")?, + block_hash: row + .try_get("block_hash") + .context("failed to read block_hash")?, + transaction_hash: row + .try_get("transaction_hash") + .context("failed to read transaction_hash")?, + log_index: row + .try_get("log_index") + .context("failed to read log_index")?, + raw_fact_ref: row + .try_get("raw_fact_ref") + .context("failed to read raw_fact_ref")?, + derivation_kind: row + .try_get("derivation_kind") + .context("failed to read derivation_kind")?, + canonicality_state: row + .try_get("canonicality_state") + .context("failed to read manifest event canonicality_state")?, + before_state: row + .try_get("before_state") + .context("failed to read before_state")?, + after_state: row + .try_get("after_state") + .context("failed to read after_state")?, + }); + if inputs.len().is_multiple_of(MANIFEST_DRIFT_PROGRESS_ROWS) + && let Some(progress) = progress.as_deref_mut() + { + progress.record(pool).await?; + } + } + if !inputs.is_empty() + && !inputs.len().is_multiple_of(MANIFEST_DRIFT_PROGRESS_ROWS) + && let Some(progress) = progress + { + progress.record(pool).await?; + } + Ok(inputs) } diff --git a/crates/manifests/src/lib/views/drift/code_hashes.rs b/crates/manifests/src/lib/views/drift/code_hashes.rs index e1df729f..6c93cfe3 100644 --- a/crates/manifests/src/lib/views/drift/code_hashes.rs +++ b/crates/manifests/src/lib/views/drift/code_hashes.rs @@ -1,16 +1,33 @@ use std::collections::BTreeSet; use anyhow::{Context, Result}; +use futures_util::TryStreamExt; use sqlx::{PgPool, Row}; -use crate::{WatchedContract, WatchedContractSource, normalize_address}; +use crate::{ManifestRuntimeProgress, WatchedContract, WatchedContractSource, normalize_address}; use super::super::types::ManifestCodeHashObservation; +const CODE_HASH_OBSERVATION_PROGRESS_ROWS: usize = 1_000; + pub async fn load_manifest_code_hash_observations( pool: &PgPool, ) -> Result> { - let rows = sqlx::query( + load_manifest_code_hash_observations_inner(pool, None).await +} + +pub async fn load_manifest_code_hash_observations_with_progress( + pool: &PgPool, + progress: &mut dyn ManifestRuntimeProgress, +) -> Result> { + load_manifest_code_hash_observations_inner(pool, Some(progress)).await +} + +async fn load_manifest_code_hash_observations_inner( + pool: &PgPool, + mut progress: Option<&mut dyn ManifestRuntimeProgress>, +) -> Result> { + let mut rows = sqlx::query( r#" WITH active_targets AS ( SELECT @@ -30,7 +47,7 @@ pub async fn load_manifest_code_hash_observations( AND cia.deactivated_at IS NULL WHERE mv.rollout_status = 'active' - UNION + UNION ALL SELECT de.chain_id AS chain, @@ -103,13 +120,33 @@ pub async fn load_manifest_code_hash_observations( ) raw_code_hashes ON TRUE "#, ) - .fetch_all(pool) - .await - .context("failed to load manifest code-hash observations")?; + .fetch(pool); - rows.into_iter() - .map(decode_manifest_code_hash_observation) - .collect() + // `UNION ALL` allows the database to stream each branch. Restore exact + // `UNION` semantics in bounded Rust progress units instead of waiting for + // one global database sort before the first heartbeat. + let mut observations = BTreeSet::new(); + let mut streamed_row_count = 0usize; + while let Some(row) = rows + .try_next() + .await + .context("failed to stream manifest code-hash observations")? + { + observations.insert(decode_manifest_code_hash_observation(row)?); + streamed_row_count += 1; + if streamed_row_count.is_multiple_of(CODE_HASH_OBSERVATION_PROGRESS_ROWS) + && let Some(progress) = progress.as_deref_mut() + { + progress.record(pool).await?; + } + } + if streamed_row_count > 0 + && !streamed_row_count.is_multiple_of(CODE_HASH_OBSERVATION_PROGRESS_ROWS) + && let Some(progress) = progress + { + progress.record(pool).await?; + } + Ok(observations.into_iter().collect()) } pub async fn load_manifest_code_hash_observations_for_watched_contracts( diff --git a/crates/manifests/src/lib/views/types.rs b/crates/manifests/src/lib/views/types.rs index 4c6625ac..c645596c 100644 --- a/crates/manifests/src/lib/views/types.rs +++ b/crates/manifests/src/lib/views/types.rs @@ -89,7 +89,7 @@ pub struct ManifestProxyImplementationDriftEdge { pub provenance: Value, } -#[derive(Clone, Debug, Eq, PartialEq)] +#[derive(Clone, Debug, Eq, Ord, PartialEq, PartialOrd)] pub struct ManifestCodeHashObservation { pub chain: String, pub source_family: String, diff --git a/crates/manifests/src/lib/views/watched.rs b/crates/manifests/src/lib/views/watched.rs index 33e1d1d0..fdd00574 100644 --- a/crates/manifests/src/lib/views/watched.rs +++ b/crates/manifests/src/lib/views/watched.rs @@ -11,27 +11,50 @@ mod scoped; #[path = "watched/selection.rs"] mod selection; +use std::{collections::BTreeSet, future::Future, pin::Pin}; + use anyhow::{Context, Result, ensure}; +use futures_util::TryStreamExt; use sqlx::{PgConnection, PgPool, Row, postgres::PgRow}; use crate::{WatchedContract, WatchedContractSource, normalize_address}; +const WATCHED_PLAN_PROGRESS_ROWS: usize = 10_000; + +pub type ManifestRuntimeProgressFuture<'a> = Pin> + Send + 'a>>; + +pub trait ManifestRuntimeProgress: Send { + fn record<'a>(&'a mut self, pool: &'a PgPool) -> ManifestRuntimeProgressFuture<'a>; +} + pub use coverage::{ RequiredWatchedTuple, UncoveredWatchedTuple, find_uncovered_required_watched_tuples, find_uncovered_required_watched_tuples_for_retention_generation, find_uncovered_required_watched_tuples_for_retention_generation_in_transaction, find_uncovered_required_watched_tuples_in_transaction, find_uncovered_watched_tuples, + has_required_watched_tuples, has_required_watched_tuples_with_progress, load_required_watched_tuples, load_required_watched_tuples_in_transaction, + load_required_watched_tuples_in_transaction_with_progress, + load_required_watched_tuples_with_progress, }; pub use frontier::{ StoredLineageCoverageCandidateSummary, StoredLineageCoverageDeltaCursor, StoredLineageCoverageDeltaPage, load_earliest_known_watched_block, + load_earliest_known_watched_block_with_progress, load_stored_lineage_coverage_candidate_delta_page, materialize_stored_lineage_coverage_candidate, + materialize_stored_lineage_coverage_candidate_delta_with_progress, + materialize_stored_lineage_coverage_candidate_with_progress, }; pub use historical::{ ENS_V2_RETAINED_HISTORY_SOURCE_FAMILIES, load_ens_v2_authoritative_discovery_bootstrap_targets, - load_ens_v2_retained_history_recovery_targets, load_historical_watched_contracts_by_chain, + load_ens_v2_authoritative_discovery_bootstrap_targets_with_progress, + load_ens_v2_retained_history_recovery_targets, + load_ens_v2_retained_history_recovery_targets_with_progress, + load_historical_watched_contracts_by_chain, + load_historical_watched_contracts_by_chain_with_progress, + load_historical_watched_contracts_for_target, + load_historical_watched_contracts_scoped_with_progress, }; pub use scoped::{ load_watched_contracts_by_addresses, load_watched_contracts_by_source_family_and_addresses, @@ -77,6 +100,30 @@ ORDER BY 1, 2, 3, 5, 6, 4 )) } +fn streaming_watched_contracts_sql() -> String { + intervals::with_streaming_watched_intervals(&format!( + r#" +SELECT + watched.chain, + watched.source_family, + watched.address, + watched.contract_instance_id, + watched.source, + watched.source_manifest_id, + watched.active_from_block_number, + watched.active_to_block_number +FROM watched_intervals watched +WHERE {current_predicate} + AND ($1::TEXT IS NULL OR watched.chain = $1) + AND ( + cardinality($2::TEXT[]) = 0 + OR watched.source_family = ANY($2::TEXT[]) + ) +"#, + current_predicate = intervals::CURRENT_WATCHED_INTERVAL_PREDICATE, + )) +} + fn manifest_declared_watched_contracts_sql() -> String { intervals::with_watched_intervals(&format!( r#" @@ -99,25 +146,26 @@ ORDER BY 1, 2, 3, 5, 6, 4 } fn resolver_profile_authority_target_cursor_sql() -> String { - let targets = intervals::with_watched_intervals(&format!( + let targets = intervals::with_streaming_watched_intervals(&format!( r#" -SELECT DISTINCT watched.chain, watched.address +SELECT watched.chain, watched.address FROM watched_intervals watched WHERE {current_predicate} AND watched.source_family IN ( 'ens_v1_resolver_l1', 'basenames_base_resolver' ) -ORDER BY watched.chain, watched.address "#, current_predicate = intervals::CURRENT_WATCHED_INTERVAL_PREDICATE, )); format!("DECLARE resolver_profile_authority_targets NO SCROLL CURSOR FOR\n{targets}") } -/// One server-side cursor over the distinct current addresses which can +/// One streaming server-side cursor over current addresses which can /// contribute ENSv1 or Basenames -/// [resolver-profile](../../../../../docs/glossary.md) authority entries. +/// [resolver-profile](../../../../../docs/glossary.md) authority entries. The +/// caller deduplicates across pages so PostgreSQL need not sort the complete +/// watched surface before returning the first page. pub struct ResolverProfileAuthorityTargetPages; impl ResolverProfileAuthorityTargetPages { @@ -165,6 +213,57 @@ pub async fn load_watched_contracts(pool: &PgPool) -> Result Result> { + load_watched_contracts_scoped_with_progress(pool, None, &[], progress).await +} + +/// Stream the current watch plan for one optional chain and source-family +/// subset. The SQL deliberately uses `UNION ALL` without a global sort; the +/// client restores exact deduplication while reporting each completed row +/// page. +pub async fn load_watched_contracts_scoped_with_progress( + pool: &PgPool, + chain: Option<&str>, + source_families: &[String], + progress: &mut dyn ManifestRuntimeProgress, +) -> Result> { + let query = streaming_watched_contracts_sql(); + let mut rows = sqlx::query(&query) + .bind(chain) + .bind(source_families) + .fetch(pool); + let mut watched_contracts = BTreeSet::new(); + let mut streamed_row_count = 0usize; + while let Some(row) = rows + .try_next() + .await + .context("failed to stream watched contracts")? + { + watched_contracts.insert(watched_contract_from_row(row)?); + streamed_row_count += 1; + if streamed_row_count.is_multiple_of(WATCHED_PLAN_PROGRESS_ROWS) { + progress.record(pool).await?; + } + } + if streamed_row_count > 0 && !streamed_row_count.is_multiple_of(WATCHED_PLAN_PROGRESS_ROWS) { + progress.record(pool).await?; + } + let mut result = Vec::with_capacity(watched_contracts.len()); + for watched_contract in watched_contracts { + result.push(watched_contract); + if result.len().is_multiple_of(WATCHED_PLAN_PROGRESS_ROWS) { + progress.record(pool).await?; + } + } + if !result.is_empty() && !result.len().is_multiple_of(WATCHED_PLAN_PROGRESS_ROWS) { + progress.record(pool).await?; + } + Ok(result) +} + pub async fn load_watched_contracts_by_chain( pool: &PgPool, chain: &str, @@ -208,38 +307,38 @@ pub async fn load_manifest_declared_watched_contracts( } fn watched_contracts_from_rows(rows: Vec) -> Result> { - rows.into_iter() - .map(|row| { - let source = row - .try_get::("source") - .context("failed to read watched contract source")?; - Ok(WatchedContract { - chain: row - .try_get("chain") - .context("failed to read watched contract chain")?, - source_family: row - .try_get("source_family") - .context("failed to read watched contract source_family")?, - address: normalize_address( - &row.try_get::("address") - .context("failed to read watched contract address")?, - ), - contract_instance_id: row - .try_get("contract_instance_id") - .context("failed to read watched contract_instance_id")?, - source: WatchedContractSource::from_db_value(&source)?, - source_manifest_id: row - .try_get("source_manifest_id") - .context("failed to read watched contract source_manifest_id")?, - active_from_block_number: row - .try_get("active_from_block_number") - .context("failed to read watched contract active_from_block_number")?, - active_to_block_number: row - .try_get("active_to_block_number") - .context("failed to read watched contract active_to_block_number")?, - }) - }) - .collect() + rows.into_iter().map(watched_contract_from_row).collect() +} + +fn watched_contract_from_row(row: PgRow) -> Result { + let source = row + .try_get::("source") + .context("failed to read watched contract source")?; + Ok(WatchedContract { + chain: row + .try_get("chain") + .context("failed to read watched contract chain")?, + source_family: row + .try_get("source_family") + .context("failed to read watched contract source_family")?, + address: normalize_address( + &row.try_get::("address") + .context("failed to read watched contract address")?, + ), + contract_instance_id: row + .try_get("contract_instance_id") + .context("failed to read watched contract_instance_id")?, + source: WatchedContractSource::from_db_value(&source)?, + source_manifest_id: row + .try_get("source_manifest_id") + .context("failed to read watched contract source_manifest_id")?, + active_from_block_number: row + .try_get("active_from_block_number") + .context("failed to read watched contract active_from_block_number")?, + active_to_block_number: row + .try_get("active_to_block_number") + .context("failed to read watched contract active_to_block_number")?, + }) } fn sort_and_dedup_watched_contracts(watched_contracts: &mut Vec) { @@ -305,6 +404,8 @@ mod query_tests { assert!(resolver_targets.contains("DECLARE resolver_profile_authority_targets")); assert!(resolver_targets.contains("'ens_v1_resolver_l1'")); assert!(resolver_targets.contains("'basenames_base_resolver'")); - assert!(resolver_targets.contains("SELECT DISTINCT watched.chain, watched.address")); + assert!(resolver_targets.contains("SELECT watched.chain, watched.address")); + assert!(resolver_targets.contains("UNION ALL")); + assert!(!resolver_targets.contains("ORDER BY watched.chain, watched.address")); } } diff --git a/crates/manifests/src/lib/views/watched/coverage.rs b/crates/manifests/src/lib/views/watched/coverage.rs index 298c7391..e9341560 100644 --- a/crates/manifests/src/lib/views/watched/coverage.rs +++ b/crates/manifests/src/lib/views/watched/coverage.rs @@ -1,6 +1,11 @@ +use std::collections::BTreeSet; + use anyhow::{Context, Result}; +use futures_util::TryStreamExt; use sqlx::{Executor, PgConnection, PgPool, Postgres, Row}; +use super::ManifestRuntimeProgress; + pub(super) fn required_watched_tuples_cte() -> String { super::intervals::with_watched_intervals(&format!( r#" @@ -36,7 +41,7 @@ required_tuples AS ( /// A historically authoritative watched tuple and the part of its block /// interval required within the evaluated range. -#[derive(Clone, Debug, Eq, PartialEq)] +#[derive(Clone, Debug, Eq, Ord, PartialEq, PartialOrd)] pub struct RequiredWatchedTuple { pub source_family: String, pub address: String, @@ -98,403 +103,290 @@ pub async fn load_required_watched_tuples_in_transaction( .await } -async fn load_required_watched_tuples_with_executor<'e, E>( - executor: E, +/// Progress-aware required-tuple load for callers that do not already own a +/// transaction. Rows stream from the two watch-authority branches without a +/// global `UNION`/`DISTINCT` sort; client-side ordering restores the exact +/// public result while each completed page can refresh loop liveness. +pub async fn load_required_watched_tuples_with_progress( + pool: &PgPool, chain: &str, from_block: i64, to_block: i64, log_producing_source_families: &[String], -) -> Result> -where - E: Executor<'e, Database = Postgres>, -{ + progress: &mut dyn ManifestRuntimeProgress, +) -> Result> { + let mut connection = pool + .acquire() + .await + .context("failed to acquire required watched tuple stream connection")?; + load_required_watched_tuples_in_transaction_with_progress( + &mut connection, + pool, + chain, + from_block, + to_block, + log_producing_source_families, + progress, + ) + .await +} + +/// Test whether watched authority requires any tuple in the interval without +/// retaining the complete mainnet-sized tuple set. +pub async fn has_required_watched_tuples( + pool: &PgPool, + chain: &str, + from_block: i64, + to_block: i64, + log_producing_source_families: &[String], +) -> Result { + has_required_watched_tuples_inner( + pool, + chain, + from_block, + to_block, + log_producing_source_families, + ) + .await +} + +/// Progress-aware existence check. A callback follows the completed probe; +/// unlike the full loader, no watched tuples are accumulated or copied. +pub async fn has_required_watched_tuples_with_progress( + pool: &PgPool, + chain: &str, + from_block: i64, + to_block: i64, + log_producing_source_families: &[String], + progress: &mut dyn ManifestRuntimeProgress, +) -> Result { + let found = has_required_watched_tuples_inner( + pool, + chain, + from_block, + to_block, + log_producing_source_families, + ) + .await?; + progress.record(pool).await?; + Ok(found) +} + +async fn has_required_watched_tuples_inner( + pool: &PgPool, + chain: &str, + from_block: i64, + to_block: i64, + log_producing_source_families: &[String], +) -> Result { if from_block > to_block { anyhow::bail!( "required watched tuple scan range start {from_block} is after end {to_block}" ); } if log_producing_source_families.is_empty() { - return Ok(Vec::new()); + return Ok(false); } - - let required_watched_tuples_cte = required_watched_tuples_cte(); - let query = format!( + let query = super::intervals::with_streaming_watched_intervals(&format!( r#" - {required_watched_tuples_cte} - SELECT - source_family, - address, - required_from_block, - required_to_block - FROM required_tuples - ORDER BY source_family, address, required_from_block - "# - ); - let rows = sqlx::query(&query) + SELECT EXISTS ( + SELECT 1 + FROM watched_intervals watched + WHERE {historical_predicate} + AND watched.chain = $1 + AND watched.source_family = ANY($4::TEXT[]) + AND COALESCE(watched.active_from_block_number, $2::BIGINT) <= $3::BIGINT + AND COALESCE(watched.active_to_block_number, $3::BIGINT) >= $2::BIGINT + AND GREATEST( + COALESCE(watched.active_from_block_number, $2::BIGINT), + $2::BIGINT + ) <= LEAST( + COALESCE(watched.active_to_block_number, $3::BIGINT), + $3::BIGINT + ) + LIMIT 1 + ) + "#, + historical_predicate = super::intervals::HISTORICAL_WATCHED_INTERVAL_PREDICATE, + )); + sqlx::query_scalar(&query) .bind(chain) .bind(from_block) .bind(to_block) .bind(log_producing_source_families) - .fetch_all(executor) + .fetch_one(pool) .await .with_context(|| { format!( - "failed to load required watched tuples for chain {chain} over {from_block}..={to_block}" + "failed to inspect required watched tuples for chain {chain} over {from_block}..={to_block}" ) - })?; - - rows.into_iter() - .map(|row| { - Ok(RequiredWatchedTuple { - source_family: row - .try_get("source_family") - .context("missing required tuple source_family")?, - address: row - .try_get("address") - .context("missing required tuple address")?, - required_from_block: row - .try_get("required_from_block") - .context("missing required tuple required_from_block")?, - required_to_block: row - .try_get("required_to_block") - .context("missing required tuple required_to_block")?, - }) }) - .collect() } -/// Compare the chain's historically authoritative contract tuples -/// (manifest-declared and discovery-edge) with durable -/// `backfill_coverage_facts`, restricted to tuples whose block-number active -/// window intersects `[from_block, to_block]` and whose source family produces -/// logs (`log_producing_source_families`). Deactivation does not erase a closed -/// historical interval while its source and mapped target manifests remain -/// active. A tuple is covered when its required interval (active window ∩ -/// evaluated range) is contained by the gap-free union of address-scoped facts -/// for that exact tuple and family-scoped facts for its family. Returns at most -/// `limit` violations ordered by (source_family, address). -pub async fn find_uncovered_watched_tuples( - pool: &PgPool, +/// Transaction-scoped progress-aware variant. The heartbeat write uses the +/// pool rather than the fenced connection, so the caller's snapshot and locks +/// remain intact while forward row progress is reported. +pub async fn load_required_watched_tuples_in_transaction_with_progress( + connection: &mut PgConnection, + progress_pool: &PgPool, chain: &str, from_block: i64, to_block: i64, log_producing_source_families: &[String], - limit: i64, -) -> Result> { + progress: &mut dyn ManifestRuntimeProgress, +) -> Result> { if from_block > to_block { anyhow::bail!( - "uncovered watched tuple scan range start {from_block} is after end {to_block}" + "required watched tuple scan range start {from_block} is after end {to_block}" ); } if log_producing_source_families.is_empty() { return Ok(Vec::new()); } - let required_watched_tuples_cte = required_watched_tuples_cte(); - let query = format!( + let query = super::intervals::with_streaming_watched_intervals(&format!( r#" - {required_watched_tuples_cte} SELECT - source_family, - address, - required_from_block, - required_to_block - FROM required_tuples watched - WHERE NOT ( - COALESCE( - ( - SELECT range_agg( - int8range( - fact.covered_from_block, - fact.covered_to_block, - '[]' - ) - ) - FROM backfill_coverage_facts fact - JOIN backfill_jobs fact_job - ON fact_job.backfill_job_id = fact.backfill_job_id - WHERE fact.chain_id = $1 - AND fact_job.status = 'completed'::backfill_lifecycle_status - AND fact_job.chain_id = fact.chain_id - AND fact.covered_from_block >= fact_job.range_start_block_number - AND fact.covered_to_block <= fact_job.range_end_block_number - AND fact.source_family = watched.source_family - AND ( - ( - fact.scope = 'address' - AND fact.address = watched.address - ) - OR ( - fact.scope = 'family' - AND fact.address IS NULL - ) - ) - AND fact.covered_from_block <= watched.required_to_block - AND fact.covered_to_block >= watched.required_from_block - ), - '{{}}'::INT8MULTIRANGE - ) @> int8range( - watched.required_from_block, - watched.required_to_block, - '[]' - ) + watched.source_family, + watched.address, + GREATEST( + COALESCE(watched.active_from_block_number, $2::BIGINT), + $2::BIGINT + ) AS required_from_block, + LEAST( + COALESCE(watched.active_to_block_number, $3::BIGINT), + $3::BIGINT + ) AS required_to_block + FROM watched_intervals watched + WHERE {historical_predicate} + AND watched.chain = $1 + AND watched.source_family = ANY($4::TEXT[]) + AND COALESCE(watched.active_from_block_number, $2::BIGINT) <= $3::BIGINT + AND COALESCE(watched.active_to_block_number, $3::BIGINT) >= $2::BIGINT + AND GREATEST( + COALESCE(watched.active_from_block_number, $2::BIGINT), + $2::BIGINT + ) <= LEAST( + COALESCE(watched.active_to_block_number, $3::BIGINT), + $3::BIGINT + ) + "#, + historical_predicate = super::intervals::HISTORICAL_WATCHED_INTERVAL_PREDICATE, + )); + let mut rows = sqlx::query(&query) + .bind(chain) + .bind(from_block) + .bind(to_block) + .bind(log_producing_source_families) + .fetch(&mut *connection); + let mut required = BTreeSet::new(); + let mut streamed = 0usize; + while let Some(row) = rows.try_next().await.with_context(|| { + format!( + "failed to stream required watched tuples for chain {chain} over {from_block}..={to_block}" ) - ORDER BY source_family, address, required_from_block - LIMIT $5 - "# - ); - let rows = sqlx::query(&query) - .bind(chain) - .bind(from_block) - .bind(to_block) - .bind(log_producing_source_families) - .bind(limit) - .fetch_all(pool) - .await - .with_context(|| { - format!("failed to scan uncovered watched tuples for chain {chain} over {from_block}..={to_block}") - })?; - - rows.into_iter() - .map(|row| { - Ok(UncoveredWatchedTuple { - source_family: row - .try_get("source_family") - .context("missing uncovered tuple source_family")?, - address: row - .try_get("address") - .context("missing uncovered tuple address")?, - required_from_block: row - .try_get("required_from_block") - .context("missing uncovered tuple required_from_block")?, - required_to_block: row - .try_get("required_to_block") - .context("missing uncovered tuple required_to_block")?, - }) - }) - .collect() -} - -/// Compare an explicit, already-diffed set of watched requirements with -/// durable coverage facts. Stored-lineage promotion uses this after a watched -/// set or topic-selector change so unchanged tuples do not need historical -/// coverage re-verification. -pub async fn find_uncovered_required_watched_tuples( - pool: &PgPool, - chain: &str, - requirements: &[RequiredWatchedTuple], - limit: i64, -) -> Result> { - let mut connection = pool - .acquire() - .await - .context("failed to acquire connection for explicit watched coverage verification")?; - find_uncovered_required_watched_tuples_with_retention_generation( - &mut connection, - chain, - requirements, - None, - limit, - ) - .await -} + })? { + required.insert(required_watched_tuple_from_row(&row)?); + streamed += 1; + if streamed.is_multiple_of(super::WATCHED_PLAN_PROGRESS_ROWS) { + progress.record(progress_pool).await?; + } + } + if streamed > 0 && !streamed.is_multiple_of(super::WATCHED_PLAN_PROGRESS_ROWS) { + progress.record(progress_pool).await?; + } -pub async fn find_uncovered_required_watched_tuples_in_transaction( - connection: &mut PgConnection, - chain: &str, - requirements: &[RequiredWatchedTuple], - limit: i64, -) -> Result> { - find_uncovered_required_watched_tuples_with_retention_generation( - connection, - chain, - requirements, - None, - limit, - ) - .await + let mut result = Vec::with_capacity(required.len()); + for requirement in required { + result.push(requirement); + if result + .len() + .is_multiple_of(super::WATCHED_PLAN_PROGRESS_ROWS) + { + progress.record(progress_pool).await?; + } + } + if !result.is_empty() + && !result + .len() + .is_multiple_of(super::WATCHED_PLAN_PROGRESS_ROWS) + { + progress.record(progress_pool).await?; + } + Ok(result) } -/// Generation-bound variant used when absence in the retained raw-log corpus -/// is replay authority. Coverage from a completed job in any older retention -/// generation is deliberately ignored even when its numeric interval and -/// watched tuple match exactly. -pub async fn find_uncovered_required_watched_tuples_for_retention_generation( - pool: &PgPool, - chain: &str, - requirements: &[RequiredWatchedTuple], - retention_generation: i64, - limit: i64, -) -> Result> { - if retention_generation < 0 { - anyhow::bail!( - "raw-log retention generation must not be negative, got {retention_generation}" - ); - } - let mut connection = pool.acquire().await.context( - "failed to acquire connection for generation-bound watched coverage verification", - )?; - find_uncovered_required_watched_tuples_for_retention_generation_in_transaction( - &mut connection, - chain, - requirements, - retention_generation, - limit, - ) - .await +fn required_watched_tuple_from_row(row: &sqlx::postgres::PgRow) -> Result { + Ok(RequiredWatchedTuple { + source_family: row + .try_get("source_family") + .context("missing required tuple source_family")?, + address: row + .try_get("address") + .context("missing required tuple address")?, + required_from_block: row + .try_get("required_from_block") + .context("missing required tuple required_from_block")?, + required_to_block: row + .try_get("required_to_block") + .context("missing required tuple required_to_block")?, + }) } -pub async fn find_uncovered_required_watched_tuples_for_retention_generation_in_transaction( - connection: &mut PgConnection, +async fn load_required_watched_tuples_with_executor<'e, E>( + executor: E, chain: &str, - requirements: &[RequiredWatchedTuple], - retention_generation: i64, - limit: i64, -) -> Result> { - if retention_generation < 0 { + from_block: i64, + to_block: i64, + log_producing_source_families: &[String], +) -> Result> +where + E: Executor<'e, Database = Postgres>, +{ + if from_block > to_block { anyhow::bail!( - "raw-log retention generation must not be negative, got {retention_generation}" + "required watched tuple scan range start {from_block} is after end {to_block}" ); } - find_uncovered_required_watched_tuples_with_retention_generation( - connection, - chain, - requirements, - Some(retention_generation), - limit, - ) - .await -} - -async fn find_uncovered_required_watched_tuples_with_retention_generation( - connection: &mut PgConnection, - chain: &str, - requirements: &[RequiredWatchedTuple], - retention_generation: Option, - limit: i64, -) -> Result> { - let requirements = requirements - .iter() - .filter(|requirement| requirement.required_from_block <= requirement.required_to_block) - .collect::>(); - if requirements.is_empty() { + if log_producing_source_families.is_empty() { return Ok(Vec::new()); } - let source_families = requirements - .iter() - .map(|requirement| requirement.source_family.clone()) - .collect::>(); - let addresses = requirements - .iter() - .map(|requirement| requirement.address.to_ascii_lowercase()) - .collect::>(); - let required_from_blocks = requirements - .iter() - .map(|requirement| requirement.required_from_block) - .collect::>(); - let required_to_blocks = requirements - .iter() - .map(|requirement| requirement.required_to_block) - .collect::>(); - let rows = sqlx::query( + let required_watched_tuples_cte = required_watched_tuples_cte(); + let query = format!( r#" - WITH required_tuples AS ( - SELECT * - FROM UNNEST( - $2::TEXT[], - $3::TEXT[], - $4::BIGINT[], - $5::BIGINT[] - ) AS watched( - source_family, - address, - required_from_block, - required_to_block - ) - ) + {required_watched_tuples_cte} SELECT source_family, address, required_from_block, required_to_block - FROM required_tuples watched - WHERE NOT ( - COALESCE( - ( - SELECT range_agg( - int8range( - fact.covered_from_block, - fact.covered_to_block, - '[]' - ) - ) - FROM backfill_coverage_facts fact - JOIN backfill_jobs fact_job - ON fact_job.backfill_job_id = fact.backfill_job_id - WHERE fact.chain_id = $1 - AND fact_job.status = 'completed'::backfill_lifecycle_status - AND fact_job.chain_id = fact.chain_id - AND ($6::BIGINT IS NULL OR fact_job.raw_log_retention_generation = $6) - AND fact.covered_from_block >= fact_job.range_start_block_number - AND fact.covered_to_block <= fact_job.range_end_block_number - AND fact.source_family = watched.source_family - AND ( - ( - fact.scope = 'address' - AND fact.address = watched.address - ) - OR ( - fact.scope = 'family' - AND fact.address IS NULL - ) - ) - AND fact.covered_from_block <= watched.required_to_block - AND fact.covered_to_block >= watched.required_from_block - ), - '{}'::INT8MULTIRANGE - ) @> int8range( - watched.required_from_block, - watched.required_to_block, - '[]' - ) - ) + FROM required_tuples ORDER BY source_family, address, required_from_block - LIMIT $7 - "#, - ) - .bind(chain) - .bind(&source_families) - .bind(&addresses) - .bind(&required_from_blocks) - .bind(&required_to_blocks) - .bind(retention_generation) - .bind(limit) - .fetch_all(connection) - .await - .with_context(|| { - format!( - "failed to scan {} explicit watched tuple coverage requirements for chain {chain}", - requirements.len() - ) - })?; + "# + ); + let rows = sqlx::query(&query) + .bind(chain) + .bind(from_block) + .bind(to_block) + .bind(log_producing_source_families) + .fetch_all(executor) + .await + .with_context(|| { + format!( + "failed to load required watched tuples for chain {chain} over {from_block}..={to_block}" + ) + })?; - rows.into_iter() - .map(|row| { - Ok(UncoveredWatchedTuple { - source_family: row - .try_get("source_family") - .context("missing uncovered tuple source_family")?, - address: row - .try_get("address") - .context("missing uncovered tuple address")?, - required_from_block: row - .try_get("required_from_block") - .context("missing uncovered tuple required_from_block")?, - required_to_block: row - .try_get("required_to_block") - .context("missing uncovered tuple required_to_block")?, - }) - }) - .collect() + rows.iter().map(required_watched_tuple_from_row).collect() } + +#[path = "coverage/uncovered.rs"] +mod uncovered; + +pub use uncovered::{ + find_uncovered_required_watched_tuples, + find_uncovered_required_watched_tuples_for_retention_generation, + find_uncovered_required_watched_tuples_for_retention_generation_in_transaction, + find_uncovered_required_watched_tuples_in_transaction, find_uncovered_watched_tuples, +}; diff --git a/crates/manifests/src/lib/views/watched/coverage/uncovered.rs b/crates/manifests/src/lib/views/watched/coverage/uncovered.rs new file mode 100644 index 00000000..c3331fe8 --- /dev/null +++ b/crates/manifests/src/lib/views/watched/coverage/uncovered.rs @@ -0,0 +1,337 @@ +use super::*; + +/// Compare the chain's historically authoritative contract tuples +/// (manifest-declared and discovery-edge) with durable +/// `backfill_coverage_facts`, restricted to tuples whose block-number active +/// window intersects `[from_block, to_block]` and whose source family produces +/// logs (`log_producing_source_families`). Deactivation does not erase a closed +/// historical interval while its source and mapped target manifests remain +/// active. A tuple is covered when its required interval (active window ∩ +/// evaluated range) is contained by the gap-free union of address-scoped facts +/// for that exact tuple and family-scoped facts for its family. Returns at most +/// `limit` violations ordered by (source_family, address). +pub async fn find_uncovered_watched_tuples( + pool: &PgPool, + chain: &str, + from_block: i64, + to_block: i64, + log_producing_source_families: &[String], + limit: i64, +) -> Result> { + if from_block > to_block { + anyhow::bail!( + "uncovered watched tuple scan range start {from_block} is after end {to_block}" + ); + } + if log_producing_source_families.is_empty() { + return Ok(Vec::new()); + } + + let required_watched_tuples_cte = required_watched_tuples_cte(); + let query = format!( + r#" + {required_watched_tuples_cte} + SELECT + source_family, + address, + required_from_block, + required_to_block + FROM required_tuples watched + WHERE NOT ( + COALESCE( + ( + SELECT range_agg( + int8range( + fact.covered_from_block, + fact.covered_to_block, + '[]' + ) + ) + FROM backfill_coverage_facts fact + JOIN backfill_jobs fact_job + ON fact_job.backfill_job_id = fact.backfill_job_id + WHERE fact.chain_id = $1 + AND fact_job.status = 'completed'::backfill_lifecycle_status + AND fact_job.chain_id = fact.chain_id + AND fact.covered_from_block >= fact_job.range_start_block_number + AND fact.covered_to_block <= fact_job.range_end_block_number + AND fact.source_family = watched.source_family + AND ( + ( + fact.scope = 'address' + AND fact.address = watched.address + ) + OR ( + fact.scope = 'family' + AND fact.address IS NULL + ) + ) + AND fact.covered_from_block <= watched.required_to_block + AND fact.covered_to_block >= watched.required_from_block + ), + '{{}}'::INT8MULTIRANGE + ) @> int8range( + watched.required_from_block, + watched.required_to_block, + '[]' + ) + ) + ORDER BY source_family, address, required_from_block + LIMIT $5 + "# + ); + let rows = sqlx::query(&query) + .bind(chain) + .bind(from_block) + .bind(to_block) + .bind(log_producing_source_families) + .bind(limit) + .fetch_all(pool) + .await + .with_context(|| { + format!("failed to scan uncovered watched tuples for chain {chain} over {from_block}..={to_block}") + })?; + + rows.into_iter() + .map(|row| { + Ok(UncoveredWatchedTuple { + source_family: row + .try_get("source_family") + .context("missing uncovered tuple source_family")?, + address: row + .try_get("address") + .context("missing uncovered tuple address")?, + required_from_block: row + .try_get("required_from_block") + .context("missing uncovered tuple required_from_block")?, + required_to_block: row + .try_get("required_to_block") + .context("missing uncovered tuple required_to_block")?, + }) + }) + .collect() +} + +/// Compare an explicit, already-diffed set of watched requirements with +/// durable coverage facts. Stored-lineage promotion uses this after a watched +/// set or topic-selector change so unchanged tuples do not need historical +/// coverage re-verification. +pub async fn find_uncovered_required_watched_tuples( + pool: &PgPool, + chain: &str, + requirements: &[RequiredWatchedTuple], + limit: i64, +) -> Result> { + let mut connection = pool + .acquire() + .await + .context("failed to acquire connection for explicit watched coverage verification")?; + find_uncovered_required_watched_tuples_with_retention_generation( + &mut connection, + chain, + requirements, + None, + limit, + ) + .await +} + +pub async fn find_uncovered_required_watched_tuples_in_transaction( + connection: &mut PgConnection, + chain: &str, + requirements: &[RequiredWatchedTuple], + limit: i64, +) -> Result> { + find_uncovered_required_watched_tuples_with_retention_generation( + connection, + chain, + requirements, + None, + limit, + ) + .await +} + +/// Generation-bound variant used when absence in the retained raw-log corpus +/// is replay authority. Coverage from a completed job in any older retention +/// generation is deliberately ignored even when its numeric interval and +/// watched tuple match exactly. +pub async fn find_uncovered_required_watched_tuples_for_retention_generation( + pool: &PgPool, + chain: &str, + requirements: &[RequiredWatchedTuple], + retention_generation: i64, + limit: i64, +) -> Result> { + if retention_generation < 0 { + anyhow::bail!( + "raw-log retention generation must not be negative, got {retention_generation}" + ); + } + let mut connection = pool.acquire().await.context( + "failed to acquire connection for generation-bound watched coverage verification", + )?; + find_uncovered_required_watched_tuples_for_retention_generation_in_transaction( + &mut connection, + chain, + requirements, + retention_generation, + limit, + ) + .await +} + +pub async fn find_uncovered_required_watched_tuples_for_retention_generation_in_transaction( + connection: &mut PgConnection, + chain: &str, + requirements: &[RequiredWatchedTuple], + retention_generation: i64, + limit: i64, +) -> Result> { + if retention_generation < 0 { + anyhow::bail!( + "raw-log retention generation must not be negative, got {retention_generation}" + ); + } + find_uncovered_required_watched_tuples_with_retention_generation( + connection, + chain, + requirements, + Some(retention_generation), + limit, + ) + .await +} + +async fn find_uncovered_required_watched_tuples_with_retention_generation( + connection: &mut PgConnection, + chain: &str, + requirements: &[RequiredWatchedTuple], + retention_generation: Option, + limit: i64, +) -> Result> { + let requirements = requirements + .iter() + .filter(|requirement| requirement.required_from_block <= requirement.required_to_block) + .collect::>(); + if requirements.is_empty() { + return Ok(Vec::new()); + } + + let source_families = requirements + .iter() + .map(|requirement| requirement.source_family.clone()) + .collect::>(); + let addresses = requirements + .iter() + .map(|requirement| requirement.address.to_ascii_lowercase()) + .collect::>(); + let required_from_blocks = requirements + .iter() + .map(|requirement| requirement.required_from_block) + .collect::>(); + let required_to_blocks = requirements + .iter() + .map(|requirement| requirement.required_to_block) + .collect::>(); + let rows = sqlx::query( + r#" + WITH required_tuples AS ( + SELECT * + FROM UNNEST( + $2::TEXT[], + $3::TEXT[], + $4::BIGINT[], + $5::BIGINT[] + ) AS watched( + source_family, + address, + required_from_block, + required_to_block + ) + ) + SELECT + source_family, + address, + required_from_block, + required_to_block + FROM required_tuples watched + WHERE NOT ( + COALESCE( + ( + SELECT range_agg( + int8range( + fact.covered_from_block, + fact.covered_to_block, + '[]' + ) + ) + FROM backfill_coverage_facts fact + JOIN backfill_jobs fact_job + ON fact_job.backfill_job_id = fact.backfill_job_id + WHERE fact.chain_id = $1 + AND fact_job.status = 'completed'::backfill_lifecycle_status + AND fact_job.chain_id = fact.chain_id + AND ($6::BIGINT IS NULL OR fact_job.raw_log_retention_generation = $6) + AND fact.covered_from_block >= fact_job.range_start_block_number + AND fact.covered_to_block <= fact_job.range_end_block_number + AND fact.source_family = watched.source_family + AND ( + ( + fact.scope = 'address' + AND fact.address = watched.address + ) + OR ( + fact.scope = 'family' + AND fact.address IS NULL + ) + ) + AND fact.covered_from_block <= watched.required_to_block + AND fact.covered_to_block >= watched.required_from_block + ), + '{}'::INT8MULTIRANGE + ) @> int8range( + watched.required_from_block, + watched.required_to_block, + '[]' + ) + ) + ORDER BY source_family, address, required_from_block + LIMIT $7 + "#, + ) + .bind(chain) + .bind(&source_families) + .bind(&addresses) + .bind(&required_from_blocks) + .bind(&required_to_blocks) + .bind(retention_generation) + .bind(limit) + .fetch_all(connection) + .await + .with_context(|| { + format!( + "failed to scan {} explicit watched tuple coverage requirements for chain {chain}", + requirements.len() + ) + })?; + + rows.into_iter() + .map(|row| { + Ok(UncoveredWatchedTuple { + source_family: row + .try_get("source_family") + .context("missing uncovered tuple source_family")?, + address: row + .try_get("address") + .context("missing uncovered tuple address")?, + required_from_block: row + .try_get("required_from_block") + .context("missing uncovered tuple required_from_block")?, + required_to_block: row + .try_get("required_to_block") + .context("missing uncovered tuple required_to_block")?, + }) + }) + .collect() +} diff --git a/crates/manifests/src/lib/views/watched/frontier.rs b/crates/manifests/src/lib/views/watched/frontier.rs index 3baa5836..db155373 100644 --- a/crates/manifests/src/lib/views/watched/frontier.rs +++ b/crates/manifests/src/lib/views/watched/frontier.rs @@ -2,7 +2,10 @@ use anyhow::{Context, Result, ensure}; use bigname_storage::STORED_LINEAGE_COVERAGE_CANDIDATE_TABLE; use sqlx::{PgConnection, PgPool, Row}; -use super::RequiredWatchedTuple; +use super::{ManifestRuntimeProgress, RequiredWatchedTuple}; + +const COVERAGE_SOURCE_PAGE_ROWS: i64 = 1_000; +const COVERAGE_DELTA_TABLE: &str = "stored_lineage_coverage_frontier_candidate_delta"; // Created and owned by bigname-storage's publication guard. Keeping manifest // authority in a temporary relation lets PostgreSQL derive, diff, and publish @@ -65,6 +68,80 @@ pub async fn load_earliest_known_watched_block( }) } +pub async fn load_earliest_known_watched_block_with_progress( + pool: &PgPool, + chain: &str, + through_block: i64, + log_producing_source_families: &[String], + progress: &mut dyn ManifestRuntimeProgress, +) -> Result> { + if log_producing_source_families.is_empty() { + return Ok(None); + } + let mut connection = pool + .acquire() + .await + .context("failed to acquire earliest watched-block scan connection")?; + let mut earliest = None; + for (source_table, source_id_column, watched_branch) in [ + ( + "contract_instance_addresses", + "contract_instance_address_id", + "manifest_watched_intervals", + ), + ( + "discovery_edges", + "discovery_edge_id", + "discovery_watched_intervals", + ), + ] { + let mut after_id = 0i64; + loop { + let source_ids = sqlx::query_scalar::<_, i64>(&format!( + "SELECT {source_id_column} FROM {source_table} WHERE {source_id_column} > $1 ORDER BY {source_id_column} LIMIT $2" + )) + .bind(after_id) + .bind(COVERAGE_SOURCE_PAGE_ROWS) + .fetch_all(&mut *connection) + .await + .with_context(|| format!("failed to page {source_table} for earliest watched block"))?; + let Some(last_id) = source_ids.last().copied() else { + break; + }; + after_id = last_id; + let query = super::intervals::with_streaming_watched_intervals(&format!( + r#" + SELECT MIN(watched.active_from_block_number)::BIGINT + FROM {watched_branch} watched + WHERE watched.source_row_id = ANY($3::BIGINT[]) + AND {historical_predicate} + AND watched.chain = $1 + AND watched.source_family = ANY($4::TEXT[]) + AND watched.active_from_block_number IS NOT NULL + AND watched.active_from_block_number <= $2 + "#, + historical_predicate = super::intervals::HISTORICAL_WATCHED_INTERVAL_PREDICATE, + )); + let page_earliest = sqlx::query_scalar::<_, Option>(&query) + .bind(chain) + .bind(through_block) + .bind(&source_ids) + .bind(log_producing_source_families) + .fetch_one(&mut *connection) + .await + .with_context(|| { + format!("failed to inspect {watched_branch} earliest watched block page") + })?; + if let Some(page_earliest) = page_earliest { + earliest = + Some(earliest.map_or(page_earliest, |value: i64| value.min(page_earliest))); + } + progress.record(pool).await?; + } + } + Ok(earliest) +} + /// Materialize the complete authoritative requirement snapshot, clipped to /// inclusive `verified_from_block..=verified_through_block`, on the storage /// publication guard's transaction-local connection. @@ -165,6 +242,204 @@ pub async fn materialize_stored_lineage_coverage_candidate( }) } +/// Progress-aware candidate materialization. Source rows are first keyset +/// paged by their primary identities; each bounded page contributes its +/// interval union to the transaction-local candidate table. +pub async fn materialize_stored_lineage_coverage_candidate_with_progress( + connection: &mut PgConnection, + progress_pool: &PgPool, + chain: &str, + verified_from_block: i64, + verified_through_block: i64, + log_producing_source_families: &[String], + progress: &mut dyn ManifestRuntimeProgress, +) -> Result { + ensure!( + !chain.trim().is_empty(), + "coverage candidate chain must not be empty" + ); + ensure!( + verified_from_block >= 0 && verified_through_block >= verified_from_block, + "coverage candidate bounds must be non-negative and ordered" + ); + ensure!( + verified_through_block < i64::MAX, + "inclusive coverage candidate upper bound cannot be represented as INT8MULTIRANGE" + ); + sqlx::query(&format!( + "TRUNCATE pg_temp.{STORED_LINEAGE_COVERAGE_CANDIDATE_TABLE}" + )) + .execute(&mut *connection) + .await + .context("failed to clear stored-lineage coverage candidate")?; + if log_producing_source_families.is_empty() { + return Ok(StoredLineageCoverageCandidateSummary { + requirement_tuple_count: 0, + }); + } + + materialize_candidate_source_pages( + connection, + progress_pool, + chain, + verified_from_block, + verified_through_block, + log_producing_source_families, + "contract_instance_addresses", + "contract_instance_address_id", + "manifest_watched_intervals", + progress, + ) + .await?; + materialize_candidate_source_pages( + connection, + progress_pool, + chain, + verified_from_block, + verified_through_block, + log_producing_source_families, + "discovery_edges", + "discovery_edge_id", + "discovery_watched_intervals", + progress, + ) + .await?; + + let mut requirement_tuple_count = 0i64; + let mut cursor = None::<(String, String)>; + loop { + let rows = sqlx::query(&format!( + r#" + SELECT source_family, address + FROM pg_temp.{STORED_LINEAGE_COVERAGE_CANDIDATE_TABLE} + WHERE $1::TEXT IS NULL OR (source_family, address) > ($1, $2) + ORDER BY source_family, address + LIMIT $3 + "#, + )) + .bind(cursor.as_ref().map(|(family, _)| family)) + .bind(cursor.as_ref().map(|(_, address)| address)) + .bind(COVERAGE_SOURCE_PAGE_ROWS) + .fetch_all(&mut *connection) + .await + .context("failed to count a stored-lineage coverage candidate page")?; + let Some(last) = rows.last() else { + break; + }; + cursor = Some((last.try_get("source_family")?, last.try_get("address")?)); + requirement_tuple_count += i64::try_from(rows.len())?; + progress.record(progress_pool).await?; + } + Ok(StoredLineageCoverageCandidateSummary { + requirement_tuple_count, + }) +} + +#[expect(clippy::too_many_arguments)] +#[path = "frontier/source_pages.rs"] +mod source_pages; + +use source_pages::materialize_candidate_source_pages; +pub async fn materialize_stored_lineage_coverage_candidate_delta_with_progress( + connection: &mut PgConnection, + progress_pool: &PgPool, + chain: &str, + topic_changed_source_families: &[String], + reverify_all: bool, + progress: &mut dyn ManifestRuntimeProgress, +) -> Result<()> { + sqlx::query(&format!( + r#" + CREATE TEMP TABLE pg_temp.{COVERAGE_DELTA_TABLE} ( + source_family TEXT NOT NULL, + address TEXT NOT NULL, + required_from_block BIGINT NOT NULL, + required_to_block BIGINT NOT NULL, + PRIMARY KEY (source_family, address, required_from_block, required_to_block) + ) ON COMMIT DROP + "#, + )) + .execute(&mut *connection) + .await + .context("failed to create stored-lineage coverage delta table")?; + let mut cursor = None::<(String, String)>; + loop { + let rows = sqlx::query(&format!( + r#" + SELECT source_family, address + FROM pg_temp.{STORED_LINEAGE_COVERAGE_CANDIDATE_TABLE} + WHERE $1::TEXT IS NULL OR (source_family, address) > ($1, $2) + ORDER BY source_family, address + LIMIT $3 + "#, + )) + .bind(cursor.as_ref().map(|(family, _)| family)) + .bind(cursor.as_ref().map(|(_, address)| address)) + .bind(COVERAGE_SOURCE_PAGE_ROWS) + .fetch_all(&mut *connection) + .await + .context("failed to page coverage candidates for delta materialization")?; + let Some(last) = rows.last() else { + break; + }; + cursor = Some((last.try_get("source_family")?, last.try_get("address")?)); + let families = rows + .iter() + .map(|row| row.try_get::("source_family")) + .collect::, _>>()?; + let addresses = rows + .iter() + .map(|row| row.try_get::("address")) + .collect::, _>>()?; + sqlx::query(&format!( + r#" + WITH page_keys AS ( + SELECT * FROM UNNEST($4::TEXT[], $5::TEXT[]) key(source_family, address) + ), delta_by_tuple AS ( + SELECT + candidate.source_family, + candidate.address, + CASE WHEN $3::BOOLEAN + OR candidate.source_family = ANY($2::TEXT[]) + THEN candidate.required_intervals + ELSE candidate.required_intervals - COALESCE( + persisted.required_intervals, + '{{}}'::INT8MULTIRANGE + ) + END AS required_intervals + FROM page_keys key + JOIN pg_temp.{STORED_LINEAGE_COVERAGE_CANDIDATE_TABLE} candidate + USING (source_family, address) + LEFT JOIN stored_lineage_coverage_frontier_requirements persisted + ON persisted.chain_id = $1 + AND persisted.source_family = candidate.source_family + AND persisted.address = candidate.address + ) + INSERT INTO pg_temp.{COVERAGE_DELTA_TABLE} ( + source_family, address, required_from_block, required_to_block + ) + SELECT + delta.source_family, + delta.address, + lower(required_range)::BIGINT, + (upper(required_range) - 1)::BIGINT + FROM delta_by_tuple delta + CROSS JOIN LATERAL unnest(delta.required_intervals) required_range + "#, + )) + .bind(chain) + .bind(topic_changed_source_families) + .bind(reverify_all) + .bind(&families) + .bind(&addresses) + .execute(&mut *connection) + .await + .context("failed to materialize a stored-lineage coverage delta page")?; + progress.record(progress_pool).await?; + } + Ok(()) +} + /// Return one bounded, ordered page of candidate intervals which need fact /// verification. Unchanged topic families return only candidate-minus-saved /// intervals. Changed topic families (or a cold/deep rebuild) return their @@ -185,8 +460,35 @@ pub async fn load_stored_lineage_coverage_candidate_delta_page( (1..=1_000).contains(&limit), "coverage delta page limit must be between 1 and 1000" ); - let rows = sqlx::query(&format!( - r#" + let delta_is_materialized = sqlx::query_scalar::<_, bool>( + "SELECT to_regclass('pg_temp.stored_lineage_coverage_frontier_candidate_delta') IS NOT NULL", + ) + .fetch_one(&mut *connection) + .await + .context("failed to inspect stored-lineage coverage delta materialization")?; + let rows = if delta_is_materialized { + sqlx::query(&format!( + r#" + SELECT source_family, address, required_from_block, required_to_block + FROM pg_temp.{COVERAGE_DELTA_TABLE} + WHERE $1::TEXT IS NULL + OR (source_family, address, required_from_block, required_to_block) + > ($1::TEXT, $2::TEXT, $3::BIGINT, $4::BIGINT) + ORDER BY source_family, address, required_from_block, required_to_block + LIMIT $5 + "#, + )) + .bind(cursor.map(|cursor| cursor.source_family.as_str())) + .bind(cursor.map(|cursor| cursor.address.as_str())) + .bind(cursor.map(|cursor| cursor.required_from_block)) + .bind(cursor.map(|cursor| cursor.required_to_block)) + .bind(limit) + .fetch_all(&mut *connection) + .await + .with_context(|| format!("failed to page materialized coverage delta for {chain}"))? + } else { + sqlx::query(&format!( + r#" WITH delta_by_tuple AS ( SELECT candidate.source_family, @@ -227,19 +529,20 @@ pub async fn load_stored_lineage_coverage_candidate_delta_page( ORDER BY source_family, address, required_from_block, required_to_block LIMIT $8 "#, - candidate_table = STORED_LINEAGE_COVERAGE_CANDIDATE_TABLE, - )) - .bind(chain) - .bind(topic_changed_source_families) - .bind(reverify_all) - .bind(cursor.map(|cursor| cursor.source_family.as_str())) - .bind(cursor.map(|cursor| cursor.address.as_str())) - .bind(cursor.map(|cursor| cursor.required_from_block)) - .bind(cursor.map(|cursor| cursor.required_to_block)) - .bind(limit) - .fetch_all(connection) - .await - .with_context(|| format!("failed to page stored-lineage coverage delta for {chain}"))?; + candidate_table = STORED_LINEAGE_COVERAGE_CANDIDATE_TABLE, + )) + .bind(chain) + .bind(topic_changed_source_families) + .bind(reverify_all) + .bind(cursor.map(|cursor| cursor.source_family.as_str())) + .bind(cursor.map(|cursor| cursor.address.as_str())) + .bind(cursor.map(|cursor| cursor.required_from_block)) + .bind(cursor.map(|cursor| cursor.required_to_block)) + .bind(limit) + .fetch_all(&mut *connection) + .await + .with_context(|| format!("failed to page stored-lineage coverage delta for {chain}"))? + }; let requirements = rows .into_iter() @@ -278,204 +581,5 @@ pub async fn load_stored_lineage_coverage_candidate_delta_page( } #[cfg(test)] -mod tests { - use std::collections::BTreeMap; - - use anyhow::Result; - use bigname_storage::{ - StoredLineageCoverageFrontierPublication, StoredLineageCoveragePublicationGuard, - begin_stored_lineage_coverage_frontier_publication, - }; - use bigname_test_support::{TestDatabase, TestDatabaseConfig}; - - use super::*; - - const CHAIN: &str = "frontier-test-chain"; - const FAMILY: &str = "frontier_test_family"; - const ADDRESS_ONE: &str = "0x0000000000000000000000000000000000000001"; - const ADDRESS_TWO: &str = "0x0000000000000000000000000000000000000002"; - - async fn database(name: &str) -> Result { - TestDatabase::create_migrated( - TestDatabaseConfig::new(name), - &bigname_storage::MIGRATOR, - "failed to apply migrations for manifest frontier test", - ) - .await - } - - fn publication() -> StoredLineageCoverageFrontierPublication { - StoredLineageCoverageFrontierPublication { - discovery_admission_epoch: 0, - verified_from_block: 10, - verified_through_block: 40, - topic0s_by_family: BTreeMap::from([(FAMILY.to_owned(), vec![format!("0x{:064x}", 1)])]), - } - } - - async fn stage( - guard: &mut StoredLineageCoveragePublicationGuard, - rows: &[(&str, i64, i64)], - ) -> Result<()> { - for (address, from, through) in rows { - sqlx::query(&format!( - r#" - INSERT INTO pg_temp.{STORED_LINEAGE_COVERAGE_CANDIDATE_TABLE} ( - source_family, - address, - required_intervals - ) - VALUES ($1, $2, int8multirange(int8range($3, $4 + 1, '[)'))) - "#, - )) - .bind(FAMILY) - .bind(address) - .bind(from) - .bind(through) - .execute(guard.connection_mut()) - .await?; - } - Ok(()) - } - - #[tokio::test] - async fn server_delta_handles_addition_removal_readmission_and_topic_change() -> Result<()> { - let database = database("manifest_frontier_delta").await?; - let mut first = - begin_stored_lineage_coverage_frontier_publication(database.pool(), CHAIN, None, 0) - .await?; - stage(&mut first, &[(ADDRESS_ONE, 10, 20)]).await?; - first.publish(&publication()).await?; - - let mut replacement = - begin_stored_lineage_coverage_frontier_publication(database.pool(), CHAIN, Some(1), 0) - .await?; - stage( - &mut replacement, - &[(ADDRESS_ONE, 10, 15), (ADDRESS_TWO, 30, 40)], - ) - .await?; - let delta = load_stored_lineage_coverage_candidate_delta_page( - replacement.connection_mut(), - CHAIN, - &[], - false, - None, - 32, - ) - .await?; - assert_eq!( - delta.requirements, - vec![RequiredWatchedTuple { - source_family: FAMILY.to_owned(), - address: ADDRESS_TWO.to_owned(), - required_from_block: 30, - required_to_block: 40, - }], - "a shortening/removal needs no fact read while an addition does" - ); - replacement.publish(&publication()).await?; - - let mut readmission = - begin_stored_lineage_coverage_frontier_publication(database.pool(), CHAIN, Some(2), 0) - .await?; - stage( - &mut readmission, - &[(ADDRESS_ONE, 10, 20), (ADDRESS_TWO, 30, 40)], - ) - .await?; - let delta = load_stored_lineage_coverage_candidate_delta_page( - readmission.connection_mut(), - CHAIN, - &[], - false, - None, - 32, - ) - .await?; - assert_eq!( - delta.requirements, - vec![RequiredWatchedTuple { - source_family: FAMILY.to_owned(), - address: ADDRESS_ONE.to_owned(), - required_from_block: 16, - required_to_block: 20, - }], - "readmission verifies only the interval absent from the saved replacement" - ); - let topic_delta = load_stored_lineage_coverage_candidate_delta_page( - readmission.connection_mut(), - CHAIN, - &[FAMILY.to_owned()], - false, - None, - 32, - ) - .await?; - assert_eq!(topic_delta.requirements.len(), 2); - assert_eq!(topic_delta.requirements[0].required_from_block, 10); - assert_eq!(topic_delta.requirements[0].required_to_block, 20); - drop(readmission); - - database.cleanup().await - } - - #[tokio::test] - async fn high_cardinality_candidate_returns_only_bounded_delta_pages() -> Result<()> { - let database = database("manifest_frontier_bounded_delta").await?; - let mut guard = - begin_stored_lineage_coverage_frontier_publication(database.pool(), CHAIN, None, 0) - .await?; - sqlx::query(&format!( - r#" - INSERT INTO pg_temp.{STORED_LINEAGE_COVERAGE_CANDIDATE_TABLE} ( - source_family, - address, - required_intervals - ) - SELECT - $1, - '0x' || lpad(to_hex(candidate), 40, '0'), - int8multirange(int8range(10, 21, '[)')) - FROM generate_series(1, 10000) candidate - "#, - )) - .bind(FAMILY) - .execute(guard.connection_mut()) - .await?; - - let first = load_stored_lineage_coverage_candidate_delta_page( - guard.connection_mut(), - CHAIN, - &[], - true, - None, - 37, - ) - .await?; - assert_eq!(first.requirements.len(), 37); - let second = load_stored_lineage_coverage_candidate_delta_page( - guard.connection_mut(), - CHAIN, - &[], - true, - first.next_cursor.as_ref(), - 37, - ) - .await?; - assert_eq!(second.requirements.len(), 37); - assert_ne!(first.requirements, second.requirements); - assert_eq!( - sqlx::query_scalar::<_, i64>(&format!( - "SELECT COUNT(*)::BIGINT FROM pg_temp.{STORED_LINEAGE_COVERAGE_CANDIDATE_TABLE}" - )) - .fetch_one(guard.connection_mut()) - .await?, - 10_000, - "the full candidate stays server-side while each returned delta is bounded" - ); - drop(guard); - - database.cleanup().await - } -} +#[path = "frontier/tests.rs"] +mod tests; diff --git a/crates/manifests/src/lib/views/watched/frontier/source_pages.rs b/crates/manifests/src/lib/views/watched/frontier/source_pages.rs new file mode 100644 index 00000000..6c1ae1d5 --- /dev/null +++ b/crates/manifests/src/lib/views/watched/frontier/source_pages.rs @@ -0,0 +1,73 @@ +use super::*; + +pub(super) async fn materialize_candidate_source_pages( + connection: &mut PgConnection, + progress_pool: &PgPool, + chain: &str, + verified_from_block: i64, + verified_through_block: i64, + log_producing_source_families: &[String], + source_table: &str, + source_id_column: &str, + watched_branch: &str, + progress: &mut dyn ManifestRuntimeProgress, +) -> Result<()> { + let mut after_id = 0i64; + loop { + let source_ids = sqlx::query_scalar::<_, i64>(&format!( + "SELECT {source_id_column} FROM {source_table} WHERE {source_id_column} > $1 ORDER BY {source_id_column} LIMIT $2" + )) + .bind(after_id) + .bind(COVERAGE_SOURCE_PAGE_ROWS) + .fetch_all(&mut *connection) + .await + .with_context(|| format!("failed to page {source_table} for coverage candidate"))?; + let Some(last_id) = source_ids.last().copied() else { + break; + }; + after_id = last_id; + let query = super::super::intervals::with_streaming_watched_intervals(&format!( + r#" + INSERT INTO pg_temp.{candidate_table} ( + source_family, + address, + required_intervals + ) + SELECT + watched.source_family, + LOWER(watched.address), + range_agg( + int8range( + GREATEST(COALESCE(watched.active_from_block_number, $2), $2), + LEAST(COALESCE(watched.active_to_block_number, $3), $3) + 1, + '[)' + ) + ) + FROM {watched_branch} watched + WHERE watched.source_row_id = ANY($5::BIGINT[]) + AND {historical_predicate} + AND watched.chain = $1 + AND watched.source_family = ANY($4::TEXT[]) + AND COALESCE(watched.active_from_block_number, $2) <= $3 + AND COALESCE(watched.active_to_block_number, $3) >= $2 + GROUP BY watched.source_family, LOWER(watched.address) + ON CONFLICT (source_family, address) DO UPDATE + SET required_intervals = {candidate_table}.required_intervals + + EXCLUDED.required_intervals + "#, + candidate_table = STORED_LINEAGE_COVERAGE_CANDIDATE_TABLE, + historical_predicate = super::super::intervals::HISTORICAL_WATCHED_INTERVAL_PREDICATE, + )); + sqlx::query(&query) + .bind(chain) + .bind(verified_from_block) + .bind(verified_through_block) + .bind(log_producing_source_families) + .bind(&source_ids) + .execute(&mut *connection) + .await + .with_context(|| format!("failed to materialize {watched_branch} coverage page"))?; + progress.record(progress_pool).await?; + } + Ok(()) +} diff --git a/crates/manifests/src/lib/views/watched/frontier/tests.rs b/crates/manifests/src/lib/views/watched/frontier/tests.rs new file mode 100644 index 00000000..0c00fa89 --- /dev/null +++ b/crates/manifests/src/lib/views/watched/frontier/tests.rs @@ -0,0 +1,197 @@ +use std::collections::BTreeMap; + +use anyhow::Result; +use bigname_storage::{ + StoredLineageCoverageFrontierPublication, StoredLineageCoveragePublicationGuard, + begin_stored_lineage_coverage_frontier_publication, +}; +use bigname_test_support::{TestDatabase, TestDatabaseConfig}; + +use super::*; + +const CHAIN: &str = "frontier-test-chain"; +const FAMILY: &str = "frontier_test_family"; +const ADDRESS_ONE: &str = "0x0000000000000000000000000000000000000001"; +const ADDRESS_TWO: &str = "0x0000000000000000000000000000000000000002"; + +async fn database(name: &str) -> Result { + TestDatabase::create_migrated( + TestDatabaseConfig::new(name), + &bigname_storage::MIGRATOR, + "failed to apply migrations for manifest frontier test", + ) + .await +} + +fn publication() -> StoredLineageCoverageFrontierPublication { + StoredLineageCoverageFrontierPublication { + discovery_admission_epoch: 0, + verified_from_block: 10, + verified_through_block: 40, + topic0s_by_family: BTreeMap::from([(FAMILY.to_owned(), vec![format!("0x{:064x}", 1)])]), + } +} + +async fn stage( + guard: &mut StoredLineageCoveragePublicationGuard, + rows: &[(&str, i64, i64)], +) -> Result<()> { + for (address, from, through) in rows { + sqlx::query(&format!( + r#" + INSERT INTO pg_temp.{STORED_LINEAGE_COVERAGE_CANDIDATE_TABLE} ( + source_family, + address, + required_intervals + ) + VALUES ($1, $2, int8multirange(int8range($3, $4 + 1, '[)'))) + "#, + )) + .bind(FAMILY) + .bind(address) + .bind(from) + .bind(through) + .execute(guard.connection_mut()) + .await?; + } + Ok(()) +} + +#[tokio::test] +async fn server_delta_handles_addition_removal_readmission_and_topic_change() -> Result<()> { + let database = database("manifest_frontier_delta").await?; + let mut first = + begin_stored_lineage_coverage_frontier_publication(database.pool(), CHAIN, None, 0).await?; + stage(&mut first, &[(ADDRESS_ONE, 10, 20)]).await?; + first.publish(&publication()).await?; + + let mut replacement = + begin_stored_lineage_coverage_frontier_publication(database.pool(), CHAIN, Some(1), 0) + .await?; + stage( + &mut replacement, + &[(ADDRESS_ONE, 10, 15), (ADDRESS_TWO, 30, 40)], + ) + .await?; + let delta = load_stored_lineage_coverage_candidate_delta_page( + replacement.connection_mut(), + CHAIN, + &[], + false, + None, + 32, + ) + .await?; + assert_eq!( + delta.requirements, + vec![RequiredWatchedTuple { + source_family: FAMILY.to_owned(), + address: ADDRESS_TWO.to_owned(), + required_from_block: 30, + required_to_block: 40, + }], + "a shortening/removal needs no fact read while an addition does" + ); + replacement.publish(&publication()).await?; + + let mut readmission = + begin_stored_lineage_coverage_frontier_publication(database.pool(), CHAIN, Some(2), 0) + .await?; + stage( + &mut readmission, + &[(ADDRESS_ONE, 10, 20), (ADDRESS_TWO, 30, 40)], + ) + .await?; + let delta = load_stored_lineage_coverage_candidate_delta_page( + readmission.connection_mut(), + CHAIN, + &[], + false, + None, + 32, + ) + .await?; + assert_eq!( + delta.requirements, + vec![RequiredWatchedTuple { + source_family: FAMILY.to_owned(), + address: ADDRESS_ONE.to_owned(), + required_from_block: 16, + required_to_block: 20, + }], + "readmission verifies only the interval absent from the saved replacement" + ); + let topic_delta = load_stored_lineage_coverage_candidate_delta_page( + readmission.connection_mut(), + CHAIN, + &[FAMILY.to_owned()], + false, + None, + 32, + ) + .await?; + assert_eq!(topic_delta.requirements.len(), 2); + assert_eq!(topic_delta.requirements[0].required_from_block, 10); + assert_eq!(topic_delta.requirements[0].required_to_block, 20); + drop(readmission); + + database.cleanup().await +} + +#[tokio::test] +async fn high_cardinality_candidate_returns_only_bounded_delta_pages() -> Result<()> { + let database = database("manifest_frontier_bounded_delta").await?; + let mut guard = + begin_stored_lineage_coverage_frontier_publication(database.pool(), CHAIN, None, 0).await?; + sqlx::query(&format!( + r#" + INSERT INTO pg_temp.{STORED_LINEAGE_COVERAGE_CANDIDATE_TABLE} ( + source_family, + address, + required_intervals + ) + SELECT + $1, + '0x' || lpad(to_hex(candidate), 40, '0'), + int8multirange(int8range(10, 21, '[)')) + FROM generate_series(1, 10000) candidate + "#, + )) + .bind(FAMILY) + .execute(guard.connection_mut()) + .await?; + + let first = load_stored_lineage_coverage_candidate_delta_page( + guard.connection_mut(), + CHAIN, + &[], + true, + None, + 37, + ) + .await?; + assert_eq!(first.requirements.len(), 37); + let second = load_stored_lineage_coverage_candidate_delta_page( + guard.connection_mut(), + CHAIN, + &[], + true, + first.next_cursor.as_ref(), + 37, + ) + .await?; + assert_eq!(second.requirements.len(), 37); + assert_ne!(first.requirements, second.requirements); + assert_eq!( + sqlx::query_scalar::<_, i64>(&format!( + "SELECT COUNT(*)::BIGINT FROM pg_temp.{STORED_LINEAGE_COVERAGE_CANDIDATE_TABLE}" + )) + .fetch_one(guard.connection_mut()) + .await?, + 10_000, + "the full candidate stays server-side while each returned delta is bounded" + ); + drop(guard); + + database.cleanup().await +} diff --git a/crates/manifests/src/lib/views/watched/historical.rs b/crates/manifests/src/lib/views/watched/historical.rs index c762c883..a437f2da 100644 --- a/crates/manifests/src/lib/views/watched/historical.rs +++ b/crates/manifests/src/lib/views/watched/historical.rs @@ -1,15 +1,16 @@ +use std::collections::{BTreeMap, BTreeSet}; + use anyhow::{Context, Result}; use bigname_domain::block_interval::InclusiveBlockInterval; +use futures_util::TryStreamExt; use sqlx::PgPool; use crate::{ - ManifestBootstrapTarget, WatchedContract, WatchedContractSource, + ManifestBootstrapTarget, ManifestRuntimeProgress, WatchedContract, WatchedContractSource, load_log_producing_source_families, normalize_address, }; -use super::{ - load_required_watched_tuples, sort_and_dedup_watched_contracts, watched_contracts_from_rows, -}; +use super::{sort_and_dedup_watched_contracts, watched_contracts_from_rows}; pub const ENS_V2_RETAINED_HISTORY_SOURCE_FAMILIES: &[&str] = &["ens_v2_root_l1", "ens_v2_registry_l1"]; @@ -35,6 +36,31 @@ pub async fn load_ens_v2_authoritative_discovery_bootstrap_targets( pool: &PgPool, chain: &str, through_block: i64, +) -> Result> { + load_ens_v2_authoritative_discovery_bootstrap_targets_inner(pool, chain, through_block, None) + .await +} + +pub async fn load_ens_v2_authoritative_discovery_bootstrap_targets_with_progress( + pool: &PgPool, + chain: &str, + through_block: i64, + progress: &mut dyn ManifestRuntimeProgress, +) -> Result> { + load_ens_v2_authoritative_discovery_bootstrap_targets_inner( + pool, + chain, + through_block, + Some(progress), + ) + .await +} + +async fn load_ens_v2_authoritative_discovery_bootstrap_targets_inner( + pool: &PgPool, + chain: &str, + through_block: i64, + mut progress: Option<&mut dyn ManifestRuntimeProgress>, ) -> Result> { if through_block < 0 { anyhow::bail!("discovery bootstrap target block must be non-negative"); @@ -48,43 +74,46 @@ pub async fn load_ens_v2_authoritative_discovery_bootstrap_targets( .any(|active| active.as_str() == *source_family) }) .map(str::to_owned) - .collect::>(); + .collect::>(); if source_families.is_empty() { return Ok(Vec::new()); } - let requirements = - load_required_watched_tuples(pool, chain, 0, through_block, &source_families).await?; - let historical = load_historical_watched_contracts_by_chain(pool, chain).await?; - let mut targets = std::collections::BTreeSet::new(); + record_progress(pool, &mut progress).await?; + let source_family_filter = source_families.iter().cloned().collect::>(); + let historical = + load_historical_with_optional_progress(pool, chain, &source_family_filter, &mut progress) + .await?; + let mut targets = BTreeSet::new(); - for contract in historical - .iter() - .filter(|contract| contract.source == WatchedContractSource::DiscoveryEdge) - { + for (index, contract) in historical.iter().enumerate() { + if (index + 1).is_multiple_of(super::WATCHED_PLAN_PROGRESS_ROWS) { + record_progress(pool, &mut progress).await?; + } + if contract.source != WatchedContractSource::DiscoveryEdge + || !source_families.contains(&contract.source_family) + { + continue; + } let Some(active_from_block) = contract.active_from_block_number else { continue; }; let address = normalize_address(&contract.address); - for requirement in requirements.iter().filter(|requirement| { - requirement.source_family == contract.source_family && requirement.address == address - }) { - let effective_from_block = active_from_block.max(requirement.required_from_block); - let effective_to_block = contract - .active_to_block_number - .unwrap_or(through_block) - .min(requirement.required_to_block); - if effective_from_block > effective_to_block { - continue; - } + let effective_from_block = active_from_block.max(0); + let effective_to_block = contract + .active_to_block_number + .unwrap_or(through_block) + .min(through_block); + if effective_from_block <= effective_to_block { targets.insert(ManifestBootstrapTarget { - source_family: requirement.source_family.clone(), + source_family: contract.source_family.clone(), contract_instance_id: contract.contract_instance_id, - address: address.clone(), + address, effective_from_block, effective_to_block: Some(effective_to_block), }); } } + record_tail_progress(pool, &mut progress, historical.len()).await?; Ok(targets.into_iter().collect()) } @@ -127,6 +156,131 @@ pub async fn load_historical_watched_contracts_by_chain( Ok(watched_contracts) } +pub async fn load_historical_watched_contracts_by_chain_with_progress( + pool: &PgPool, + chain: &str, + progress: &mut dyn ManifestRuntimeProgress, +) -> Result> { + load_historical_watched_contracts_scoped_with_progress(pool, chain, &[], progress).await +} + +/// Stream retained watch intervals for one chain and optional source-family +/// subset without a server-side union sort. This is the progress-aware route +/// for adapter preloads whose cardinality grows with admitted discovery. +pub async fn load_historical_watched_contracts_scoped_with_progress( + pool: &PgPool, + chain: &str, + source_families: &[String], + progress: &mut dyn ManifestRuntimeProgress, +) -> Result> { + let query = super::intervals::with_streaming_watched_intervals(&format!( + r#" + SELECT + watched.chain, + watched.source_family, + watched.address, + watched.contract_instance_id, + watched.source, + watched.source_manifest_id, + watched.active_from_block_number, + watched.active_to_block_number + FROM watched_intervals watched + WHERE {historical_predicate} + AND watched.chain = $1 + AND ( + cardinality($2::TEXT[]) = 0 + OR watched.source_family = ANY($2::TEXT[]) + ) + "#, + historical_predicate = super::intervals::HISTORICAL_WATCHED_INTERVAL_PREDICATE, + )); + let mut rows = sqlx::query(&query) + .bind(chain) + .bind(source_families) + .fetch(pool); + let mut watched_contracts = BTreeSet::new(); + let mut streamed_row_count = 0usize; + while let Some(row) = rows + .try_next() + .await + .with_context(|| format!("failed to stream historical watched contracts for {chain}"))? + { + watched_contracts.insert(super::watched_contract_from_row(row)?); + streamed_row_count += 1; + if streamed_row_count.is_multiple_of(super::WATCHED_PLAN_PROGRESS_ROWS) { + progress.record(pool).await?; + } + } + if streamed_row_count > 0 + && !streamed_row_count.is_multiple_of(super::WATCHED_PLAN_PROGRESS_ROWS) + { + progress.record(pool).await?; + } + + let mut result = Vec::with_capacity(watched_contracts.len()); + for watched_contract in watched_contracts { + result.push(watched_contract); + if result + .len() + .is_multiple_of(super::WATCHED_PLAN_PROGRESS_ROWS) + { + progress.record(pool).await?; + } + } + if !result.is_empty() + && !result + .len() + .is_multiple_of(super::WATCHED_PLAN_PROGRESS_ROWS) + { + progress.record(pool).await?; + } + Ok(result) +} + +/// Load the retained intervals for one exact watched target. This avoids a +/// whole-chain historical scan when live coverage recovery already knows the +/// family and address whose proof is missing. +pub async fn load_historical_watched_contracts_for_target( + pool: &PgPool, + chain: &str, + source_family: &str, + address: &str, +) -> Result> { + let query = super::intervals::with_watched_intervals(&format!( + r#" + SELECT + watched.chain, + watched.source_family, + watched.address, + watched.contract_instance_id, + watched.source, + watched.source_manifest_id, + watched.active_from_block_number, + watched.active_to_block_number + FROM watched_intervals watched + WHERE {historical_predicate} + AND watched.chain = $1 + AND watched.source_family = $2 + AND watched.address = $3 + ORDER BY 1, 2, 3, 5, 6, 4, 7, 8 + "#, + historical_predicate = super::intervals::HISTORICAL_WATCHED_INTERVAL_PREDICATE, + )); + let rows = sqlx::query(&query) + .bind(chain) + .bind(source_family) + .bind(normalize_address(address)) + .fetch_all(pool) + .await + .with_context(|| { + format!("failed to load historical watched target {chain}/{source_family}/{address}") + })?; + + let mut watched_contracts = watched_contracts_from_rows(rows)?; + sort_and_dedup_watched_contracts(&mut watched_contracts); + Ok(watched_contracts) +} + /// Build the finite, historically authoritative ENSv2 root/registry targets /// needed to recover a retained-history proof through `through_block`. /// @@ -139,6 +293,25 @@ pub async fn load_ens_v2_retained_history_recovery_targets( pool: &PgPool, chain: &str, through_block: i64, +) -> Result> { + load_ens_v2_retained_history_recovery_targets_inner(pool, chain, through_block, None).await +} + +pub async fn load_ens_v2_retained_history_recovery_targets_with_progress( + pool: &PgPool, + chain: &str, + through_block: i64, + progress: &mut dyn ManifestRuntimeProgress, +) -> Result> { + load_ens_v2_retained_history_recovery_targets_inner(pool, chain, through_block, Some(progress)) + .await +} + +async fn load_ens_v2_retained_history_recovery_targets_inner( + pool: &PgPool, + chain: &str, + through_block: i64, + mut progress: Option<&mut dyn ManifestRuntimeProgress>, ) -> Result> { if through_block < 0 { anyhow::bail!("retained-history recovery target block must be non-negative"); @@ -147,71 +320,113 @@ pub async fn load_ens_v2_retained_history_recovery_targets( let source_families = ENS_V2_RETAINED_HISTORY_SOURCE_FAMILIES .iter() .map(|source_family| (*source_family).to_owned()) - .collect::>(); - let requirements = - load_required_watched_tuples(pool, chain, 0, through_block, &source_families).await?; - let historical = load_historical_watched_contracts_by_chain(pool, chain).await?; - let mut targets = std::collections::BTreeSet::new(); - - for requirement in requirements { - let required_interval = InclusiveBlockInterval::new( - requirement.required_from_block, - requirement.required_to_block, - ) - .with_context(|| { - format!( - "required retained-history tuple {}/{} has an inverted interval {}..={}", - requirement.source_family, - requirement.address, - requirement.required_from_block, - requirement.required_to_block - ) - })?; - let mut covered_intervals = Vec::new(); - for contract in historical.iter().filter(|contract| { - contract.source_family == requirement.source_family - && normalize_address(&contract.address) == requirement.address - }) { - let Some(active_from_block) = contract.active_from_block_number else { - continue; - }; - let effective_from_block = active_from_block.max(requirement.required_from_block); - let effective_to_block = contract - .active_to_block_number - .unwrap_or(through_block) - .min(requirement.required_to_block); - if effective_from_block > effective_to_block { - continue; - } - - covered_intervals.push( - InclusiveBlockInterval::new(effective_from_block, effective_to_block) - .expect("clamped historical watched interval must not be inverted"), - ); + .collect::>(); + let source_family_filter = source_families.iter().cloned().collect::>(); + let historical = + load_historical_with_optional_progress(pool, chain, &source_family_filter, &mut progress) + .await?; + let mut required = BTreeMap::<(String, String), Vec>::new(); + let mut covered = BTreeMap::<(String, String), Vec>::new(); + let mut targets = BTreeSet::new(); + for (index, contract) in historical.iter().enumerate() { + if (index + 1).is_multiple_of(super::WATCHED_PLAN_PROGRESS_ROWS) { + record_progress(pool, &mut progress).await?; + } + if !source_families.contains(&contract.source_family) { + continue; + } + let address = normalize_address(&contract.address); + let required_from_block = contract.active_from_block_number.unwrap_or(0).max(0); + let required_to_block = contract + .active_to_block_number + .unwrap_or(through_block) + .min(through_block); + if required_from_block > required_to_block { + continue; + } + let key = (contract.source_family.clone(), address.clone()); + required.entry(key.clone()).or_default().push( + InclusiveBlockInterval::new(required_from_block, required_to_block) + .expect("clamped required watched interval must not be inverted"), + ); + if let Some(active_from_block) = contract.active_from_block_number { + let effective_from_block = active_from_block.max(required_from_block); + let interval = InclusiveBlockInterval::new(effective_from_block, required_to_block) + .expect("known-start historical watched interval must not be inverted"); + covered.entry(key).or_default().push(interval); targets.insert(ManifestBootstrapTarget { - source_family: requirement.source_family.clone(), + source_family: contract.source_family.clone(), contract_instance_id: contract.contract_instance_id, - address: requirement.address.clone(), + address, effective_from_block, - effective_to_block: Some(effective_to_block), + effective_to_block: Some(required_to_block), }); } + } + record_tail_progress(pool, &mut progress, historical.len()).await?; - if !retained_requirement_is_covered(required_interval, covered_intervals) { - anyhow::bail!( - "required retained-history tuple {}/{} over {}..={} has no gap-free known-start historical contract identity", - requirement.source_family, - requirement.address, - requirement.required_from_block, - requirement.required_to_block - ); + for ((source_family, address), required_intervals) in required { + let covered_intervals = covered.get(&(source_family.clone(), address.clone())); + for required_interval in required_intervals { + if !retained_requirement_is_covered( + required_interval, + covered_intervals.into_iter().flatten().copied(), + ) { + anyhow::bail!( + "required retained-history tuple {source_family}/{address} over {}..={} has no gap-free known-start historical contract identity", + required_interval.from_block(), + required_interval.through_block() + ); + } } + record_progress(pool, &mut progress).await?; } Ok(targets.into_iter().collect()) } +async fn load_historical_with_optional_progress( + pool: &PgPool, + chain: &str, + source_families: &[String], + progress: &mut Option<&mut dyn ManifestRuntimeProgress>, +) -> Result> { + match progress.as_deref_mut() { + Some(progress) => { + load_historical_watched_contracts_scoped_with_progress( + pool, + chain, + source_families, + progress, + ) + .await + } + None => load_historical_watched_contracts_by_chain(pool, chain).await, + } +} + +async fn record_progress( + pool: &PgPool, + progress: &mut Option<&mut dyn ManifestRuntimeProgress>, +) -> Result<()> { + if let Some(progress) = progress.as_deref_mut() { + progress.record(pool).await?; + } + Ok(()) +} + +async fn record_tail_progress( + pool: &PgPool, + progress: &mut Option<&mut dyn ManifestRuntimeProgress>, + row_count: usize, +) -> Result<()> { + if row_count > 0 && !row_count.is_multiple_of(super::WATCHED_PLAN_PROGRESS_ROWS) { + record_progress(pool, progress).await?; + } + Ok(()) +} + fn retained_requirement_is_covered( required: InclusiveBlockInterval, covered: impl IntoIterator, diff --git a/crates/manifests/src/lib/views/watched/intervals.rs b/crates/manifests/src/lib/views/watched/intervals.rs index a32ecf8c..7cf5e3c4 100644 --- a/crates/manifests/src/lib/views/watched/intervals.rs +++ b/crates/manifests/src/lib/views/watched/intervals.rs @@ -9,6 +9,7 @@ pub(super) const WATCHED_INTERVALS_CTES: &str = r#" manifest_watched_intervals AS ( SELECT + cia.contract_instance_address_id AS source_row_id, mv.chain AS chain, mv.source_family AS source_family, LOWER(cia.address) AS address, @@ -58,6 +59,7 @@ manifest_watched_intervals AS ( ), discovery_watched_intervals AS ( SELECT + de.discovery_edge_id AS source_row_id, de.chain_id AS chain, COALESCE(target_mv.source_family, mv.source_family) AS source_family, LOWER(cia.address) AS address, @@ -135,9 +137,35 @@ discovery_watched_intervals AS ( AND cia.chain_id = de.chain_id ), watched_intervals AS ( - SELECT * FROM manifest_watched_intervals + SELECT + chain, + source_family, + address, + contract_instance_id, + source, + source_manifest_id, + active_from_block_number, + active_to_block_number, + rollout_eligible, + interval_eligible, + current_eligible, + historical_eligible + FROM manifest_watched_intervals UNION - SELECT * FROM discovery_watched_intervals + SELECT + chain, + source_family, + address, + contract_instance_id, + source, + source_manifest_id, + active_from_block_number, + active_to_block_number, + rollout_eligible, + interval_eligible, + current_eligible, + historical_eligible + FROM discovery_watched_intervals ) "#; @@ -156,3 +184,16 @@ AND watched.historical_eligible pub(super) fn with_watched_intervals(query_body: &str) -> String { format!("WITH\n{WATCHED_INTERVALS_CTES}\n{query_body}") } + +/// Stream the two disjoint source kinds without forcing PostgreSQL to +/// materialize and sort the multi-million-row union before returning its +/// first row. The progress-aware caller restores exact `UNION` deduplication +/// in a sorted set while rows arrive. +pub(super) fn with_streaming_watched_intervals(query_body: &str) -> String { + let streaming_ctes = WATCHED_INTERVALS_CTES.replacen( + "\n FROM manifest_watched_intervals\n UNION\n", + "\n FROM manifest_watched_intervals\n UNION ALL\n", + 1, + ); + format!("WITH\n{streaming_ctes}\n{query_body}") +} diff --git a/crates/manifests/src/lib/views/watched/selection.rs b/crates/manifests/src/lib/views/watched/selection.rs index b45a304a..48be78fe 100644 --- a/crates/manifests/src/lib/views/watched/selection.rs +++ b/crates/manifests/src/lib/views/watched/selection.rs @@ -10,10 +10,23 @@ use crate::{ }; use super::{ - load_historical_watched_contracts_by_chain, load_manifest_declared_watched_contracts, - load_watched_contracts, load_watched_contracts_by_source_family, + ManifestRuntimeProgress, load_historical_watched_contracts_by_chain, + load_manifest_declared_watched_contracts, load_watched_contracts, + load_watched_contracts_by_source_family, load_watched_contracts_with_progress, }; +#[path = "selection/progress.rs"] +mod progress; +pub use progress::load_historical_watched_source_selector_plan_with_progress; + +#[derive(Default)] +struct ChainPlanAccumulator { + addresses: BTreeSet, + manifest_root_entry_count: usize, + manifest_contract_entry_count: usize, + discovery_edge_entry_count: usize, +} + pub fn summarize_watched_contracts( watched_contracts: &[WatchedContract], ) -> WatchedContractSummary { @@ -75,14 +88,6 @@ pub fn summarize_watched_contracts( } pub fn plan_watched_contracts(watched_contracts: &[WatchedContract]) -> Vec { - #[derive(Default)] - struct ChainPlanAccumulator { - addresses: BTreeSet, - manifest_root_entry_count: usize, - manifest_contract_entry_count: usize, - discovery_edge_entry_count: usize, - } - let mut plans = BTreeMap::::new(); for watched_contract in watched_contracts { @@ -124,6 +129,134 @@ pub fn plan_watched_contracts(watched_contracts: &[WatchedContract]) -> Vec Result { + let mut unique_contracts = HashSet::new(); + let mut unique_addresses_by_chain = BTreeMap::>::new(); + let mut chains = BTreeMap::::new(); + let mut manifest_root_count = 0; + let mut manifest_contract_count = 0; + let mut discovery_edge_count = 0; + + for (index, watched_contract) in watched_contracts.iter().enumerate() { + unique_contracts.insert(( + watched_contract.chain.clone(), + watched_contract.address.clone(), + )); + unique_addresses_by_chain + .entry(watched_contract.chain.clone()) + .or_default() + .insert(watched_contract.address.clone()); + let chain_summary = chains + .entry(watched_contract.chain.clone()) + .or_insert_with(|| WatchedContractChainSummary { + chain: watched_contract.chain.clone(), + unique_contract_count: 0, + manifest_root_count: 0, + manifest_contract_count: 0, + discovery_edge_count: 0, + }); + match watched_contract.source { + WatchedContractSource::ManifestRoot => { + manifest_root_count += 1; + chain_summary.manifest_root_count += 1; + } + WatchedContractSource::ManifestContract => { + manifest_contract_count += 1; + chain_summary.manifest_contract_count += 1; + } + WatchedContractSource::DiscoveryEdge => { + discovery_edge_count += 1; + chain_summary.discovery_edge_count += 1; + } + } + if (index + 1).is_multiple_of(super::WATCHED_PLAN_PROGRESS_ROWS) { + progress.record(pool).await?; + } + } + for (chain, addresses) in unique_addresses_by_chain { + chains + .get_mut(&chain) + .expect("unique address chain must have a summary") + .unique_contract_count = addresses.len(); + } + if !watched_contracts.is_empty() + && !watched_contracts + .len() + .is_multiple_of(super::WATCHED_PLAN_PROGRESS_ROWS) + { + progress.record(pool).await?; + } + + Ok(WatchedContractSummary { + unique_contract_count: unique_contracts.len(), + source_entry_count: watched_contracts.len(), + manifest_root_count, + manifest_contract_count, + discovery_edge_count, + chains: chains.into_values().collect(), + }) +} + +async fn plan_watched_contracts_with_progress( + pool: &sqlx::PgPool, + watched_contracts: &[WatchedContract], + progress: &mut dyn ManifestRuntimeProgress, +) -> Result> { + let mut plans = BTreeMap::::new(); + for (index, watched_contract) in watched_contracts.iter().enumerate() { + let plan = plans.entry(watched_contract.chain.clone()).or_default(); + plan.addresses.insert(watched_contract.address.clone()); + match watched_contract.source { + WatchedContractSource::ManifestRoot => plan.manifest_root_entry_count += 1, + WatchedContractSource::ManifestContract => plan.manifest_contract_entry_count += 1, + WatchedContractSource::DiscoveryEdge => plan.discovery_edge_entry_count += 1, + } + if (index + 1).is_multiple_of(super::WATCHED_PLAN_PROGRESS_ROWS) { + progress.record(pool).await?; + } + } + if !watched_contracts.is_empty() + && !watched_contracts + .len() + .is_multiple_of(super::WATCHED_PLAN_PROGRESS_ROWS) + { + progress.record(pool).await?; + } + + let mut result = Vec::with_capacity(plans.len()); + for (chain, accumulator) in plans { + let mut addresses = Vec::with_capacity(accumulator.addresses.len()); + for address in accumulator.addresses { + addresses.push(address); + if addresses + .len() + .is_multiple_of(super::WATCHED_PLAN_PROGRESS_ROWS) + { + progress.record(pool).await?; + } + } + if !addresses.is_empty() + && !addresses + .len() + .is_multiple_of(super::WATCHED_PLAN_PROGRESS_ROWS) + { + progress.record(pool).await?; + } + result.push(WatchedChainPlan { + chain, + addresses, + manifest_root_entry_count: accumulator.manifest_root_entry_count, + manifest_contract_entry_count: accumulator.manifest_contract_entry_count, + discovery_edge_entry_count: accumulator.discovery_edge_entry_count, + }); + } + Ok(result) +} + pub fn resolve_watched_source_selector( watched_contracts: &[WatchedContract], chain: &str, @@ -372,6 +505,17 @@ pub async fn load_watched_contract_summary_and_chain_plan( )) } +pub async fn load_watched_contract_summary_and_chain_plan_with_progress( + pool: &sqlx::PgPool, + progress: &mut dyn ManifestRuntimeProgress, +) -> Result<(WatchedContractSummary, Vec)> { + let watched_contracts = load_watched_contracts_with_progress(pool, progress).await?; + let summary = + summarize_watched_contracts_with_progress(pool, &watched_contracts, progress).await?; + let plan = plan_watched_contracts_with_progress(pool, &watched_contracts, progress).await?; + Ok((summary, plan)) +} + pub async fn load_manifest_declared_watched_chain_plan( pool: &sqlx::PgPool, ) -> Result> { diff --git a/crates/manifests/src/lib/views/watched/selection/progress.rs b/crates/manifests/src/lib/views/watched/selection/progress.rs new file mode 100644 index 00000000..43eadf5a --- /dev/null +++ b/crates/manifests/src/lib/views/watched/selection/progress.rs @@ -0,0 +1,259 @@ +use std::collections::{BTreeMap, BTreeSet}; + +use anyhow::{Result, bail}; +use uuid::Uuid; + +use crate::{ + ManifestRuntimeProgress, WatchedBackfillTarget, WatchedChainPlan, WatchedContract, + WatchedSourceSelector, WatchedSourceSelectorPlan, WatchedTargetIdentity, + load_historical_watched_contracts_by_chain_with_progress, +}; + +pub async fn load_historical_watched_source_selector_plan_with_progress( + pool: &sqlx::PgPool, + chain: &str, + selector: WatchedSourceSelector, + range_start_block_number: i64, + range_end_block_number: i64, + progress: &mut dyn ManifestRuntimeProgress, +) -> Result { + let watched_contracts = + load_historical_watched_contracts_by_chain_with_progress(pool, chain, progress).await?; + resolve_watched_source_selector_with_progress( + pool, + &watched_contracts, + chain, + selector, + range_start_block_number, + range_end_block_number, + progress, + ) + .await +} + +async fn resolve_watched_source_selector_with_progress( + pool: &sqlx::PgPool, + watched_contracts: &[WatchedContract], + chain: &str, + selector: WatchedSourceSelector, + range_start_block_number: i64, + range_end_block_number: i64, + progress: &mut dyn ManifestRuntimeProgress, +) -> Result { + if range_start_block_number < 0 { + bail!("watched source selector range start must be non-negative"); + } + if range_end_block_number < 0 { + bail!("watched source selector range end must be non-negative"); + } + if range_start_block_number > range_end_block_number { + bail!( + "watched source selector range start {range_start_block_number} is after end {range_end_block_number}" + ); + } + + let selector_kind = selector.kind(); + let source_family = match &selector { + WatchedSourceSelector::SourceFamily(source_family) => Some(source_family.clone()), + _ => None, + }; + let requested_watched_targets = normalize_requested_targets(pool, &selector, progress).await?; + let requested_target_ids = requested_watched_targets + .iter() + .map(|target| target.contract_instance_id) + .collect::>(); + + let mut selected_contracts = Vec::new(); + for (index, watched_contract) in watched_contracts.iter().enumerate() { + let selected = watched_contract.chain == chain + && super::watched_contract_range_intersects( + watched_contract, + range_start_block_number, + range_end_block_number, + ) + && match &selector { + WatchedSourceSelector::WholeActiveWatchedChain => true, + WatchedSourceSelector::SourceFamily(source_family) => { + watched_contract.source_family == *source_family + } + WatchedSourceSelector::WatchedTargetSet(_) => { + requested_target_ids.contains(&watched_contract.contract_instance_id) + } + }; + if selected { + selected_contracts.push(watched_contract.clone()); + } + record_every(pool, progress, index + 1).await?; + } + record_tail(pool, progress, watched_contracts.len()).await?; + + validate_selection( + pool, + chain, + &selector, + &requested_watched_targets, + &selected_contracts, + progress, + ) + .await?; + let selected_targets = selected_backfill_targets_with_progress( + pool, + &selected_contracts, + range_start_block_number, + range_end_block_number, + progress, + ) + .await?; + let watched_chain_plan = + super::plan_watched_contracts_with_progress(pool, &selected_contracts, progress) + .await? + .into_iter() + .next() + .unwrap_or_else(|| WatchedChainPlan { + chain: chain.to_owned(), + addresses: Vec::new(), + manifest_root_entry_count: 0, + manifest_contract_entry_count: 0, + discovery_edge_entry_count: 0, + }); + + Ok(WatchedSourceSelectorPlan { + chain: chain.to_owned(), + selector_kind, + source_family, + requested_watched_targets, + selected_targets, + watched_chain_plan, + }) +} + +async fn normalize_requested_targets( + pool: &sqlx::PgPool, + selector: &WatchedSourceSelector, + progress: &mut dyn ManifestRuntimeProgress, +) -> Result> { + let targets = match selector { + WatchedSourceSelector::WatchedTargetSet(targets) => targets, + _ => return Ok(Vec::new()), + }; + let mut unique = BTreeSet::new(); + for (index, target) in targets.iter().enumerate() { + unique.insert(target.clone()); + record_every(pool, progress, index + 1).await?; + } + record_tail(pool, progress, targets.len()).await?; + Ok(unique.into_iter().collect()) +} + +async fn validate_selection( + pool: &sqlx::PgPool, + chain: &str, + selector: &WatchedSourceSelector, + requested: &[WatchedTargetIdentity], + selected: &[WatchedContract], + progress: &mut dyn ManifestRuntimeProgress, +) -> Result<()> { + match selector { + WatchedSourceSelector::WholeActiveWatchedChain if selected.is_empty() => bail!( + "watched source selector whole_active_watched_chain found no active watched targets for chain {chain}" + ), + WatchedSourceSelector::SourceFamily(source_family) if selected.is_empty() => bail!( + "watched source selector source_family {source_family} found no active watched targets for chain {chain}" + ), + WatchedSourceSelector::WatchedTargetSet(_) => { + if requested.is_empty() { + bail!("watched_target_set selector must include at least one contract_instance_id"); + } + let selected_ids = selected + .iter() + .map(|contract| contract.contract_instance_id) + .collect::>(); + for (index, requested_target) in requested.iter().enumerate() { + if !selected_ids.contains(&requested_target.contract_instance_id) { + bail!( + "watched target {} is not active for chain {chain} in the selected range", + requested_target.contract_instance_id + ); + } + record_every(pool, progress, index + 1).await?; + } + record_tail(pool, progress, requested.len()).await?; + } + _ => {} + } + Ok(()) +} + +async fn selected_backfill_targets_with_progress( + pool: &sqlx::PgPool, + watched_contracts: &[WatchedContract], + range_start_block_number: i64, + range_end_block_number: i64, + progress: &mut dyn ManifestRuntimeProgress, +) -> Result> { + let mut addresses_by_identity = BTreeMap::<(String, Uuid), String>::new(); + let mut selected_targets = BTreeSet::new(); + for (index, watched_contract) in watched_contracts.iter().enumerate() { + let Some((effective_from_block, effective_to_block)) = + super::watched_contract_effective_range( + watched_contract, + range_start_block_number, + range_end_block_number, + ) + else { + continue; + }; + let target = WatchedBackfillTarget { + source_family: watched_contract.source_family.clone(), + contract_instance_id: watched_contract.contract_instance_id, + address: watched_contract.address.clone(), + effective_from_block, + effective_to_block, + }; + let identity = (target.source_family.clone(), target.contract_instance_id); + if let Some(existing_address) = addresses_by_identity.get(&identity) { + if existing_address != &target.address { + bail!( + "source identity conflict for watched target {} in source family {}", + target.contract_instance_id, + target.source_family + ); + } + } else { + addresses_by_identity.insert(identity, target.address.clone()); + } + selected_targets.insert(target); + record_every(pool, progress, index + 1).await?; + } + record_tail(pool, progress, watched_contracts.len()).await?; + + let mut result = Vec::with_capacity(selected_targets.len()); + for target in selected_targets { + result.push(target); + record_every(pool, progress, result.len()).await?; + } + record_tail(pool, progress, result.len()).await?; + Ok(result) +} + +async fn record_every( + pool: &sqlx::PgPool, + progress: &mut dyn ManifestRuntimeProgress, + completed: usize, +) -> Result<()> { + if completed.is_multiple_of(super::super::WATCHED_PLAN_PROGRESS_ROWS) { + progress.record(pool).await?; + } + Ok(()) +} + +async fn record_tail( + pool: &sqlx::PgPool, + progress: &mut dyn ManifestRuntimeProgress, + completed: usize, +) -> Result<()> { + if completed > 0 && !completed.is_multiple_of(super::super::WATCHED_PLAN_PROGRESS_ROWS) { + progress.record(pool).await?; + } + Ok(()) +} diff --git a/crates/storage/src/backfill_jobs.rs b/crates/storage/src/backfill_jobs.rs index 758cbbb1..07da3b7d 100644 --- a/crates/storage/src/backfill_jobs.rs +++ b/crates/storage/src/backfill_jobs.rs @@ -12,9 +12,11 @@ mod validate; pub use complete::{ complete_backfill_job, complete_backfill_range, complete_backfill_range_recording_coverage, + complete_backfill_range_recording_coverage_with_progress, }; pub use coverage_facts::{ - BackfillCoverageFactDerivation, BackfillCoverageFactScope, BackfillCoverageFactWrite, + BackfillCoverageFactDerivation, BackfillCoverageFactScope, BackfillCoverageFactStreamItem, + BackfillCoverageFactWrite, BackfillCoverageProgress, BackfillCoverageProgressFuture, load_backfill_coverage_fact_counts, write_backfill_coverage_facts, }; pub use create::{ diff --git a/crates/storage/src/backfill_jobs/complete.rs b/crates/storage/src/backfill_jobs/complete.rs index 9effe643..d46c111c 100644 --- a/crates/storage/src/backfill_jobs/complete.rs +++ b/crates/storage/src/backfill_jobs/complete.rs @@ -5,7 +5,8 @@ use tracing::warn; use super::{ coverage_facts::{ - BackfillCoverageFactDerivation, BackfillCoverageFactWrite, + BackfillCoverageFactDerivation, BackfillCoverageFactStreamItem, BackfillCoverageFactWrite, + BackfillCoverageProgress, write_backfill_coverage_fact_stream, write_backfill_coverage_facts_from_iter, }, decode::{decode_backfill_job, decode_backfill_range}, @@ -119,6 +120,92 @@ where Ok(range) } +/// Progress-aware completion for whole-active plans. The stream may emit +/// progress markers for examined inputs that do not produce facts; database +/// insert chunks also trigger the supplied hook. +pub async fn complete_backfill_range_recording_coverage_with_progress( + pool: &PgPool, + backfill_range_id: i64, + lease_token: &str, + coverage_facts: F, + progress: &mut dyn BackfillCoverageProgress, +) -> Result +where + F: FnOnce(&BackfillJob) -> I, + I: Iterator, +{ + validate_non_empty("lease_token", lease_token)?; + + let mut transaction = pool + .begin() + .await + .context("failed to open transaction for backfill range completion")?; + let backfill_job_id = load_backfill_range_job_id(&mut *transaction, backfill_range_id) + .await? + .with_context(|| format!("missing backfill range {backfill_range_id}"))?; + load_backfill_job_for_update(&mut *transaction, backfill_job_id) + .await? + .with_context(|| { + format!("missing backfill job {backfill_job_id} for range {backfill_range_id}") + })?; + let current = load_backfill_range_for_update(&mut *transaction, backfill_range_id) + .await? + .with_context(|| format!("missing backfill range {backfill_range_id}"))?; + if current.status == BackfillLifecycleStatus::Completed { + transaction + .commit() + .await + .context("failed to commit completed backfill range no-op")?; + return Ok(current); + } + if current.status == BackfillLifecycleStatus::Failed { + bail!("failed backfill range {backfill_range_id} must be reserved again before completion"); + } + ensure_lease_matches(¤t, lease_token)?; + if current.checkpoint_block_number != current.range_end_block_number { + bail!( + "backfill range {backfill_range_id} checkpoint {} has not reached declared range end {}", + current.checkpoint_block_number, + current.range_end_block_number + ); + } + + let complete_sql = backfill_range_returning_sql( + r#" + UPDATE backfill_ranges + SET + status = 'completed'::backfill_lifecycle_status, + lease_token = NULL, + lease_owner = NULL, + lease_expires_at = NULL, + failure_reason = NULL, + failure_metadata = '{}'::jsonb, + completed_at = COALESCE(completed_at, now()), + updated_at = now() + WHERE backfill_range_id = $1 + "#, + ); + let range = sqlx::query(&complete_sql) + .bind(backfill_range_id) + .fetch_one(&mut *transaction) + .await + .with_context(|| format!("failed to complete backfill range {backfill_range_id}"))?; + let range = decode_backfill_range(range)?; + + maybe_complete_backfill_job_with_progress( + &mut transaction, + range.backfill_job_id, + coverage_facts, + progress, + ) + .await?; + transaction + .commit() + .await + .context("failed to commit backfill range completion")?; + Ok(range) +} + /// Complete a job when all child range checkpoints have reached their declared /// ends. This is idempotent when the job is already complete. pub async fn complete_backfill_job(pool: &PgPool, backfill_job_id: i64) -> Result { @@ -255,6 +342,40 @@ where Ok(()) } +async fn maybe_complete_backfill_job_with_progress( + executor: &mut sqlx::Transaction<'_, Postgres>, + backfill_job_id: i64, + coverage_facts: F, + progress: &mut dyn BackfillCoverageProgress, +) -> Result<()> +where + F: FnOnce(&BackfillJob) -> I, + I: Iterator, +{ + let incomplete_count = incomplete_range_count(&mut **executor, backfill_job_id).await?; + if incomplete_count != 0 { + return Ok(()); + } + + let job = set_backfill_job_completed(executor, backfill_job_id).await?; + let inserted_fact_count = write_backfill_coverage_fact_stream( + executor, + job.backfill_job_id, + BackfillCoverageFactDerivation::JobCompletion, + coverage_facts(&job), + &mut Some(progress), + ) + .await?; + if inserted_fact_count == 0 { + warn_backfill_job_coverage_fact_gap( + &job, + "complete_backfill_range", + "backfill job completion recorded zero coverage facts", + ); + } + Ok(()) +} + pub(super) async fn set_backfill_job_completed( executor: &mut sqlx::Transaction<'_, Postgres>, backfill_job_id: i64, diff --git a/crates/storage/src/backfill_jobs/coverage_facts.rs b/crates/storage/src/backfill_jobs/coverage_facts.rs index df7aefb1..84628374 100644 --- a/crates/storage/src/backfill_jobs/coverage_facts.rs +++ b/crates/storage/src/backfill_jobs/coverage_facts.rs @@ -1,3 +1,5 @@ +use std::{future::Future, pin::Pin}; + use anyhow::{Context, Result, bail}; use sqlx::{PgConnection, PgPool, Postgres, QueryBuilder}; @@ -53,6 +55,23 @@ pub struct BackfillCoverageFactWrite { pub covered_to_block: i64, } +/// One item consumed while a large in-memory backfill plan is converted into +/// durable coverage. `Progress` means the source iterator examined another +/// bounded unit even when that unit did not yield a fact (for example, an +/// address covered by a family-wide scan). +pub enum BackfillCoverageFactStreamItem { + Fact(BackfillCoverageFactWrite), + Progress, +} + +pub type BackfillCoverageProgressFuture<'a> = Pin> + Send + 'a>>; + +/// Progress hook for whole-plan coverage publication. Callers must only emit a +/// hook after input was consumed or a database insert chunk completed. +pub trait BackfillCoverageProgress: Send { + fn record<'a>(&'a mut self) -> BackfillCoverageProgressFuture<'a>; +} + struct BackfillCoverageAuthority { chain_id: String, status: String, @@ -105,23 +124,56 @@ pub(super) async fn write_backfill_coverage_facts_from_iter( backfill_job_id: i64, derivation: BackfillCoverageFactDerivation, facts: impl Iterator, +) -> Result { + write_backfill_coverage_fact_stream( + conn, + backfill_job_id, + derivation, + facts.map(BackfillCoverageFactStreamItem::Fact), + &mut None, + ) + .await +} + +pub(super) async fn write_backfill_coverage_fact_stream( + conn: &mut PgConnection, + backfill_job_id: i64, + derivation: BackfillCoverageFactDerivation, + facts: impl Iterator, + progress: &mut Option<&mut dyn BackfillCoverageProgress>, ) -> Result { let mut inserted = 0_u64; let mut chunk = Vec::new(); let authority = load_backfill_job_coverage_authority(conn, backfill_job_id).await?; - for fact in facts { - chunk.push(fact); - if chunk.len() == COVERAGE_FACT_INSERT_CHUNK_ROWS { - validate_backfill_job_coverage_authority(backfill_job_id, &authority, chunk.iter())?; - inserted += insert_backfill_coverage_fact_chunk( - conn, - backfill_job_id, - &authority.chain_id, - derivation, - &chunk, - ) - .await?; - chunk.clear(); + for item in facts { + match item { + BackfillCoverageFactStreamItem::Fact(fact) => { + chunk.push(fact); + if chunk.len() == COVERAGE_FACT_INSERT_CHUNK_ROWS { + validate_backfill_job_coverage_authority( + backfill_job_id, + &authority, + chunk.iter(), + )?; + inserted += insert_backfill_coverage_fact_chunk( + conn, + backfill_job_id, + &authority.chain_id, + derivation, + &chunk, + ) + .await?; + chunk.clear(); + if let Some(progress) = progress.as_deref_mut() { + progress.record().await?; + } + } + } + BackfillCoverageFactStreamItem::Progress => { + if let Some(progress) = progress.as_deref_mut() { + progress.record().await?; + } + } } } if !chunk.is_empty() { @@ -134,6 +186,9 @@ pub(super) async fn write_backfill_coverage_facts_from_iter( &chunk, ) .await?; + if let Some(progress) = progress.as_deref_mut() { + progress.record().await?; + } } Ok(inserted) } diff --git a/crates/storage/src/execution.rs b/crates/storage/src/execution.rs index fa87b37c..6d0792cd 100644 --- a/crates/storage/src/execution.rs +++ b/crates/storage/src/execution.rs @@ -10,9 +10,11 @@ mod trace_validation; mod types; pub use invalidation::{ + ExecutionOutcomeInvalidationProgress, ExecutionOutcomeInvalidationProgressFuture, invalidate_execution_outcomes_for_manifest_version, invalidate_execution_outcomes_for_manifest_version_and_request_key, invalidate_execution_outcomes_for_orphaned_blocks, + invalidate_execution_outcomes_for_orphaned_blocks_with_progress, invalidate_execution_outcomes_for_record_boundary, invalidate_execution_outcomes_for_record_boundary_and_request_key, invalidate_execution_outcomes_for_topology_boundary, diff --git a/crates/storage/src/execution/invalidation.rs b/crates/storage/src/execution/invalidation.rs index 50900a6c..1ced3c87 100644 --- a/crates/storage/src/execution/invalidation.rs +++ b/crates/storage/src/execution/invalidation.rs @@ -1,4 +1,4 @@ -use std::collections::BTreeSet; +use std::{collections::BTreeSet, future::Future, pin::Pin}; use anyhow::{Context, Result, bail}; use serde_json::Value; @@ -22,9 +22,18 @@ const REORG_INVALIDATION_BATCH_SIZE: i64 = 500; #[cfg(test)] const REORG_INVALIDATION_BATCH_SIZE: i64 = 2; +pub type ExecutionOutcomeInvalidationProgressFuture<'a> = + Pin> + Send + 'a>>; + +pub trait ExecutionOutcomeInvalidationProgress: Send { + fn record<'a>(&'a mut self, pool: &'a PgPool) + -> ExecutionOutcomeInvalidationProgressFuture<'a>; +} + #[derive(Clone, Debug)] struct ExecutionOutcomeReorgInvalidationCandidate { execution_cache_key: String, + request_type: String, request_key: String, requested_chain_positions: Value, topology_version_boundary: Value, @@ -180,13 +189,26 @@ pub async fn invalidate_execution_outcomes_for_record_boundary_and_request_key( /// cache dependencies reference a block identity marked `orphaned`. pub async fn invalidate_execution_outcomes_for_orphaned_blocks( pool: &PgPool, +) -> Result { + invalidate_execution_outcomes_for_orphaned_blocks_inner(pool, &mut None).await +} + +pub async fn invalidate_execution_outcomes_for_orphaned_blocks_with_progress( + pool: &PgPool, + progress: &mut dyn ExecutionOutcomeInvalidationProgress, +) -> Result { + invalidate_execution_outcomes_for_orphaned_blocks_inner(pool, &mut Some(progress)).await +} + +async fn invalidate_execution_outcomes_for_orphaned_blocks_inner( + pool: &PgPool, + progress: &mut Option<&mut dyn ExecutionOutcomeInvalidationProgress>, ) -> Result { let mut transaction = pool .begin() .await .context("failed to open transaction for execution reorg invalidation")?; - let orphaned_blocks = load_orphaned_block_dependencies_internal(&mut *transaction).await?; let mut deleted_outcome_count = 0; let mut last_seen_cache_key = None; loop { @@ -201,7 +223,15 @@ pub async fn invalidate_execution_outcomes_for_orphaned_blocks( last_seen_cache_key = Some(last_outcome.execution_cache_key.clone()); let mut cache_keys = Vec::new(); + let mut parsed_dependencies = Vec::new(); + let mut candidate_dependencies = BTreeSet::new(); for outcome in outcomes { + if !matches!( + outcome.request_type.as_str(), + "verified_resolution" | "verified_primary_name" + ) { + continue; + } let dependencies = match execution_outcome_block_dependencies( &outcome.request_key, &outcome.requested_chain_positions, @@ -214,16 +244,24 @@ pub async fn invalidate_execution_outcomes_for_orphaned_blocks( continue; } }; + candidate_dependencies.extend(dependencies.iter().cloned()); + parsed_dependencies.push((outcome.execution_cache_key, dependencies)); + } + let orphaned_dependencies = + load_orphaned_block_dependencies_internal(&mut *transaction, &candidate_dependencies) + .await?; + for (execution_cache_key, dependencies) in parsed_dependencies { if dependencies .iter() - .any(|dependency| orphaned_blocks.contains(dependency)) + .any(|dependency| orphaned_dependencies.contains(dependency)) { - cache_keys.push(outcome.execution_cache_key); + cache_keys.push(execution_cache_key); } } deleted_outcome_count += delete_execution_outcomes_by_keys(&mut transaction, &cache_keys).await?; + record_reorg_invalidation_progress(pool, progress).await?; } transaction @@ -236,6 +274,16 @@ pub async fn invalidate_execution_outcomes_for_orphaned_blocks( }) } +async fn record_reorg_invalidation_progress( + pool: &PgPool, + progress: &mut Option<&mut dyn ExecutionOutcomeInvalidationProgress>, +) -> Result<()> { + if let Some(progress) = progress.as_deref_mut() { + progress.record(pool).await?; + } + Ok(()) +} + impl ExecutionManifestInvalidation { fn identity_key(&self) -> String { manifest_version_identity_key( @@ -399,13 +447,13 @@ where r#" SELECT execution_cache_key, + request_type, request_key, requested_chain_positions, topology_version_boundary, record_version_boundary FROM execution_cache_outcomes - WHERE request_type IN ('verified_resolution', 'verified_primary_name') - AND ($1::text IS NULL OR execution_cache_key > $1) + WHERE ($1::text IS NULL OR execution_cache_key > $1) ORDER BY execution_cache_key LIMIT $2 "#, @@ -423,21 +471,37 @@ where async fn load_orphaned_block_dependencies_internal<'e, E>( executor: E, + dependencies: &BTreeSet<(String, String)>, ) -> Result> where E: Executor<'e, Database = Postgres>, { + if dependencies.is_empty() { + return Ok(BTreeSet::new()); + } + let chains = dependencies + .iter() + .map(|(chain, _)| chain.as_str()) + .collect::>(); + let block_hashes = dependencies + .iter() + .map(|(_, block_hash)| block_hash.as_str()) + .collect::>(); let rows = sqlx::query( r#" - SELECT chain_id, block_hash - FROM chain_lineage - WHERE canonicality_state = 'orphaned'::canonicality_state - ORDER BY chain_id, block_hash + SELECT lineage.chain_id, lineage.block_hash + FROM UNNEST($1::TEXT[], $2::TEXT[]) dependency(chain_id, block_hash) + JOIN chain_lineage lineage + ON lineage.chain_id = dependency.chain_id + AND lineage.block_hash = dependency.block_hash + WHERE lineage.canonicality_state = 'orphaned'::canonicality_state "#, ) + .bind(chains) + .bind(block_hashes) .fetch_all(executor) .await - .context("failed to load orphaned block identities for execution reorg invalidation")?; + .context("failed to match execution dependencies against orphaned block identities")?; rows.into_iter() .map(|row| { @@ -481,6 +545,9 @@ fn decode_execution_outcome_reorg_invalidation_candidate_row( execution_cache_key: row .try_get("execution_cache_key") .context("execution outcome row missing execution_cache_key")?, + request_type: row + .try_get("request_type") + .context("execution outcome row missing request_type")?, request_key: row .try_get("request_key") .context("execution outcome row missing request_key")?, diff --git a/crates/storage/src/execution/tests.rs b/crates/storage/src/execution/tests.rs index 7464e8cb..caf76be3 100644 --- a/crates/storage/src/execution/tests.rs +++ b/crates/storage/src/execution/tests.rs @@ -19,6 +19,23 @@ use crate::{ static NEXT_TEST_ID: AtomicU64 = AtomicU64::new(0); +#[derive(Default)] +struct CountingExecutionInvalidationProgress { + count: usize, +} + +impl ExecutionOutcomeInvalidationProgress for CountingExecutionInvalidationProgress { + fn record<'a>( + &'a mut self, + _pool: &'a PgPool, + ) -> ExecutionOutcomeInvalidationProgressFuture<'a> { + Box::pin(async move { + self.count += 1; + Ok(()) + }) + } +} + #[test] fn migration_prunes_noncanonical_addr_selector_execution_artifacts() { let migration = include_str!( @@ -2619,8 +2636,17 @@ async fn invalidates_verified_execution_outcomes_for_orphaned_block_dependencies }]); insert_trace_and_outcome(&database, &out_of_scope_trace, &out_of_scope_outcome).await?; - let summary = invalidate_execution_outcomes_for_orphaned_blocks(database.pool()).await?; + let mut progress = CountingExecutionInvalidationProgress::default(); + let summary = invalidate_execution_outcomes_for_orphaned_blocks_with_progress( + database.pool(), + &mut progress, + ) + .await?; assert_eq!(summary.deleted_outcome_count, 3); + assert!( + progress.count >= 3, + "reorg invalidation must report each completed cache-candidate source page" + ); assert_eq!( load_execution_outcome(database.pool(), &requested_outcome.cache_key).await?, diff --git a/crates/storage/src/lib.rs b/crates/storage/src/lib.rs index f66d5fa3..6680caa6 100644 --- a/crates/storage/src/lib.rs +++ b/crates/storage/src/lib.rs @@ -1,7 +1,5 @@ //! Shared PostgreSQL storage and migration utilities. - use std::{str::FromStr, time::Duration}; - mod address_names; mod audit; mod backfill_jobs; @@ -38,16 +36,6 @@ pub mod sql_row; mod stored_lineage_coverage; mod time; mod versions; - -use anyhow::{Context, Result, ensure}; -use clap::Args; -use sqlx::{ - PgPool, Postgres, - pool::PoolConnection, - postgres::{PgConnectOptions, PgPoolOptions}, -}; -use tracing::info; - pub use address_names::{ AddressNameCurrentEntry, AddressNameCurrentRow, AddressNameRelation, AddressNamesCurrentAddressReplacement, AddressNamesCurrentCountFilter, @@ -68,6 +56,7 @@ pub use address_names::{ rebuild_address_names_current_identity_sidecars, replace_address_names_current_logical_names, upsert_address_names_current_rows, }; +use anyhow::{Context, Result, ensure}; pub use audit::{ CanonicalityInspection, CanonicalityInspectionStatus, ManifestDriftAlertInspection, ManifestDriftAlertKind, ManifestDriftAlertObservation, RawFactAuditCounts, @@ -75,18 +64,7 @@ pub use audit::{ inspect_canonicality_range, list_manifest_drift_alert_observations, list_raw_payload_cache_audit_metadata, list_stored_lineage_range, }; -pub use backfill_jobs::{ - BackfillCoverageFactDerivation, BackfillCoverageFactScope, BackfillCoverageFactWrite, - BackfillJob, BackfillJobCreate, BackfillJobRecord, BackfillLifecycleStatus, BackfillRange, - BackfillRangeSpec, BackfillTopicCoverageRequirement, BackfillTopicCoverageViolation, - MAX_BACKFILL_TOPIC_EVIDENCE_REQUIREMENTS, advance_backfill_range, complete_backfill_job, - complete_backfill_range, complete_backfill_range_recording_coverage, create_backfill_job, - create_generation_scoped_backfill_job, ensure_and_load_raw_log_retention_generation, - fail_backfill_job, fail_backfill_range, find_backfill_topic_coverage_violations, - load_backfill_coverage_fact_counts, load_backfill_job, load_backfill_ranges, - load_completed_backfill_jobs_intersecting_range, materialize_completed_backfill_topic_evidence, - reserve_backfill_range, write_backfill_coverage_facts, -}; +mod long_operation_exports; pub use base_normalized_rederive::{ BASE_NORMALIZED_REDERIVE_ADAPTER, BASE_NORMALIZED_REDERIVE_BACKLOG_CURSOR_KIND, BASE_NORMALIZED_REDERIVE_CHAIN_ID, BASE_NORMALIZED_REDERIVE_CURSOR_KIND, @@ -125,6 +103,7 @@ pub use children::{ load_children_current_page, load_children_current_summaries, stream_canonical_declared_child_sources, upsert_children_current_rows, }; +use clap::Args; pub use evm_primitives::{ens_namehash_label_bytes, normalize_evm_address, normalize_evm_b256}; pub use execution::{ ExecutionBoundaryInvalidation, ExecutionCacheKey, ExecutionManifestInvalidation, @@ -143,6 +122,15 @@ pub use execution::{ upsert_execution_outcome, upsert_execution_outcome_in_transaction, upsert_execution_trace, upsert_execution_trace_in_transaction, }; +pub use long_operation_exports::*; +use sqlx::{ + PgPool, Postgres, + pool::PoolConnection, + postgres::{PgConnectOptions, PgPoolOptions}, +}; +use tracing::info; +#[rustfmt::skip] +pub use execution::{ExecutionOutcomeInvalidationProgress, ExecutionOutcomeInvalidationProgressFuture, invalidate_execution_outcomes_for_orphaned_blocks_with_progress}; pub use history::{ EventHistoryAddressFilter, EventHistoryFilter, HistoryChainPositionSample, HistoryCursor, HistoryEvent, HistoryPage, HistoryScope, HistorySummary, HistorySummaryMode, @@ -311,6 +299,8 @@ pub use resolver_profile_authority_journal::{ load_resolver_profile_authority_family_target_page, load_resolver_profile_authority_journal, resolver_profile_authority_entry_key, }; +#[rustfmt::skip] +pub use resolver_profile_authority_journal::{ResolverProfileAuthorityJournalProgress, ResolverProfileAuthorityJournalProgressFuture}; pub use resolver_profile_input_changes::{ ResolverProfileInputChange, ResolverProfileReconciliationTarget, acknowledge_resolver_profile_input_changes, enqueue_resolver_profile_reconciliations, @@ -332,14 +322,6 @@ pub use snapshot_selection::{ SnapshotSelectorInput, ensure_projection_chain_positions_match, parse_rfc3339_utc_timestamp, resolve_exact_name_snapshot_selection, }; -pub use stored_lineage_coverage::{ - STORED_LINEAGE_COVERAGE_CANDIDATE_TABLE, STORED_LINEAGE_COVERAGE_PROOF_FORMAT_VERSION, - StoredLineageCoverageFrontierHeader, StoredLineageCoverageFrontierPublication, - StoredLineageCoveragePublicationGuard, StoredLineageCoveragePublicationOutcome, - begin_stored_lineage_coverage_frontier_publication, - load_stored_lineage_coverage_frontier_header, - stored_lineage_coverage_frontier_requirements_are_valid, -}; pub use versions::{CURRENT_PROJECTION_REPLAY_VERSION, latest_migration_version}; /// Checked-in database migrations. diff --git a/crates/storage/src/long_operation_exports.rs b/crates/storage/src/long_operation_exports.rs new file mode 100644 index 00000000..270dc491 --- /dev/null +++ b/crates/storage/src/long_operation_exports.rs @@ -0,0 +1,25 @@ +pub use crate::backfill_jobs::{ + BackfillCoverageFactDerivation, BackfillCoverageFactScope, BackfillCoverageFactStreamItem, + BackfillCoverageFactWrite, BackfillCoverageProgress, BackfillCoverageProgressFuture, + BackfillJob, BackfillJobCreate, BackfillJobRecord, BackfillLifecycleStatus, BackfillRange, + BackfillRangeSpec, BackfillTopicCoverageRequirement, BackfillTopicCoverageViolation, + MAX_BACKFILL_TOPIC_EVIDENCE_REQUIREMENTS, advance_backfill_range, complete_backfill_job, + complete_backfill_range, complete_backfill_range_recording_coverage, + complete_backfill_range_recording_coverage_with_progress, create_backfill_job, + create_generation_scoped_backfill_job, ensure_and_load_raw_log_retention_generation, + fail_backfill_job, fail_backfill_range, find_backfill_topic_coverage_violations, + load_backfill_coverage_fact_counts, load_backfill_job, load_backfill_ranges, + load_completed_backfill_jobs_intersecting_range, materialize_completed_backfill_topic_evidence, + reserve_backfill_range, write_backfill_coverage_facts, +}; + +pub use crate::stored_lineage_coverage::{ + STORED_LINEAGE_COVERAGE_CANDIDATE_TABLE, STORED_LINEAGE_COVERAGE_PROOF_FORMAT_VERSION, + StoredLineageCoverageFrontierHeader, StoredLineageCoverageFrontierPublication, + StoredLineageCoverageProgress, StoredLineageCoverageProgressFuture, + StoredLineageCoveragePublicationGuard, StoredLineageCoveragePublicationOutcome, + begin_stored_lineage_coverage_frontier_publication, + load_stored_lineage_coverage_frontier_header, + stored_lineage_coverage_frontier_requirements_are_valid, + stored_lineage_coverage_frontier_requirements_are_valid_with_progress, +}; diff --git a/crates/storage/src/resolver_profile_authority_journal.rs b/crates/storage/src/resolver_profile_authority_journal.rs index 7ad449f4..e26e98ea 100644 --- a/crates/storage/src/resolver_profile_authority_journal.rs +++ b/crates/storage/src/resolver_profile_authority_journal.rs @@ -9,6 +9,7 @@ mod reads; pub use advance::{ ResolverProfileAuthorityJournalAdvance, ResolverProfileAuthorityJournalAdvanceSummary, + ResolverProfileAuthorityJournalProgress, ResolverProfileAuthorityJournalProgressFuture, }; pub use reads::{ load_resolver_profile_authority_entries_for_targets, diff --git a/crates/storage/src/resolver_profile_authority_journal/advance.rs b/crates/storage/src/resolver_profile_authority_journal/advance.rs index 7dec3337..b503d016 100644 --- a/crates/storage/src/resolver_profile_authority_journal/advance.rs +++ b/crates/storage/src/resolver_profile_authority_journal/advance.rs @@ -1,3 +1,5 @@ +use std::{future::Future, pin::Pin}; + use anyhow::{Context, Result, ensure}; use serde_json::Value; use sqlx::{PgConnection, PgPool, Postgres, QueryBuilder, Transaction}; @@ -15,6 +17,25 @@ mod diff; mod mutations; const RESOLVER_PROFILE_AUTHORITY_TARGET_BATCH_SIZE: usize = 1_000; +const RESOLVER_PROFILE_AUTHORITY_INTERNAL_PAGE_SIZE: usize = 1_000; + +pub type ResolverProfileAuthorityJournalProgressFuture<'a> = + Pin> + Send + 'a>>; + +/// Reports a completed bounded unit inside an authority-journal transaction. +/// Implementations publish liveness only when this callback is invoked. +pub trait ResolverProfileAuthorityJournalProgress: Send { + fn record(&mut self) -> ResolverProfileAuthorityJournalProgressFuture<'_>; +} + +async fn record_journal_progress( + progress: &mut Option<&mut dyn ResolverProfileAuthorityJournalProgress>, +) -> Result<()> { + if let Some(progress) = progress.as_deref_mut() { + progress.record().await?; + } + Ok(()) +} #[derive(Clone, Copy)] pub(super) struct ResolverProfileAuthorityJournalBatchSizes { @@ -138,11 +159,73 @@ impl ResolverProfileAuthorityJournalAdvance { Ok(count) } + pub async fn begin_staged_entry_diff(&mut self) -> Result<()> { + ensure!( + self.changed_entry_count.is_none(), + "resolver-profile authority entry diff is already complete" + ); + diff::create_changed_entry_table(&mut self.transaction).await + } + + pub async fn materialize_staged_entry_diff_page( + &mut self, + after: Option<&str>, + limit: usize, + ) -> Result> { + ensure!( + limit > 0, + "resolver-profile authority diff page limit must be positive" + ); + ensure!( + self.changed_entry_count.is_none(), + "resolver-profile authority entry diff is already complete" + ); + diff::materialize_changed_entry_key_page( + &mut self.transaction, + RESOLVER_PROFILE_AUTHORITY_JOURNAL_KEY, + after, + limit, + ) + .await + } + + pub async fn finish_staged_entry_diff(&mut self, count: i64) -> Result { + ensure!( + count >= 0, + "resolver-profile authority diff count must not be negative" + ); + ensure!( + self.changed_entry_count.is_none(), + "resolver-profile authority entry diff is already complete" + ); + diff::finish_changed_entry_keys(&mut self.transaction).await?; + self.changed_entry_count = Some(count); + self.summary.changed_entry_count = count; + Ok(count) + } + /// Queue the staged diff, apply exact entry mutations in bounded /// statements, and finally compare-and-set the journal header. pub async fn publish( + self, + discovery_epoch_snapshot: &Value, + ) -> Result> { + self.publish_inner(discovery_epoch_snapshot, None).await + } + + pub async fn publish_with_progress( + self, + discovery_epoch_snapshot: &Value, + progress: &mut dyn ResolverProfileAuthorityJournalProgress, + ) -> Result> { + self.publish_inner(discovery_epoch_snapshot, Some(progress)) + .await + } + + async fn publish_inner( mut self, discovery_epoch_snapshot: &Value, + mut progress: Option<&mut dyn ResolverProfileAuthorityJournalProgress>, ) -> Result> { validate_journal_header(self.expected_revision, discovery_epoch_snapshot)?; let changed_entry_count = self.staged_entry_diff_count().await?; @@ -150,15 +233,17 @@ impl ResolverProfileAuthorityJournalAdvance { diff::materialize_reconciliation_targets( &mut self.transaction, RESOLVER_PROFILE_AUTHORITY_JOURNAL_KEY, + &mut progress, ) .await?; - self.enqueue_reconciliation_targets().await?; + self.enqueue_reconciliation_targets(&mut progress).await?; } let mutation_summary = mutations::apply_entry_diff( &mut self.transaction, RESOLVER_PROFILE_AUTHORITY_JOURNAL_KEY, self.batch_sizes.entry_mutation, + &mut progress, ) .await?; self.summary.upserted_entry_count = mutation_summary.upserted_entry_count; @@ -207,7 +292,10 @@ impl ResolverProfileAuthorityJournalAdvance { .context("failed to roll back resolver-profile authority journal capture") } - async fn enqueue_reconciliation_targets(&mut self) -> Result<()> { + async fn enqueue_reconciliation_targets( + &mut self, + progress: &mut Option<&mut dyn ResolverProfileAuthorityJournalProgress>, + ) -> Result<()> { let mut after = None::<(String, String)>; loop { let page = diff::load_reconciliation_target_page( @@ -229,6 +317,7 @@ impl ResolverProfileAuthorityJournalAdvance { self.summary.target_enqueue_statement_count += 1; self.summary.max_target_enqueue_batch_size = self.summary.max_target_enqueue_batch_size.max(page.len()); + record_journal_progress(progress).await?; } } } diff --git a/crates/storage/src/resolver_profile_authority_journal/advance/diff.rs b/crates/storage/src/resolver_profile_authority_journal/advance/diff.rs index 744b1ec2..c7d431e6 100644 --- a/crates/storage/src/resolver_profile_authority_journal/advance/diff.rs +++ b/crates/storage/src/resolver_profile_authority_journal/advance/diff.rs @@ -20,6 +20,19 @@ pub(super) async fn create_after_entries_table( .execute(&mut **transaction) .await .context("failed to create resolver-profile authority staging table")?; + sqlx::query( + r#" + CREATE INDEX resolver_profile_authority_after_entries_family_idx + ON pg_temp.resolver_profile_authority_after_entries ( + ((entry_payload ->> 'chain') COLLATE "C"), + ((entry_payload ->> 'source_family') COLLATE "C"), + ((entry_payload ->> 'address') COLLATE "C") + ) + "#, + ) + .execute(&mut **transaction) + .await + .context("failed to index resolver-profile authority staging families")?; Ok(()) } @@ -27,6 +40,26 @@ pub(super) async fn materialize_changed_entry_keys( transaction: &mut Transaction<'_, Postgres>, journal_key: &str, ) -> Result { + create_changed_entry_table(transaction).await?; + let mut after = None::; + let mut changed_entry_count = 0_i64; + loop { + let Some((last_key, inserted_count)) = + materialize_changed_entry_key_page(transaction, journal_key, after.as_deref(), 10_000) + .await? + else { + break; + }; + after = Some(last_key); + changed_entry_count += inserted_count; + } + finish_changed_entry_keys(transaction).await?; + Ok(changed_entry_count) +} + +pub(super) async fn create_changed_entry_table( + transaction: &mut Transaction<'_, Postgres>, +) -> Result<()> { sqlx::query( r#" CREATE TEMP TABLE pg_temp.resolver_profile_authority_changed_entries ( @@ -38,43 +71,90 @@ pub(super) async fn materialize_changed_entry_keys( .await .context("failed to create resolver-profile authority changed-entry table")?; - let changed_entry_count = sqlx::query_scalar::<_, i64>( + Ok(()) +} + +pub(super) async fn materialize_changed_entry_key_page( + transaction: &mut Transaction<'_, Postgres>, + journal_key: &str, + after: Option<&str>, + limit: usize, +) -> Result> { + let (last_key, inserted_count) = sqlx::query_as::<_, (Option, i64)>( r#" - WITH inserted AS ( - INSERT INTO pg_temp.resolver_profile_authority_changed_entries (entry_key) - SELECT COALESCE(before.entry_key, after.entry_key) + WITH before_keys AS ( + SELECT entry_key + FROM resolver_profile_authority_journal_entries + WHERE journal_key = $1 + AND ($2::TEXT IS NULL OR entry_key > $2) + ORDER BY entry_key + LIMIT $3 + ), + after_keys AS ( + SELECT entry_key + FROM pg_temp.resolver_profile_authority_after_entries + WHERE $2::TEXT IS NULL OR entry_key > $2 + ORDER BY entry_key + LIMIT $3 + ), + candidate_keys AS ( + SELECT entry_key FROM ( - SELECT entry_key, entry_payload - FROM resolver_profile_authority_journal_entries - WHERE journal_key = $1 - ) before - FULL OUTER JOIN pg_temp.resolver_profile_authority_after_entries after - USING (entry_key) + SELECT entry_key FROM before_keys + + UNION ALL + + SELECT entry_key FROM after_keys + ) bounded_keys + GROUP BY entry_key + ORDER BY entry_key + LIMIT $3 + ), + inserted AS ( + INSERT INTO pg_temp.resolver_profile_authority_changed_entries (entry_key) + SELECT candidate.entry_key + FROM candidate_keys candidate + LEFT JOIN resolver_profile_authority_journal_entries before + ON before.journal_key = $1 + AND before.entry_key = candidate.entry_key + LEFT JOIN pg_temp.resolver_profile_authority_after_entries after + ON after.entry_key = candidate.entry_key WHERE before.entry_payload IS DISTINCT FROM after.entry_payload + ON CONFLICT (entry_key) DO NOTHING RETURNING 1 ) - SELECT COUNT(*)::BIGINT FROM inserted + SELECT + (SELECT MAX(entry_key) FROM candidate_keys), + (SELECT COUNT(*)::BIGINT FROM inserted) "#, ) .bind(journal_key) + .bind(after) + .bind(i64::try_from(limit)?) .fetch_one(&mut **transaction) .await - .context("failed to materialize resolver-profile authority changed-entry keys")?; + .context("failed to materialize resolver-profile authority changed-entry key page")?; + Ok(last_key.map(|last_key| (last_key, inserted_count))) +} +pub(super) async fn finish_changed_entry_keys( + transaction: &mut Transaction<'_, Postgres>, +) -> Result<()> { sqlx::query("ANALYZE pg_temp.resolver_profile_authority_changed_entries") .execute(&mut **transaction) .await .context("failed to analyze resolver-profile authority changed-entry keys")?; - Ok(changed_entry_count) + Ok(()) } pub(super) async fn materialize_reconciliation_targets( transaction: &mut Transaction<'_, Postgres>, journal_key: &str, + progress: &mut Option<&mut dyn super::ResolverProfileAuthorityJournalProgress>, ) -> Result<()> { create_target_tables(transaction).await?; - materialize_direct_targets_and_seed_families(transaction, journal_key).await?; - materialize_seed_family_targets(transaction, journal_key).await?; + materialize_direct_targets_and_seed_families(transaction, journal_key, progress).await?; + materialize_seed_family_targets(transaction, journal_key, progress).await?; sqlx::query("ANALYZE pg_temp.resolver_profile_authority_changed_targets") .execute(&mut **transaction) .await @@ -113,103 +193,193 @@ async fn create_target_tables(transaction: &mut Transaction<'_, Postgres>) -> Re async fn materialize_direct_targets_and_seed_families( transaction: &mut Transaction<'_, Postgres>, journal_key: &str, + progress: &mut Option<&mut dyn super::ResolverProfileAuthorityJournalProgress>, ) -> Result<()> { - sqlx::query( + let mut after = None::; + loop { + let last_key = materialize_direct_target_page( + transaction, + journal_key, + after.as_deref(), + super::RESOLVER_PROFILE_AUTHORITY_INTERNAL_PAGE_SIZE, + ) + .await?; + let Some(last_key) = last_key else { + break; + }; + after = Some(last_key); + super::record_journal_progress(progress).await?; + } + Ok(()) +} + +async fn materialize_direct_target_page( + transaction: &mut Transaction<'_, Postgres>, + journal_key: &str, + after: Option<&str>, + limit: usize, +) -> Result> { + sqlx::query_scalar::<_, Option>( r#" - WITH changed_payloads AS ( + WITH candidate_keys AS ( + SELECT entry_key + FROM pg_temp.resolver_profile_authority_changed_entries + WHERE $2::TEXT IS NULL OR entry_key > $2 + ORDER BY entry_key + LIMIT $3 + ), + changed_payloads AS ( SELECT before.entry_payload AS payload - FROM pg_temp.resolver_profile_authority_changed_entries changed + FROM candidate_keys candidate JOIN resolver_profile_authority_journal_entries before ON before.journal_key = $1 - AND before.entry_key = changed.entry_key + AND before.entry_key = candidate.entry_key UNION ALL SELECT after.entry_payload AS payload - FROM pg_temp.resolver_profile_authority_changed_entries changed + FROM candidate_keys candidate JOIN pg_temp.resolver_profile_authority_after_entries after - ON after.entry_key = changed.entry_key - ) - INSERT INTO pg_temp.resolver_profile_authority_changed_targets ( - chain_id, - contract_address + ON after.entry_key = candidate.entry_key + ), + inserted_targets AS ( + INSERT INTO pg_temp.resolver_profile_authority_changed_targets ( + chain_id, + contract_address + ) + SELECT DISTINCT payload ->> 'chain', payload ->> 'address' + FROM changed_payloads + ON CONFLICT (chain_id, contract_address) DO NOTHING + RETURNING 1 + ), + inserted_seed_families AS ( + INSERT INTO pg_temp.resolver_profile_authority_changed_seed_families ( + chain_id, + source_family + ) + SELECT DISTINCT payload ->> 'chain', payload ->> 'source_family' + FROM changed_payloads + WHERE (payload ->> 'is_seed')::BOOLEAN + ON CONFLICT (chain_id, source_family) DO NOTHING + RETURNING 1 ) - SELECT DISTINCT payload ->> 'chain', payload ->> 'address' - FROM changed_payloads - ON CONFLICT (chain_id, contract_address) DO NOTHING + SELECT MAX(entry_key) + FROM candidate_keys + WHERE (SELECT COUNT(*) FROM inserted_targets) >= 0 + AND (SELECT COUNT(*) FROM inserted_seed_families) >= 0 "#, ) .bind(journal_key) - .execute(&mut **transaction) + .bind(after) + .bind(i64::try_from(limit)?) + .fetch_one(&mut **transaction) .await - .context("failed to materialize direct resolver-profile authority targets")?; + .context("failed to materialize a direct resolver-profile authority target page") +} - sqlx::query( +async fn materialize_seed_family_targets( + transaction: &mut Transaction<'_, Postgres>, + journal_key: &str, + progress: &mut Option<&mut dyn super::ResolverProfileAuthorityJournalProgress>, +) -> Result<()> { + let families = sqlx::query_as::<_, (String, String)>( r#" - WITH changed_payloads AS ( - SELECT before.entry_payload AS payload - FROM pg_temp.resolver_profile_authority_changed_entries changed - JOIN resolver_profile_authority_journal_entries before - ON before.journal_key = $1 - AND before.entry_key = changed.entry_key - - UNION ALL - - SELECT after.entry_payload AS payload - FROM pg_temp.resolver_profile_authority_changed_entries changed - JOIN pg_temp.resolver_profile_authority_after_entries after - ON after.entry_key = changed.entry_key - ) - INSERT INTO pg_temp.resolver_profile_authority_changed_seed_families ( - chain_id, - source_family - ) - SELECT DISTINCT payload ->> 'chain', payload ->> 'source_family' - FROM changed_payloads - WHERE (payload ->> 'is_seed')::BOOLEAN - ON CONFLICT (chain_id, source_family) DO NOTHING + SELECT chain_id, source_family + FROM pg_temp.resolver_profile_authority_changed_seed_families + ORDER BY chain_id, source_family "#, ) - .bind(journal_key) - .execute(&mut **transaction) + .fetch_all(&mut **transaction) .await - .context("failed to materialize changed resolver-profile seed families")?; + .context("failed to load changed resolver-profile seed families")?; + + for (chain_id, source_family) in families { + let mut after = None::; + loop { + let last_address = materialize_seed_family_target_page( + transaction, + journal_key, + &chain_id, + &source_family, + after.as_deref(), + super::RESOLVER_PROFILE_AUTHORITY_INTERNAL_PAGE_SIZE, + ) + .await?; + let Some(last_address) = last_address else { + break; + }; + after = Some(last_address); + super::record_journal_progress(progress).await?; + } + } Ok(()) } -async fn materialize_seed_family_targets( +async fn materialize_seed_family_target_page( transaction: &mut Transaction<'_, Postgres>, journal_key: &str, -) -> Result<()> { - sqlx::query( + chain_id: &str, + source_family: &str, + after: Option<&str>, + limit: usize, +) -> Result> { + sqlx::query_scalar::<_, Option>( r#" - WITH all_payloads AS ( - SELECT entry_payload AS payload + WITH before_addresses AS ( + SELECT (entry_payload ->> 'address') COLLATE "C" AS address FROM resolver_profile_authority_journal_entries WHERE journal_key = $1 + AND (entry_payload ->> 'chain') COLLATE "C" = $2 + AND (entry_payload ->> 'source_family') COLLATE "C" = $3 + AND ($4::TEXT IS NULL OR (entry_payload ->> 'address') COLLATE "C" > $4) + ORDER BY (entry_payload ->> 'address') COLLATE "C" + LIMIT $5 + ), + after_addresses AS ( + SELECT (entry_payload ->> 'address') COLLATE "C" AS address + FROM pg_temp.resolver_profile_authority_after_entries + WHERE (entry_payload ->> 'chain') COLLATE "C" = $2 + AND (entry_payload ->> 'source_family') COLLATE "C" = $3 + AND ($4::TEXT IS NULL OR (entry_payload ->> 'address') COLLATE "C" > $4) + ORDER BY (entry_payload ->> 'address') COLLATE "C" + LIMIT $5 + ), + candidate_addresses AS ( + SELECT address + FROM ( + SELECT address FROM before_addresses - UNION ALL + UNION ALL - SELECT entry_payload AS payload - FROM pg_temp.resolver_profile_authority_after_entries - ) - INSERT INTO pg_temp.resolver_profile_authority_changed_targets ( - chain_id, - contract_address + SELECT address FROM after_addresses + ) bounded_addresses + GROUP BY address + ORDER BY address + LIMIT $5 + ), + inserted AS ( + INSERT INTO pg_temp.resolver_profile_authority_changed_targets ( + chain_id, + contract_address + ) + SELECT $2, address + FROM candidate_addresses + ON CONFLICT (chain_id, contract_address) DO NOTHING + RETURNING 1 ) - SELECT DISTINCT payload ->> 'chain', payload ->> 'address' - FROM all_payloads - JOIN pg_temp.resolver_profile_authority_changed_seed_families family - ON family.chain_id = payload ->> 'chain' - AND family.source_family = payload ->> 'source_family' - ON CONFLICT (chain_id, contract_address) DO NOTHING + SELECT MAX(address) + FROM candidate_addresses + WHERE (SELECT COUNT(*) FROM inserted) >= 0 "#, ) .bind(journal_key) - .execute(&mut **transaction) + .bind(chain_id) + .bind(source_family) + .bind(after) + .bind(i64::try_from(limit)?) + .fetch_one(&mut **transaction) .await - .context("failed to expand changed resolver-profile seed families")?; - Ok(()) + .context("failed to expand a resolver-profile seed-family target page") } pub(super) async fn load_reconciliation_target_page( diff --git a/crates/storage/src/resolver_profile_authority_journal/advance/mutations.rs b/crates/storage/src/resolver_profile_authority_journal/advance/mutations.rs index 89595f58..dc5f450b 100644 --- a/crates/storage/src/resolver_profile_authority_journal/advance/mutations.rs +++ b/crates/storage/src/resolver_profile_authority_journal/advance/mutations.rs @@ -13,6 +13,7 @@ pub(super) async fn apply_entry_diff( transaction: &mut Transaction<'_, Postgres>, journal_key: &str, batch_size: usize, + progress: &mut Option<&mut dyn super::ResolverProfileAuthorityJournalProgress>, ) -> Result { ensure!(batch_size > 0, "journal entry batch size must be positive"); let mut summary = EntryMutationSummary::default(); @@ -26,6 +27,7 @@ pub(super) async fn apply_entry_diff( after_key = Some(last_key); record_batch(&mut summary, count)?; summary.upserted_entry_count += count; + super::record_journal_progress(progress).await?; } let mut after_key = None::; @@ -38,6 +40,7 @@ pub(super) async fn apply_entry_diff( after_key = Some(last_key); record_batch(&mut summary, count)?; summary.deleted_entry_count += count; + super::record_journal_progress(progress).await?; } Ok(summary) } diff --git a/crates/storage/src/resolver_profile_authority_journal/tests.rs b/crates/storage/src/resolver_profile_authority_journal/tests.rs index 5cf409a6..014416a5 100644 --- a/crates/storage/src/resolver_profile_authority_journal/tests.rs +++ b/crates/storage/src/resolver_profile_authority_journal/tests.rs @@ -7,6 +7,20 @@ use serde_json::{Value, json}; use super::advance::ResolverProfileAuthorityJournalBatchSizes; use super::*; +#[derive(Default)] +struct CountingJournalProgress { + record_count: usize, +} + +impl ResolverProfileAuthorityJournalProgress for CountingJournalProgress { + fn record(&mut self) -> ResolverProfileAuthorityJournalProgressFuture<'_> { + Box::pin(async move { + self.record_count += 1; + Ok(()) + }) + } +} + fn entry_payload( chain: &str, source_family: &str, @@ -327,7 +341,11 @@ async fn batched_diff_preserves_seed_family_target_expansion() -> Result<()> { ) .await?; stage_entries(&mut advance, &after).await?; - let summary = advance.publish(&json!({})).await?.unwrap(); + let mut progress = CountingJournalProgress::default(); + let summary = advance + .publish_with_progress(&json!({}), &mut progress) + .await? + .unwrap(); assert_eq!(summary.changed_entry_count, 4); assert_eq!(summary.enqueued_target_count, 5); assert_eq!(summary.target_enqueue_statement_count, 3); @@ -336,6 +354,10 @@ async fn batched_diff_preserves_seed_family_target_expansion() -> Result<()> { assert_eq!(summary.deleted_entry_count, 1); assert_eq!(summary.entry_mutation_statement_count, 3); assert_eq!(summary.max_entry_mutation_batch_size, 2); + assert!( + progress.record_count >= 8, + "authority journal publication must report bounded diff, target, queue, and mutation pages" + ); let queued = sqlx::query_as::<_, (String, String)>( r#" diff --git a/crates/storage/src/stored_lineage_coverage.rs b/crates/storage/src/stored_lineage_coverage.rs index 48719bf9..389bae43 100644 --- a/crates/storage/src/stored_lineage_coverage.rs +++ b/crates/storage/src/stored_lineage_coverage.rs @@ -1,4 +1,4 @@ -use std::collections::BTreeMap; +use std::{collections::BTreeMap, future::Future, pin::Pin}; use anyhow::{Context, Result, ensure}; use serde_json::Value; @@ -12,6 +12,14 @@ mod integrity; pub const STORED_LINEAGE_COVERAGE_CANDIDATE_TABLE: &str = "stored_lineage_coverage_frontier_candidate_requirements"; pub const STORED_LINEAGE_COVERAGE_PROOF_FORMAT_VERSION: &str = "stored_lineage_coverage_v1"; +const COVERAGE_PUBLICATION_PAGE_ROWS: i64 = 1_000; + +pub type StoredLineageCoverageProgressFuture<'a> = + Pin> + Send + 'a>>; + +pub trait StoredLineageCoverageProgress: Send { + fn record<'a>(&'a mut self) -> StoredLineageCoverageProgressFuture<'a>; +} #[derive(Clone, Debug, Eq, PartialEq)] pub struct StoredLineageCoverageFrontierHeader { @@ -60,8 +68,24 @@ impl StoredLineageCoveragePublicationGuard { } pub async fn publish( + self, + publication: &StoredLineageCoverageFrontierPublication, + ) -> Result { + self.publish_inner(publication, &mut None).await + } + + pub async fn publish_with_progress( + self, + publication: &StoredLineageCoverageFrontierPublication, + progress: &mut dyn StoredLineageCoverageProgress, + ) -> Result { + self.publish_inner(publication, &mut Some(progress)).await + } + + async fn publish_inner( mut self, publication: &StoredLineageCoverageFrontierPublication, + progress: &mut Option<&mut dyn StoredLineageCoverageProgress>, ) -> Result { validate_publication(publication)?; ensure!( @@ -71,13 +95,27 @@ impl StoredLineageCoveragePublicationGuard { self.expected_discovery_admission_epoch ); - let candidate_integrity = integrity::validate_candidate_and_load_integrity( - self.transaction.as_mut(), - &self.chain, - publication.verified_from_block, - publication.verified_through_block, - ) - .await?; + let candidate_integrity = match progress.as_deref_mut() { + Some(progress) => { + integrity::validate_candidate_and_load_integrity_with_progress( + self.transaction.as_mut(), + &self.chain, + publication.verified_from_block, + publication.verified_through_block, + progress, + ) + .await? + } + None => { + integrity::validate_candidate_and_load_integrity( + self.transaction.as_mut(), + &self.chain, + publication.verified_from_block, + publication.verified_through_block, + ) + .await? + } + }; // Candidate derivation and immutable coverage-fact verification are // deliberately optimistic. Take the shared admission fence only for @@ -203,40 +241,8 @@ impl StoredLineageCoveragePublicationGuard { return Ok(StoredLineageCoveragePublicationOutcome::Conflict); }; - sqlx::query( - "DELETE FROM stored_lineage_coverage_frontier_requirements WHERE chain_id = $1", - ) - .bind(&self.chain) - .execute(self.transaction.as_mut()) - .await - .with_context(|| { - format!( - "failed to replace prior stored-lineage coverage requirements for {}", - self.chain - ) - })?; - sqlx::query( - r#" - INSERT INTO stored_lineage_coverage_frontier_requirements ( - chain_id, - source_family, - address, - required_intervals - ) - SELECT $1, source_family, address, required_intervals - FROM pg_temp.stored_lineage_coverage_frontier_candidate_requirements - ORDER BY source_family, address - "#, - ) - .bind(&self.chain) - .execute(self.transaction.as_mut()) - .await - .with_context(|| { - format!( - "failed to publish stored-lineage coverage requirements for {}", - self.chain - ) - })?; + replace_requirements_with_progress(self.transaction.as_mut(), &self.chain, progress) + .await?; self.transaction.commit().await.with_context(|| { format!( @@ -297,6 +303,109 @@ pub async fn stored_lineage_coverage_frontier_requirements_are_valid( integrity::saved_snapshot_is_valid(pool, header).await } +pub async fn stored_lineage_coverage_frontier_requirements_are_valid_with_progress( + pool: &PgPool, + header: &StoredLineageCoverageFrontierHeader, + progress: &mut dyn StoredLineageCoverageProgress, +) -> Result { + integrity::saved_snapshot_is_valid_with_progress(pool, header, progress).await +} + +async fn replace_requirements_with_progress( + connection: &mut PgConnection, + chain: &str, + progress: &mut Option<&mut dyn StoredLineageCoverageProgress>, +) -> Result<()> { + loop { + let deleted = sqlx::query( + r#" + DELETE FROM stored_lineage_coverage_frontier_requirements target + WHERE target.ctid IN ( + SELECT candidate.ctid + FROM stored_lineage_coverage_frontier_requirements candidate + WHERE candidate.chain_id = $1 + ORDER BY candidate.source_family, candidate.address + LIMIT $2 + ) + "#, + ) + .bind(chain) + .bind(COVERAGE_PUBLICATION_PAGE_ROWS) + .execute(&mut *connection) + .await + .with_context(|| { + format!("failed to delete a prior stored-lineage coverage page for {chain}") + })? + .rows_affected(); + if deleted == 0 { + break; + } + record_publication_progress(progress).await?; + } + + let mut cursor = None::<(String, String)>; + loop { + let rows = sqlx::query( + r#" + SELECT source_family, address + FROM pg_temp.stored_lineage_coverage_frontier_candidate_requirements + WHERE $1::TEXT IS NULL OR (source_family, address) > ($1, $2) + ORDER BY source_family, address + LIMIT $3 + "#, + ) + .bind(cursor.as_ref().map(|(family, _)| family)) + .bind(cursor.as_ref().map(|(_, address)| address)) + .bind(COVERAGE_PUBLICATION_PAGE_ROWS) + .fetch_all(&mut *connection) + .await + .with_context(|| format!("failed to page coverage publication candidate for {chain}"))?; + let Some(last) = rows.last() else { + break; + }; + cursor = Some((last.try_get("source_family")?, last.try_get("address")?)); + let families = rows + .iter() + .map(|row| row.try_get::("source_family")) + .collect::, _>>()?; + let addresses = rows + .iter() + .map(|row| row.try_get::("address")) + .collect::, _>>()?; + sqlx::query( + r#" + WITH page_keys AS ( + SELECT * FROM UNNEST($2::TEXT[], $3::TEXT[]) key(source_family, address) + ) + INSERT INTO stored_lineage_coverage_frontier_requirements ( + chain_id, source_family, address, required_intervals + ) + SELECT $1, candidate.source_family, candidate.address, candidate.required_intervals + FROM page_keys key + JOIN pg_temp.stored_lineage_coverage_frontier_candidate_requirements candidate + USING (source_family, address) + "#, + ) + .bind(chain) + .bind(&families) + .bind(&addresses) + .execute(&mut *connection) + .await + .with_context(|| format!("failed to insert a stored-lineage coverage page for {chain}"))?; + record_publication_progress(progress).await?; + } + Ok(()) +} + +async fn record_publication_progress( + progress: &mut Option<&mut dyn StoredLineageCoverageProgress>, +) -> Result<()> { + if let Some(progress) = progress.as_deref_mut() { + progress.record().await?; + } + Ok(()) +} + pub async fn begin_stored_lineage_coverage_frontier_publication( pool: &PgPool, chain: &str, diff --git a/crates/storage/src/stored_lineage_coverage/integrity.rs b/crates/storage/src/stored_lineage_coverage/integrity.rs index 03bc3ad3..1e7c31c2 100644 --- a/crates/storage/src/stored_lineage_coverage/integrity.rs +++ b/crates/storage/src/stored_lineage_coverage/integrity.rs @@ -1,7 +1,9 @@ use anyhow::{Context, Result, ensure}; use sqlx::{PgConnection, PgPool, Row}; -use super::StoredLineageCoverageFrontierHeader; +use super::{StoredLineageCoverageFrontierHeader, StoredLineageCoverageProgress}; + +const INTEGRITY_PAGE_ROWS: i64 = 1_000; #[derive(Clone, Debug, Eq, PartialEq)] pub(super) struct RequirementSnapshotIntegrity { @@ -85,6 +87,77 @@ pub(super) async fn validate_candidate_and_load_integrity( }) } +pub(super) async fn validate_candidate_and_load_integrity_with_progress( + connection: &mut PgConnection, + chain: &str, + verified_from_block: i64, + verified_through_block: i64, + progress: &mut dyn StoredLineageCoverageProgress, +) -> Result { + let mut cursor = None::<(String, String)>; + let mut row_count = 0i64; + let mut invalid_count = 0i64; + let mut digest_left = 0u64; + let mut digest_right = 0u64; + loop { + let rows = sqlx::query( + r#" + SELECT + source_family, + address, + ( + source_family = '' + OR address <> lower(address) + OR required_intervals = '{}'::INT8MULTIRANGE + OR lower(required_intervals) IS NULL + OR upper(required_intervals) IS NULL + OR lower(required_intervals) < $3 + OR upper(required_intervals) > $4 + 1 + ) AS invalid, + md5( + jsonb_build_array( + source_family, + address, + required_intervals::TEXT + )::TEXT + ) AS row_hash + FROM pg_temp.stored_lineage_coverage_frontier_candidate_requirements + WHERE $1::TEXT IS NULL OR (source_family, address) > ($1, $2) + ORDER BY source_family, address + LIMIT $5 + "#, + ) + .bind(cursor.as_ref().map(|(family, _)| family)) + .bind(cursor.as_ref().map(|(_, address)| address)) + .bind(verified_from_block) + .bind(verified_through_block) + .bind(INTEGRITY_PAGE_ROWS) + .fetch_all(&mut *connection) + .await + .with_context(|| format!("failed to validate a coverage candidate page for {chain}"))?; + let Some(last) = rows.last() else { + break; + }; + cursor = Some((last.try_get("source_family")?, last.try_get("address")?)); + accumulate_integrity_page( + &rows, + &mut row_count, + &mut invalid_count, + &mut digest_left, + &mut digest_right, + )?; + progress.record().await?; + } + ensure!( + invalid_count == 0, + "stored-lineage coverage candidate for {chain} has {invalid_count} invalid rows" + ); + Ok(RequirementSnapshotIntegrity { + row_count, + digest: format!("{digest_left:016x}{digest_right:016x}"), + }) +} + pub(super) async fn saved_snapshot_is_valid( pool: &PgPool, header: &StoredLineageCoverageFrontierHeader, @@ -156,3 +229,109 @@ pub(super) async fn saved_snapshot_is_valid( && row_count == header.requirement_row_count && digest == header.requirement_digest) } + +pub(super) async fn saved_snapshot_is_valid_with_progress( + pool: &PgPool, + header: &StoredLineageCoverageFrontierHeader, + progress: &mut dyn StoredLineageCoverageProgress, +) -> Result { + if !header.is_well_formed { + return Ok(false); + } + let mut transaction = pool + .begin() + .await + .context("failed to begin stored-lineage integrity snapshot")?; + sqlx::query("SET TRANSACTION ISOLATION LEVEL REPEATABLE READ READ ONLY") + .execute(transaction.as_mut()) + .await + .context("failed to pin stored-lineage integrity snapshot")?; + let mut cursor = None::<(String, String)>; + let mut row_count = 0i64; + let mut invalid_count = 0i64; + let mut digest_left = 0u64; + let mut digest_right = 0u64; + loop { + let rows = sqlx::query( + r#" + SELECT + source_family, + address, + ( + required_intervals = '{}'::INT8MULTIRANGE + OR lower(required_intervals) IS NULL + OR upper(required_intervals) IS NULL + OR lower(required_intervals) < $4 + OR upper(required_intervals) > $5 + 1 + ) AS invalid, + md5( + jsonb_build_array( + source_family, + address, + required_intervals::TEXT + )::TEXT + ) AS row_hash + FROM stored_lineage_coverage_frontier_requirements + WHERE chain_id = $1 + AND ($2::TEXT IS NULL OR (source_family, address) > ($2, $3)) + ORDER BY source_family, address + LIMIT $6 + "#, + ) + .bind(&header.chain_id) + .bind(cursor.as_ref().map(|(family, _)| family)) + .bind(cursor.as_ref().map(|(_, address)| address)) + .bind(header.verified_from_block) + .bind(header.verified_through_block) + .bind(INTEGRITY_PAGE_ROWS) + .fetch_all(transaction.as_mut()) + .await + .with_context(|| { + format!( + "failed to validate a stored-lineage coverage page for {} revision {}", + header.chain_id, header.snapshot_revision + ) + })?; + let Some(last) = rows.last() else { + break; + }; + cursor = Some((last.try_get("source_family")?, last.try_get("address")?)); + accumulate_integrity_page( + &rows, + &mut row_count, + &mut invalid_count, + &mut digest_left, + &mut digest_right, + )?; + progress.record().await?; + } + transaction + .commit() + .await + .context("failed to close stored-lineage integrity snapshot")?; + Ok(invalid_count == 0 + && row_count == header.requirement_row_count + && format!("{digest_left:016x}{digest_right:016x}") == header.requirement_digest) +} + +fn accumulate_integrity_page( + rows: &[sqlx::postgres::PgRow], + row_count: &mut i64, + invalid_count: &mut i64, + digest_left: &mut u64, + digest_right: &mut u64, +) -> Result<()> { + for row in rows { + *row_count += 1; + if row.try_get::("invalid")? { + *invalid_count += 1; + } + let row_hash: String = row.try_get("row_hash")?; + ensure!(row_hash.len() == 32, "coverage row hash is malformed"); + *digest_left ^= u64::from_str_radix(&row_hash[..16], 16) + .context("failed to decode coverage row hash prefix")?; + *digest_right ^= u64::from_str_radix(&row_hash[16..], 16) + .context("failed to decode coverage row hash suffix")?; + } + Ok(()) +} diff --git a/docs/deployment.md b/docs/deployment.md index 8b435f6b..8bde7d59 100644 --- a/docs/deployment.md +++ b/docs/deployment.md @@ -1200,10 +1200,8 @@ the abort-status migration is installed, resume/complete the run or restore the database to a consistent pre-run snapshot before running migrations or writers. Guarded writer processes require at least two database pool connections so the held advisory lock connection cannot starve ordinary writer work. The indexer -requires at least three: resolver-profile authority journaling shares its -transaction with the bounded target cursor and keeps one connection available -for bounded admission reads, while resolver-profile convergence keeps capacity -for its guarded adapter writes and bounded journal/event reads. Each path +requires at least four so its permanent runtime writer guard, nested bounded +work guards, and progress-heartbeat writer cannot exhaust the pool. Each path rejects a smaller pool before starting that work. 1. Stop the indexer and worker services, leaving PostgreSQL and the API online diff --git a/docs/storage.md b/docs/storage.md index 3b0c27cb..972de168 100644 --- a/docs/storage.md +++ b/docs/storage.md @@ -531,6 +531,16 @@ instance cannot hide a stopped process. These rows are mutable operational signa not raw facts, [replay](glossary.md) checkpoints, chain checkpoints, or projection freshness evidence. +The worker's parent projection loop and required spawned hydration or +invalidation-derivation work share an exclusive process-heartbeat ownership +gate. The indexer's parent poll and automatic normalized-event replay catch-up +share the same kind of gate. A parent work unit or required child iteration +acquires ownership before it starts and releases ownership before an idle poll +sleep. Only the current owner can advance the row, and it can do so only after +its own bounded work commits. A wedged child therefore cannot be hidden by an +otherwise healthy parent, and a progressing child cannot hide a wedged parent. +No timer generates beats on either operation's behalf. + Full worker rebuild heartbeat routes are explicit: | Rebuild step | Bounded progress heartbeat | Named monolithic phases | @@ -543,6 +553,72 @@ Full worker rebuild heartbeat routes are explicit: | `address_names_current` | each completed surface binding; staged writes remain in 2,000-row batches | `address_names_current.prepare`, `address_names_current.publish`, `address_names_current.count_published_addresses` | | `primary_names_current` | each streamed tuple; legacy hydration beats during 1,000-candidate planning, configured provider batches, resolver-edge batches, and 1,000-row upserts | `primary_names_current.count_existing`, `primary_names_current.invalidate_execution_cache`, `primary_names_current.publish`, `primary_names_current.legacy_hydration.load_reverse_claim_candidates`, `primary_names_current.legacy_hydration.load_resolver_edge_candidates` | +The following table is the maintained coverage audit for work performed by the +worker and indexer service loops, including their spawned tasks. `covered` +means the progress route predates the systematic audit; `newly-covered` means +the audit added the route. Both advance liveness only after a completed unit of +work. `cannot-exceed-20s-with-reason` is reserved for operations whose input +cardinality is fixed independently of mainnet history. `n-a` means the operation does not own +the service-loop heartbeat or has no smaller successful progress unit. In +particular, a slow or stuck atomic database statement or RPC is not progress +and must be allowed to make the heartbeat stale. + +| Service stage | Operation | Disposition | Progress boundary or scale reason | +| --- | --- | --- | --- | +| worker pre-registration | parse fixed CLI hydration and chain/provider configuration | cannot-exceed-20s-with-reason | Linear in the finite configured chain, resolver-address, and RPC-URL lists; it does not read chain history. | +| worker pre-registration | connect to PostgreSQL and register the service loop | n-a | No heartbeat row exists yet, and each connection or registration statement is atomic. A stalled database operation has no completed progress unit and must not be kept green. | +| worker startup/poll | replay-readiness and handoff checks | cannot-exceed-20s-with-reason | Scalar counts over configured replay cursors, seven replay markers, and PostgreSQL index metadata; cardinality does not grow with names or events. | +| worker rebuild | `name_current` | covered (#213) | Completed name tasks and the named load/publication phases listed above. | +| worker rebuild | `children_current` | covered (#213) | Completed child-source tasks and the named count/publication phases listed above. | +| worker rebuild | `permissions_current` | covered (#213) | Completed resource tasks and the named count/publication phases listed above. | +| worker rebuild | `record_inventory_current` | covered (#213) | Completed resource tasks and the named count/publication phases listed above. | +| worker rebuild | `resolver_current` | covered (#213) | Completed resolver-target tasks and the named load/count/publication phases listed above. | +| worker rebuild | `address_names_current` | covered (#213) | Completed surface-binding tasks and the named prepare/count/publication phases listed above. | +| worker rebuild | `primary_names_current` | covered (#213) | Streamed tuples and the named count/cache-invalidation/publication phases listed above. | +| worker hydration | record-inventory text values | covered (#213) | Each bounded 500-row input/provider/persistence page. | +| worker hydration | legacy reverse-resolver primary names, including the spawned continuing pass | covered (#213) | Each 1,000-candidate planning or upsert page, configured provider batch, and resolver-edge batch; the spawned task uses the same shared progress recorder. | +| worker invalidation | derive normalized-event changes, including the spawned continuing pass | newly-covered (#229; bounded derive beats originated in #213) | Each successfully derived bounded change unit. Parent work and the spawned pass mutually exclude each other's heartbeat writes while either owns a work unit, so neither can mask the other's wedge; exit and panic remain supervised (#242). | +| worker invalidation | claim, dispatch, and complete queued keys | covered (#213) | Each completed claimed key or bounded address group. Claim-lease refreshes are separate queue ownership evidence, not service-loop liveness. | +| worker invalidation | targeted rebuild work inside an applied key or address group | newly-covered (#229) | Completed projection-specific loads, source/target rows, bounded writes, and cleanup pages for all seven projection families. | +| worker spawned task | route-local primary-name execution pruning | n-a | Detached best-effort maintenance does not own worker readiness; every call is capped by the configured delete batch, and the parent loop remains independently observable. | +| worker/indexer main loop | idle poll sleep and subtask supervision | n-a | Sleeping is intentional inactivity; loop ticks record liveness only while no required spawned task owns the row. Spawned-task exit and panic are reported by the existing supervisor (#242), not by synthetic progress beats. | +| indexer pre-registration and live poll | parse the checked-in manifest repository | cannot-exceed-20s-with-reason | Bounded by the finite checked-in deployment files, not chain history; live parsing runs before deciding whether stored manifest state needs refresh. | +| indexer pre-registration | connect to PostgreSQL and register the service loop | n-a | No heartbeat row exists yet, and each connection or registration statement is atomic. A stalled database operation has no completed progress unit and must not be kept green. `indexer run` rejects pools below four connections before opening them so the permanent runtime writer guard, nested bounded work guards, and progress heartbeat writer cannot exhaust the pool. | +| indexer startup | synchronize manifest declarations, source graph, active addresses, and stale discovery rows | newly-covered (#229) | Completed manifest, discovery-edge, active-address, and stale-row stream pages through `ManifestRuntimeProgress`. Full-source discovery reconciliation pages the active-edge summary, desired/insert and deactivation diffs, candidate observations, same-assignment retention, set-based historical-successor resolution, historical materialization, and final active-edge summary; the candidate allocation cap is checked before extending the retained vector with each bounded page. | +| indexer startup | load discovery admission, watched contracts, drift inputs, and manifest-derived normalized events | newly-covered (#229) | Completed 1,000-row or 10,000-row database pages and bounded event-upsert pages. | +| indexer startup | build and persist intake tasks from the watched-chain plan | newly-covered (#229) | Each completed 10,000-address plan copy/comparison chunk and persisted chain task. | +| indexer startup adapter sync | ENSv1 reverse claim | covered (#221/#237) | Completed raw-log, decode, existing-event, and normalized-event pages. | +| indexer startup adapter sync | ENSv1 subregistry discovery | covered (#221/#237) | Completed checkpoint raw-log pages plus bounded discovery, identity, binding, and event finalization units. | +| indexer startup adapter sync | ENSv1 unwrapped authority | covered (#221/#237) | Completed checkpoint raw-log pages plus bounded resolver-profile, identity, and event finalization units. | +| indexer startup adapter sync | ENSv2 registry resource/surface | covered (#221/#237) | Completed raw-log/processing pages and bounded lineage, resource, surface, binding, and event writes. | +| indexer startup adapter sync | ENSv2 registrar | covered (#221/#237) | Completed raw-log, processing, and normalized-event pages. | +| indexer startup adapter sync | ENSv2 resolver | covered (#221/#237) | Completed raw-log, processing, and normalized-event pages. | +| indexer startup adapter sync | ENSv2 permissions | covered (#221/#237) | Completed raw-log/processing pages and bounded resource/event writes. | +| indexer startup | plan source identities, targets, segments, reservations, and coverage for hash-pinned bootstrap | newly-covered (#229) | Completed 1,000-target planning/identity pages, durable reservation and coverage pages, and completed worker-result handling. | +| indexer startup | execute hash-pinned bootstrap provider units | covered (#213) | Provider-backed ranges are divided into at-most-32-block progress units; a beat follows only a fully persisted unit. Worker completion and failure remain coordinator-owned. | +| indexer startup | post-bootstrap adapter synchronization | covered (#213/#221/#237) | Reuses every adapter's page progress route and records completed family boundaries. | +| indexer startup | widen bootstrap admission to the live watch plan | newly-covered (#229) | Reuses manifest/discovery page progress and records each 10,000-address plan/task copy or comparison chunk. | +| indexer startup | initial resolver authority journal and resolver-profile convergence | newly-covered (#229) | Uses the authority and convergence boundaries described below. | +| indexer live poll | refresh manifest declarations, source graph, active addresses, and stale discovery rows | newly-covered (#229) | Same `ManifestRuntimeProgress` pages as startup; no timer runs beside the refresh. | +| indexer live poll | refresh discovery admission, watched plan, and intake tasks | newly-covered (#229) | Completed discovery pages, 10,000-address plan builds/copies/comparisons, and persisted chain tasks. | +| indexer live poll | refresh code-hash drift and manifest-derived normalized events | newly-covered (#229) | Completed 1,000-row code-hash, address, discovery-parent, event-input, and event-upsert pages. | +| indexer live poll | full-corpus adapter refresh after manifest/discovery changes | newly-covered (#229) | Existing adapter page callbacks now remain connected through live reconciliation; completed family, reconciliation, and finalization boundaries also beat. | +| indexer live poll | post-replay adapter backlog and cursor publication | newly-covered (#229) | Completed adapter families and bounded publication/cursor units; publication beats only after its transaction commits. | +| indexer live poll | fetch the provider head, safe head, and finalized head | n-a | A fixed set of atomic RPC results is required before reconciliation can advance. A provider call taking more than 20 seconds represents no completed progress and must not receive a beat. | +| indexer canonical reconciliation | walk provider parents and fill a contiguous gap | newly-covered (#229) | Each fetched parent and each completed block-persistence unit; non-reorg adapter work is split into at-most-32-block chunks. | +| indexer canonical reconciliation | prove stored lineage and retained-history coverage | newly-covered (#229) | Completed lineage blocks and bounded coverage candidate, companion, fork, and delta pages. | +| indexer canonical reconciliation | orphan the losing branch | newly-covered (#229) | Each losing block plus bounded raw-fact, normalized-event, identity, and execution-invalidation pages. | +| indexer canonical reconciliation | persist winning raw blocks, transactions, receipts, logs, and event-silent calls | newly-covered (#229) | Each block and each 1,000-row persistence page. | +| indexer canonical reconciliation | observe code hashes and replay adapter-owned state | newly-covered (#229) | Completed code-observation pages, at-most-32-block adapter chunks, and internal raw-log/processing/persistence pages for block-derived events and all selected ENSv1/ENSv2 adapter families. | +| indexer canonical reconciliation | recover exact missing retained-history coverage | newly-covered (#229) | Exact watched-target pages and provider-backed sequential backfill units of at most 32 blocks; coverage is recorded before the beat. | +| indexer canonical reconciliation | publish chain checkpoints | newly-covered (#229) | A beat follows the committed checkpoint and admission-epoch fence; the atomic publication itself has no synthetic keepalive. | +| indexer resolver-profile convergence | compare and advance the authority journal | newly-covered (#229) | Completed authority-entry, seed-family expansion, staged-diff, mutation, and forced-target pages. | +| indexer resolver-profile convergence | drain input changes and materialize reconciliation targets | newly-covered (#229) | Each 1,000-input page and each completed target/family page. A progress-enabled drain rejects pools below four connections, reserving capacity for the runtime writer guard, reconciliation guard, bounded authority/event reads, and heartbeat writes. | +| indexer resolver-profile convergence | replay resolver targets, stage/publish events, and enqueue invalidations | newly-covered (#229) | Completed target/log/state pages, 1,000-row staged-event and invalidation pages, family publication, and acknowledged input pages. | +| indexer spawned task | automatic normalized-event replay catch-up | newly-covered (#229) | Completed stateless replay pages, adapter-internal pages, full-closure adapter boundaries, replay chunks, durable cursor publications, and checkpoint cleanup advance the shared process row. Parent work and the spawned iteration mutually exclude each other's heartbeat writes while either owns a work unit, including failure journaling, so neither can mask the other's wedge; idle catch-up sleep releases ownership. Exit and panic remain supervised (#242). | +| indexer spawned task | normalized-replay projection-index preparation and restoration DDL statements | n-a | Each PostgreSQL catalog check and `CREATE INDEX` or `DROP INDEX` statement is atomic at the application boundary, with no safe successful progress callback inside the statement. The replay iteration retains heartbeat ownership, so a slow or stuck statement intentionally ages the process row instead of receiving a synthetic beat. | +| indexer one-shot modes | manual backfill, replay, rewind, repair, and operational catch-up commands | n-a | These commands do not run inside the registered `indexer run` service loop, so service-loop heartbeat coverage does not describe their completion. Backfill range leases retain their separate ownership heartbeat. | + A named phase is a distinct `scope_kind='phase'` row for the worker instance. Its `heartbeat_at` is the phase start rather than a free-running timer, so a crash or wedge still ages out. Worker and API checks use the separately diff --git a/scripts/rust-file-size-baseline.toml b/scripts/rust-file-size-baseline.toml index e5fd36b0..7804b278 100644 --- a/scripts/rust-file-size-baseline.toml +++ b/scripts/rust-file-size-baseline.toml @@ -27,7 +27,7 @@ justification = "Phase 7 allowlist for an oversized production file; future chan [[files]] path = "crates/adapters/src/ens_v1_unwrapped_authority/loading/active_emitters.rs" -loc = 892 +loc = 808 justification = "Phase 7 allowlist for an oversized production file; future changes should split or shrink it." [[files]] From b78ebac5e85d4e75335837d9ab59c8ab8072d477 Mon Sep 17 00:00:00 2001 From: TateB Date: Thu, 23 Jul 2026 02:09:18 +0000 Subject: [PATCH 02/12] chore: refresh conformance lockfile for workspace dep changes Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01C9RTnbCyhqEUSPc7xCK76p --- tests/conformance/Cargo.lock | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/conformance/Cargo.lock b/tests/conformance/Cargo.lock index 00a59e84..1535c713 100644 --- a/tests/conformance/Cargo.lock +++ b/tests/conformance/Cargo.lock @@ -740,6 +740,7 @@ dependencies = [ "anyhow", "bigname-domain", "bigname-storage", + "futures-util", "serde", "serde_json", "sqlx", From 29bc8e0ac8577967e48e3e973477351412a253a8 Mon Sep 17 00:00:00 2001 From: TateB Date: Thu, 23 Jul 2026 03:14:17 +0000 Subject: [PATCH 03/12] fix(storage): scope reorg invalidation scan Keep heartbeat reporting per candidate batch while restricting keyset pagination to verified resolution and verified primary-name outcomes in SQL. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01C9RTnbCyhqEUSPc7xCK76p --- crates/storage/src/execution/invalidation.rs | 3 +- crates/storage/src/execution/tests.rs | 43 ++++++++++++++++++-- 2 files changed, 42 insertions(+), 4 deletions(-) diff --git a/crates/storage/src/execution/invalidation.rs b/crates/storage/src/execution/invalidation.rs index 1ced3c87..7ce2f0e1 100644 --- a/crates/storage/src/execution/invalidation.rs +++ b/crates/storage/src/execution/invalidation.rs @@ -453,7 +453,8 @@ where topology_version_boundary, record_version_boundary FROM execution_cache_outcomes - WHERE ($1::text IS NULL OR execution_cache_key > $1) + WHERE request_type IN ('verified_resolution', 'verified_primary_name') + AND ($1::text IS NULL OR execution_cache_key > $1) ORDER BY execution_cache_key LIMIT $2 "#, diff --git a/crates/storage/src/execution/tests.rs b/crates/storage/src/execution/tests.rs index caf76be3..1927dd5b 100644 --- a/crates/storage/src/execution/tests.rs +++ b/crates/storage/src/execution/tests.rs @@ -2635,6 +2635,43 @@ async fn invalidates_verified_execution_outcomes_for_orphaned_block_dependencies "block_hash": "0xrequested-orphan" }]); insert_trace_and_outcome(&database, &out_of_scope_trace, &out_of_scope_outcome).await?; + sqlx::query( + r#" + INSERT INTO execution_cache_outcomes ( + execution_cache_key, + request_key, + requested_chain_positions, + manifest_versions, + topology_version_boundary, + record_version_boundary, + execution_trace_id, + request_type, + namespace, + outcome_payload, + failure_payload, + finished_at + ) + SELECT + 'dominant-declared-outcome-' || series::TEXT, + request_key, + requested_chain_positions, + manifest_versions, + topology_version_boundary, + record_version_boundary, + execution_trace_id, + request_type, + namespace, + outcome_payload, + failure_payload, + finished_at + FROM execution_cache_outcomes + CROSS JOIN generate_series(1, 5) AS series + WHERE request_type = 'declared_resolution' + "#, + ) + .execute(database.pool()) + .await + .context("failed to make out-of-scope execution outcomes dominate the invalidation fixture")?; let mut progress = CountingExecutionInvalidationProgress::default(); let summary = invalidate_execution_outcomes_for_orphaned_blocks_with_progress( @@ -2643,9 +2680,9 @@ async fn invalidates_verified_execution_outcomes_for_orphaned_block_dependencies ) .await?; assert_eq!(summary.deleted_outcome_count, 3); - assert!( - progress.count >= 3, - "reorg invalidation must report each completed cache-candidate source page" + assert_eq!( + progress.count, 2, + "reorg invalidation must page only the four verified cache candidates" ); assert_eq!( From b1723589967a79812bfe0f4988bf6db4f7329dfa Mon Sep 17 00:00:00 2001 From: TateB Date: Thu, 23 Jul 2026 03:14:33 +0000 Subject: [PATCH 04/12] fix(indexer): preserve persisted backfill identities Limit the progressive compact identity to source-family plans and retain the pre-existing whole-active and watched-target-set payloads and hashes while recording heartbeat progress. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01C9RTnbCyhqEUSPc7xCK76p --- .../backfill/reservation_execution/digest.rs | 103 ++++++++--- .../reservation_execution/identity.rs | 88 +++++++++- apps/indexer/src/main/tests/backfill.rs | 164 ++++++++++++++++++ 3 files changed, 333 insertions(+), 22 deletions(-) diff --git a/apps/indexer/src/main/backfill/reservation_execution/digest.rs b/apps/indexer/src/main/backfill/reservation_execution/digest.rs index 87725ad7..e2edcd43 100644 --- a/apps/indexer/src/main/backfill/reservation_execution/digest.rs +++ b/apps/indexer/src/main/backfill/reservation_execution/digest.rs @@ -53,40 +53,75 @@ pub(super) async fn keccak256_json_value_digest_with_progress( value: &serde_json::Value, progress: &mut dyn StartupAdapterProgress, ) -> Result { + let mut writer = Keccak256Writer::default(); + write_json_value_with_progress(pool, &mut writer, value, progress).await?; + Ok(format!("keccak256:{}", hex_string(&writer.finalize()))) +} + +pub(super) async fn fnv1a64_json_value_digest_with_progress( + pool: &sqlx::PgPool, + value: &serde_json::Value, + progress: &mut dyn StartupAdapterProgress, +) -> Result { + let mut writer = Fnv1a64Writer::default(); + write_json_value_with_progress(pool, &mut writer, value, progress).await?; + Ok(format!("fnv1a64:{:016x}", writer.finalize())) +} + +async fn write_json_value_with_progress( + pool: &sqlx::PgPool, + writer: &mut impl Write, + value: &serde_json::Value, + progress: &mut dyn StartupAdapterProgress, +) -> Result<()> { + if let Some(items) = value.as_array() { + return write_json_array_with_progress(pool, writer, items, progress).await; + } let Some(fields) = value.as_object() else { - return keccak256_json_digest(value); + serde_json::to_writer(writer, value).context("failed to serialize JSON digest value")?; + return Ok(()); }; - let mut writer = Keccak256Writer::default(); + writer.write_all(b"{")?; for (field_index, (key, value)) in fields.iter().enumerate() { if field_index > 0 { writer.write_all(b",")?; } - serde_json::to_writer(&mut writer, key).context("failed to serialize JSON digest key")?; + serde_json::to_writer(&mut *writer, key).context("failed to serialize JSON digest key")?; writer.write_all(b":")?; if let Some(items) = value.as_array() { - writer.write_all(b"[")?; - for (item_index, item) in items.iter().enumerate() { - if item_index > 0 { - writer.write_all(b",")?; - } - serde_json::to_writer(&mut writer, item) - .context("failed to serialize JSON digest array item")?; - if (item_index + 1).is_multiple_of(DIGEST_PROGRESS_ITEMS) { - progress.record(pool).await?; - } - } - writer.write_all(b"]")?; - if !items.is_empty() && !items.len().is_multiple_of(DIGEST_PROGRESS_ITEMS) { - progress.record(pool).await?; - } + write_json_array_with_progress(pool, writer, items, progress).await?; } else { - serde_json::to_writer(&mut writer, value) + serde_json::to_writer(&mut *writer, value) .context("failed to serialize JSON digest value")?; } } writer.write_all(b"}")?; - Ok(format!("keccak256:{}", hex_string(&writer.finalize()))) + Ok(()) +} + +async fn write_json_array_with_progress( + pool: &sqlx::PgPool, + writer: &mut impl Write, + items: &[serde_json::Value], + progress: &mut dyn StartupAdapterProgress, +) -> Result<()> { + writer.write_all(b"[")?; + for (item_index, item) in items.iter().enumerate() { + if item_index > 0 { + writer.write_all(b",")?; + } + serde_json::to_writer(&mut *writer, item) + .context("failed to serialize JSON digest array item")?; + if (item_index + 1).is_multiple_of(DIGEST_PROGRESS_ITEMS) { + progress.record(pool).await?; + } + } + writer.write_all(b"]")?; + if !items.is_empty() && !items.len().is_multiple_of(DIGEST_PROGRESS_ITEMS) { + progress.record(pool).await?; + } + Ok(()) } #[derive(Default)] @@ -111,6 +146,34 @@ impl Write for Keccak256Writer { } } +struct Fnv1a64Writer(u64); + +impl Default for Fnv1a64Writer { + fn default() -> Self { + Self(0xcbf29ce484222325) + } +} + +impl Fnv1a64Writer { + const fn finalize(self) -> u64 { + self.0 + } +} + +impl Write for Fnv1a64Writer { + fn write(&mut self, buf: &[u8]) -> io::Result { + for byte in buf { + self.0 ^= u64::from(*byte); + self.0 = self.0.wrapping_mul(0x100000001b3); + } + Ok(buf.len()) + } + + fn flush(&mut self) -> io::Result<()> { + Ok(()) + } +} + fn hex_string(bytes: &[u8]) -> String { format!("0x{}", hex::encode(bytes)) } diff --git a/apps/indexer/src/main/backfill/reservation_execution/identity.rs b/apps/indexer/src/main/backfill/reservation_execution/identity.rs index 38e286c8..faff8f01 100644 --- a/apps/indexer/src/main/backfill/reservation_execution/identity.rs +++ b/apps/indexer/src/main/backfill/reservation_execution/identity.rs @@ -1,7 +1,9 @@ use super::digest::{ - keccak256_json_value_digest_with_progress, keccak256_selected_targets_digest_with_progress, + fnv1a64_json_value_digest_with_progress, keccak256_json_value_digest_with_progress, + keccak256_selected_targets_digest_with_progress, }; use super::*; +use bigname_manifests::WatchedSourceSelectorKind; pub(crate) fn backfill_job_source_identity_payload( source_plan: &WatchedSourceSelectorPlan, @@ -13,7 +15,9 @@ pub(crate) fn backfill_job_source_identity_payload( return generic_topic_scan_source_identity_payload(source_plan); } - if source_plan.selected_targets.len() <= COMPACT_SOURCE_IDENTITY_SELECTED_TARGET_THRESHOLD { + if source_plan.selector_kind != WatchedSourceSelectorKind::SourceFamily + || source_plan.selected_targets.len() <= COMPACT_SOURCE_IDENTITY_SELECTED_TARGET_THRESHOLD + { return Ok(source_plan.source_identity_payload()); } @@ -57,6 +61,9 @@ pub(crate) async fn backfill_job_source_identity_payload_with_progress( ) .await; } + if source_plan.selector_kind != WatchedSourceSelectorKind::SourceFamily { + return source_identity_payload_with_progress(pool, source_plan, progress).await; + } if source_plan.selected_targets.len() <= COMPACT_SOURCE_IDENTITY_SELECTED_TARGET_THRESHOLD { return Ok(source_plan.source_identity_payload()); } @@ -94,6 +101,61 @@ pub(crate) async fn backfill_job_source_identity_payload_with_progress( Ok(payload) } +async fn source_identity_payload_with_progress( + pool: &sqlx::PgPool, + source_plan: &WatchedSourceSelectorPlan, + progress: &mut dyn StartupAdapterProgress, +) -> Result { + let requested_watched_targets = + requested_watched_targets_value_with_progress(pool, source_plan, progress).await?; + let selected_targets = + selected_targets_value_with_progress(pool, source_plan, progress).await?; + let mut full_payload = json!({ + "selector_kind": source_plan.selector_kind.as_str(), + "source_family": &source_plan.source_family, + "requested_watched_targets": requested_watched_targets, + "selected_targets": selected_targets, + }); + let source_identity_hash = + fnv1a64_json_value_digest_with_progress(pool, &full_payload, progress) + .await + .context("failed to digest backfill source identity")?; + + if source_plan.selector_kind == WatchedSourceSelectorKind::WholeActiveWatchedChain + && source_plan.selected_targets.len() > COMPACT_SOURCE_IDENTITY_SELECTED_TARGET_THRESHOLD + { + let selected_targets_digest = keccak256_json_value_digest_with_progress( + pool, + full_payload + .get("selected_targets") + .expect("full source identity payload must contain selected targets"), + progress, + ) + .await + .context("failed to digest whole-active backfill source selected targets")?; + return Ok(json!({ + "selector_kind": source_plan.selector_kind.as_str(), + "source_family": &source_plan.source_family, + "requested_watched_targets": &source_plan.requested_watched_targets, + "selected_target_count": source_plan.selected_targets.len(), + "selected_targets_digest_algorithm": "keccak256", + "selected_targets_digest": selected_targets_digest, + "selected_targets_sample": selected_targets_sample(&source_plan.selected_targets), + "source_identity_payload_format": "selected_targets_digest_v1", + "source_identity_hash": source_identity_hash, + })); + } + + full_payload + .as_object_mut() + .expect("full source identity payload must be an object") + .insert( + "source_identity_hash".to_owned(), + Value::String(source_identity_hash), + ); + Ok(full_payload) +} + pub(super) async fn requested_watched_targets_value_with_progress( pool: &sqlx::PgPool, source_plan: &WatchedSourceSelectorPlan, @@ -115,3 +177,25 @@ pub(super) async fn requested_watched_targets_value_with_progress( } Ok(Value::Array(targets)) } + +async fn selected_targets_value_with_progress( + pool: &sqlx::PgPool, + source_plan: &WatchedSourceSelectorPlan, + progress: &mut dyn StartupAdapterProgress, +) -> Result { + const PROGRESS_TARGETS: usize = 1_000; + let mut targets = Vec::with_capacity(source_plan.selected_targets.len()); + for target in &source_plan.selected_targets { + targets.push( + serde_json::to_value(target) + .context("failed to serialize selected backfill target identity")?, + ); + if targets.len().is_multiple_of(PROGRESS_TARGETS) { + progress.record(pool).await?; + } + } + if !targets.is_empty() && !targets.len().is_multiple_of(PROGRESS_TARGETS) { + progress.record(pool).await?; + } + Ok(Value::Array(targets)) +} diff --git a/apps/indexer/src/main/tests/backfill.rs b/apps/indexer/src/main/tests/backfill.rs index aa08c08a..d81ecaba 100644 --- a/apps/indexer/src/main/tests/backfill.rs +++ b/apps/indexer/src/main/tests/backfill.rs @@ -58,6 +58,23 @@ struct MaterializedRawFactSet { payload_cache_metadata_count: i64, } +#[derive(Default)] +struct CountingSourceIdentityProgress { + count: usize, +} + +impl bigname_adapters::StartupAdapterProgress for CountingSourceIdentityProgress { + fn record<'a>( + &'a mut self, + _pool: &'a PgPool, + ) -> bigname_adapters::StartupAdapterProgressFuture<'a> { + Box::pin(async move { + self.count += 1; + Ok(()) + }) + } +} + #[tokio::test] async fn pruned_primary_code_fallback_persists_like_primary_observation() -> Result<()> { let database = TestDatabase::new().await?; @@ -330,6 +347,153 @@ fn large_source_family_backfill_source_identity_uses_compact_digest() -> Result< Ok(()) } +#[tokio::test] +async fn large_backfill_source_identities_match_pre_heartbeat_values() -> Result<()> { + let selected_targets = (0..=backfill::COMPACT_SOURCE_IDENTITY_SELECTED_TARGET_THRESHOLD) + .map(|index| WatchedBackfillTarget { + source_family: "identity_fixture".to_owned(), + contract_instance_id: Uuid::from_u128(index as u128 + 1), + address: format!("0x{index:040x}"), + effective_from_block: index as i64, + effective_to_block: index as i64 + 10, + }) + .collect::>(); + let watched_chain_plan = WatchedChainPlan { + chain: "identity-chain".to_owned(), + addresses: Vec::new(), + manifest_root_entry_count: 0, + manifest_contract_entry_count: 0, + discovery_edge_entry_count: 0, + }; + let mut source_plan = WatchedSourceSelectorPlan { + chain: "identity-chain".to_owned(), + selector_kind: WatchedSourceSelectorKind::SourceFamily, + source_family: Some("identity_fixture".to_owned()), + requested_watched_targets: Vec::new(), + selected_targets, + watched_chain_plan, + }; + + let source_family_expected = json!({ + "selector_kind": "source_family", + "source_family": "identity_fixture", + "requested_watched_targets": [], + "selected_target_count": source_plan.selected_targets.len(), + "selected_targets_digest_algorithm": "keccak256", + "selected_targets_digest": + "keccak256:0x7d4a66659d48822809252feb8566d43fea06ce8b07ef8f3d47230c63c18bac44", + "selected_targets_sample": { + "first": source_plan.selected_targets.first(), + "last": source_plan.selected_targets.last(), + }, + "source_identity_payload_format": "selected_targets_digest_v1", + "source_identity_hash": + "keccak256:0xd836c01edfa5b2f5c4f575bc79e480a84ec4b0ebc1b2b40f6412eeb1541b4d26", + }); + + source_plan.selector_kind = WatchedSourceSelectorKind::WholeActiveWatchedChain; + source_plan.source_family = None; + let whole_active_expected = json!({ + "selector_kind": "whole_active_watched_chain", + "source_family": null, + "requested_watched_targets": [], + "selected_target_count": source_plan.selected_targets.len(), + "selected_targets_digest_algorithm": "keccak256", + "selected_targets_digest": + "keccak256:0xbecb80264009c12d3996d435bf591edb21c6467a014e1ecbe830bd2b0f63f547", + "selected_targets_sample": { + "first": source_plan.selected_targets.first(), + "last": source_plan.selected_targets.last(), + }, + "source_identity_payload_format": "selected_targets_digest_v1", + "source_identity_hash": "fnv1a64:557cee9b843c194a", + }); + + source_plan.selector_kind = WatchedSourceSelectorKind::WatchedTargetSet; + source_plan.requested_watched_targets = vec![ + source_plan.selected_targets[0].contract_instance_id.into(), + source_plan + .selected_targets + .last() + .expect("identity fixture has selected targets") + .contract_instance_id + .into(), + ]; + let watched_target_set_expected = json!({ + "selector_kind": "watched_target_set", + "source_family": null, + "requested_watched_targets": &source_plan.requested_watched_targets, + "selected_targets": &source_plan.selected_targets, + "source_identity_hash": "fnv1a64:8be4681e2c671f3a", + }); + + let pool = PgPoolOptions::new().connect_lazy(default_database_url())?; + let mut progress = CountingSourceIdentityProgress::default(); + + source_plan.selector_kind = WatchedSourceSelectorKind::SourceFamily; + source_plan.source_family = Some("identity_fixture".to_owned()); + source_plan.requested_watched_targets.clear(); + assert_eq!( + backfill::backfill_job_source_identity_payload(&source_plan)?, + source_family_expected + ); + assert_eq!( + backfill::backfill_job_source_identity_payload_with_progress( + &pool, + &source_plan, + &mut progress, + ) + .await?, + source_family_expected + ); + + source_plan.selector_kind = WatchedSourceSelectorKind::WholeActiveWatchedChain; + source_plan.source_family = None; + assert_eq!( + backfill::backfill_job_source_identity_payload(&source_plan)?, + whole_active_expected + ); + assert_eq!( + backfill::backfill_job_source_identity_payload_with_progress( + &pool, + &source_plan, + &mut progress, + ) + .await?, + whole_active_expected + ); + + source_plan.selector_kind = WatchedSourceSelectorKind::WatchedTargetSet; + source_plan.requested_watched_targets = vec![ + source_plan.selected_targets[0].contract_instance_id.into(), + source_plan + .selected_targets + .last() + .expect("identity fixture has selected targets") + .contract_instance_id + .into(), + ]; + assert_eq!( + backfill::backfill_job_source_identity_payload(&source_plan)?, + watched_target_set_expected + ); + assert_eq!( + backfill::backfill_job_source_identity_payload_with_progress( + &pool, + &source_plan, + &mut progress, + ) + .await?, + watched_target_set_expected + ); + assert!( + progress.count > 0, + "large progress-aware identity construction must retain heartbeat boundaries" + ); + + Ok(()) +} + #[test] fn large_whole_active_backfill_source_identity_uses_compact_digest() -> Result<()> { let selected_targets = (0..=backfill::COMPACT_SOURCE_IDENTITY_SELECTED_TARGET_THRESHOLD) From 4da5c7da806c0ab9fd9a38a09568d233232027c4 Mon Sep 17 00:00:00 2001 From: TateB Date: Thu, 23 Jul 2026 03:14:42 +0000 Subject: [PATCH 05/12] fix(indexer): restore atomic losing-branch orphaning Validate the losing lineage path with progress heartbeats before applying one transactional range update, so failures cannot leave mixed canonicality. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01C9RTnbCyhqEUSPc7xCK76p --- .../reconciliation/canonical/orphaning.rs | 12 +- apps/indexer/src/main/tests/reorg.rs | 105 ++++++++++++++++++ 2 files changed, 109 insertions(+), 8 deletions(-) diff --git a/apps/indexer/src/main/reconciliation/canonical/orphaning.rs b/apps/indexer/src/main/reconciliation/canonical/orphaning.rs index e6847fc2..d6f7d87b 100644 --- a/apps/indexer/src/main/reconciliation/canonical/orphaning.rs +++ b/apps/indexer/src/main/reconciliation/canonical/orphaning.rs @@ -43,7 +43,6 @@ pub(super) async fn orphan_canonical_branch_with_progress( stop_before_hash: Option<&str>, progress: &mut Option<&mut dyn StartupAdapterProgress>, ) -> Result { - let mut orphaned_block_count = 0usize; let mut cursor_hash = Some(from_hash.to_owned()); while let Some(block_hash) = cursor_hash { @@ -58,16 +57,13 @@ pub(super) async fn orphan_canonical_branch_with_progress( "missing stored lineage row for chain {chain} block {block_hash} while orphaning the losing branch" ) })?; - let parent_hash = block.parent_hash.clone(); - let snapshots = - mark_chain_lineage_range_orphaned(pool, chain, &block_hash, parent_hash.as_deref()) - .await?; - orphaned_block_count += snapshots.len(); - cursor_hash = parent_hash; + cursor_hash = block.parent_hash; record_live_progress(pool, progress).await?; } - Ok(orphaned_block_count) + let snapshots = + mark_chain_lineage_range_orphaned(pool, chain, from_hash, stop_before_hash).await?; + Ok(snapshots.len()) } #[allow(dead_code)] diff --git a/apps/indexer/src/main/tests/reorg.rs b/apps/indexer/src/main/tests/reorg.rs index 5ff040f3..b745d543 100644 --- a/apps/indexer/src/main/tests/reorg.rs +++ b/apps/indexer/src/main/tests/reorg.rs @@ -979,6 +979,111 @@ async fn reorg_common_ancestor_must_be_on_current_canonical_branch() -> Result<( database.cleanup().await } +#[tokio::test] +async fn losing_branch_orphaning_rolls_back_every_block_on_mid_branch_failure() -> Result<()> { + let database = TestDatabase::new().await?; + let chain = "ethereum-mainnet"; + let ancestor = provider_block( + "0x1010101010101010101010101010101010101010101010101010101010101010", + None, + 1, + ); + let losing_parent = provider_block( + "0x2020202020202020202020202020202020202020202020202020202020202020", + Some(&ancestor.block_hash), + 2, + ); + let losing_head = provider_block( + "0x3030303030303030303030303030303030303030303030303030303030303030", + Some(&losing_parent.block_hash), + 3, + ); + upsert_chain_lineage_blocks( + database.pool(), + &[ + provider_block_to_lineage(chain, &ancestor, CanonicalityState::Canonical), + provider_block_to_lineage(chain, &losing_parent, CanonicalityState::Canonical), + provider_block_to_lineage(chain, &losing_head, CanonicalityState::Canonical), + ], + ) + .await?; + sqlx::query( + r#" + CREATE FUNCTION fail_middle_losing_branch_orphaning() + RETURNS TRIGGER + LANGUAGE plpgsql + AS $$ + BEGIN + IF NEW.block_hash = + '0x2020202020202020202020202020202020202020202020202020202020202020' + AND NEW.canonicality_state = 'orphaned'::canonicality_state + THEN + RAISE EXCEPTION 'injected failure while orphaning the middle losing block'; + END IF; + RETURN NEW; + END; + $$ + "#, + ) + .execute(database.pool()) + .await?; + sqlx::query( + r#" + CREATE TRIGGER fail_middle_losing_branch_orphaning + BEFORE UPDATE OF canonicality_state ON chain_lineage + FOR EACH ROW + EXECUTE FUNCTION fail_middle_losing_branch_orphaning() + "#, + ) + .execute(database.pool()) + .await?; + + let error = orphan_canonical_branch( + database.pool(), + chain, + &losing_head.block_hash, + Some(&ancestor.block_hash), + ) + .await + .expect_err("the injected middle-block failure must abort losing-branch orphaning"); + assert!( + format!("{error:#}").contains("injected failure while orphaning the middle losing block"), + "unexpected orphaning error: {error:#}" + ); + + let states = sqlx::query_as::<_, (String, String)>( + r#" + SELECT block_hash, canonicality_state::TEXT + FROM chain_lineage + WHERE chain_id = $1 + AND block_hash = ANY($2::TEXT[]) + ORDER BY block_number + "#, + ) + .bind(chain) + .bind([losing_parent.block_hash, losing_head.block_hash]) + .fetch_all(database.pool()) + .await?; + assert_eq!( + states, + vec![ + ( + "0x2020202020202020202020202020202020202020202020202020202020202020" + .to_owned(), + "canonical".to_owned(), + ), + ( + "0x3030303030303030303030303030303030303030303030303030303030303030" + .to_owned(), + "canonical".to_owned(), + ), + ], + "a failed losing-branch repair must not leave mixed canonicality" + ); + + database.cleanup().await +} + #[tokio::test] async fn awaiting_ancestor_raw_persistence_preserves_walked_orphaned_lineage() -> Result<()> { let database = TestDatabase::new().await?; From 9b19ddd0f75f9621b16eed5aed7f9a5a25077013 Mon Sep 17 00:00:00 2001 From: TateB Date: Thu, 23 Jul 2026 04:50:44 +0000 Subject: [PATCH 06/12] fix(paging): scope heartbeat page sources Apply operation scope before LIMIT across retained witnesses, discovery reconciliation, watched coverage, resolver convergence, and related audit sweep sites. Add dominated-table exact-page-count regressions and correct the exact-boundary requirement beat. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01C9RTnbCyhqEUSPc7xCK76p --- Cargo.lock | 1 + .../invalidations/capture.rs | 120 +++--- .../resolver_profile_convergence/tests.rs | 348 ++++++++++++++++++ crates/adapters/Cargo.toml | 1 + .../resolver_profile_reconciliation.rs | 34 +- .../progress_tests.rs | 210 +++++++++++ .../targets.rs | 25 +- .../targets/tests.rs | 124 +++++++ .../completeness/requirements/witnesses.rs | 37 +- .../requirements/witnesses/tests.rs | 235 ++++++++++++ .../src/lib/discovery/loading/progress.rs | 38 +- .../lib/discovery/loading/progress/tests.rs | 154 ++++++++ .../discovery/reconciliation/streamed/diff.rs | 23 +- .../reconciliation/streamed/diff/tests.rs | 123 +++++++ .../reconciliation/streamed/progress.rs | 9 +- .../reconciliation/streamed/progress/tests.rs | 107 ++++++ .../src/lib/managed_edges/source_graph.rs | 35 +- .../lib/managed_edges/source_graph/tests.rs | 148 ++++++++ .../src/lib/views/watched/frontier.rs | 51 +-- .../views/watched/frontier/source_pages.rs | 37 +- .../src/lib/views/watched/frontier/tests.rs | 224 +++++++++++ 21 files changed, 1932 insertions(+), 152 deletions(-) create mode 100644 crates/adapters/src/ens_v1_unwrapped_authority/resolver_profile_reconciliation/progress_tests.rs create mode 100644 crates/adapters/src/ens_v1_unwrapped_authority/resolver_profile_reconciliation/targets/tests.rs create mode 100644 crates/adapters/src/ens_v2_registry/live/completeness/requirements/witnesses/tests.rs create mode 100644 crates/manifests/src/lib/discovery/loading/progress/tests.rs create mode 100644 crates/manifests/src/lib/discovery/reconciliation/streamed/diff/tests.rs create mode 100644 crates/manifests/src/lib/discovery/reconciliation/streamed/progress/tests.rs create mode 100644 crates/manifests/src/lib/managed_edges/source_graph/tests.rs diff --git a/Cargo.lock b/Cargo.lock index 23aa2c61..c12bbd94 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1667,6 +1667,7 @@ dependencies = [ "bigname-domain", "bigname-manifests", "bigname-storage", + "bigname-test-support", "futures-util", "serde", "serde_json", diff --git a/apps/indexer/src/main/resolver_profile_convergence/invalidations/capture.rs b/apps/indexer/src/main/resolver_profile_convergence/invalidations/capture.rs index a34d6fa5..0df1d555 100644 --- a/apps/indexer/src/main/resolver_profile_convergence/invalidations/capture.rs +++ b/apps/indexer/src/main/resolver_profile_convergence/invalidations/capture.rs @@ -132,35 +132,12 @@ async fn materialize_bound_names( r#" WITH source_page AS MATERIALIZED ( SELECT - normalized_event_id, - chain_id, - logical_name_id, - event_kind, - canonicality_state, - before_state, - after_state - FROM normalized_events - WHERE normalized_event_id > $1 - AND normalized_event_id <= $2 - ORDER BY normalized_event_id - LIMIT $3 - ), - page_end AS ( - SELECT MAX(normalized_event_id) AS last_id - FROM source_page - ), - bound_names AS ( - SELECT DISTINCT event.logical_name_id - FROM source_page event - JOIN LATERAL ( - VALUES - (event.before_state ->> 'resolver'), - (event.after_state ->> 'resolver') - ) resolver(resolver_address) ON TRUE - JOIN resolver_profile_reconciliation_targets target - ON target.run_id = $4 - AND target.resolver_address = lower(resolver.resolver_address) - WHERE event.chain_id = $5 + event.normalized_event_id, + event.logical_name_id + FROM normalized_events event + WHERE event.normalized_event_id > $1 + AND event.normalized_event_id <= $2 + AND event.chain_id = $5 AND event.event_kind = 'ResolverChanged' AND event.logical_name_id IS NOT NULL AND event.canonicality_state IN ( @@ -168,6 +145,25 @@ async fn materialize_bound_names( 'safe'::canonicality_state, 'finalized'::canonicality_state ) + AND EXISTS ( + SELECT 1 + FROM resolver_profile_reconciliation_targets target + WHERE target.run_id = $4 + AND target.resolver_address IN ( + lower(event.before_state ->> 'resolver'), + lower(event.after_state ->> 'resolver') + ) + ) + ORDER BY event.normalized_event_id + LIMIT $3 + ), + page_end AS ( + SELECT MAX(normalized_event_id) AS last_id + FROM source_page + ), + bound_names AS ( + SELECT DISTINCT logical_name_id + FROM source_page ) SELECT page_end.last_id, bound.logical_name_id FROM page_end @@ -212,25 +208,15 @@ async fn stage_event_resources( let rows = sqlx::query_as::<_, (Option, Option)>(&format!( r#" WITH source_page AS MATERIALIZED ( - SELECT normalized_event_id, logical_name_id, resource_id, canonicality_state - FROM normalized_events - WHERE normalized_event_id > $1 - AND normalized_event_id <= $2 - ORDER BY normalized_event_id - LIMIT $3 - ), - page_end AS ( - SELECT MAX(normalized_event_id) AS last_id - FROM source_page - ), - resources AS ( - SELECT DISTINCT event.resource_id - FROM source_page event + SELECT event.normalized_event_id, event.resource_id + FROM normalized_events event JOIN {TEMP_BOUND_NAMES} name ON name.logical_name_id = event.logical_name_id JOIN resources resource ON resource.resource_id = event.resource_id - WHERE event.resource_id IS NOT NULL + WHERE event.normalized_event_id > $1 + AND event.normalized_event_id <= $2 + AND event.resource_id IS NOT NULL AND event.canonicality_state IN ( 'canonical'::canonicality_state, 'safe'::canonicality_state, @@ -241,6 +227,16 @@ async fn stage_event_resources( 'safe'::canonicality_state, 'finalized'::canonicality_state ) + ORDER BY event.normalized_event_id + LIMIT $3 + ), + page_end AS ( + SELECT MAX(normalized_event_id) AS last_id + FROM source_page + ), + resources AS ( + SELECT DISTINCT resource_id + FROM source_page ) SELECT page_end.last_id, resource.resource_id FROM page_end @@ -282,29 +278,15 @@ async fn stage_binding_resources( r#" WITH source_page AS MATERIALIZED ( SELECT - surface_binding_id, - logical_name_id, - resource_id, - canonicality_state - FROM surface_bindings - WHERE ($1::UUID IS NULL OR surface_binding_id > $1) - ORDER BY surface_binding_id - LIMIT $2 - ), - page_end AS ( - SELECT surface_binding_id AS last_id - FROM source_page - ORDER BY surface_binding_id DESC - LIMIT 1 - ), - resources AS ( - SELECT DISTINCT binding.resource_id - FROM source_page binding + binding.surface_binding_id, + binding.resource_id + FROM surface_bindings binding JOIN {TEMP_BOUND_NAMES} name ON name.logical_name_id = binding.logical_name_id JOIN resources resource ON resource.resource_id = binding.resource_id - WHERE binding.canonicality_state IN ( + WHERE ($1::UUID IS NULL OR binding.surface_binding_id > $1) + AND binding.canonicality_state IN ( 'canonical'::canonicality_state, 'safe'::canonicality_state, 'finalized'::canonicality_state @@ -314,6 +296,18 @@ async fn stage_binding_resources( 'safe'::canonicality_state, 'finalized'::canonicality_state ) + ORDER BY binding.surface_binding_id + LIMIT $2 + ), + page_end AS ( + SELECT surface_binding_id AS last_id + FROM source_page + ORDER BY surface_binding_id DESC + LIMIT 1 + ), + resources AS ( + SELECT DISTINCT resource_id + FROM source_page ) SELECT page_end.last_id, resource.resource_id FROM page_end diff --git a/apps/indexer/src/main/resolver_profile_convergence/tests.rs b/apps/indexer/src/main/resolver_profile_convergence/tests.rs index 2ba1ddfe..7baa4add 100644 --- a/apps/indexer/src/main/resolver_profile_convergence/tests.rs +++ b/apps/indexer/src/main/resolver_profile_convergence/tests.rs @@ -11,8 +11,24 @@ use super::{ drain_resolver_profile_input_changes, drain_resolver_profile_input_changes_with_progress, expanded_reconciliation_targets, expanded_reconciliation_targets_with_family_count, input_requires_reconciliation, + invalidations::stage_resolver_profile_projection_invalidations_with_progress, }; +#[derive(Default)] +struct CountingStartupProgress { + count: usize, +} + +impl bigname_adapters::StartupAdapterProgress for CountingStartupProgress { + fn record<'a>( + &'a mut self, + _pool: &'a sqlx::PgPool, + ) -> bigname_adapters::StartupAdapterProgressFuture<'a> { + self.count += 1; + Box::pin(async { Ok(()) }) + } +} + #[test] fn completion_guard_refuses_only_the_deferred_chain() { let summary = ResolverProfileConvergenceSummary { @@ -94,6 +110,338 @@ fn authority_index(authority: ResolverProfileAuthoritySnapshot) -> ResolverProfi ResolverProfileAuthorityIndex::from_snapshot(authority) } +#[tokio::test] +async fn invalidation_capture_pages_only_bound_names_and_their_resources() -> anyhow::Result<()> { + let database = TestDatabase::create_migrated( + TestDatabaseConfig::new("resolver_invalidation_scoped_pages"), + &bigname_storage::MIGRATOR, + "failed to migrate resolver invalidation paging test database", + ) + .await?; + let pool = database.pool(); + let chain = "resolver-invalidation-chain"; + let resolver = "0x0000000000000000000000000000000000000d01"; + let run_id = Uuid::from_u128(0xd01); + let target_resource = Uuid::from_u128(0xd02); + let other_resource = Uuid::from_u128(0xd03); + + sqlx::query( + r#" + INSERT INTO resolver_profile_reconciliation_runs ( + run_id, + chain_id, + first_block_number, + last_block_number, + resolver_address_count, + resolver_address_set_digest + ) + VALUES ($1, $2, 0, 0, 1, 'test-digest') + "#, + ) + .bind(run_id) + .bind(chain) + .execute(pool) + .await?; + sqlx::query( + r#" + INSERT INTO name_surfaces ( + logical_name_id, + namespace, + input_name, + canonical_display_name, + normalized_name, + dns_encoded_name, + namehash, + labelhashes, + normalizer_version, + chain_id, + block_hash, + block_number, + canonicality_state + ) + SELECT + 'ens:other-' || value, + 'ens', + 'other-' || value, + 'other-' || value, + 'other-' || value, + '\x00', + 'other-namehash-' || value, + ARRAY[]::TEXT[], + 'test', + 'other-chain', + 'other-block', + 1, + 'canonical'::canonicality_state + FROM generate_series(1, 2001) value + "#, + ) + .execute(pool) + .await?; + sqlx::query( + "INSERT INTO resolver_profile_reconciliation_targets (run_id, resolver_address) VALUES ($1, $2)", + ) + .bind(run_id) + .bind(resolver) + .execute(pool) + .await?; + sqlx::query( + r#" + INSERT INTO normalized_events ( + event_identity, + namespace, + event_kind, + source_family, + manifest_version, + chain_id, + raw_fact_ref, + derivation_kind, + canonicality_state + ) + SELECT + 'out-of-scope-invalidation-event-' || value, + 'ens', + 'Other', + 'other_family', + 1, + 'other-chain', + '{}'::JSONB, + 'other_derivation', + 'canonical'::canonicality_state + FROM generate_series(1, 2001) value + "#, + ) + .execute(pool) + .await?; + sqlx::query( + r#" + INSERT INTO name_surfaces ( + logical_name_id, + namespace, + input_name, + canonical_display_name, + normalized_name, + dns_encoded_name, + namehash, + labelhashes, + normalizer_version, + chain_id, + block_hash, + block_number, + canonicality_state + ) + VALUES + ( + 'ens:target', + 'ens', + 'target', + 'target', + 'target', + '\x00', + 'target-namehash', + ARRAY[]::TEXT[], + 'test', + $1, + 'target-block', + 10, + 'canonical'::canonicality_state + ), + ( + 'ens:other', + 'ens', + 'other', + 'other', + 'other', + '\x00', + 'other-namehash', + ARRAY[]::TEXT[], + 'test', + 'other-chain', + 'other-block', + 1, + 'canonical'::canonicality_state + ) + "#, + ) + .bind(chain) + .execute(pool) + .await?; + sqlx::query( + r#" + INSERT INTO resources ( + resource_id, + chain_id, + block_hash, + block_number, + canonicality_state + ) + VALUES + ($1, $3, 'target-block', 10, 'canonical'::canonicality_state), + ($2, 'other-chain', 'other-block', 1, 'canonical'::canonicality_state) + "#, + ) + .bind(target_resource) + .bind(other_resource) + .bind(chain) + .execute(pool) + .await?; + sqlx::query( + r#" + INSERT INTO normalized_events ( + event_identity, + namespace, + logical_name_id, + event_kind, + source_family, + manifest_version, + chain_id, + raw_fact_ref, + derivation_kind, + canonicality_state, + before_state, + after_state + ) + VALUES ( + 'target-resolver-change', + 'ens', + 'ens:target', + 'ResolverChanged', + 'ens_v1_resolver_l1', + 1, + $1, + '{}'::JSONB, + 'ens_v1_unwrapped_authority', + 'canonical'::canonicality_state, + '{}'::JSONB, + jsonb_build_object('resolver', $2) + ) + "#, + ) + .bind(chain) + .bind(resolver) + .execute(pool) + .await?; + sqlx::query( + r#" + INSERT INTO normalized_events ( + event_identity, + namespace, + logical_name_id, + resource_id, + event_kind, + source_family, + manifest_version, + chain_id, + raw_fact_ref, + derivation_kind, + canonicality_state + ) + VALUES ( + 'target-resource-event', + 'ens', + 'ens:target', + $1, + 'RecordChanged', + 'ens_v1_resolver_l1', + 1, + $2, + '{}'::JSONB, + 'ens_v1_unwrapped_authority', + 'canonical'::canonicality_state + ) + "#, + ) + .bind(target_resource) + .bind(chain) + .execute(pool) + .await?; + sqlx::query( + r#" + INSERT INTO surface_bindings ( + surface_binding_id, + logical_name_id, + resource_id, + binding_kind, + active_from, + chain_id, + block_hash, + block_number, + canonicality_state + ) + SELECT + md5('out-of-scope-binding-' || value)::UUID, + 'ens:other-' || value, + $1, + 'observed_only', + now(), + 'other-chain', + 'other-block', + 1, + 'canonical'::canonicality_state + FROM generate_series(1, 2001) value + "#, + ) + .bind(other_resource) + .execute(pool) + .await?; + sqlx::query( + r#" + INSERT INTO surface_bindings ( + surface_binding_id, + logical_name_id, + resource_id, + binding_kind, + active_from, + chain_id, + block_hash, + block_number, + canonicality_state + ) + VALUES ( + $1, + 'ens:target', + $2, + 'observed_only', + now(), + $3, + 'target-block', + 10, + 'canonical'::canonicality_state + ) + "#, + ) + .bind(Uuid::from_u128(0xd04)) + .bind(target_resource) + .bind(chain) + .execute(pool) + .await?; + + let mut progress = CountingStartupProgress::default(); + stage_resolver_profile_projection_invalidations_with_progress( + pool, + run_id, + chain, + &mut progress, + ) + .await?; + + assert_eq!( + progress.count, 4, + "one target, bound-name, event-resource, and binding-resource page must beat" + ); + assert_eq!( + sqlx::query_scalar::<_, i64>( + "SELECT COUNT(*)::BIGINT FROM resolver_profile_reconciliation_invalidation_keys WHERE chain_id = $1", + ) + .bind(chain) + .fetch_one(pool) + .await?, + 2, + "the resolver and record-inventory keys must be staged once" + ); + database.cleanup().await +} + #[test] fn candidate_change_reconciles_only_the_dirty_address() { let dirty = "0x0000000000000000000000000000000000000002"; diff --git a/crates/adapters/Cargo.toml b/crates/adapters/Cargo.toml index 6ab8897e..cd14feac 100644 --- a/crates/adapters/Cargo.toml +++ b/crates/adapters/Cargo.toml @@ -21,5 +21,6 @@ tokio.workspace = true tracing.workspace = true [dev-dependencies] +bigname-test-support.workspace = true tokio.workspace = true uuid.workspace = true diff --git a/crates/adapters/src/ens_v1_unwrapped_authority/resolver_profile_reconciliation.rs b/crates/adapters/src/ens_v1_unwrapped_authority/resolver_profile_reconciliation.rs index 5ea62737..a1edf8b3 100644 --- a/crates/adapters/src/ens_v1_unwrapped_authority/resolver_profile_reconciliation.rs +++ b/crates/adapters/src/ens_v1_unwrapped_authority/resolver_profile_reconciliation.rs @@ -352,20 +352,8 @@ async fn publish_resolver_profile_events( let (last_event_id, page_orphaned_count) = sqlx::query_as::<_, (Option, i64)>( r#" WITH source_page AS MATERIALIZED ( - SELECT * - FROM normalized_events - WHERE normalized_event_id > $5 - AND normalized_event_id <= $6 - ORDER BY normalized_event_id - LIMIT 1000 - ), - page_end AS ( - SELECT MAX(normalized_event_id) AS last_event_id - FROM source_page - ), - stale AS ( - SELECT DISTINCT event.normalized_event_id - FROM source_page event + SELECT event.normalized_event_id + FROM normalized_events event JOIN raw_logs raw_log ON raw_log.chain_id = event.chain_id AND raw_log.block_hash = event.block_hash @@ -377,7 +365,9 @@ async fn publish_resolver_profile_events( JOIN resolver_profile_reconciliation_targets target ON target.run_id = $4 AND target.resolver_address = LOWER(raw_log.emitting_address) - WHERE event.chain_id = $1 + WHERE event.normalized_event_id > $5 + AND event.normalized_event_id <= $6 + AND event.chain_id = $1 AND event.derivation_kind = 'ens_v1_unwrapped_authority' AND event.source_family IN ('ens_v1_resolver_l1', 'basenames_base_resolver') AND event.raw_fact_ref->>'kind' = 'raw_log' @@ -392,12 +382,18 @@ async fn publish_resolver_profile_events( AND event.canonicality_state IN ('canonical', 'safe', 'finalized') AND raw_log.canonicality_state IN ('canonical', 'safe', 'finalized') AND lineage.canonicality_state IN ('canonical', 'safe', 'finalized') + ORDER BY event.normalized_event_id + LIMIT 1000 + ), + page_end AS ( + SELECT MAX(normalized_event_id) AS last_event_id + FROM source_page ), updated AS ( UPDATE normalized_events event SET canonicality_state = 'orphaned'::canonicality_state, observed_at = now() - FROM stale - WHERE event.normalized_event_id = stale.normalized_event_id + FROM source_page + WHERE event.normalized_event_id = source_page.normalized_event_id RETURNING 1 ) SELECT @@ -442,3 +438,7 @@ async fn record_progress( } Ok(()) } + +#[cfg(test)] +#[path = "resolver_profile_reconciliation/progress_tests.rs"] +mod progress_tests; diff --git a/crates/adapters/src/ens_v1_unwrapped_authority/resolver_profile_reconciliation/progress_tests.rs b/crates/adapters/src/ens_v1_unwrapped_authority/resolver_profile_reconciliation/progress_tests.rs new file mode 100644 index 00000000..c2e23398 --- /dev/null +++ b/crates/adapters/src/ens_v1_unwrapped_authority/resolver_profile_reconciliation/progress_tests.rs @@ -0,0 +1,210 @@ +use bigname_test_support::{TestDatabase, TestDatabaseConfig}; + +use super::*; + +#[derive(Default)] +struct CountingProgress { + count: usize, +} + +impl StartupAdapterProgress for CountingProgress { + fn record<'a>( + &'a mut self, + _pool: &'a PgPool, + ) -> crate::checkpoint_context::StartupAdapterProgressFuture<'a> { + self.count += 1; + Box::pin(async { Ok(()) }) + } +} + +#[tokio::test] +async fn resolver_event_orphaning_pages_only_stale_target_events() -> Result<()> { + let database = TestDatabase::create_migrated( + TestDatabaseConfig::new("resolver_event_orphan_scoped_pages"), + &bigname_storage::MIGRATOR, + "failed to migrate resolver-event orphan paging test database", + ) + .await?; + let pool = database.pool(); + let chain = "resolver-orphan-chain"; + let resolver = "0x0000000000000000000000000000000000000701"; + let run_id = Uuid::from_u128(0x701); + + sqlx::query( + r#" + INSERT INTO resolver_profile_reconciliation_runs ( + run_id, + chain_id, + first_block_number, + last_block_number, + resolver_address_count, + resolver_address_set_digest, + status + ) + VALUES ($1, $2, 10, 10, 1, 'test-digest', 'replay_complete') + "#, + ) + .bind(run_id) + .bind(chain) + .execute(pool) + .await?; + sqlx::query( + "INSERT INTO resolver_profile_reconciliation_targets (run_id, resolver_address) VALUES ($1, $2)", + ) + .bind(run_id) + .bind(resolver) + .execute(pool) + .await?; + + sqlx::query( + r#" + INSERT INTO normalized_events ( + event_identity, + namespace, + event_kind, + source_family, + manifest_version, + chain_id, + block_number, + block_hash, + transaction_hash, + log_index, + raw_fact_ref, + derivation_kind, + canonicality_state + ) + SELECT + 'out-of-scope-orphan-event-' || value, + 'ens', + 'Other', + 'other_family', + 1, + 'other-chain', + 1, + 'out-of-scope-block', + 'out-of-scope-transaction-' || value, + value, + '{}'::JSONB, + 'other_derivation', + 'canonical'::canonicality_state + FROM generate_series(1, 2001) value + "#, + ) + .execute(pool) + .await?; + sqlx::query( + r#" + INSERT INTO chain_lineage ( + chain_id, + block_hash, + block_number, + block_timestamp, + canonicality_state + ) + VALUES ($1, 'target-block', 10, now(), 'canonical'::canonicality_state) + "#, + ) + .bind(chain) + .execute(pool) + .await?; + sqlx::query( + r#" + INSERT INTO raw_logs ( + chain_id, + block_hash, + block_number, + transaction_hash, + transaction_index, + log_index, + emitting_address, + canonicality_state + ) + VALUES ( + $1, + 'target-block', + 10, + 'target-transaction', + 0, + 0, + $2, + 'canonical'::canonicality_state + ) + "#, + ) + .bind(chain) + .bind(resolver) + .execute(pool) + .await?; + sqlx::query( + r#" + INSERT INTO normalized_events ( + event_identity, + namespace, + event_kind, + source_family, + manifest_version, + chain_id, + block_number, + block_hash, + transaction_hash, + log_index, + raw_fact_ref, + derivation_kind, + canonicality_state + ) + VALUES ( + 'stale-target-event', + 'ens', + 'AddrChanged', + 'ens_v1_resolver_l1', + 1, + $1, + 10, + 'target-block', + 'target-transaction', + 0, + '{"kind":"raw_log"}'::JSONB, + 'ens_v1_unwrapped_authority', + 'canonical'::canonicality_state + ) + "#, + ) + .bind(chain) + .execute(pool) + .await?; + + let mut transaction = pool.begin().await?; + let mut progress = CountingProgress::default(); + let mut progress_ref = Some(&mut progress as &mut dyn StartupAdapterProgress); + let result = publish_resolver_profile_events( + pool, + &mut transaction, + chain, + ResolverEmitterReplayRange { + first_block_number: 10, + last_block_number: 10, + resolver_block_count: 1, + }, + run_id, + "test-digest", + 1, + &mut progress_ref, + ) + .await?; + transaction.commit().await?; + + assert_eq!(result, (0, 1)); + assert_eq!( + progress.count, 1, + "only the scoped stale-event page must beat" + ); + assert_eq!( + sqlx::query_scalar::<_, String>( + "SELECT canonicality_state::TEXT FROM normalized_events WHERE event_identity = 'stale-target-event'", + ) + .fetch_one(pool) + .await?, + "orphaned" + ); + database.cleanup().await +} diff --git a/crates/adapters/src/ens_v1_unwrapped_authority/resolver_profile_reconciliation/targets.rs b/crates/adapters/src/ens_v1_unwrapped_authority/resolver_profile_reconciliation/targets.rs index 3f0095e5..2fe401bc 100644 --- a/crates/adapters/src/ens_v1_unwrapped_authority/resolver_profile_reconciliation/targets.rs +++ b/crates/adapters/src/ens_v1_unwrapped_authority/resolver_profile_reconciliation/targets.rs @@ -312,13 +312,24 @@ async fn load_resolver_emitter_replay_range( loop { let raw_log_ids = sqlx::query_scalar::<_, i64>( r#" - SELECT raw_log_id - FROM raw_logs - WHERE raw_log_id > $1 - ORDER BY raw_log_id - LIMIT $2 + SELECT raw_log.raw_log_id + FROM raw_logs raw_log + JOIN resolver_profile_reconciliation_targets target + ON target.run_id = $2 + AND target.resolver_address = LOWER(raw_log.emitting_address) + WHERE raw_log.chain_id = $1 + AND raw_log.raw_log_id > $3 + AND raw_log.canonicality_state IN ( + 'canonical'::canonicality_state, + 'safe'::canonicality_state, + 'finalized'::canonicality_state + ) + ORDER BY raw_log.raw_log_id + LIMIT $4 "#, ) + .bind(chain) + .bind(run_id) .bind(after_raw_log_id) .bind(i64::try_from(TARGET_BATCH_SIZE)?) .fetch_all(transaction.as_mut()) @@ -415,3 +426,7 @@ async fn load_resolver_emitter_replay_range( resolver_block_count, })) } + +#[cfg(test)] +#[path = "targets/tests.rs"] +mod tests; diff --git a/crates/adapters/src/ens_v1_unwrapped_authority/resolver_profile_reconciliation/targets/tests.rs b/crates/adapters/src/ens_v1_unwrapped_authority/resolver_profile_reconciliation/targets/tests.rs new file mode 100644 index 00000000..fb34261a --- /dev/null +++ b/crates/adapters/src/ens_v1_unwrapped_authority/resolver_profile_reconciliation/targets/tests.rs @@ -0,0 +1,124 @@ +use bigname_test_support::{TestDatabase, TestDatabaseConfig}; + +use super::*; + +#[derive(Default)] +struct CountingProgress { + count: usize, +} + +impl StartupAdapterProgress for CountingProgress { + fn record<'a>( + &'a mut self, + _pool: &'a PgPool, + ) -> crate::checkpoint_context::StartupAdapterProgressFuture<'a> { + self.count += 1; + Box::pin(async { Ok(()) }) + } +} + +#[tokio::test] +async fn resolver_replay_range_pages_only_target_emitters_on_the_requested_chain() -> Result<()> { + let database = TestDatabase::create_migrated( + TestDatabaseConfig::new("resolver_replay_range_scoped_pages"), + &bigname_storage::MIGRATOR, + "failed to migrate resolver replay-range paging test database", + ) + .await?; + let pool = database.pool(); + let chain = "resolver-target-chain"; + let resolver = "0x0000000000000000000000000000000000000601"; + + sqlx::query( + r#" + INSERT INTO raw_logs ( + chain_id, + block_hash, + block_number, + transaction_hash, + transaction_index, + log_index, + emitting_address, + canonicality_state + ) + SELECT + 'other-chain', + 'out-of-scope-block', + 1, + 'out-of-scope-transaction', + 0, + value, + '0x0000000000000000000000000000000000000999', + 'canonical'::canonicality_state + FROM generate_series(1, 2001) value + "#, + ) + .execute(pool) + .await?; + sqlx::query( + r#" + INSERT INTO chain_lineage ( + chain_id, + block_hash, + block_number, + block_timestamp, + canonicality_state + ) + VALUES ($1, 'target-block', 10, now(), 'canonical'::canonicality_state) + "#, + ) + .bind(chain) + .execute(pool) + .await?; + sqlx::query( + r#" + INSERT INTO raw_logs ( + chain_id, + block_hash, + block_number, + transaction_hash, + transaction_index, + log_index, + emitting_address, + canonicality_state + ) + VALUES ( + $1, + 'target-block', + 10, + 'target-transaction', + 0, + 0, + $2, + 'canonical'::canonicality_state + ) + "#, + ) + .bind(chain) + .bind(resolver) + .execute(pool) + .await?; + + let mut reconciliation = begin_reconciliation(pool, chain).await?; + reconciliation + .stage_addresses(&[resolver.to_owned()]) + .await?; + let mut progress = CountingProgress::default(); + let mut progress_ref = Some(&mut progress as &mut dyn StartupAdapterProgress); + let prepared = reconciliation.prepare(&mut progress_ref).await?; + + assert_eq!( + prepared.replay_range, + Some(ResolverEmitterReplayRange { + first_block_number: 10, + last_block_number: 10, + resolver_block_count: 1, + }) + ); + assert_eq!( + progress.count, 2, + "one target page and one scoped raw-log page must beat" + ); + drop(reconciliation); + database.cleanup().await +} diff --git a/crates/adapters/src/ens_v2_registry/live/completeness/requirements/witnesses.rs b/crates/adapters/src/ens_v2_registry/live/completeness/requirements/witnesses.rs index 141b1700..8b836757 100644 --- a/crates/adapters/src/ens_v2_registry/live/completeness/requirements/witnesses.rs +++ b/crates/adapters/src/ens_v2_registry/live/completeness/requirements/witnesses.rs @@ -100,7 +100,8 @@ async fn requirement_indexes( .push(interval); addresses.entry(address).or_default().push(interval); } - if (index + 1).is_multiple_of(WITNESS_PAGE_ROWS as usize) { + if index + 1 < requirements.len() && (index + 1).is_multiple_of(WITNESS_PAGE_ROWS as usize) + { record_progress(pool, progress).await?; } } @@ -127,9 +128,23 @@ async fn verify_event_witnesses( let mut after_id = 0i64; loop { let ids = sqlx::query_scalar::<_, i64>( - "SELECT normalized_event_id FROM normalized_events WHERE normalized_event_id > $1 ORDER BY normalized_event_id LIMIT $2", + r#" + SELECT normalized_event_id + FROM normalized_events + WHERE normalized_event_id > $1 + AND chain_id = $2 + AND derivation_kind = $3 + AND raw_fact_ref ->> 'kind' = 'raw_log' + AND block_number <= $4 + AND canonicality_state IN ('canonical', 'safe', 'finalized') + ORDER BY normalized_event_id + LIMIT $5 + "#, ) .bind(after_id) + .bind(chain) + .bind(DERIVATION_KIND_ENS_V2_REGISTRY_RESOURCE_SURFACE) + .bind(through_block) .bind(WITNESS_PAGE_ROWS) .fetch_all(&mut *connection) .await @@ -211,9 +226,21 @@ async fn verify_discovery_witnesses( let mut after_id = 0i64; loop { let ids = sqlx::query_scalar::<_, i64>( - "SELECT discovery_edge_id FROM discovery_edges WHERE discovery_edge_id > $1 ORDER BY discovery_edge_id LIMIT $2", + r#" + SELECT discovery_edge_id + FROM discovery_edges + WHERE discovery_edge_id > $1 + AND chain_id = $2 + AND discovery_source LIKE 'ens_v2_registry_%' + AND provenance ->> 'source' = 'raw_log' + AND active_from_block_number <= $3 + ORDER BY discovery_edge_id + LIMIT $4 + "#, ) .bind(after_id) + .bind(chain) + .bind(through_block) .bind(WITNESS_PAGE_ROWS) .fetch_all(&mut *connection) .await @@ -295,3 +322,7 @@ async fn record_progress( } Ok(()) } + +#[cfg(test)] +#[path = "witnesses/tests.rs"] +mod tests; diff --git a/crates/adapters/src/ens_v2_registry/live/completeness/requirements/witnesses/tests.rs b/crates/adapters/src/ens_v2_registry/live/completeness/requirements/witnesses/tests.rs new file mode 100644 index 00000000..c4015b7e --- /dev/null +++ b/crates/adapters/src/ens_v2_registry/live/completeness/requirements/witnesses/tests.rs @@ -0,0 +1,235 @@ +use bigname_test_support::{TestDatabase, TestDatabaseConfig}; +use uuid::Uuid; + +use super::*; + +#[derive(Default)] +struct CountingProgress { + count: usize, +} + +impl StartupAdapterProgress for CountingProgress { + fn record<'a>( + &'a mut self, + _pool: &'a PgPool, + ) -> crate::checkpoint_context::StartupAdapterProgressFuture<'a> { + self.count += 1; + Box::pin(async { Ok(()) }) + } +} + +#[tokio::test] +async fn requirement_index_exact_page_records_one_progress_beat() -> Result<()> { + let pool = sqlx::postgres::PgPoolOptions::new() + .connect_lazy("postgres://postgres:postgres@localhost/bigname_test")?; + let requirements = (0..WITNESS_PAGE_ROWS) + .map(|index| RequiredWatchedTuple { + source_family: "ens_v2_registry_l1".to_owned(), + address: format!("0x{index:040x}"), + required_from_block: 0, + required_to_block: 20, + }) + .collect::>(); + let mut progress = CountingProgress::default(); + let mut progress_ref = Some(&mut progress as &mut dyn StartupAdapterProgress); + + requirement_indexes(Some(&pool), &requirements, &mut progress_ref).await?; + + assert_eq!( + progress.count, 1, + "one exact requirement page must record exactly one completed-work beat" + ); + pool.close().await; + Ok(()) +} + +#[tokio::test] +async fn witness_pages_skip_rows_outside_the_ens_v2_chain_scope() -> Result<()> { + let database = TestDatabase::create_migrated( + TestDatabaseConfig::new("ens_v2_witness_scoped_pages"), + &bigname_storage::MIGRATOR, + "failed to migrate ENSv2 witness paging test database", + ) + .await?; + let pool = database.pool(); + let chain = "witness-target-chain"; + let from_id = Uuid::from_u128(0x501); + let to_id = Uuid::from_u128(0x502); + + sqlx::query( + r#" + INSERT INTO normalized_events ( + event_identity, + namespace, + event_kind, + source_family, + manifest_version, + chain_id, + block_number, + block_hash, + transaction_hash, + log_index, + raw_fact_ref, + derivation_kind, + canonicality_state + ) + SELECT + 'out-of-scope-event-' || value, + 'ens', + 'Other', + 'other_family', + 1, + 'other-chain', + 1, + 'out-of-scope-block', + 'out-of-scope-transaction-' || value, + value, + '{}'::JSONB, + 'other_derivation', + 'canonical'::canonicality_state + FROM generate_series(1, 2001) value + "#, + ) + .execute(pool) + .await?; + sqlx::query( + r#" + INSERT INTO normalized_events ( + event_identity, + namespace, + event_kind, + source_family, + manifest_version, + chain_id, + block_number, + block_hash, + transaction_hash, + log_index, + raw_fact_ref, + derivation_kind, + canonicality_state + ) + VALUES ( + 'target-event', + 'ens', + 'RegistryEvent', + 'ens_v2_registry_l1', + 1, + $1, + 10, + 'target-block', + 'target-transaction', + 0, + '{"kind":"raw_log","emitting_address":"0x0000000000000000000000000000000000000501"}'::JSONB, + $2, + 'canonical'::canonicality_state + ) + "#, + ) + .bind(chain) + .bind(DERIVATION_KIND_ENS_V2_REGISTRY_RESOURCE_SURFACE) + .execute(pool) + .await?; + + sqlx::query( + r#" + INSERT INTO contract_instances ( + contract_instance_id, + chain_id, + contract_kind + ) + VALUES ($1, $3, 'test'), ($2, $3, 'test') + "#, + ) + .bind(from_id) + .bind(to_id) + .bind(chain) + .execute(pool) + .await?; + sqlx::query( + r#" + INSERT INTO discovery_edges ( + chain_id, + edge_kind, + from_contract_instance_id, + to_contract_instance_id, + discovery_source, + admission, + active_from_block_number, + provenance + ) + SELECT + 'other-chain', + 'test', + $1, + $2, + 'other_source', + 'test', + 1, + '{}'::JSONB + FROM generate_series(1, 2001) + "#, + ) + .bind(from_id) + .bind(to_id) + .execute(pool) + .await?; + sqlx::query( + r#" + INSERT INTO discovery_edges ( + chain_id, + edge_kind, + from_contract_instance_id, + to_contract_instance_id, + discovery_source, + admission, + active_from_block_number, + provenance + ) + VALUES ( + $1, + 'subregistry', + $2, + $3, + 'ens_v2_registry_test', + 'reachable_from_root', + 10, + '{ + "source":"raw_log", + "from_address":"0x0000000000000000000000000000000000000501", + "block_hash":"target-block", + "log_index":"0" + }'::JSONB + ) + "#, + ) + .bind(chain) + .bind(from_id) + .bind(to_id) + .execute(pool) + .await?; + + let requirements = [RequiredWatchedTuple { + source_family: "ens_v2_registry_l1".to_owned(), + address: "0x0000000000000000000000000000000000000501".to_owned(), + required_from_block: 0, + required_to_block: 20, + }]; + let mut connection = pool.acquire().await?; + let mut progress = CountingProgress::default(); + ensure_retained_semantic_witnesses_with_progress( + pool, + &mut connection, + chain, + &requirements, + 20, + &mut progress, + ) + .await?; + + assert_eq!( + progress.count, 3, + "one requirement page and one scoped page per witness table must beat" + ); + database.cleanup().await +} diff --git a/crates/manifests/src/lib/discovery/loading/progress.rs b/crates/manifests/src/lib/discovery/loading/progress.rs index bde394cf..d15ea8fe 100644 --- a/crates/manifests/src/lib/discovery/loading/progress.rs +++ b/crates/manifests/src/lib/discovery/loading/progress.rs @@ -48,13 +48,37 @@ pub(super) async fn load_active_discovered_parent_rows_with_progress( loop { let edge_ids = sqlx::query_scalar::<_, i64>( r#" - SELECT discovery_edge_id - FROM discovery_edges - WHERE discovery_edge_id > $1 - ORDER BY discovery_edge_id - LIMIT $2 + SELECT de.discovery_edge_id + FROM discovery_edges de + JOIN manifest_versions mv ON mv.manifest_id = de.source_manifest_id + WHERE de.discovery_edge_id > $5 + AND mv.rollout_status = 'active' + AND de.deactivated_at IS NULL + AND de.edge_kind = $4 + AND de.admission = $1 + AND de.provenance ? $2 + AND ($3::TEXT IS NULL OR de.discovery_source <> $3) + AND EXISTS ( + SELECT 1 + FROM contract_instance_addresses cia + WHERE cia.contract_instance_id = de.to_contract_instance_id + AND cia.deactivated_at IS NULL + ) + AND NOT EXISTS ( + SELECT 1 + FROM chain_lineage start_block + WHERE start_block.chain_id = de.chain_id + AND start_block.block_hash = de.active_from_block_hash + AND start_block.canonicality_state = 'orphaned'::canonicality_state + ) + ORDER BY de.discovery_edge_id + LIMIT $6 "#, ) + .bind(REACHABLE_FROM_ROOT_ADMISSION) + .bind(PROPAGATED_ROLE_PROVENANCE_FIELD) + .bind(excluded_discovery_source) + .bind(TRANSITIVE_DISCOVERY_EDGE_KIND) .bind(after_id) .bind(ADMISSION_LOAD_PAGE_SIZE) .fetch_all(&mut *executor) @@ -202,3 +226,7 @@ fn candidate_precedes(left: KnownAddressCandidate, right: KnownAddressCandidate) || (left.admitted_at == right.admitted_at && left.contract_instance_id < right.contract_instance_id) } + +#[cfg(test)] +#[path = "progress/tests.rs"] +mod tests; diff --git a/crates/manifests/src/lib/discovery/loading/progress/tests.rs b/crates/manifests/src/lib/discovery/loading/progress/tests.rs new file mode 100644 index 00000000..f0c19309 --- /dev/null +++ b/crates/manifests/src/lib/discovery/loading/progress/tests.rs @@ -0,0 +1,154 @@ +use bigname_test_support::{TestDatabase, TestDatabaseConfig}; +use uuid::Uuid; + +use super::*; + +#[derive(Default)] +struct CountingProgress { + count: usize, +} + +impl AdmissionStateProgress for CountingProgress { + fn record(&mut self) -> AdmissionStateProgressFuture<'_> { + self.count += 1; + Box::pin(async { Ok(()) }) + } +} + +#[tokio::test] +async fn active_parent_loader_pages_only_admitted_transitive_edges() -> Result<()> { + let database = TestDatabase::create_migrated( + TestDatabaseConfig::new("active_parent_scoped_pages"), + &bigname_storage::MIGRATOR, + "failed to migrate active-parent paging test database", + ) + .await?; + let pool = database.pool(); + let from_id = Uuid::from_u128(0xa01); + let to_id = Uuid::from_u128(0xa02); + sqlx::query( + r#" + INSERT INTO contract_instances (contract_instance_id, chain_id, contract_kind) + VALUES ($1, 'parent-chain', 'test'), ($2, 'parent-chain', 'test') + "#, + ) + .bind(from_id) + .bind(to_id) + .execute(pool) + .await?; + sqlx::query( + r#" + INSERT INTO contract_instance_addresses ( + contract_instance_id, + chain_id, + address + ) + VALUES ( + $1, + 'parent-chain', + '0x0000000000000000000000000000000000000a02' + ) + "#, + ) + .bind(to_id) + .execute(pool) + .await?; + sqlx::query( + r#" + INSERT INTO discovery_edges ( + chain_id, + edge_kind, + from_contract_instance_id, + to_contract_instance_id, + discovery_source, + admission + ) + SELECT + 'other-chain', + 'test', + $1, + $2, + 'other-source', + 'test' + FROM generate_series(1, 2001) + "#, + ) + .bind(from_id) + .bind(to_id) + .execute(pool) + .await?; + let manifest_id = sqlx::query_scalar::<_, i64>( + r#" + INSERT INTO manifest_versions ( + manifest_version, + namespace, + source_family, + chain, + deployment_epoch, + rollout_status, + normalizer_version, + file_path, + manifest_payload + ) + VALUES ( + 1, + 'ens', + 'parent_family', + 'parent-chain', + 'test', + 'active', + 'test', + 'test.toml', + '{}'::JSONB + ) + RETURNING manifest_id + "#, + ) + .fetch_one(pool) + .await?; + sqlx::query( + r#" + INSERT INTO discovery_edges ( + chain_id, + edge_kind, + from_contract_instance_id, + to_contract_instance_id, + discovery_source, + source_manifest_id, + admission, + provenance + ) + VALUES ( + 'parent-chain', + $1, + $2, + $3, + 'target-source', + $4, + $5, + '{"propagated_role":"registry"}'::JSONB + ) + "#, + ) + .bind(TRANSITIVE_DISCOVERY_EDGE_KIND) + .bind(from_id) + .bind(to_id) + .bind(manifest_id) + .bind(REACHABLE_FROM_ROOT_ADMISSION) + .execute(pool) + .await?; + + let mut connection = pool.acquire().await?; + let mut progress = CountingProgress::default(); + let rows = + load_active_discovered_parent_rows_with_progress(&mut connection, None, &mut progress) + .await?; + + assert_eq!(rows.len(), 1); + assert_eq!( + progress.count, 1, + "only the one admitted transitive-parent page must beat" + ); + drop(connection); + database.cleanup().await +} diff --git a/crates/manifests/src/lib/discovery/reconciliation/streamed/diff.rs b/crates/manifests/src/lib/discovery/reconciliation/streamed/diff.rs index 51fbb8d4..b63ca43a 100644 --- a/crates/manifests/src/lib/discovery/reconciliation/streamed/diff.rs +++ b/crates/manifests/src/lib/discovery/reconciliation/streamed/diff.rs @@ -132,13 +132,22 @@ pub(super) async fn load_streamed_deactivation_source_page( ) -> Result { let edge_ids = sqlx::query_scalar::<_, i64>( r#" - SELECT discovery_edge_id - FROM discovery_edges - WHERE discovery_edge_id > $1 - ORDER BY discovery_edge_id - LIMIT $2 + SELECT de.discovery_edge_id + FROM discovery_edges de + WHERE de.discovery_source = $1 + AND de.deactivated_at IS NULL + AND de.discovery_edge_id > $2 + AND EXISTS ( + SELECT 1 + FROM contract_instance_addresses cia + WHERE cia.contract_instance_id = de.to_contract_instance_id + AND cia.deactivated_at IS NULL + ) + ORDER BY de.discovery_edge_id + LIMIT $3 "#, ) + .bind(discovery_source) .bind(after_edge_id) .bind(limit) .fetch_all(&mut *executor) @@ -553,3 +562,7 @@ where } Ok(historical_edges) } + +#[cfg(test)] +#[path = "diff/tests.rs"] +mod tests; diff --git a/crates/manifests/src/lib/discovery/reconciliation/streamed/diff/tests.rs b/crates/manifests/src/lib/discovery/reconciliation/streamed/diff/tests.rs new file mode 100644 index 00000000..299e3659 --- /dev/null +++ b/crates/manifests/src/lib/discovery/reconciliation/streamed/diff/tests.rs @@ -0,0 +1,123 @@ +use bigname_test_support::{TestDatabase, TestDatabaseConfig}; +use uuid::Uuid; + +use super::*; + +#[tokio::test] +async fn deactivation_source_pages_only_active_edges_for_the_requested_source() -> Result<()> { + let database = TestDatabase::create_migrated( + TestDatabaseConfig::new("streamed_deactivation_scoped_pages"), + &bigname_storage::MIGRATOR, + "failed to migrate streamed deactivation paging test database", + ) + .await?; + let pool = database.pool(); + let from_id = Uuid::from_u128(0x901); + let to_id = Uuid::from_u128(0x902); + sqlx::query( + r#" + INSERT INTO contract_instances (contract_instance_id, chain_id, contract_kind) + VALUES ($1, 'deactivation-chain', 'test'), ($2, 'deactivation-chain', 'test') + "#, + ) + .bind(from_id) + .bind(to_id) + .execute(pool) + .await?; + sqlx::query( + r#" + INSERT INTO contract_instance_addresses ( + contract_instance_id, + chain_id, + address + ) + VALUES ( + $1, + 'deactivation-chain', + '0x0000000000000000000000000000000000000902' + ) + "#, + ) + .bind(to_id) + .execute(pool) + .await?; + sqlx::query( + r#" + INSERT INTO discovery_edges ( + chain_id, + edge_kind, + from_contract_instance_id, + to_contract_instance_id, + discovery_source, + admission + ) + SELECT + 'other-chain', + 'test', + $1, + $2, + 'other-source', + 'test' + FROM generate_series(1, 2001) + "#, + ) + .bind(from_id) + .bind(to_id) + .execute(pool) + .await?; + sqlx::query( + r#" + INSERT INTO discovery_edges ( + chain_id, + edge_kind, + from_contract_instance_id, + to_contract_instance_id, + discovery_source, + admission, + provenance + ) + VALUES ( + 'deactivation-chain', + 'subregistry', + $1, + $2, + 'target-source', + 'reachable_from_root', + '{"observation_key":"target-edge"}'::JSONB + ) + "#, + ) + .bind(from_id) + .bind(to_id) + .execute(pool) + .await?; + + let mut transaction = pool.begin().await?; + super::super::staging::create_streamed_reconcile_temp_tables(transaction.as_mut()).await?; + let mut after_id = 0; + let mut page_count = 0; + let mut candidate_count = 0; + loop { + let page = load_streamed_deactivation_source_page( + transaction.as_mut(), + "target-source", + after_id, + 1, + ) + .await?; + let Some(last_id) = page.last_edge_id else { + break; + }; + after_id = last_id; + page_count += 1; + candidate_count += page.candidates.len(); + } + + assert_eq!(candidate_count, 1); + assert_eq!( + page_count, 1, + "only the requested source's one active edge must form a page" + ); + transaction.rollback().await?; + database.cleanup().await +} diff --git a/crates/manifests/src/lib/discovery/reconciliation/streamed/progress.rs b/crates/manifests/src/lib/discovery/reconciliation/streamed/progress.rs index fd80b948..96489f2e 100644 --- a/crates/manifests/src/lib/discovery/reconciliation/streamed/progress.rs +++ b/crates/manifests/src/lib/discovery/reconciliation/streamed/progress.rs @@ -41,11 +41,14 @@ where SELECT discovery_edge_id FROM discovery_edges WHERE discovery_edge_id > $1 + AND discovery_source = $2 + AND deactivated_at IS NULL ORDER BY discovery_edge_id - LIMIT $2 + LIMIT $3 "#, ) .bind(after_edge_id) + .bind(discovery_source) .bind(page_limit.max(1)) .fetch_all(&mut *executor) .await @@ -77,3 +80,7 @@ where } Ok((active_edge_count, chains)) } + +#[cfg(test)] +#[path = "progress/tests.rs"] +mod tests; diff --git a/crates/manifests/src/lib/discovery/reconciliation/streamed/progress/tests.rs b/crates/manifests/src/lib/discovery/reconciliation/streamed/progress/tests.rs new file mode 100644 index 00000000..f706eb81 --- /dev/null +++ b/crates/manifests/src/lib/discovery/reconciliation/streamed/progress/tests.rs @@ -0,0 +1,107 @@ +use std::sync::atomic::{AtomicUsize, Ordering}; + +use bigname_test_support::{TestDatabase, TestDatabaseConfig}; +use uuid::Uuid; + +use super::*; +use crate::DiscoveryObservation; + +#[derive(Default)] +struct CountingPageSource { + progress_count: AtomicUsize, +} + +impl DiscoveryObservationPageSource for CountingPageSource { + async fn load_page( + &self, + _after_key: Option<&str>, + _limit: i64, + ) -> Result> { + Ok(Vec::new()) + } + + async fn record_progress(&self) -> Result<()> { + self.progress_count.fetch_add(1, Ordering::Relaxed); + Ok(()) + } +} + +#[tokio::test] +async fn active_edge_summary_pages_only_the_requested_source() -> Result<()> { + let database = TestDatabase::create_migrated( + TestDatabaseConfig::new("streamed_active_summary_scoped_pages"), + &bigname_storage::MIGRATOR, + "failed to migrate streamed active-summary paging test database", + ) + .await?; + let pool = database.pool(); + let from_id = Uuid::from_u128(0x801); + let to_id = Uuid::from_u128(0x802); + sqlx::query( + r#" + INSERT INTO contract_instances (contract_instance_id, chain_id, contract_kind) + VALUES ($1, 'summary-chain', 'test'), ($2, 'summary-chain', 'test') + "#, + ) + .bind(from_id) + .bind(to_id) + .execute(pool) + .await?; + sqlx::query( + r#" + INSERT INTO discovery_edges ( + chain_id, + edge_kind, + from_contract_instance_id, + to_contract_instance_id, + discovery_source, + admission + ) + SELECT + 'other-chain', + 'test', + $1, + $2, + 'other-source', + 'test' + FROM generate_series(1, 2001) + "#, + ) + .bind(from_id) + .bind(to_id) + .execute(pool) + .await?; + sqlx::query( + r#" + INSERT INTO discovery_edges ( + chain_id, + edge_kind, + from_contract_instance_id, + to_contract_instance_id, + discovery_source, + admission + ) + VALUES ('summary-chain', 'test', $1, $2, 'target-source', 'test') + "#, + ) + .bind(from_id) + .bind(to_id) + .execute(pool) + .await?; + + let source = CountingPageSource::default(); + let mut connection = pool.acquire().await?; + let (count, chains) = + load_active_edge_summary_with_progress(&mut connection, "target-source", 1_000, &source) + .await?; + + assert_eq!(count, 1); + assert_eq!(chains, BTreeSet::from(["summary-chain".to_owned()])); + assert_eq!( + source.progress_count.load(Ordering::Relaxed), + 1, + "only the one scoped active-edge page must beat" + ); + drop(connection); + database.cleanup().await +} diff --git a/crates/manifests/src/lib/managed_edges/source_graph.rs b/crates/manifests/src/lib/managed_edges/source_graph.rs index 65b4dca2..af7fb8b3 100644 --- a/crates/manifests/src/lib/managed_edges/source_graph.rs +++ b/crates/manifests/src/lib/managed_edges/source_graph.rs @@ -420,20 +420,20 @@ async fn deactivate_discovery_edges_without_active_source_manifest( let mut deactivated_edge_count = 0usize; let mut mutated_chains = BTreeSet::new(); loop { - let page = sqlx::query_as::<_, (i64, String, bool)>( + let page = sqlx::query_as::<_, (i64, String)>( r#" SELECT de.discovery_edge_id, - de.chain_id, - de.deactivated_at IS NULL - AND NOT EXISTS ( - SELECT 1 - FROM manifest_versions mv - WHERE mv.manifest_id = de.source_manifest_id - AND mv.rollout_status = 'active' - ) AS stale_source + de.chain_id FROM discovery_edges de WHERE de.discovery_edge_id > $1 + AND de.deactivated_at IS NULL + AND NOT EXISTS ( + SELECT 1 + FROM manifest_versions mv + WHERE mv.manifest_id = de.source_manifest_id + AND mv.rollout_status = 'active' + ) ORDER BY de.discovery_edge_id LIMIT $2 "#, @@ -443,14 +443,11 @@ async fn deactivate_discovery_edges_without_active_source_manifest( .fetch_all(&mut *executor) .await .context("failed to scan discovery edges for inactive source manifests")?; - let Some((last_id, _, _)) = page.last() else { + let Some((last_id, _)) = page.last() else { break; }; cursor = *last_id; - let stale_ids = page - .iter() - .filter_map(|(edge_id, _, stale)| stale.then_some(*edge_id)) - .collect::>(); + let stale_ids = page.iter().map(|(edge_id, _)| *edge_id).collect::>(); if !stale_ids.is_empty() { let result = sqlx::query( r#" @@ -465,11 +462,7 @@ async fn deactivate_discovery_edges_without_active_source_manifest( .await .context("failed to deactivate discovery edges without an active source manifest")?; deactivated_edge_count += result.rows_affected() as usize; - mutated_chains.extend( - page.iter() - .filter(|(_, _, stale)| *stale) - .map(|(_, chain, _)| chain.clone()), - ); + mutated_chains.extend(page.iter().map(|(_, chain)| chain.clone())); } record_progress(pool, progress).await?; } @@ -487,3 +480,7 @@ async fn record_progress( } Ok(()) } + +#[cfg(test)] +#[path = "source_graph/tests.rs"] +mod tests; diff --git a/crates/manifests/src/lib/managed_edges/source_graph/tests.rs b/crates/manifests/src/lib/managed_edges/source_graph/tests.rs new file mode 100644 index 00000000..be463f3a --- /dev/null +++ b/crates/manifests/src/lib/managed_edges/source_graph/tests.rs @@ -0,0 +1,148 @@ +use bigname_test_support::{TestDatabase, TestDatabaseConfig}; +use uuid::Uuid; + +use super::*; + +#[derive(Default)] +struct CountingProgress { + count: usize, +} + +impl ManifestRuntimeProgress for CountingProgress { + fn record<'a>(&'a mut self, _pool: &'a PgPool) -> crate::ManifestRuntimeProgressFuture<'a> { + self.count += 1; + Box::pin(async { Ok(()) }) + } +} + +#[tokio::test] +async fn stale_source_cleanup_pages_only_edges_without_an_active_manifest() -> Result<()> { + let database = TestDatabase::create_migrated( + TestDatabaseConfig::new("stale_source_scoped_pages"), + &bigname_storage::MIGRATOR, + "failed to migrate stale-source paging test database", + ) + .await?; + let pool = database.pool(); + let from_id = Uuid::from_u128(0xb01); + let to_id = Uuid::from_u128(0xb02); + sqlx::query( + r#" + INSERT INTO contract_instances (contract_instance_id, chain_id, contract_kind) + VALUES ($1, 'managed-chain', 'test'), ($2, 'managed-chain', 'test') + "#, + ) + .bind(from_id) + .bind(to_id) + .execute(pool) + .await?; + let manifest_id = sqlx::query_scalar::<_, i64>( + r#" + INSERT INTO manifest_versions ( + manifest_version, + namespace, + source_family, + chain, + deployment_epoch, + rollout_status, + normalizer_version, + file_path, + manifest_payload + ) + VALUES ( + 1, + 'ens', + 'managed_family', + 'managed-chain', + 'test', + 'active', + 'test', + 'test.toml', + '{}'::JSONB + ) + RETURNING manifest_id + "#, + ) + .fetch_one(pool) + .await?; + sqlx::query( + r#" + INSERT INTO discovery_edges ( + chain_id, + edge_kind, + from_contract_instance_id, + to_contract_instance_id, + discovery_source, + source_manifest_id, + admission + ) + SELECT + 'managed-chain', + 'test', + $1, + $2, + 'active-source', + $3, + 'test' + FROM generate_series(1, 10001) + "#, + ) + .bind(from_id) + .bind(to_id) + .bind(manifest_id) + .execute(pool) + .await?; + let stale_edge_id = sqlx::query_scalar::<_, i64>( + r#" + INSERT INTO discovery_edges ( + chain_id, + edge_kind, + from_contract_instance_id, + to_contract_instance_id, + discovery_source, + admission + ) + VALUES ('managed-chain', 'test', $1, $2, 'stale-source', 'test') + RETURNING discovery_edge_id + "#, + ) + .bind(from_id) + .bind(to_id) + .fetch_one(pool) + .await?; + + let mut connection = pool.acquire().await?; + let mut progress = CountingProgress::default(); + let mut progress_ref = Some(&mut progress as &mut dyn ManifestRuntimeProgress); + let deactivated = deactivate_discovery_edges_without_active_source_manifest( + &mut connection, + pool, + &mut progress_ref, + ) + .await?; + + assert_eq!(deactivated, 1); + assert_eq!( + progress.count, 1, + "only the one stale-source page must beat" + ); + assert!( + sqlx::query_scalar::<_, bool>( + "SELECT deactivated_at IS NOT NULL FROM discovery_edges WHERE discovery_edge_id = $1", + ) + .bind(stale_edge_id) + .fetch_one(pool) + .await? + ); + assert_eq!( + sqlx::query_scalar::<_, i64>( + "SELECT COUNT(*)::BIGINT FROM discovery_edges WHERE source_manifest_id = $1 AND deactivated_at IS NULL", + ) + .bind(manifest_id) + .fetch_one(pool) + .await?, + 10001 + ); + drop(connection); + database.cleanup().await +} diff --git a/crates/manifests/src/lib/views/watched/frontier.rs b/crates/manifests/src/lib/views/watched/frontier.rs index db155373..8e57ad9d 100644 --- a/crates/manifests/src/lib/views/watched/frontier.rs +++ b/crates/manifests/src/lib/views/watched/frontier.rs @@ -83,28 +83,35 @@ pub async fn load_earliest_known_watched_block_with_progress( .await .context("failed to acquire earliest watched-block scan connection")?; let mut earliest = None; - for (source_table, source_id_column, watched_branch) in [ - ( - "contract_instance_addresses", - "contract_instance_address_id", - "manifest_watched_intervals", - ), - ( - "discovery_edges", - "discovery_edge_id", - "discovery_watched_intervals", - ), - ] { + for watched_branch in ["manifest_watched_intervals", "discovery_watched_intervals"] { let mut after_id = 0i64; loop { - let source_ids = sqlx::query_scalar::<_, i64>(&format!( - "SELECT {source_id_column} FROM {source_table} WHERE {source_id_column} > $1 ORDER BY {source_id_column} LIMIT $2" - )) - .bind(after_id) - .bind(COVERAGE_SOURCE_PAGE_ROWS) - .fetch_all(&mut *connection) - .await - .with_context(|| format!("failed to page {source_table} for earliest watched block"))?; + let source_page_query = super::intervals::with_streaming_watched_intervals(&format!( + r#" + SELECT DISTINCT watched.source_row_id + FROM {watched_branch} watched + WHERE watched.source_row_id > $3 + AND {historical_predicate} + AND watched.chain = $1 + AND watched.source_family = ANY($4::TEXT[]) + AND watched.active_from_block_number IS NOT NULL + AND watched.active_from_block_number <= $2 + ORDER BY watched.source_row_id + LIMIT $5 + "#, + historical_predicate = super::intervals::HISTORICAL_WATCHED_INTERVAL_PREDICATE, + )); + let source_ids = sqlx::query_scalar::<_, i64>(&source_page_query) + .bind(chain) + .bind(through_block) + .bind(after_id) + .bind(log_producing_source_families) + .bind(COVERAGE_SOURCE_PAGE_ROWS) + .fetch_all(&mut *connection) + .await + .with_context(|| { + format!("failed to page {watched_branch} for earliest watched block") + })?; let Some(last_id) = source_ids.last().copied() else { break; }; @@ -285,8 +292,6 @@ pub async fn materialize_stored_lineage_coverage_candidate_with_progress( verified_from_block, verified_through_block, log_producing_source_families, - "contract_instance_addresses", - "contract_instance_address_id", "manifest_watched_intervals", progress, ) @@ -298,8 +303,6 @@ pub async fn materialize_stored_lineage_coverage_candidate_with_progress( verified_from_block, verified_through_block, log_producing_source_families, - "discovery_edges", - "discovery_edge_id", "discovery_watched_intervals", progress, ) diff --git a/crates/manifests/src/lib/views/watched/frontier/source_pages.rs b/crates/manifests/src/lib/views/watched/frontier/source_pages.rs index 6c1ae1d5..98eac36b 100644 --- a/crates/manifests/src/lib/views/watched/frontier/source_pages.rs +++ b/crates/manifests/src/lib/views/watched/frontier/source_pages.rs @@ -7,21 +7,38 @@ pub(super) async fn materialize_candidate_source_pages( verified_from_block: i64, verified_through_block: i64, log_producing_source_families: &[String], - source_table: &str, - source_id_column: &str, watched_branch: &str, progress: &mut dyn ManifestRuntimeProgress, ) -> Result<()> { let mut after_id = 0i64; loop { - let source_ids = sqlx::query_scalar::<_, i64>(&format!( - "SELECT {source_id_column} FROM {source_table} WHERE {source_id_column} > $1 ORDER BY {source_id_column} LIMIT $2" - )) - .bind(after_id) - .bind(COVERAGE_SOURCE_PAGE_ROWS) - .fetch_all(&mut *connection) - .await - .with_context(|| format!("failed to page {source_table} for coverage candidate"))?; + let source_page_query = + super::super::intervals::with_streaming_watched_intervals(&format!( + r#" + SELECT DISTINCT watched.source_row_id + FROM {watched_branch} watched + WHERE watched.source_row_id > $5 + AND {historical_predicate} + AND watched.chain = $1 + AND watched.source_family = ANY($4::TEXT[]) + AND COALESCE(watched.active_from_block_number, $2) <= $3 + AND COALESCE(watched.active_to_block_number, $3) >= $2 + ORDER BY watched.source_row_id + LIMIT $6 + "#, + historical_predicate = + super::super::intervals::HISTORICAL_WATCHED_INTERVAL_PREDICATE, + )); + let source_ids = sqlx::query_scalar::<_, i64>(&source_page_query) + .bind(chain) + .bind(verified_from_block) + .bind(verified_through_block) + .bind(log_producing_source_families) + .bind(after_id) + .bind(COVERAGE_SOURCE_PAGE_ROWS) + .fetch_all(&mut *connection) + .await + .with_context(|| format!("failed to page {watched_branch} for coverage candidate"))?; let Some(last_id) = source_ids.last().copied() else { break; }; diff --git a/crates/manifests/src/lib/views/watched/frontier/tests.rs b/crates/manifests/src/lib/views/watched/frontier/tests.rs index 0c00fa89..d71e4b9b 100644 --- a/crates/manifests/src/lib/views/watched/frontier/tests.rs +++ b/crates/manifests/src/lib/views/watched/frontier/tests.rs @@ -6,6 +6,9 @@ use bigname_storage::{ begin_stored_lineage_coverage_frontier_publication, }; use bigname_test_support::{TestDatabase, TestDatabaseConfig}; +use uuid::Uuid; + +use crate::ManifestRuntimeProgressFuture; use super::*; @@ -14,6 +17,18 @@ const FAMILY: &str = "frontier_test_family"; const ADDRESS_ONE: &str = "0x0000000000000000000000000000000000000001"; const ADDRESS_TWO: &str = "0x0000000000000000000000000000000000000002"; +#[derive(Default)] +struct CountingProgress { + count: usize, +} + +impl ManifestRuntimeProgress for CountingProgress { + fn record<'a>(&'a mut self, _pool: &'a PgPool) -> ManifestRuntimeProgressFuture<'a> { + self.count += 1; + Box::pin(async { Ok(()) }) + } +} + async fn database(name: &str) -> Result { TestDatabase::create_migrated( TestDatabaseConfig::new(name), @@ -195,3 +210,212 @@ async fn high_cardinality_candidate_returns_only_bounded_delta_pages() -> Result database.cleanup().await } + +#[tokio::test] +async fn watched_frontier_pages_only_rows_in_the_requested_coverage_scope() -> Result<()> { + let database = database("manifest_frontier_scoped_source_pages").await?; + let pool = database.pool(); + let manifest_contract_id = Uuid::from_u128(0xc01); + let discovered_contract_id = Uuid::from_u128(0xc02); + + sqlx::query( + r#" + INSERT INTO contract_instances ( + contract_instance_id, + chain_id, + contract_kind + ) + SELECT + md5('out-of-scope-contract-' || value)::UUID, + 'other-chain', + 'test' + FROM generate_series(1, 2001) value + "#, + ) + .execute(pool) + .await?; + sqlx::query( + r#" + INSERT INTO contract_instance_addresses ( + contract_instance_id, + chain_id, + address, + active_from_block_number + ) + SELECT + md5('out-of-scope-contract-' || value)::UUID, + 'other-chain', + '0x' || lpad(to_hex(value), 40, '0'), + 1 + FROM generate_series(1, 2001) value + "#, + ) + .execute(pool) + .await?; + sqlx::query( + r#" + INSERT INTO discovery_edges ( + chain_id, + edge_kind, + from_contract_instance_id, + to_contract_instance_id, + discovery_source, + admission, + active_from_block_number + ) + SELECT + 'other-chain', + 'test', + md5('out-of-scope-contract-1')::UUID, + md5('out-of-scope-contract-2')::UUID, + 'other-source', + 'test', + 1 + FROM generate_series(1, 2001) + "#, + ) + .execute(pool) + .await?; + + sqlx::query( + r#" + INSERT INTO contract_instances (contract_instance_id, chain_id, contract_kind) + VALUES ($1, $3, 'test'), ($2, $3, 'test') + "#, + ) + .bind(manifest_contract_id) + .bind(discovered_contract_id) + .bind(CHAIN) + .execute(pool) + .await?; + let manifest_id = sqlx::query_scalar::<_, i64>( + r#" + INSERT INTO manifest_versions ( + manifest_version, + namespace, + source_family, + chain, + deployment_epoch, + rollout_status, + normalizer_version, + file_path, + manifest_payload + ) + VALUES ( + 1, + 'ens', + $1, + $2, + 'test', + 'active', + 'test', + 'test.toml', + '{"contracts":[{"role":"registry","start_block":10}]}'::JSONB + ) + RETURNING manifest_id + "#, + ) + .bind(FAMILY) + .bind(CHAIN) + .fetch_one(pool) + .await?; + sqlx::query( + r#" + INSERT INTO manifest_contract_instances ( + manifest_id, + declaration_kind, + declaration_name, + contract_instance_id, + declared_address, + role, + proxy_kind + ) + VALUES ($1, 'contract', 'registry', $2, $3, 'registry', 'none') + "#, + ) + .bind(manifest_id) + .bind(manifest_contract_id) + .bind(ADDRESS_ONE) + .execute(pool) + .await?; + sqlx::query( + r#" + INSERT INTO contract_instance_addresses ( + contract_instance_id, + chain_id, + address, + active_from_block_number, + source_manifest_id + ) + VALUES + ($1, $3, $4, 10, $5), + ($2, $3, $6, 20, $5) + "#, + ) + .bind(manifest_contract_id) + .bind(discovered_contract_id) + .bind(CHAIN) + .bind(ADDRESS_ONE) + .bind(manifest_id) + .bind(ADDRESS_TWO) + .execute(pool) + .await?; + sqlx::query( + r#" + INSERT INTO discovery_edges ( + chain_id, + edge_kind, + from_contract_instance_id, + to_contract_instance_id, + discovery_source, + source_manifest_id, + admission, + active_from_block_number + ) + VALUES ($1, 'subregistry', $2, $3, 'target-source', $4, 'reachable_from_root', 20) + "#, + ) + .bind(CHAIN) + .bind(manifest_contract_id) + .bind(discovered_contract_id) + .bind(manifest_id) + .execute(pool) + .await?; + + let families = [FAMILY.to_owned()]; + let mut earliest_progress = CountingProgress::default(); + let earliest = load_earliest_known_watched_block_with_progress( + pool, + CHAIN, + 40, + &families, + &mut earliest_progress, + ) + .await?; + assert_eq!(earliest, Some(10)); + assert_eq!( + earliest_progress.count, 2, + "one scoped manifest page and one scoped discovery page must beat" + ); + + let mut guard = + begin_stored_lineage_coverage_frontier_publication(pool, CHAIN, None, 0).await?; + let mut materialize_progress = CountingProgress::default(); + let summary = materialize_stored_lineage_coverage_candidate_with_progress( + guard.connection_mut(), + pool, + CHAIN, + 0, + 40, + &families, + &mut materialize_progress, + ) + .await?; + assert_eq!(summary.requirement_tuple_count, 2); + assert_eq!( + materialize_progress.count, 3, + "the two scoped source pages and one candidate-count page must beat" + ); + drop(guard); + database.cleanup().await +} From de4e741b61c11e2d4823783891337ef0152a8f66 Mon Sep 17 00:00:00 2001 From: TateB Date: Thu, 23 Jul 2026 04:51:06 +0000 Subject: [PATCH 07/12] fix(indexer): preserve ENSv2 block-hash progress Add progress-aware ENSv2 registry block-hash entrypoints, route every reconciliation mode through them when a heartbeat handle is present, and lock the coverage with an out-of-scope-dominated exact-beat regression. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01C9RTnbCyhqEUSPc7xCK76p --- apps/indexer/src/main/reconciliation.rs | 7 +- .../src/main/reconciliation/adapter_sync.rs | 4 + .../adapter_sync/ens_v2_registry.rs | 108 ++++++++++---- .../main/reconciliation/adapter_sync/mode.rs | 2 +- apps/indexer/src/main/tests/adapter_sync.rs | 139 ++++++++++++++++++ .../src/ens_v2_registry/entrypoints.rs | 78 ++++++++++ docs/storage.md | 2 +- 7 files changed, 308 insertions(+), 32 deletions(-) diff --git a/apps/indexer/src/main/reconciliation.rs b/apps/indexer/src/main/reconciliation.rs index 535806c6..7f555851 100644 --- a/apps/indexer/src/main/reconciliation.rs +++ b/apps/indexer/src/main/reconciliation.rs @@ -30,9 +30,10 @@ pub(crate) use adapter_sync::{ }; #[cfg(test)] pub(crate) use adapter_sync::{ - install_after_stateless_failure, install_backlog_after_adapter_sync_test_hook, - install_ownership_release_test_hook, install_post_discovery_mutation_failure_for_test, - install_stateless_page_observer, + PersistedRawPayloadAdapterSyncModeForTest, install_after_stateless_failure, + install_backlog_after_adapter_sync_test_hook, install_ownership_release_test_hook, + install_post_discovery_mutation_failure_for_test, install_stateless_page_observer, + sync_ens_v2_registry_for_mode_for_test, sync_full_closure_normalized_events_from_persisted_raw_payloads, sync_live_adapter_backlog_after_normalized_replay, sync_manual_full_closure_normalized_events_from_persisted_raw_payloads, diff --git a/apps/indexer/src/main/reconciliation/adapter_sync.rs b/apps/indexer/src/main/reconciliation/adapter_sync.rs index e147ebc4..e7638033 100644 --- a/apps/indexer/src/main/reconciliation/adapter_sync.rs +++ b/apps/indexer/src/main/reconciliation/adapter_sync.rs @@ -48,6 +48,8 @@ pub(crate) use backlog::{ }; use block_scoped::{sync_ens_v1_unwrapped_authority_for_scope, sync_ens_v2_registrar_for_scope}; use ens_v1_subregistry::{ens_v1_subregistry_sync_operation, sync_ens_v1_subregistry_for_mode}; +#[cfg(test)] +pub(crate) use ens_v2_registry::sync_ens_v2_registry_for_mode as sync_ens_v2_registry_for_mode_for_test; use ens_v2_registry::{ens_v2_registry_sync_operation, sync_ens_v2_registry_for_mode}; use ens_v2_tail::sync_ens_v2_tail_adapters; #[allow(unused_imports)] @@ -71,6 +73,8 @@ pub(crate) use full_closure::{ install_stateless_page_observer, sync_full_closure_normalized_events_from_persisted_raw_payloads, }; +#[cfg(test)] +pub(crate) use mode::PersistedRawPayloadAdapterSyncMode as PersistedRawPayloadAdapterSyncModeForTest; use mode::{PersistedRawPayloadAdapterSyncMode, ensure_raw_fact_adapter_allowed}; use progress::{journal_authority_epoch_with_progress, record_adapter_progress}; use scope::load_live_adapter_source_scope; diff --git a/apps/indexer/src/main/reconciliation/adapter_sync/ens_v2_registry.rs b/apps/indexer/src/main/reconciliation/adapter_sync/ens_v2_registry.rs index b21d8c40..75ce8652 100644 --- a/apps/indexer/src/main/reconciliation/adapter_sync/ens_v2_registry.rs +++ b/apps/indexer/src/main/reconciliation/adapter_sync/ens_v2_registry.rs @@ -5,7 +5,7 @@ use super::{ }; #[expect(clippy::too_many_arguments)] -pub(super) async fn sync_ens_v2_registry_for_mode( +pub(crate) async fn sync_ens_v2_registry_for_mode( pool: &sqlx::PgPool, live_deployment_profile: Option<&str>, chain: &str, @@ -69,38 +69,92 @@ pub(super) async fn sync_ens_v2_registry_for_mode( } } (PersistedRawPayloadAdapterSyncMode::RawFactReplay { .. }, Some(source_scope)) => { - bigname_adapters::EnsV2RegistryResourceSurfaceSyncSummary::sync_for_block_hashes_with_source_scope_canonical_only( - pool, - chain, - block_hashes, - source_scope, - ) - .await + match progress.as_deref_mut() { + Some(progress) => { + bigname_adapters::EnsV2RegistryResourceSurfaceSyncSummary::sync_for_block_hashes_with_source_scope_canonical_only_and_progress( + pool, + chain, + block_hashes, + source_scope, + progress, + ) + .await + } + None => { + bigname_adapters::EnsV2RegistryResourceSurfaceSyncSummary::sync_for_block_hashes_with_source_scope_canonical_only( + pool, + chain, + block_hashes, + source_scope, + ) + .await + } + } } (PersistedRawPayloadAdapterSyncMode::RawFactReplay { .. }, None) => { - bigname_adapters::EnsV2RegistryResourceSurfaceSyncSummary::sync_for_block_hashes_canonical_only( - pool, - chain, - block_hashes, - ) - .await + match progress.as_deref_mut() { + Some(progress) => { + bigname_adapters::EnsV2RegistryResourceSurfaceSyncSummary::sync_for_block_hashes_canonical_only_with_progress( + pool, + chain, + block_hashes, + progress, + ) + .await + } + None => { + bigname_adapters::EnsV2RegistryResourceSurfaceSyncSummary::sync_for_block_hashes_canonical_only( + pool, + chain, + block_hashes, + ) + .await + } + } } (_, Some(source_scope)) => { - bigname_adapters::EnsV2RegistryResourceSurfaceSyncSummary::sync_for_block_hashes_with_source_scope( - pool, - chain, - block_hashes, - source_scope, - ) - .await + match progress.as_deref_mut() { + Some(progress) => { + bigname_adapters::EnsV2RegistryResourceSurfaceSyncSummary::sync_for_block_hashes_with_source_scope_and_progress( + pool, + chain, + block_hashes, + source_scope, + progress, + ) + .await + } + None => { + bigname_adapters::EnsV2RegistryResourceSurfaceSyncSummary::sync_for_block_hashes_with_source_scope( + pool, + chain, + block_hashes, + source_scope, + ) + .await + } + } } (_, None) => { - bigname_adapters::EnsV2RegistryResourceSurfaceSyncSummary::sync_for_block_hashes( - pool, - chain, - block_hashes, - ) - .await + match progress.as_deref_mut() { + Some(progress) => { + bigname_adapters::EnsV2RegistryResourceSurfaceSyncSummary::sync_for_block_hashes_with_progress( + pool, + chain, + block_hashes, + progress, + ) + .await + } + None => { + bigname_adapters::EnsV2RegistryResourceSurfaceSyncSummary::sync_for_block_hashes( + pool, + chain, + block_hashes, + ) + .await + } + } } } } diff --git a/apps/indexer/src/main/reconciliation/adapter_sync/mode.rs b/apps/indexer/src/main/reconciliation/adapter_sync/mode.rs index a420fcd5..be47c62e 100644 --- a/apps/indexer/src/main/reconciliation/adapter_sync/mode.rs +++ b/apps/indexer/src/main/reconciliation/adapter_sync/mode.rs @@ -5,7 +5,7 @@ use super::super::replay::{ }; #[derive(Clone, Copy, Debug, Eq, PartialEq)] -pub(super) enum PersistedRawPayloadAdapterSyncMode { +pub(crate) enum PersistedRawPayloadAdapterSyncMode { LivePoll, LiveOrBackfill, RawFactReplay { diff --git a/apps/indexer/src/main/tests/adapter_sync.rs b/apps/indexer/src/main/tests/adapter_sync.rs index 98b421cd..bbb9c6fd 100644 --- a/apps/indexer/src/main/tests/adapter_sync.rs +++ b/apps/indexer/src/main/tests/adapter_sync.rs @@ -73,6 +73,145 @@ async fn scoped_ens_v2_registry_sync_emits_registry_permission_events() -> Resul database.cleanup().await } +#[tokio::test] +async fn ens_v2_block_hash_dispatch_preserves_progress_for_large_live_chunks() -> Result<()> { + #[derive(Default)] + struct CountingProgress(usize); + + impl bigname_adapters::StartupAdapterProgress for CountingProgress { + fn record<'a>( + &'a mut self, + _pool: &'a PgPool, + ) -> bigname_adapters::StartupAdapterProgressFuture<'a> { + self.0 += 1; + Box::pin(async { Ok(()) }) + } + } + + let database = TestDatabase::new().await?; + let chain = "ens-v2-block-hash-progress-chain"; + let registry_contract_instance_id = Uuid::from_u128(0x346); + let registry_address = "0x0000000000000000000000000000000000000346"; + let out_of_scope_block = provider_block( + "0xc5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5", + None, + 63, + ); + let block = provider_block( + "0xd6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6", + Some(&out_of_scope_block.block_hash), + 64, + ); + + insert_active_replay_manifest_contract( + database.pool(), + 1, + "ens", + "ens_v2_registry_l1", + chain, + "ens_v2", + registry_contract_instance_id, + registry_address, + "registry", + ) + .await?; + sqlx::query("UPDATE manifest_versions SET manifest_payload = $2 WHERE manifest_id = $1") + .bind(1_i64) + .bind(test_manifest_payload()) + .execute(database.pool()) + .await?; + upsert_raw_blocks( + database.pool(), + &[ + provider_block_to_raw_block( + chain, + &out_of_scope_block, + CanonicalityState::Canonical, + ), + provider_block_to_raw_block(chain, &block, CanonicalityState::Canonical), + ], + ) + .await?; + let out_of_scope_logs = (0..2001) + .map(|log_index| RawLog { + chain_id: chain.to_owned(), + block_hash: out_of_scope_block.block_hash.clone(), + block_number: out_of_scope_block.block_number, + transaction_hash: transaction_hash_for_block(&out_of_scope_block), + transaction_index: 0, + log_index, + emitting_address: registry_address.to_owned(), + topics: vec![format!("0x{:064x}", 0xdead_u64)], + data: Vec::new(), + canonicality_state: CanonicalityState::Canonical, + }) + .collect::>(); + upsert_raw_logs(database.pool(), &out_of_scope_logs).await?; + let logs = (0..1001) + .map(|log_index| RawLog { + chain_id: chain.to_owned(), + block_hash: block.block_hash.clone(), + block_number: block.block_number, + transaction_hash: transaction_hash_for_block(&block), + transaction_index: 0, + log_index, + emitting_address: registry_address.to_owned(), + topics: vec![format!("0x{:064x}", 0xdead_u64)], + data: Vec::new(), + canonicality_state: CanonicalityState::Canonical, + }) + .collect::>(); + upsert_raw_logs(database.pool(), &logs).await?; + + let mut progress = CountingProgress::default(); + let mut progress_ref = + Some(&mut progress as &mut dyn bigname_adapters::StartupAdapterProgress); + let summary = sync_ens_v2_registry_for_mode_for_test( + database.pool(), + None, + chain, + std::slice::from_ref(&block.block_hash), + None, + PersistedRawPayloadAdapterSyncModeForTest::LiveOrBackfill, + false, + &mut progress_ref, + ) + .await?; + + assert_eq!(summary.scanned_log_count, 1001); + assert_eq!( + progress.0, 15, + "the dispatcher must preserve every emitter, raw-log, processing, and publication beat" + ); + + let source_scope = [( + "ens_v2_registry_l1".to_owned(), + registry_address.to_owned(), + 0, + block.block_number, + )]; + let mut scoped_progress = CountingProgress::default(); + let mut scoped_progress_ref = + Some(&mut scoped_progress as &mut dyn bigname_adapters::StartupAdapterProgress); + let scoped_summary = sync_ens_v2_registry_for_mode_for_test( + database.pool(), + None, + chain, + std::slice::from_ref(&block.block_hash), + Some(&source_scope), + PersistedRawPayloadAdapterSyncModeForTest::LiveOrBackfill, + false, + &mut scoped_progress_ref, + ) + .await?; + assert_eq!(scoped_summary.scanned_log_count, 1001); + assert_eq!( + scoped_progress.0, 15, + "the scoped LiveOrBackfill block-hash arm must preserve the same completed-work beats" + ); + database.cleanup().await +} + #[tokio::test] async fn sync_adapter_owned_raw_log_state_backfills_reverse_claims_from_stored_raw_logs() -> Result<()> { diff --git a/crates/adapters/src/ens_v2_registry/entrypoints.rs b/crates/adapters/src/ens_v2_registry/entrypoints.rs index 8ffe724d..f49cf85a 100644 --- a/crates/adapters/src/ens_v2_registry/entrypoints.rs +++ b/crates/adapters/src/ens_v2_registry/entrypoints.rs @@ -38,6 +38,25 @@ impl EnsV2RegistryResourceSurfaceSyncSummary { .await } + pub async fn sync_for_block_hashes_with_progress( + pool: &PgPool, + chain: &str, + block_hashes: &[String], + progress: &mut dyn StartupAdapterProgress, + ) -> Result { + sync_ens_v2_registry_resource_surface_with_scope( + pool, + chain, + true, + block_hashes, + None, + RawLogCanonicalityFilter::IncludeObserved, + None, + Some(progress), + ) + .await + } + pub async fn sync_for_block_hashes_canonical_only( pool: &PgPool, chain: &str, @@ -56,6 +75,25 @@ impl EnsV2RegistryResourceSurfaceSyncSummary { .await } + pub async fn sync_for_block_hashes_canonical_only_with_progress( + pool: &PgPool, + chain: &str, + block_hashes: &[String], + progress: &mut dyn StartupAdapterProgress, + ) -> Result { + sync_ens_v2_registry_resource_surface_with_scope( + pool, + chain, + true, + block_hashes, + None, + RawLogCanonicalityFilter::CanonicalOnly, + None, + Some(progress), + ) + .await + } + pub async fn sync_for_block_hashes_with_source_scope( pool: &PgPool, chain: &str, @@ -75,6 +113,26 @@ impl EnsV2RegistryResourceSurfaceSyncSummary { .await } + pub async fn sync_for_block_hashes_with_source_scope_and_progress( + pool: &PgPool, + chain: &str, + block_hashes: &[String], + source_scope: &[(String, String, i64, i64)], + progress: &mut dyn StartupAdapterProgress, + ) -> Result { + sync_ens_v2_registry_resource_surface_with_scope( + pool, + chain, + true, + block_hashes, + Some(source_scope), + RawLogCanonicalityFilter::IncludeObserved, + None, + Some(progress), + ) + .await + } + pub async fn sync_for_block_hashes_with_source_scope_canonical_only( pool: &PgPool, chain: &str, @@ -93,6 +151,26 @@ impl EnsV2RegistryResourceSurfaceSyncSummary { ) .await } + + pub async fn sync_for_block_hashes_with_source_scope_canonical_only_and_progress( + pool: &PgPool, + chain: &str, + block_hashes: &[String], + source_scope: &[(String, String, i64, i64)], + progress: &mut dyn StartupAdapterProgress, + ) -> Result { + sync_ens_v2_registry_resource_surface_with_scope( + pool, + chain, + true, + block_hashes, + Some(source_scope), + RawLogCanonicalityFilter::CanonicalOnly, + None, + Some(progress), + ) + .await + } } pub async fn sync_ens_v2_registry_resource_surface( diff --git a/docs/storage.md b/docs/storage.md index 972de168..7965f921 100644 --- a/docs/storage.md +++ b/docs/storage.md @@ -609,7 +609,7 @@ and must be allowed to make the heartbeat stale. | indexer canonical reconciliation | prove stored lineage and retained-history coverage | newly-covered (#229) | Completed lineage blocks and bounded coverage candidate, companion, fork, and delta pages. | | indexer canonical reconciliation | orphan the losing branch | newly-covered (#229) | Each losing block plus bounded raw-fact, normalized-event, identity, and execution-invalidation pages. | | indexer canonical reconciliation | persist winning raw blocks, transactions, receipts, logs, and event-silent calls | newly-covered (#229) | Each block and each 1,000-row persistence page. | -| indexer canonical reconciliation | observe code hashes and replay adapter-owned state | newly-covered (#229) | Completed code-observation pages, at-most-32-block adapter chunks, and internal raw-log/processing/persistence pages for block-derived events and all selected ENSv1/ENSv2 adapter families. | +| indexer canonical reconciliation | observe code hashes and replay adapter-owned state | newly-covered (#229) | Completed code-observation pages, at-most-32-block adapter chunks, and internal raw-log/processing/persistence pages for block-derived events and all selected ENSv1/ENSv2 adapter families, including ENSv2 registry block-hash entrypoints. | | indexer canonical reconciliation | recover exact missing retained-history coverage | newly-covered (#229) | Exact watched-target pages and provider-backed sequential backfill units of at most 32 blocks; coverage is recorded before the beat. | | indexer canonical reconciliation | publish chain checkpoints | newly-covered (#229) | A beat follows the committed checkpoint and admission-epoch fence; the atomic publication itself has no synthetic keepalive. | | indexer resolver-profile convergence | compare and advance the authority journal | newly-covered (#229) | Completed authority-entry, seed-family expansion, staged-diff, mutation, and forced-target pages. | From 8254619d392df23d8db3dd95b34193ee2a83e1c6 Mon Sep 17 00:00:00 2001 From: TateB Date: Thu, 23 Jul 2026 06:10:01 +0000 Subject: [PATCH 08/12] fix(indexer): heartbeat post-bootstrap replay Route completed bootstrap raw-range replay through the progress-aware normalized-event entrypoint and cover multi-page callbacks at the caller boundary. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01C9RTnbCyhqEUSPc7xCK76p --- apps/indexer/src/main/bootstrap_backfill.rs | 7 +- .../main/bootstrap_backfill/orchestration.rs | 27 +++--- .../src/main/bootstrap_backfill/replay.rs | 23 +++++ apps/indexer/src/main/tests/bootstrap.rs | 87 +++++++++++++++++++ docs/storage.md | 1 + 5 files changed, 133 insertions(+), 12 deletions(-) create mode 100644 apps/indexer/src/main/bootstrap_backfill/replay.rs diff --git a/apps/indexer/src/main/bootstrap_backfill.rs b/apps/indexer/src/main/bootstrap_backfill.rs index a33c59ad..36e6b7b6 100644 --- a/apps/indexer/src/main/bootstrap_backfill.rs +++ b/apps/indexer/src/main/bootstrap_backfill.rs @@ -18,7 +18,6 @@ use crate::{ reconciliation::{ HeaderAuditMode, RawFactNormalizedEventReplayRequest, RawFactNormalizedEventReplaySelection, log_raw_fact_normalized_event_replay_outcome, - replay_raw_fact_normalized_events, }, run::startup_heartbeat::{StartupAdapterHeartbeat, StartupHeartbeat}, runtime::{IntakeChainTask, validate_provider_registry_for_intake_tasks}, @@ -40,6 +39,8 @@ mod planning; mod progress; #[path = "bootstrap_backfill/recovery.rs"] mod recovery; +#[path = "bootstrap_backfill/replay.rs"] +mod replay; use checkpoints::{load_bootstrap_segment_checkpoint, load_bootstrap_target_checkpoint}; #[cfg(test)] @@ -78,6 +79,10 @@ pub(crate) use recovery::{ automatic_backfill_retention_snapshot_is_stable, converge_ens_v2_retained_history_through_block, load_bootstrap_retention_snapshot, }; +#[cfg(not(test))] +use replay::replay_completed_bootstrap_raw_range; +#[cfg(test)] +pub(crate) use replay::replay_completed_bootstrap_raw_range; const BOOTSTRAP_BACKFILL_LEASE_DURATION_SECS: u64 = 300; pub(crate) const DEFAULT_BOOTSTRAP_BACKFILL_WORKERS: usize = 0; pub(crate) const DEFAULT_BOOTSTRAP_BACKFILL_RANGE_BLOCKS: i64 = 50_000; diff --git a/apps/indexer/src/main/bootstrap_backfill/orchestration.rs b/apps/indexer/src/main/bootstrap_backfill/orchestration.rs index c27f5ed1..7e8de267 100644 --- a/apps/indexer/src/main/bootstrap_backfill/orchestration.rs +++ b/apps/indexer/src/main/bootstrap_backfill/orchestration.rs @@ -483,9 +483,7 @@ pub(super) async fn run_startup_bootstrap_backfills_inner( }; outcome.add_job(&job_outcome); if replay_completed_raw_ranges && job_outcome.raw_log_count > 0 { - let replay_outcome = replay_raw_fact_normalized_events( - pool, - RawFactNormalizedEventReplayRequest { + let replay_request = RawFactNormalizedEventReplayRequest { deployment_profile: deployment_profile.clone(), chain: task.chain.clone(), selection: RawFactNormalizedEventReplaySelection::ScopedBlockRange { @@ -497,15 +495,22 @@ pub(super) async fn run_startup_bootstrap_backfills_inner( job_outcome.to_block, ), }, - }, - ) - .await - .with_context(|| { - format!( - "failed to replay normalized events after bootstrap raw backfill for chain {} range {}..={}", - task.chain, job_outcome.from_block, job_outcome.to_block + }; + let replay_heartbeat = heartbeat + .as_deref_mut() + .map(|heartbeat| (heartbeat, heartbeat_chain_ids.as_slice())); + let replay_outcome = replay_completed_bootstrap_raw_range( + pool, + replay_request, + replay_heartbeat, ) - })?; + .await + .with_context(|| { + format!( + "failed to replay normalized events after bootstrap raw backfill for chain {} range {}..={}", + task.chain, job_outcome.from_block, job_outcome.to_block + ) + })?; log_raw_fact_normalized_event_replay_outcome(&replay_outcome); outcome.normalized_replay_job_count += 1; outcome.normalized_replay_synced_count += diff --git a/apps/indexer/src/main/bootstrap_backfill/replay.rs b/apps/indexer/src/main/bootstrap_backfill/replay.rs new file mode 100644 index 00000000..5f2d8a2c --- /dev/null +++ b/apps/indexer/src/main/bootstrap_backfill/replay.rs @@ -0,0 +1,23 @@ +use anyhow::Result; + +use crate::{ + reconciliation::{ + RawFactNormalizedEventReplayOutcome, RawFactNormalizedEventReplayRequest, + replay_raw_fact_normalized_events, replay_raw_fact_normalized_events_with_progress, + }, + run::startup_heartbeat::{StartupAdapterHeartbeat, StartupHeartbeat}, +}; + +pub(crate) async fn replay_completed_bootstrap_raw_range( + pool: &sqlx::PgPool, + request: RawFactNormalizedEventReplayRequest, + heartbeat: Option<(&mut StartupHeartbeat, &[String])>, +) -> Result { + match heartbeat { + Some((heartbeat, chain_ids)) => { + let mut progress = StartupAdapterHeartbeat::new(heartbeat, chain_ids); + replay_raw_fact_normalized_events_with_progress(pool, request, &mut progress).await + } + None => replay_raw_fact_normalized_events(pool, request).await, + } +} diff --git a/apps/indexer/src/main/tests/bootstrap.rs b/apps/indexer/src/main/tests/bootstrap.rs index b7bae450..07b1b8e5 100644 --- a/apps/indexer/src/main/tests/bootstrap.rs +++ b/apps/indexer/src/main/tests/bootstrap.rs @@ -69,6 +69,93 @@ fn bootstrap_planning_uses_finalized_head_and_fails_closed_without_it() -> Resul Ok(()) } +#[tokio::test] +async fn post_bootstrap_normalized_replay_preserves_per_page_heartbeats() -> Result<()> { + let database = TestDatabase::new().await?; + let chain = "ethereum-mainnet"; + let reverse_address = "0x0000000000000000000000000000000000000229"; + let block = provider_block( + "0x2290229022902290229022902290229022902290229022902290229022902290", + None, + 229, + ); + insert_active_replay_manifest_contract( + database.pool(), + 229, + "ens", + "ens_v1_reverse_l1", + chain, + "ens_v1", + Uuid::from_u128(0x229), + reverse_address, + "reverse_registrar", + ) + .await?; + upsert_raw_blocks( + database.pool(), + &[provider_block_to_raw_block( + chain, + &block, + CanonicalityState::Canonical, + )], + ) + .await?; + let raw_logs = (0_i64..1_001) + .map(|log_index| RawLog { + chain_id: chain.to_owned(), + block_hash: block.block_hash.clone(), + block_number: block.block_number, + transaction_hash: + "0x2291229122912291229122912291229122912291229122912291229122912291" + .to_owned(), + transaction_index: 0, + log_index, + emitting_address: reverse_address.to_owned(), + topics: Vec::new(), + data: Vec::new(), + canonicality_state: CanonicalityState::Canonical, + }) + .collect::>(); + upsert_raw_logs(database.pool(), &raw_logs).await?; + + let instance_id = "post-bootstrap-normalized-replay-progress-test"; + install_stale_indexer_heartbeat(database.pool(), instance_id).await?; + let mut heartbeat = crate::run::startup_heartbeat::StartupHeartbeat::new( + instance_id.to_owned(), + tokio::time::Duration::ZERO, + ); + let chain_ids = vec![chain.to_owned()]; + let progress_before = heartbeat.adapter_progress_count(); + let outcome = replay_completed_bootstrap_raw_range( + database.pool(), + RawFactNormalizedEventReplayRequest { + deployment_profile: "mainnet".to_owned(), + chain: chain.to_owned(), + selection: RawFactNormalizedEventReplaySelection::ScopedBlockRange { + from_block: block.block_number, + to_block: block.block_number, + source_scope: vec![crate::source_scope::SourceScopeTarget { + source_family: "ens_v1_reverse_l1".to_owned(), + address: reverse_address.to_owned(), + from_block: block.block_number, + to_block: block.block_number, + }], + }, + }, + Some((&mut heartbeat, &chain_ids)), + ) + .await?; + let replay_progress_count = heartbeat.adapter_progress_count() - progress_before; + + assert_eq!(outcome.canonical_raw_log_count, 1_001); + assert!( + replay_progress_count >= 4, + "1,001 reverse-claim inputs must report both raw-log pages and both processing pages through the post-bootstrap caller, got {replay_progress_count} beats" + ); + + database.cleanup().await +} + #[test] fn ensure_manifest_root_ready_accepts_loaded_root() -> Result<()> { ensure_manifest_root_ready(&manifest_load_summary(ManifestLoadStatus::Loaded)) diff --git a/docs/storage.md b/docs/storage.md index 7965f921..1ac1fad7 100644 --- a/docs/storage.md +++ b/docs/storage.md @@ -596,6 +596,7 @@ and must be allowed to make the heartbeat stale. | indexer startup adapter sync | ENSv2 permissions | covered (#221/#237) | Completed raw-log/processing pages and bounded resource/event writes. | | indexer startup | plan source identities, targets, segments, reservations, and coverage for hash-pinned bootstrap | newly-covered (#229) | Completed 1,000-target planning/identity pages, durable reservation and coverage pages, and completed worker-result handling. | | indexer startup | execute hash-pinned bootstrap provider units | covered (#213) | Provider-backed ranges are divided into at-most-32-block progress units; a beat follows only a fully persisted unit. Worker completion and failure remain coordinator-owned. | +| indexer startup | replay normalized events from each completed hash-pinned bootstrap range | newly-covered (#229) | Reuses the normalized-event replay progress entrypoint so completed raw-log, processing, and persistence pages beat before the enclosing bootstrap segment completes. | | indexer startup | post-bootstrap adapter synchronization | covered (#213/#221/#237) | Reuses every adapter's page progress route and records completed family boundaries. | | indexer startup | widen bootstrap admission to the live watch plan | newly-covered (#229) | Reuses manifest/discovery page progress and records each 10,000-address plan/task copy or comparison chunk. | | indexer startup | initial resolver authority journal and resolver-profile convergence | newly-covered (#229) | Uses the authority and convergence boundaries described below. | From f1eadc06885d1d84706d9d582f2ff49aba033183 Mon Sep 17 00:00:00 2001 From: TateB Date: Thu, 23 Jul 2026 06:18:14 +0000 Subject: [PATCH 09/12] fix(indexer): restore payload branch atomicity Restore origin/main's range-transaction boundaries for losing-branch payload families and winning-lineage re-canonicalization while retaining read-side and post-commit progress beats. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01C9RTnbCyhqEUSPc7xCK76p --- .../src/main/reconciliation/canonical.rs | 6 +- .../canonical/contiguous_gap.rs | 6 +- .../reconciliation/canonical/orphaning.rs | 103 +--- apps/indexer/src/main/tests/reorg.rs | 469 ++++++++++++++++++ docs/storage.md | 11 +- 5 files changed, 503 insertions(+), 92 deletions(-) diff --git a/apps/indexer/src/main/reconciliation/canonical.rs b/apps/indexer/src/main/reconciliation/canonical.rs index 8629e5fa..d25a6f7b 100644 --- a/apps/indexer/src/main/reconciliation/canonical.rs +++ b/apps/indexer/src/main/reconciliation/canonical.rs @@ -580,10 +580,8 @@ pub(super) async fn reconcile_canonical_head_with_progress( ) }) .collect::>(); - for block_chunk in lineage_blocks.chunks(32) { - upsert_recanonicalized_lineage_blocks_without_snapshots(pool, block_chunk).await?; - record_live_progress(pool, progress).await?; - } + upsert_recanonicalized_lineage_blocks_without_snapshots(pool, &lineage_blocks).await?; + record_live_progress(pool, progress).await?; Ok(CanonicalReconciliation { status, diff --git a/apps/indexer/src/main/reconciliation/canonical/contiguous_gap.rs b/apps/indexer/src/main/reconciliation/canonical/contiguous_gap.rs index 0d0441f9..c4abb5c7 100644 --- a/apps/indexer/src/main/reconciliation/canonical/contiguous_gap.rs +++ b/apps/indexer/src/main/reconciliation/canonical/contiguous_gap.rs @@ -87,10 +87,8 @@ pub(super) async fn reconcile_contiguous_checkpoint_gap( ) }) .collect::>(); - for lineage_chunk in lineage_blocks.chunks(LIVE_GAP_PROGRESS_BLOCKS) { - upsert_recanonicalized_lineage_blocks_without_snapshots(pool, lineage_chunk).await?; - record_progress(pool, progress).await?; - } + upsert_recanonicalized_lineage_blocks_without_snapshots(pool, &lineage_blocks).await?; + record_progress(pool, progress).await?; let status = if path.len() == 1 { CanonicalReconciliationStatus::Appended diff --git a/apps/indexer/src/main/reconciliation/canonical/orphaning.rs b/apps/indexer/src/main/reconciliation/canonical/orphaning.rs index d6f7d87b..fefaef62 100644 --- a/apps/indexer/src/main/reconciliation/canonical/orphaning.rs +++ b/apps/indexer/src/main/reconciliation/canonical/orphaning.rs @@ -2,11 +2,10 @@ use anyhow::{Context, Result}; use bigname_adapters::StartupAdapterProgress; use bigname_storage::{ ExecutionOutcomeInvalidationProgress, ExecutionOutcomeInvalidationProgressFuture, - IdentityOrphanCounts, RawFactOrphanCounts, invalidate_execution_outcomes_for_orphaned_blocks, + invalidate_execution_outcomes_for_orphaned_blocks, invalidate_execution_outcomes_for_orphaned_blocks_with_progress, load_chain_lineage_block, - load_raw_block, mark_block_derived_normalized_events_range_orphaned, - mark_chain_lineage_range_orphaned, mark_identity_rows_range_orphaned, - mark_raw_block_facts_range_orphaned, + mark_block_derived_normalized_events_range_orphaned, mark_chain_lineage_range_orphaned, + mark_identity_rows_range_orphaned, mark_raw_block_facts_range_orphaned, }; use tracing::info; @@ -121,42 +120,25 @@ async fn orphan_reorg_losing_branch_payloads_inner( progress, ) .await?; - let losing_blocks = load_losing_raw_block_path( + let orphaned_raw_facts = mark_raw_block_facts_range_orphaned( pool, chain, current_canonical_hash, stop_before_hash, - progress, ) .await?; - - let mut orphaned_raw_facts = RawFactOrphanCounts::default(); - for (block_hash, parent_hash) in &losing_blocks { - let counts = mark_raw_block_facts_range_orphaned( - pool, - chain, - block_hash, - parent_hash.as_deref(), - ) - .await?; - add_raw_fact_counts(&mut orphaned_raw_facts, &counts); - record_live_progress(pool, progress).await?; - } if orphaned_raw_facts.code_hash_count > 0 { coverage_frontiers.invalidate_raw_code_baseline_frontier(chain); } + record_live_progress(pool, progress).await?; - let mut orphaned_normalized_event_count = 0u64; - for (block_hash, parent_hash) in &losing_blocks { - orphaned_normalized_event_count += mark_block_derived_normalized_events_range_orphaned( - pool, - chain, - block_hash, - parent_hash.as_deref(), - ) - .await?; - record_live_progress(pool, progress).await?; - } + let orphaned_normalized_event_count = mark_block_derived_normalized_events_range_orphaned( + pool, + chain, + current_canonical_hash, + stop_before_hash, + ) + .await?; if orphaned_normalized_event_count > 0 { info!( service = "indexer", @@ -165,15 +147,15 @@ async fn orphan_reorg_losing_branch_payloads_inner( "block-derived normalized events orphaned for the losing branch" ); } + record_live_progress(pool, progress).await?; - let mut orphaned_identity_counts = IdentityOrphanCounts::default(); - for (block_hash, parent_hash) in &losing_blocks { - let counts = - mark_identity_rows_range_orphaned(pool, chain, block_hash, parent_hash.as_deref()) - .await?; - add_identity_counts(&mut orphaned_identity_counts, &counts); - record_live_progress(pool, progress).await?; - } + let orphaned_identity_counts = mark_identity_rows_range_orphaned( + pool, + chain, + current_canonical_hash, + stop_before_hash, + ) + .await?; if orphaned_identity_counts.token_lineage_count > 0 || orphaned_identity_counts.resource_count > 0 || orphaned_identity_counts.name_surface_count > 0 @@ -189,6 +171,7 @@ async fn orphan_reorg_losing_branch_payloads_inner( "identity rows orphaned for the losing branch" ); } + record_live_progress(pool, progress).await?; } let execution_invalidation_summary = match progress.as_deref_mut() { @@ -211,47 +194,3 @@ async fn orphan_reorg_losing_branch_payloads_inner( Ok(()) } - -async fn load_losing_raw_block_path( - pool: &sqlx::PgPool, - chain: &str, - from_hash: &str, - stop_before_hash: Option<&str>, - progress: &mut Option<&mut dyn StartupAdapterProgress>, -) -> Result)>> { - let mut blocks = Vec::new(); - let mut cursor_hash = Some(from_hash.to_owned()); - while let Some(block_hash) = cursor_hash { - if Some(block_hash.as_str()) == stop_before_hash { - break; - } - let raw_block = load_raw_block(pool, chain, &block_hash) - .await? - .with_context(|| { - format!( - "missing stored raw block for chain {chain} block {block_hash} while orphaning losing-branch payloads" - ) - })?; - cursor_hash = raw_block.parent_hash.clone(); - blocks.push((raw_block.block_hash, raw_block.parent_hash)); - record_live_progress(pool, progress).await?; - } - Ok(blocks) -} - -fn add_raw_fact_counts(total: &mut RawFactOrphanCounts, next: &RawFactOrphanCounts) { - total.block_count += next.block_count; - total.code_hash_count += next.code_hash_count; - total.transaction_count += next.transaction_count; - total.receipt_count += next.receipt_count; - total.log_count += next.log_count; - total.call_snapshot_count += next.call_snapshot_count; - total.payload_cache_metadata_count += next.payload_cache_metadata_count; -} - -fn add_identity_counts(total: &mut IdentityOrphanCounts, next: &IdentityOrphanCounts) { - total.token_lineage_count += next.token_lineage_count; - total.resource_count += next.resource_count; - total.name_surface_count += next.name_surface_count; - total.surface_binding_count += next.surface_binding_count; -} diff --git a/apps/indexer/src/main/tests/reorg.rs b/apps/indexer/src/main/tests/reorg.rs index b745d543..b8ffb878 100644 --- a/apps/indexer/src/main/tests/reorg.rs +++ b/apps/indexer/src/main/tests/reorg.rs @@ -1084,6 +1084,475 @@ async fn losing_branch_orphaning_rolls_back_every_block_on_mid_branch_failure() database.cleanup().await } +#[tokio::test] +async fn losing_branch_payload_families_roll_back_every_block_on_mid_branch_failure() -> Result<()> +{ + let database = TestDatabase::new().await?; + let chain = "ethereum-mainnet"; + let ancestor = provider_block( + "0x4111111111111111111111111111111111111111111111111111111111111111", + None, + 1, + ); + let losing_parent = provider_block( + "0x4222222222222222222222222222222222222222222222222222222222222222", + Some(&ancestor.block_hash), + 2, + ); + let losing_middle = provider_block( + "0x4333333333333333333333333333333333333333333333333333333333333333", + Some(&losing_parent.block_hash), + 3, + ); + let losing_head = provider_block( + "0x4444444444444444444444444444444444444444444444444444444444444444", + Some(&losing_middle.block_hash), + 4, + ); + let losing_blocks = [ + losing_parent.clone(), + losing_middle.clone(), + losing_head.clone(), + ]; + let raw_blocks = std::iter::once(&ancestor) + .chain(losing_blocks.iter()) + .map(|block| provider_block_to_raw_block(chain, block, CanonicalityState::Canonical)) + .collect::>(); + upsert_raw_blocks(database.pool(), &raw_blocks).await?; + + let raw_logs = losing_blocks + .iter() + .map(|block| RawLog { + chain_id: chain.to_owned(), + block_hash: block.block_hash.clone(), + block_number: block.block_number, + transaction_hash: transaction_hash_for_block(block), + transaction_index: 0, + log_index: 0, + emitting_address: "0x0000000000000000000000000000000000000444".to_owned(), + topics: vec![ + "0xaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + .to_owned(), + ], + data: Vec::new(), + canonicality_state: CanonicalityState::Canonical, + }) + .collect::>(); + upsert_raw_logs(database.pool(), &raw_logs).await?; + + for block in &losing_blocks { + sqlx::query( + r#" + INSERT INTO normalized_events ( + event_identity, + namespace, + event_kind, + source_family, + manifest_version, + source_manifest_id, + chain_id, + block_number, + block_hash, + transaction_hash, + log_index, + raw_fact_ref, + derivation_kind, + canonicality_state, + before_state, + after_state + ) + VALUES ( + $1, + 'ens', + 'AtomicityTest', + 'ens_test', + 1, + NULL, + $2, + $3, + $4, + $5, + 0, + '{"kind":"raw_log"}'::jsonb, + 'raw_log_preimage_observation', + 'canonical'::canonicality_state, + '{}'::jsonb, + '{}'::jsonb + ) + "#, + ) + .bind(format!("payload-atomicity:{}", block.block_hash)) + .bind(chain) + .bind(block.block_number) + .bind(&block.block_hash) + .bind(transaction_hash_for_block(block)) + .execute(database.pool()) + .await?; + } + + let token_lineages = losing_blocks + .iter() + .zip(0_u128..) + .map(|(block, index)| TokenLineage { + token_lineage_id: Uuid::from_u128(0x4400 + index), + chain_id: chain.to_owned(), + block_hash: block.block_hash.clone(), + block_number: block.block_number, + provenance: json!({"test": "payload_atomicity"}), + canonicality_state: CanonicalityState::Canonical, + }) + .collect::>(); + upsert_token_lineages(database.pool(), &token_lineages).await?; + let resources = losing_blocks + .iter() + .zip(token_lineages.iter()) + .zip(0_u128..) + .map(|((block, token_lineage), index)| Resource { + resource_id: Uuid::from_u128(0x4500 + index), + token_lineage_id: Some(token_lineage.token_lineage_id), + chain_id: chain.to_owned(), + block_hash: block.block_hash.clone(), + block_number: block.block_number, + provenance: json!({"test": "payload_atomicity"}), + canonicality_state: CanonicalityState::Canonical, + }) + .collect::>(); + upsert_resources(database.pool(), &resources).await?; + + sqlx::raw_sql( + r#" + CREATE TABLE payload_atomicity_failure_target ( + singleton BOOLEAN PRIMARY KEY DEFAULT true CHECK (singleton), + table_name TEXT NOT NULL + ); + INSERT INTO payload_atomicity_failure_target (table_name) + VALUES ('raw_logs'); + + CREATE FUNCTION fail_middle_payload_orphaning() + RETURNS TRIGGER + LANGUAGE plpgsql + AS $$ + BEGIN + IF TG_TABLE_NAME = ( + SELECT table_name + FROM payload_atomicity_failure_target + WHERE singleton + ) + AND NEW.block_hash = + '0x4333333333333333333333333333333333333333333333333333333333333333' + AND NEW.canonicality_state = 'orphaned'::canonicality_state + THEN + RAISE EXCEPTION 'injected mid-branch payload failure in %', TG_TABLE_NAME; + END IF; + RETURN NEW; + END; + $$; + + CREATE TRIGGER fail_middle_raw_log_orphaning + BEFORE UPDATE OF canonicality_state ON raw_logs + FOR EACH ROW + EXECUTE FUNCTION fail_middle_payload_orphaning(); + + CREATE TRIGGER fail_middle_normalized_event_orphaning + BEFORE UPDATE OF canonicality_state ON normalized_events + FOR EACH ROW + EXECUTE FUNCTION fail_middle_payload_orphaning(); + + CREATE TRIGGER fail_middle_resource_orphaning + BEFORE UPDATE OF canonicality_state ON resources + FOR EACH ROW + EXECUTE FUNCTION fail_middle_payload_orphaning(); + "#, + ) + .execute(database.pool()) + .await?; + + let expected_canonical = losing_blocks + .iter() + .map(|block| (block.block_hash.clone(), "canonical".to_owned())) + .collect::>(); + let expected_orphaned = losing_blocks + .iter() + .map(|block| (block.block_hash.clone(), "orphaned".to_owned())) + .collect::>(); + let coverage_frontiers = ChainCoverageFrontiers::default(); + + let raw_error = orphan_reorg_losing_branch_payloads( + database.pool(), + chain, + Some(&losing_head.block_hash), + Some(&ancestor.block_hash), + &coverage_frontiers, + ) + .await + .expect_err("the injected raw-log failure must abort whole-branch raw-fact orphaning"); + assert!( + format!("{raw_error:#}").contains("injected mid-branch payload failure in raw_logs"), + "unexpected raw-fact orphaning error: {raw_error:#}" + ); + assert_eq!( + sqlx::query_as::<_, (String, String)>( + r#" + SELECT block_hash, canonicality_state::TEXT + FROM raw_logs + WHERE chain_id = $1 + ORDER BY block_number + "#, + ) + .bind(chain) + .fetch_all(database.pool()) + .await?, + expected_canonical, + "raw-fact orphaning must update the complete losing branch or none of it" + ); + + sqlx::query( + "UPDATE payload_atomicity_failure_target SET table_name = 'normalized_events' WHERE singleton", + ) + .execute(database.pool()) + .await?; + let event_error = orphan_reorg_losing_branch_payloads( + database.pool(), + chain, + Some(&losing_head.block_hash), + Some(&ancestor.block_hash), + &coverage_frontiers, + ) + .await + .expect_err("the injected normalized-event failure must abort whole-branch event orphaning"); + assert!( + format!("{event_error:#}") + .contains("injected mid-branch payload failure in normalized_events"), + "unexpected normalized-event orphaning error: {event_error:#}" + ); + assert_eq!( + sqlx::query_as::<_, (String, String)>( + r#" + SELECT block_hash, canonicality_state::TEXT + FROM normalized_events + WHERE chain_id = $1 + ORDER BY block_number + "#, + ) + .bind(chain) + .fetch_all(database.pool()) + .await?, + expected_canonical, + "normalized-event orphaning must update the complete losing branch or none of it" + ); + assert_eq!( + sqlx::query_as::<_, (String, String)>( + r#" + SELECT block_hash, canonicality_state::TEXT + FROM raw_logs + WHERE chain_id = $1 + ORDER BY block_number + "#, + ) + .bind(chain) + .fetch_all(database.pool()) + .await?, + expected_orphaned, + "the preceding raw-fact family retains its independently committed range transaction" + ); + + sqlx::query( + "UPDATE payload_atomicity_failure_target SET table_name = 'resources' WHERE singleton", + ) + .execute(database.pool()) + .await?; + let identity_error = orphan_reorg_losing_branch_payloads( + database.pool(), + chain, + Some(&losing_head.block_hash), + Some(&ancestor.block_hash), + &coverage_frontiers, + ) + .await + .expect_err("the injected resource failure must abort whole-branch identity orphaning"); + assert!( + format!("{identity_error:#}").contains("injected mid-branch payload failure in resources"), + "unexpected identity orphaning error: {identity_error:#}" + ); + assert_eq!( + sqlx::query_as::<_, (String, String)>( + r#" + SELECT block_hash, canonicality_state::TEXT + FROM resources + WHERE chain_id = $1 + ORDER BY block_number + "#, + ) + .bind(chain) + .fetch_all(database.pool()) + .await?, + expected_canonical, + "identity orphaning must update the complete losing branch or none of it" + ); + assert_eq!( + sqlx::query_as::<_, (String, String)>( + r#" + SELECT block_hash, canonicality_state::TEXT + FROM token_lineages + WHERE chain_id = $1 + ORDER BY block_number + "#, + ) + .bind(chain) + .fetch_all(database.pool()) + .await?, + expected_canonical, + "a later identity-table failure must roll back earlier tables in the same branch transaction" + ); + + database.cleanup().await +} + +#[tokio::test] +async fn contiguous_winning_branch_recanonicalization_rolls_back_prior_chunks_on_failure() +-> Result<()> { + assert_winning_branch_recanonicalization_rolls_back_prior_chunks(false).await +} + +#[tokio::test] +async fn reorg_winning_branch_recanonicalization_rolls_back_prior_chunks_on_failure() -> Result<()> +{ + assert_winning_branch_recanonicalization_rolls_back_prior_chunks(true).await +} + +async fn assert_winning_branch_recanonicalization_rolls_back_prior_chunks( + force_reorg_walk: bool, +) -> Result<()> { + let database = TestDatabase::new().await?; + let chain = "ethereum-mainnet"; + let ancestor = provider_block( + "0x5100000000000000000000000000000000000000000000000000000000000000", + None, + 100, + ); + let mut parent_hash = ancestor.block_hash.clone(); + let mut winning_blocks = Vec::new(); + for block_number in 101_i64..=133 { + let block_hash = format!("0x52{block_number:062x}"); + let block = provider_block(&block_hash, Some(&parent_hash), block_number); + parent_hash = block.block_hash.clone(); + winning_blocks.push(block); + } + let losing_checkpoint = force_reorg_walk.then(|| { + provider_block( + "0x5300000000000000000000000000000000000000000000000000000000000065", + Some(&ancestor.block_hash), + 101, + ) + }); + let mut lineage_blocks = vec![provider_block_to_lineage( + chain, + &ancestor, + CanonicalityState::Canonical, + )]; + if let Some(losing_checkpoint) = &losing_checkpoint { + lineage_blocks.push(provider_block_to_lineage( + chain, + losing_checkpoint, + CanonicalityState::Canonical, + )); + } + lineage_blocks.extend(winning_blocks.iter().map(|block| { + provider_block_to_lineage(chain, block, CanonicalityState::Orphaned) + })); + upsert_chain_lineage_blocks(database.pool(), &lineage_blocks).await?; + + let failing_hash = &winning_blocks + .last() + .expect("winning branch must have a final block") + .block_hash; + sqlx::raw_sql(&format!( + r#" + CREATE FUNCTION fail_late_winning_branch_recanonicalization() + RETURNS TRIGGER + LANGUAGE plpgsql + AS $$ + BEGIN + IF NEW.block_hash = '{failing_hash}' + AND OLD.canonicality_state = 'orphaned'::canonicality_state + AND NEW.canonicality_state = 'canonical'::canonicality_state + THEN + RAISE EXCEPTION 'injected failure in the second winning-branch chunk'; + END IF; + RETURN NEW; + END; + $$; + + CREATE TRIGGER fail_late_winning_branch_recanonicalization + BEFORE UPDATE OF canonicality_state ON chain_lineage + FOR EACH ROW + EXECUTE FUNCTION fail_late_winning_branch_recanonicalization(); + "#, + )) + .execute(database.pool()) + .await?; + + let latest_head = winning_blocks + .last() + .expect("winning branch must have a head") + .clone(); + let checkpoint_block = losing_checkpoint.as_ref().unwrap_or(&ancestor); + let (provider, server) = bundle_provider(winning_blocks.clone()).await?; + let error = reconcile_canonical_head( + database.pool(), + &provider, + chain, + &ChainCheckpoint { + chain_id: chain.to_owned(), + canonical_block_hash: Some(checkpoint_block.block_hash.clone()), + canonical_block_number: Some(checkpoint_block.block_number), + safe_block_hash: None, + safe_block_number: None, + finalized_block_hash: None, + finalized_block_number: None, + }, + &latest_head, + HeaderAuditMode::Minimal, + &[], + &ChainCoverageFrontiers::default(), + ) + .await + .expect_err("the injected late failure must abort winning-branch re-canonicalization"); + assert!( + format!("{error:#}").contains("injected failure in the second winning-branch chunk"), + "unexpected winning-branch error: {error:#}" + ); + + let states = sqlx::query_as::<_, (String, String)>( + r#" + SELECT block_hash, canonicality_state::TEXT + FROM chain_lineage + WHERE chain_id = $1 + AND block_hash = ANY($2::TEXT[]) + ORDER BY block_number + "#, + ) + .bind(chain) + .bind( + winning_blocks + .iter() + .map(|block| block.block_hash.clone()) + .collect::>(), + ) + .fetch_all(database.pool()) + .await?; + assert_eq!(states.len(), 33); + assert!( + states + .iter() + .all(|(_, state)| state == CanonicalityState::Orphaned.as_str()), + "a failed winning-branch transaction must leave every prior application chunk orphaned: {states:?}" + ); + + server.abort(); + database.cleanup().await +} + #[tokio::test] async fn awaiting_ancestor_raw_persistence_preserves_walked_orphaned_lineage() -> Result<()> { let database = TestDatabase::new().await?; diff --git a/docs/storage.md b/docs/storage.md index 1ac1fad7..23e4873e 100644 --- a/docs/storage.md +++ b/docs/storage.md @@ -606,9 +606,9 @@ and must be allowed to make the heartbeat stale. | indexer live poll | full-corpus adapter refresh after manifest/discovery changes | newly-covered (#229) | Existing adapter page callbacks now remain connected through live reconciliation; completed family, reconciliation, and finalization boundaries also beat. | | indexer live poll | post-replay adapter backlog and cursor publication | newly-covered (#229) | Completed adapter families and bounded publication/cursor units; publication beats only after its transaction commits. | | indexer live poll | fetch the provider head, safe head, and finalized head | n-a | A fixed set of atomic RPC results is required before reconciliation can advance. A provider call taking more than 20 seconds represents no completed progress and must not receive a beat. | -| indexer canonical reconciliation | walk provider parents and fill a contiguous gap | newly-covered (#229) | Each fetched parent and each completed block-persistence unit; non-reorg adapter work is split into at-most-32-block chunks. | +| indexer canonical reconciliation | walk provider parents and fill a contiguous gap | newly-covered (#229) | Each fetched parent and each completed block-persistence unit; non-reorg adapter work is split into at-most-32-block chunks, while winning-branch lineage re-canonicalization beats after its branch-atomic transaction commits. | | indexer canonical reconciliation | prove stored lineage and retained-history coverage | newly-covered (#229) | Completed lineage blocks and bounded coverage candidate, companion, fork, and delta pages. | -| indexer canonical reconciliation | orphan the losing branch | newly-covered (#229) | Each losing block plus bounded raw-fact, normalized-event, identity, and execution-invalidation pages. | +| indexer canonical reconciliation | orphan the losing branch | newly-covered (#229) | Each losing block during read-only path validation, each committed branch-wide raw-fact, normalized-event, or identity transaction, and each bounded execution-invalidation page. | | indexer canonical reconciliation | persist winning raw blocks, transactions, receipts, logs, and event-silent calls | newly-covered (#229) | Each block and each 1,000-row persistence page. | | indexer canonical reconciliation | observe code hashes and replay adapter-owned state | newly-covered (#229) | Completed code-observation pages, at-most-32-block adapter chunks, and internal raw-log/processing/persistence pages for block-derived events and all selected ENSv1/ENSv2 adapter families, including ENSv2 registry block-hash entrypoints. | | indexer canonical reconciliation | recover exact missing retained-history coverage | newly-covered (#229) | Exact watched-target pages and provider-backed sequential backfill units of at most 32 blocks; coverage is recorded before the beat. | @@ -984,6 +984,13 @@ Rules: Reorg repair preserves audit truth: orphaned rows persist for explanation and rebuild, not deletion. The losing branch's lineage, identity rows, and normalized events stay canonical-state `orphaned` so explain and history routes can still reconstruct what was observed. +Reconciliation validates the complete losing path before changing canonicality. +The selected lineage branch, each dependent raw-fact, normalized-event, or +identity storage family, and winning-lineage re-canonicalization each commit +their complete branch range in one transaction. Progress heartbeats occur +during the read-only path walk and after those transactions commit, never +inside a write window that could expose only part of one branch. + Registry discovery authority is repaired before the winning chain checkpoint advances. Reconciliation forces complete canonical ENSv1, Basenames, and ENSv2 registry-source passes through the winning head even when that head is event-silent, so a losing `NewOwner`, `NewResolver`, `SubregistryUpdated`, or resolver assignment cannot remain in active discovery edges, contract addresses, or the watch plan merely because log-derived live scope is empty. ENSv1 and Basenames replay closed historical registry-emitter intervals through that boundary so a canonical subregistry branch closed by the losing fork can be restored in full. Raw facts after the winning head do not enter the repair, and existing non-orphaned later discovery assignments are preserved. Discovery mutations remain manifest/discovery-owned and use the normal reachability cascade and admission-epoch fence. Generation-zero retained input is replayable from its original boundary. After destructive retention rotation, ENSv1 and Basenames may recover this target-bounded repair only from gap-free current-generation coverage for every required registry-emitter interval under an unchanged discovery-admission epoch; ENSv2 uses its generation-bound root/registry proof. Missing or stale coverage, topic drift, generation drift, or epoch drift leaves the winning checkpoint unpublished for retry. Other stateful ENSv1 replay still requires its own documented generation-bound proof or versioned snapshot. Execution cache rows follow the same hash-first canonicality rule. When reorg repair marks a block identity `orphaned`, synchronous indexer/reorg repair invalidates or deletes any reusable `execution_cache_outcomes` row whose dependency set includes that `(chain_id, block_hash)` or a boundary resolved through it. The invalidation makes the cached outcome ineligible for reuse; it does not delete raw facts, traces, steps, attachments, or any execution-owned audit artifact. From 04c08fc92fa5a8fbb10c73fc98649297088ba140 Mon Sep 17 00:00:00 2001 From: TateB Date: Thu, 23 Jul 2026 07:29:54 +0000 Subject: [PATCH 10/12] fix(adapters): make replay emitter selection path independent Use total emitter identity ordering after both watched-contract loader paths and restore the ordinary full-replay page size while retaining small startup heartbeat pages. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01C9RTnbCyhqEUSPc7xCK76p --- apps/indexer/src/main/runtime/adapter_sync.rs | 5 +- .../ens_v1_unwrapped_authority/pipeline.rs | 2 +- crates/adapters/src/ens_v2_common.rs | 25 +++++-- .../adapters/src/ens_v2_permissions/load.rs | 53 +++++++++---- .../src/ens_v2_registrar/active_emitters.rs | 75 ++++++++++++++++--- 5 files changed, 120 insertions(+), 40 deletions(-) diff --git a/apps/indexer/src/main/runtime/adapter_sync.rs b/apps/indexer/src/main/runtime/adapter_sync.rs index 9f831776..488f1b42 100644 --- a/apps/indexer/src/main/runtime/adapter_sync.rs +++ b/apps/indexer/src/main/runtime/adapter_sync.rs @@ -15,9 +15,8 @@ use super::logging::{ log_ens_v2_resolver_sync_summary, }; -// At roughly 2 KiB per log, the 100,000-log target is about 200 MB page-resident: -// about 10x smaller than normalized replay's resident set and 400x smaller than -// the #218 OOM class. Ownership/control pages preserve whole blocks, so an +// Startup heartbeat replay uses small pages so long ownership/control scans +// report liveness between pages. These pages preserve whole blocks, so an // unusually dense block can exceed this target. pub(crate) const DEFAULT_STARTUP_DISCOVERY_PAGE_LOGS: usize = 1_000; diff --git a/crates/adapters/src/ens_v1_unwrapped_authority/pipeline.rs b/crates/adapters/src/ens_v1_unwrapped_authority/pipeline.rs index 9e771e8a..1b9b5073 100644 --- a/crates/adapters/src/ens_v1_unwrapped_authority/pipeline.rs +++ b/crates/adapters/src/ens_v1_unwrapped_authority/pipeline.rs @@ -22,7 +22,7 @@ use profile_stream::{ResolverProfileStreamInput, sync_resolver_profile_stream}; use summary::empty_summary; const FULL_REPLAY_RAW_LOG_STREAM_MAX_BLOCK_SCAN_SPAN: i64 = 262_144; -const FULL_REPLAY_RAW_LOG_STREAM_DEFAULT_MAX_LOGS_PER_PAGE: usize = 1_000; +const FULL_REPLAY_RAW_LOG_STREAM_DEFAULT_MAX_LOGS_PER_PAGE: usize = 100_000; pub(super) async fn sync_ens_v1_unwrapped_authority_with_scope( pool: &PgPool, diff --git a/crates/adapters/src/ens_v2_common.rs b/crates/adapters/src/ens_v2_common.rs index 126b43de..4cdd7afb 100644 --- a/crates/adapters/src/ens_v2_common.rs +++ b/crates/adapters/src/ens_v2_common.rs @@ -17,16 +17,16 @@ use crate::{ startup_progress::{STARTUP_ADAPTER_PROGRESS_PAGE_ROWS, StartupManifestProgress}, }; -#[derive(Clone, Debug, Eq, PartialEq)] +#[derive(Clone, Debug, Eq, Ord, PartialEq, PartialOrd)] pub(crate) struct ActiveEmitter { pub(crate) address: String, - pub(crate) contract_instance_id: Uuid, - pub(crate) source_manifest_id: i64, - pub(crate) namespace: String, pub(crate) source_family: String, - pub(crate) manifest_version: i64, pub(crate) active_from_block_number: Option, pub(crate) active_to_block_number: Option, + pub(crate) source_manifest_id: i64, + pub(crate) contract_instance_id: Uuid, + pub(crate) namespace: String, + pub(crate) manifest_version: i64, } /// Identifies one distinct discovery/watched interval for an emitting address: @@ -87,9 +87,7 @@ fn insert_distinct_emitter( } Entry::Occupied(mut entry) => { let existing = entry.get_mut(); - if (emitter.source_manifest_id, emitter.contract_instance_id) - < (existing.source_manifest_id, existing.contract_instance_id) - { + if emitter < *existing { *existing = emitter; } } @@ -471,5 +469,16 @@ mod tests { let kept = by_scope.values().next().expect("one emitter"); assert_eq!(kept.source_manifest_id, 2); assert_eq!(kept.contract_instance_id, Uuid::from_u128(3)); + + let preferred = emitter_with("0xresolver", 3, 2, Some(100), Some(200)); + let mut alternate = preferred.clone(); + alternate.namespace = "z-namespace".to_owned(); + let mut forward_order = BTreeMap::new(); + insert_distinct_emitter(&mut forward_order, preferred.clone()); + insert_distinct_emitter(&mut forward_order, alternate.clone()); + let mut reverse_order = BTreeMap::new(); + insert_distinct_emitter(&mut reverse_order, alternate); + insert_distinct_emitter(&mut reverse_order, preferred); + assert_eq!(reverse_order, forward_order); } } diff --git a/crates/adapters/src/ens_v2_permissions/load.rs b/crates/adapters/src/ens_v2_permissions/load.rs index 959dfae1..f63048fb 100644 --- a/crates/adapters/src/ens_v2_permissions/load.rs +++ b/crates/adapters/src/ens_v2_permissions/load.rs @@ -1,4 +1,4 @@ -use std::collections::{BTreeMap, HashSet}; +use std::collections::HashSet; use anyhow::{Context, Result}; use bigname_manifests::{load_watched_contracts, load_watched_contracts_scoped_with_progress}; @@ -211,23 +211,11 @@ pub(super) async fn load_active_emitters( // `active_emitter_for_block`'s first-match selection nondeterministic for overlapping windows. // This matches the resolver path's earliest-activation-first ordering (ens_v2_common). let emitter_count = emitters.len(); - let mut ordered = BTreeMap::new(); - for (index, emitter) in emitters.into_iter().enumerate() { - ordered.insert( - ( - emitter.address.clone(), - emitter.source_family.clone(), - emitter.active_from_block_number, - emitter.active_to_block_number, - emitter.source_manifest_id, - emitter.contract_instance_id, - index, - ), - emitter, - ); + emitters.sort(); + for index in 0..emitter_count { record_emitter_progress(pool, progress, index + 1, emitter_count).await?; } - Ok(ordered.into_values().collect()) + Ok(emitters) } async fn load_registry_active_emitters( @@ -328,3 +316,36 @@ fn source_scope_bindings( } (addresses, from_blocks, to_blocks) } + +#[cfg(test)] +mod tests { + use super::*; + use sqlx::types::Uuid; + + fn tied_emitter(namespace: &str) -> ActiveEmitter { + ActiveEmitter { + address: "0xresolver".to_owned(), + source_family: SOURCE_FAMILY_ENS_V2_RESOLVER_L1.to_owned(), + active_from_block_number: Some(100), + active_to_block_number: Some(200), + source_manifest_id: 7, + contract_instance_id: Uuid::from_u128(1), + namespace: namespace.to_owned(), + manifest_version: 1, + } + } + + #[test] + fn exact_sort_tie_has_the_same_winner_for_both_loader_orders() { + let preferred = tied_emitter("a-namespace"); + let alternate = tied_emitter("z-namespace"); + let mut non_progress = vec![preferred.clone(), alternate.clone()]; + let mut progress = vec![alternate, preferred.clone()]; + + non_progress.sort(); + progress.sort(); + + assert_eq!(progress, non_progress); + assert_eq!(active_emitter_for_block(&progress, 150), Some(&preferred)); + } +} diff --git a/crates/adapters/src/ens_v2_registrar/active_emitters.rs b/crates/adapters/src/ens_v2_registrar/active_emitters.rs index 33239fc9..3fd16bac 100644 --- a/crates/adapters/src/ens_v2_registrar/active_emitters.rs +++ b/crates/adapters/src/ens_v2_registrar/active_emitters.rs @@ -2,7 +2,7 @@ use std::collections::{BTreeMap, HashSet}; use anyhow::{Context, Result}; use bigname_manifests::{load_watched_contracts, load_watched_contracts_scoped_with_progress}; -use sqlx::PgPool; +use sqlx::{PgPool, types::Uuid}; use crate::adapter_manifest::{ active_manifest_for_watched_contract, ensure_watched_contract_manifest_chain, @@ -15,14 +15,17 @@ use crate::{ use super::SOURCE_FAMILY_ENS_V2_REGISTRAR_L1; -#[derive(Clone, Debug, Eq, PartialEq)] +#[derive(Clone, Debug, Eq, Ord, PartialEq, PartialOrd)] pub(super) struct ActiveEmitter { pub(super) address: String, + pub(super) source_rank: i32, pub(super) source_manifest_id: i64, + pub(super) contract_instance_id: Uuid, + pub(super) active_from_block_number: Option, + pub(super) active_to_block_number: Option, pub(super) namespace: String, pub(super) source_family: String, pub(super) manifest_version: i64, - pub(super) source_rank: i32, } pub(super) async fn load_active_emitters( @@ -75,19 +78,17 @@ pub(super) async fn load_active_emitters( let candidate = ActiveEmitter { address: watched_contract.address.clone(), + source_rank: source_rank(watched_contract.source), source_manifest_id, + contract_instance_id: watched_contract.contract_instance_id, + active_from_block_number: watched_contract.active_from_block_number, + active_to_block_number: watched_contract.active_to_block_number, namespace: manifest.namespace.clone(), source_family: manifest.source_family.clone(), manifest_version: manifest.manifest_version, - source_rank: source_rank(watched_contract.source), }; - match emitters_by_address.get(&candidate.address) { - Some(current) if !candidate_precedes(&candidate, current) => {} - _ => { - emitters_by_address.insert(candidate.address.clone(), candidate); - } - } + insert_preferred_emitter(&mut emitters_by_address, candidate); record_emitter_progress(pool, progress, index + 1, watched_contract_count).await?; } @@ -106,7 +107,57 @@ async fn record_emitter_progress( Ok(()) } +fn insert_preferred_emitter( + emitters_by_address: &mut BTreeMap, + candidate: ActiveEmitter, +) { + match emitters_by_address.get(&candidate.address) { + Some(current) if !candidate_precedes(&candidate, current) => {} + _ => { + emitters_by_address.insert(candidate.address.clone(), candidate); + } + } +} + fn candidate_precedes(candidate: &ActiveEmitter, current: &ActiveEmitter) -> bool { - (candidate.source_rank, candidate.source_manifest_id) - < (current.source_rank, current.source_manifest_id) + candidate < current +} + +#[cfg(test)] +mod tests { + use super::*; + + fn tied_emitter(contract_instance_id: u128) -> ActiveEmitter { + ActiveEmitter { + address: "0xregistrar".to_owned(), + source_rank: 1, + source_manifest_id: 7, + contract_instance_id: Uuid::from_u128(contract_instance_id), + active_from_block_number: Some(100), + active_to_block_number: None, + namespace: "ens".to_owned(), + source_family: SOURCE_FAMILY_ENS_V2_REGISTRAR_L1.to_owned(), + manifest_version: 1, + } + } + + fn winner(candidates: impl IntoIterator) -> ActiveEmitter { + let mut emitters = BTreeMap::new(); + for candidate in candidates { + insert_preferred_emitter(&mut emitters, candidate); + } + emitters.into_values().next().expect("one address winner") + } + + #[test] + fn exact_rank_tie_has_the_same_winner_for_both_loader_orders() { + let lower_identity = tied_emitter(1); + let higher_identity = tied_emitter(2); + + let non_progress = winner([lower_identity.clone(), higher_identity.clone()]); + let progress = winner([higher_identity, lower_identity.clone()]); + + assert_eq!(progress, non_progress); + assert_eq!(progress, lower_identity); + } } From fa01aa65ab419db3d9a96eeb3a8be8f00612712d Mon Sep 17 00:00:00 2001 From: TateB Date: Thu, 23 Jul 2026 10:51:06 +0000 Subject: [PATCH 11/12] fix(manifests): scope historical successor pages Filter staged desired edges by a qualifying active successor before the page limit, and cover page progression with a non-historical-dominated fixture. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01C9RTnbCyhqEUSPc7xCK76p --- .../discovery/reconciliation/streamed/diff.rs | 57 ++++-- .../reconciliation/streamed/diff/tests.rs | 189 ++++++++++++++++++ 2 files changed, 225 insertions(+), 21 deletions(-) diff --git a/crates/manifests/src/lib/discovery/reconciliation/streamed/diff.rs b/crates/manifests/src/lib/discovery/reconciliation/streamed/diff.rs index b63ca43a..15c94732 100644 --- a/crates/manifests/src/lib/discovery/reconciliation/streamed/diff.rs +++ b/crates/manifests/src/lib/discovery/reconciliation/streamed/diff.rs @@ -119,6 +119,33 @@ const STREAMED_STARTS_NO_LATER_SQL: &str = r#" ) "#; +const STREAMED_SUCCESSOR_IDENTITY_SQL: &str = r#" + de.provenance ->> 'observation_key' = desired.observation_key + AND de.chain_id = desired.chain_id + AND de.edge_kind = desired.edge_kind + AND de.from_contract_instance_id = desired.from_contract_instance_id +"#; + +const STREAMED_STARTS_AFTER_SQL: &str = r#" + ( + de.active_from_block_number > desired.active_from_block_number + OR ( + de.active_from_block_number = desired.active_from_block_number + AND desired.active_from_transaction_index IS NOT NULL + AND desired.active_from_log_index IS NOT NULL + AND (de.provenance ->> 'transaction_index')::BIGINT IS NOT NULL + AND (de.provenance ->> 'log_index')::BIGINT IS NOT NULL + AND ( + (de.provenance ->> 'transaction_index')::BIGINT, + (de.provenance ->> 'log_index')::BIGINT + ) > ( + desired.active_from_transaction_index, + desired.active_from_log_index + ) + ) + ) +"#; + pub(super) struct StreamedDeactivationSourcePage { pub(super) last_edge_id: Option, pub(super) candidates: Vec, @@ -424,6 +451,13 @@ where FROM pg_temp.reconcile_desired_edges desired WHERE desired.desired_row_id > $2 AND desired.active_from_block_number IS NOT NULL + AND EXISTS ( + SELECT 1 + {STREAMED_ACTIVE_EDGE_FROM_SQL} + AND {STREAMED_SUCCESSOR_IDENTITY_SQL} + AND NOT {STREAMED_EDGE_IS_ORPHANED_SQL} + AND {STREAMED_STARTS_AFTER_SQL} + ) ORDER BY desired.desired_row_id LIMIT $3 "# @@ -480,28 +514,9 @@ where (de.provenance ->> 'transaction_index')::BIGINT AS transaction_index, (de.provenance ->> 'log_index')::BIGINT AS log_index {STREAMED_ACTIVE_EDGE_FROM_SQL} - AND de.provenance ->> 'observation_key' = desired.observation_key - AND de.chain_id = desired.chain_id - AND de.edge_kind = desired.edge_kind - AND de.from_contract_instance_id = desired.from_contract_instance_id + AND {STREAMED_SUCCESSOR_IDENTITY_SQL} AND NOT {STREAMED_EDGE_IS_ORPHANED_SQL} - AND ( - de.active_from_block_number > desired.active_from_block_number - OR ( - de.active_from_block_number = desired.active_from_block_number - AND desired.active_from_transaction_index IS NOT NULL - AND desired.active_from_log_index IS NOT NULL - AND (de.provenance ->> 'transaction_index')::BIGINT IS NOT NULL - AND (de.provenance ->> 'log_index')::BIGINT IS NOT NULL - AND ( - (de.provenance ->> 'transaction_index')::BIGINT, - (de.provenance ->> 'log_index')::BIGINT - ) > ( - desired.active_from_transaction_index, - desired.active_from_log_index - ) - ) - ) + AND {STREAMED_STARTS_AFTER_SQL} ORDER BY de.active_from_block_number, (de.provenance ->> 'transaction_index')::BIGINT NULLS FIRST, diff --git a/crates/manifests/src/lib/discovery/reconciliation/streamed/diff/tests.rs b/crates/manifests/src/lib/discovery/reconciliation/streamed/diff/tests.rs index 299e3659..df06fed2 100644 --- a/crates/manifests/src/lib/discovery/reconciliation/streamed/diff/tests.rs +++ b/crates/manifests/src/lib/discovery/reconciliation/streamed/diff/tests.rs @@ -1,7 +1,30 @@ +use std::sync::atomic::{AtomicUsize, Ordering}; + use bigname_test_support::{TestDatabase, TestDatabaseConfig}; use uuid::Uuid; use super::*; +use crate::DiscoveryObservation; + +#[derive(Default)] +struct CountingPageSource { + progress_count: AtomicUsize, +} + +impl DiscoveryObservationPageSource for CountingPageSource { + async fn load_page( + &self, + _after_key: Option<&str>, + _limit: i64, + ) -> Result> { + Ok(Vec::new()) + } + + async fn record_progress(&self) -> Result<()> { + self.progress_count.fetch_add(1, Ordering::Relaxed); + Ok(()) + } +} #[tokio::test] async fn deactivation_source_pages_only_active_edges_for_the_requested_source() -> Result<()> { @@ -121,3 +144,169 @@ async fn deactivation_source_pages_only_active_edges_for_the_requested_source() transaction.rollback().await?; database.cleanup().await } + +#[tokio::test] +async fn historical_pages_only_desired_edges_with_successors() -> Result<()> { + let database = TestDatabase::create_migrated( + TestDatabaseConfig::new("streamed_historical_successor_scoped_pages"), + &bigname_storage::MIGRATOR, + "failed to migrate streamed historical paging test database", + ) + .await?; + let pool = database.pool(); + let from_id = Uuid::from_u128(0xa01); + let to_id = Uuid::from_u128(0xa02); + sqlx::query( + r#" + INSERT INTO contract_instances (contract_instance_id, chain_id, contract_kind) + VALUES ($1, 'historical-chain', 'test'), ($2, 'historical-chain', 'test') + "#, + ) + .bind(from_id) + .bind(to_id) + .execute(pool) + .await?; + sqlx::query( + r#" + INSERT INTO contract_instance_addresses ( + contract_instance_id, + chain_id, + address, + active_from_block_number + ) + VALUES ( + $1, + 'historical-chain', + '0x0000000000000000000000000000000000000a02', + 20 + ) + "#, + ) + .bind(to_id) + .execute(pool) + .await?; + let successor_edge_id = sqlx::query_scalar::<_, i64>( + r#" + INSERT INTO discovery_edges ( + chain_id, + edge_kind, + from_contract_instance_id, + to_contract_instance_id, + discovery_source, + admission, + active_from_block_number, + active_from_block_hash, + provenance + ) + VALUES ( + 'historical-chain', + 'subregistry', + $1, + $2, + 'target-source', + 'reachable_from_root', + 20, + '0x20', + '{"observation_key":"historical"}'::JSONB + ) + RETURNING discovery_edge_id + "#, + ) + .bind(from_id) + .bind(to_id) + .fetch_one(pool) + .await?; + + let mut transaction = pool.begin().await?; + super::super::staging::create_streamed_reconcile_temp_tables(transaction.as_mut()).await?; + sqlx::query( + r#" + INSERT INTO pg_temp.reconcile_desired_edges ( + observation_key, + chain_id, + edge_kind, + from_contract_instance_id, + to_contract_instance_id, + discovery_source, + source_manifest_id, + admission, + active_from_block_number, + active_from_block_hash, + provenance_json + ) + SELECT + format('non-historical-%s', series), + 'historical-chain', + 'subregistry', + $1, + $2, + 'target-source', + -1, + 'reachable_from_root', + 10, + '0x10', + format('{"observation_key":"non-historical-%s"}', series) + FROM generate_series(1, 2001) AS series + "#, + ) + .bind(from_id) + .bind(to_id) + .execute(transaction.as_mut()) + .await?; + sqlx::query( + r#" + INSERT INTO pg_temp.reconcile_desired_edges ( + observation_key, + chain_id, + edge_kind, + from_contract_instance_id, + to_contract_instance_id, + discovery_source, + source_manifest_id, + admission, + active_from_block_number, + active_from_block_hash, + provenance_json + ) + VALUES ( + 'historical', + 'historical-chain', + 'subregistry', + $1, + $2, + 'target-source', + -1, + 'reachable_from_root', + 10, + '0x10', + '{"observation_key":"historical"}' + ) + "#, + ) + .bind(from_id) + .bind(to_id) + .execute(transaction.as_mut()) + .await?; + + let source = CountingPageSource::default(); + let mut retained_newer_edge_ids = HashSet::new(); + let historical = collect_streamed_historical_edges( + transaction.as_mut(), + "target-source", + 1_000, + &mut retained_newer_edge_ids, + &source, + ) + .await?; + + assert_eq!(historical.len(), 1); + assert_eq!(historical[0].1.observation_key, "historical"); + assert_eq!(retained_newer_edge_ids, HashSet::from([successor_edge_id])); + assert_eq!( + source.progress_count.load(Ordering::Relaxed), + 2, + "only the one true historical-successor page must beat" + ); + transaction.rollback().await?; + database.cleanup().await +} From 77abf637fbd00345a37ae88bef56fda15fd8cd6d Mon Sep 17 00:00:00 2001 From: TateB Date: Thu, 23 Jul 2026 10:51:19 +0000 Subject: [PATCH 12/12] fix(indexer): propagate final recovery heartbeats Route normalized replay ENSv2 recovery, scoped ENSv1 emitter conversion, and startup authority checkpoint deletion through their progress-aware paths. Add caller-level per-page coverage and scoped-emitter regressions. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01C9RTnbCyhqEUSPc7xCK76p --- .../src/main/normalized_replay_catchup.rs | 2 + .../coverage_recovery.rs | 37 ++++++- apps/indexer/src/main/reconciliation.rs | 1 + apps/indexer/src/main/startup_heartbeat.rs | 16 ++- .../main/tests/normalized_replay_catchup.rs | 104 ++++++++++++++++++ .../loader/active_emitters.rs | 19 +++- .../src/ens_v1_subregistry_discovery/tests.rs | 37 +++++++ .../ens_v1_unwrapped_authority/checkpoint.rs | 4 +- .../checkpoint/startup_events.rs | 4 +- 9 files changed, 210 insertions(+), 14 deletions(-) diff --git a/apps/indexer/src/main/normalized_replay_catchup.rs b/apps/indexer/src/main/normalized_replay_catchup.rs index 43cbf227..4ed4330a 100644 --- a/apps/indexer/src/main/normalized_replay_catchup.rs +++ b/apps/indexer/src/main/normalized_replay_catchup.rs @@ -44,6 +44,8 @@ pub(crate) use test_hook::{ install_before_cursor_failure_record as install_before_cursor_failure_record_test_hook, }; +#[cfg(test)] +pub(crate) use coverage_recovery::recover_ens_v2_live_coverage_requirement_for_replay; use coverage_recovery::replay_full_closure_with_coverage_recovery; use cursors::{ advance_cursor, ensure_cursor, record_cursor_failure, diff --git a/apps/indexer/src/main/normalized_replay_catchup/coverage_recovery.rs b/apps/indexer/src/main/normalized_replay_catchup/coverage_recovery.rs index 9c26f483..b8fa6b81 100644 --- a/apps/indexer/src/main/normalized_replay_catchup/coverage_recovery.rs +++ b/apps/indexer/src/main/normalized_replay_catchup/coverage_recovery.rs @@ -11,11 +11,45 @@ use crate::{ reconciliation::{ EnsV2LiveCoverageRecoveryStatus, HeaderAuditMode, RawFactNormalizedEventReplayOutcome, automatic_stateless_replay_completed, recover_ens_v2_live_coverage_requirement, + recover_ens_v2_live_coverage_requirement_with_progress, }, }; const MAX_COVERAGE_RECOVERY_ATTEMPTS: usize = 32; +pub(crate) async fn recover_ens_v2_live_coverage_requirement_for_replay( + pool: &sqlx::PgPool, + deployment_profile: &str, + provider: &(impl ChainProviderOps + ?Sized), + header_audit_mode: HeaderAuditMode, + requirement: &bigname_adapters::EnsV2MissingCoverage, + progress: &mut Option<&mut NormalizedReplayHeartbeat>, +) -> Result { + match progress.as_deref_mut() { + Some(progress) => { + recover_ens_v2_live_coverage_requirement_with_progress( + pool, + deployment_profile, + provider, + header_audit_mode, + requirement, + progress, + ) + .await + } + None => { + recover_ens_v2_live_coverage_requirement( + pool, + deployment_profile, + provider, + header_audit_mode, + requirement, + ) + .await + } + } +} + #[expect(clippy::too_many_arguments)] pub(super) async fn replay_full_closure_with_coverage_recovery( pool: &sqlx::PgPool, @@ -67,12 +101,13 @@ pub(super) async fn replay_full_closure_with_coverage_recovery( }; recovery_attempt += 1; - let status = match recover_ens_v2_live_coverage_requirement( + let status = match recover_ens_v2_live_coverage_requirement_for_replay( pool, deployment_profile, provider, header_audit_mode, &requirement, + progress, ) .await { diff --git a/apps/indexer/src/main/reconciliation.rs b/apps/indexer/src/main/reconciliation.rs index 7f555851..bad48f07 100644 --- a/apps/indexer/src/main/reconciliation.rs +++ b/apps/indexer/src/main/reconciliation.rs @@ -47,6 +47,7 @@ pub(crate) use canonical::{ poll_provider_heads_with_adapter_sync, poll_provider_heads_with_adapter_sync_and_progress, reconcile_canonical_head, reconcile_fetched_heads, reconcile_fetched_heads_with_adapter_sync, reconcile_intake_chain_task, recover_ens_v2_live_coverage_requirement, + recover_ens_v2_live_coverage_requirement_with_progress, }; #[allow(unused_imports)] pub(crate) use lineage::{ diff --git a/apps/indexer/src/main/startup_heartbeat.rs b/apps/indexer/src/main/startup_heartbeat.rs index 4f2035f0..63da5673 100644 --- a/apps/indexer/src/main/startup_heartbeat.rs +++ b/apps/indexer/src/main/startup_heartbeat.rs @@ -70,6 +70,11 @@ impl NormalizedReplayHeartbeat { chain_ids: Arc::new(chain_ids), } } + + #[cfg(test)] + pub(crate) async fn adapter_progress_count(&self) -> usize { + self.heartbeat.lock().await.adapter_progress_count() + } } impl<'a> StartupAdapterHeartbeat<'a> { @@ -117,11 +122,12 @@ impl bigname_adapters::StartupAdapterProgress for NormalizedReplayHeartbeat { pool: &'a PgPool, ) -> bigname_adapters::StartupAdapterProgressFuture<'a> { Box::pin(async move { - self.heartbeat - .lock() - .await - .record_if_due(pool, &self.chain_ids) - .await + let mut heartbeat = self.heartbeat.lock().await; + #[cfg(test)] + { + heartbeat.adapter_progress_count += 1; + } + heartbeat.record_if_due(pool, &self.chain_ids).await }) } } diff --git a/apps/indexer/src/main/tests/normalized_replay_catchup.rs b/apps/indexer/src/main/tests/normalized_replay_catchup.rs index 49cd9a0d..bf75eccf 100644 --- a/apps/indexer/src/main/tests/normalized_replay_catchup.rs +++ b/apps/indexer/src/main/tests/normalized_replay_catchup.rs @@ -308,6 +308,110 @@ async fn spawned_normalized_replay_beats_on_progress_and_exposes_a_later_wedge() database.cleanup().await } +#[tokio::test] +async fn normalized_replay_coverage_recovery_preserves_per_page_heartbeats() -> Result<()> { + let database = TestDatabase::new().await?; + create_ops_catchup_backfill_job_tables(database.pool()).await?; + let chain = "normalized-replay-coverage-progress-chain"; + let source_family = "ens_v2_registry_l1"; + let address = "0x0000000000000000000000000000000000005230"; + let contract_instance_id = Uuid::from_u128(52_300); + let through_block = crate::backfill::DEFAULT_HASH_PINNED_BACKFILL_CHUNK_BLOCKS + 1; + insert_active_replay_manifest_contract( + database.pool(), + 52_300, + "ens", + source_family, + chain, + "ens_v2", + contract_instance_id, + address, + "registry", + ) + .await?; + sqlx::query( + r#" + UPDATE contract_instance_addresses + SET active_from_block_number = 1 + WHERE contract_instance_id = $1 + "#, + ) + .bind(contract_instance_id) + .execute(database.pool()) + .await?; + create_complete_raw_log_staging_input_fixture(database.pool(), chain, 0).await?; + sqlx::query( + "INSERT INTO discovery_admission_epochs (chain_id, epoch) VALUES ($1, 0)", + ) + .bind(chain) + .execute(database.pool()) + .await?; + + let fixtures = linear_provider_blocks(through_block) + .into_iter() + .map(|block| ProviderBlockFixture { + block, + logs: Vec::new(), + }) + .collect(); + let (provider, server) = bundle_provider_with_fixtures(fixtures).await?; + let instance_id = "normalized-replay-coverage-progress-test"; + install_stale_indexer_heartbeat(database.pool(), instance_id).await?; + let mut heartbeat = crate::run::startup_heartbeat::NormalizedReplayHeartbeat::new( + instance_id.to_owned(), + tokio::time::Duration::ZERO, + vec![chain.to_owned()], + ); + let progress_before = heartbeat.adapter_progress_count().await; + let requirement = bigname_adapters::EnsV2MissingCoverage { + chain: chain.to_owned(), + retention_generation: 0, + source_family: source_family.to_owned(), + address: address.to_owned(), + required_from_block: 1, + required_to_block: through_block, + }; + let status = { + let mut progress = Some(&mut heartbeat); + normalized_replay_catchup::recover_ens_v2_live_coverage_requirement_for_replay( + database.pool(), + "sepolia", + &provider, + HeaderAuditMode::Minimal, + &requirement, + &mut progress, + ) + .await? + }; + let recovery_progress_count = heartbeat.adapter_progress_count().await - progress_before; + + assert_eq!(status, EnsV2LiveCoverageRecoveryStatus::Recovered); + assert!( + recovery_progress_count >= 2, + "a 1,025-block generation-bound recovery must preserve at least two chunk-page beats through the normalized-replay caller, got {recovery_progress_count}" + ); + assert_eq!( + sqlx::query_as::<_, (i64, i64)>( + r#" + SELECT covered_from_block, covered_to_block + FROM backfill_coverage_facts + WHERE chain_id = $1 + AND source_family = $2 + AND address = $3 + "#, + ) + .bind(chain) + .bind(source_family) + .bind(address) + .fetch_one(database.pool()) + .await?, + (1, through_block) + ); + + server.abort(); + database.cleanup().await +} + #[tokio::test] async fn normalized_replay_failure_journal_keeps_child_heartbeat_ownership() -> Result<()> { let database = TestDatabase::new().await?; diff --git a/crates/adapters/src/ens_v1_subregistry_discovery/loader/active_emitters.rs b/crates/adapters/src/ens_v1_subregistry_discovery/loader/active_emitters.rs index f121c120..3877bc6b 100644 --- a/crates/adapters/src/ens_v1_subregistry_discovery/loader/active_emitters.rs +++ b/crates/adapters/src/ens_v1_subregistry_discovery/loader/active_emitters.rs @@ -39,12 +39,13 @@ pub(in crate::ens_v1_subregistry_discovery) async fn load_active_emitters( if has_source_scope { let mut manifest_emitters = - load_manifest_declared_active_emitters(pool, chain, source_scope).await?; + load_manifest_declared_active_emitters(pool, chain, source_scope, progress).await?; if source_scope_covered_by_emitters(source_scope, &manifest_emitters) { return Ok(manifest_emitters); } - manifest_emitters - .extend(load_scoped_discovery_active_emitters(pool, chain, source_scope).await?); + manifest_emitters.extend( + load_scoped_discovery_active_emitters(pool, chain, source_scope, progress).await?, + ); sort_active_emitters(&mut manifest_emitters); return Ok(manifest_emitters); } @@ -231,6 +232,7 @@ async fn load_scoped_discovery_active_emitters( pool: &PgPool, chain: &str, source_scope: &[RegistryRawLogSourceScopeTarget], + progress: &mut Option<&mut dyn StartupAdapterProgress>, ) -> Result> { let scoped_targets = source_scope .iter() @@ -471,13 +473,17 @@ async fn load_scoped_discovery_active_emitters( format!("failed to load scoped discovery ENSv1 registry emitters for {chain}") })?; - active_emitters_from_rows(rows) + match progress.as_deref_mut() { + Some(progress) => active_emitters_from_rows_with_progress(pool, rows, progress).await, + None => active_emitters_from_rows(rows), + } } async fn load_manifest_declared_active_emitters( pool: &PgPool, chain: &str, source_scope: &[RegistryRawLogSourceScopeTarget], + progress: &mut Option<&mut dyn StartupAdapterProgress>, ) -> Result> { let scoped_source_families = source_scope .iter() @@ -571,5 +577,8 @@ async fn load_manifest_declared_active_emitters( format!("failed to load manifest-declared ENSv1 registry emitters for {chain}") })?; - active_emitters_from_rows(rows) + match progress.as_deref_mut() { + Some(progress) => active_emitters_from_rows_with_progress(pool, rows, progress).await, + None => active_emitters_from_rows(rows), + } } diff --git a/crates/adapters/src/ens_v1_subregistry_discovery/tests.rs b/crates/adapters/src/ens_v1_subregistry_discovery/tests.rs index ab6f1261..0ecce8db 100644 --- a/crates/adapters/src/ens_v1_subregistry_discovery/tests.rs +++ b/crates/adapters/src/ens_v1_subregistry_discovery/tests.rs @@ -806,6 +806,43 @@ async fn canonical_new_owner_log_persists_one_active_subregistry_edge_and_expand database.cleanup().await } +#[tokio::test] +async fn source_scoped_emitter_loading_preserves_startup_progress() -> Result<()> { + let _permit = crate::acquire_test_db_permit().await; + let test_dir = TestDir::new()?; + let database = TestDatabase::new().await?; + let chain = "ethereum-mainnet"; + let registry_address = "0x00000000000C2E074eC69A0dFb2997BA6C7d2E1E"; + + test_dir.write_manifest("ens", "ens_v1_registry_l1", "v1", &manifest_contents(true))?; + sync_repository(database.pool(), &load_repository(&test_dir.path)?).await?; + let source_scope = normalized_registry_source_scope_targets(&[( + ENS_V1_REGISTRY_SOURCE_FAMILY.to_owned(), + registry_address.to_owned(), + 42, + 42, + )]); + let mut progress = CountingStartupAdapterProgress::default(); + let emitters = { + let mut progress_ref = Some(&mut progress as &mut dyn StartupAdapterProgress); + load_active_emitters( + database.pool(), + chain, + Some(&source_scope), + false, + &mut progress_ref, + ) + .await? + }; + + assert!(!emitters.is_empty()); + assert!( + progress.record_count > 0, + "source-scoped emitter loading must preserve the progress-aware row path" + ); + database.cleanup().await +} + #[tokio::test] async fn startup_checkpointed_subregistry_matches_uncheckpointed_edges_and_events() -> Result<()> { let _permit = crate::acquire_test_db_permit().await; diff --git a/crates/adapters/src/ens_v1_unwrapped_authority/checkpoint.rs b/crates/adapters/src/ens_v1_unwrapped_authority/checkpoint.rs index 70ce2e4e..db71bc93 100644 --- a/crates/adapters/src/ens_v1_unwrapped_authority/checkpoint.rs +++ b/crates/adapters/src/ens_v1_unwrapped_authority/checkpoint.rs @@ -20,8 +20,8 @@ use crate::checkpoint_codec::JsonbCheckpointCodec; use items::{checkpoint_item_rows, checkpoint_pending_observation_delete_keys}; use items::{ checkpoint_item_rows_with_progress, checkpoint_pending_observation_delete_keys_with_progress, - delete_checkpoint_items, delete_checkpoint_items_with_progress, - insert_checkpoint_items_with_progress, update_checkpoint_progress, + delete_checkpoint_items_with_progress, insert_checkpoint_items_with_progress, + update_checkpoint_progress, }; pub(super) use payload::{decode_item, encode_item}; use payload::{flushed_events_from_payload, summary_from_payload, summary_payload}; diff --git a/crates/adapters/src/ens_v1_unwrapped_authority/checkpoint/startup_events.rs b/crates/adapters/src/ens_v1_unwrapped_authority/checkpoint/startup_events.rs index b9d58b77..30cbb51e 100644 --- a/crates/adapters/src/ens_v1_unwrapped_authority/checkpoint/startup_events.rs +++ b/crates/adapters/src/ens_v1_unwrapped_authority/checkpoint/startup_events.rs @@ -137,11 +137,13 @@ impl UnwrappedAuthorityReplayCheckpoint { &events, ) .await?; - delete_checkpoint_items( + delete_checkpoint_items_with_progress( + pool, &mut transaction, self, ITEM_KIND_STARTUP_PENDING_EVENT, &item_keys, + startup_progress, ) .await?; let next_inserted_count = staged_events