diff --git a/.env.example b/.env.example index 641d7217..e1871dfd 100644 --- a/.env.example +++ b/.env.example @@ -8,10 +8,21 @@ DATABASE_URL=postgresql://bigname:bigname@127.0.0.1:5432/bigname BIGNAME_DATABASE_URL=postgresql://bigname:bigname@127.0.0.1:5432/bigname # Per-service primary pool ceiling; the API reserves one additional readiness connection. BIGNAME_DATABASE_MAX_CONNECTIONS=10 +# Stable service identities used by docker-compose.server.yml. +BIGNAME_INDEXER_HEARTBEAT_INSTANCE_ID=indexer +BIGNAME_WORKER_HEARTBEAT_INSTANCE_ID=worker # API bind address for local bigname-api serve. BIGNAME_API_BIND_ADDR=127.0.0.1:3000 -# API live execution RPC entries, as comma-delimited = pairs. +BIGNAME_API_HEARTBEAT_MAX_AGE_SECS=20 +BIGNAME_API_WORKER_REBUILD_PHASE_MAX_AGE_SECS=43200 +BIGNAME_API_STATUS_PROVIDER_TIMEOUT_MS=750 +BIGNAME_API_STATUS_PROVIDER_REFRESH_SECS=5 +BIGNAME_API_STATUS_PROVIDER_CACHE_TTL_SECS=30 +BIGNAME_API_STATUS_MAX_BLOCK_LAG=30 +BIGNAME_API_STATUS_MAX_LAG_SECS=60 +# API live execution and status-readiness RPC entries, as comma-delimited = pairs. +# Status remains degraded for every expected status chain omitted here. # BIGNAME_API_CHAIN_RPC_URLS=ethereum-mainnet=http://127.0.0.1:8545 # Maximum addresses accepted by the native identity batch endpoint. BIGNAME_API_IDENTITY_BATCH_LIMIT=1000 @@ -45,6 +56,7 @@ BIGNAME_API_TRUST_X_FORWARDED_FOR=false BIGNAME_INDEXER_MANIFESTS_ROOT=manifests/mainnet # Poll interval in seconds for live indexer loops. BIGNAME_INDEXER_POLL_INTERVAL_SECS=5 +BIGNAME_INDEXER_HEARTBEAT_MAX_AGE_SECS=20 # Maximum watched addresses verified per chain by one code-observation sweep tick. BIGNAME_INDEXER_RAW_CODE_BASELINE_MAX_ADDRESSES_PER_TICK=2048 # Block span for hash-pinned backfill ranges. @@ -145,6 +157,8 @@ BIGNAME_INDEXER_LIVE_ADAPTER_BACKLOG_BLOCK_BATCH_SIZE=1 # Worker loop interval in seconds. BIGNAME_WORKER_POLL_INTERVAL_SECS=5 +BIGNAME_WORKER_HEARTBEAT_MAX_AGE_SECS=20 +BIGNAME_WORKER_REBUILD_PHASE_MAX_AGE_SECS=43200 # Optional worker RPC for ENSv1 legacy text hydration during projection replay/apply. # BIGNAME_WORKER_CHAIN_RPC_URLS=ethereum-mainnet=http://127.0.0.1:8545 # Multicall3 address for projection-owned text hydration. diff --git a/.env.server.example b/.env.server.example index 9f0a3d2a..b0e838a7 100644 --- a/.env.server.example +++ b/.env.server.example @@ -8,12 +8,22 @@ POSTGRES_PASSWORD=change-me BIGNAME_DATABASE_URL=postgresql://bigname:change-me@postgres:5432/bigname # Per-service primary pool ceiling; the API reserves one additional readiness connection. BIGNAME_DATABASE_MAX_CONNECTIONS=10 +# Stable process identities shared by each service and its healthcheck command. +BIGNAME_INDEXER_HEARTBEAT_INSTANCE_ID=indexer +BIGNAME_WORKER_HEARTBEAT_INSTANCE_ID=worker # API binding. The container listens on 3000; these control the host binding. # Keep the API's host-published port private; Caddy is the public path. BIGNAME_API_HOST=127.0.0.1 BIGNAME_API_PORT=3000 BIGNAME_API_BIND_ADDR=0.0.0.0:3000 +BIGNAME_API_HEARTBEAT_MAX_AGE_SECS=20 +BIGNAME_API_WORKER_REBUILD_PHASE_MAX_AGE_SECS=43200 +BIGNAME_API_STATUS_PROVIDER_TIMEOUT_MS=750 +BIGNAME_API_STATUS_PROVIDER_REFRESH_SECS=5 +BIGNAME_API_STATUS_PROVIDER_CACHE_TTL_SECS=30 +BIGNAME_API_STATUS_MAX_BLOCK_LAG=30 +BIGNAME_API_STATUS_MAX_LAG_SECS=60 # Maximum addresses accepted by the native identity batch endpoint. BIGNAME_API_IDENTITY_BATCH_LIMIT=1000 # Maximum inputs accepted by the v2 lookup endpoint. @@ -37,7 +47,8 @@ BIGNAME_PUBLIC_SITE_ADDRESS=:80 BIGNAME_PUBLIC_HTTP_PORT=80 BIGNAME_PUBLIC_HTTPS_PORT=443 -# Comma-delimited = entries for API-triggered live execution. +# Comma-delimited = entries for API-triggered live execution and +# status readiness. Every expected status chain must be present in production. # ENS verified resolution currently needs ethereum-mainnet when no matching # persisted execution output exists. # API JSON-RPC support accepts http:// and https:// endpoints. @@ -46,6 +57,7 @@ BIGNAME_PUBLIC_HTTPS_PORT=443 # Runtime profile selection. Use /app/manifests/sepolia for the ENSv2 Sepolia profile. BIGNAME_INDEXER_MANIFESTS_ROOT=/app/manifests/mainnet BIGNAME_INDEXER_POLL_INTERVAL_SECS=5 +BIGNAME_INDEXER_HEARTBEAT_MAX_AGE_SECS=20 BIGNAME_INDEXER_HASH_PINNED_BACKFILL_CHUNK_BLOCKS=1024 BIGNAME_INDEXER_HASH_PINNED_BACKFILL_MAX_LOGS_PER_PUSH=10000 # Legacy alias for BIGNAME_INDEXER_HASH_PINNED_BACKFILL_MAX_LOGS_PER_PUSH. @@ -115,6 +127,8 @@ BIGNAME_INDEXER_LIVE_ADAPTER_BACKLOG_BLOCK_BATCH_SIZE=1 # Use the same extra address list as BIGNAME_WORKER_PRIMARY_NAME_LEGACY_REVERSE_RESOLVER_ADDRESSES. # BIGNAME_INDEXER_EVENT_SILENT_REVERSE_RESOLVER_ADDRESSES=0x... BIGNAME_WORKER_POLL_INTERVAL_SECS=5 +BIGNAME_WORKER_HEARTBEAT_MAX_AGE_SECS=20 +BIGNAME_WORKER_REBUILD_PHASE_MAX_AGE_SECS=43200 # Optional worker RPC for projection-owned ENSv1 legacy text hydration and # legacy event-silent reverse resolver primary-name hydration. diff --git a/apps/api/src/cli.rs b/apps/api/src/cli.rs index e91484b5..fc53a583 100644 --- a/apps/api/src/cli.rs +++ b/apps/api/src/cli.rs @@ -19,7 +19,7 @@ pub(crate) struct Cli { #[derive(Subcommand, Debug)] pub(crate) enum Command { - Serve(ServeArgs), + Serve(Box), PrintOpenapi, } @@ -47,6 +47,48 @@ pub(crate) struct ServeArgs { pub(crate) rpc_timeout_ms: u64, #[command(flatten)] pub(crate) bounds: ApiBoundsConfig, + #[arg( + long, + env = "BIGNAME_API_HEARTBEAT_MAX_AGE_SECS", + default_value_t = 20_i64 + )] + pub(crate) heartbeat_max_age_secs: i64, + #[arg( + long, + env = "BIGNAME_API_WORKER_REBUILD_PHASE_MAX_AGE_SECS", + default_value_t = bigname_storage::DEFAULT_WORKER_REBUILD_PHASE_MAX_AGE_SECS + )] + pub(crate) worker_rebuild_phase_max_age_secs: i64, + #[arg( + long, + env = "BIGNAME_API_STATUS_PROVIDER_TIMEOUT_MS", + default_value_t = crate::status_freshness::DEFAULT_PROVIDER_TIMEOUT_MS + )] + pub(crate) status_provider_timeout_ms: u64, + #[arg( + long, + env = "BIGNAME_API_STATUS_PROVIDER_REFRESH_SECS", + default_value_t = crate::status_freshness::DEFAULT_PROVIDER_REFRESH_SECS + )] + pub(crate) status_provider_refresh_secs: u64, + #[arg( + long, + env = "BIGNAME_API_STATUS_PROVIDER_CACHE_TTL_SECS", + default_value_t = crate::status_freshness::DEFAULT_PROVIDER_CACHE_TTL_SECS + )] + pub(crate) status_provider_cache_ttl_secs: u64, + #[arg( + long, + env = "BIGNAME_API_STATUS_MAX_BLOCK_LAG", + default_value_t = crate::status_freshness::DEFAULT_MAX_BLOCK_LAG + )] + pub(crate) status_max_block_lag: i64, + #[arg( + long, + env = "BIGNAME_API_STATUS_MAX_LAG_SECS", + default_value_t = crate::status_freshness::DEFAULT_MAX_LAG_SECS + )] + pub(crate) status_max_lag_secs: i64, #[command(flatten)] pub(crate) database: DatabaseConfig, } diff --git a/apps/api/src/handlers/app_facing/identity/mod.rs b/apps/api/src/handlers/app_facing/identity/mod.rs index 202c172c..c6a5cc28 100644 --- a/apps/api/src/handlers/app_facing/identity/mod.rs +++ b/apps/api/src/handlers/app_facing/identity/mod.rs @@ -24,5 +24,5 @@ async fn load_indexing_status_response(state: &AppState) -> ApiResult, axum::Extension(health_pool): axum::Extension, ) -> (StatusCode, Json) { let database_reachable = match tokio::time::timeout( @@ -36,28 +37,85 @@ pub(super) async fn health( } }; - let (http_status, status, database) = match database_reachable { - true => ( - StatusCode::OK, - "ready", - HealthDatabaseResponse { + let (database, loops, loops_ready) = match database_reachable { + true => { + let database = HealthDatabaseResponse { status: "reachable", reachable: true, check: "select_1", error: None, - }, - ), - false => ( - StatusCode::SERVICE_UNAVAILABLE, - "degraded", - HealthDatabaseResponse { + }; + match bigname_storage::load_preferred_service_loop_heartbeats( + &health_pool.0, + &[ + bigname_storage::INDEXER_SERVICE_NAME, + bigname_storage::WORKER_SERVICE_NAME, + ], + state.heartbeat_max_age_secs, + state.worker_rebuild_phase_max_age_secs, + ) + .await + { + Ok(heartbeats) => { + let indexer = loop_health_response( + heartbeats.iter().find(|heartbeat| { + heartbeat.service_name == bigname_storage::INDEXER_SERVICE_NAME + }), + state.heartbeat_max_age_secs, + state.heartbeat_max_age_secs, + ); + let worker = loop_health_response( + heartbeats.iter().find(|heartbeat| { + heartbeat.service_name == bigname_storage::WORKER_SERVICE_NAME + }), + state.heartbeat_max_age_secs, + state.worker_rebuild_phase_max_age_secs, + ); + let loops_ready = indexer.status == "running" && worker.status == "running"; + (database, HealthLoopsResponse { indexer, worker }, loops_ready) + } + Err(readiness_error) => { + warn!( + service = "api", + build_sha = BUILD_SHA, + error = ?readiness_error, + "service loop heartbeat readiness probe failed" + ); + ( + database, + unavailable_loop_health(state.heartbeat_max_age_secs), + false, + ) + } + } + } + false => { + let database = HealthDatabaseResponse { status: "unreachable", reachable: false, check: "select_1", error: Some("database readiness query failed"), - }, - ), + }; + ( + database, + unavailable_loop_health(state.heartbeat_max_age_secs), + false, + ) + } }; + let api_ready = database.reachable; + let aggregate_ready = api_ready && loops_ready; + let http_status = if api_ready { + StatusCode::OK + } else { + StatusCode::SERVICE_UNAVAILABLE + }; + let status = if aggregate_ready { + "ready" + } else { + "degraded" + }; + let api_status = if api_ready { "ready" } else { "degraded" }; ( http_status, @@ -65,8 +123,68 @@ pub(super) async fn health( service: "api", identity: HealthIdentityResponse::current(), status, + api_status, process: HealthProcessResponse { status: "running" }, database, + loops, }), ) } + +fn loop_health_response( + heartbeat: Option<&bigname_storage::ServiceLoopHeartbeat>, + max_age_seconds: i64, + phase_max_age_seconds: i64, +) -> HealthLoopResponse { + let Some(heartbeat) = heartbeat else { + return HealthLoopResponse { + status: "not_started", + phase: None, + started_at: None, + heartbeat_at: None, + heartbeat_age_seconds: None, + max_age_seconds, + }; + }; + if let Some(phase) = heartbeat.active_phase.as_ref() { + return HealthLoopResponse { + status: if phase.age_seconds <= phase_max_age_seconds { + "running" + } else { + "stale" + }, + phase: Some(phase.phase.clone()), + started_at: Some(format_timestamp(phase.started_at)), + heartbeat_at: Some(format_timestamp(phase.heartbeat_at)), + heartbeat_age_seconds: Some(phase.age_seconds), + max_age_seconds: phase_max_age_seconds, + }; + } + HealthLoopResponse { + status: if heartbeat.age_seconds <= max_age_seconds { + "running" + } else { + "stale" + }, + phase: None, + started_at: Some(format_timestamp(heartbeat.started_at)), + heartbeat_at: Some(format_timestamp(heartbeat.heartbeat_at)), + heartbeat_age_seconds: Some(heartbeat.age_seconds), + max_age_seconds, + } +} + +fn unavailable_loop_health(max_age_seconds: i64) -> HealthLoopsResponse { + let unavailable = || HealthLoopResponse { + status: "unavailable", + phase: None, + started_at: None, + heartbeat_at: None, + heartbeat_age_seconds: None, + max_age_seconds, + }; + HealthLoopsResponse { + indexer: unavailable(), + worker: unavailable(), + } +} diff --git a/apps/api/src/main.rs b/apps/api/src/main.rs index 00bfa4e0..319dd4d7 100644 --- a/apps/api/src/main.rs +++ b/apps/api/src/main.rs @@ -46,6 +46,7 @@ mod pagination; mod query; mod routes; mod state; +mod status_freshness; mod types; mod v2; @@ -87,7 +88,7 @@ async fn main() -> Result<()> { match Cli::parse().command { Command::Serve(args) => { init_tracing("bigname-api"); - serve(args).await + serve(*args).await } Command::PrintOpenapi => { print!("{}", render_openapi_document()); diff --git a/apps/api/src/openapi/schemas.rs b/apps/api/src/openapi/schemas.rs index 01e406d2..a3822587 100644 --- a/apps/api/src/openapi/schemas.rs +++ b/apps/api/src/openapi/schemas.rs @@ -25,8 +25,8 @@ use identity_native::{ native_identity_record_schema, normalization_info_schema, }; use health::{ - health_database_schema, health_identity_schema, health_process_schema, - health_projection_publication_versions_schema, health_response_schema, + health_database_schema, health_identity_schema, health_loop_schema, health_loops_schema, + health_process_schema, health_projection_publication_versions_schema, health_response_schema, }; use primary_name::{ primary_name_claimed_result_schema, primary_name_route_provenance_schema, @@ -356,6 +356,8 @@ pub(super) fn openapi_components() -> JsonValue { "HealthIdentity": health_identity_schema(), "HealthProcess": health_process_schema(), "HealthDatabase": health_database_schema(), + "HealthLoop": health_loop_schema(), + "HealthLoops": health_loops_schema(), "HealthResponse": health_response_schema(), "ErrorBody": json!({ "type": "object", diff --git a/apps/api/src/openapi/schemas/health.rs b/apps/api/src/openapi/schemas/health.rs index 8ce1ae20..b0fe913e 100644 --- a/apps/api/src/openapi/schemas/health.rs +++ b/apps/api/src/openapi/schemas/health.rs @@ -73,16 +73,65 @@ pub(super) fn health_database_schema() -> JsonValue { }) } +pub(super) fn health_loop_schema() -> JsonValue { + json!({ + "type": "object", + "required": [ + "status", + "phase", + "started_at", + "heartbeat_at", + "heartbeat_age_seconds", + "max_age_seconds", + ], + "properties": { + "status": { + "type": "string", + "enum": ["running", "stale", "not_started", "unavailable"], + }, + "phase": { + "type": ["string", "null"], + "description": "Named worker rebuild phase when the loop is inside a monolithic operation governed by its separately configurable maximum age; null for normal loop progress.", + }, + "started_at": { "type": ["string", "null"], "format": "date-time" }, + "heartbeat_at": { "type": ["string", "null"], "format": "date-time" }, + "heartbeat_age_seconds": { "type": ["integer", "null"], "minimum": 0 }, + "max_age_seconds": { "type": "integer", "minimum": 1 }, + }, + }) +} + +pub(super) fn health_loops_schema() -> JsonValue { + json!({ + "type": "object", + "required": ["indexer", "worker"], + "properties": { + "indexer": schema_ref("HealthLoop"), + "worker": schema_ref("HealthLoop"), + }, + }) +} + pub(super) fn health_response_schema() -> JsonValue { json!({ "type": "object", - "required": ["service", "identity", "status", "process", "database"], + "required": ["service", "identity", "status", "api_status", "process", "database", "loops"], "properties": { "service": { "type": "string" }, "identity": schema_ref("HealthIdentity"), - "status": { "type": "string" }, + "status": { + "type": "string", + "enum": ["ready", "degraded"], + "description": "Aggregate database, indexer-loop, and worker-loop health for status consumers.", + }, + "api_status": { + "type": "string", + "enum": ["ready", "degraded"], + "description": "API-local readiness: ready when this process is serving and its database reachability query succeeds; independent of indexer and worker loop state.", + }, "process": schema_ref("HealthProcess"), "database": schema_ref("HealthDatabase"), + "loops": schema_ref("HealthLoops"), }, }) } diff --git a/apps/api/src/openapi/schemas/identity.rs b/apps/api/src/openapi/schemas/identity.rs index 8d7c3fa5..efea14e2 100644 --- a/apps/api/src/openapi/schemas/identity.rs +++ b/apps/api/src/openapi/schemas/identity.rs @@ -20,12 +20,29 @@ pub(super) fn name_record_status_schema() -> JsonValue { pub(super) fn indexing_status_response_schema() -> JsonValue { json!({ "type": "object", - "required": ["status", "chains"], + "required": [ + "status", + "pending_invalidation_count", + "pending_invalidation_count_capped", + "dead_letter_count", + "chains", + ], "properties": { "status": { "type": "string", "enum": ["ready", "degraded", "stale"], }, + "pending_invalidation_count": { + "type": "integer", + "minimum": 0, + "maximum": bigname_storage::PENDING_INVALIDATION_COUNT_CAP, + "description": "Exact live queue row count when pending_invalidation_count_capped is false. When capped is true, this field equals the cap and the queue contains at least one additional row.", + }, + "pending_invalidation_count_capped": { + "type": "boolean", + "description": "True when the bounded status query stopped after observing more rows than pending_invalidation_count reports.", + }, + "dead_letter_count": { "type": "integer", "minimum": 0 }, "chains": { "type": "object", "additionalProperties": { @@ -38,6 +55,12 @@ pub(super) fn indexing_status_response_schema() -> JsonValue { "latest_projected_timestamp", "projection_lag_blocks", "projection_lag_seconds", + "network_block", + "network_head_observed_at", + "network_head_age_seconds", + "network_head_status", + "ingestion_lag_blocks", + "ingestion_lag_seconds", ], "properties": { "canonical_block": { "type": ["integer", "null"] }, @@ -50,6 +73,33 @@ pub(super) fn indexing_status_response_schema() -> JsonValue { }, "projection_lag_blocks": { "type": ["integer", "null"] }, "projection_lag_seconds": { "type": ["integer", "null"] }, + "network_block": { "type": ["integer", "null"], "minimum": 0 }, + "network_head_observed_at": { + "type": ["string", "null"], + "format": "date-time", + }, + "network_head_age_seconds": { + "type": ["integer", "null"], + "minimum": 0, + }, + "network_head_status": { + "type": "string", + "enum": [ + "fresh", + "stale", + "unavailable", + "pending", + "unconfigured", + ], + }, + "ingestion_lag_blocks": { + "type": ["integer", "null"], + "minimum": 0, + }, + "ingestion_lag_seconds": { + "type": ["integer", "null"], + "minimum": 0, + }, }, }, }, diff --git a/apps/api/src/openapi/server.rs b/apps/api/src/openapi/server.rs index e8052e63..7d1cbe2b 100644 --- a/apps/api/src/openapi/server.rs +++ b/apps/api/src/openapi/server.rs @@ -1,13 +1,14 @@ -use anyhow::{Context, Result}; +use anyhow::{Context, Result, ensure}; use axum::{Json, response::Html, routing::get}; use serde_json::{Map as JsonMap, json}; use sqlx::types::JsonValue; use tower_http::cors::CorsLayer; -use tracing::info; +use tracing::{info, warn}; use crate::{ API_ROUTE_DEFINITIONS, ApiBoundsConfig, AppState, BUILD_SHA, Router, SOFTWARE_VERSION, ServeArgs, HEALTH_DATABASE_CHECK_TIMEOUT, HealthDatabasePool, shutdown_signal, + status_freshness::{StatusFreshnessConfig, missing_status_rpc_chains}, warm_compact_records_route_sql_path, }; @@ -30,10 +31,41 @@ pub(crate) async fn serve(args: ServeArgs) -> Result<()> { HEALTH_DATABASE_CHECK_TIMEOUT, ) .await?; - let state = AppState { - pool, - chain_rpc_urls, - }; + let expected_status_chain_ids = + bigname_storage::load_expected_status_chain_ids(&pool).await?; + let missing_status_rpc_chains = + missing_status_rpc_chains(&expected_status_chain_ids, &chain_rpc_urls); + if !missing_status_rpc_chains.is_empty() { + warn!( + service = "api", + configuration = "BIGNAME_API_CHAIN_RPC_URLS", + missing_chain_ids = ?missing_status_rpc_chains, + expected_chain_ids = ?expected_status_chain_ids, + "status network-head RPC configuration is incomplete; indexing status remains fail-closed for the named chains" + ); + } + ensure!( + args.heartbeat_max_age_secs > 0, + "BIGNAME_API_HEARTBEAT_MAX_AGE_SECS must be greater than zero" + ); + ensure!( + args.worker_rebuild_phase_max_age_secs > 0, + "BIGNAME_API_WORKER_REBUILD_PHASE_MAX_AGE_SECS must be greater than zero" + ); + let status_freshness_config = StatusFreshnessConfig::new( + args.status_provider_timeout_ms, + args.status_provider_refresh_secs, + args.status_provider_cache_ttl_secs, + args.status_max_block_lag, + args.status_max_lag_secs, + )?; + let state = AppState::new(pool, chain_rpc_urls) + .with_heartbeat_max_age_secs(args.heartbeat_max_age_secs) + .with_worker_rebuild_phase_max_age_secs(args.worker_rebuild_phase_max_age_secs) + .with_status_freshness_config(status_freshness_config); + state + .status_freshness + .spawn_refresh(state.chain_rpc_urls.clone()); warm_compact_records_route_sql_path(&state, args.database.max_connections) .await .context("failed to warm compact records route SQL path")?; diff --git a/apps/api/src/responses/app_facing/identity.rs b/apps/api/src/responses/app_facing/identity.rs index 5849b8f2..a3cc8dce 100644 --- a/apps/api/src/responses/app_facing/identity.rs +++ b/apps/api/src/responses/app_facing/identity.rs @@ -77,38 +77,74 @@ fn build_name_record_response_for_coin_type( } } -pub(crate) fn build_indexing_status_response( +pub(crate) async fn build_indexing_status_response( read: &bigname_storage::IndexingStatusRead, + state: &AppState, ) -> IndexingStatusResponse { - let chains = read - .chains - .iter() - .map(|row| { - let projection_lag_blocks = row - .canonical_block - .zip(row.latest_projected_block) - .map(|(canonical, projected)| canonical.saturating_sub(projected)); - let projection_lag_seconds = row - .canonical_timestamp - .zip(row.latest_projected_timestamp) - .map(|(canonical, projected)| (canonical - projected).whole_seconds().max(0)); - ( - row.chain_id.clone(), - IndexingStatusChainResponse { - canonical_block: row.canonical_block, - safe_block: row.safe_block, - finalized_block: row.finalized_block, - latest_projected_block: row.latest_projected_block, - latest_projected_timestamp: row.latest_projected_timestamp.map(format_timestamp), - projection_lag_blocks, - projection_lag_seconds, - }, + let mut chains = BTreeMap::new(); + let mut saw_stale = false; + let mut saw_degraded = read.has_unscoped_pending_invalidations; + for row in &read.chains { + let projection_lag_blocks = row + .canonical_block + .zip(row.latest_projected_block) + .map(|(canonical, projected)| canonical.saturating_sub(projected)); + let projection_lag_seconds = row + .canonical_timestamp + .zip(row.latest_projected_timestamp) + .map(|(canonical, projected)| (canonical - projected).whole_seconds().max(0)); + let network_head = state + .status_freshness + .compare( + &state.chain_rpc_urls, + &row.chain_id, + row.canonical_block, + row.canonical_timestamp, ) - }) - .collect::>(); + .await; + match crate::status_freshness::status_readiness( + row.canonical_block, + row.latest_projected_block, + projection_lag_blocks, + &network_head, + ) { + crate::status_freshness::StatusReadiness::Ready => {} + crate::status_freshness::StatusReadiness::Degraded => saw_degraded = true, + crate::status_freshness::StatusReadiness::Stale => saw_stale = true, + } + chains.insert( + row.chain_id.clone(), + IndexingStatusChainResponse { + canonical_block: row.canonical_block, + safe_block: row.safe_block, + finalized_block: row.finalized_block, + latest_projected_block: row.latest_projected_block, + latest_projected_timestamp: row.latest_projected_timestamp.map(format_timestamp), + projection_lag_blocks, + projection_lag_seconds, + network_block: network_head.block, + network_head_observed_at: network_head.observed_at.map(format_timestamp), + network_head_age_seconds: network_head.age_seconds, + network_head_status: network_head.status.as_str().to_owned(), + ingestion_lag_blocks: network_head.ingestion_lag_blocks, + ingestion_lag_seconds: network_head.ingestion_lag_seconds, + }, + ); + } + + let status = if saw_stale { + "stale" + } else if saw_degraded || chains.is_empty() { + "degraded" + } else { + "ready" + }; IndexingStatusResponse { - status: indexing_status_label(chains.values(), read.has_unscoped_pending_invalidations), + status: status.to_owned(), + pending_invalidation_count: read.pending_invalidation_count, + pending_invalidation_count_capped: read.pending_invalidation_count_capped, + dead_letter_count: read.dead_letter_count, chains, } } @@ -242,27 +278,3 @@ fn identity_as_of_timestamp(record: &bigname_storage::IdentityNameRecordRow) -> .map(format_timestamp) .unwrap_or_else(|| format_timestamp(OffsetDateTime::now_utc())) } - -fn indexing_status_label<'a>( - chains: impl Iterator, - has_unscoped_pending_invalidations: bool, -) -> String { - let mut saw_degraded = has_unscoped_pending_invalidations; - let mut saw_chain = false; - for chain in chains { - saw_chain = true; - if chain.canonical_block.is_none() || chain.latest_projected_block.is_none() { - saw_degraded = true; - continue; - } - if chain.projection_lag_blocks.unwrap_or_default() > 0 { - return "stale".to_owned(); - } - } - - if saw_degraded || !saw_chain { - "degraded".to_owned() - } else { - "ready".to_owned() - } -} diff --git a/apps/api/src/state.rs b/apps/api/src/state.rs index 4af45cd6..ce0bdc39 100644 --- a/apps/api/src/state.rs +++ b/apps/api/src/state.rs @@ -1,8 +1,47 @@ use bigname_execution::ChainRpcUrls; use sqlx::PgPool; +use crate::status_freshness::{StatusFreshness, StatusFreshnessConfig}; + #[derive(Clone)] pub(crate) struct AppState { pub(crate) pool: PgPool, pub(crate) chain_rpc_urls: ChainRpcUrls, + pub(crate) heartbeat_max_age_secs: i64, + pub(crate) worker_rebuild_phase_max_age_secs: i64, + pub(crate) status_freshness: StatusFreshness, +} + +impl AppState { + pub(crate) fn new(pool: PgPool, chain_rpc_urls: ChainRpcUrls) -> Self { + Self { + pool, + chain_rpc_urls, + heartbeat_max_age_secs: 20, + worker_rebuild_phase_max_age_secs: + bigname_storage::DEFAULT_WORKER_REBUILD_PHASE_MAX_AGE_SECS, + status_freshness: StatusFreshness::new(StatusFreshnessConfig::default()), + } + } + + pub(crate) fn with_worker_rebuild_phase_max_age_secs( + mut self, + worker_rebuild_phase_max_age_secs: i64, + ) -> Self { + self.worker_rebuild_phase_max_age_secs = worker_rebuild_phase_max_age_secs; + self + } + + pub(crate) fn with_heartbeat_max_age_secs(mut self, heartbeat_max_age_secs: i64) -> Self { + self.heartbeat_max_age_secs = heartbeat_max_age_secs; + self + } + + pub(crate) fn with_status_freshness_config( + mut self, + status_freshness_config: StatusFreshnessConfig, + ) -> Self { + self.status_freshness = StatusFreshness::new(status_freshness_config); + self + } } diff --git a/apps/api/src/status_freshness.rs b/apps/api/src/status_freshness.rs new file mode 100644 index 00000000..c64ec959 --- /dev/null +++ b/apps/api/src/status_freshness.rs @@ -0,0 +1,355 @@ +use std::{collections::BTreeMap, sync::Arc, time::Duration}; + +use anyhow::{Result, ensure}; +use bigname_execution::ChainRpcUrls; +use sqlx::types::time::OffsetDateTime; +use tokio::{sync::RwLock, task::JoinSet, time::MissedTickBehavior}; +use tracing::warn; + +pub(crate) const DEFAULT_PROVIDER_TIMEOUT_MS: u64 = 750; +pub(crate) const DEFAULT_PROVIDER_REFRESH_SECS: u64 = 5; +pub(crate) const DEFAULT_PROVIDER_CACHE_TTL_SECS: u64 = 30; +pub(crate) const DEFAULT_MAX_BLOCK_LAG: i64 = 30; +pub(crate) const DEFAULT_MAX_LAG_SECS: i64 = 60; + +#[derive(Clone, Debug)] +pub(crate) struct StatusFreshnessConfig { + provider_timeout: Duration, + provider_refresh: Duration, + provider_cache_ttl: Duration, + max_block_lag: i64, + max_lag_seconds: i64, +} + +impl StatusFreshnessConfig { + pub(crate) fn new( + provider_timeout_ms: u64, + provider_refresh_secs: u64, + provider_cache_ttl_secs: u64, + max_block_lag: i64, + max_lag_seconds: i64, + ) -> Result { + ensure!( + provider_timeout_ms > 0, + "status provider timeout must be positive" + ); + ensure!( + provider_refresh_secs > 0, + "status provider refresh must be positive" + ); + ensure!( + provider_cache_ttl_secs > 0, + "status provider cache TTL must be positive" + ); + ensure!( + max_block_lag >= 0, + "status maximum block lag must not be negative" + ); + ensure!( + max_lag_seconds >= 0, + "status maximum seconds lag must not be negative" + ); + + Ok(Self { + provider_timeout: Duration::from_millis(provider_timeout_ms), + provider_refresh: Duration::from_secs(provider_refresh_secs), + provider_cache_ttl: Duration::from_secs(provider_cache_ttl_secs), + max_block_lag, + max_lag_seconds, + }) + } +} + +pub(crate) fn missing_status_rpc_chains( + expected_chain_ids: &[String], + chain_rpc_urls: &ChainRpcUrls, +) -> Vec { + expected_chain_ids + .iter() + .filter(|chain_id| chain_rpc_urls.url_for(chain_id).is_none()) + .cloned() + .collect() +} + +impl Default for StatusFreshnessConfig { + fn default() -> Self { + Self::new( + DEFAULT_PROVIDER_TIMEOUT_MS, + DEFAULT_PROVIDER_REFRESH_SECS, + DEFAULT_PROVIDER_CACHE_TTL_SECS, + DEFAULT_MAX_BLOCK_LAG, + DEFAULT_MAX_LAG_SECS, + ) + .expect("default status freshness configuration must be valid") + } +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub(crate) enum NetworkHeadStatus { + Fresh, + Stale, + Unavailable, + Pending, + Unconfigured, +} + +impl NetworkHeadStatus { + pub(crate) const fn as_str(self) -> &'static str { + match self { + Self::Fresh => "fresh", + Self::Stale => "stale", + Self::Unavailable => "unavailable", + Self::Pending => "pending", + Self::Unconfigured => "unconfigured", + } + } +} + +#[derive(Clone, Debug, Eq, PartialEq)] +pub(crate) struct NetworkHeadComparison { + pub(crate) status: NetworkHeadStatus, + pub(crate) block: Option, + pub(crate) observed_at: Option, + pub(crate) age_seconds: Option, + pub(crate) ingestion_lag_blocks: Option, + pub(crate) ingestion_lag_seconds: Option, + pub(crate) data_is_stale: bool, +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub(crate) enum StatusReadiness { + Ready, + Degraded, + Stale, +} + +pub(crate) fn status_readiness( + canonical_block: Option, + latest_projected_block: Option, + projection_lag_blocks: Option, + network_head: &NetworkHeadComparison, +) -> StatusReadiness { + if canonical_block.is_none() || latest_projected_block.is_none() { + return StatusReadiness::Degraded; + } + if projection_lag_blocks.is_some_and(|lag| lag > 0) || network_head.data_is_stale { + return StatusReadiness::Stale; + } + if network_head.status != NetworkHeadStatus::Fresh { + return StatusReadiness::Degraded; + } + StatusReadiness::Ready +} + +#[derive(Clone, Debug)] +struct SuccessfulNetworkHead { + block: i64, + observed_at: OffsetDateTime, +} + +#[derive(Clone, Debug, Default)] +struct CachedNetworkHead { + attempted: bool, + latest_attempt_failed: bool, + successful: Option, +} + +#[derive(Clone, Debug)] +pub(crate) struct StatusFreshness { + config: StatusFreshnessConfig, + cache: Arc>>, +} + +impl StatusFreshness { + pub(crate) fn new(config: StatusFreshnessConfig) -> Self { + Self { + config, + cache: Arc::new(RwLock::new(BTreeMap::new())), + } + } + + pub(crate) fn spawn_refresh(&self, chain_rpc_urls: ChainRpcUrls) { + let freshness = self.clone(); + tokio::spawn(async move { + let mut interval = tokio::time::interval(freshness.config.provider_refresh); + interval.set_missed_tick_behavior(MissedTickBehavior::Skip); + loop { + interval.tick().await; + freshness.refresh_once(&chain_rpc_urls).await; + } + }); + } + + async fn refresh_once(&self, chain_rpc_urls: &ChainRpcUrls) { + let mut probes = JoinSet::new(); + for (chain_id, endpoint) in chain_rpc_urls.iter() { + let chain_id = chain_id.to_owned(); + let endpoint = endpoint.to_owned(); + let timeout = self.config.provider_timeout; + probes.spawn(async move { + let result = tokio::time::timeout( + timeout, + bigname_execution::fetch_network_head_block_number(&endpoint), + ) + .await + .map_err(|_| ()) + .and_then(|result| result.map_err(|_| ())); + (chain_id, result) + }); + } + + while let Some(probe) = probes.join_next().await { + let Ok((chain_id, result)) = probe else { + warn!( + service = "api", + probe = "network_head", + "network-head refresh task failed" + ); + continue; + }; + let mut cache = self.cache.write().await; + let entry = cache.entry(chain_id.clone()).or_default(); + entry.attempted = true; + match result { + Ok(block) => { + entry.latest_attempt_failed = false; + entry.successful = Some(SuccessfulNetworkHead { + block, + observed_at: OffsetDateTime::now_utc(), + }); + } + Err(()) => { + entry.latest_attempt_failed = true; + warn!( + service = "api", + chain_id, + probe = "network_head", + "network-head refresh failed or timed out" + ); + } + } + } + } + + pub(crate) async fn compare( + &self, + chain_rpc_urls: &ChainRpcUrls, + chain_id: &str, + canonical_block: Option, + canonical_timestamp: Option, + ) -> NetworkHeadComparison { + if chain_rpc_urls.url_for(chain_id).is_none() { + return empty_comparison(NetworkHeadStatus::Unconfigured); + } + + let cached = self.cache.read().await.get(chain_id).cloned(); + let Some(cached) = cached else { + return empty_comparison(NetworkHeadStatus::Pending); + }; + let Some(successful) = cached.successful else { + return empty_comparison(if cached.attempted { + NetworkHeadStatus::Unavailable + } else { + NetworkHeadStatus::Pending + }); + }; + + let mut comparison = + self.compare_successful(successful, canonical_block, canonical_timestamp); + if cached.latest_attempt_failed { + comparison.status = NetworkHeadStatus::Unavailable; + comparison.data_is_stale = false; + } + comparison + } + + fn compare_successful( + &self, + successful: SuccessfulNetworkHead, + canonical_block: Option, + canonical_timestamp: Option, + ) -> NetworkHeadComparison { + let age_seconds = (OffsetDateTime::now_utc() - successful.observed_at) + .whole_seconds() + .max(0); + let status = if age_seconds + > i64::try_from(self.config.provider_cache_ttl.as_secs()).unwrap_or(i64::MAX) + { + NetworkHeadStatus::Stale + } else { + NetworkHeadStatus::Fresh + }; + let ingestion_lag_blocks = + canonical_block.map(|canonical| successful.block.saturating_sub(canonical).max(0)); + let ingestion_lag_seconds = + canonical_block + .zip(canonical_timestamp) + .map(|(canonical, canonical_timestamp)| { + if successful.block <= canonical { + 0 + } else { + (successful.observed_at - canonical_timestamp) + .whole_seconds() + .max(0) + } + }); + let data_is_stale = status == NetworkHeadStatus::Fresh + && (ingestion_lag_blocks.is_some_and(|lag| lag > self.config.max_block_lag) + || ingestion_lag_seconds.is_some_and(|lag| lag > self.config.max_lag_seconds)); + + NetworkHeadComparison { + status, + block: Some(successful.block), + observed_at: Some(successful.observed_at), + age_seconds: Some(age_seconds), + ingestion_lag_blocks, + ingestion_lag_seconds, + data_is_stale, + } + } + + #[cfg(test)] + pub(crate) async fn seed_success( + &self, + chain_id: &str, + block: i64, + observed_at: OffsetDateTime, + ) { + self.cache.write().await.insert( + chain_id.to_owned(), + CachedNetworkHead { + attempted: true, + latest_attempt_failed: false, + successful: Some(SuccessfulNetworkHead { block, observed_at }), + }, + ); + } + + #[cfg(test)] + pub(crate) async fn seed_unavailable(&self, chain_id: &str) { + self.cache.write().await.insert( + chain_id.to_owned(), + CachedNetworkHead { + attempted: true, + latest_attempt_failed: true, + successful: None, + }, + ); + } +} + +fn empty_comparison(status: NetworkHeadStatus) -> NetworkHeadComparison { + NetworkHeadComparison { + status, + block: None, + observed_at: None, + age_seconds: None, + ingestion_lag_blocks: None, + ingestion_lag_seconds: None, + data_is_stale: false, + } +} + +#[cfg(test)] +#[path = "status_freshness/tests.rs"] +mod tests; diff --git a/apps/api/src/status_freshness/tests.rs b/apps/api/src/status_freshness/tests.rs new file mode 100644 index 00000000..0fd3ef1d --- /dev/null +++ b/apps/api/src/status_freshness/tests.rs @@ -0,0 +1,236 @@ +use tokio::net::TcpListener; + +use super::*; + +#[test] +fn default_block_lag_tolerates_fast_chain_poll_and_provider_skew() { + assert_eq!(StatusFreshnessConfig::default().max_block_lag, 30); +} + +#[test] +fn missing_status_rpc_chains_names_every_expected_unconfigured_chain() -> Result<()> { + let urls = ChainRpcUrls::from_entries(&[ + "ethereum-mainnet=http://rpc.test".to_owned(), + "base-mainnet=http://base.test".to_owned(), + ])?; + + assert_eq!( + missing_status_rpc_chains( + &[ + "base-mainnet".to_owned(), + "ethereum-mainnet".to_owned(), + "ethereum-sepolia".to_owned(), + ], + &urls, + ), + vec!["ethereum-sepolia".to_owned()] + ); + Ok(()) +} + +#[tokio::test] +async fn comparison_distinguishes_probe_states_and_applies_both_lag_thresholds() -> Result<()> { + let urls = ChainRpcUrls::from_entries(&["ethereum-mainnet=http://rpc.test".to_owned()])?; + let freshness = StatusFreshness::new(StatusFreshnessConfig::new(50, 5, 30, 5, 60)?); + assert_eq!( + freshness + .compare(&urls, "ethereum-mainnet", Some(100), None) + .await + .status, + NetworkHeadStatus::Pending + ); + + freshness.seed_unavailable("ethereum-mainnet").await; + assert_eq!( + freshness + .compare(&urls, "ethereum-mainnet", Some(100), None) + .await + .status, + NetworkHeadStatus::Unavailable + ); + + let observed_at = OffsetDateTime::now_utc(); + freshness + .seed_success("ethereum-mainnet", 106, observed_at) + .await; + let block_lag = freshness + .compare(&urls, "ethereum-mainnet", Some(100), Some(observed_at)) + .await; + assert_eq!(block_lag.status, NetworkHeadStatus::Fresh); + assert_eq!(block_lag.ingestion_lag_blocks, Some(6)); + assert_eq!(block_lag.ingestion_lag_seconds, Some(0)); + assert!(block_lag.data_is_stale); + assert_eq!( + status_readiness(Some(100), Some(100), Some(0), &block_lag), + StatusReadiness::Stale + ); + + freshness + .seed_success("ethereum-mainnet", 101, observed_at) + .await; + let time_lag = freshness + .compare( + &urls, + "ethereum-mainnet", + Some(100), + Some(observed_at - Duration::from_secs(61)), + ) + .await; + assert_eq!(time_lag.ingestion_lag_blocks, Some(1)); + assert_eq!(time_lag.ingestion_lag_seconds, Some(61)); + assert!(time_lag.data_is_stale); + + freshness.seed_unavailable("ethereum-mainnet").await; + let unavailable = freshness + .compare(&urls, "ethereum-mainnet", Some(100), Some(observed_at)) + .await; + assert_eq!( + status_readiness(Some(100), Some(100), Some(0), &unavailable), + StatusReadiness::Degraded + ); + + let unconfigured = freshness + .compare( + &ChainRpcUrls::default(), + "ethereum-mainnet", + Some(100), + None, + ) + .await; + assert_eq!(unconfigured.status, NetworkHeadStatus::Unconfigured); + assert_eq!( + status_readiness(Some(100), Some(100), Some(0), &unconfigured), + StatusReadiness::Degraded + ); + assert_eq!( + status_readiness(Some(100), Some(99), Some(1), &unconfigured), + StatusReadiness::Stale, + "known projection lag must not be masked by an unconfigured provider" + ); + Ok(()) +} + +#[tokio::test] +async fn slow_provider_refresh_is_bounded_and_never_blocks_cached_comparison() -> Result<()> { + let listener = TcpListener::bind("127.0.0.1:0").await?; + let endpoint = format!("http://{}", listener.local_addr()?); + let server = tokio::spawn(async move { + let (_stream, _) = listener.accept().await.expect("slow probe must connect"); + std::future::pending::<()>().await; + }); + let urls = ChainRpcUrls::from_entries(&[format!("ethereum-mainnet={endpoint}")])?; + let freshness = StatusFreshness::new(StatusFreshnessConfig::new(500, 5, 30, 5, 60)?); + let refresh = { + let freshness = freshness.clone(); + let urls = urls.clone(); + tokio::spawn(async move { freshness.refresh_once(&urls).await }) + }; + tokio::task::yield_now().await; + + let comparison = tokio::time::timeout( + Duration::from_millis(100), + freshness.compare(&urls, "ethereum-mainnet", Some(100), None), + ) + .await + .expect("cached status comparison must not wait for the provider probe"); + assert_eq!(comparison.status, NetworkHeadStatus::Pending); + + tokio::time::timeout(Duration::from_millis(750), refresh) + .await + .expect("provider timeout must bound the asynchronous refresh") + .expect("refresh task must not panic"); + assert_eq!( + freshness + .compare(&urls, "ethereum-mainnet", Some(100), None) + .await + .status, + NetworkHeadStatus::Unavailable + ); + server.abort(); + Ok(()) +} + +#[tokio::test] +async fn successful_refresh_caches_eth_block_number_and_cache_age_is_explicit() -> Result<()> { + use tokio::io::{AsyncReadExt, AsyncWriteExt}; + + let listener = TcpListener::bind("127.0.0.1:0").await?; + let endpoint = format!("http://{}", listener.local_addr()?); + let server = tokio::spawn(async move { + let (mut stream, _) = listener.accept().await.expect("probe must connect"); + let mut request = vec![0_u8; 4096]; + let count = stream.read(&mut request).await.expect("request must read"); + let request = String::from_utf8_lossy(&request[..count]); + assert!(request.contains("eth_blockNumber")); + let body = r#"{"jsonrpc":"2.0","id":1,"result":"0x2a"}"#; + let response = format!( + "HTTP/1.1 200 OK\r\ncontent-type: application/json\r\nconnection: close\r\ncontent-length: {}\r\n\r\n{}", + body.len(), + body + ); + stream + .write_all(response.as_bytes()) + .await + .expect("response must write"); + }); + let urls = ChainRpcUrls::from_entries(&[format!("ethereum-mainnet={endpoint}")])?; + let freshness = StatusFreshness::new(StatusFreshnessConfig::new(200, 5, 30, 5, 60)?); + + freshness.refresh_once(&urls).await; + server.await.expect("provider task must not panic"); + let comparison = freshness + .compare(&urls, "ethereum-mainnet", Some(42), None) + .await; + assert_eq!(comparison.status, NetworkHeadStatus::Fresh); + assert_eq!(comparison.block, Some(42)); + assert_eq!(comparison.ingestion_lag_blocks, Some(0)); + assert!(comparison.observed_at.is_some()); + assert!(comparison.age_seconds.is_some_and(|age| age <= 1)); + + freshness + .seed_success( + "ethereum-mainnet", + 42, + OffsetDateTime::now_utc() - Duration::from_secs(31), + ) + .await; + let stale_cache = freshness + .compare(&urls, "ethereum-mainnet", Some(42), None) + .await; + assert_eq!(stale_cache.status, NetworkHeadStatus::Stale); + assert_eq!( + status_readiness(Some(42), Some(42), Some(0), &stale_cache), + StatusReadiness::Degraded + ); + Ok(()) +} + +#[tokio::test] +async fn failed_refresh_degrades_but_retains_the_last_successful_head_as_evidence() -> Result<()> { + let listener = TcpListener::bind("127.0.0.1:0").await?; + let endpoint = format!("http://{}", listener.local_addr()?); + let server = tokio::spawn(async move { + let (stream, _) = listener.accept().await.expect("failed probe must connect"); + drop(stream); + }); + let urls = ChainRpcUrls::from_entries(&[format!("ethereum-mainnet={endpoint}")])?; + let freshness = StatusFreshness::new(StatusFreshnessConfig::new(200, 5, 30, 5, 60)?); + freshness + .seed_success("ethereum-mainnet", 42, OffsetDateTime::now_utc()) + .await; + + freshness.refresh_once(&urls).await; + server.await.expect("failed provider task must not panic"); + let comparison = freshness + .compare(&urls, "ethereum-mainnet", Some(42), None) + .await; + + assert_eq!(comparison.status, NetworkHeadStatus::Unavailable); + assert_eq!(comparison.block, Some(42)); + assert!(comparison.observed_at.is_some()); + assert_eq!( + status_readiness(Some(42), Some(42), Some(0), &comparison), + StatusReadiness::Degraded + ); + Ok(()) +} diff --git a/apps/api/src/tests.rs b/apps/api/src/tests.rs index 8c34883f..0176e295 100644 --- a/apps/api/src/tests.rs +++ b/apps/api/src/tests.rs @@ -12,9 +12,20 @@ fn expected_health_identity() -> Value { }) } +async fn register_ready_health_loops(database: &TestDatabase) -> Result<()> { + for (service_name, instance_id) in [ + (bigname_storage::INDEXER_SERVICE_NAME, "api-health-indexer"), + (bigname_storage::WORKER_SERVICE_NAME, "api-health-worker"), + ] { + bigname_storage::register_service_loop(&database.pool, service_name, instance_id).await?; + } + Ok(()) +} + #[tokio::test] async fn healthz_reports_ready_when_database_is_reachable() -> Result<()> { let database = TestDatabase::new_migrated().await?; + register_ready_health_loops(&database).await?; let response = app_router(database.app_state()) .oneshot( @@ -31,6 +42,7 @@ async fn healthz_reports_ready_when_database_is_reachable() -> Result<()> { assert_eq!(payload.get("identity"), Some(&expected_health_identity())); assert!(payload.get("phase").is_none()); assert_eq!(payload.get("status"), Some(&json!("ready"))); + assert_eq!(payload.get("api_status"), Some(&json!("ready"))); assert_eq!( payload.get("process"), Some(&json!({ @@ -46,6 +58,15 @@ async fn healthz_reports_ready_when_database_is_reachable() -> Result<()> { "error": null, })) ); + for service_name in ["indexer", "worker"] { + let loop_health = &payload["loops"][service_name]; + assert_eq!(loop_health["status"], json!("running")); + assert_eq!(loop_health["phase"], Value::Null); + assert!(loop_health["started_at"].is_string()); + assert!(loop_health["heartbeat_at"].is_string()); + assert!(loop_health["heartbeat_age_seconds"].is_number()); + assert_eq!(loop_health["max_age_seconds"], json!(20)); + } database.cleanup().await?; Ok(()) @@ -54,6 +75,7 @@ async fn healthz_reports_ready_when_database_is_reachable() -> Result<()> { #[tokio::test] async fn healthz_returns_ready_within_probe_window_when_request_pool_is_exhausted() -> Result<()> { let database = TestDatabase::new_migrated().await?; + register_ready_health_loops(&database).await?; let request_pool = bigname_storage::connect_with_application_name_and_statement_timeout( &database.database_config(2)?, "bigname-api-exhausted-pool-test", @@ -66,10 +88,10 @@ async fn healthz_returns_ready_within_probe_window_when_request_pool_is_exhauste HEALTH_DATABASE_CHECK_TIMEOUT, ) .await?; - let state = AppState { - pool: request_pool.clone(), - chain_rpc_urls: bigname_execution::ChainRpcUrls::default(), - }; + let state = AppState::new( + request_pool.clone(), + bigname_execution::ChainRpcUrls::default(), + ); let mut held_connections = Vec::new(); for _ in 0..request_pool.options().get_max_connections() { @@ -169,6 +191,7 @@ async fn healthz_reports_degraded_when_database_is_unreachable() -> Result<()> { assert_eq!(payload.get("identity"), Some(&expected_health_identity())); assert!(payload.get("phase").is_none()); assert_eq!(payload.get("status"), Some(&json!("degraded"))); + assert_eq!(payload.get("api_status"), Some(&json!("degraded"))); assert_eq!( payload.get("process"), Some(&json!({ @@ -184,6 +207,66 @@ async fn healthz_reports_degraded_when_database_is_unreachable() -> Result<()> { "error": "database readiness query failed", })) ); + assert_eq!(payload["loops"]["indexer"]["status"], json!("unavailable")); + assert_eq!(payload["loops"]["worker"]["status"], json!("unavailable")); + + database.cleanup().await?; + Ok(()) +} + +#[tokio::test] +async fn healthz_distinguishes_not_started_and_stale_service_loops() -> Result<()> { + let database = TestDatabase::new_migrated().await?; + + let response = app_router(database.app_state()) + .oneshot( + Request::builder() + .uri("/healthz") + .body(Body::empty()) + .unwrap(), + ) + .await?; + assert_eq!(response.status(), StatusCode::OK); + let payload: Value = read_json(response).await?; + assert_eq!(payload["status"], json!("degraded")); + assert_eq!(payload["api_status"], json!("ready")); + assert_eq!(payload["loops"]["indexer"]["status"], json!("not_started")); + assert_eq!(payload["loops"]["worker"]["status"], json!("not_started")); + + register_ready_health_loops(&database).await?; + sqlx::query( + r#" + UPDATE service_loop_heartbeats + SET started_at = clock_timestamp() - INTERVAL '2 minutes', + heartbeat_at = clock_timestamp() - INTERVAL '1 minute' + WHERE service_name = 'indexer' + AND instance_id = 'api-health-indexer' + "#, + ) + .execute(&database.pool) + .await?; + + let response = app_router(database.app_state()) + .oneshot( + Request::builder() + .uri("/healthz") + .body(Body::empty()) + .unwrap(), + ) + .await?; + assert_eq!(response.status(), StatusCode::OK); + let payload: Value = read_json(response).await?; + assert_eq!(payload["status"], json!("degraded")); + assert_eq!(payload["api_status"], json!("ready")); + assert_eq!(payload["loops"]["indexer"]["status"], json!("stale")); + assert_eq!(payload["loops"]["worker"]["status"], json!("running")); + assert!(payload["loops"]["indexer"]["started_at"].is_string()); + assert!(payload["loops"]["indexer"]["heartbeat_at"].is_string()); + assert!( + payload["loops"]["indexer"]["heartbeat_age_seconds"] + .as_i64() + .is_some_and(|age| age >= 60) + ); database.cleanup().await?; Ok(()) @@ -224,6 +307,202 @@ async fn api_pool_applies_statement_timeout_to_every_connection() -> Result<()> Ok(()) } +#[tokio::test] +async fn healthz_uses_the_worker_phase_threshold_during_monolithic_rebuild_work() -> Result<()> { + let database = TestDatabase::new_migrated().await?; + register_ready_health_loops(&database).await?; + bigname_storage::begin_service_loop_phase( + &database.pool, + bigname_storage::WORKER_SERVICE_NAME, + "api-health-worker", + "resolver_current.publish", + ) + .await?; + sqlx::query( + r#" + UPDATE service_loop_heartbeats + SET started_at = clock_timestamp() - INTERVAL '2 minutes', + heartbeat_at = clock_timestamp() - INTERVAL '1 minute' + WHERE service_name = 'worker' + AND instance_id = 'api-health-worker' + AND scope_kind = 'process' + "#, + ) + .execute(&database.pool) + .await?; + + let response = app_router(database.app_state()) + .oneshot( + Request::builder() + .uri("/healthz") + .body(Body::empty()) + .unwrap(), + ) + .await?; + assert_eq!(response.status(), StatusCode::OK); + let payload: Value = read_json(response).await?; + assert_eq!(payload["status"], json!("ready")); + assert_eq!(payload["api_status"], json!("ready")); + assert_eq!(payload["loops"]["worker"]["status"], json!("running")); + assert_eq!( + payload["loops"]["worker"]["phase"], + json!("resolver_current.publish") + ); + assert_eq!( + payload["loops"]["worker"]["max_age_seconds"], + json!(bigname_storage::DEFAULT_WORKER_REBUILD_PHASE_MAX_AGE_SECS) + ); + + database.cleanup().await +} + +#[tokio::test] +async fn healthz_reaps_a_dead_worker_phase_when_a_replacement_registers() -> Result<()> { + let database = TestDatabase::new_migrated().await?; + bigname_storage::register_service_loop( + &database.pool, + bigname_storage::INDEXER_SERVICE_NAME, + "api-health-indexer", + ) + .await?; + bigname_storage::register_service_loop( + &database.pool, + bigname_storage::WORKER_SERVICE_NAME, + "dead-mid-phase-worker", + ) + .await?; + bigname_storage::begin_service_loop_phase( + &database.pool, + bigname_storage::WORKER_SERVICE_NAME, + "dead-mid-phase-worker", + "name_current.publish", + ) + .await?; + sqlx::query( + r#" + UPDATE service_loop_heartbeats + SET started_at = clock_timestamp() - INTERVAL '2 minutes', + heartbeat_at = clock_timestamp() - INTERVAL '1 minute' + WHERE service_name = 'worker' + AND instance_id = 'dead-mid-phase-worker' + "#, + ) + .execute(&database.pool) + .await?; + + bigname_storage::register_service_loop( + &database.pool, + bigname_storage::WORKER_SERVICE_NAME, + "replacement-worker", + ) + .await?; + let orphaned_phase_count = sqlx::query_scalar::<_, i64>( + r#" + SELECT COUNT(*)::BIGINT + FROM service_loop_heartbeats + WHERE service_name = 'worker' + AND instance_id = 'dead-mid-phase-worker' + AND scope_kind = 'phase' + "#, + ) + .fetch_one(&database.pool) + .await?; + assert_eq!( + orphaned_phase_count, 0, + "replacement registration must reap the dead predecessor's phase" + ); + + sqlx::query( + r#" + UPDATE service_loop_heartbeats + SET started_at = clock_timestamp() - INTERVAL '2 minutes', + heartbeat_at = clock_timestamp() - INTERVAL '1 minute' + WHERE service_name = 'worker' + AND scope_kind = 'process' + "#, + ) + .execute(&database.pool) + .await?; + + let response = app_router(database.app_state()) + .oneshot( + Request::builder() + .uri("/healthz") + .body(Body::empty()) + .unwrap(), + ) + .await?; + assert_eq!(response.status(), StatusCode::OK); + let payload: Value = read_json(response).await?; + assert_eq!(payload["status"], json!("degraded")); + assert_eq!(payload["loops"]["worker"]["status"], json!("stale")); + assert_eq!(payload["loops"]["worker"]["phase"], Value::Null); + + database.cleanup().await +} + +#[tokio::test] +async fn healthz_prefers_a_healthy_worker_phase_over_a_newer_stale_replica() -> Result<()> { + let database = TestDatabase::new_migrated().await?; + register_ready_health_loops(&database).await?; + + bigname_storage::register_service_loop( + &database.pool, + bigname_storage::WORKER_SERVICE_NAME, + "newer-stale-worker", + ) + .await?; + sqlx::query( + r#" + UPDATE service_loop_heartbeats + SET started_at = clock_timestamp() - INTERVAL '90 minutes', + heartbeat_at = clock_timestamp() - INTERVAL '1 hour' + WHERE service_name = 'worker' + AND instance_id = 'newer-stale-worker' + "#, + ) + .execute(&database.pool) + .await?; + + bigname_storage::begin_service_loop_phase( + &database.pool, + bigname_storage::WORKER_SERVICE_NAME, + "api-health-worker", + "resolver_current.publish", + ) + .await?; + sqlx::query( + r#" + UPDATE service_loop_heartbeats + SET started_at = clock_timestamp() - INTERVAL '3 hours', + heartbeat_at = clock_timestamp() - INTERVAL '2 hours' + WHERE service_name = 'worker' + AND instance_id = 'api-health-worker' + "#, + ) + .execute(&database.pool) + .await?; + + let response = app_router(database.app_state()) + .oneshot( + Request::builder() + .uri("/healthz") + .body(Body::empty()) + .unwrap(), + ) + .await?; + assert_eq!(response.status(), StatusCode::OK); + let payload: Value = read_json(response).await?; + assert_eq!(payload["status"], json!("ready")); + assert_eq!(payload["loops"]["worker"]["status"], json!("running")); + assert_eq!( + payload["loops"]["worker"]["phase"], + json!("resolver_current.publish") + ); + + database.cleanup().await +} + include!("tests/exact_name.rs"); include!("tests/resolution.rs"); diff --git a/apps/api/src/tests/identity.rs b/apps/api/src/tests/identity.rs index 33783a96..2b0a1187 100644 --- a/apps/api/src/tests/identity.rs +++ b/apps/api/src/tests/identity.rs @@ -1732,6 +1732,8 @@ async fn indexing_status_degrades_without_chain_readiness_data() -> Result<()> { assert_eq!(response.status(), StatusCode::OK); let payload: Value = read_json(response).await?; assert_eq!(payload["data"]["status"], json!("degraded")); + assert_eq!(payload["data"]["pending_invalidation_count"], json!(0)); + assert_eq!(payload["data"]["dead_letter_count"], json!(0)); assert_eq!( payload["data"]["chains"] .as_object() @@ -1755,7 +1757,6 @@ async fn indexing_status_degrades_for_chain_without_checkpoint() -> Result<()> { ) .execute(&database.pool) .await?; - let response = app_router(database.app_state()) .oneshot( Request::builder() @@ -1768,6 +1769,8 @@ async fn indexing_status_degrades_for_chain_without_checkpoint() -> Result<()> { assert_eq!(response.status(), StatusCode::OK); let payload: Value = read_json(response).await?; assert_eq!(payload["data"]["status"], json!("degraded")); + assert_eq!(payload["data"]["pending_invalidation_count"], json!(0)); + assert_eq!(payload["data"]["dead_letter_count"], json!(0)); assert_eq!( payload["data"]["chains"]["ethereum-mainnet"]["canonical_block"], Value::Null @@ -1916,6 +1919,12 @@ async fn indexing_status_degrades_for_direct_pending_invalidations() -> Result<( assert_eq!(response.status(), StatusCode::OK); let payload: Value = read_json(response).await?; assert_eq!(payload["data"]["status"], json!("degraded")); + assert_eq!(payload["data"]["pending_invalidation_count"], json!(1)); + assert_eq!( + payload["data"]["pending_invalidation_count_capped"], + json!(false) + ); + assert_eq!(payload["data"]["dead_letter_count"], json!(0)); assert_eq!( payload["data"]["chains"]["ethereum-mainnet"]["latest_projected_block"], json!(10) @@ -1925,6 +1934,50 @@ async fn indexing_status_degrades_for_direct_pending_invalidations() -> Result<( Ok(()) } +#[tokio::test] +async fn indexing_status_caps_the_pending_invalidation_count_with_an_explicit_marker() -> Result<()> +{ + let database = TestDatabase::new_migrated().await?; + sqlx::query( + r#" + INSERT INTO projection_invalidations ( + projection, + projection_key, + key_payload + ) + SELECT + 'name_current', + 'bounded-status-' || sequence, + '{}'::jsonb + FROM generate_series(1, $1 + 1) AS sequence + "#, + ) + .bind(bigname_storage::PENDING_INVALIDATION_COUNT_CAP) + .execute(&database.pool) + .await?; + + let response = app_router(database.app_state()) + .oneshot( + Request::builder() + .uri("/v1/status") + .body(Body::empty()) + .expect("request must build"), + ) + .await?; + assert_eq!(response.status(), StatusCode::OK); + let payload: Value = read_json(response).await?; + assert_eq!( + payload["data"]["pending_invalidation_count"], + json!(bigname_storage::PENDING_INVALIDATION_COUNT_CAP) + ); + assert_eq!( + payload["data"]["pending_invalidation_count_capped"], + json!(true) + ); + + database.cleanup().await +} + #[tokio::test] async fn indexing_status_reports_projection_lag_by_chain() -> Result<()> { let database = TestDatabase::new_migrated().await?; @@ -2055,6 +2108,32 @@ async fn indexing_status_reports_projection_lag_by_chain() -> Result<()> { ) .execute(&database.pool) .await?; + sqlx::query( + r#" + INSERT INTO projection_invalidation_dead_letters ( + projection, + projection_key, + generation, + attempt_count, + last_changed_at, + invalidated_at, + last_failure_reason, + last_failure_at + ) + VALUES ( + 'name_current', + 'dead-lettered-status.eth', + 1, + 5, + '2026-04-17T00:00:10Z', + '2026-04-17T00:00:10Z', + 'status test failure', + '2026-04-17T00:00:10Z' + ) + "#, + ) + .execute(&database.pool) + .await?; let response = app_router(database.app_state()) .oneshot( @@ -2068,6 +2147,8 @@ async fn indexing_status_reports_projection_lag_by_chain() -> Result<()> { assert_eq!(response.status(), StatusCode::OK); let payload: Value = read_json(response).await?; assert_eq!(payload["data"]["status"], json!("stale")); + assert_eq!(payload["data"]["pending_invalidation_count"], json!(1)); + assert_eq!(payload["data"]["dead_letter_count"], json!(1)); assert_eq!( payload["data"]["chains"]["ethereum-mainnet"]["canonical_block"], json!(10) @@ -2080,6 +2161,14 @@ async fn indexing_status_reports_projection_lag_by_chain() -> Result<()> { payload["data"]["chains"]["ethereum-mainnet"]["projection_lag_blocks"], json!(1) ); + assert_eq!( + payload["data"]["chains"]["ethereum-mainnet"]["network_head_status"], + json!("unconfigured") + ); + assert_eq!( + payload["data"]["chains"]["ethereum-mainnet"]["network_block"], + Value::Null + ); assert_eq!( payload["data"]["chains"]["ethereum-mainnet"]["projection_lag_seconds"], json!(1) @@ -2099,6 +2188,8 @@ async fn indexing_status_reports_projection_lag_by_chain() -> Result<()> { assert!(payload.get("page").is_none()); assert_eq!(payload["meta"], json!({})); assert_eq!(payload["data"]["status"], json!("stale")); + assert_eq!(payload["data"]["pending_invalidation_count"], json!(1)); + assert_eq!(payload["data"]["dead_letter_count"], json!(1)); let chains = payload["data"]["chains"] .as_object() @@ -2114,6 +2205,8 @@ async fn indexing_status_reports_projection_lag_by_chain() -> Result<()> { assert_eq!(chain.get("finalized_block"), Some(&json!(9))); assert_eq!(chain.get("lag_blocks"), Some(&json!(1))); assert_eq!(chain.get("lag_seconds"), Some(&json!(1))); + assert_eq!(chain.get("network_head_status"), Some(&json!("unconfigured"))); + assert_eq!(chain.get("network_block"), Some(&Value::Null)); assert_eq!(chain.get("status"), Some(&json!("stale"))); assert!(chain.get("canonical_block").is_none()); assert!(chain.get("latest_projected_block").is_none()); @@ -2121,7 +2214,76 @@ async fn indexing_status_reports_projection_lag_by_chain() -> Result<()> { sqlx::query("DELETE FROM projection_invalidations") .execute(&database.pool) .await?; - let response = app_router(database.app_state()) + let chain_rpc_urls = bigname_execution::ChainRpcUrls::from_entries(&[ + "ethereum-mainnet=http://status-provider.test".to_owned(), + ])?; + let ready_state = database.app_state_with_chain_rpc_urls(chain_rpc_urls); + ready_state + .status_freshness + .seed_success( + "ethereum-mainnet", + 16, + OffsetDateTime::now_utc(), + ) + .await; + let response = app_router(ready_state.clone()) + .oneshot( + Request::builder() + .uri("/v1/status") + .body(Body::empty()) + .expect("request must build"), + ) + .await + .context("network-lag indexing status request failed")?; + assert_eq!(response.status(), StatusCode::OK); + let payload: Value = read_json(response).await?; + assert_eq!(payload["data"]["status"], json!("stale")); + assert_eq!( + payload["data"]["chains"]["ethereum-mainnet"]["network_block"], + json!(16) + ); + assert_eq!( + payload["data"]["chains"]["ethereum-mainnet"]["network_head_status"], + json!("fresh") + ); + assert_eq!( + payload["data"]["chains"]["ethereum-mainnet"]["ingestion_lag_blocks"], + json!(6) + ); + assert!( + payload["data"]["chains"]["ethereum-mainnet"]["ingestion_lag_seconds"] + .as_i64() + .is_some_and(|lag| lag > 60) + ); + + let response = app_router(ready_state.clone()) + .oneshot( + Request::builder() + .uri("/v2/status") + .body(Body::empty()) + .expect("request must build"), + ) + .await + .context("network-lag v2 indexing status request failed")?; + assert_eq!(response.status(), StatusCode::OK); + let payload: Value = read_json(response).await?; + assert_eq!(payload["data"]["status"], json!("stale")); + assert_eq!(payload["data"]["chains"]["1"]["network_block"], json!(16)); + assert_eq!( + payload["data"]["chains"]["1"]["ingestion_lag_blocks"], + json!(6) + ); + assert_eq!(payload["data"]["chains"]["1"]["status"], json!("stale")); + + ready_state + .status_freshness + .seed_success( + "ethereum-mainnet", + 10, + OffsetDateTime::now_utc(), + ) + .await; + let response = app_router(ready_state.clone()) .oneshot( Request::builder() .uri("/v1/status") @@ -2133,6 +2295,8 @@ async fn indexing_status_reports_projection_lag_by_chain() -> Result<()> { assert_eq!(response.status(), StatusCode::OK); let payload: Value = read_json(response).await?; assert_eq!(payload["data"]["status"], json!("ready")); + assert_eq!(payload["data"]["pending_invalidation_count"], json!(0)); + assert_eq!(payload["data"]["dead_letter_count"], json!(1)); assert_eq!( payload["data"]["chains"]["ethereum-mainnet"]["latest_projected_block"], json!(10) @@ -2141,6 +2305,35 @@ async fn indexing_status_reports_projection_lag_by_chain() -> Result<()> { payload["data"]["chains"]["ethereum-mainnet"]["projection_lag_blocks"], json!(0) ); + assert_eq!( + payload["data"]["chains"]["ethereum-mainnet"]["network_head_status"], + json!("fresh") + ); + assert_eq!( + payload["data"]["chains"]["ethereum-mainnet"]["ingestion_lag_blocks"], + json!(0) + ); + + let response = app_router(ready_state) + .oneshot( + Request::builder() + .uri("/v2/status") + .body(Body::empty()) + .expect("request must build"), + ) + .await + .context("ready v2 indexing status request failed")?; + assert_eq!(response.status(), StatusCode::OK); + let payload: Value = read_json(response).await?; + assert_eq!(payload["data"]["status"], json!("ready")); + assert_eq!(payload["data"]["pending_invalidation_count"], json!(0)); + assert_eq!(payload["data"]["dead_letter_count"], json!(1)); + assert_eq!(payload["data"]["chains"]["1"]["network_block"], json!(10)); + assert_eq!( + payload["data"]["chains"]["1"]["network_head_status"], + json!("fresh") + ); + assert_eq!(payload["data"]["chains"]["1"]["ingestion_lag_blocks"], json!(0)); database.cleanup().await?; Ok(()) diff --git a/apps/api/src/tests/openapi.rs b/apps/api/src/tests/openapi.rs index 2cae4a5e..6e34d6b5 100644 --- a/apps/api/src/tests/openapi.rs +++ b/apps/api/src/tests/openapi.rs @@ -349,10 +349,41 @@ fn openapi_document_uses_package_version_and_freezes_health_identity() { Some(&json!(["string", "null"])) ); + let service_loop = openapi_schema(&document, "HealthLoop"); + assert_eq!( + required_fields(service_loop), + vec![ + "status", + "phase", + "started_at", + "heartbeat_at", + "heartbeat_age_seconds", + "max_age_seconds", + ] + ); + assert_eq!( + service_loop.pointer("/properties/status/enum"), + Some(&json!(["running", "stale", "not_started", "unavailable"])) + ); + let service_loops = openapi_schema(&document, "HealthLoops"); + assert_eq!(required_fields(service_loops), vec!["indexer", "worker"]); + let response = openapi_schema(&document, "HealthResponse"); assert_eq!( required_fields(response), - vec!["service", "identity", "status", "process", "database"] + vec![ + "service", + "identity", + "status", + "api_status", + "process", + "database", + "loops", + ] + ); + assert_eq!( + response.pointer("/properties/api_status/enum"), + Some(&json!(["ready", "degraded"])) ); assert_eq!( response.pointer("/properties/process"), @@ -362,7 +393,41 @@ fn openapi_document_uses_package_version_and_freezes_health_identity() { response.pointer("/properties/database"), Some(&json!({ "$ref": "#/components/schemas/HealthDatabase" })) ); + assert_eq!( + response.pointer("/properties/loops"), + Some(&json!({ "$ref": "#/components/schemas/HealthLoops" })) + ); assert!(!property_names(response).contains(&"phase")); + + let indexing_status = openapi_schema(&document, "IndexingStatusResponse"); + assert_eq!( + required_fields(indexing_status), + vec![ + "status", + "pending_invalidation_count", + "pending_invalidation_count_capped", + "dead_letter_count", + "chains", + ] + ); + assert_eq!( + indexing_status.pointer("/properties/pending_invalidation_count/maximum"), + Some(&json!( + bigname_storage::PENDING_INVALIDATION_COUNT_CAP + )) + ); + assert_eq!( + indexing_status.pointer( + "/properties/chains/additionalProperties/properties/network_head_status/enum" + ), + Some(&json!([ + "fresh", + "stale", + "unavailable", + "pending", + "unconfigured", + ])) + ); } #[test] @@ -1347,9 +1412,9 @@ async fn openapi_docs_route_serves_viewer() -> Result<()> { } fn openapi_docs_test_state() -> AppState { - AppState { - pool: PgPool::connect_lazy("postgres://bigname:bigname@127.0.0.1:5432/bigname") + AppState::new( + PgPool::connect_lazy("postgres://bigname:bigname@127.0.0.1:5432/bigname") .expect("OpenAPI helper route tests only need a lazily parsed pool"), - chain_rpc_urls: bigname_execution::ChainRpcUrls::default(), - } + bigname_execution::ChainRpcUrls::default(), + ) } diff --git a/apps/api/src/tests/support.rs b/apps/api/src/tests/support.rs index 8ae59d4f..f38dcb47 100644 --- a/apps/api/src/tests/support.rs +++ b/apps/api/src/tests/support.rs @@ -607,10 +607,7 @@ impl TestDatabase { &self, chain_rpc_urls: bigname_execution::ChainRpcUrls, ) -> AppState { - AppState { - pool: self.pool.clone(), - chain_rpc_urls, - } + AppState::new(self.pool.clone(), chain_rpc_urls) } async fn seed_history_binding( diff --git a/apps/api/src/tests/v2_diagnostics_names.rs b/apps/api/src/tests/v2_diagnostics_names.rs index 89fb810b..77c45e3f 100644 --- a/apps/api/src/tests/v2_diagnostics_names.rs +++ b/apps/api/src/tests/v2_diagnostics_names.rs @@ -962,11 +962,11 @@ async fn v2_diagnostics_name_routes_honor_namespace_override() -> Result<()> { #[tokio::test] async fn v2_diagnostics_name_routes_reject_malformed_name() -> Result<()> { - let state = AppState { - pool: PgPool::connect_lazy("postgres://bigname:bigname@127.0.0.1:5432/bigname") + let state = AppState::new( + PgPool::connect_lazy("postgres://bigname:bigname@127.0.0.1:5432/bigname") .expect("name normalization rejection does not use the database"), - chain_rpc_urls: bigname_execution::ChainRpcUrls::default(), - }; + bigname_execution::ChainRpcUrls::default(), + ); for suffix in ["coverage", "binding", "authority", "records"] { let uri = format!("/v2/diagnostics/names/bad%20name.eth/{suffix}"); @@ -990,11 +990,11 @@ async fn v2_diagnostics_name_routes_reject_malformed_name() -> Result<()> { #[tokio::test] async fn v2_diagnostics_name_routes_reject_undocumented_query_params() -> Result<()> { - let state = AppState { - pool: PgPool::connect_lazy("postgres://bigname:bigname@127.0.0.1:5432/bigname") + let state = AppState::new( + PgPool::connect_lazy("postgres://bigname:bigname@127.0.0.1:5432/bigname") .expect("query rejection does not use the database"), - chain_rpc_urls: bigname_execution::ChainRpcUrls::default(), - }; + bigname_execution::ChainRpcUrls::default(), + ); for suffix in ["coverage", "binding", "authority"] { for (query, expected_message) in [ @@ -1057,11 +1057,11 @@ async fn v2_diagnostics_name_routes_reject_undocumented_query_params() -> Result #[tokio::test] async fn v2_diagnostics_name_records_rejects_malformed_duplicate_and_unknown_query_params() -> Result<()> { - let state = AppState { - pool: PgPool::connect_lazy("postgres://bigname:bigname@127.0.0.1:5432/bigname") + let state = AppState::new( + PgPool::connect_lazy("postgres://bigname:bigname@127.0.0.1:5432/bigname") .expect("query rejection does not use the database"), - chain_rpc_urls: bigname_execution::ChainRpcUrls::default(), - }; + bigname_execution::ChainRpcUrls::default(), + ); for (uri, expected_message) in [ ( @@ -1133,11 +1133,11 @@ async fn v2_diagnostics_name_routes_reject_invalid_namespace_and_at() -> Result< #[tokio::test] async fn v2_diagnostics_name_execution_requires_keys() -> Result<()> { - let state = AppState { - pool: PgPool::connect_lazy("postgres://bigname:bigname@127.0.0.1:5432/bigname") + let state = AppState::new( + PgPool::connect_lazy("postgres://bigname:bigname@127.0.0.1:5432/bigname") .expect("keys rejection does not use the database"), - chain_rpc_urls: bigname_execution::ChainRpcUrls::default(), - }; + bigname_execution::ChainRpcUrls::default(), + ); for uri in [ "/v2/diagnostics/names/alice.eth/execution", @@ -1169,11 +1169,11 @@ async fn v2_diagnostics_name_execution_requires_keys() -> Result<()> { #[tokio::test] async fn v2_diagnostics_name_execution_rejects_malformed_duplicate_and_unknown_query_params() -> Result<()> { - let state = AppState { - pool: PgPool::connect_lazy("postgres://bigname:bigname@127.0.0.1:5432/bigname") + let state = AppState::new( + PgPool::connect_lazy("postgres://bigname:bigname@127.0.0.1:5432/bigname") .expect("query rejection does not use the database"), - chain_rpc_urls: bigname_execution::ChainRpcUrls::default(), - }; + bigname_execution::ChainRpcUrls::default(), + ); for (uri, expected_message) in [ ( diff --git a/apps/api/src/types.rs b/apps/api/src/types.rs index 93840640..00ba4142 100644 --- a/apps/api/src/types.rs +++ b/apps/api/src/types.rs @@ -11,7 +11,8 @@ use crate::pagination::HistoryPageResponse; mod health; pub(crate) use health::{ - HealthDatabaseResponse, HealthIdentityResponse, HealthProcessResponse, HealthResponse, + HealthDatabaseResponse, HealthIdentityResponse, HealthLoopResponse, HealthLoopsResponse, + HealthProcessResponse, HealthResponse, }; fn json_value_is_null(value: &JsonValue) -> bool { @@ -146,6 +147,9 @@ pub(crate) struct IdentityAsOfResponse { #[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)] pub(crate) struct IndexingStatusResponse { pub(crate) status: String, + pub(crate) pending_invalidation_count: i64, + pub(crate) pending_invalidation_count_capped: bool, + pub(crate) dead_letter_count: i64, pub(crate) chains: BTreeMap, } @@ -163,6 +167,12 @@ pub(crate) struct IndexingStatusChainResponse { pub(crate) latest_projected_timestamp: Option, pub(crate) projection_lag_blocks: Option, pub(crate) projection_lag_seconds: Option, + pub(crate) network_block: Option, + pub(crate) network_head_observed_at: Option, + pub(crate) network_head_age_seconds: Option, + pub(crate) network_head_status: String, + pub(crate) ingestion_lag_blocks: Option, + pub(crate) ingestion_lag_seconds: Option, } #[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)] diff --git a/apps/api/src/types/health.rs b/apps/api/src/types/health.rs index b8d5962f..4411e61e 100644 --- a/apps/api/src/types/health.rs +++ b/apps/api/src/types/health.rs @@ -7,8 +7,10 @@ pub(crate) struct HealthResponse { pub(crate) service: &'static str, pub(crate) identity: HealthIdentityResponse, pub(crate) status: &'static str, + pub(crate) api_status: &'static str, pub(crate) process: HealthProcessResponse, pub(crate) database: HealthDatabaseResponse, + pub(crate) loops: HealthLoopsResponse, } #[derive(Serialize)] @@ -51,3 +53,19 @@ pub(crate) struct HealthDatabaseResponse { pub(crate) check: &'static str, pub(crate) error: Option<&'static str>, } + +#[derive(Serialize)] +pub(crate) struct HealthLoopsResponse { + pub(crate) indexer: HealthLoopResponse, + pub(crate) worker: HealthLoopResponse, +} + +#[derive(Serialize)] +pub(crate) struct HealthLoopResponse { + pub(crate) status: &'static str, + pub(crate) phase: Option, + pub(crate) started_at: Option, + pub(crate) heartbeat_at: Option, + pub(crate) heartbeat_age_seconds: Option, + pub(crate) max_age_seconds: i64, +} diff --git a/apps/api/src/v2/namespaces.rs b/apps/api/src/v2/namespaces.rs index 3683ff04..ca50a542 100644 --- a/apps/api/src/v2/namespaces.rs +++ b/apps/api/src/v2/namespaces.rs @@ -284,11 +284,11 @@ mod tests { #[tokio::test] async fn get_namespace_returns_not_found_for_unsupported_namespace() { - let state = AppState { - pool: PgPool::connect_lazy("postgres://bigname:bigname@127.0.0.1:5432/bigname") + let state = AppState::new( + PgPool::connect_lazy("postgres://bigname:bigname@127.0.0.1:5432/bigname") .expect("unsupported namespace validation does not use the database"), - chain_rpc_urls: bigname_execution::ChainRpcUrls::default(), - }; + bigname_execution::ChainRpcUrls::default(), + ); let error = get_namespace(Path("unknown".to_owned()), NoQueryParams, State(state)) .await diff --git a/apps/api/src/v2/router.rs b/apps/api/src/v2/router.rs index 4b9b268e..4e3e76d4 100644 --- a/apps/api/src/v2/router.rs +++ b/apps/api/src/v2/router.rs @@ -76,11 +76,11 @@ mod tests { #[tokio::test] async fn status_route_rejects_query_params_with_v2_error_envelope() { - let state = AppState { - pool: PgPool::connect_lazy("postgres://bigname:bigname@127.0.0.1:5432/bigname") + let state = AppState::new( + PgPool::connect_lazy("postgres://bigname:bigname@127.0.0.1:5432/bigname") .expect("query rejection does not use the database"), - chain_rpc_urls: bigname_execution::ChainRpcUrls::default(), - }; + bigname_execution::ChainRpcUrls::default(), + ); let response = router() .with_state(state) diff --git a/apps/api/src/v2/search/tests.rs b/apps/api/src/v2/search/tests.rs index d43b5770..35048f76 100644 --- a/apps/api/src/v2/search/tests.rs +++ b/apps/api/src/v2/search/tests.rs @@ -532,10 +532,10 @@ impl SearchDatabase { } fn app_state(&self) -> crate::AppState { - crate::AppState { - pool: self.pool().clone(), - chain_rpc_urls: bigname_execution::ChainRpcUrls::default(), - } + crate::AppState::new( + self.pool().clone(), + bigname_execution::ChainRpcUrls::default(), + ) } async fn cleanup(self) -> Result<()> { diff --git a/apps/api/src/v2/status.rs b/apps/api/src/v2/status.rs index 7131ba16..4c900272 100644 --- a/apps/api/src/v2/status.rs +++ b/apps/api/src/v2/status.rs @@ -6,11 +6,16 @@ use tracing::error; use crate::AppState; -use super::{Envelope, Meta, NoQueryParams, OpsStatus, V2Error, V2Result, slug_to_numeric}; +use super::{ + Envelope, Meta, NoQueryParams, OpsStatus, V2Error, V2Result, format_timestamp, slug_to_numeric, +}; #[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)] pub(crate) struct StatusData { pub(crate) status: OpsStatus, + pub(crate) pending_invalidation_count: i64, + pub(crate) pending_invalidation_count_capped: bool, + pub(crate) dead_letter_count: i64, pub(crate) chains: BTreeMap, } @@ -22,6 +27,12 @@ pub(crate) struct ChainStatus { pub(crate) finalized_block: Option, pub(crate) lag_blocks: Option, pub(crate) lag_seconds: Option, + pub(crate) network_block: Option, + pub(crate) network_head_observed_at: Option, + pub(crate) network_head_age_seconds: Option, + pub(crate) network_head_status: String, + pub(crate) ingestion_lag_blocks: Option, + pub(crate) ingestion_lag_seconds: Option, pub(crate) status: OpsStatus, } @@ -41,20 +52,36 @@ pub(crate) async fn get_status( })?; Ok(Json(Envelope { - data: build_status_data(&read)?, + data: build_status_data(&read, &state).await?, page: None, meta: Meta::default(), })) } -fn build_status_data(read: &bigname_storage::IndexingStatusRead) -> V2Result { +async fn build_status_data( + read: &bigname_storage::IndexingStatusRead, + state: &AppState, +) -> V2Result { let mut chains = BTreeMap::new(); for row in &read.chains { - chains.insert(status_chain_key(&row.chain_id)?, build_chain_status(row)); + let chain_key = status_chain_key(&row.chain_id)?; + let network_head = state + .status_freshness + .compare( + &state.chain_rpc_urls, + &row.chain_id, + row.canonical_block, + row.canonical_timestamp, + ) + .await; + chains.insert(chain_key, build_chain_status(row, network_head)); } Ok(StatusData { status: root_status(chains.values(), read.has_unscoped_pending_invalidations), + pending_invalidation_count: read.pending_invalidation_count, + pending_invalidation_count_capped: read.pending_invalidation_count_capped, + dead_letter_count: read.dead_letter_count, chains, }) } @@ -69,7 +96,10 @@ fn status_chain_key(storage_chain_id: &str) -> V2Result { Ok(numeric.to_string()) } -fn build_chain_status(row: &bigname_storage::IndexingStatusChainRow) -> ChainStatus { +fn build_chain_status( + row: &bigname_storage::IndexingStatusChainRow, + network_head: crate::status_freshness::NetworkHeadComparison, +) -> ChainStatus { let lag_blocks = row .canonical_block .zip(row.latest_projected_block) @@ -78,7 +108,12 @@ fn build_chain_status(row: &bigname_storage::IndexingStatusChainRow) -> ChainSta .canonical_timestamp .zip(row.latest_projected_timestamp) .map(|(canonical, projected)| (canonical - projected).whole_seconds().max(0)); - let status = chain_status(row.canonical_block, row.latest_projected_block, lag_blocks); + let status = chain_status( + row.canonical_block, + row.latest_projected_block, + lag_blocks, + &network_head, + ); ChainStatus { latest_block: row.canonical_block, @@ -87,6 +122,12 @@ fn build_chain_status(row: &bigname_storage::IndexingStatusChainRow) -> ChainSta finalized_block: row.finalized_block, lag_blocks, lag_seconds, + network_block: network_head.block, + network_head_observed_at: network_head.observed_at.map(format_timestamp), + network_head_age_seconds: network_head.age_seconds, + network_head_status: network_head.status.as_str().to_owned(), + ingestion_lag_blocks: network_head.ingestion_lag_blocks, + ingestion_lag_seconds: network_head.ingestion_lag_seconds, status, } } @@ -95,11 +136,17 @@ fn chain_status( latest_block: Option, indexed_block: Option, lag_blocks: Option, + network_head: &crate::status_freshness::NetworkHeadComparison, ) -> OpsStatus { - match (latest_block, indexed_block, lag_blocks) { - (Some(_), Some(_), Some(0)) => OpsStatus::Ready, - (Some(_), Some(_), Some(_)) => OpsStatus::Stale, - _ => OpsStatus::Degraded, + match crate::status_freshness::status_readiness( + latest_block, + indexed_block, + lag_blocks, + network_head, + ) { + crate::status_freshness::StatusReadiness::Ready => OpsStatus::Ready, + crate::status_freshness::StatusReadiness::Degraded => OpsStatus::Degraded, + crate::status_freshness::StatusReadiness::Stale => OpsStatus::Stale, } } @@ -133,12 +180,12 @@ mod tests { http::{Request, StatusCode}, response::IntoResponse, }; - use sqlx::types::time::OffsetDateTime; + use sqlx::{PgPool, types::time::OffsetDateTime}; use super::*; - #[test] - fn status_data_maps_chains_by_numeric_chain_id_and_derives_chain_statuses() { + #[tokio::test] + async fn status_data_maps_chains_by_numeric_chain_id_and_derives_chain_statuses() { let read = bigname_storage::IndexingStatusRead { chains: vec![ row( @@ -158,11 +205,41 @@ mod tests { row("base-sepolia", Some(100), Some(95), Some(90), Some(80)), ], has_unscoped_pending_invalidations: false, + pending_invalidation_count: 7, + pending_invalidation_count_capped: false, + dead_letter_count: 2, }; + let chain_rpc_urls = bigname_execution::ChainRpcUrls::from_entries(&[ + "ethereum-mainnet=http://rpc.test".to_owned(), + "base-mainnet=http://rpc.test".to_owned(), + "base-sepolia=http://rpc.test".to_owned(), + ]) + .expect("test RPC map must be valid"); + let state = AppState::new( + PgPool::connect_lazy("postgres://bigname:bigname@127.0.0.1:5432/bigname") + .expect("status builder test does not use the database"), + chain_rpc_urls, + ); + let observed_at = OffsetDateTime::now_utc(); + for (chain_id, block) in [ + (bigname_storage::ETHEREUM_MAINNET_CHAIN_ID, 100), + (bigname_storage::BASE_MAINNET_CHAIN_ID, 50), + ("base-sepolia", 100), + ] { + state + .status_freshness + .seed_success(chain_id, block, observed_at) + .await; + } - let data = build_status_data(&read).expect("known storage chain slugs must map"); + let data = build_status_data(&read, &state) + .await + .expect("known storage chain slugs must map"); assert_eq!(data.status, OpsStatus::Stale); + assert_eq!(data.pending_invalidation_count, 7); + assert!(!data.pending_invalidation_count_capped); + assert_eq!(data.dead_letter_count, 2); assert_eq!( data.chains.keys().collect::>(), vec!["1", "8453", "84532"] @@ -172,6 +249,10 @@ mod tests { assert_eq!(data.chains["1"].indexed_block, Some(100)); assert_eq!(data.chains["1"].lag_blocks, Some(0)); assert_eq!(data.chains["1"].lag_seconds, Some(0)); + assert_eq!(data.chains["1"].network_block, Some(100)); + assert_eq!(data.chains["1"].network_head_status, "fresh"); + assert_eq!(data.chains["1"].ingestion_lag_blocks, Some(0)); + assert_eq!(data.chains["1"].ingestion_lag_seconds, Some(0)); assert_eq!(data.chains["8453"].status, OpsStatus::Degraded); assert_eq!(data.chains["8453"].lag_blocks, None); assert_eq!(data.chains["84532"].status, OpsStatus::Stale); @@ -179,8 +260,8 @@ mod tests { assert_eq!(data.chains["84532"].lag_seconds, Some(50)); } - #[test] - fn status_data_rejects_unmapped_storage_chain_slugs() { + #[tokio::test] + async fn status_data_rejects_unmapped_storage_chain_slugs() { let read = bigname_storage::IndexingStatusRead { chains: vec![row( "future-mainnet", @@ -190,9 +271,19 @@ mod tests { Some(80), )], has_unscoped_pending_invalidations: false, + pending_invalidation_count: 0, + pending_invalidation_count_capped: false, + dead_letter_count: 0, }; + let state = AppState::new( + PgPool::connect_lazy("postgres://bigname:bigname@127.0.0.1:5432/bigname") + .expect("status builder test does not use the database"), + bigname_execution::ChainRpcUrls::default(), + ); - let error = build_status_data(&read).expect_err("unknown storage slug must fail loudly"); + let error = build_status_data(&read, &state) + .await + .expect_err("unknown storage slug must fail loudly"); assert_eq!(error.envelope().error.code, "internal_error"); } @@ -206,6 +297,12 @@ mod tests { finalized_block: Some(80), lag_blocks: Some(0), lag_seconds: Some(0), + network_block: Some(100), + network_head_observed_at: None, + network_head_age_seconds: Some(0), + network_head_status: "fresh".to_owned(), + ingestion_lag_blocks: Some(0), + ingestion_lag_seconds: Some(0), status: OpsStatus::Ready, }; let degraded = ChainStatus { @@ -215,6 +312,12 @@ mod tests { finalized_block: Some(80), lag_blocks: None, lag_seconds: None, + network_block: None, + network_head_observed_at: None, + network_head_age_seconds: None, + network_head_status: "unavailable".to_owned(), + ingestion_lag_blocks: None, + ingestion_lag_seconds: None, status: OpsStatus::Degraded, }; let stale = ChainStatus { @@ -224,6 +327,12 @@ mod tests { finalized_block: Some(80), lag_blocks: Some(1), lag_seconds: Some(10), + network_block: Some(100), + network_head_observed_at: None, + network_head_age_seconds: Some(0), + network_head_status: "fresh".to_owned(), + ingestion_lag_blocks: Some(0), + ingestion_lag_seconds: Some(0), status: OpsStatus::Stale, }; diff --git a/apps/indexer/src/main/backfill.rs b/apps/indexer/src/main/backfill.rs index 0b742187..2dc011e9 100644 --- a/apps/indexer/src/main/backfill.rs +++ b/apps/indexer/src/main/backfill.rs @@ -36,6 +36,7 @@ pub(crate) use coinbase_sql::{ pub(crate) use concurrent_execution::{ run_resumable_coinbase_sql_backfill_job_concurrently, run_resumable_hash_pinned_backfill_job_concurrently, + run_resumable_hash_pinned_backfill_job_concurrently_with_heartbeat, }; pub(crate) use coverage_facts::{covered_block_interval, merged_covered_block_segments}; #[cfg(test)] diff --git a/apps/indexer/src/main/backfill/concurrent_execution.rs b/apps/indexer/src/main/backfill/concurrent_execution.rs index ce3a8f4a..fd2b9bec 100644 --- a/apps/indexer/src/main/backfill/concurrent_execution.rs +++ b/apps/indexer/src/main/backfill/concurrent_execution.rs @@ -7,6 +7,7 @@ use tokio::task::JoinSet; use tracing::info; use crate::provider::ChainProvider; +use crate::run::startup_heartbeat::StartupHeartbeat; use super::{ BackfillJobRunConfig, BackfillJobRunOutcome, BackfillTopicPlan, CoinbaseSqlBackfillConfig, @@ -17,17 +18,61 @@ use super::{ create_hash_pinned_backfill_job_with_ranges, effective_coinbase_sql_adapter_sync_mode, ensure_coinbase_sql_registry_range_start_is_replay_safe, refreshed_backfill_lease_expires_at, run_reserved_coinbase_sql_backfill_range, - run_reserved_hash_pinned_backfill_range, validate_hash_pinned_chunk_blocks, + run_reserved_hash_pinned_backfill_range_with_progress, validate_hash_pinned_chunk_blocks, }, }; pub(crate) async fn run_resumable_hash_pinned_backfill_job_concurrently( + pool: &sqlx::PgPool, + source_plan: &WatchedSourceSelectorPlan, + provider: &ChainProvider, + config: BackfillJobRunConfig, + ranges: Vec, + worker_count: usize, +) -> Result { + run_resumable_hash_pinned_backfill_job_concurrently_inner( + pool, + source_plan, + provider, + config, + ranges, + worker_count, + None, + ) + .await +} + +#[expect(clippy::too_many_arguments)] +pub(crate) async fn run_resumable_hash_pinned_backfill_job_concurrently_with_heartbeat( + pool: &sqlx::PgPool, + source_plan: &WatchedSourceSelectorPlan, + provider: &ChainProvider, + config: BackfillJobRunConfig, + ranges: Vec, + worker_count: usize, + heartbeat: &mut StartupHeartbeat, + heartbeat_chain_ids: &[String], +) -> Result { + run_resumable_hash_pinned_backfill_job_concurrently_inner( + pool, + source_plan, + provider, + config, + ranges, + worker_count, + Some((heartbeat, heartbeat_chain_ids)), + ) + .await +} + +async fn run_resumable_hash_pinned_backfill_job_concurrently_inner( pool: &sqlx::PgPool, source_plan: &WatchedSourceSelectorPlan, provider: &ChainProvider, mut config: BackfillJobRunConfig, ranges: Vec, worker_count: usize, + mut heartbeat: Option<(&mut StartupHeartbeat, &[String])>, ) -> Result { if worker_count == 0 { bail!("hash-pinned backfill worker count must be positive"); @@ -71,11 +116,13 @@ pub(crate) async fn run_resumable_hash_pinned_backfill_job_concurrently( ); let mut workers = JoinSet::new(); + let (progress_tx, mut progress_rx) = tokio::sync::mpsc::unbounded_channel(); let backfill_job_id = record.job.backfill_job_id; for worker_index in 0..active_worker_count { let pool = pool.clone(); let source_plan = source_plan.clone(); let provider = provider.clone(); + let progress_tx = progress_tx.clone(); let mut worker_config = config.clone(); worker_config.lease_owner = format!("{}:worker-{worker_index}", config.lease_owner); worker_config.lease_token = format!("{}:worker-{worker_index}", config.lease_token); @@ -97,13 +144,14 @@ pub(crate) async fn run_resumable_hash_pinned_backfill_job_concurrently( }; outcome.reserved_range_count += 1; - run_reserved_hash_pinned_backfill_range( + run_reserved_hash_pinned_backfill_range_with_progress( &pool, &source_plan, &provider, &worker_config, &reserved_range, &mut outcome, + &progress_tx, ) .await?; outcome.completed_range_count += 1; @@ -112,8 +160,25 @@ pub(crate) async fn run_resumable_hash_pinned_backfill_job_concurrently( Ok::<_, anyhow::Error>(outcome) }); } + drop(progress_tx); - while let Some(result) = workers.join_next().await { + let mut progress_open = true; + while !workers.is_empty() { + let result = tokio::select! { + progress = progress_rx.recv(), if progress_open => { + match progress { + Some(()) => { + if let Some((heartbeat, chain_ids)) = heartbeat.as_mut() { + heartbeat.record_if_due(pool, chain_ids).await?; + } + } + None => progress_open = false, + } + continue; + } + result = workers.join_next() => result + .expect("hash-pinned backfill worker set must be non-empty"), + }; let worker_outcome = match result { Ok(Ok(outcome)) => outcome, Ok(Err(error)) => { diff --git a/apps/indexer/src/main/backfill/reservation_execution.rs b/apps/indexer/src/main/backfill/reservation_execution.rs index affa728b..d0b360ab 100644 --- a/apps/indexer/src/main/backfill/reservation_execution.rs +++ b/apps/indexer/src/main/backfill/reservation_execution.rs @@ -8,6 +8,8 @@ mod generic_topic_identity; mod lease_heartbeat; #[path = "reservation_execution/scan_all.rs"] mod scan_all; +#[path = "reservation_execution/startup_progress.rs"] +mod startup_progress; #[cfg(test)] #[path = "reservation_execution/tests.rs"] mod tests; @@ -20,7 +22,6 @@ use bigname_storage::{ create_generation_scoped_backfill_job, load_backfill_job, reserve_backfill_range, }; use serde_json::{Value, json}; -use sqlx::types::time::OffsetDateTime; use tracing::info; use crate::{ @@ -48,6 +49,7 @@ pub(crate) use coinbase_sql_execution::{ run_reserved_coinbase_sql_backfill_range, run_resumable_coinbase_sql_backfill_job, }; pub(super) use lease_heartbeat::{ + backfill_lease_duration_secs, refreshed_backfill_lease_expires_at, run_with_backfill_lease_heartbeat, validate_hash_pinned_chunk_blocks, }; pub(crate) use scan_all::effective_hash_pinned_adapter_sync_mode; @@ -409,6 +411,48 @@ pub(super) async fn run_reserved_hash_pinned_backfill_range( config: &BackfillJobRunConfig, reserved_range: &BackfillRange, aggregate: &mut BackfillJobRunOutcome, +) -> Result<()> { + run_reserved_hash_pinned_backfill_range_inner( + pool, + source_plan, + provider, + config, + reserved_range, + aggregate, + None, + ) + .await +} + +pub(super) async fn run_reserved_hash_pinned_backfill_range_with_progress( + pool: &sqlx::PgPool, + source_plan: &WatchedSourceSelectorPlan, + provider: &(impl ChainProviderOps + ?Sized), + config: &BackfillJobRunConfig, + reserved_range: &BackfillRange, + aggregate: &mut BackfillJobRunOutcome, + progress: &tokio::sync::mpsc::UnboundedSender<()>, +) -> Result<()> { + run_reserved_hash_pinned_backfill_range_inner( + pool, + source_plan, + provider, + config, + reserved_range, + aggregate, + Some(progress), + ) + .await +} + +async fn run_reserved_hash_pinned_backfill_range_inner( + pool: &sqlx::PgPool, + source_plan: &WatchedSourceSelectorPlan, + provider: &(impl ChainProviderOps + ?Sized), + config: &BackfillJobRunConfig, + reserved_range: &BackfillRange, + aggregate: &mut BackfillJobRunOutcome, + progress: Option<&tokio::sync::mpsc::UnboundedSender<()>>, ) -> Result<()> { let mut active_range = reserved_range.clone(); let mut block_number = active_range @@ -446,45 +490,50 @@ pub(super) async fn run_reserved_hash_pinned_backfill_range( .unwrap_or(active_range.range_end_block_number) .min(active_range.range_end_block_number); let chunk_range = BackfillBlockRange::new(block_number, chunk_end)?; - let selected_target_addresses_for_chunk = scan_all::chunk_addresses_for_plan( - source_plan, - &mut selected_target_range_cursor, - chunk_range, - ); - let chunk_outcome = match run_with_backfill_lease_heartbeat( - pool, - &active_range, - config, - run_hash_pinned_backfill_range( - pool, + let progress_ranges = + startup_progress::heartbeat_progress_ranges(chunk_range, progress.is_some())?; + for progress_range in progress_ranges { + let selected_target_addresses = scan_all::chunk_addresses_for_plan( source_plan, - &selected_target_index, - &selected_target_addresses_for_chunk, - provider, - chunk_range, - canonicality_evidence.clone(), - config.adapter_sync_mode, - config.header_audit_mode, - ), - ) - .await - { - Ok(outcome) => outcome, - Err(error) => { - return Err(record_reserved_range_failure(ReservedRangeFailure { + &mut selected_target_range_cursor, + progress_range, + ); + let outcome = run_with_backfill_lease_heartbeat( + pool, + &active_range, + config, + run_hash_pinned_backfill_range( pool, - reserved_range: &active_range, - config, - failure_reason: "hash-pinned backfill failed", - block_number: Some(block_number), - attempted_range: Some(chunk_range), - phase: "hash_pinned_intake", - error, - }) - .await); + source_plan, + &selected_target_index, + &selected_target_addresses, + provider, + progress_range, + canonicality_evidence.clone(), + config.adapter_sync_mode, + config.header_audit_mode, + ), + ) + .await + .map_err(|error| ReservedRangeFailure { + pool, + reserved_range: &active_range, + config, + failure_reason: "hash-pinned backfill failed", + block_number: Some(progress_range.from_block), + attempted_range: Some(progress_range), + phase: "hash_pinned_intake", + error, + }); + let outcome = match outcome { + Ok(outcome) => outcome, + Err(failure) => return Err(record_reserved_range_failure(failure).await), + }; + aggregate.add_range_outcome(&outcome); + if let Some(progress) = progress { + let _ = progress.send(()); } - }; - aggregate.add_range_outcome(&chunk_outcome); + } active_range = match advance_backfill_range( pool, @@ -509,7 +558,6 @@ pub(super) async fn run_reserved_hash_pinned_backfill_range( .await); } }; - if chunk_end == active_range.range_end_block_number { break; } @@ -528,21 +576,3 @@ pub(super) async fn run_reserved_hash_pinned_backfill_range( ) .await } -pub(super) fn backfill_lease_duration_secs(lease_expires_at: OffsetDateTime) -> Result { - let duration_secs = lease_expires_at - .unix_timestamp() - .checked_sub(OffsetDateTime::now_utc().unix_timestamp()) - .context("backfill lease duration timestamp underflowed")?; - if duration_secs <= 0 { - bail!("lease_expires_at must be in the future"); - } - Ok(duration_secs) -} -pub(super) fn refreshed_backfill_lease_expires_at(duration_secs: i64) -> Result { - let deadline = OffsetDateTime::now_utc() - .unix_timestamp() - .checked_add(duration_secs) - .context("backfill lease expiry timestamp overflowed while refreshing range lease")?; - OffsetDateTime::from_unix_timestamp(deadline) - .context("refreshed backfill lease expiry timestamp is out of range") -} diff --git a/apps/indexer/src/main/backfill/reservation_execution/lease_heartbeat.rs b/apps/indexer/src/main/backfill/reservation_execution/lease_heartbeat.rs index c658fac7..feca773a 100644 --- a/apps/indexer/src/main/backfill/reservation_execution/lease_heartbeat.rs +++ b/apps/indexer/src/main/backfill/reservation_execution/lease_heartbeat.rs @@ -2,6 +2,7 @@ use std::{future::Future, time::Duration}; use anyhow::{Context, Result, bail}; use bigname_storage::{BackfillRange, advance_backfill_range}; +use sqlx::types::time::OffsetDateTime; use crate::backfill::BackfillJobRunConfig; @@ -69,3 +70,23 @@ pub(crate) fn validate_hash_pinned_chunk_blocks(chunk_blocks: i64) -> Result<()> Ok(()) } + +pub(crate) fn backfill_lease_duration_secs(lease_expires_at: OffsetDateTime) -> Result { + let duration_secs = lease_expires_at + .unix_timestamp() + .checked_sub(OffsetDateTime::now_utc().unix_timestamp()) + .context("backfill lease duration timestamp underflowed")?; + if duration_secs <= 0 { + bail!("lease_expires_at must be in the future"); + } + Ok(duration_secs) +} + +pub(crate) fn refreshed_backfill_lease_expires_at(duration_secs: i64) -> Result { + let deadline = OffsetDateTime::now_utc() + .unix_timestamp() + .checked_add(duration_secs) + .context("backfill lease expiry timestamp overflowed while refreshing range lease")?; + OffsetDateTime::from_unix_timestamp(deadline) + .context("refreshed backfill lease expiry timestamp is out of range") +} diff --git a/apps/indexer/src/main/backfill/reservation_execution/startup_progress.rs b/apps/indexer/src/main/backfill/reservation_execution/startup_progress.rs new file mode 100644 index 00000000..d3cf57b7 --- /dev/null +++ b/apps/indexer/src/main/backfill/reservation_execution/startup_progress.rs @@ -0,0 +1,82 @@ +use anyhow::Result; + +use crate::backfill::BackfillBlockRange; + +const STARTUP_HEARTBEAT_PROGRESS_BLOCKS: i64 = 32; + +pub(super) fn heartbeat_progress_ranges( + range: BackfillBlockRange, + progress_requested: bool, +) -> Result> { + if !progress_requested { + return Ok(vec![range]); + } + + let mut ranges = Vec::new(); + let mut from_block = range.from_block; + loop { + let to_block = from_block + .checked_add(STARTUP_HEARTBEAT_PROGRESS_BLOCKS - 1) + .unwrap_or(range.to_block) + .min(range.to_block); + ranges.push(BackfillBlockRange::new(from_block, to_block)?); + if to_block == range.to_block { + return Ok(ranges); + } + from_block = to_block + .checked_add(1) + .expect("non-terminal progress range end cannot overflow"); + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::backfill::DEFAULT_HASH_PINNED_BACKFILL_CHUNK_BLOCKS; + + #[test] + fn startup_heartbeat_progress_splits_the_default_hash_pinned_chunk() -> Result<()> { + let configured_chunk = + BackfillBlockRange::new(1, DEFAULT_HASH_PINNED_BACKFILL_CHUNK_BLOCKS)?; + + let progress_ranges = heartbeat_progress_ranges(configured_chunk, true)?; + + assert!( + progress_ranges.len() > 1, + "startup liveness must advance before a full configured chunk completes" + ); + assert_eq!( + progress_ranges.first().map(|range| range.from_block), + Some(1) + ); + assert!( + progress_ranges.iter().all(|range| { + range.to_block - range.from_block < STARTUP_HEARTBEAT_PROGRESS_BLOCKS + }), + "startup progress units must stay within the documented 32-block bound" + ); + assert_eq!( + progress_ranges.last().map(|range| range.to_block), + Some(DEFAULT_HASH_PINNED_BACKFILL_CHUNK_BLOCKS) + ); + assert!( + progress_ranges + .windows(2) + .all(|ranges| ranges[0].to_block + 1 == ranges[1].from_block), + "progress ranges must cover the configured chunk contiguously" + ); + Ok(()) + } + + #[test] + fn ordinary_backfill_preserves_the_configured_chunk() -> Result<()> { + let configured_chunk = + BackfillBlockRange::new(1, DEFAULT_HASH_PINNED_BACKFILL_CHUNK_BLOCKS)?; + + assert_eq!( + heartbeat_progress_ranges(configured_chunk, false)?, + vec![configured_chunk] + ); + Ok(()) + } +} diff --git a/apps/indexer/src/main/bootstrap_backfill.rs b/apps/indexer/src/main/bootstrap_backfill.rs index ebc13c54..16684e7f 100644 --- a/apps/indexer/src/main/bootstrap_backfill.rs +++ b/apps/indexer/src/main/bootstrap_backfill.rs @@ -1,8 +1,8 @@ -use std::{collections::BTreeMap, path::Path}; +use std::path::Path; use anyhow::{Context, Result}; use bigname_manifests::{ - ManifestBootstrapSkippedTarget, load_ens_v2_authoritative_discovery_bootstrap_targets, + load_ens_v2_authoritative_discovery_bootstrap_targets, load_ens_v2_retained_history_recovery_targets, load_manifest_declared_bootstrap_targets, load_manifest_skipped_bootstrap_targets, }; @@ -12,20 +12,24 @@ use crate::{ backfill::{ BackfillAdapterSyncMode, BackfillBlockRange, backfill_job_source_identity_payload, hash_pinned_backfill_range_specs, run_resumable_hash_pinned_backfill_job_concurrently, + run_resumable_hash_pinned_backfill_job_concurrently_with_heartbeat, }, backfill_lease_expires_at, default_backfill_lease_owner, deployment_profile_from_manifest_root, generated_backfill_lease_token, - provider::{ChainProviderOps, ProviderBlock, ProviderRegistry}, + provider::{ChainProviderOps, ProviderRegistry}, reconciliation::{ HeaderAuditMode, RawFactNormalizedEventReplayRequest, RawFactNormalizedEventReplaySelection, log_raw_fact_normalized_event_replay_outcome, replay_raw_fact_normalized_events, }, + run::startup_heartbeat::StartupHeartbeat, runtime::{IntakeChainTask, validate_provider_registry_for_intake_tasks}, }; #[path = "bootstrap_backfill/checkpoints.rs"] mod checkpoints; +#[path = "bootstrap_backfill/entrypoints.rs"] +mod entrypoints; #[path = "bootstrap_backfill/identity.rs"] mod identity; #[path = "bootstrap_backfill/planning.rs"] @@ -37,6 +41,11 @@ use checkpoints::{ bootstrap_segment_target_ids, load_bootstrap_segment_checkpoint, load_bootstrap_target_checkpoint, }; +#[cfg(test)] +pub(crate) use entrypoints::run_startup_bootstrap_backfills; +pub(crate) use entrypoints::{ + BootstrapBackfillOutcome, run_startup_bootstrap_backfills_with_heartbeat, +}; pub(crate) use identity::bootstrap_backfill_idempotency_key; use identity::{ partitioned_bootstrap_backfill_idempotency_key, replay_source_scope_from_source_plan, @@ -63,50 +72,8 @@ pub(crate) use recovery::{ const BOOTSTRAP_BACKFILL_LEASE_DURATION_SECS: u64 = 300; pub(crate) const DEFAULT_BOOTSTRAP_BACKFILL_WORKERS: usize = 0; pub(crate) const DEFAULT_BOOTSTRAP_BACKFILL_RANGE_BLOCKS: i64 = 50_000; -#[derive(Clone, Debug, Default, Eq, PartialEq)] -pub(crate) struct BootstrapBackfillOutcome { - pub(crate) latched_finalized_heads: BTreeMap, - pub(crate) active_chain_count: usize, - pub(crate) provider_configured_chain_count: usize, - pub(crate) missing_provider_chain_count: usize, - pub(crate) eligible_target_count: usize, - pub(crate) skipped_unknown_start_target_count: usize, - pub(crate) skipped_unknown_start_targets: Vec, - pub(crate) drained_job_count: usize, - pub(crate) skipped_future_target_count: usize, - pub(crate) reserved_range_count: usize, - pub(crate) completed_range_count: usize, - pub(crate) resolved_block_count: usize, - pub(crate) raw_block_count: usize, - pub(crate) raw_transaction_count: usize, - pub(crate) raw_receipt_count: usize, - pub(crate) raw_log_count: usize, - pub(crate) raw_code_hash_count: usize, - pub(crate) normalized_replay_job_count: usize, - pub(crate) normalized_replay_synced_count: usize, - pub(crate) normalized_replay_inserted_count: usize, - pub(crate) requested_worker_count: usize, - pub(crate) effective_worker_count: usize, - pub(crate) range_partition_block_count: i64, -} - -impl BootstrapBackfillOutcome { - fn add_job(&mut self, outcome: &crate::backfill::BackfillJobRunOutcome) { - self.drained_job_count += 1; - self.reserved_range_count += outcome.reserved_range_count; - self.completed_range_count += outcome.completed_range_count; - self.resolved_block_count += outcome.resolved_block_count; - self.raw_block_count += outcome.raw_block_count; - self.raw_transaction_count += outcome.raw_transaction_count; - self.raw_receipt_count += outcome.raw_receipt_count; - self.raw_log_count += outcome.raw_log_count; - self.raw_code_hash_count += outcome.raw_code_hash_count; - } -} - -// Startup orchestration keeps provider, replay, audit, and worker settings explicit. #[expect(clippy::too_many_arguments)] -pub(crate) async fn run_startup_bootstrap_backfills( +async fn run_startup_bootstrap_backfills_inner( pool: &sqlx::PgPool, manifests_root: &Path, intake_chain_tasks: &[IntakeChainTask], @@ -117,8 +84,14 @@ pub(crate) async fn run_startup_bootstrap_backfills( header_audit_mode: HeaderAuditMode, bootstrap_backfill_workers: usize, bootstrap_backfill_range_blocks: i64, + mut heartbeat: Option<&mut StartupHeartbeat>, ) -> Result { validate_provider_registry_for_intake_tasks(intake_chain_tasks, provider_registry)?; + let heartbeat_chain_ids = intake_chain_tasks + .iter() + .map(|task| task.chain.clone()) + .collect::>(); + record_bootstrap_progress(pool, &mut heartbeat, &heartbeat_chain_ids).await?; let backfill_adapter_sync_mode = adapter_sync_mode.startup_hash_pinned_backfill_mode(); let deployment_profile = deployment_profile_from_manifest_root(manifests_root); let lease_owner = format!("{}:bootstrap-backfill", default_backfill_lease_owner()); @@ -487,15 +460,29 @@ pub(crate) async fn run_startup_bootstrap_backfills( header_audit_mode, }; - let job_outcome = run_resumable_hash_pinned_backfill_job_concurrently( - pool, - &source_plan, - provider, - config, - range_specs, - effective_worker_count, - ) - .await?; + let job_outcome = if let Some(heartbeat) = heartbeat.as_deref_mut() { + run_resumable_hash_pinned_backfill_job_concurrently_with_heartbeat( + pool, + &source_plan, + provider, + config, + range_specs, + effective_worker_count, + heartbeat, + &heartbeat_chain_ids, + ) + .await? + } else { + run_resumable_hash_pinned_backfill_job_concurrently( + pool, + &source_plan, + provider, + config, + range_specs, + effective_worker_count, + ) + .await? + }; outcome.add_job(&job_outcome); if replay_completed_raw_ranges && job_outcome.raw_log_count > 0 { let replay_outcome = replay_raw_fact_normalized_events( @@ -528,6 +515,7 @@ pub(crate) async fn run_startup_bootstrap_backfills( outcome.normalized_replay_inserted_count += replay_outcome.normalized_event_inserted_count; } + record_bootstrap_progress(pool, &mut heartbeat, &heartbeat_chain_ids).await?; } let pass_status = finish_bootstrap_convergence_pass( @@ -596,3 +584,14 @@ pub(crate) async fn run_startup_bootstrap_backfills( Ok(outcome) } + +async fn record_bootstrap_progress( + pool: &sqlx::PgPool, + heartbeat: &mut Option<&mut StartupHeartbeat>, + chain_ids: &[String], +) -> Result<()> { + if let Some(heartbeat) = heartbeat.as_deref_mut() { + heartbeat.record_if_due(pool, chain_ids).await?; + } + Ok(()) +} diff --git a/apps/indexer/src/main/bootstrap_backfill/entrypoints.rs b/apps/indexer/src/main/bootstrap_backfill/entrypoints.rs new file mode 100644 index 00000000..8c0a3c91 --- /dev/null +++ b/apps/indexer/src/main/bootstrap_backfill/entrypoints.rs @@ -0,0 +1,116 @@ +use std::{collections::BTreeMap, path::Path}; + +use anyhow::Result; +use bigname_manifests::ManifestBootstrapSkippedTarget; + +use crate::{ + backfill::BackfillAdapterSyncMode, + provider::{ProviderBlock, ProviderRegistry}, + reconciliation::HeaderAuditMode, + run::startup_heartbeat::StartupHeartbeat, + runtime::IntakeChainTask, +}; + +use super::run_startup_bootstrap_backfills_inner; + +#[derive(Clone, Debug, Default, Eq, PartialEq)] +pub(crate) struct BootstrapBackfillOutcome { + pub(crate) latched_finalized_heads: BTreeMap, + pub(crate) active_chain_count: usize, + pub(crate) provider_configured_chain_count: usize, + pub(crate) missing_provider_chain_count: usize, + pub(crate) eligible_target_count: usize, + pub(crate) skipped_unknown_start_target_count: usize, + pub(crate) skipped_unknown_start_targets: Vec, + pub(crate) drained_job_count: usize, + pub(crate) skipped_future_target_count: usize, + pub(crate) reserved_range_count: usize, + pub(crate) completed_range_count: usize, + pub(crate) resolved_block_count: usize, + pub(crate) raw_block_count: usize, + pub(crate) raw_transaction_count: usize, + pub(crate) raw_receipt_count: usize, + pub(crate) raw_log_count: usize, + pub(crate) raw_code_hash_count: usize, + pub(crate) normalized_replay_job_count: usize, + pub(crate) normalized_replay_synced_count: usize, + pub(crate) normalized_replay_inserted_count: usize, + pub(crate) requested_worker_count: usize, + pub(crate) effective_worker_count: usize, + pub(crate) range_partition_block_count: i64, +} + +impl BootstrapBackfillOutcome { + pub(super) fn add_job(&mut self, outcome: &crate::backfill::BackfillJobRunOutcome) { + self.drained_job_count += 1; + self.reserved_range_count += outcome.reserved_range_count; + self.completed_range_count += outcome.completed_range_count; + self.resolved_block_count += outcome.resolved_block_count; + self.raw_block_count += outcome.raw_block_count; + self.raw_transaction_count += outcome.raw_transaction_count; + self.raw_receipt_count += outcome.raw_receipt_count; + self.raw_log_count += outcome.raw_log_count; + self.raw_code_hash_count += outcome.raw_code_hash_count; + } +} + +// Startup orchestration keeps provider, replay, audit, and worker settings explicit. +#[expect(clippy::too_many_arguments)] +#[cfg(test)] +pub(crate) async fn run_startup_bootstrap_backfills( + pool: &sqlx::PgPool, + manifests_root: &Path, + intake_chain_tasks: &[IntakeChainTask], + provider_registry: &ProviderRegistry, + hash_pinned_chunk_blocks: i64, + adapter_sync_mode: BackfillAdapterSyncMode, + replay_completed_raw_ranges: bool, + header_audit_mode: HeaderAuditMode, + bootstrap_backfill_workers: usize, + bootstrap_backfill_range_blocks: i64, +) -> Result { + run_startup_bootstrap_backfills_inner( + pool, + manifests_root, + intake_chain_tasks, + provider_registry, + hash_pinned_chunk_blocks, + adapter_sync_mode, + replay_completed_raw_ranges, + header_audit_mode, + bootstrap_backfill_workers, + bootstrap_backfill_range_blocks, + None, + ) + .await +} + +#[expect(clippy::too_many_arguments)] +pub(crate) async fn run_startup_bootstrap_backfills_with_heartbeat( + pool: &sqlx::PgPool, + manifests_root: &Path, + intake_chain_tasks: &[IntakeChainTask], + provider_registry: &ProviderRegistry, + hash_pinned_chunk_blocks: i64, + adapter_sync_mode: BackfillAdapterSyncMode, + replay_completed_raw_ranges: bool, + header_audit_mode: HeaderAuditMode, + bootstrap_backfill_workers: usize, + bootstrap_backfill_range_blocks: i64, + heartbeat: &mut StartupHeartbeat, +) -> Result { + run_startup_bootstrap_backfills_inner( + pool, + manifests_root, + intake_chain_tasks, + provider_registry, + hash_pinned_chunk_blocks, + adapter_sync_mode, + replay_completed_raw_ranges, + header_audit_mode, + bootstrap_backfill_workers, + bootstrap_backfill_range_blocks, + Some(heartbeat), + ) + .await +} diff --git a/apps/indexer/src/main/cli.rs b/apps/indexer/src/main/cli.rs index 75fdd4b0..9bb29882 100644 --- a/apps/indexer/src/main/cli.rs +++ b/apps/indexer/src/main/cli.rs @@ -35,6 +35,11 @@ use crate::repair::{ }; use crate::runtime::DEFAULT_STARTUP_DISCOVERY_PAGE_LOGS; +#[path = "cli/healthcheck.rs"] +mod healthcheck_args; + +pub(crate) use healthcheck_args::HealthcheckArgs; + fn parse_positive_usize(value: &str) -> Result { let value = value.parse::().map_err(|error| error.to_string())?; let maximum = usize::try_from(i64::MAX - 1).unwrap_or(usize::MAX); @@ -80,6 +85,8 @@ pub(crate) struct RunArgs { default_value_t = 5_u64 )] pub(crate) poll_interval_secs: u64, + #[arg(long, env = "BIGNAME_HEARTBEAT_INSTANCE_ID")] + pub(crate) heartbeat_instance_id: Option, #[arg( long = "chain-rpc-url", env = "BIGNAME_INDEXER_CHAIN_RPC_URLS", @@ -175,18 +182,6 @@ pub(crate) struct RunArgs { pub(crate) retain_header_audit_fields: bool, } -#[derive(Args, Debug)] -pub(crate) struct HealthcheckArgs { - #[command(flatten)] - pub(crate) database: DatabaseConfig, - #[arg( - long, - env = "BIGNAME_INDEXER_MANIFESTS_ROOT", - default_value = "manifests/mainnet" - )] - pub(crate) manifests_root: PathBuf, -} - #[derive(Args, Debug)] pub(crate) struct BackfillArgs { #[command(flatten)] diff --git a/apps/indexer/src/main/cli/healthcheck.rs b/apps/indexer/src/main/cli/healthcheck.rs new file mode 100644 index 00000000..575659c6 --- /dev/null +++ b/apps/indexer/src/main/cli/healthcheck.rs @@ -0,0 +1,24 @@ +use std::path::PathBuf; + +use bigname_storage::DatabaseConfig; +use clap::Args; + +#[derive(Args, Debug)] +pub(crate) struct HealthcheckArgs { + #[command(flatten)] + pub(crate) database: DatabaseConfig, + #[arg( + long, + env = "BIGNAME_INDEXER_MANIFESTS_ROOT", + default_value = "manifests/mainnet" + )] + pub(crate) manifests_root: PathBuf, + #[arg(long, env = "BIGNAME_HEARTBEAT_INSTANCE_ID")] + pub(crate) heartbeat_instance_id: Option, + #[arg( + long, + env = "BIGNAME_INDEXER_HEARTBEAT_MAX_AGE_SECS", + default_value_t = 20_i64 + )] + pub(crate) heartbeat_max_age_secs: i64, +} diff --git a/apps/indexer/src/main/healthcheck.rs b/apps/indexer/src/main/healthcheck.rs index 124b3b04..32be8e2b 100644 --- a/apps/indexer/src/main/healthcheck.rs +++ b/apps/indexer/src/main/healthcheck.rs @@ -15,6 +15,15 @@ pub(crate) async fn healthcheck(args: HealthcheckArgs) -> Result<()> { let pool = bigname_storage::connect(&args.database).await?; verify_database_reachable(&pool).await?; verify_migrations_current(&pool).await?; + let instance_id = + bigname_storage::resolve_service_instance_id(args.heartbeat_instance_id.as_deref())?; + bigname_storage::ensure_service_loop_heartbeat_recent( + &pool, + bigname_storage::INDEXER_SERVICE_NAME, + &instance_id, + args.heartbeat_max_age_secs, + ) + .await?; println!("ok"); Ok(()) } @@ -153,6 +162,8 @@ mod tests { Command::Healthcheck(args) => { assert_eq!(args.manifests_root, PathBuf::from("manifests/mainnet")); assert_eq!(args.database.max_connections, 10); + assert_eq!(args.heartbeat_instance_id, None); + assert_eq!(args.heartbeat_max_age_secs, 20); } other => panic!("expected healthcheck command, got {other:?}"), } @@ -167,9 +178,38 @@ mod tests { ) .await?; let manifest_root = TestManifestRoot::create()?; + bigname_storage::register_service_loop( + database.pool(), + bigname_storage::INDEXER_SERVICE_NAME, + "indexer-healthcheck-test", + ) + .await?; + bigname_storage::record_service_loop_heartbeat( + database.pool(), + bigname_storage::INDEXER_SERVICE_NAME, + "indexer-healthcheck-test", + &["ethereum-mainnet".to_owned()], + ) + .await?; + assert_eq!( + sqlx::query_scalar::<_, i64>( + r#" + SELECT COUNT(*) + FROM service_loop_heartbeats + WHERE service_name = 'indexer' + AND instance_id = 'indexer-healthcheck-test' + "#, + ) + .fetch_one(database.pool()) + .await?, + 2, + "one batched loop tick must retain process and per-chain heartbeats", + ); let result = healthcheck(HealthcheckArgs { database: database_config(&database)?, manifests_root: manifest_root.path().to_path_buf(), + heartbeat_instance_id: Some("indexer-healthcheck-test".to_owned()), + heartbeat_max_age_secs: 20, }) .await; database.cleanup().await?; @@ -186,6 +226,8 @@ mod tests { let error = healthcheck(HealthcheckArgs { database: database_config(&database)?, manifests_root: manifest_root.path().to_path_buf(), + heartbeat_instance_id: Some("indexer-healthcheck-unmigrated".to_owned()), + heartbeat_max_age_secs: 20, }) .await .expect_err("unmigrated database must fail healthcheck"); @@ -200,6 +242,62 @@ mod tests { Ok(()) } + #[tokio::test] + async fn healthcheck_distinguishes_missing_and_stale_loop_heartbeats() -> Result<()> { + let database = bigname_test_support::TestDatabase::create_migrated( + bigname_test_support::TestDatabaseConfig::new("bigname_indexer_healthcheck_heartbeat"), + &bigname_storage::MIGRATOR, + "failed to apply migrations for indexer heartbeat healthcheck test", + ) + .await?; + let manifest_root = TestManifestRoot::create()?; + let missing_error = healthcheck(HealthcheckArgs { + database: database_config(&database)?, + manifests_root: manifest_root.path().to_path_buf(), + heartbeat_instance_id: Some("missing-indexer".to_owned()), + heartbeat_max_age_secs: 20, + }) + .await + .expect_err("an indexer loop that never started must fail healthcheck"); + assert!( + missing_error.to_string().contains("never started"), + "unexpected error: {missing_error:#}" + ); + + bigname_storage::register_service_loop( + database.pool(), + bigname_storage::INDEXER_SERVICE_NAME, + "stale-indexer", + ) + .await?; + sqlx::query( + r#" + UPDATE service_loop_heartbeats + SET started_at = clock_timestamp() - INTERVAL '2 minutes', + heartbeat_at = clock_timestamp() - INTERVAL '1 minute' + WHERE service_name = 'indexer' + AND instance_id = 'stale-indexer' + "#, + ) + .execute(database.pool()) + .await?; + let stale_error = healthcheck(HealthcheckArgs { + database: database_config(&database)?, + manifests_root: manifest_root.path().to_path_buf(), + heartbeat_instance_id: Some("stale-indexer".to_owned()), + heartbeat_max_age_secs: 20, + }) + .await + .expect_err("a wedged indexer loop must fail healthcheck"); + assert!( + stale_error.to_string().contains("stopped or wedged"), + "unexpected error: {stale_error:#}" + ); + + database.cleanup().await?; + Ok(()) + } + #[tokio::test] async fn healthcheck_rejects_missing_manifest_root() { let missing_root = std::env::temp_dir().join(format!( @@ -209,6 +307,8 @@ mod tests { let error = healthcheck(HealthcheckArgs { database: DatabaseConfig::default(), manifests_root: missing_root, + heartbeat_instance_id: Some("missing-manifests".to_owned()), + heartbeat_max_age_secs: 20, }) .await .expect_err("missing manifest root must fail healthcheck before database access"); diff --git a/apps/indexer/src/main/run.rs b/apps/indexer/src/main/run.rs index 1b29400f..884f921f 100644 --- a/apps/indexer/src/main/run.rs +++ b/apps/indexer/src/main/run.rs @@ -1,9 +1,13 @@ use anyhow::Result; +use tokio::time::Duration; use tracing::info; +#[path = "startup_heartbeat.rs"] +pub(crate) mod startup_heartbeat; + use crate::{ backfill::BackfillAdapterSyncMode, - bootstrap_backfill::run_startup_bootstrap_backfills, + bootstrap_backfill::run_startup_bootstrap_backfills_with_heartbeat, cli::RunArgs, normalized_replay_catchup::{NormalizedReplayCatchupConfig, run_normalized_replay_catchup}, provider::{ChainProviderKind, ProviderRegistry}, @@ -17,13 +21,18 @@ use crate::{ ensure_manifest_root_ready, intake_runtime_state, load_manifest_repository, log_intake_chain_tasks, log_manifest_runtime_state, log_manifest_summary, log_provider_registry, log_watched_chain_plan, manifest_normalized_event_kind_count, - run_poll_loop, sync_discovery_adapter_owned_raw_log_state, sync_intake_chain_tasks, - sync_startup_adapter_owned_raw_log_state, validate_provider_registry_for_intake_tasks, - watched_chain_plan_state, widen_runtime_state_to_live_watch_scope_with_admission_epochs, + run_poll_loop, sync_adapter_owned_raw_log_state_with_heartbeat, + sync_discovery_adapter_owned_raw_log_state_with_heartbeat, sync_intake_chain_tasks, + validate_provider_registry_for_intake_tasks, watched_chain_plan_state, + widen_runtime_state_to_live_watch_scope_with_admission_epochs, }, }; +use startup_heartbeat::StartupHeartbeat; + pub(crate) async fn run(args: RunArgs) -> Result<()> { + let heartbeat_instance_id = + bigname_storage::resolve_service_instance_id(args.heartbeat_instance_id.as_deref())?; let manifest_repository = load_manifest_repository(&args.manifests_root)?; let deployment_profile = deployment_profile_from_manifest_root(&args.manifests_root); let manifest_summary = manifest_repository.summary().clone(); @@ -36,6 +45,16 @@ pub(crate) async fn run(args: RunArgs) -> Result<()> { "bigname-indexer", ) .await?; + bigname_storage::register_service_loop( + &pool, + bigname_storage::INDEXER_SERVICE_NAME, + &heartbeat_instance_id, + ) + .await?; + let mut startup_heartbeat = StartupHeartbeat::new( + heartbeat_instance_id.clone(), + Duration::from_secs(args.poll_interval_secs.max(1)), + ); let adapter_sync_mode = BackfillAdapterSyncMode::parse(&args.hash_pinned_adapter_sync)?; let header_audit_mode = HeaderAuditMode::from_retain_audit_fields(args.retain_header_audit_fields); @@ -46,12 +65,22 @@ pub(crate) async fn run(args: RunArgs) -> Result<()> { run_mode.bootstrap_watch_scope, ) .await?; + let bootstrap_chain_ids = manifest_runtime_state + .watched_chain_plan + .iter() + .map(|chain| chain.chain.clone()) + .collect::>(); + startup_heartbeat + .record(&pool, &bootstrap_chain_ids) + .await?; if run_mode.sync_adapter_before_startup_backfill { - sync_startup_adapter_owned_raw_log_state( + sync_adapter_owned_raw_log_state_with_heartbeat( &pool, &deployment_profile, &manifest_runtime_state.watched_chain_plan, args.startup_discovery_page_logs, + &mut startup_heartbeat, + &bootstrap_chain_ids, ) .await?; } else { @@ -66,13 +95,18 @@ pub(crate) async fn run(args: RunArgs) -> Result<()> { log_watched_chain_plan("startup", &manifest_runtime_state.watched_chain_plan); let intake_chain_tasks = sync_intake_chain_tasks(&pool, &manifest_runtime_state.watched_chain_plan).await?; + let startup_chain_ids = intake_chain_tasks + .iter() + .map(|task| task.chain.clone()) + .collect::>(); + startup_heartbeat.record(&pool, &startup_chain_ids).await?; log_intake_chain_tasks("startup", &intake_chain_tasks); let provider_registry = args.provider_registry()?; validate_provider_registry_for_intake_tasks(&intake_chain_tasks, &provider_registry)?; log_provider_registry("startup", &intake_chain_tasks, &provider_registry); // Automatic normalized catch-up replays bounded chunks after raw bootstrap drains. let replay_completed_startup_raw_ranges = false; - let bootstrap_backfill_outcome = run_startup_bootstrap_backfills( + let bootstrap_backfill_outcome = run_startup_bootstrap_backfills_with_heartbeat( &pool, &args.manifests_root, &intake_chain_tasks, @@ -83,39 +117,19 @@ pub(crate) async fn run(args: RunArgs) -> Result<()> { header_audit_mode, args.bootstrap_backfill_workers, args.bootstrap_backfill_range_blocks, + &mut startup_heartbeat, + ) + .await?; + sync_post_bootstrap_adapter_state_with_heartbeat( + &pool, + &deployment_profile, + &manifest_runtime_state.watched_chain_plan, + args.startup_discovery_page_logs, + &run_mode, + &mut startup_heartbeat, + &startup_chain_ids, ) .await?; - if run_mode.sync_adapter_after_startup_backfill { - info!( - service = "indexer", - adapter_sync_mode = adapter_sync_mode.as_str(), - effective_backfill_adapter_sync_mode = - run_mode.startup_backfill_adapter_sync_mode.as_str(), - "startup bootstrap backfill drained; syncing adapter-owned raw-log state before live polling" - ); - sync_startup_adapter_owned_raw_log_state( - &pool, - &deployment_profile, - &manifest_runtime_state.watched_chain_plan, - args.startup_discovery_page_logs, - ) - .await?; - } else if run_mode.sync_discovery_adapters_after_startup_backfill { - info!( - service = "indexer", - adapter_sync_mode = adapter_sync_mode.as_str(), - effective_backfill_adapter_sync_mode = - run_mode.startup_backfill_adapter_sync_mode.as_str(), - "startup bootstrap backfill drained; syncing only discovery-materializing adapter families before the live-plan widen" - ); - sync_discovery_adapter_owned_raw_log_state( - &pool, - &deployment_profile, - &manifest_runtime_state.watched_chain_plan, - args.startup_discovery_page_logs, - ) - .await?; - } // Bootstrap backfill has drained, so the narrow bootstrap scope has served its purpose. Widen // before spawning replay catch-up: both reconcile `contract_instance_addresses`, and the widen @@ -129,6 +143,11 @@ pub(crate) async fn run(args: RunArgs) -> Result<()> { &provider_registry, ) .await?; + let live_chain_ids = intake_chain_tasks + .iter() + .map(|task| task.chain.clone()) + .collect::>(); + startup_heartbeat.record(&pool, &live_chain_ids).await?; if adapter_sync_mode != BackfillAdapterSyncMode::RawOnly && !run_mode.normalized_replay_catchup_enabled { @@ -268,12 +287,14 @@ pub(crate) async fn run(args: RunArgs) -> Result<()> { run_poll_loop( &pool, + &mut startup_heartbeat, args.manifests_root, manifest_runtime_state, intake_chain_tasks, watched_plan_admission_epochs, &provider_registry, args.poll_interval_secs, + args.startup_discovery_page_logs, run_mode.live_watch_scope, run_mode.broad_runtime_refresh_enabled, run_mode.live_poll_adapter_sync_enabled, @@ -293,6 +314,53 @@ pub(crate) async fn run(args: RunArgs) -> Result<()> { .await } +async fn sync_post_bootstrap_adapter_state_with_heartbeat( + pool: &sqlx::PgPool, + deployment_profile: &str, + watched_chain_plan: &[bigname_manifests::WatchedChainPlan], + startup_discovery_page_logs: usize, + run_mode: &IndexerRunMode, + heartbeat: &mut StartupHeartbeat, + heartbeat_chain_ids: &[String], +) -> Result<()> { + if run_mode.sync_adapter_after_startup_backfill { + info!( + service = "indexer", + adapter_sync_mode = run_mode.adapter_sync_mode.as_str(), + effective_backfill_adapter_sync_mode = + run_mode.startup_backfill_adapter_sync_mode.as_str(), + "startup bootstrap backfill drained; syncing adapter-owned raw-log state before live polling" + ); + sync_adapter_owned_raw_log_state_with_heartbeat( + pool, + deployment_profile, + watched_chain_plan, + startup_discovery_page_logs, + heartbeat, + heartbeat_chain_ids, + ) + .await?; + } else if run_mode.sync_discovery_adapters_after_startup_backfill { + info!( + service = "indexer", + adapter_sync_mode = run_mode.adapter_sync_mode.as_str(), + effective_backfill_adapter_sync_mode = + run_mode.startup_backfill_adapter_sync_mode.as_str(), + "startup bootstrap backfill drained; syncing only discovery-materializing adapter families before the live-plan widen" + ); + sync_discovery_adapter_owned_raw_log_state_with_heartbeat( + pool, + deployment_profile, + watched_chain_plan, + startup_discovery_page_logs, + heartbeat, + heartbeat_chain_ids, + ) + .await?; + } + Ok(()) +} + async fn widen_to_live_watch_scope( pool: &sqlx::PgPool, run_mode: &IndexerRunMode, @@ -334,3 +402,76 @@ async fn widen_to_live_watch_scope( watched_plan_admission_epochs, )) } + +#[cfg(test)] +mod tests { + use anyhow::Context; + use bigname_test_support::{TestDatabase, TestDatabaseConfig}; + + use super::*; + + #[tokio::test] + async fn post_bootstrap_discovery_sync_records_page_progress() -> Result<()> { + let database = TestDatabase::create_migrated( + TestDatabaseConfig::new("bigname_indexer_post_bootstrap_heartbeat_test"), + &bigname_storage::MIGRATOR, + "failed to migrate post-bootstrap heartbeat test database", + ) + .await?; + let instance_id = "post-bootstrap-page-progress-test"; + bigname_storage::register_service_loop( + database.pool(), + bigname_storage::INDEXER_SERVICE_NAME, + instance_id, + ) + .await?; + sqlx::query( + r#" + UPDATE service_loop_heartbeats + SET started_at = clock_timestamp() - INTERVAL '2 minutes', + heartbeat_at = clock_timestamp() - INTERVAL '1 minute' + WHERE service_name = 'indexer' + AND instance_id = $1 + "#, + ) + .bind(instance_id) + .execute(database.pool()) + .await?; + + let watched_chain_plan = vec![bigname_manifests::WatchedChainPlan { + chain: "ethereum-mainnet".to_owned(), + addresses: Vec::new(), + manifest_root_entry_count: 0, + manifest_contract_entry_count: 0, + discovery_edge_entry_count: 0, + }]; + let chain_ids = vec!["ethereum-mainnet".to_owned()]; + let mut heartbeat = StartupHeartbeat::new(instance_id.to_owned(), Duration::ZERO); + sync_post_bootstrap_adapter_state_with_heartbeat( + database.pool(), + "test", + &watched_chain_plan, + 1, + &IndexerRunMode::new(BackfillAdapterSyncMode::Auto, false), + &mut heartbeat, + &chain_ids, + ) + .await?; + assert!( + heartbeat.adapter_progress_count() > 0, + "the production post-bootstrap branch must forward page progress" + ); + let heartbeat = bigname_storage::load_service_loop_heartbeat( + database.pool(), + bigname_storage::INDEXER_SERVICE_NAME, + instance_id, + ) + .await? + .context("post-bootstrap sync must retain its registered heartbeat")?; + assert!( + heartbeat.age_seconds <= 1, + "page progress must refresh the post-bootstrap heartbeat" + ); + database.cleanup().await + } +} diff --git a/apps/indexer/src/main/runtime.rs b/apps/indexer/src/main/runtime.rs index dc085fc3..91e8237a 100644 --- a/apps/indexer/src/main/runtime.rs +++ b/apps/indexer/src/main/runtime.rs @@ -19,10 +19,13 @@ pub(crate) const BUILD_SHA: &str = match option_env!("BIGNAME_BUILD_SHA") { None => "unknown", }; +#[cfg(test)] +pub(crate) use adapter_sync::sync_discovery_adapter_owned_raw_log_state; #[allow(unused_imports)] pub(crate) use adapter_sync::{ DEFAULT_STARTUP_DISCOVERY_PAGE_LOGS, sync_adapter_owned_raw_log_state, - sync_discovery_adapter_owned_raw_log_state, sync_startup_adapter_owned_raw_log_state, + sync_adapter_owned_raw_log_state_with_heartbeat, + sync_discovery_adapter_owned_raw_log_state_with_heartbeat, }; #[allow(unused_imports)] pub(crate) use intake::{ diff --git a/apps/indexer/src/main/runtime/adapter_sync.rs b/apps/indexer/src/main/runtime/adapter_sync.rs index 0c0ee8b5..476d56c3 100644 --- a/apps/indexer/src/main/runtime/adapter_sync.rs +++ b/apps/indexer/src/main/runtime/adapter_sync.rs @@ -1,7 +1,10 @@ use anyhow::{Context, Result}; use bigname_manifests::WatchedChainPlan; -use crate::resolver_profile_convergence::journal_resolver_profile_authority; +use crate::{ + resolver_profile_convergence::journal_resolver_profile_authority, + run::startup_heartbeat::{StartupAdapterHeartbeat, StartupHeartbeat}, +}; use super::logging::{ log_ens_v1_reverse_claim_sync_summary, log_ens_v1_subregistry_discovery_sync_summary, @@ -20,19 +23,24 @@ pub(crate) async fn sync_adapter_owned_raw_log_state( pool: &sqlx::PgPool, watched_chain_plan: &[WatchedChainPlan], ) -> Result<()> { - sync_adapter_owned_raw_log_state_with_startup_context(pool, watched_chain_plan, None).await + sync_adapter_owned_raw_log_state_with_startup_context(pool, watched_chain_plan, None, None) + .await } -pub(crate) async fn sync_startup_adapter_owned_raw_log_state( +pub(crate) async fn sync_adapter_owned_raw_log_state_with_heartbeat( pool: &sqlx::PgPool, deployment_profile: &str, watched_chain_plan: &[WatchedChainPlan], startup_discovery_page_logs: usize, + heartbeat: &mut StartupHeartbeat, + heartbeat_chain_ids: &[String], ) -> Result<()> { + heartbeat.record(pool, heartbeat_chain_ids).await?; sync_adapter_owned_raw_log_state_with_startup_context( pool, watched_chain_plan, Some((deployment_profile, startup_discovery_page_logs)), + Some((heartbeat, heartbeat_chain_ids)), ) .await } @@ -41,7 +49,9 @@ async fn sync_adapter_owned_raw_log_state_with_startup_context( pool: &sqlx::PgPool, watched_chain_plan: &[WatchedChainPlan], startup_context: Option<(&str, usize)>, + mut startup_heartbeat: Option<(&mut StartupHeartbeat, &[String])>, ) -> Result<()> { + record_startup_sync_progress(pool, &mut startup_heartbeat).await?; // Broad startup/timer passes also recover any prior discovery transaction // that committed before its caller could journal the epoch change. journal_resolver_profile_authority(pool).await?; @@ -64,17 +74,31 @@ async fn sync_adapter_owned_raw_log_state_with_startup_context( ) })?; log_ens_v1_reverse_claim_sync_summary(&chain.chain, &summary); + record_startup_sync_progress(pool, &mut startup_heartbeat).await?; let summary = match startup_checkpoint.as_ref() { - Some((checkpoint, page_logs)) => { - bigname_adapters::sync_ens_v1_subregistry_discovery_with_startup_checkpoint_and_log_limit( - pool, - &chain.chain, - checkpoint, - *page_logs, - ) - .await - } + Some((checkpoint, page_logs)) => match startup_heartbeat.as_mut() { + Some((heartbeat, chain_ids)) => { + let mut progress = StartupAdapterHeartbeat::new(heartbeat, chain_ids); + bigname_adapters::sync_ens_v1_subregistry_discovery_with_startup_checkpoint_and_log_limit_and_progress( + pool, + &chain.chain, + checkpoint, + *page_logs, + &mut progress, + ) + .await + } + None => { + bigname_adapters::sync_ens_v1_subregistry_discovery_with_startup_checkpoint_and_log_limit( + pool, + &chain.chain, + checkpoint, + *page_logs, + ) + .await + } + }, None => bigname_adapters::sync_ens_v1_subregistry_discovery(pool, &chain.chain).await, } .with_context(|| { @@ -84,17 +108,31 @@ async fn sync_adapter_owned_raw_log_state_with_startup_context( ) })?; log_ens_v1_subregistry_discovery_sync_summary(&chain.chain, &summary); + record_startup_sync_progress(pool, &mut startup_heartbeat).await?; let summary = match startup_checkpoint.as_ref() { - Some((checkpoint, page_logs)) => { - bigname_adapters::sync_ens_v1_unwrapped_authority_with_startup_checkpoint_and_log_limit( - pool, - &chain.chain, - checkpoint, - *page_logs, - ) - .await - } + Some((checkpoint, page_logs)) => match startup_heartbeat.as_mut() { + Some((heartbeat, chain_ids)) => { + let mut progress = StartupAdapterHeartbeat::new(heartbeat, chain_ids); + bigname_adapters::sync_ens_v1_unwrapped_authority_with_startup_checkpoint_and_log_limit_and_progress( + pool, + &chain.chain, + checkpoint, + *page_logs, + &mut progress, + ) + .await + } + None => { + bigname_adapters::sync_ens_v1_unwrapped_authority_with_startup_checkpoint_and_log_limit( + pool, + &chain.chain, + checkpoint, + *page_logs, + ) + .await + } + }, None => bigname_adapters::sync_ens_v1_unwrapped_authority(pool, &chain.chain).await, } .with_context(|| { @@ -104,6 +142,7 @@ async fn sync_adapter_owned_raw_log_state_with_startup_context( ) })?; log_ens_v1_unwrapped_authority_sync_summary(&chain.chain, &summary); + record_startup_sync_progress(pool, &mut startup_heartbeat).await?; let summary = bigname_adapters::sync_ens_v2_registry_resource_surface(pool, &chain.chain) .await @@ -114,6 +153,7 @@ async fn sync_adapter_owned_raw_log_state_with_startup_context( ) })?; log_ens_v2_registry_resource_surface_sync_summary(&chain.chain, &summary); + record_startup_sync_progress(pool, &mut startup_heartbeat).await?; let summary = bigname_adapters::sync_ens_v2_registrar(pool, &chain.chain) .await @@ -124,6 +164,7 @@ async fn sync_adapter_owned_raw_log_state_with_startup_context( ) })?; log_ens_v2_registrar_sync_summary(&chain.chain, &summary); + record_startup_sync_progress(pool, &mut startup_heartbeat).await?; let summary = bigname_adapters::sync_ens_v2_resolver(pool, &chain.chain) .await @@ -134,6 +175,7 @@ async fn sync_adapter_owned_raw_log_state_with_startup_context( ) })?; log_ens_v2_resolver_sync_summary(&chain.chain, &summary); + record_startup_sync_progress(pool, &mut startup_heartbeat).await?; let summary = bigname_adapters::sync_ens_v2_permissions(pool, &chain.chain) .await @@ -144,6 +186,7 @@ async fn sync_adapter_owned_raw_log_state_with_startup_context( ) })?; log_ens_v2_permissions_sync_summary(&chain.chain, &summary); + record_startup_sync_progress(pool, &mut startup_heartbeat).await?; if let Some((checkpoint, _)) = startup_checkpoint { completed_startup_checkpoints.push((chain.chain.clone(), checkpoint)); @@ -152,38 +195,92 @@ async fn sync_adapter_owned_raw_log_state_with_startup_context( journal_resolver_profile_authority(pool).await?; clear_completed_startup_adapter_checkpoints(pool, &completed_startup_checkpoints).await?; + record_startup_sync_progress(pool, &mut startup_heartbeat).await?; Ok(()) } /// Materialize only the discovery edges needed by the post-bootstrap live-plan /// widen. Auto bootstrap stores raw facts without adapter work; replay catch-up /// owns the remaining historical adapter families. +#[cfg(test)] pub(crate) async fn sync_discovery_adapter_owned_raw_log_state( pool: &sqlx::PgPool, deployment_profile: &str, watched_chain_plan: &[WatchedChainPlan], startup_discovery_page_logs: usize, ) -> Result<()> { + sync_discovery_adapter_owned_raw_log_state_inner( + pool, + deployment_profile, + watched_chain_plan, + startup_discovery_page_logs, + None, + ) + .await +} + +pub(crate) async fn sync_discovery_adapter_owned_raw_log_state_with_heartbeat( + pool: &sqlx::PgPool, + deployment_profile: &str, + watched_chain_plan: &[WatchedChainPlan], + startup_discovery_page_logs: usize, + heartbeat: &mut StartupHeartbeat, + heartbeat_chain_ids: &[String], +) -> Result<()> { + heartbeat.record(pool, heartbeat_chain_ids).await?; + sync_discovery_adapter_owned_raw_log_state_inner( + pool, + deployment_profile, + watched_chain_plan, + startup_discovery_page_logs, + Some((heartbeat, heartbeat_chain_ids)), + ) + .await +} + +async fn sync_discovery_adapter_owned_raw_log_state_inner( + pool: &sqlx::PgPool, + deployment_profile: &str, + watched_chain_plan: &[WatchedChainPlan], + startup_discovery_page_logs: usize, + mut startup_heartbeat: Option<(&mut StartupHeartbeat, &[String])>, +) -> Result<()> { + record_startup_sync_progress(pool, &mut startup_heartbeat).await?; journal_resolver_profile_authority(pool).await?; let mut completed_startup_checkpoints = Vec::new(); for chain in watched_chain_plan { let startup_checkpoint = load_startup_adapter_checkpoint_context(pool, deployment_profile, &chain.chain).await?; - let summary = - bigname_adapters::sync_ens_v1_subregistry_discovery_with_startup_checkpoint_and_log_limit( - pool, - &chain.chain, - &startup_checkpoint, - startup_discovery_page_logs, - ) - .await - .with_context(|| { + let summary = match startup_heartbeat.as_mut() { + Some((heartbeat, chain_ids)) => { + let mut progress = StartupAdapterHeartbeat::new(heartbeat, chain_ids); + bigname_adapters::sync_ens_v1_subregistry_discovery_with_startup_checkpoint_and_log_limit_and_progress( + pool, + &chain.chain, + &startup_checkpoint, + startup_discovery_page_logs, + &mut progress, + ) + .await + } + None => { + bigname_adapters::sync_ens_v1_subregistry_discovery_with_startup_checkpoint_and_log_limit( + pool, + &chain.chain, + &startup_checkpoint, + startup_discovery_page_logs, + ) + .await + } + } + .with_context(|| { format!( "failed to sync ENSv1 registry discovery from stored raw logs for chain {}", chain.chain ) })?; log_ens_v1_subregistry_discovery_sync_summary(&chain.chain, &summary); + record_startup_sync_progress(pool, &mut startup_heartbeat).await?; let summary = bigname_adapters::sync_ens_v2_registry_resource_surface(pool, &chain.chain) .await @@ -194,10 +291,22 @@ pub(crate) async fn sync_discovery_adapter_owned_raw_log_state( ) })?; log_ens_v2_registry_resource_surface_sync_summary(&chain.chain, &summary); + record_startup_sync_progress(pool, &mut startup_heartbeat).await?; completed_startup_checkpoints.push((chain.chain.clone(), startup_checkpoint)); } journal_resolver_profile_authority(pool).await?; clear_completed_startup_adapter_checkpoints(pool, &completed_startup_checkpoints).await?; + record_startup_sync_progress(pool, &mut startup_heartbeat).await?; + Ok(()) +} + +async fn record_startup_sync_progress( + pool: &sqlx::PgPool, + startup_heartbeat: &mut Option<(&mut StartupHeartbeat, &[String])>, +) -> Result<()> { + if let Some((heartbeat, chain_ids)) = startup_heartbeat.as_mut() { + heartbeat.record_if_due(pool, chain_ids).await?; + } Ok(()) } diff --git a/apps/indexer/src/main/runtime/poll_loop.rs b/apps/indexer/src/main/runtime/poll_loop.rs index 230ff78c..510ae7e0 100644 --- a/apps/indexer/src/main/runtime/poll_loop.rs +++ b/apps/indexer/src/main/runtime/poll_loop.rs @@ -9,8 +9,9 @@ use crate::reconciliation::{ }; use crate::replay::deployment_profile_from_manifest_root; use crate::resolver_profile_convergence::drain_resolver_profile_input_changes; +use crate::run::startup_heartbeat::StartupHeartbeat; -use super::adapter_sync::sync_adapter_owned_raw_log_state; +use super::adapter_sync::sync_adapter_owned_raw_log_state_with_heartbeat; use super::intake::{ IntakeChainTask, intake_runtime_state, sync_intake_chain_tasks, validate_provider_registry_for_intake_tasks, watched_chain_plan_state, @@ -30,10 +31,9 @@ mod discovery_refresh; #[path = "poll_loop/replay_handoff.rs"] mod replay_handoff; -#[cfg(not(test))] -use discovery_refresh::refresh_discovery_watch_state; #[cfg(test)] pub(crate) use discovery_refresh::refresh_discovery_watch_state; +use discovery_refresh::refresh_discovery_watch_state_with_heartbeat; #[cfg(test)] pub(crate) use replay_handoff::{ ReplayHandoffLatchStatus, install_replay_handoff_before_latch_test_hook, @@ -45,12 +45,14 @@ use replay_handoff::{ #[expect(clippy::too_many_arguments)] pub(crate) async fn run_poll_loop( pool: &sqlx::PgPool, + heartbeat: &mut StartupHeartbeat, manifests_root: PathBuf, mut manifest_runtime_state: ManifestRuntimeState, mut intake_chain_tasks: Vec, initial_watched_plan_admission_epochs: BTreeMap, provider_registry: &ProviderRegistry, poll_interval_secs: u64, + adapter_sync_page_logs: usize, runtime_watch_scope: RuntimeWatchScope, adapter_sync_on_manifest_refresh: bool, adapter_sync_on_live_poll: bool, @@ -80,6 +82,12 @@ pub(crate) async fn run_poll_loop( return Ok(()); } _ = interval.tick() => { + let heartbeat_chains = intake_chain_tasks + .iter() + .map(|task| task.chain.clone()) + .collect::>(); + heartbeat.record(pool, &heartbeat_chains).await?; + match load_manifest_repository(&manifests_root) { Ok(manifest_repository) => { let manifest_summary = manifest_repository.summary().clone(); @@ -125,9 +133,13 @@ pub(crate) async fn run_poll_loop( if adapter_sync_on_manifest_refresh && (manifest_state_changed || watched_plan_changed) - && let Err(error) = sync_adapter_owned_raw_log_state( + && let Err(error) = sync_adapter_owned_raw_log_state_with_heartbeat( pool, + &deployment_profile, &next_manifest_runtime_state.watched_chain_plan, + adapter_sync_page_logs, + heartbeat, + &heartbeat_chains, ) .await { @@ -481,7 +493,11 @@ pub(crate) async fn run_poll_loop( } if discovery_refresh_enabled || effective_adapter_sync_on_live_poll { - refresh_discovery_watch_state( + let heartbeat_chains = intake_chain_tasks + .iter() + .map(|task| task.chain.clone()) + .collect::>(); + refresh_discovery_watch_state_with_heartbeat( pool, provider_registry, &mut manifest_runtime_state, @@ -489,6 +505,10 @@ pub(crate) async fn run_poll_loop( resync_adapter_owned_state_on_discovery_refresh, resolver_profile_convergence_enabled, &mut watched_plan_admission_epochs, + &deployment_profile, + adapter_sync_page_logs, + heartbeat, + &heartbeat_chains, ) .await?; } diff --git a/apps/indexer/src/main/runtime/poll_loop/discovery_refresh.rs b/apps/indexer/src/main/runtime/poll_loop/discovery_refresh.rs index a9b8e975..5f373673 100644 --- a/apps/indexer/src/main/runtime/poll_loop/discovery_refresh.rs +++ b/apps/indexer/src/main/runtime/poll_loop/discovery_refresh.rs @@ -7,8 +7,9 @@ use crate::provider::ProviderRegistry; use crate::resolver_profile_convergence::{ ResolverProfileConvergenceSummary, drain_resolver_profile_input_changes, }; +use crate::run::startup_heartbeat::StartupHeartbeat; -use super::super::adapter_sync::sync_adapter_owned_raw_log_state; +use super::super::adapter_sync::sync_adapter_owned_raw_log_state_with_heartbeat; use super::super::intake::{ IntakeChainTask, intake_runtime_state, validate_provider_registry_for_intake_tasks, watched_chain_plan_state, @@ -47,11 +48,79 @@ pub(crate) async fn refresh_discovery_watch_state( sync_adapter_state_before_refresh: bool, resolver_profile_convergence_enabled: bool, last_admission_epochs: &mut Option>, +) -> Result { + refresh_discovery_watch_state_inner( + pool, + provider_registry, + manifest_runtime_state, + intake_chain_tasks, + sync_adapter_state_before_refresh, + resolver_profile_convergence_enabled, + last_admission_epochs, + None, + ) + .await +} + +#[expect(clippy::too_many_arguments)] +pub(crate) async fn refresh_discovery_watch_state_with_heartbeat( + pool: &sqlx::PgPool, + provider_registry: &ProviderRegistry, + manifest_runtime_state: &mut ManifestRuntimeState, + intake_chain_tasks: &mut Vec, + sync_adapter_state_before_refresh: bool, + resolver_profile_convergence_enabled: bool, + last_admission_epochs: &mut Option>, + deployment_profile: &str, + adapter_sync_page_logs: usize, + heartbeat: &mut StartupHeartbeat, + heartbeat_chain_ids: &[String], +) -> Result { + refresh_discovery_watch_state_inner( + pool, + provider_registry, + manifest_runtime_state, + intake_chain_tasks, + sync_adapter_state_before_refresh, + resolver_profile_convergence_enabled, + last_admission_epochs, + Some(( + deployment_profile, + adapter_sync_page_logs, + heartbeat, + heartbeat_chain_ids, + )), + ) + .await +} + +type AdapterSyncHeartbeat<'a> = (&'a str, usize, &'a mut StartupHeartbeat, &'a [String]); + +#[expect(clippy::too_many_arguments)] +async fn refresh_discovery_watch_state_inner( + pool: &sqlx::PgPool, + provider_registry: &ProviderRegistry, + manifest_runtime_state: &mut ManifestRuntimeState, + intake_chain_tasks: &mut Vec, + sync_adapter_state_before_refresh: bool, + resolver_profile_convergence_enabled: bool, + last_admission_epochs: &mut Option>, + adapter_sync_heartbeat: Option>, ) -> Result { // The whole-corpus re-derivation must run before the sentinel read: it is // what materializes new edges (and bumps epochs) on the broad-refresh path. let adapter_sync_result: Result<()> = if sync_adapter_state_before_refresh { - sync_adapter_owned_raw_log_state(pool, &manifest_runtime_state.watched_chain_plan).await + let (deployment_profile, page_logs, heartbeat, chain_ids) = adapter_sync_heartbeat + .context("discovery adapter refresh requires a live loop heartbeat")?; + sync_adapter_owned_raw_log_state_with_heartbeat( + pool, + deployment_profile, + &manifest_runtime_state.watched_chain_plan, + page_logs, + heartbeat, + chain_ids, + ) + .await } else { Ok(()) }; diff --git a/apps/indexer/src/main/startup_heartbeat.rs b/apps/indexer/src/main/startup_heartbeat.rs new file mode 100644 index 00000000..e77bd0c1 --- /dev/null +++ b/apps/indexer/src/main/startup_heartbeat.rs @@ -0,0 +1,144 @@ +use anyhow::Result; +use sqlx::PgPool; +use tokio::time::{Duration, Instant}; + +pub(crate) struct StartupHeartbeat { + instance_id: String, + interval: Duration, + last_recorded_at: Instant, + #[cfg(test)] + adapter_progress_count: usize, +} + +pub(crate) struct StartupAdapterHeartbeat<'a> { + heartbeat: &'a mut StartupHeartbeat, + chain_ids: &'a [String], +} + +impl<'a> StartupAdapterHeartbeat<'a> { + pub(crate) fn new(heartbeat: &'a mut StartupHeartbeat, chain_ids: &'a [String]) -> Self { + Self { + heartbeat, + chain_ids, + } + } +} + +impl bigname_adapters::StartupAdapterProgress for StartupAdapterHeartbeat<'_> { + fn record<'a>( + &'a mut self, + pool: &'a PgPool, + ) -> bigname_adapters::StartupAdapterProgressFuture<'a> { + Box::pin(async move { + #[cfg(test)] + { + self.heartbeat.adapter_progress_count += 1; + } + self.heartbeat.record_if_due(pool, self.chain_ids).await + }) + } +} + +impl StartupHeartbeat { + pub(crate) fn new(instance_id: String, interval: Duration) -> Self { + Self { + instance_id, + interval, + last_recorded_at: Instant::now(), + #[cfg(test)] + adapter_progress_count: 0, + } + } + + #[cfg(test)] + pub(crate) fn adapter_progress_count(&self) -> usize { + self.adapter_progress_count + } + + pub(crate) async fn record_if_due( + &mut self, + pool: &PgPool, + chain_ids: &[String], + ) -> Result<()> { + if self.last_recorded_at.elapsed() < self.interval { + return Ok(()); + } + self.record(pool, chain_ids).await + } + + pub(crate) async fn record(&mut self, pool: &PgPool, chain_ids: &[String]) -> Result<()> { + bigname_storage::record_service_loop_heartbeat( + pool, + bigname_storage::INDEXER_SERVICE_NAME, + &self.instance_id, + chain_ids, + ) + .await?; + self.last_recorded_at = Instant::now(); + Ok(()) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[tokio::test] + async fn bootstrap_progress_refreshes_the_registered_indexer_loop() -> Result<()> { + let database = bigname_test_support::TestDatabase::create_migrated( + bigname_test_support::TestDatabaseConfig::new("bigname_indexer_startup_heartbeat_test"), + &bigname_storage::MIGRATOR, + "failed to migrate indexer startup-heartbeat test database", + ) + .await?; + bigname_storage::register_service_loop( + database.pool(), + bigname_storage::INDEXER_SERVICE_NAME, + "bootstrap-test", + ) + .await?; + sqlx::query( + r#" + UPDATE service_loop_heartbeats + SET started_at = clock_timestamp() - INTERVAL '3 minutes', + heartbeat_at = clock_timestamp() - INTERVAL '2 minutes' + WHERE service_name = 'indexer' + AND instance_id = 'bootstrap-test' + "#, + ) + .execute(database.pool()) + .await?; + + let mut heartbeat = + StartupHeartbeat::new("bootstrap-test".to_owned(), Duration::from_secs(0)); + let chain_ids = vec!["ethereum-mainnet".to_owned(), "ethereum-mainnet".to_owned()]; + let mut progress = StartupAdapterHeartbeat::new(&mut heartbeat, &chain_ids); + bigname_adapters::StartupAdapterProgress::record(&mut progress, database.pool()).await?; + + let observed = bigname_storage::load_service_loop_heartbeat( + database.pool(), + bigname_storage::INDEXER_SERVICE_NAME, + "bootstrap-test", + ) + .await? + .expect("registered startup heartbeat must exist"); + assert!(observed.age_seconds < 5); + let chain_row_count = sqlx::query_scalar::<_, i64>( + r#" + SELECT COUNT(*) + FROM service_loop_heartbeats + WHERE service_name = 'indexer' + AND instance_id = 'bootstrap-test' + AND scope_kind = 'chain' + "#, + ) + .fetch_one(database.pool()) + .await?; + assert_eq!( + chain_row_count, 1, + "duplicate chain ids must be deduplicated" + ); + + database.cleanup().await + } +} diff --git a/apps/indexer/src/main/tests/adapter_sync.rs b/apps/indexer/src/main/tests/adapter_sync.rs index f21ef28c..e8ef1e35 100644 --- a/apps/indexer/src/main/tests/adapter_sync.rs +++ b/apps/indexer/src/main/tests/adapter_sync.rs @@ -1482,13 +1482,49 @@ async fn sync_adapter_owned_raw_log_state_backfills_wrapper_authority_from_store let watched_plan = load_watched_chain_plan(database.pool()).await?; sync_adapter_owned_raw_log_state(database.pool(), &watched_plan).await?; sync_adapter_owned_raw_log_state(database.pool(), &watched_plan).await?; - sync_startup_adapter_owned_raw_log_state( + sqlx::query( + r#" + CREATE TABLE service_loop_heartbeats ( + service_name TEXT NOT NULL, + instance_id TEXT NOT NULL, + scope_kind TEXT NOT NULL, + scope_id TEXT NOT NULL, + started_at TIMESTAMPTZ NOT NULL, + heartbeat_at TIMESTAMPTZ NOT NULL, + PRIMARY KEY (service_name, instance_id, scope_kind, scope_id) + ) + "#, + ) + .execute(database.pool()) + .await?; + let heartbeat_instance_id = "live-adapter-page-heartbeat-test"; + bigname_storage::register_service_loop( + database.pool(), + bigname_storage::INDEXER_SERVICE_NAME, + heartbeat_instance_id, + ) + .await?; + let heartbeat_chain_ids = watched_plan + .iter() + .map(|chain| chain.chain.clone()) + .collect::>(); + let mut heartbeat = crate::run::startup_heartbeat::StartupHeartbeat::new( + heartbeat_instance_id.to_owned(), + tokio::time::Duration::ZERO, + ); + sync_adapter_owned_raw_log_state_with_heartbeat( database.pool(), "test", &watched_plan, DEFAULT_STARTUP_DISCOVERY_PAGE_LOGS, + &mut heartbeat, + &heartbeat_chain_ids, ) .await?; + assert!( + heartbeat.adapter_progress_count() > 0, + "a live full-family re-sync must heartbeat from inside checkpointed adapter work" + ); assert_eq!( sqlx::query_scalar::<_, i64>( "SELECT COUNT(*)::BIGINT FROM normalized_replay_adapter_checkpoints diff --git a/apps/worker/src/address_names.rs b/apps/worker/src/address_names.rs index ab8614ab..c50f00aa 100644 --- a/apps/worker/src/address_names.rs +++ b/apps/worker/src/address_names.rs @@ -8,6 +8,7 @@ mod relations; mod source_policy; mod util; +pub(crate) use rebuild::rebuild_address_names_current_with_heartbeat; pub use rebuild::{ rebuild_address_names_current, rebuild_address_names_current_logical_name, rebuild_address_names_current_logical_names, diff --git a/apps/worker/src/address_names/rebuild.rs b/apps/worker/src/address_names/rebuild.rs index 50a88c6d..65d7cf87 100644 --- a/apps/worker/src/address_names/rebuild.rs +++ b/apps/worker/src/address_names/rebuild.rs @@ -12,6 +12,10 @@ use futures_util::{TryStreamExt, pin_mut}; use sqlx::PgPool; use tokio::task::JoinSet; +use crate::primary_name::rebuild_heartbeat::{ + LoopHeartbeat, record_rebuild_progress, run_rebuild_phase, +}; + use super::{ AddressNamesCurrentRebuildSummary, load::{ @@ -29,10 +33,26 @@ const ADDRESS_NAMES_CURRENT_REBUILD_CONCURRENCY: usize = 8; pub async fn rebuild_address_names_current( pool: &PgPool, address: Option<&str>, +) -> Result { + rebuild_address_names_current_inner(pool, address, None).await +} + +pub(crate) async fn rebuild_address_names_current_with_heartbeat( + pool: &PgPool, + address: Option<&str>, + loop_heartbeat: &mut LoopHeartbeat, +) -> Result { + rebuild_address_names_current_inner(pool, address, Some(loop_heartbeat)).await +} + +async fn rebuild_address_names_current_inner( + pool: &PgPool, + address: Option<&str>, + loop_heartbeat: Option<&mut LoopHeartbeat>, ) -> Result { match address { Some(address) => rebuild_one_address(pool, address).await, - None => rebuild_all_addresses(pool).await, + None => rebuild_all_addresses(pool, loop_heartbeat).await, } } @@ -83,8 +103,17 @@ pub async fn rebuild_address_names_current_logical_names( }) } -async fn rebuild_all_addresses(pool: &PgPool) -> Result { - let rebuild = begin_address_names_current_full_rebuild(pool).await?; +async fn rebuild_all_addresses( + pool: &PgPool, + mut loop_heartbeat: Option<&mut LoopHeartbeat>, +) -> Result { + let rebuild = run_rebuild_phase( + pool, + &mut loop_heartbeat, + "address_names_current.prepare", + begin_address_names_current_full_rebuild(pool), + ) + .await?; let deleted_row_count = rebuild.previous_row_count(); tracing::info!( projection = "address_names_current", @@ -92,7 +121,7 @@ async fn rebuild_all_addresses(pool: &PgPool) -> Result staged, Err(error) => { if let Err(drop_error) = drop_address_names_current_full_rebuild(pool, &rebuild).await { @@ -106,22 +135,26 @@ async fn rebuild_all_addresses(pool: &PgPool) -> Result published, - Err(error) => { - if let Err(drop_error) = - drop_address_names_current_full_rebuild(pool, &rebuild).await - { - tracing::warn!( - projection = "address_names_current", - error = %drop_error, - "failed to drop address_names_current full rebuild staging table after publish error" - ); - } - return Err(error); + let (_deleted_row_count, published_row_count) = match run_rebuild_phase( + pool, + &mut loop_heartbeat, + "address_names_current.publish", + publish_address_names_current_full_rebuild(pool, &rebuild), + ) + .await + { + Ok(published) => published, + Err(error) => { + if let Err(drop_error) = drop_address_names_current_full_rebuild(pool, &rebuild).await { + tracing::warn!( + projection = "address_names_current", + error = %drop_error, + "failed to drop address_names_current full rebuild staging table after publish error" + ); } - }; + return Err(error); + } + }; if let Err(error) = drop_address_names_current_full_rebuild(pool, &rebuild).await { tracing::warn!( projection = "address_names_current", @@ -136,7 +169,13 @@ async fn rebuild_all_addresses(pool: &PgPool) -> Result, ) -> Result { let mut queued_binding_count = 0usize; let mut completed_binding_count = 0usize; @@ -173,6 +213,7 @@ async fn stage_all_address_rows( while let Some(result) = tasks.join_next().await { completed_binding_count += 1; let binding_rows = result??; + record_rebuild_progress(pool, loop_heartbeat).await; rows.extend(binding_rows); if rows.len() >= ADDRESS_NAMES_CURRENT_REBUILD_BATCH_SIZE { diff --git a/apps/worker/src/automatic_projection_replay.rs b/apps/worker/src/automatic_projection_replay.rs index b7e8dac5..57157d86 100644 --- a/apps/worker/src/automatic_projection_replay.rs +++ b/apps/worker/src/automatic_projection_replay.rs @@ -12,12 +12,21 @@ use tokio::sync::Mutex; use tokio::time::{Duration, sleep}; use tracing::{debug, info, warn}; +use crate::primary_name::rebuild_heartbeat as heartbeat; use crate::{cli::RunArgs, primary_name, projection_apply, record_inventory, replay}; +#[path = "automatic_projection_replay/bootstrap_replay.rs"] +mod bootstrap_replay; #[path = "automatic_projection_replay/primary_hydration.rs"] mod primary_hydration; #[path = "automatic_projection_replay/primary_hydration_loop.rs"] mod primary_hydration_loop; +#[path = "automatic_projection_replay/shutdown.rs"] +mod shutdown; + +#[cfg(test)] +use bootstrap_replay::replay_all_current_projections_when_ready; +use bootstrap_replay::replay_all_current_projections_when_ready_with_heartbeat; const CURSOR_KIND_RAW_FACT_NORMALIZED_EVENTS: &str = "raw_fact_normalized_events"; const ALL_CURRENT_PROJECTIONS_MIN_DATABASE_CONNECTIONS: u32 = 64; @@ -28,6 +37,8 @@ const ACTIVE_INDEX_BUILDS_QUERY: &str = r#" WHERE datname = current_database() "#; +type SharedLoopHeartbeat = Arc>; + pub(crate) fn all_current_projections_database_config( mut database: DatabaseConfig, ) -> DatabaseConfig { @@ -38,6 +49,8 @@ pub(crate) fn all_current_projections_database_config( } pub(crate) async fn run_worker(args: RunArgs) -> Result<()> { + let heartbeat_instance_id = + bigname_storage::resolve_service_instance_id(args.heartbeat_instance_id.as_deref())?; let database = all_current_projections_database_config(args.database); let (pool, _runtime_rederive_guard) = bigname_storage::connect_with_base_normalized_rederive_writer_guard( @@ -75,28 +88,37 @@ pub(crate) async fn run_worker(args: RunArgs) -> Result<()> { "worker booted" ); - tokio::select! { - () = run_automatic_current_projection_replay( - pool, + bigname_storage::register_service_loop( + &pool, + bigname_storage::WORKER_SERVICE_NAME, + &heartbeat_instance_id, + ) + .await?; + + let replay_pool = pool.clone(); + let replay_heartbeat_instance_id = heartbeat_instance_id.clone(); + shutdown::run_until_shutdown( + &pool, + &heartbeat_instance_id, + run_automatic_current_projection_replay( + replay_pool, + replay_heartbeat_instance_id, args.poll_interval_secs, text_hydration_config, primary_hydration_config, - ) => {} - signal = tokio::signal::ctrl_c() => { - signal.context("failed to listen for shutdown signal")?; - } - } - - info!(service = "worker", "shutdown signal received"); - Ok(()) + ), + shutdown::shutdown_signal(), + ) + .await } pub(crate) async fn run_automatic_current_projection_replay( pool: PgPool, + heartbeat_instance_id: String, poll_interval_secs: u64, text_hydration_config: Option, mut primary_hydration_config: Option, -) { +) -> Result<()> { let poll_interval = Duration::from_secs(poll_interval_secs.max(1)); let mut bootstrap_completed = false; let mut bootstrap_text_hydration_completed = text_hydration_config.is_none(); @@ -104,8 +126,14 @@ pub(crate) async fn run_automatic_current_projection_replay( let mut projection_derivation_started = false; let projection_apply_generation = Arc::new(AtomicU64::new(0)); let projection_apply_hydration_lock = Arc::new(Mutex::new(())); + let loop_heartbeat = Arc::new(Mutex::new(heartbeat::LoopHeartbeat::new( + heartbeat_instance_id, + poll_interval, + ))); loop { + record_loop_heartbeat_if_due(&pool, &loop_heartbeat).await; + let mut progressed = false; if !bootstrap_completed { match projection_bootstrap_already_handed_off_to_apply(&pool).await { @@ -131,13 +159,17 @@ pub(crate) async fn run_automatic_current_projection_replay( } if !bootstrap_completed { - match replay_all_current_projections_when_ready( - &pool, - text_hydration_config.as_ref(), - primary_hydration_config.as_ref(), - ) - .await - { + let replay_result = { + let mut loop_heartbeat = loop_heartbeat.lock().await; + replay_all_current_projections_when_ready_with_heartbeat( + &pool, + text_hydration_config.as_ref(), + primary_hydration_config.as_ref(), + &mut loop_heartbeat, + ) + .await + }; + match replay_result { Ok(true) => { bootstrap_completed = true; progressed = true; @@ -152,6 +184,7 @@ pub(crate) async fn run_automatic_current_projection_replay( ); } } + record_loop_heartbeat_if_due(&pool, &loop_heartbeat).await; } if bootstrap_completed { @@ -172,6 +205,7 @@ pub(crate) async fn run_automatic_current_projection_replay( if let Some(config) = primary_hydration_config.take() { primary_hydration_loop::spawn( pool.clone(), + Arc::clone(&loop_heartbeat), poll_interval_secs, config, Arc::clone(&projection_apply_generation), @@ -182,12 +216,16 @@ pub(crate) async fn run_automatic_current_projection_replay( } if hydration_schedule.run_text_hydration { - match hydrate_record_inventory_text_values_after_bootstrap( - &pool, - text_hydration_config.as_ref(), - ) - .await - { + let hydration_result = { + let mut loop_heartbeat = loop_heartbeat.lock().await; + hydrate_record_inventory_text_values_after_bootstrap( + &pool, + text_hydration_config.as_ref(), + &mut loop_heartbeat, + ) + .await + }; + match hydration_result { Ok(()) => { bootstrap_text_hydration_completed = true; progressed = true; @@ -201,10 +239,20 @@ pub(crate) async fn run_automatic_current_projection_replay( ); } } + record_loop_heartbeat_if_due(&pool, &loop_heartbeat).await; } let _apply_hydration_guard = projection_apply_hydration_lock.lock().await; - match projection_apply::run_once(&pool, text_hydration_config.as_ref()).await { + let apply_result = { + let mut loop_heartbeat = loop_heartbeat.lock().await; + projection_apply::run_once( + &pool, + text_hydration_config.as_ref(), + &mut loop_heartbeat, + ) + .await + }; + match apply_result { Ok(summary) => { let apply_progressed = summary.made_progress(); if apply_progressed { @@ -221,6 +269,7 @@ pub(crate) async fn run_automatic_current_projection_replay( ); } } + record_loop_heartbeat_if_due(&pool, &loop_heartbeat).await; } if !progressed { @@ -229,6 +278,10 @@ pub(crate) async fn run_automatic_current_projection_replay( } } +async fn record_loop_heartbeat_if_due(pool: &PgPool, loop_heartbeat: &SharedLoopHeartbeat) { + loop_heartbeat.lock().await.record_if_due(pool).await; +} + fn spawn_continuous_projection_invalidation_derivation(pool: PgPool, poll_interval_secs: u64) { tokio::spawn(async move { run_continuous_projection_invalidation_derivation(pool, poll_interval_secs).await; @@ -293,113 +346,22 @@ fn bootstrap_hydration_schedule( async fn hydrate_record_inventory_text_values_after_bootstrap( pool: &PgPool, text_hydration_config: Option<&record_inventory::RecordInventoryTextHydrationConfig>, + loop_heartbeat: &mut heartbeat::LoopHeartbeat, ) -> Result<()> { let Some(config) = text_hydration_config else { return Ok(()); }; - let summary = - record_inventory::hydrate_record_inventory_text_values(pool, None, config.clone()).await?; + let summary = record_inventory::hydrate_record_inventory_text_values_with_heartbeat( + pool, + None, + config.clone(), + loop_heartbeat, + ) + .await?; record_inventory::log_text_hydration_summary(None, &summary); Ok(()) } -async fn replay_all_current_projections_when_ready( - pool: &PgPool, - text_hydration_config: Option<&record_inventory::RecordInventoryTextHydrationConfig>, - primary_hydration_config: Option<&primary_name::PrimaryNameLegacyReverseHydrationConfig>, -) -> Result { - let readiness = load_projection_replay_readiness(pool).await?; - if !readiness.is_ready() { - debug!( - service = "worker", - replay = "all_current_projections", - normalized_replay_cursor_count = readiness.normalized_replay_cursor_count, - incomplete_normalized_replay_cursor_count = - readiness.incomplete_normalized_replay_cursor_count, - failed_normalized_replay_cursor_count = readiness.failed_normalized_replay_cursor_count, - active_index_build_count = readiness.active_index_build_count, - missing_projection_index_count = readiness.missing_projection_index_count, - "automatic all-current projection replay is waiting for normalized replay readiness" - ); - return Ok(false); - } - - let Some(mut replay_lock) = try_acquire_replay_lock(pool).await? else { - debug!( - service = "worker", - replay = "all_current_projections", - "automatic all-current projection replay skipped because another worker holds the replay lock" - ); - return Ok(false); - }; - - let readiness = load_projection_replay_readiness(pool).await?; - if !readiness.is_ready() { - release_replay_lock(&mut replay_lock).await?; - return Ok(false); - } - - let replay_result: Result<_> = async { - let cursor_exists = projection_apply::normalized_event_cursor_exists(pool).await?; - let captured_watermark = - projection_apply::load_normalized_event_change_watermark(pool).await?; - let chain_checkpoint_max_block = - projection_apply::load_chain_checkpoint_max_block(pool).await?; - let replay_target_block = projection_bootstrap_replay_target_block( - readiness.normalized_replay_max_target_block, - chain_checkpoint_max_block, - ); - let reusable_marker_count = - load_current_projection_replay_marker_count(pool, replay_target_block).await?; - let cursor_seed = projection_bootstrap_apply_cursor_seed( - cursor_exists, - reusable_marker_count, - captured_watermark, - ); - - info!( - service = "worker", - replay = "all_current_projections", - normalized_replay_cursor_count = readiness.normalized_replay_cursor_count, - normalized_replay_max_target_block = readiness.normalized_replay_max_target_block, - chain_checkpoint_max_block, - projection_replay_target_block = replay_target_block, - captured_change_watermark = captured_watermark.change_id, - reusable_marker_count, - cursor_seed_change_id = cursor_seed.map(|cursor| cursor.change_id), - "automatic all-current projection replay started" - ); - let summary = replay::rebuild_pending_all_current_projections( - pool, - replay_target_block, - text_hydration_config, - primary_hydration_config, - ) - .await - .context("failed to automatically replay all current projections")?; - if let Some(cursor_seed) = cursor_seed { - projection_apply::seed_normalized_event_cursor_if_absent(pool, cursor_seed).await?; - } - Ok(summary) - } - .await; - release_replay_lock(&mut replay_lock).await?; - - let summary = replay_result?; - info!( - service = "worker", - replay = "all_current_projections", - projection_order = ?summary.projection_order(), - projection_count = summary.steps.len(), - total_requested_key_count = summary.total_requested_key_count(), - total_upserted_row_count = summary.total_upserted_row_count(), - total_deleted_row_count = summary.total_deleted_row_count(), - "automatic all-current projection replay completed" - ); - - Ok(true) -} - async fn projection_bootstrap_already_handed_off_to_apply(pool: &PgPool) -> Result { let cursor_exists = projection_apply::normalized_event_cursor_exists(pool).await?; if !cursor_exists { diff --git a/apps/worker/src/automatic_projection_replay/bootstrap_replay.rs b/apps/worker/src/automatic_projection_replay/bootstrap_replay.rs new file mode 100644 index 00000000..2f505c97 --- /dev/null +++ b/apps/worker/src/automatic_projection_replay/bootstrap_replay.rs @@ -0,0 +1,140 @@ +use super::*; + +#[cfg(test)] +pub(super) async fn replay_all_current_projections_when_ready( + pool: &PgPool, + text_hydration_config: Option<&record_inventory::RecordInventoryTextHydrationConfig>, + primary_hydration_config: Option<&primary_name::PrimaryNameLegacyReverseHydrationConfig>, +) -> Result { + replay_all_current_projections_when_ready_inner( + pool, + text_hydration_config, + primary_hydration_config, + None, + ) + .await +} + +pub(super) async fn replay_all_current_projections_when_ready_with_heartbeat( + pool: &PgPool, + text_hydration_config: Option<&record_inventory::RecordInventoryTextHydrationConfig>, + primary_hydration_config: Option<&primary_name::PrimaryNameLegacyReverseHydrationConfig>, + loop_heartbeat: &mut heartbeat::LoopHeartbeat, +) -> Result { + replay_all_current_projections_when_ready_inner( + pool, + text_hydration_config, + primary_hydration_config, + Some(loop_heartbeat), + ) + .await +} + +async fn replay_all_current_projections_when_ready_inner( + pool: &PgPool, + text_hydration_config: Option<&record_inventory::RecordInventoryTextHydrationConfig>, + primary_hydration_config: Option<&primary_name::PrimaryNameLegacyReverseHydrationConfig>, + mut loop_heartbeat: Option<&mut heartbeat::LoopHeartbeat>, +) -> Result { + let readiness = load_projection_replay_readiness(pool).await?; + if !readiness.is_ready() { + debug!( + service = "worker", + replay = "all_current_projections", + normalized_replay_cursor_count = readiness.normalized_replay_cursor_count, + incomplete_normalized_replay_cursor_count = + readiness.incomplete_normalized_replay_cursor_count, + failed_normalized_replay_cursor_count = readiness.failed_normalized_replay_cursor_count, + active_index_build_count = readiness.active_index_build_count, + missing_projection_index_count = readiness.missing_projection_index_count, + "automatic all-current projection replay is waiting for normalized replay readiness" + ); + return Ok(false); + } + + let Some(mut replay_lock) = try_acquire_replay_lock(pool).await? else { + debug!( + service = "worker", + replay = "all_current_projections", + "automatic all-current projection replay skipped because another worker holds the replay lock" + ); + return Ok(false); + }; + + let readiness = load_projection_replay_readiness(pool).await?; + if !readiness.is_ready() { + release_replay_lock(&mut replay_lock).await?; + return Ok(false); + } + + let replay_result: Result<_> = async { + let cursor_exists = projection_apply::normalized_event_cursor_exists(pool).await?; + let captured_watermark = + projection_apply::load_normalized_event_change_watermark(pool).await?; + let chain_checkpoint_max_block = + projection_apply::load_chain_checkpoint_max_block(pool).await?; + let replay_target_block = projection_bootstrap_replay_target_block( + readiness.normalized_replay_max_target_block, + chain_checkpoint_max_block, + ); + let reusable_marker_count = + load_current_projection_replay_marker_count(pool, replay_target_block).await?; + let cursor_seed = projection_bootstrap_apply_cursor_seed( + cursor_exists, + reusable_marker_count, + captured_watermark, + ); + + info!( + service = "worker", + replay = "all_current_projections", + normalized_replay_cursor_count = readiness.normalized_replay_cursor_count, + normalized_replay_max_target_block = readiness.normalized_replay_max_target_block, + chain_checkpoint_max_block, + projection_replay_target_block = replay_target_block, + captured_change_watermark = captured_watermark.change_id, + reusable_marker_count, + cursor_seed_change_id = cursor_seed.map(|cursor| cursor.change_id), + "automatic all-current projection replay started" + ); + let summary = if let Some(loop_heartbeat) = loop_heartbeat.as_deref_mut() { + replay::rebuild_pending_all_current_projections_with_heartbeat( + pool, + replay_target_block, + text_hydration_config, + primary_hydration_config, + loop_heartbeat, + ) + .await + } else { + replay::rebuild_pending_all_current_projections( + pool, + replay_target_block, + text_hydration_config, + primary_hydration_config, + ) + .await + } + .context("failed to automatically replay all current projections")?; + if let Some(cursor_seed) = cursor_seed { + projection_apply::seed_normalized_event_cursor_if_absent(pool, cursor_seed).await?; + } + Ok(summary) + } + .await; + release_replay_lock(&mut replay_lock).await?; + + let summary = replay_result?; + info!( + service = "worker", + replay = "all_current_projections", + projection_order = ?summary.projection_order(), + projection_count = summary.steps.len(), + total_requested_key_count = summary.total_requested_key_count(), + total_upserted_row_count = summary.total_upserted_row_count(), + total_deleted_row_count = summary.total_deleted_row_count(), + "automatic all-current projection replay completed" + ); + + Ok(true) +} diff --git a/apps/worker/src/automatic_projection_replay/primary_hydration.rs b/apps/worker/src/automatic_projection_replay/primary_hydration.rs index d2f48bba..51c3538c 100644 --- a/apps/worker/src/automatic_projection_replay/primary_hydration.rs +++ b/apps/worker/src/automatic_projection_replay/primary_hydration.rs @@ -1,7 +1,7 @@ use anyhow::Result; use sqlx::PgPool; -use crate::primary_name; +use crate::primary_name::{self, rebuild_heartbeat::LoopHeartbeat}; pub(super) type LegacyReverseHydrationTriggerState = Option>; @@ -10,6 +10,7 @@ pub(super) async fn hydrate_after_bootstrap( pool: &PgPool, primary_hydration_config: Option<&primary_name::PrimaryNameLegacyReverseHydrationConfig>, last_trigger: &mut LegacyReverseHydrationTriggerState, + loop_heartbeat: &mut LoopHeartbeat, ) -> Result { let trigger_before = match primary_hydration_config { Some(config) => { @@ -17,7 +18,7 @@ pub(super) async fn hydrate_after_bootstrap( } None => Vec::new(), }; - let summary = hydrate(pool, primary_hydration_config).await?; + let summary = hydrate(pool, primary_hydration_config, loop_heartbeat).await?; if primary_hydration_config.is_some() && hydration_cause_consumed(&summary) { *last_trigger = Some(trigger_before); } @@ -29,6 +30,7 @@ pub(super) async fn hydrate_if_projection_changed_or_triggered( primary_hydration_config: Option<&primary_name::PrimaryNameLegacyReverseHydrationConfig>, last_trigger: &mut LegacyReverseHydrationTriggerState, projection_apply_changed: &mut bool, + loop_heartbeat: &mut LoopHeartbeat, ) -> Result { let Some(config) = primary_hydration_config else { return Ok(primary_name::PrimaryNameLegacyReverseHydrationSummary::default()); @@ -39,7 +41,7 @@ pub(super) async fn hydrate_if_projection_changed_or_triggered( return Ok(primary_name::PrimaryNameLegacyReverseHydrationSummary::default()); } - let summary = hydrate(pool, Some(config)).await?; + let summary = hydrate(pool, Some(config), loop_heartbeat).await?; if hydration_cause_consumed(&summary) { *last_trigger = Some(current_trigger); *projection_apply_changed = false; @@ -50,12 +52,17 @@ pub(super) async fn hydrate_if_projection_changed_or_triggered( async fn hydrate( pool: &PgPool, primary_hydration_config: Option<&primary_name::PrimaryNameLegacyReverseHydrationConfig>, + loop_heartbeat: &mut LoopHeartbeat, ) -> Result { let Some(config) = primary_hydration_config else { return Ok(primary_name::PrimaryNameLegacyReverseHydrationSummary::default()); }; - let summary = - primary_name::hydrate_legacy_reverse_resolver_primary_names(pool, config.clone()).await?; + let summary = primary_name::hydrate_legacy_reverse_resolver_primary_names_with_heartbeat( + pool, + config.clone(), + loop_heartbeat, + ) + .await?; if summary.candidate_tuple_count > 0 || summary.failed_lookup_count > 0 { primary_name::log_legacy_reverse_hydration_summary(&summary); } diff --git a/apps/worker/src/automatic_projection_replay/primary_hydration_loop.rs b/apps/worker/src/automatic_projection_replay/primary_hydration_loop.rs index b36c75e0..c369cb12 100644 --- a/apps/worker/src/automatic_projection_replay/primary_hydration_loop.rs +++ b/apps/worker/src/automatic_projection_replay/primary_hydration_loop.rs @@ -8,11 +8,12 @@ use tokio::sync::Mutex; use tokio::time::{Duration, sleep}; use tracing::{info, warn}; -use super::primary_hydration; +use super::{SharedLoopHeartbeat, primary_hydration}; use crate::{primary_name, projection_apply}; pub(super) fn spawn( pool: PgPool, + loop_heartbeat: SharedLoopHeartbeat, poll_interval_secs: u64, config: primary_name::PrimaryNameLegacyReverseHydrationConfig, projection_apply_generation: Arc, @@ -21,6 +22,7 @@ pub(super) fn spawn( tokio::spawn(async move { run( pool, + loop_heartbeat, poll_interval_secs, config, projection_apply_generation, @@ -32,6 +34,7 @@ pub(super) fn spawn( async fn run( pool: PgPool, + loop_heartbeat: SharedLoopHeartbeat, poll_interval_secs: u64, config: primary_name::PrimaryNameLegacyReverseHydrationConfig, projection_apply_generation: Arc, @@ -73,13 +76,17 @@ async fn run( if !bootstrap_completed { let hydration_generation = projection_apply_generation.load(Ordering::Acquire); - match primary_hydration::hydrate_after_bootstrap( - &pool, - Some(&config), - &mut last_trigger, - ) - .await - { + let hydration_result = { + let mut loop_heartbeat = loop_heartbeat.lock().await; + primary_hydration::hydrate_after_bootstrap( + &pool, + Some(&config), + &mut last_trigger, + &mut loop_heartbeat, + ) + .await + }; + match hydration_result { Ok(summary) => { bootstrap_completed = summary.failed_lookup_count == 0; progressed |= primary_hydration::bootstrap_hydration_made_progress(&summary); @@ -99,14 +106,18 @@ async fn run( } else { let current_generation = projection_apply_generation.load(Ordering::Acquire); let mut projection_apply_changed = current_generation != hydrated_projection_generation; - match primary_hydration::hydrate_if_projection_changed_or_triggered( - &pool, - Some(&config), - &mut last_trigger, - &mut projection_apply_changed, - ) - .await - { + let hydration_result = { + let mut loop_heartbeat = loop_heartbeat.lock().await; + primary_hydration::hydrate_if_projection_changed_or_triggered( + &pool, + Some(&config), + &mut last_trigger, + &mut projection_apply_changed, + &mut loop_heartbeat, + ) + .await + }; + match hydration_result { Ok(summary) => { if !projection_apply_changed { hydrated_projection_generation = current_generation; @@ -130,3 +141,172 @@ async fn run( } } } + +#[cfg(test)] +mod tests { + use anyhow::{Context, Result}; + use bigname_test_support::{TestDatabase, TestDatabaseConfig}; + use tokio::sync::Notify; + + use super::*; + + async fn test_database() -> Result { + TestDatabase::create_migrated( + TestDatabaseConfig::new("bigname_worker_primary_hydration_loop_test"), + &bigname_storage::MIGRATOR, + "failed to apply migrations for primary hydration loop tests", + ) + .await + } + + #[tokio::test] + async fn idle_primary_hydration_does_not_mask_a_stalled_main_loop() -> Result<()> { + let database = test_database().await?; + let instance_id = "idle-primary-hydration-heartbeat-test"; + bigname_storage::register_service_loop( + database.pool(), + bigname_storage::WORKER_SERVICE_NAME, + instance_id, + ) + .await?; + + let mut config = primary_name::PrimaryNameLegacyReverseHydrationConfig::new( + bigname_execution::ChainRpcUrls::default(), + ); + config.resolver_addresses.clear(); + let hydration_task = tokio::spawn(run( + database.pool().clone(), + Arc::new(Mutex::new( + crate::primary_name::rebuild_heartbeat::LoopHeartbeat::new( + instance_id.to_owned(), + Duration::from_secs(1), + ), + )), + 1, + config, + Arc::new(AtomicU64::new(0)), + Arc::new(Mutex::new(())), + )); + + sleep(Duration::from_millis(250)).await; + sqlx::query( + r#" + UPDATE service_loop_heartbeats + SET started_at = clock_timestamp() - INTERVAL '2 minutes', + heartbeat_at = clock_timestamp() - INTERVAL '1 minute' + WHERE service_name = 'worker' + AND instance_id = $1 + AND scope_kind = 'process' + "#, + ) + .bind(instance_id) + .execute(database.pool()) + .await?; + sleep(Duration::from_millis(1_250)).await; + + let heartbeat_age = bigname_storage::load_service_loop_heartbeat( + database.pool(), + bigname_storage::WORKER_SERVICE_NAME, + instance_id, + ) + .await? + .context("worker heartbeat must remain registered")? + .age_seconds; + hydration_task.abort(); + let _ = hydration_task.await; + database.cleanup().await?; + + assert!( + heartbeat_age >= 30, + "an idle detached hydration loop refreshed the worker heartbeat and masked a stalled main loop" + ); + Ok(()) + } + + #[tokio::test] + async fn primary_hydration_preserves_an_active_shared_heartbeat_phase() -> Result<()> { + let database = test_database().await?; + let instance_id = "primary-hydration-phase-owner-test"; + bigname_storage::register_service_loop( + database.pool(), + bigname_storage::WORKER_SERVICE_NAME, + instance_id, + ) + .await?; + + let loop_heartbeat = Arc::new(Mutex::new( + crate::primary_name::rebuild_heartbeat::LoopHeartbeat::new( + instance_id.to_owned(), + Duration::ZERO, + ), + )); + let phase_started = Arc::new(Notify::new()); + let phase_release = Arc::new(Notify::new()); + let phase_task = tokio::spawn({ + let pool = database.pool().clone(); + let loop_heartbeat = Arc::clone(&loop_heartbeat); + let phase_started = Arc::clone(&phase_started); + let phase_release = Arc::clone(&phase_release); + async move { + let mut loop_heartbeat = loop_heartbeat.lock().await; + loop_heartbeat + .run_phase(&pool, "test_primary_hydration_serialization", async move { + phase_started.notify_one(); + phase_release.notified().await; + Ok(()) + }) + .await + } + }); + phase_started.notified().await; + + let mut config = primary_name::PrimaryNameLegacyReverseHydrationConfig::new( + bigname_execution::ChainRpcUrls::default(), + ); + config.resolver_addresses.clear(); + let projection_apply_hydration_lock = Arc::new(Mutex::new(())); + let hydration_task = tokio::spawn(run( + database.pool().clone(), + Arc::clone(&loop_heartbeat), + 1, + config, + Arc::new(AtomicU64::new(0)), + Arc::clone(&projection_apply_hydration_lock), + )); + tokio::time::timeout(Duration::from_secs(1), async { + loop { + if projection_apply_hydration_lock.try_lock().is_err() { + break; + } + tokio::task::yield_now().await; + } + }) + .await + .context("primary hydration did not enter its serialized work section")?; + + let active_phase = bigname_storage::load_service_loop_heartbeat( + database.pool(), + bigname_storage::WORKER_SERVICE_NAME, + instance_id, + ) + .await? + .context("worker heartbeat must remain registered")? + .active_phase + .map(|phase| phase.phase); + + hydration_task.abort(); + let _ = hydration_task.await; + phase_release.notify_one(); + phase_task + .await + .context("shared heartbeat phase task failed to join")??; + database.cleanup().await?; + + assert_eq!( + active_phase.as_deref(), + Some("test_primary_hydration_serialization"), + "primary hydration must not retire a phase owned by another shared-heartbeat operation" + ); + Ok(()) + } +} diff --git a/apps/worker/src/automatic_projection_replay/shutdown.rs b/apps/worker/src/automatic_projection_replay/shutdown.rs new file mode 100644 index 00000000..cb501559 --- /dev/null +++ b/apps/worker/src/automatic_projection_replay/shutdown.rs @@ -0,0 +1,53 @@ +use std::future::Future; + +use anyhow::{Context, Result}; +use sqlx::PgPool; +use tracing::info; + +pub(super) async fn run_until_shutdown( + pool: &PgPool, + heartbeat_instance_id: &str, + work: Work, + shutdown: Shutdown, +) -> Result<()> +where + Work: Future>, + Shutdown: Future>, +{ + tokio::select! { + result = work => return result, + signal = shutdown => { + signal?; + } + } + + bigname_storage::deregister_service_loop( + pool, + bigname_storage::WORKER_SERVICE_NAME, + heartbeat_instance_id, + ) + .await?; + info!(service = "worker", "shutdown signal received"); + Ok(()) +} + +#[cfg(unix)] +pub(super) async fn shutdown_signal() -> Result<()> { + let mut terminate = tokio::signal::unix::signal(tokio::signal::unix::SignalKind::terminate()) + .context("failed to listen for termination signal")?; + tokio::select! { + signal = tokio::signal::ctrl_c() => { + signal.context("failed to listen for interrupt signal") + } + signal = terminate.recv() => { + signal.context("termination signal listener closed") + } + } +} + +#[cfg(not(unix))] +pub(super) async fn shutdown_signal() -> Result<()> { + tokio::signal::ctrl_c() + .await + .context("failed to listen for shutdown signal") +} diff --git a/apps/worker/src/automatic_projection_replay/tests.rs b/apps/worker/src/automatic_projection_replay/tests.rs index 2f3d1dad..eead8a78 100644 --- a/apps/worker/src/automatic_projection_replay/tests.rs +++ b/apps/worker/src/automatic_projection_replay/tests.rs @@ -1,6 +1,7 @@ use super::*; use anyhow::{Context, Result}; use bigname_test_support::{TestDatabase, TestDatabaseConfig}; +use tokio::time::{Duration, sleep}; async fn test_database() -> Result { TestDatabase::create_migrated( @@ -27,6 +28,197 @@ fn ready_status() -> ProjectionReplayReadiness { } } +#[tokio::test] +async fn graceful_shutdown_removes_the_active_worker_phase() -> Result<()> { + let database = test_database().await?; + let instance_id = "graceful-shutdown-phase-test"; + bigname_storage::register_service_loop( + database.pool(), + bigname_storage::WORKER_SERVICE_NAME, + instance_id, + ) + .await?; + bigname_storage::begin_service_loop_phase( + database.pool(), + bigname_storage::WORKER_SERVICE_NAME, + instance_id, + "name_current.publish", + ) + .await?; + + shutdown::run_until_shutdown( + database.pool(), + instance_id, + std::future::pending::>(), + std::future::ready(Ok(())), + ) + .await?; + + let heartbeat_result = bigname_storage::record_service_loop_heartbeat( + database.pool(), + bigname_storage::WORKER_SERVICE_NAME, + instance_id, + &[], + ) + .await; + assert!( + heartbeat_result.is_err(), + "a deregistered loop must not recreate its process row" + ); + + let (process_count, phase_count) = sqlx::query_as::<_, (i64, i64)>( + r#" + SELECT + COUNT(*) FILTER (WHERE scope_kind = 'process')::BIGINT, + COUNT(*) FILTER (WHERE scope_kind = 'phase')::BIGINT + FROM service_loop_heartbeats + WHERE service_name = 'worker' + AND instance_id = $1 + "#, + ) + .bind(instance_id) + .fetch_one(database.pool()) + .await?; + assert_eq!( + process_count, 0, + "graceful shutdown must deregister the loop" + ); + assert_eq!(phase_count, 0, "graceful shutdown must not orphan a phase"); + + database.cleanup().await +} + +#[tokio::test] +async fn graceful_shutdown_cannot_leave_a_concurrently_inserted_phase() -> Result<()> { + let database = test_database().await?; + let instance_id = "graceful-shutdown-phase-race-test"; + bigname_storage::register_service_loop( + database.pool(), + bigname_storage::WORKER_SERVICE_NAME, + instance_id, + ) + .await?; + bigname_storage::begin_service_loop_phase( + database.pool(), + bigname_storage::WORKER_SERVICE_NAME, + instance_id, + "name_current.publish", + ) + .await?; + + let mut phase_lock = database.pool().begin().await?; + sqlx::query( + r#" + SELECT 1 + FROM service_loop_heartbeats + WHERE service_name = 'worker' + AND instance_id = $1 + AND scope_kind = 'phase' + FOR UPDATE + "#, + ) + .bind(instance_id) + .fetch_one(&mut *phase_lock) + .await?; + + let shutdown_pool = database.pool().clone(); + let shutdown_task = tokio::spawn(async move { + shutdown::run_until_shutdown( + &shutdown_pool, + instance_id, + std::future::pending::>(), + std::future::ready(Ok(())), + ) + .await + }); + tokio::time::timeout(Duration::from_secs(5), async { + loop { + let cleanup_is_waiting = sqlx::query_scalar::<_, bool>( + r#" + SELECT EXISTS ( + SELECT 1 + FROM pg_stat_activity + WHERE datname = current_database() + AND pid <> pg_backend_pid() + AND query LIKE '%DELETE FROM service_loop_heartbeats%' + AND wait_event_type = 'Lock' + ) + "#, + ) + .fetch_one(database.pool()) + .await?; + if cleanup_is_waiting { + return Ok::<_, anyhow::Error>(()); + } + sleep(Duration::from_millis(10)).await; + } + }) + .await + .context("shutdown phase cleanup did not reach the locked row")??; + + let phase_writer_pool = database.pool().clone(); + let phase_writer = tokio::spawn(async move { + bigname_storage::begin_service_loop_phase( + &phase_writer_pool, + bigname_storage::WORKER_SERVICE_NAME, + instance_id, + "resolver_current.publish", + ) + .await + }); + tokio::time::timeout(Duration::from_secs(5), async { + loop { + let phase_writer_is_waiting = sqlx::query_scalar::<_, bool>( + r#" + SELECT EXISTS ( + SELECT 1 + FROM pg_stat_activity + WHERE datname = current_database() + AND pid <> pg_backend_pid() + AND query LIKE '%begin_service_loop_phase_registration_fence%' + AND wait_event_type = 'Lock' + ) + "#, + ) + .fetch_one(database.pool()) + .await?; + if phase_writer_is_waiting { + return Ok::<_, anyhow::Error>(()); + } + sleep(Duration::from_millis(10)).await; + } + }) + .await + .context("concurrent phase writer did not reach the locked row")??; + + phase_lock.commit().await?; + shutdown_task.await??; + let phase_writer_result = phase_writer.await?; + assert!( + phase_writer_result.is_err(), + "a deregistered loop must reject a concurrent phase writer" + ); + + let phase_count = sqlx::query_scalar::<_, i64>( + r#" + SELECT COUNT(*)::BIGINT + FROM service_loop_heartbeats + WHERE service_name = 'worker' + AND instance_id = $1 + AND scope_kind = 'phase' + "#, + ) + .bind(instance_id) + .fetch_one(database.pool()) + .await?; + assert_eq!( + phase_count, 0, + "graceful shutdown must fence concurrent phase writers before cleanup" + ); + + database.cleanup().await +} + #[test] fn all_current_projection_pool_size_raises_low_default() { let database = all_current_projections_database_config(DatabaseConfig { diff --git a/apps/worker/src/children.rs b/apps/worker/src/children.rs index 2775211f..f70dc75b 100644 --- a/apps/worker/src/children.rs +++ b/apps/worker/src/children.rs @@ -15,7 +15,10 @@ use tokio::task::JoinSet; #[path = "staged_rebuild.rs"] mod staged_rebuild; -use crate::projection_json::format_timestamp; +use crate::{ + primary_name::rebuild_heartbeat::{LoopHeartbeat, record_rebuild_progress, run_rebuild_phase}, + projection_json::format_timestamp, +}; use staged_rebuild::{ CHILDREN_CURRENT_COLUMNS, count_rows, create_stage_table, drop_stage_table, publish_stage_table, stage_children_current_rows, @@ -37,23 +40,47 @@ pub struct ChildrenCurrentRebuildSummary { pub async fn rebuild_children_current( pool: &PgPool, parent_logical_name_id: Option<&str>, +) -> Result { + rebuild_children_current_inner(pool, parent_logical_name_id, None).await +} + +pub(crate) async fn rebuild_children_current_with_heartbeat( + pool: &PgPool, + parent_logical_name_id: Option<&str>, + loop_heartbeat: &mut LoopHeartbeat, +) -> Result { + rebuild_children_current_inner(pool, parent_logical_name_id, Some(loop_heartbeat)).await +} + +async fn rebuild_children_current_inner( + pool: &PgPool, + parent_logical_name_id: Option<&str>, + loop_heartbeat: Option<&mut LoopHeartbeat>, ) -> Result { match parent_logical_name_id { Some(parent_logical_name_id) => rebuild_one_parent(pool, parent_logical_name_id).await, - None => rebuild_all_parents(pool).await, + None => rebuild_all_parents(pool, loop_heartbeat).await, } } -async fn rebuild_all_parents(pool: &PgPool) -> Result { +async fn rebuild_all_parents( + pool: &PgPool, + mut loop_heartbeat: Option<&mut LoopHeartbeat>, +) -> Result { let mut conn = pool .acquire() .await .context("failed to acquire children_current staging connection")?; let stage_table = create_stage_table(&mut conn, "children_current").await?; - let previous_row_count = count_rows( - &mut conn, - "children_current", - Some("WHERE surface_class = 'declared'"), + let previous_row_count = run_rebuild_phase( + pool, + &mut loop_heartbeat, + "children_current.count_existing", + count_rows( + &mut conn, + "children_current", + Some("WHERE surface_class = 'declared'"), + ), ) .await?; let mut rows = Vec::with_capacity(CHILDREN_CURRENT_REBUILD_BATCH_SIZE); @@ -76,6 +103,7 @@ async fn rebuild_all_parents(pool: &PgPool) -> Result= CHILDREN_CURRENT_REBUILD_BATCH_SIZE { upserted_row_count += stage_children_current_rows(&mut conn, &stage_table, &rows).await? as usize; @@ -105,18 +133,29 @@ async fn rebuild_all_parents(pool: &PgPool) -> Result, #[arg( long = "chain-rpc-url", env = "BIGNAME_WORKER_CHAIN_RPC_URLS", @@ -86,6 +88,20 @@ pub(crate) struct RunArgs { pub(crate) struct HealthcheckArgs { #[command(flatten)] pub(crate) database: DatabaseConfig, + #[arg(long, env = "BIGNAME_HEARTBEAT_INSTANCE_ID")] + pub(crate) heartbeat_instance_id: Option, + #[arg( + long, + env = "BIGNAME_WORKER_HEARTBEAT_MAX_AGE_SECS", + default_value_t = 20_i64 + )] + pub(crate) heartbeat_max_age_secs: i64, + #[arg( + long, + env = "BIGNAME_WORKER_REBUILD_PHASE_MAX_AGE_SECS", + default_value_t = bigname_storage::DEFAULT_WORKER_REBUILD_PHASE_MAX_AGE_SECS + )] + pub(crate) rebuild_phase_max_age_secs: i64, } #[derive(Args, Debug)] diff --git a/apps/worker/src/healthcheck.rs b/apps/worker/src/healthcheck.rs index edb5efae..3da602ff 100644 --- a/apps/worker/src/healthcheck.rs +++ b/apps/worker/src/healthcheck.rs @@ -9,6 +9,16 @@ pub(crate) async fn healthcheck(args: HealthcheckArgs) -> Result<()> { let pool = bigname_storage::connect(&args.database).await?; verify_database_reachable(&pool).await?; verify_migrations_current(&pool).await?; + let instance_id = + bigname_storage::resolve_service_instance_id(args.heartbeat_instance_id.as_deref())?; + bigname_storage::ensure_service_loop_heartbeat_recent_with_phase( + &pool, + bigname_storage::WORKER_SERVICE_NAME, + &instance_id, + args.heartbeat_max_age_secs, + args.rebuild_phase_max_age_secs, + ) + .await?; println!("ok"); Ok(()) } @@ -104,6 +114,9 @@ mod tests { match cli.command { Command::Healthcheck(args) => { assert_eq!(args.database.max_connections, 10); + assert_eq!(args.heartbeat_instance_id, None); + assert_eq!(args.heartbeat_max_age_secs, 20); + assert_eq!(args.rebuild_phase_max_age_secs, 43_200); } other => panic!("expected healthcheck command, got {other:?}"), } @@ -117,8 +130,17 @@ mod tests { "failed to apply migrations for worker healthcheck test", ) .await?; + bigname_storage::register_service_loop( + database.pool(), + bigname_storage::WORKER_SERVICE_NAME, + "worker-healthcheck-test", + ) + .await?; let result = healthcheck(HealthcheckArgs { database: database_config(&database)?, + heartbeat_instance_id: Some("worker-healthcheck-test".to_owned()), + heartbeat_max_age_secs: 20, + rebuild_phase_max_age_secs: 43_200, }) .await; database.cleanup().await?; @@ -133,6 +155,9 @@ mod tests { .await?; let error = healthcheck(HealthcheckArgs { database: database_config(&database)?, + heartbeat_instance_id: Some("worker-healthcheck-unmigrated".to_owned()), + heartbeat_max_age_secs: 20, + rebuild_phase_max_age_secs: 43_200, }) .await .expect_err("unmigrated database must fail healthcheck"); @@ -146,4 +171,132 @@ mod tests { ); Ok(()) } + + #[tokio::test] + async fn healthcheck_distinguishes_missing_and_stale_loop_heartbeats() -> Result<()> { + let database = bigname_test_support::TestDatabase::create_migrated( + bigname_test_support::TestDatabaseConfig::new("bigname_worker_healthcheck_heartbeat"), + &bigname_storage::MIGRATOR, + "failed to apply migrations for worker heartbeat healthcheck test", + ) + .await?; + + let missing_error = healthcheck(HealthcheckArgs { + database: database_config(&database)?, + heartbeat_instance_id: Some("missing-worker".to_owned()), + heartbeat_max_age_secs: 20, + rebuild_phase_max_age_secs: 43_200, + }) + .await + .expect_err("a worker loop that never started must fail healthcheck"); + assert!( + missing_error.to_string().contains("never started"), + "unexpected error: {missing_error:#}" + ); + + bigname_storage::register_service_loop( + database.pool(), + bigname_storage::WORKER_SERVICE_NAME, + "stale-worker", + ) + .await?; + sqlx::query( + r#" + UPDATE service_loop_heartbeats + SET started_at = clock_timestamp() - INTERVAL '2 minutes', + heartbeat_at = clock_timestamp() - INTERVAL '1 minute' + WHERE service_name = 'worker' + AND instance_id = 'stale-worker' + "#, + ) + .execute(database.pool()) + .await?; + let stale_error = healthcheck(HealthcheckArgs { + database: database_config(&database)?, + heartbeat_instance_id: Some("stale-worker".to_owned()), + heartbeat_max_age_secs: 20, + rebuild_phase_max_age_secs: 43_200, + }) + .await + .expect_err("a wedged worker loop must fail healthcheck"); + assert!( + stale_error.to_string().contains("stopped or wedged"), + "unexpected error: {stale_error:#}" + ); + + database.cleanup().await?; + Ok(()) + } + + #[tokio::test] + async fn healthcheck_uses_the_distinct_rebuild_phase_age_limit() -> Result<()> { + let database = bigname_test_support::TestDatabase::create_migrated( + bigname_test_support::TestDatabaseConfig::new( + "bigname_worker_healthcheck_rebuild_phase", + ), + &bigname_storage::MIGRATOR, + "failed to apply migrations for worker phase healthcheck test", + ) + .await?; + bigname_storage::register_service_loop( + database.pool(), + bigname_storage::WORKER_SERVICE_NAME, + "phase-worker", + ) + .await?; + bigname_storage::begin_service_loop_phase( + database.pool(), + bigname_storage::WORKER_SERVICE_NAME, + "phase-worker", + "name_current.publish", + ) + .await?; + sqlx::query( + r#" + UPDATE service_loop_heartbeats + SET started_at = clock_timestamp() - INTERVAL '2 minutes', + heartbeat_at = clock_timestamp() - INTERVAL '1 minute' + WHERE service_name = 'worker' + AND instance_id = 'phase-worker' + AND scope_kind = 'process' + "#, + ) + .execute(database.pool()) + .await?; + + healthcheck(HealthcheckArgs { + database: database_config(&database)?, + heartbeat_instance_id: Some("phase-worker".to_owned()), + heartbeat_max_age_secs: 20, + rebuild_phase_max_age_secs: 120, + }) + .await?; + + sqlx::query( + r#" + UPDATE service_loop_heartbeats + SET started_at = clock_timestamp() - INTERVAL '2 minutes', + heartbeat_at = clock_timestamp() - INTERVAL '1 minute' + WHERE service_name = 'worker' + AND instance_id = 'phase-worker' + AND scope_kind = 'phase' + "#, + ) + .execute(database.pool()) + .await?; + let stale_error = healthcheck(HealthcheckArgs { + database: database_config(&database)?, + heartbeat_instance_id: Some("phase-worker".to_owned()), + heartbeat_max_age_secs: 20, + rebuild_phase_max_age_secs: 20, + }) + .await + .expect_err("a rebuild phase older than its own limit must fail healthcheck"); + assert!( + stale_error.to_string().contains("name_current.publish"), + "unexpected error: {stale_error:#}" + ); + + database.cleanup().await + } } diff --git a/apps/worker/src/name_current.rs b/apps/worker/src/name_current.rs index 0d0268f1..8eff7b88 100644 --- a/apps/worker/src/name_current.rs +++ b/apps/worker/src/name_current.rs @@ -48,6 +48,10 @@ use types::RelevantEvent; #[cfg(test)] use uuid::Uuid; +use crate::primary_name::rebuild_heartbeat::{ + LoopHeartbeat, record_rebuild_progress, run_rebuild_phase, +}; + const ENS_NAMESPACE: &str = "ens"; const BASENAMES_NAMESPACE: &str = "basenames"; const ENS_V1_AUTHORITY_DERIVATION_KIND: &str = "ens_v1_unwrapped_authority"; @@ -114,15 +118,40 @@ pub struct NameCurrentRebuildSummary { pub async fn rebuild_name_current( pool: &PgPool, logical_name_id: Option<&str>, +) -> Result { + rebuild_name_current_inner(pool, logical_name_id, None).await +} + +pub(crate) async fn rebuild_name_current_with_heartbeat( + pool: &PgPool, + logical_name_id: Option<&str>, + loop_heartbeat: &mut LoopHeartbeat, +) -> Result { + rebuild_name_current_inner(pool, logical_name_id, Some(loop_heartbeat)).await +} + +async fn rebuild_name_current_inner( + pool: &PgPool, + logical_name_id: Option<&str>, + loop_heartbeat: Option<&mut LoopHeartbeat>, ) -> Result { match logical_name_id { Some(logical_name_id) => rebuild_one_name_current(pool, logical_name_id).await, - None => rebuild_all_name_current(pool).await, + None => rebuild_all_name_current(pool, loop_heartbeat).await, } } -async fn rebuild_all_name_current(pool: &PgPool) -> Result { - let names = load_canonical_name_surfaces(pool).await?; +async fn rebuild_all_name_current( + pool: &PgPool, + mut loop_heartbeat: Option<&mut LoopHeartbeat>, +) -> Result { + let names = run_rebuild_phase( + pool, + &mut loop_heartbeat, + "name_current.load_inputs", + load_canonical_name_surfaces(pool), + ) + .await?; let requested_name_count = names.len(); let mut replacement = NameCurrentReplacement::begin(pool).await?; let mut rows = Vec::with_capacity(NAME_CURRENT_REBUILD_STAGE_BATCH_SIZE); @@ -140,6 +169,7 @@ async fn rebuild_all_name_current(pool: &PgPool) -> Result= NAME_CURRENT_REBUILD_STAGE_BATCH_SIZE { replacement.stage_rows(&rows).await?; rows.clear(); @@ -163,7 +193,13 @@ async fn rebuild_all_name_current(pool: &PgPool) -> Result, +) -> Result { + rebuild_permissions_current_inner(pool, resource_id, None).await +} + +pub(crate) async fn rebuild_permissions_current_with_heartbeat( + pool: &PgPool, + resource_id: Option<&str>, + loop_heartbeat: &mut LoopHeartbeat, +) -> Result { + rebuild_permissions_current_inner(pool, resource_id, Some(loop_heartbeat)).await +} + +async fn rebuild_permissions_current_inner( + pool: &PgPool, + resource_id: Option<&str>, + loop_heartbeat: Option<&mut LoopHeartbeat>, ) -> Result { match resource_id { Some(resource_id) => rebuild_one_resource(pool, resource_id).await, - None => rebuild_all_resources(pool).await, + None => rebuild_all_resources(pool, loop_heartbeat).await, } } -async fn rebuild_all_resources(pool: &PgPool) -> Result { +async fn rebuild_all_resources( + pool: &PgPool, + mut loop_heartbeat: Option<&mut LoopHeartbeat>, +) -> Result { let mut conn = pool .acquire() .await @@ -70,7 +93,13 @@ async fn rebuild_all_resources(pool: &PgPool) -> Result Result Result Result { + hydration::hydrate_legacy_reverse_resolver_primary_names_with_heartbeat( + pool, + config, + loop_heartbeat, + ) + .await +} + pub async fn load_legacy_reverse_resolver_call_triggers( pool: &PgPool, config: &PrimaryNameLegacyReverseHydrationConfig, diff --git a/apps/worker/src/primary_name/hydration.rs b/apps/worker/src/primary_name/hydration.rs index 96e8f33f..945f64cf 100644 --- a/apps/worker/src/primary_name/hydration.rs +++ b/apps/worker/src/primary_name/hydration.rs @@ -6,14 +6,12 @@ use bigname_execution::{ EnsReverseNameMulticallRequest, EnsReverseNameMulticallResult, MULTICALL3_ADDRESS, execute_ens_reverse_name_multicall, lookup_ens_forward_address_at_block, }; -use bigname_storage::{ - ENS_LEGACY_EVENT_SILENT_REVERSE_RESOLVER_ADDRESSES, ETHEREUM_MAINNET_CHAIN_ID, - normalize_evm_address, -}; +use bigname_storage::{ENS_LEGACY_EVENT_SILENT_REVERSE_RESOLVER_ADDRESSES, normalize_evm_address}; use futures_util::{FutureExt, future::BoxFuture}; use serde_json::{Value, json}; -use sqlx::{PgPool, Row}; +use sqlx::PgPool; +use super::rebuild_heartbeat::{LoopHeartbeat, record_rebuild_progress, run_rebuild_phase}; use super::{ PrimaryNameLegacyReverseHydrationSummary, projection::{primary_name_row, primary_name_row_with_provenance_extensions}, @@ -28,10 +26,13 @@ mod invalidation; mod resolver_edge; #[path = "hydration/resolver_edge_query.rs"] mod resolver_edge_query; +#[path = "hydration/triggers.rs"] +mod triggers; use hydration_query::load_legacy_reverse_hydration_candidates; use invalidation::invalidate_changed_hydration_snapshots; use resolver_edge::hydrate_resolver_edge_candidates; use resolver_edge_query::load_legacy_reverse_resolver_edge_hydration_candidates; +pub(super) use triggers::load_legacy_reverse_resolver_call_triggers; #[cfg(test)] const LEGACY_EVENT_SILENT_REVERSE_RESOLVER_ADDRESS: &str = @@ -136,6 +137,10 @@ enum ReverseNameHydrationOutcome { } trait ReverseNameHydrationClient: Sync { + fn batch_size(&self) -> usize { + usize::MAX + } + fn hydrate<'a>( &'a self, chain_id: &'a str, @@ -158,6 +163,10 @@ struct MulticallReverseNameHydrationClient { } impl ReverseNameHydrationClient for MulticallReverseNameHydrationClient { + fn batch_size(&self) -> usize { + self.config.batch_size.max(1) + } + fn hydrate<'a>( &'a self, chain_id: &'a str, @@ -241,92 +250,60 @@ pub(super) async fn hydrate_legacy_reverse_resolver_primary_names( .await } -pub(super) async fn load_legacy_reverse_resolver_call_triggers( +pub(super) async fn hydrate_legacy_reverse_resolver_primary_names_with_heartbeat( pool: &PgPool, - config: &PrimaryNameLegacyReverseHydrationConfig, -) -> Result> { + config: PrimaryNameLegacyReverseHydrationConfig, + loop_heartbeat: &mut LoopHeartbeat, +) -> Result { let resolver_addresses = normalize_resolver_addresses(&config.resolver_addresses); - if resolver_addresses.is_empty() { - return Ok(Vec::new()); - } - - let rows = sqlx::query( - r#" - WITH chain_positions AS ( - SELECT - chain_id, - canonical_block_number AS hydration_block_number, - canonical_block_hash AS hydration_block_hash - FROM chain_checkpoints - WHERE chain_id = $2 - AND canonical_block_number IS NOT NULL - AND canonical_block_hash IS NOT NULL - ) - SELECT DISTINCT ON (LOWER(esc.resolver_address)) - LOWER(esc.resolver_address) AS resolver_address, - esc.block_number, - esc.block_hash, - esc.transaction_hash, - esc.transaction_index - FROM event_silent_resolver_call_observations esc - JOIN chain_positions - ON chain_positions.chain_id = esc.chain_id - AND esc.block_number <= chain_positions.hydration_block_number - WHERE esc.chain_id = $2 - AND LOWER(esc.resolver_address) = ANY($1::TEXT[]) - AND esc.canonicality_state IN ( - 'canonical'::canonicality_state, - 'safe'::canonicality_state, - 'finalized'::canonicality_state - ) - ORDER BY - LOWER(esc.resolver_address) ASC, - esc.block_number DESC, - esc.transaction_index DESC, - esc.transaction_hash DESC - "#, + let client = MulticallReverseNameHydrationClient { config }; + hydrate_legacy_reverse_resolver_primary_names_with_client_inner( + pool, + &resolver_addresses, + &client, + Some(loop_heartbeat), ) - .bind(&resolver_addresses) - .bind(ETHEREUM_MAINNET_CHAIN_ID) - .fetch_all(pool) .await - .context("failed to load latest legacy reverse-resolver direct-call triggers")?; - - rows.into_iter() - .map(|row| { - Ok(PrimaryNameLegacyReverseHydrationTrigger { - resolver_address: row - .try_get("resolver_address") - .context("missing legacy reverse hydration trigger resolver_address")?, - block_number: row - .try_get("block_number") - .context("missing legacy reverse hydration trigger block_number")?, - block_hash: row - .try_get("block_hash") - .context("missing legacy reverse hydration trigger block_hash")?, - transaction_hash: row - .try_get("transaction_hash") - .context("missing legacy reverse hydration trigger transaction_hash")?, - transaction_index: row - .try_get("transaction_index") - .context("missing legacy reverse hydration trigger transaction_index")?, - }) - }) - .collect() } async fn hydrate_legacy_reverse_resolver_primary_names_with_client( pool: &PgPool, resolver_addresses: &[String], client: &dyn ReverseNameHydrationClient, +) -> Result { + hydrate_legacy_reverse_resolver_primary_names_with_client_inner( + pool, + resolver_addresses, + client, + None, + ) + .await +} + +async fn hydrate_legacy_reverse_resolver_primary_names_with_client_inner( + pool: &PgPool, + resolver_addresses: &[String], + client: &dyn ReverseNameHydrationClient, + mut loop_heartbeat: Option<&mut LoopHeartbeat>, ) -> Result { if resolver_addresses.is_empty() { return Ok(PrimaryNameLegacyReverseHydrationSummary::default()); } - let candidates = load_legacy_reverse_hydration_candidates(pool, resolver_addresses).await?; - let resolver_edge_candidates = - load_legacy_reverse_resolver_edge_hydration_candidates(pool, resolver_addresses).await?; + let candidates = run_rebuild_phase( + pool, + &mut loop_heartbeat, + "primary_names_current.legacy_hydration.load_reverse_claim_candidates", + load_legacy_reverse_hydration_candidates(pool, resolver_addresses), + ) + .await?; + let resolver_edge_candidates = run_rebuild_phase( + pool, + &mut loop_heartbeat, + "primary_names_current.legacy_hydration.load_resolver_edge_candidates", + load_legacy_reverse_resolver_edge_hydration_candidates(pool, resolver_addresses), + ) + .await?; let mut summary = PrimaryNameLegacyReverseHydrationSummary { candidate_tuple_count: candidates.len() + resolver_edge_candidates.len(), ..PrimaryNameLegacyReverseHydrationSummary::default() @@ -336,22 +313,24 @@ async fn hydrate_legacy_reverse_resolver_primary_names_with_client( } let mut snapshots = Vec::new(); - for candidate in candidates + for (candidate_index, candidate) in candidates .iter() .filter(|candidate| candidate.hydration_target.is_none()) + .enumerate() { let snapshot = baseline_snapshot(candidate)?; add_snapshot_status(&mut summary, &snapshot); snapshots.push(snapshot); + if candidate_index % LEGACY_REVERSE_HYDRATION_UPSERT_BATCH_SIZE == 0 { + record_rebuild_progress(pool, &mut loop_heartbeat).await; + } } - let calls_by_position = candidates.iter().enumerate().fold( - BTreeMap::<(String, i64, String), Vec<(usize, ReverseNameHydrationCall)>>::new(), - |mut by_position, (index, candidate)| { - let Some(target) = candidate.hydration_target.as_ref() else { - return by_position; - }; - by_position + let mut calls_by_position = + BTreeMap::<(String, i64, String), Vec<(usize, ReverseNameHydrationCall)>>::new(); + for (index, candidate) in candidates.iter().enumerate() { + if let Some(target) = candidate.hydration_target.as_ref() { + calls_by_position .entry(( target.chain_id.clone(), target.position.block_number, @@ -365,86 +344,92 @@ async fn hydrate_legacy_reverse_resolver_primary_names_with_client( reverse_node: target.reverse_node.clone(), }, )); - by_position - }, - ); + } + if index % LEGACY_REVERSE_HYDRATION_UPSERT_BATCH_SIZE == 0 { + record_rebuild_progress(pool, &mut loop_heartbeat).await; + } + } for ((chain_id, block_number, block_hash), calls_with_refs) in calls_by_position { let position = ReverseNameHydrationChainPosition { block_number, block_hash, }; - let calls = calls_with_refs - .iter() - .map(|(_, call)| call.clone()) - .collect::>(); - summary.queried_tuple_count += calls.len(); - let outcomes = match client.hydrate(&chain_id, &position, &calls).await { - Ok(outcomes) => outcomes, - Err(error) => { - summary.failed_lookup_count += calls.len(); - tracing::warn!( - service = "worker", - projection = "primary_names_current", - chain_id, - error = %format!("{error:#}"), - failed_lookup_count = calls.len(), - "legacy reverse-resolver primary-name hydration batch failed" - ); - for (candidate_index, _) in &calls_with_refs { - let candidate = candidates.get(*candidate_index).context( - "legacy reverse-resolver hydration candidate reference is out of bounds", - )?; - if candidate.has_existing_hydration { - let snapshot = baseline_snapshot(candidate)?; - add_snapshot_status(&mut summary, &snapshot); - snapshots.push(snapshot); + for calls_chunk in calls_with_refs.chunks(client.batch_size().max(1)) { + let calls = calls_chunk + .iter() + .map(|(_, call)| call.clone()) + .collect::>(); + summary.queried_tuple_count += calls.len(); + let outcomes = match client.hydrate(&chain_id, &position, &calls).await { + Ok(outcomes) => outcomes, + Err(error) => { + summary.failed_lookup_count += calls.len(); + tracing::warn!( + service = "worker", + projection = "primary_names_current", + chain_id, + error = %format!("{error:#}"), + failed_lookup_count = calls.len(), + "legacy reverse-resolver primary-name hydration batch failed" + ); + for (candidate_index, _) in calls_chunk { + let candidate = candidates.get(*candidate_index).context( + "legacy reverse-resolver hydration candidate reference is out of bounds", + )?; + if candidate.has_existing_hydration { + let snapshot = baseline_snapshot(candidate)?; + add_snapshot_status(&mut summary, &snapshot); + snapshots.push(snapshot); + } } + record_rebuild_progress(pool, &mut loop_heartbeat).await; + continue; } - continue; + }; + if outcomes.len() != calls_chunk.len() { + anyhow::bail!( + "legacy reverse-resolver hydration provider returned {} outcomes for {} calls on {chain_id}", + outcomes.len(), + calls_chunk.len() + ); } - }; - if outcomes.len() != calls_with_refs.len() { - anyhow::bail!( - "legacy reverse-resolver hydration provider returned {} outcomes for {} calls on {chain_id}", - outcomes.len(), - calls_with_refs.len() - ); - } - for ((candidate_index, _), outcome) in calls_with_refs.iter().zip(outcomes) { - let candidate = candidates.get(*candidate_index).context( - "legacy reverse-resolver hydration candidate reference is out of bounds", - )?; - let target = candidate.hydration_target.as_ref().context( - "legacy reverse-resolver hydration candidate is missing hydration target", - )?; - let raw_name = match outcome { - ReverseNameHydrationOutcome::Success(value) => value, - ReverseNameHydrationOutcome::NotFound => String::new(), - ReverseNameHydrationOutcome::Failed(_) => { - summary.failed_lookup_count += 1; - if candidate.has_existing_hydration { - let snapshot = baseline_snapshot(candidate)?; - add_snapshot_status(&mut summary, &snapshot); - snapshots.push(snapshot); + for ((candidate_index, _), outcome) in calls_chunk.iter().zip(outcomes) { + let candidate = candidates.get(*candidate_index).context( + "legacy reverse-resolver hydration candidate reference is out of bounds", + )?; + let target = candidate.hydration_target.as_ref().context( + "legacy reverse-resolver hydration candidate is missing hydration target", + )?; + let raw_name = match outcome { + ReverseNameHydrationOutcome::Success(value) => value, + ReverseNameHydrationOutcome::NotFound => String::new(), + ReverseNameHydrationOutcome::Failed(_) => { + summary.failed_lookup_count += 1; + if candidate.has_existing_hydration { + let snapshot = baseline_snapshot(candidate)?; + add_snapshot_status(&mut summary, &snapshot); + snapshots.push(snapshot); + } + continue; } - continue; - } - }; - let claim_observation = NameClaimObservation { - key: candidate.tuple.key.clone(), - raw_name: Some(raw_name), - primary_claim_source: target.primary_claim_source.clone(), - }; - let hydration_provenance = hydration_provenance(target, &position); - let snapshot = primary_name_row_with_provenance_extensions( - &candidate.tuple, - Some(&claim_observation), - [(HYDRATION_PROVENANCE_KEY, hydration_provenance)], - )?; - add_snapshot_status(&mut summary, &snapshot); - snapshots.push(snapshot); + }; + let claim_observation = NameClaimObservation { + key: candidate.tuple.key.clone(), + raw_name: Some(raw_name), + primary_claim_source: target.primary_claim_source.clone(), + }; + let hydration_provenance = hydration_provenance(target, &position); + let snapshot = primary_name_row_with_provenance_extensions( + &candidate.tuple, + Some(&claim_observation), + [(HYDRATION_PROVENANCE_KEY, hydration_provenance)], + )?; + add_snapshot_status(&mut summary, &snapshot); + snapshots.push(snapshot); + } + record_rebuild_progress(pool, &mut loop_heartbeat).await; } } @@ -454,22 +439,36 @@ async fn hydrate_legacy_reverse_resolver_primary_names_with_client( client, &mut summary, &mut snapshots, + &mut loop_heartbeat, ) .await?; - summary.upserted_row_count = upsert_hydration_snapshots_in_batches( + summary.upserted_row_count = upsert_hydration_snapshots_in_batches_inner( pool, &snapshots, LEGACY_REVERSE_HYDRATION_UPSERT_BATCH_SIZE, + &mut loop_heartbeat, ) .await?; Ok(summary) } +#[cfg(test)] async fn upsert_hydration_snapshots_in_batches( pool: &PgPool, snapshots: &[bigname_storage::PrimaryNameCurrentSnapshot], batch_size: usize, +) -> Result { + let mut loop_heartbeat = None; + upsert_hydration_snapshots_in_batches_inner(pool, snapshots, batch_size, &mut loop_heartbeat) + .await +} + +async fn upsert_hydration_snapshots_in_batches_inner( + pool: &PgPool, + snapshots: &[bigname_storage::PrimaryNameCurrentSnapshot], + batch_size: usize, + loop_heartbeat: &mut Option<&mut LoopHeartbeat>, ) -> Result { if snapshots.is_empty() { return Ok(0); @@ -487,6 +486,7 @@ async fn upsert_hydration_snapshots_in_batches( format!("failed to upsert legacy reverse hydration snapshot batch {batch_index}") })? .len(); + record_rebuild_progress(pool, loop_heartbeat).await; } Ok(upserted_row_count) } diff --git a/apps/worker/src/primary_name/hydration/resolver_edge.rs b/apps/worker/src/primary_name/hydration/resolver_edge.rs index 531dc4de..367b455c 100644 --- a/apps/worker/src/primary_name/hydration/resolver_edge.rs +++ b/apps/worker/src/primary_name/hydration/resolver_edge.rs @@ -9,6 +9,7 @@ use bigname_storage::{ENS_NAMESPACE, normalize_evm_address}; use serde_json::{Value, json}; use sqlx::PgPool; +use super::super::rebuild_heartbeat::LoopHeartbeat; use super::super::{ PrimaryNameLegacyReverseHydrationSummary, projection::primary_name_row_with_provenance_extensions, @@ -19,7 +20,7 @@ use super::{ ResolverEdgeHydrationCandidate, ResolverEdgeHydrationTarget, ReverseNameHydrationCall, ReverseNameHydrationChainPosition, ReverseNameHydrationClient, ReverseNameHydrationOutcome, SOURCE_FAMILY_ENS_V1_REVERSE_L1, TUPLE_SOURCE_RESOLVER_EDGE_FORWARD_CONFIRMED, - add_snapshot_status, + add_snapshot_status, record_rebuild_progress, }; pub(super) async fn hydrate_resolver_edge_candidates( @@ -28,6 +29,7 @@ pub(super) async fn hydrate_resolver_edge_candidates( client: &dyn ReverseNameHydrationClient, summary: &mut PrimaryNameLegacyReverseHydrationSummary, snapshots: &mut Vec, + loop_heartbeat: &mut Option<&mut LoopHeartbeat>, ) -> Result<()> { for candidate in candidates .iter() @@ -46,16 +48,15 @@ pub(super) async fn hydrate_resolver_edge_candidates( &existing_key.coin_type, ) .await?; + record_rebuild_progress(pool, loop_heartbeat).await; } } - let calls_by_position = candidates.iter().enumerate().fold( - BTreeMap::<(String, i64, String), Vec<(usize, ReverseNameHydrationCall)>>::new(), - |mut by_position, (index, candidate)| { - let Some(target) = candidate.hydration_target.as_ref() else { - return by_position; - }; - by_position + let mut calls_by_position = + BTreeMap::<(String, i64, String), Vec<(usize, ReverseNameHydrationCall)>>::new(); + for (index, candidate) in candidates.iter().enumerate() { + if let Some(target) = candidate.hydration_target.as_ref() { + calls_by_position .entry(( target.chain_id.clone(), target.position.block_number, @@ -69,131 +70,138 @@ pub(super) async fn hydrate_resolver_edge_candidates( reverse_node: target.reverse_node.clone(), }, )); - by_position - }, - ); + } + if index % 1_000 == 0 { + record_rebuild_progress(pool, loop_heartbeat).await; + } + } for ((chain_id, block_number, block_hash), calls_with_refs) in calls_by_position { let position = ReverseNameHydrationChainPosition { block_number, block_hash, }; - let calls = calls_with_refs - .iter() - .map(|(_, call)| call.clone()) - .collect::>(); - summary.queried_tuple_count += calls.len(); - let outcomes = match client.hydrate(&chain_id, &position, &calls).await { - Ok(outcomes) => outcomes, - Err(error) => { - summary.failed_lookup_count += calls.len(); - tracing::warn!( - service = "worker", - projection = "primary_names_current", - chain_id, - error = %format!("{error:#}"), - failed_lookup_count = calls.len(), - "legacy reverse-resolver resolver-edge hydration batch failed" - ); - continue; - } - }; - if outcomes.len() != calls_with_refs.len() { - anyhow::bail!( - "legacy reverse-resolver resolver-edge hydration provider returned {} outcomes for {} calls on {chain_id}", - outcomes.len(), - calls_with_refs.len() - ); - } - - for ((candidate_index, _), outcome) in calls_with_refs.iter().zip(outcomes) { - let candidate = candidates.get(*candidate_index).context( - "legacy reverse-resolver resolver-edge candidate reference is out of bounds", - )?; - let target = candidate.hydration_target.as_ref().context( - "legacy reverse-resolver resolver-edge candidate is missing hydration target", - )?; - let raw_name = match outcome { - ReverseNameHydrationOutcome::Success(value) => value, - ReverseNameHydrationOutcome::NotFound => { - delete_existing_resolver_edge_row(pool, candidate, summary).await?; - continue; - } - ReverseNameHydrationOutcome::Failed(_) => { - summary.failed_lookup_count += 1; + for calls_chunk in calls_with_refs.chunks(client.batch_size().max(1)) { + let calls = calls_chunk + .iter() + .map(|(_, call)| call.clone()) + .collect::>(); + summary.queried_tuple_count += calls.len(); + let outcomes = match client.hydrate(&chain_id, &position, &calls).await { + Ok(outcomes) => outcomes, + Err(error) => { + summary.failed_lookup_count += calls.len(); + tracing::warn!( + service = "worker", + projection = "primary_names_current", + chain_id, + error = %format!("{error:#}"), + failed_lookup_count = calls.len(), + "legacy reverse-resolver resolver-edge hydration batch failed" + ); + record_rebuild_progress(pool, loop_heartbeat).await; continue; } }; - let Some(normalized_name) = normalize_hydrated_name(&raw_name) else { - delete_existing_resolver_edge_row(pool, candidate, summary).await?; - continue; - }; - if raw_name != normalized_name { - summary.claim_not_normalized_count += 1; - tracing::debug!( - service = "worker", - projection = "primary_names_current", - chain_id, - reverse_node = target.reverse_node, - raw_name, - normalized_name, - failure_reason = VERIFIED_PRIMARY_NAME_CLAIM_NOT_NORMALIZED_REASON, - "legacy reverse-resolver resolver-edge claim is not already ENSIP-15 normalized" + if outcomes.len() != calls_chunk.len() { + anyhow::bail!( + "legacy reverse-resolver resolver-edge hydration provider returned {} outcomes for {} calls on {chain_id}", + outcomes.len(), + calls_chunk.len() ); - delete_existing_resolver_edge_row(pool, candidate, summary).await?; - continue; } - let address = match client - .lookup_forward_address(&chain_id, &position, &normalized_name) - .await - { - Ok(Some(address)) => address, - Ok(None) => { + + for ((candidate_index, _), outcome) in calls_chunk.iter().zip(outcomes) { + let candidate = candidates.get(*candidate_index).context( + "legacy reverse-resolver resolver-edge candidate reference is out of bounds", + )?; + let target = candidate.hydration_target.as_ref().context( + "legacy reverse-resolver resolver-edge candidate is missing hydration target", + )?; + let raw_name = match outcome { + ReverseNameHydrationOutcome::Success(value) => value, + ReverseNameHydrationOutcome::NotFound => { + delete_existing_resolver_edge_row(pool, candidate, summary).await?; + continue; + } + ReverseNameHydrationOutcome::Failed(_) => { + summary.failed_lookup_count += 1; + continue; + } + }; + let Some(normalized_name) = normalize_hydrated_name(&raw_name) else { + delete_existing_resolver_edge_row(pool, candidate, summary).await?; + continue; + }; + if raw_name != normalized_name { + summary.claim_not_normalized_count += 1; + tracing::debug!( + service = "worker", + projection = "primary_names_current", + chain_id, + reverse_node = target.reverse_node, + raw_name, + normalized_name, + failure_reason = VERIFIED_PRIMARY_NAME_CLAIM_NOT_NORMALIZED_REASON, + "legacy reverse-resolver resolver-edge claim is not already ENSIP-15 normalized" + ); delete_existing_resolver_edge_row(pool, candidate, summary).await?; continue; } - Err(error) => { - if is_universal_resolver_non_confirmation(&error) - || is_new_row_offchain_non_confirmation(candidate, &error) - { + let address = match client + .lookup_forward_address(&chain_id, &position, &normalized_name) + .await + { + Ok(Some(address)) => address, + Ok(None) => { delete_existing_resolver_edge_row(pool, candidate, summary).await?; - } else { - summary.failed_lookup_count += 1; - tracing::warn!( - service = "worker", - projection = "primary_names_current", - chain_id, - reverse_node = target.reverse_node, - normalized_name, - error = %format!("{error:#}"), - "legacy reverse-resolver resolver-edge forward confirmation failed" - ); + continue; + } + Err(error) => { + if is_universal_resolver_non_confirmation(&error) + || is_new_row_offchain_non_confirmation(candidate, &error) + { + delete_existing_resolver_edge_row(pool, candidate, summary).await?; + } else { + summary.failed_lookup_count += 1; + tracing::warn!( + service = "worker", + projection = "primary_names_current", + chain_id, + reverse_node = target.reverse_node, + normalized_name, + error = %format!("{error:#}"), + "legacy reverse-resolver resolver-edge forward confirmation failed" + ); + } + continue; } + }; + let normalized_address = normalize_evm_address(&address); + if !forward_address_matches_reverse_node(&normalized_address, &target.reverse_node)? + { + delete_existing_resolver_edge_row(pool, candidate, summary).await?; continue; } - }; - let normalized_address = normalize_evm_address(&address); - if !forward_address_matches_reverse_node(&normalized_address, &target.reverse_node)? { - delete_existing_resolver_edge_row(pool, candidate, summary).await?; - continue; - } - let tuple = resolver_edge_tuple(&normalized_address); - let primary_claim_source = - resolver_edge_primary_claim_source(&tuple.key, &target.reverse_node); - let claim_observation = NameClaimObservation { - key: tuple.key.clone(), - raw_name: Some(raw_name), - primary_claim_source, - }; - let hydration_provenance = resolver_edge_hydration_provenance(target, &position); - let snapshot = primary_name_row_with_provenance_extensions( - &tuple, - Some(&claim_observation), - [(HYDRATION_PROVENANCE_KEY, hydration_provenance)], - )?; - add_snapshot_status(summary, &snapshot); - snapshots.push(snapshot); + let tuple = resolver_edge_tuple(&normalized_address); + let primary_claim_source = + resolver_edge_primary_claim_source(&tuple.key, &target.reverse_node); + let claim_observation = NameClaimObservation { + key: tuple.key.clone(), + raw_name: Some(raw_name), + primary_claim_source, + }; + let hydration_provenance = resolver_edge_hydration_provenance(target, &position); + let snapshot = primary_name_row_with_provenance_extensions( + &tuple, + Some(&claim_observation), + [(HYDRATION_PROVENANCE_KEY, hydration_provenance)], + )?; + add_snapshot_status(summary, &snapshot); + snapshots.push(snapshot); + } + record_rebuild_progress(pool, loop_heartbeat).await; } } diff --git a/apps/worker/src/primary_name/hydration/triggers.rs b/apps/worker/src/primary_name/hydration/triggers.rs new file mode 100644 index 00000000..7aec442d --- /dev/null +++ b/apps/worker/src/primary_name/hydration/triggers.rs @@ -0,0 +1,81 @@ +use anyhow::{Context, Result}; +use sqlx::{PgPool, Row}; + +use super::{ + PrimaryNameLegacyReverseHydrationConfig, PrimaryNameLegacyReverseHydrationTrigger, + normalize_resolver_addresses, +}; + +pub(crate) async fn load_legacy_reverse_resolver_call_triggers( + pool: &PgPool, + config: &PrimaryNameLegacyReverseHydrationConfig, +) -> Result> { + let resolver_addresses = normalize_resolver_addresses(&config.resolver_addresses); + if resolver_addresses.is_empty() { + return Ok(Vec::new()); + } + + let rows = sqlx::query( + r#" + WITH chain_positions AS ( + SELECT + chain_id, + canonical_block_number AS hydration_block_number, + canonical_block_hash AS hydration_block_hash + FROM chain_checkpoints + WHERE chain_id = $2 + AND canonical_block_number IS NOT NULL + AND canonical_block_hash IS NOT NULL + ) + SELECT DISTINCT ON (LOWER(esc.resolver_address)) + LOWER(esc.resolver_address) AS resolver_address, + esc.block_number, + esc.block_hash, + esc.transaction_hash, + esc.transaction_index + FROM event_silent_resolver_call_observations esc + JOIN chain_positions + ON chain_positions.chain_id = esc.chain_id + AND esc.block_number <= chain_positions.hydration_block_number + WHERE esc.chain_id = $2 + AND LOWER(esc.resolver_address) = ANY($1::TEXT[]) + AND esc.canonicality_state IN ( + 'canonical'::canonicality_state, + 'safe'::canonicality_state, + 'finalized'::canonicality_state + ) + ORDER BY + LOWER(esc.resolver_address) ASC, + esc.block_number DESC, + esc.transaction_index DESC, + esc.transaction_hash DESC + "#, + ) + .bind(&resolver_addresses) + .bind(bigname_storage::ETHEREUM_MAINNET_CHAIN_ID) + .fetch_all(pool) + .await + .context("failed to load latest legacy reverse-resolver direct-call triggers")?; + + rows.into_iter() + .map(|row| { + Ok(PrimaryNameLegacyReverseHydrationTrigger { + resolver_address: row + .try_get("resolver_address") + .context("missing legacy reverse hydration trigger resolver_address")?, + block_number: row + .try_get("block_number") + .context("missing legacy reverse hydration trigger block_number")?, + block_hash: row + .try_get("block_hash") + .context("missing legacy reverse hydration trigger block_hash")?, + transaction_hash: row + .try_get("transaction_hash") + .context("missing legacy reverse hydration trigger transaction_hash")?, + transaction_index: row + .try_get("transaction_index") + .context("missing legacy reverse hydration trigger transaction_index")?, + }) + }) + .collect() +} diff --git a/apps/worker/src/primary_name/projection.rs b/apps/worker/src/primary_name/projection.rs index 6f22cdeb..0d738891 100644 --- a/apps/worker/src/primary_name/projection.rs +++ b/apps/worker/src/primary_name/projection.rs @@ -10,6 +10,8 @@ use futures_util::{TryStreamExt, pin_mut}; use serde_json::{Map, Value, json}; use sqlx::{PgConnection, PgPool}; +use super::rebuild_heartbeat::{LoopHeartbeat, record_rebuild_progress, run_rebuild_phase}; + #[allow(clippy::duplicate_mod)] #[path = "../staged_rebuild.rs"] mod staged_rebuild; @@ -77,25 +79,55 @@ pub async fn rebuild_primary_names_current( address: Option<&str>, namespace: Option<&str>, coin_type: Option<&str>, +) -> Result { + rebuild_primary_names_current_inner(pool, address, namespace, coin_type, None).await +} + +pub(crate) async fn rebuild_primary_names_current_with_heartbeat( + pool: &PgPool, + address: Option<&str>, + namespace: Option<&str>, + coin_type: Option<&str>, + loop_heartbeat: &mut LoopHeartbeat, +) -> Result { + rebuild_primary_names_current_inner(pool, address, namespace, coin_type, Some(loop_heartbeat)) + .await +} + +async fn rebuild_primary_names_current_inner( + pool: &PgPool, + address: Option<&str>, + namespace: Option<&str>, + coin_type: Option<&str>, + loop_heartbeat: Option<&mut LoopHeartbeat>, ) -> Result { match (address, namespace, coin_type) { (Some(address), Some(namespace), Some(coin_type)) => { rebuild_one_primary_name(pool, address, namespace, coin_type).await } - (None, None, None) => rebuild_all_primary_names(pool).await, + (None, None, None) => rebuild_all_primary_names(pool, loop_heartbeat).await, _ => bail!( "primary_names_current rebuild requires address, namespace, and coin_type together when targeting one tuple" ), } } -async fn rebuild_all_primary_names(pool: &PgPool) -> Result { +async fn rebuild_all_primary_names( + pool: &PgPool, + mut loop_heartbeat: Option<&mut LoopHeartbeat>, +) -> Result { let mut conn = pool .acquire() .await .context("failed to acquire primary_names_current staging connection")?; let stage_table = create_stage_table(&mut conn, "primary_names_current").await?; - let previous_row_count = count_rows(&mut conn, "primary_names_current", None).await?; + let previous_row_count = run_rebuild_phase( + pool, + &mut loop_heartbeat, + "primary_names_current.count_existing", + count_rows(&mut conn, "primary_names_current", None), + ) + .await?; let mut projections = Vec::with_capacity(PRIMARY_NAMES_CURRENT_REBUILD_BATCH_SIZE); let mut status_counts = StatusCounts::default(); let mut requested_tuple_count = 0usize; @@ -107,6 +139,7 @@ async fn rebuild_all_primary_names(pool: &PgPool) -> Result Result, + loop_heartbeat: &mut LoopHeartbeat, ) -> Result { - let derived = derive_once(pool).await?; - let applied = apply::apply_pending_invalidations( + let derived = derive_once_with_heartbeat(pool, loop_heartbeat).await?; + let applied = apply::apply_pending_invalidations_with_heartbeat( pool, PROJECTION_APPLY_BATCH_LIMIT, text_hydration_config, + loop_heartbeat, ) .await?; @@ -88,7 +92,44 @@ pub(crate) async fn run_once( pub(crate) async fn derive_once( pool: &PgPool, ) -> Result { - derive::derive_normalized_event_invalidations(pool, NORMALIZED_EVENT_DERIVE_BATCH_LIMIT).await + derive_once_inner(pool, None).await +} + +async fn derive_once_with_heartbeat( + pool: &PgPool, + loop_heartbeat: &mut LoopHeartbeat, +) -> Result { + derive_once_inner(pool, Some(loop_heartbeat)).await +} + +async fn derive_once_inner( + pool: &PgPool, + mut loop_heartbeat: Option<&mut LoopHeartbeat>, +) -> Result { + let complete_upper = derive::capture_normalized_event_change_watermark(pool).await?; + let mut remaining = NORMALIZED_EVENT_DERIVE_BATCH_LIMIT; + let mut summary = derive::ProjectionInvalidationDeriveSummary::default(); + + while remaining > 0 { + let derived = derive::derive_normalized_event_invalidations_through( + pool, + remaining.min(NORMALIZED_EVENT_DERIVE_PROGRESS_LIMIT), + complete_upper, + ) + .await?; + if derived.scanned_event_count == 0 { + break; + } + + remaining -= derived.scanned_event_count; + summary.scanned_event_count += derived.scanned_event_count; + summary.enqueued_invalidation_count += derived.enqueued_invalidation_count; + if let Some(loop_heartbeat) = loop_heartbeat.as_deref_mut() { + loop_heartbeat.record_if_due(pool).await; + } + } + + Ok(summary) } pub(crate) async fn has_primary_hydration_blocking_work(pool: &PgPool) -> Result { @@ -160,3 +201,246 @@ pub(crate) async fn load_chain_checkpoint_max_block(pool: &PgPool) -> Result Result<(i64, Transaction<'static, Postgres>)> { + sqlx::query( + r#" + INSERT INTO normalized_events ( + event_identity, + namespace, + logical_name_id, + event_kind, + source_family, + manifest_version, + raw_fact_ref, + derivation_kind, + canonicality_state, + before_state, + after_state, + observed_at + ) + SELECT + 'projection-derive-heartbeat-' || series::TEXT, + 'ens', + CASE WHEN series = $1 + 1 THEN 'ens:blocked.eth' END, + 'HeartbeatProgress', + 'test', + 1, + '{}'::jsonb, + 'heartbeat_progress_test', + 'canonical'::canonicality_state, + '{}'::jsonb, + '{}'::jsonb, + clock_timestamp() + FROM generate_series(1::BIGINT, $1 + 1) AS series + "#, + ) + .bind(NORMALIZED_EVENT_DERIVE_PROGRESS_LIMIT) + .execute(database.pool()) + .await?; + let first_progress_change_id = sqlx::query_scalar::<_, i64>( + r#" + SELECT change_id + FROM projection_normalized_event_changes + ORDER BY change_id + LIMIT 1 OFFSET $1 + "#, + ) + .bind(NORMALIZED_EVENT_DERIVE_PROGRESS_LIMIT - 1) + .fetch_one(database.pool()) + .await?; + + sqlx::query( + r#" + INSERT INTO projection_invalidations ( + projection, + projection_key, + key_payload + ) + VALUES ( + 'name_current', + 'ens:blocked.eth', + '{"logical_name_id":"ens:blocked.eth"}'::jsonb + ) + "#, + ) + .execute(database.pool()) + .await?; + let mut later_unit_blocker = database.pool().begin().await?; + sqlx::query( + r#" + UPDATE projection_invalidations + SET generation = generation + WHERE projection = 'name_current' + AND projection_key = 'ens:blocked.eth' + "#, + ) + .execute(&mut *later_unit_blocker) + .await?; + + Ok((first_progress_change_id, later_unit_blocker)) + } + + async fn wait_for_derive_cursor(database: &TestDatabase, expected: i64) -> Result<()> { + loop { + let cursor = sqlx::query_scalar::<_, i64>( + r#" + SELECT last_change_id + FROM projection_apply_cursors + WHERE cursor_name = $1 + "#, + ) + .bind(NORMALIZED_EVENT_CURSOR) + .fetch_optional(database.pool()) + .await?; + if cursor == Some(expected) { + return Ok(()); + } + sleep(Duration::from_millis(10)).await; + } + } + + async fn wait_for_fresh_worker_heartbeat( + database: &TestDatabase, + instance_id: &str, + ) -> Result<()> { + loop { + let heartbeat = bigname_storage::load_service_loop_heartbeat( + database.pool(), + bigname_storage::WORKER_SERVICE_NAME, + instance_id, + ) + .await? + .context("projection derive must retain its registered heartbeat")?; + if heartbeat.age_seconds <= 1 { + return Ok(()); + } + sleep(Duration::from_millis(10)).await; + } + } + + #[tokio::test] + async fn detached_derive_commits_before_a_later_progress_unit_finishes() -> Result<()> { + let database = TestDatabase::create_migrated( + TestDatabaseConfig::new("bigname_worker_detached_projection_derive_progress_test") + .pool_max_connections(5), + &bigname_storage::MIGRATOR, + "failed to migrate detached projection derive progress test database", + ) + .await?; + let (first_progress_change_id, later_unit_blocker) = + seed_blocked_later_progress_unit(&database).await?; + + let derive_pool = database.pool().clone(); + let derive = tokio::spawn(async move { derive_once(&derive_pool).await }); + match timeout( + Duration::from_secs(2), + wait_for_derive_cursor(&database, first_progress_change_id), + ) + .await + { + Ok(result) => result?, + Err(error) => { + derive.abort(); + later_unit_blocker.rollback().await?; + let _ = derive.await; + database.cleanup().await?; + return Err(error) + .context("detached derive did not commit a bounded progress unit"); + } + } + assert!( + !derive.is_finished(), + "the later derive unit must still be blocked after earlier progress commits" + ); + + later_unit_blocker.commit().await?; + let summary = timeout(Duration::from_secs(10), derive) + .await + .context("detached derive did not finish after the later unit was released")? + .context("detached derive task failed")??; + assert_eq!( + summary.scanned_event_count, + NORMALIZED_EVENT_DERIVE_PROGRESS_LIMIT + 1 + ); + database.cleanup().await + } + + #[tokio::test] + async fn large_derive_batch_beats_before_a_later_progress_unit_finishes() -> Result<()> { + let database = TestDatabase::create_migrated( + TestDatabaseConfig::new("bigname_worker_projection_derive_heartbeat_test") + .pool_max_connections(5), + &bigname_storage::MIGRATOR, + "failed to migrate projection derive heartbeat test database", + ) + .await?; + let instance_id = "projection-derive-progress-test"; + bigname_storage::register_service_loop( + database.pool(), + bigname_storage::WORKER_SERVICE_NAME, + instance_id, + ) + .await?; + sqlx::query( + r#" + UPDATE service_loop_heartbeats + SET started_at = clock_timestamp() - INTERVAL '2 minutes', + heartbeat_at = clock_timestamp() - INTERVAL '1 minute' + WHERE service_name = 'worker' + AND instance_id = $1 + "#, + ) + .bind(instance_id) + .execute(database.pool()) + .await?; + let (first_progress_change_id, later_unit_blocker) = + seed_blocked_later_progress_unit(&database).await?; + + let derive_pool = database.pool().clone(); + let derive = tokio::spawn(async move { + let mut heartbeat = LoopHeartbeat::new(instance_id.to_owned(), Duration::ZERO); + derive_once_with_heartbeat(&derive_pool, &mut heartbeat).await + }); + timeout( + Duration::from_secs(10), + wait_for_derive_cursor(&database, first_progress_change_id), + ) + .await + .context("first bounded derive unit did not commit")??; + timeout( + Duration::from_secs(10), + wait_for_fresh_worker_heartbeat(&database, instance_id), + ) + .await + .context("first bounded derive unit did not record a heartbeat")??; + assert!( + !derive.is_finished(), + "the later derive unit must still be blocked when the heartbeat is inspected" + ); + + later_unit_blocker.commit().await?; + let summary = timeout(Duration::from_secs(10), derive) + .await + .context("derive did not finish after the later unit was released")? + .context("derive task failed")??; + assert_eq!( + summary.scanned_event_count, + NORMALIZED_EVENT_DERIVE_PROGRESS_LIMIT + 1 + ); + database.cleanup().await + } +} diff --git a/apps/worker/src/projection_apply/apply.rs b/apps/worker/src/projection_apply/apply.rs index 0bc22754..30e4eb9d 100644 --- a/apps/worker/src/projection_apply/apply.rs +++ b/apps/worker/src/projection_apply/apply.rs @@ -7,7 +7,8 @@ use tokio::time::timeout; use uuid::Uuid; use crate::{ - address_names, children, name_current, permissions, primary_name, record_inventory, resolver, + address_names, children, name_current, permissions, primary_name, + primary_name::rebuild_heartbeat::LoopHeartbeat, record_inventory, resolver, }; use super::{ @@ -43,10 +44,35 @@ pub(super) struct ClaimedInvalidation { pub(super) attempt_count: i64, } +#[cfg(test)] pub(super) async fn apply_pending_invalidations( pool: &PgPool, batch_limit: i64, text_hydration_config: Option<&record_inventory::RecordInventoryTextHydrationConfig>, +) -> Result { + apply_pending_invalidations_inner(pool, batch_limit, text_hydration_config, None).await +} + +pub(super) async fn apply_pending_invalidations_with_heartbeat( + pool: &PgPool, + batch_limit: i64, + text_hydration_config: Option<&record_inventory::RecordInventoryTextHydrationConfig>, + loop_heartbeat: &mut LoopHeartbeat, +) -> Result { + apply_pending_invalidations_inner( + pool, + batch_limit, + text_hydration_config, + Some(loop_heartbeat), + ) + .await +} + +async fn apply_pending_invalidations_inner( + pool: &PgPool, + batch_limit: i64, + text_hydration_config: Option<&record_inventory::RecordInventoryTextHydrationConfig>, + mut loop_heartbeat: Option<&mut LoopHeartbeat>, ) -> Result { if batch_limit <= 0 { bail!("projection apply batch limit must be positive, got {batch_limit}"); @@ -89,6 +115,7 @@ pub(super) async fn apply_pending_invalidations( let unlock = release_invalidation_apply_locks(&mut locks).await; finish?; unlock?; + record_loop_progress(pool, &mut loop_heartbeat).await; continue; } @@ -118,6 +145,7 @@ pub(super) async fn apply_pending_invalidations( let unlock = release_invalidation_apply_locks(&mut locks).await; finish?; unlock?; + record_loop_progress(pool, &mut loop_heartbeat).await; } Ok(summary) @@ -127,6 +155,12 @@ pub(super) async fn apply_pending_invalidations( result } +async fn record_loop_progress(pool: &PgPool, loop_heartbeat: &mut Option<&mut LoopHeartbeat>) { + if let Some(loop_heartbeat) = loop_heartbeat.as_deref_mut() { + loop_heartbeat.record_if_due(pool).await; + } +} + fn drain_address_names_group( invalidations: &mut Vec, ) -> Vec { diff --git a/apps/worker/src/projection_apply/derive.rs b/apps/worker/src/projection_apply/derive.rs index 29a858be..b6e4f1ff 100644 --- a/apps/worker/src/projection_apply/derive.rs +++ b/apps/worker/src/projection_apply/derive.rs @@ -54,6 +54,7 @@ pub(crate) async fn seed_normalized_event_cursor_if_absent( Ok(inserted > 0) } +#[cfg(test)] pub(super) async fn derive_normalized_event_invalidations( pool: &PgPool, batch_limit: i64, diff --git a/apps/worker/src/rebuild_heartbeat.rs b/apps/worker/src/rebuild_heartbeat.rs new file mode 100644 index 00000000..0c4ba8a0 --- /dev/null +++ b/apps/worker/src/rebuild_heartbeat.rs @@ -0,0 +1,186 @@ +use std::future::Future; + +use anyhow::{Context, Result}; +use sqlx::PgPool; +use tokio::time::{Duration, Instant}; +use tracing::warn; + +pub(crate) struct LoopHeartbeat { + instance_id: String, + interval: Duration, + last_recorded_at: Option, +} + +impl LoopHeartbeat { + pub(crate) fn new(instance_id: String, interval: Duration) -> Self { + Self { + instance_id, + interval, + last_recorded_at: None, + } + } + + pub(crate) async fn record_if_due(&mut self, pool: &PgPool) { + if !self.is_due() { + return; + } + + let result = bigname_storage::record_service_loop_heartbeat( + pool, + bigname_storage::WORKER_SERVICE_NAME, + &self.instance_id, + &[], + ) + .await; + match result { + Ok(()) => self.last_recorded_at = Some(Instant::now()), + Err(error) => warn!( + service = "worker", + heartbeat_instance_id = %self.instance_id, + error = %format!("{error:#}"), + "failed to record worker loop heartbeat; continuing so the missed beat degrades liveness without restarting the worker" + ), + } + } + + pub(crate) async fn run_phase( + &mut self, + pool: &PgPool, + phase: &'static str, + future: Fut, + ) -> Result + where + Fut: Future>, + { + self.begin_phase(pool, phase).await?; + let result = future.await; + self.finish_phase(pool, phase).await; + result + } + + async fn begin_phase(&mut self, pool: &PgPool, phase: &'static str) -> Result<()> { + bigname_storage::begin_service_loop_phase( + pool, + bigname_storage::WORKER_SERVICE_NAME, + &self.instance_id, + phase, + ) + .await + .with_context(|| format!("failed to establish worker loop phase {phase}"))?; + self.last_recorded_at = Some(Instant::now()); + Ok(()) + } + + async fn finish_phase(&mut self, pool: &PgPool, phase: &'static str) { + match bigname_storage::finish_service_loop_phase( + pool, + bigname_storage::WORKER_SERVICE_NAME, + &self.instance_id, + phase, + ) + .await + { + Ok(()) => self.last_recorded_at = Some(Instant::now()), + Err(error) => warn!( + service = "worker", + heartbeat_instance_id = %self.instance_id, + phase, + error = %format!("{error:#}"), + "failed to finish worker loop phase heartbeat; continuing with degraded liveness evidence" + ), + } + } + + fn is_due(&self) -> bool { + self.last_recorded_at + .map(|recorded_at| recorded_at.elapsed() >= self.interval) + .unwrap_or(true) + } +} + +pub(crate) async fn record_rebuild_progress( + pool: &PgPool, + loop_heartbeat: &mut Option<&mut LoopHeartbeat>, +) { + if let Some(loop_heartbeat) = loop_heartbeat.as_deref_mut() { + loop_heartbeat.record_if_due(pool).await; + } +} + +pub(crate) async fn run_rebuild_phase( + pool: &PgPool, + loop_heartbeat: &mut Option<&mut LoopHeartbeat>, + phase: &'static str, + future: Fut, +) -> Result +where + Fut: Future>, +{ + if let Some(loop_heartbeat) = loop_heartbeat.as_deref_mut() { + loop_heartbeat.run_phase(pool, phase, future).await + } else { + future.await + } +} + +#[cfg(test)] +mod tests { + use std::sync::{ + Arc, + atomic::{AtomicBool, Ordering}, + }; + + use super::*; + + #[test] + fn heartbeat_is_due_initially_but_not_again_inside_the_poll_interval() { + let mut heartbeat = LoopHeartbeat::new("worker-test".to_owned(), Duration::from_secs(5)); + assert!(heartbeat.is_due()); + + heartbeat.last_recorded_at = Some(Instant::now()); + assert!(!heartbeat.is_due()); + } + + #[tokio::test] + async fn heartbeat_write_failure_is_warn_and_continue() { + let pool = PgPool::connect_lazy("postgres://bigname:bigname@127.0.0.1:5432/bigname") + .expect("test pool URL must parse"); + pool.close().await; + let mut heartbeat = + LoopHeartbeat::new("worker-closed-pool".to_owned(), Duration::from_secs(5)); + + heartbeat.record_if_due(&pool).await; + + assert!( + heartbeat.is_due(), + "a failed beat must remain due so the next progress boundary retries" + ); + } + + #[tokio::test] + async fn rebuild_phase_does_not_start_without_a_durable_phase_marker() { + let pool = PgPool::connect_lazy("postgres://bigname:bigname@127.0.0.1:5432/bigname") + .expect("test pool URL must parse"); + pool.close().await; + let mut heartbeat = + LoopHeartbeat::new("worker-closed-pool".to_owned(), Duration::from_secs(5)); + let work_started = Arc::new(AtomicBool::new(false)); + let observed_work_started = Arc::clone(&work_started); + + let result = heartbeat + .run_phase(&pool, "test_monolithic_phase", async move { + observed_work_started.store(true, Ordering::SeqCst); + Ok(()) + }) + .await; + + assert!( + result.is_err(), + "phase registration failure must abort the attempt" + ); + assert!( + !work_started.load(Ordering::SeqCst), + "monolithic work must not start without its phase evidence" + ); + } +} diff --git a/apps/worker/src/record_inventory.rs b/apps/worker/src/record_inventory.rs index dd182ffd..50d6c486 100644 --- a/apps/worker/src/record_inventory.rs +++ b/apps/worker/src/record_inventory.rs @@ -12,6 +12,8 @@ use bigname_execution::ChainRpcUrls; use sqlx::PgPool; use tracing::info; +use crate::primary_name::rebuild_heartbeat::LoopHeartbeat; + pub use hydration::RecordInventoryTextHydrationConfig; pub use types::{RecordInventoryCurrentRebuildSummary, RecordInventoryTextHydrationSummary}; @@ -22,6 +24,15 @@ pub async fn rebuild_record_inventory_current( projection::rebuild_record_inventory_current(pool, resource_id).await } +pub(crate) async fn rebuild_record_inventory_current_with_heartbeat( + pool: &PgPool, + resource_id: Option<&str>, + loop_heartbeat: &mut LoopHeartbeat, +) -> Result { + projection::rebuild_record_inventory_current_with_heartbeat(pool, resource_id, loop_heartbeat) + .await +} + pub async fn hydrate_record_inventory_text_values( pool: &PgPool, resource_id: Option<&str>, @@ -30,6 +41,21 @@ pub async fn hydrate_record_inventory_text_values( hydration::hydrate_record_inventory_text_values(pool, resource_id, config).await } +pub(crate) async fn hydrate_record_inventory_text_values_with_heartbeat( + pool: &PgPool, + resource_id: Option<&str>, + config: RecordInventoryTextHydrationConfig, + loop_heartbeat: &mut LoopHeartbeat, +) -> Result { + hydration::hydrate_record_inventory_text_values_with_heartbeat( + pool, + resource_id, + config, + loop_heartbeat, + ) + .await +} + impl RecordInventoryTextHydrationConfig { pub fn from_chain_rpc_url_entries( chain_rpc_url_entries: &[String], diff --git a/apps/worker/src/record_inventory/hydration.rs b/apps/worker/src/record_inventory/hydration.rs index 73c428c0..78bfe048 100644 --- a/apps/worker/src/record_inventory/hydration.rs +++ b/apps/worker/src/record_inventory/hydration.rs @@ -12,8 +12,14 @@ use serde_json::{Value, json}; use sqlx::{PgPool, Row}; use uuid::Uuid; +use crate::primary_name::rebuild_heartbeat::{LoopHeartbeat, record_rebuild_progress}; + use super::{constants::*, types::RecordInventoryTextHydrationSummary}; +#[path = "hydration/chain_positions.rs"] +mod chain_positions; +use chain_positions::{TextHydrationChainPosition, load_text_hydration_chain_positions}; + const DEFAULT_TEXT_HYDRATION_BATCH_SIZE: usize = 250; const DEFAULT_TEXT_HYDRATION_ROW_BATCH_SIZE: i64 = 500; @@ -53,12 +59,6 @@ struct TextHydrationCall { text_key: String, } -#[derive(Clone, Debug, Eq, PartialEq)] -struct TextHydrationChainPosition { - block_number: i64, - block_hash: String, -} - #[derive(Clone, Debug, Eq, PartialEq)] enum TextHydrationOutcome { Success(String), @@ -147,10 +147,35 @@ pub(super) async fn hydrate_record_inventory_text_values( hydrate_record_inventory_text_values_with_client(pool, resource_id, &client).await } +pub(super) async fn hydrate_record_inventory_text_values_with_heartbeat( + pool: &PgPool, + resource_id: Option<&str>, + config: RecordInventoryTextHydrationConfig, + loop_heartbeat: &mut LoopHeartbeat, +) -> Result { + let client = MulticallTextHydrationClient { config }; + hydrate_record_inventory_text_values_with_client_inner( + pool, + resource_id, + &client, + Some(loop_heartbeat), + ) + .await +} + async fn hydrate_record_inventory_text_values_with_client( pool: &PgPool, resource_id: Option<&str>, client: &dyn TextHydrationClient, +) -> Result { + hydrate_record_inventory_text_values_with_client_inner(pool, resource_id, client, None).await +} + +async fn hydrate_record_inventory_text_values_with_client_inner( + pool: &PgPool, + resource_id: Option<&str>, + client: &dyn TextHydrationClient, + mut loop_heartbeat: Option<&mut LoopHeartbeat>, ) -> Result { let resource_id = resource_id .map(Uuid::parse_str) @@ -299,62 +324,12 @@ async fn hydrate_record_inventory_text_values_with_client( update_record_inventory_entries(pool, row).await?; summary.updated_row_count += 1; } + record_rebuild_progress(pool, &mut loop_heartbeat).await; } Ok(summary) } -async fn load_text_hydration_chain_positions( - pool: &PgPool, - chain_ids: &[String], -) -> Result> { - if chain_ids.is_empty() { - return Ok(BTreeMap::new()); - } - - let rows = sqlx::query( - r#" - SELECT - chain_id, - canonical_block_number, - canonical_block_hash - FROM chain_checkpoints - WHERE chain_id = ANY($1::TEXT[]) - "#, - ) - .bind(chain_ids) - .fetch_all(pool) - .await - .context("failed to load text hydration chain checkpoints")?; - - let mut positions = BTreeMap::new(); - for row in rows { - let chain_id: String = row.try_get("chain_id")?; - let block_number: Option = row.try_get("canonical_block_number")?; - let block_hash: Option = row.try_get("canonical_block_hash")?; - let Some((block_number, block_hash)) = block_number.zip(block_hash) else { - continue; - }; - positions.insert( - chain_id, - TextHydrationChainPosition { - block_number, - block_hash, - }, - ); - } - - for chain_id in chain_ids { - if !positions.contains_key(chain_id) { - anyhow::bail!( - "record_inventory_current text hydration requires a canonical chain checkpoint for {chain_id}" - ); - } - } - - Ok(positions) -} - async fn load_supported_ensv1_text_resolvers( pool: &PgPool, rows: &[HydrationRow], diff --git a/apps/worker/src/record_inventory/hydration/chain_positions.rs b/apps/worker/src/record_inventory/hydration/chain_positions.rs new file mode 100644 index 00000000..9f362460 --- /dev/null +++ b/apps/worker/src/record_inventory/hydration/chain_positions.rs @@ -0,0 +1,61 @@ +use std::collections::BTreeMap; + +use anyhow::{Context, Result}; +use sqlx::{PgPool, Row}; + +#[derive(Clone, Debug, Eq, PartialEq)] +pub(super) struct TextHydrationChainPosition { + pub(super) block_number: i64, + pub(super) block_hash: String, +} + +pub(super) async fn load_text_hydration_chain_positions( + pool: &PgPool, + chain_ids: &[String], +) -> Result> { + if chain_ids.is_empty() { + return Ok(BTreeMap::new()); + } + + let rows = sqlx::query( + r#" + SELECT + chain_id, + canonical_block_number, + canonical_block_hash + FROM chain_checkpoints + WHERE chain_id = ANY($1::TEXT[]) + "#, + ) + .bind(chain_ids) + .fetch_all(pool) + .await + .context("failed to load text hydration chain checkpoints")?; + + let mut positions = BTreeMap::new(); + for row in rows { + let chain_id: String = row.try_get("chain_id")?; + let block_number: Option = row.try_get("canonical_block_number")?; + let block_hash: Option = row.try_get("canonical_block_hash")?; + let Some((block_number, block_hash)) = block_number.zip(block_hash) else { + continue; + }; + positions.insert( + chain_id, + TextHydrationChainPosition { + block_number, + block_hash, + }, + ); + } + + for chain_id in chain_ids { + if !positions.contains_key(chain_id) { + anyhow::bail!( + "record_inventory_current text hydration requires a canonical chain checkpoint for {chain_id}" + ); + } + } + + Ok(positions) +} diff --git a/apps/worker/src/record_inventory/projection.rs b/apps/worker/src/record_inventory/projection.rs index 5778a686..66d4ffcc 100644 --- a/apps/worker/src/record_inventory/projection.rs +++ b/apps/worker/src/record_inventory/projection.rs @@ -8,6 +8,10 @@ use sqlx::{PgPool, types::time::OffsetDateTime}; use tokio::task::JoinSet; use uuid::Uuid; +use crate::primary_name::rebuild_heartbeat::{ + LoopHeartbeat, record_rebuild_progress, run_rebuild_phase, +}; + #[allow(clippy::duplicate_mod)] #[path = "../staged_rebuild.rs"] mod staged_rebuild; @@ -79,20 +83,45 @@ const RECORD_INVENTORY_CURRENT_REBUILD_CONCURRENCY: usize = 8; pub(super) async fn rebuild_record_inventory_current( pool: &PgPool, resource_id: Option<&str>, +) -> Result { + rebuild_record_inventory_current_inner(pool, resource_id, None).await +} + +pub(super) async fn rebuild_record_inventory_current_with_heartbeat( + pool: &PgPool, + resource_id: Option<&str>, + loop_heartbeat: &mut LoopHeartbeat, +) -> Result { + rebuild_record_inventory_current_inner(pool, resource_id, Some(loop_heartbeat)).await +} + +async fn rebuild_record_inventory_current_inner( + pool: &PgPool, + resource_id: Option<&str>, + loop_heartbeat: Option<&mut LoopHeartbeat>, ) -> Result { match resource_id { Some(resource_id) => rebuild_one_resource(pool, resource_id).await, - None => rebuild_all_resources(pool).await, + None => rebuild_all_resources(pool, loop_heartbeat).await, } } -async fn rebuild_all_resources(pool: &PgPool) -> Result { +async fn rebuild_all_resources( + pool: &PgPool, + mut loop_heartbeat: Option<&mut LoopHeartbeat>, +) -> Result { let mut conn = pool .acquire() .await .context("failed to acquire record_inventory_current staging connection")?; let stage_table = create_stage_table(&mut conn, "record_inventory_current").await?; - let previous_row_count = count_rows(&mut conn, "record_inventory_current", None).await?; + let previous_row_count = run_rebuild_phase( + pool, + &mut loop_heartbeat, + "record_inventory_current.count_existing", + count_rows(&mut conn, "record_inventory_current", None), + ) + .await?; let mut rows = Vec::with_capacity(RECORD_INVENTORY_CURRENT_REBUILD_BATCH_SIZE); let mut requested_resource_count = 0usize; let mut completed_resource_count = 0usize; @@ -112,7 +141,9 @@ async fn rebuild_all_resources(pool: &PgPool) -> Result Result, + text_hydration_config: Option<&record_inventory::RecordInventoryTextHydrationConfig>, + primary_hydration_config: Option<&primary_name::PrimaryNameLegacyReverseHydrationConfig>, + loop_heartbeat: &mut LoopHeartbeat, +) -> Result { + rebuild_all_current_projections_inner( + pool, + normalized_target_block, + true, + text_hydration_config, + primary_hydration_config, + Some(loop_heartbeat), ) .await } @@ -54,6 +75,7 @@ async fn rebuild_all_current_projections_inner( skip_completed: bool, text_hydration_config: Option<&record_inventory::RecordInventoryTextHydrationConfig>, primary_hydration_config: Option<&primary_name::PrimaryNameLegacyReverseHydrationConfig>, + mut loop_heartbeat: Option<&mut LoopHeartbeat>, ) -> Result { let mut steps = Vec::with_capacity(ALL_CURRENT_PROJECTION_ORDER.len()); @@ -82,19 +104,22 @@ async fn rebuild_all_current_projections_inner( steps.push(replay_step!( "name_current", - name_current::rebuild_name_current(pool, None), + rebuild_name_current(pool, &mut loop_heartbeat), requested_name_count )); + record_loop_progress(pool, &mut loop_heartbeat).await; steps.push(replay_step!( "children_current", - children::rebuild_children_current(pool, None), + rebuild_children_current(pool, &mut loop_heartbeat), requested_parent_count )); + record_loop_progress(pool, &mut loop_heartbeat).await; steps.push(replay_step!( "permissions_current", - permissions::rebuild_permissions_current(pool, None), + rebuild_permissions_current(pool, &mut loop_heartbeat), requested_resource_count )); + record_loop_progress(pool, &mut loop_heartbeat).await; steps.push( replay_projection_step( pool, @@ -102,14 +127,14 @@ async fn rebuild_all_current_projections_inner( normalized_target_block, skip_completed, async { - let summary = record_inventory::rebuild_record_inventory_current(pool, None) + let summary = rebuild_record_inventory_current(pool, &mut loop_heartbeat) .await .context("failed to replay record_inventory_current")?; if let Some(config) = text_hydration_config { - let hydration_summary = record_inventory::hydrate_record_inventory_text_values( + let hydration_summary = hydrate_record_inventory_text_values( pool, - None, config.clone(), + &mut loop_heartbeat, ) .await .context("failed to hydrate record_inventory_current text values")?; @@ -125,16 +150,19 @@ async fn rebuild_all_current_projections_inner( ) .await?, ); + record_loop_progress(pool, &mut loop_heartbeat).await; steps.push(replay_step!( "resolver_current", - resolver::rebuild_resolver_current(pool, None, None), + rebuild_resolver_current(pool, &mut loop_heartbeat), requested_resolver_count )); + record_loop_progress(pool, &mut loop_heartbeat).await; steps.push(replay_step!( "address_names_current", - address_names::rebuild_address_names_current(pool, None), + rebuild_address_names_current(pool, &mut loop_heartbeat), requested_address_count )); + record_loop_progress(pool, &mut loop_heartbeat).await; steps.push( replay_projection_step( pool, @@ -142,19 +170,19 @@ async fn rebuild_all_current_projections_inner( normalized_target_block, skip_completed, async { - let summary = primary_name::rebuild_primary_names_current(pool, None, None, None) + let summary = rebuild_primary_names_current(pool, &mut loop_heartbeat) .await .context("failed to replay primary_names_current")?; if let Some(config) = primary_hydration_config { - let hydration_summary = - primary_name::hydrate_legacy_reverse_resolver_primary_names( - pool, - config.clone(), - ) - .await - .context( - "failed to hydrate primary_names_current legacy reverse-resolver claims", - )?; + let hydration_summary = hydrate_primary_names_current( + pool, + config.clone(), + &mut loop_heartbeat, + ) + .await + .context( + "failed to hydrate primary_names_current legacy reverse-resolver claims", + )?; if hydration_summary.candidate_tuple_count > 0 || hydration_summary.failed_lookup_count > 0 { @@ -171,6 +199,7 @@ async fn rebuild_all_current_projections_inner( ) .await?, ); + record_loop_progress(pool, &mut loop_heartbeat).await; debug_assert_eq!( steps.iter().map(|step| step.projection).collect::>(), @@ -180,6 +209,137 @@ async fn rebuild_all_current_projections_inner( Ok(AllCurrentProjectionsReplaySummary { steps }) } +async fn record_loop_progress(pool: &PgPool, loop_heartbeat: &mut Option<&mut LoopHeartbeat>) { + if let Some(loop_heartbeat) = loop_heartbeat.as_deref_mut() { + loop_heartbeat.record_if_due(pool).await; + } +} + +async fn rebuild_name_current( + pool: &PgPool, + loop_heartbeat: &mut Option<&mut LoopHeartbeat>, +) -> Result { + if let Some(loop_heartbeat) = loop_heartbeat.as_deref_mut() { + name_current::rebuild_name_current_with_heartbeat(pool, None, loop_heartbeat).await + } else { + name_current::rebuild_name_current(pool, None).await + } +} + +async fn rebuild_children_current( + pool: &PgPool, + loop_heartbeat: &mut Option<&mut LoopHeartbeat>, +) -> Result { + if let Some(loop_heartbeat) = loop_heartbeat.as_deref_mut() { + children::rebuild_children_current_with_heartbeat(pool, None, loop_heartbeat).await + } else { + children::rebuild_children_current(pool, None).await + } +} + +async fn rebuild_permissions_current( + pool: &PgPool, + loop_heartbeat: &mut Option<&mut LoopHeartbeat>, +) -> Result { + if let Some(loop_heartbeat) = loop_heartbeat.as_deref_mut() { + permissions::rebuild_permissions_current_with_heartbeat(pool, None, loop_heartbeat).await + } else { + permissions::rebuild_permissions_current(pool, None).await + } +} + +async fn rebuild_record_inventory_current( + pool: &PgPool, + loop_heartbeat: &mut Option<&mut LoopHeartbeat>, +) -> Result { + if let Some(loop_heartbeat) = loop_heartbeat.as_deref_mut() { + record_inventory::rebuild_record_inventory_current_with_heartbeat( + pool, + None, + loop_heartbeat, + ) + .await + } else { + record_inventory::rebuild_record_inventory_current(pool, None).await + } +} + +async fn hydrate_record_inventory_text_values( + pool: &PgPool, + config: record_inventory::RecordInventoryTextHydrationConfig, + loop_heartbeat: &mut Option<&mut LoopHeartbeat>, +) -> Result { + if let Some(loop_heartbeat) = loop_heartbeat.as_deref_mut() { + record_inventory::hydrate_record_inventory_text_values_with_heartbeat( + pool, + None, + config, + loop_heartbeat, + ) + .await + } else { + record_inventory::hydrate_record_inventory_text_values(pool, None, config).await + } +} + +async fn rebuild_resolver_current( + pool: &PgPool, + loop_heartbeat: &mut Option<&mut LoopHeartbeat>, +) -> Result { + if let Some(loop_heartbeat) = loop_heartbeat.as_deref_mut() { + resolver::rebuild_resolver_current_with_heartbeat(pool, None, None, loop_heartbeat).await + } else { + resolver::rebuild_resolver_current(pool, None, None).await + } +} + +async fn rebuild_address_names_current( + pool: &PgPool, + loop_heartbeat: &mut Option<&mut LoopHeartbeat>, +) -> Result { + if let Some(loop_heartbeat) = loop_heartbeat.as_deref_mut() { + address_names::rebuild_address_names_current_with_heartbeat(pool, None, loop_heartbeat) + .await + } else { + address_names::rebuild_address_names_current(pool, None).await + } +} + +async fn rebuild_primary_names_current( + pool: &PgPool, + loop_heartbeat: &mut Option<&mut LoopHeartbeat>, +) -> Result { + if let Some(loop_heartbeat) = loop_heartbeat.as_deref_mut() { + primary_name::rebuild_primary_names_current_with_heartbeat( + pool, + None, + None, + None, + loop_heartbeat, + ) + .await + } else { + primary_name::rebuild_primary_names_current(pool, None, None, None).await + } +} + +async fn hydrate_primary_names_current( + pool: &PgPool, + config: primary_name::PrimaryNameLegacyReverseHydrationConfig, + loop_heartbeat: &mut Option<&mut LoopHeartbeat>, +) -> Result { + if let Some(loop_heartbeat) = loop_heartbeat.as_deref_mut() { + primary_name::hydrate_legacy_reverse_resolver_primary_names_with_heartbeat( + pool, + config, + loop_heartbeat, + ) + .await + } else { + primary_name::hydrate_legacy_reverse_resolver_primary_names(pool, config).await + } +} + async fn replay_projection_step( pool: &PgPool, projection: &'static str, diff --git a/apps/worker/src/replay/tests.rs b/apps/worker/src/replay/tests.rs index 966d0812..c6d55b65 100644 --- a/apps/worker/src/replay/tests.rs +++ b/apps/worker/src/replay/tests.rs @@ -1,6 +1,6 @@ mod support; -use std::collections::BTreeMap; +use std::{collections::BTreeMap, time::Duration}; use anyhow::{Context, Result}; use serde_json::Value; @@ -113,6 +113,57 @@ fn all_current_projection_json_summary_has_frozen_shape_order_counts_and_totals( Ok(()) } +#[tokio::test] +async fn automatic_replay_refreshes_worker_heartbeat_between_projection_steps() -> Result<()> { + let database = TestDatabase::new().await?; + seed_replay_inputs(database.pool()).await?; + let instance_id = "automatic-replay-heartbeat-test"; + bigname_storage::register_service_loop( + database.pool(), + bigname_storage::WORKER_SERVICE_NAME, + instance_id, + ) + .await?; + sqlx::query( + r#" + UPDATE service_loop_heartbeats + SET started_at = clock_timestamp() - INTERVAL '2 minutes', + heartbeat_at = clock_timestamp() - INTERVAL '1 minute' + WHERE service_name = 'worker' + AND instance_id = $1 + "#, + ) + .bind(instance_id) + .execute(database.pool()) + .await?; + + let mut heartbeat = crate::primary_name::rebuild_heartbeat::LoopHeartbeat::new( + instance_id.to_owned(), + Duration::ZERO, + ); + rebuild_pending_all_current_projections_with_heartbeat( + database.pool(), + Some(108), + None, + None, + &mut heartbeat, + ) + .await?; + + let heartbeat = bigname_storage::load_service_loop_heartbeat( + database.pool(), + bigname_storage::WORKER_SERVICE_NAME, + instance_id, + ) + .await? + .context("automatic replay must retain its worker heartbeat")?; + assert!( + heartbeat.age_seconds <= 1, + "projection-step progress must refresh the worker heartbeat" + ); + Ok(()) +} + #[tokio::test] async fn all_current_projection_replay_clears_stale_rows_and_is_idempotent() -> Result<()> { let database = TestDatabase::new().await?; diff --git a/apps/worker/src/resolver.rs b/apps/worker/src/resolver.rs index a4a7423a..e2c58da5 100644 --- a/apps/worker/src/resolver.rs +++ b/apps/worker/src/resolver.rs @@ -3,6 +3,10 @@ use bigname_storage::{ResolverCurrentRow, delete_resolver_current, upsert_resolv use sqlx::PgPool; use tokio::task::JoinSet; +use crate::primary_name::rebuild_heartbeat::{ + LoopHeartbeat, record_rebuild_progress, run_rebuild_phase, +}; + #[allow(clippy::duplicate_mod)] #[path = "staged_rebuild.rs"] mod staged_rebuild; @@ -78,25 +82,64 @@ pub async fn rebuild_resolver_current( pool: &PgPool, chain_id: Option<&str>, resolver_address: Option<&str>, +) -> Result { + rebuild_resolver_current_inner(pool, chain_id, resolver_address, None).await +} + +pub(crate) async fn rebuild_resolver_current_with_heartbeat( + pool: &PgPool, + chain_id: Option<&str>, + resolver_address: Option<&str>, + loop_heartbeat: &mut LoopHeartbeat, +) -> Result { + rebuild_resolver_current_inner(pool, chain_id, resolver_address, Some(loop_heartbeat)).await +} + +async fn rebuild_resolver_current_inner( + pool: &PgPool, + chain_id: Option<&str>, + resolver_address: Option<&str>, + loop_heartbeat: Option<&mut LoopHeartbeat>, ) -> Result { match (chain_id, resolver_address) { (Some(chain_id), Some(resolver_address)) => { rebuild_one_resolver(pool, chain_id, resolver_address).await } - (None, None) => rebuild_all_resolvers(pool).await, + (None, None) => rebuild_all_resolvers(pool, loop_heartbeat).await, _ => bail!( "resolver_current rebuild requires both chain_id and resolver_address when targeting one resolver" ), } } -async fn rebuild_all_resolvers(pool: &PgPool) -> Result { - let profile_gate = ResolverProfileGate::load(pool).await?; - let targets = load_target_resolvers(pool).await?; +async fn rebuild_all_resolvers( + pool: &PgPool, + mut loop_heartbeat: Option<&mut LoopHeartbeat>, +) -> Result { + let profile_gate = run_rebuild_phase( + pool, + &mut loop_heartbeat, + "resolver_current.load_profile", + ResolverProfileGate::load(pool), + ) + .await?; + let targets = run_rebuild_phase( + pool, + &mut loop_heartbeat, + "resolver_current.load_targets", + load_target_resolvers(pool), + ) + .await?; let requested_resolver_count = targets.len(); let mut conn = pool.acquire().await.map_err(anyhow::Error::from)?; let stage_table = create_stage_table(&mut conn, "resolver_current").await?; - let previous_row_count = count_rows(&mut conn, "resolver_current", None).await?; + let previous_row_count = run_rebuild_phase( + pool, + &mut loop_heartbeat, + "resolver_current.count_existing", + count_rows(&mut conn, "resolver_current", None), + ) + .await?; tracing::info!( projection = "resolver_current", requested_resolver_count, @@ -119,7 +162,9 @@ async fn rebuild_all_resolvers(pool: &PgPool) -> Result Result = Pin> + Send + 'a>>; + +pub trait StartupAdapterProgress: Send { + fn record<'a>(&'a mut self, pool: &'a PgPool) -> StartupAdapterProgressFuture<'a>; +} + +pub(crate) async fn record_startup_adapter_progress( + pool: &PgPool, + progress: &mut Option<&mut dyn StartupAdapterProgress>, +) -> Result<()> { + if let Some(progress) = progress.as_deref_mut() { + progress.record(pool).await?; + } + Ok(()) +} + #[derive(Clone, Debug, Eq, PartialEq)] pub struct ReplayAdapterCheckpointContext { pub deployment_profile: String, diff --git a/crates/adapters/src/ens_v1_subregistry_discovery.rs b/crates/adapters/src/ens_v1_subregistry_discovery.rs index 1c144348..4294b618 100644 --- a/crates/adapters/src/ens_v1_subregistry_discovery.rs +++ b/crates/adapters/src/ens_v1_subregistry_discovery.rs @@ -4,7 +4,9 @@ use anyhow::{Context, Result, ensure}; use bigname_manifests::{FullDiscoveryReconciliationOptions, reconcile_discovery_observations}; use sqlx::PgPool; -use crate::checkpoint_context::AdapterCheckpointContext; +use crate::checkpoint_context::{ + AdapterCheckpointContext, StartupAdapterProgress, record_startup_adapter_progress, +}; use crate::registry_migration_cache::MigratedRegistryNodes; mod assignment; @@ -55,7 +57,7 @@ const EVENT_KIND_RESOLVER_CHANGED: &str = "ResolverChanged"; const DERIVATION_KIND_ENS_V1_SUBREGISTRY_CHANGED: &str = "ens_v1_subregistry_changed"; const DERIVATION_KIND_ENS_V1_REGISTRY_RESOLVER_CHANGED: &str = "ens_v1_registry_resolver_changed"; -#[derive(Clone, Debug, Eq, PartialEq)] +#[derive(Clone, Debug, Default, Eq, PartialEq)] pub struct EnsV1SubregistryDiscoverySyncSummary { pub scanned_log_count: usize, pub matched_log_count: usize, @@ -82,7 +84,10 @@ pub use replay::{ sync_ens_v1_subregistry_discovery_with_replay_checkpoint, sync_ens_v1_subregistry_discovery_with_replay_checkpoint_and_log_limit, }; -pub use startup::sync_ens_v1_subregistry_discovery_with_startup_checkpoint_and_log_limit; +pub use startup::{ + sync_ens_v1_subregistry_discovery_with_startup_checkpoint_and_log_limit, + sync_ens_v1_subregistry_discovery_with_startup_checkpoint_and_log_limit_and_progress, +}; #[derive(Clone, Copy, Debug, Eq, PartialEq)] pub(super) enum DiscoveryEdgeMutation { @@ -102,6 +107,7 @@ pub(super) async fn sync_ens_v1_subregistry_discovery_with_scope( checkpoint_context: Option<&AdapterCheckpointContext>, checkpoint_page_limit: i64, checkpoint_progress_log_every_pages: Option, + startup_progress: &mut Option<&mut dyn StartupAdapterProgress>, ) -> Result { ensure!( full_source_through_block.is_none() @@ -127,7 +133,7 @@ pub(super) async fn sync_ens_v1_subregistry_discovery_with_scope( && discovery_edge_mutation == DiscoveryEdgeMutation::Reconcile && checkpoint_context.is_some(); if source_scope.as_ref().is_some_and(Vec::is_empty) { - return Ok((empty_sync_summary(), false)); + return Ok((EnsV1SubregistryDiscoverySyncSummary::default(), false)); } let emitters = load_active_emitters( @@ -180,6 +186,7 @@ pub(super) async fn sync_ens_v1_subregistry_discovery_with_scope( checkpoint, checkpoint_page_limit, checkpoint_progress_log_every_pages, + startup_progress, &mut migrated_registry_nodes, ) .await?; @@ -237,7 +244,7 @@ pub(super) async fn sync_ens_v1_subregistry_discovery_with_scope( ) .await?; if source_scope.is_some() && raw_logs.is_empty() { - return Ok((empty_sync_summary(), false)); + return Ok((EnsV1SubregistryDiscoverySyncSummary::default(), false)); } scanned_log_count = raw_logs.len(); let preload_migrated_registry_nodes = raw_logs @@ -321,6 +328,7 @@ pub(super) async fn sync_ens_v1_subregistry_discovery_with_scope( .expect("finalizing checkpoint should be present"), &discovery_sources, &mut reconciliation, + startup_progress, ) .await?; if reconciliation.inserted_edge_count > 0 { @@ -404,6 +412,7 @@ pub(super) async fn sync_ens_v1_subregistry_discovery_with_scope( .as_ref() .expect("finalizing checkpoint should be present"), &discovery_sources, + startup_progress, ) .await? } else { @@ -414,25 +423,12 @@ pub(super) async fn sync_ens_v1_subregistry_discovery_with_scope( if let Some(checkpoint) = active_checkpoint.as_mut() { checkpoint.mark_completed(pool, &reconciliation).await?; + record_startup_adapter_progress(pool, startup_progress).await?; } Ok((reconciliation, false)) } -fn empty_sync_summary() -> EnsV1SubregistryDiscoverySyncSummary { - EnsV1SubregistryDiscoverySyncSummary { - scanned_log_count: 0, - matched_log_count: 0, - active_observation_count: 0, - active_edge_count: 0, - admitted_edge_count: 0, - inserted_edge_count: 0, - deactivated_edge_count: 0, - total_normalized_event_count: 0, - total_normalized_event_inserted_count: 0, - } -} - async fn sync_checkpointed_registry_raw_logs( pool: &PgPool, chain: &str, @@ -441,6 +437,7 @@ async fn sync_checkpointed_registry_raw_logs( checkpoint: &mut SubregistryReplayCheckpoint, checkpoint_page_limit: i64, progress_log_every_pages: Option, + startup_progress: &mut Option<&mut dyn StartupAdapterProgress>, migrated_registry_nodes: &mut MigratedRegistryNodes, ) -> Result<(usize, usize)> { if checkpoint.stream_complete() { @@ -469,6 +466,7 @@ async fn sync_checkpointed_registry_raw_logs( checkpoint .mark_stream_complete(pool, scanned_log_count, matched_log_count) .await?; + record_startup_adapter_progress(pool, startup_progress).await?; break; }; @@ -517,6 +515,7 @@ async fn sync_checkpointed_registry_raw_logs( scanned_log_count, matched_log_count, ); + record_startup_adapter_progress(pool, startup_progress).await?; start_after = Some(last_position); } diff --git a/crates/adapters/src/ens_v1_subregistry_discovery/checkpoint.rs b/crates/adapters/src/ens_v1_subregistry_discovery/checkpoint.rs index 2da6896e..9058d26f 100644 --- a/crates/adapters/src/ens_v1_subregistry_discovery/checkpoint.rs +++ b/crates/adapters/src/ens_v1_subregistry_discovery/checkpoint.rs @@ -6,7 +6,7 @@ use bigname_storage::{ raw_log_staging_block_range_changed_since, }; use serde_json::{Value, json}; -use sqlx::{PgPool, Postgres, Row}; +use sqlx::{PgPool, Row}; use crate::{ checkpoint_context::AdapterCheckpointContext, registry_migration_cache::MigratedRegistryNodes, @@ -20,11 +20,13 @@ use super::{ mod cleanup; mod items; mod payload; +mod persistence; pub use cleanup::clear_replay_adapter_checkpoints; pub(super) use cleanup::delete_checkpoint; use items::insert_checkpoint_items; use payload::{assignment_from_payload, assignment_payload, summary_from_payload, summary_payload}; +use persistence::{load_checkpoint_row, update_checkpoint_progress}; const ADAPTER: &str = "ens_v1_subregistry_discovery"; const ITEM_KIND_LATEST_ASSIGNMENT: &str = "latest_assignment"; @@ -514,163 +516,3 @@ impl SubregistryReplayCheckpoint { Ok(()) } } - -async fn load_checkpoint_row( - pool: &PgPool, - chain: &str, - context: &AdapterCheckpointContext, -) -> Result> { - let row = sqlx::query( - r#" - SELECT - replay_start_block_number, - replay_target_block_number, - last_block_number, - last_transaction_index, - last_log_index, - last_emitting_address, - scanned_log_count, - matched_log_count, - staged_item_count, - status, - state_payload, - raw_log_retention_generation, - raw_log_input_revision - FROM normalized_replay_adapter_checkpoints - WHERE deployment_profile = $1 - AND chain_id = $2 - AND cursor_kind = $3 - AND adapter = $4 - AND checkpoint_scope = $5 - "#, - ) - .bind(&context.deployment_profile) - .bind(chain) - .bind(&context.cursor_kind) - .bind(ADAPTER) - .bind(context.checkpoint_scope) - .fetch_optional(pool) - .await - .with_context(|| { - format!( - "failed to load {ADAPTER} replay checkpoint for {}/{}", - context.deployment_profile, chain - ) - })?; - - row.map(|row| checkpoint_from_row(chain, context, row)) - .transpose() -} - -fn checkpoint_from_row( - chain: &str, - context: &AdapterCheckpointContext, - row: sqlx::postgres::PgRow, -) -> Result { - let range_start_block_number = row.try_get("replay_start_block_number")?; - let target_block_number = row.try_get("replay_target_block_number")?; - let last_block_number: Option = row.try_get("last_block_number")?; - let last_transaction_index: Option = row.try_get("last_transaction_index")?; - let last_log_index: Option = row.try_get("last_log_index")?; - let last_emitting_address: Option = row.try_get("last_emitting_address")?; - let last_position = match ( - last_block_number, - last_transaction_index, - last_log_index, - last_emitting_address, - ) { - (Some(block_number), Some(transaction_index), Some(log_index), Some(emitting_address)) => { - Some(RegistryRawLogPosition { - block_number, - transaction_index, - log_index, - emitting_address, - }) - } - _ => None, - }; - - Ok(SubregistryReplayCheckpoint { - context: AdapterCheckpointContext { - deployment_profile: context.deployment_profile.clone(), - cursor_kind: context.cursor_kind.clone(), - checkpoint_scope: context.checkpoint_scope, - range_start_block_number, - target_block_number, - startup_discovery_admission_epoch: context.startup_discovery_admission_epoch, - }, - chain: chain.to_owned(), - status: row.try_get("status")?, - last_position, - scanned_log_count: usize::try_from(row.try_get::("scanned_log_count")?) - .context("checkpoint scanned log count overflowed usize")?, - matched_log_count: usize::try_from(row.try_get::("matched_log_count")?) - .context("checkpoint matched log count overflowed usize")?, - staged_item_count: usize::try_from(row.try_get::("staged_item_count")?) - .context("checkpoint staged item count overflowed usize")?, - state_payload: row.try_get("state_payload")?, - raw_log_input_version: RawLogStagingInputVersion { - retention_generation: row.try_get("raw_log_retention_generation")?, - revision: row.try_get("raw_log_input_revision")?, - }, - }) -} - -async fn update_checkpoint_progress( - transaction: &mut sqlx::Transaction<'_, Postgres>, - checkpoint: &SubregistryReplayCheckpoint, - status: &str, - last_position: Option<&RegistryRawLogPosition>, - scanned_log_count: usize, - matched_log_count: usize, - staged_item_count: usize, - staged_aux_item_count: usize, - state_payload: Value, -) -> Result<()> { - sqlx::query( - r#" - UPDATE normalized_replay_adapter_checkpoints - SET - status = $6, - last_block_number = $7, - last_transaction_index = $8, - last_log_index = $9, - last_emitting_address = $10, - staged_item_count = $11, - staged_aux_item_count = $12, - scanned_log_count = $13, - matched_log_count = $14, - state_payload = $15, - raw_log_retention_generation = $16, - raw_log_input_revision = $17, - updated_at = now(), - last_failure_reason = NULL - WHERE deployment_profile = $1 - AND chain_id = $2 - AND cursor_kind = $3 - AND adapter = $4 - AND checkpoint_scope = $5 - "#, - ) - .bind(&checkpoint.context.deployment_profile) - .bind(&checkpoint.chain) - .bind(&checkpoint.context.cursor_kind) - .bind(ADAPTER) - .bind(checkpoint.context.checkpoint_scope) - .bind(status) - .bind(last_position.map(|position| position.block_number)) - .bind(last_position.map(|position| position.transaction_index)) - .bind(last_position.map(|position| position.log_index)) - .bind(last_position.map(|position| position.emitting_address.as_str())) - .bind(i64::try_from(staged_item_count).context("staged item count overflowed i64")?) - .bind(i64::try_from(staged_aux_item_count).context("staged aux item count overflowed i64")?) - .bind(i64::try_from(scanned_log_count).context("scanned log count overflowed i64")?) - .bind(i64::try_from(matched_log_count).context("matched log count overflowed i64")?) - .bind(state_payload) - .bind(checkpoint.raw_log_input_version.retention_generation) - .bind(checkpoint.raw_log_input_version.revision) - .execute(transaction.as_mut()) - .await - .context("failed to update replay adapter checkpoint progress")?; - Ok(()) -} diff --git a/crates/adapters/src/ens_v1_subregistry_discovery/checkpoint/persistence.rs b/crates/adapters/src/ens_v1_subregistry_discovery/checkpoint/persistence.rs new file mode 100644 index 00000000..a6b49bf5 --- /dev/null +++ b/crates/adapters/src/ens_v1_subregistry_discovery/checkpoint/persistence.rs @@ -0,0 +1,168 @@ +use anyhow::{Context, Result}; +use bigname_storage::RawLogStagingInputVersion; +use serde_json::Value; +use sqlx::{PgPool, Postgres, Row}; + +use crate::checkpoint_context::AdapterCheckpointContext; + +use super::{ADAPTER, RegistryRawLogPosition, SubregistryReplayCheckpoint}; + +pub(super) async fn load_checkpoint_row( + pool: &PgPool, + chain: &str, + context: &AdapterCheckpointContext, +) -> Result> { + let row = sqlx::query( + r#" + SELECT + replay_start_block_number, + replay_target_block_number, + last_block_number, + last_transaction_index, + last_log_index, + last_emitting_address, + scanned_log_count, + matched_log_count, + staged_item_count, + status, + state_payload, + raw_log_retention_generation, + raw_log_input_revision + FROM normalized_replay_adapter_checkpoints + WHERE deployment_profile = $1 + AND chain_id = $2 + AND cursor_kind = $3 + AND adapter = $4 + AND checkpoint_scope = $5 + "#, + ) + .bind(&context.deployment_profile) + .bind(chain) + .bind(&context.cursor_kind) + .bind(ADAPTER) + .bind(context.checkpoint_scope) + .fetch_optional(pool) + .await + .with_context(|| { + format!( + "failed to load {ADAPTER} replay checkpoint for {}/{}", + context.deployment_profile, chain + ) + })?; + + row.map(|row| checkpoint_from_row(chain, context, row)) + .transpose() +} + +fn checkpoint_from_row( + chain: &str, + context: &AdapterCheckpointContext, + row: sqlx::postgres::PgRow, +) -> Result { + let range_start_block_number = row.try_get("replay_start_block_number")?; + let target_block_number = row.try_get("replay_target_block_number")?; + let last_block_number: Option = row.try_get("last_block_number")?; + let last_transaction_index: Option = row.try_get("last_transaction_index")?; + let last_log_index: Option = row.try_get("last_log_index")?; + let last_emitting_address: Option = row.try_get("last_emitting_address")?; + let last_position = match ( + last_block_number, + last_transaction_index, + last_log_index, + last_emitting_address, + ) { + (Some(block_number), Some(transaction_index), Some(log_index), Some(emitting_address)) => { + Some(RegistryRawLogPosition { + block_number, + transaction_index, + log_index, + emitting_address, + }) + } + _ => None, + }; + + Ok(SubregistryReplayCheckpoint { + context: AdapterCheckpointContext { + deployment_profile: context.deployment_profile.clone(), + cursor_kind: context.cursor_kind.clone(), + checkpoint_scope: context.checkpoint_scope, + range_start_block_number, + target_block_number, + startup_discovery_admission_epoch: context.startup_discovery_admission_epoch, + }, + chain: chain.to_owned(), + status: row.try_get("status")?, + last_position, + scanned_log_count: usize::try_from(row.try_get::("scanned_log_count")?) + .context("checkpoint scanned log count overflowed usize")?, + matched_log_count: usize::try_from(row.try_get::("matched_log_count")?) + .context("checkpoint matched log count overflowed usize")?, + staged_item_count: usize::try_from(row.try_get::("staged_item_count")?) + .context("checkpoint staged item count overflowed usize")?, + state_payload: row.try_get("state_payload")?, + raw_log_input_version: RawLogStagingInputVersion { + retention_generation: row.try_get("raw_log_retention_generation")?, + revision: row.try_get("raw_log_input_revision")?, + }, + }) +} + +pub(super) async fn update_checkpoint_progress( + transaction: &mut sqlx::Transaction<'_, Postgres>, + checkpoint: &SubregistryReplayCheckpoint, + status: &str, + last_position: Option<&RegistryRawLogPosition>, + scanned_log_count: usize, + matched_log_count: usize, + staged_item_count: usize, + staged_aux_item_count: usize, + state_payload: Value, +) -> Result<()> { + sqlx::query( + r#" + UPDATE normalized_replay_adapter_checkpoints + SET + status = $6, + last_block_number = $7, + last_transaction_index = $8, + last_log_index = $9, + last_emitting_address = $10, + staged_item_count = $11, + staged_aux_item_count = $12, + scanned_log_count = $13, + matched_log_count = $14, + state_payload = $15, + raw_log_retention_generation = $16, + raw_log_input_revision = $17, + updated_at = now(), + last_failure_reason = NULL + WHERE deployment_profile = $1 + AND chain_id = $2 + AND cursor_kind = $3 + AND adapter = $4 + AND checkpoint_scope = $5 + "#, + ) + .bind(&checkpoint.context.deployment_profile) + .bind(&checkpoint.chain) + .bind(&checkpoint.context.cursor_kind) + .bind(ADAPTER) + .bind(checkpoint.context.checkpoint_scope) + .bind(status) + .bind(last_position.map(|position| position.block_number)) + .bind(last_position.map(|position| position.transaction_index)) + .bind(last_position.map(|position| position.log_index)) + .bind(last_position.map(|position| position.emitting_address.as_str())) + .bind(i64::try_from(staged_item_count).context("staged item count overflowed i64")?) + .bind(i64::try_from(staged_aux_item_count).context("staged aux item count overflowed i64")?) + .bind(i64::try_from(scanned_log_count).context("scanned log count overflowed i64")?) + .bind(i64::try_from(matched_log_count).context("matched log count overflowed i64")?) + .bind(state_payload) + .bind(checkpoint.raw_log_input_version.retention_generation) + .bind(checkpoint.raw_log_input_version.revision) + .execute(transaction.as_mut()) + .await + .context("failed to update replay adapter checkpoint progress")?; + Ok(()) +} diff --git a/crates/adapters/src/ens_v1_subregistry_discovery/emitter.rs b/crates/adapters/src/ens_v1_subregistry_discovery/emitter.rs index 56436677..301a6982 100644 --- a/crates/adapters/src/ens_v1_subregistry_discovery/emitter.rs +++ b/crates/adapters/src/ens_v1_subregistry_discovery/emitter.rs @@ -4,6 +4,8 @@ use anyhow::Result; use bigname_storage::upsert_normalized_events_with_summary; use sqlx::PgPool; +use crate::checkpoint_context::{StartupAdapterProgress, record_startup_adapter_progress}; + use super::{ assignment::ObservedRegistryAssignment, checkpoint::{EVENT_PAGE_LIMIT, SubregistryReplayCheckpoint}, @@ -25,6 +27,7 @@ pub(super) async fn emit_registry_changed_events( latest_assignments: &BTreeMap, discovery_sources: &[String], ) -> Result { + let mut startup_progress = None; let discovery_sources = discovery_sources .iter() .map(String::as_str) @@ -38,13 +41,26 @@ pub(super) async fn emit_registry_changed_events( } assignments.push(assignment); if assignments.len() >= NORMALIZED_EVENT_UPSERT_CHUNK_SIZE { - emit_registry_changed_event_chunk(pool, &assignments, &mut events, &mut summary) - .await?; + emit_registry_changed_event_chunk( + pool, + &assignments, + &mut events, + &mut summary, + &mut startup_progress, + ) + .await?; assignments.clear(); } } - emit_registry_changed_event_chunk(pool, &assignments, &mut events, &mut summary).await?; - flush_registry_changed_events(pool, &mut events, &mut summary).await?; + emit_registry_changed_event_chunk( + pool, + &assignments, + &mut events, + &mut summary, + &mut startup_progress, + ) + .await?; + flush_registry_changed_events(pool, &mut events, &mut summary, &mut startup_progress).await?; Ok(summary) } @@ -55,6 +71,7 @@ pub(super) async fn emit_registry_changed_events_from_checkpoint( pool: &PgPool, checkpoint: &SubregistryReplayCheckpoint, discovery_sources: &[String], + startup_progress: &mut Option<&mut dyn StartupAdapterProgress>, ) -> Result { let mut events = Vec::with_capacity(usize::try_from(EVENT_PAGE_LIMIT)?); let mut summary = RegistryChangedEventEmitSummary::default(); @@ -77,11 +94,18 @@ pub(super) async fn emit_registry_changed_events_from_checkpoint( .iter() .map(|(_, assignment)| assignment) .collect::>(); - emit_registry_changed_event_chunk(pool, &assignments, &mut events, &mut summary) - .await?; + emit_registry_changed_event_chunk( + pool, + &assignments, + &mut events, + &mut summary, + startup_progress, + ) + .await?; + record_startup_adapter_progress(pool, startup_progress).await?; } } - flush_registry_changed_events(pool, &mut events, &mut summary).await?; + flush_registry_changed_events(pool, &mut events, &mut summary, startup_progress).await?; Ok(summary) } @@ -90,6 +114,7 @@ pub(super) async fn emit_registry_changed_event_chunk( assignments: &[&ObservedRegistryAssignment], events: &mut Vec, summary: &mut RegistryChangedEventEmitSummary, + startup_progress: &mut Option<&mut dyn StartupAdapterProgress>, ) -> Result<()> { if assignments.is_empty() { return Ok(()); @@ -109,7 +134,7 @@ pub(super) async fn emit_registry_changed_event_chunk( events.push(event); } if events.len() >= NORMALIZED_EVENT_UPSERT_CHUNK_SIZE { - flush_registry_changed_events(pool, events, summary).await?; + flush_registry_changed_events(pool, events, summary, startup_progress).await?; } } @@ -120,6 +145,7 @@ pub(super) async fn flush_registry_changed_events( pool: &PgPool, events: &mut Vec, summary: &mut RegistryChangedEventEmitSummary, + startup_progress: &mut Option<&mut dyn StartupAdapterProgress>, ) -> Result<()> { if events.is_empty() { return Ok(()); @@ -129,6 +155,7 @@ pub(super) async fn flush_registry_changed_events( summary.synced_count += events.len(); summary.inserted_count += upsert.inserted_count; events.clear(); + record_startup_adapter_progress(pool, startup_progress).await?; Ok(()) } diff --git a/crates/adapters/src/ens_v1_subregistry_discovery/entrypoints.rs b/crates/adapters/src/ens_v1_subregistry_discovery/entrypoints.rs index b2670ab4..83ed3d16 100644 --- a/crates/adapters/src/ens_v1_subregistry_discovery/entrypoints.rs +++ b/crates/adapters/src/ens_v1_subregistry_discovery/entrypoints.rs @@ -22,6 +22,7 @@ pub async fn sync_ens_v1_subregistry_discovery( None, checkpoint::PAGE_LIMIT, None, + &mut None, ) .await .map(|(summary, _)| summary) @@ -49,6 +50,7 @@ pub async fn sync_ens_v1_subregistry_discovery_through_block( None, checkpoint::PAGE_LIMIT, None, + &mut None, ) .await .map(|(summary, _)| summary) @@ -75,6 +77,7 @@ pub async fn sync_ens_v1_subregistry_discovery_through_block_with_expected_admis None, checkpoint::PAGE_LIMIT, None, + &mut None, ) .await .map(|(summary, _)| summary) @@ -99,6 +102,7 @@ impl EnsV1SubregistryDiscoverySyncSummary { None, checkpoint::PAGE_LIMIT, None, + &mut None, ) .await .map(|(summary, _)| summary) @@ -122,6 +126,7 @@ impl EnsV1SubregistryDiscoverySyncSummary { None, checkpoint::PAGE_LIMIT, None, + &mut None, ) .await .map(|(summary, _)| summary) @@ -144,6 +149,7 @@ impl EnsV1SubregistryDiscoverySyncSummary { None, checkpoint::PAGE_LIMIT, None, + &mut None, ) .await .map(|(summary, _)| summary) diff --git a/crates/adapters/src/ens_v1_subregistry_discovery/reconciliation.rs b/crates/adapters/src/ens_v1_subregistry_discovery/reconciliation.rs index 1b337129..7e37e1f6 100644 --- a/crates/adapters/src/ens_v1_subregistry_discovery/reconciliation.rs +++ b/crates/adapters/src/ens_v1_subregistry_discovery/reconciliation.rs @@ -10,17 +10,19 @@ use super::{ EnsV1SubregistryDiscoverySyncSummary, checkpoint::{RECONCILIATION_PAGE_LIMIT, SubregistryReplayCheckpoint}, }; +use crate::checkpoint_context::{StartupAdapterProgress, record_startup_adapter_progress}; /// Pages one discovery source's staged latest-per-key assignments straight /// from the checkpoint items, so the finalize reconcile never materializes a /// source's observations in memory (#168). -struct CheckpointAssignmentPageSource<'a> { +struct CheckpointAssignmentPageSource<'a, 'progress> { pool: &'a PgPool, checkpoint: &'a SubregistryReplayCheckpoint, discovery_source: &'a str, + startup_progress: &'a tokio::sync::Mutex>, } -impl DiscoveryObservationPageSource for CheckpointAssignmentPageSource<'_> { +impl DiscoveryObservationPageSource for CheckpointAssignmentPageSource<'_, '_> { async fn load_page( &self, after_key: Option<&str>, @@ -34,6 +36,11 @@ impl DiscoveryObservationPageSource for CheckpointAssignmentPageSource<'_> { .map(|(item_key, assignment)| Ok((item_key, assignment.discovery_observation()?))) .collect() } + + async fn record_progress(&self) -> Result<()> { + let mut startup_progress = self.startup_progress.lock().await; + record_startup_adapter_progress(self.pool, &mut startup_progress).await + } } pub(super) async fn reconcile_subregistry_discovery_from_checkpoint( @@ -41,21 +48,30 @@ pub(super) async fn reconcile_subregistry_discovery_from_checkpoint( checkpoint: &SubregistryReplayCheckpoint, discovery_sources: &[String], reconciliation: &mut EnsV1SubregistryDiscoverySyncSummary, + startup_progress: &mut Option<&mut dyn StartupAdapterProgress>, ) -> Result<()> { - for discovery_source in discovery_sources { - let page_source = CheckpointAssignmentPageSource { - pool, - checkpoint, - discovery_source, - }; - let source_reconciliation = - reconcile_discovery_observations_streamed(pool, discovery_source, &page_source).await?; - reconciliation.active_edge_count += source_reconciliation.active_edge_count; - reconciliation.admitted_edge_count += source_reconciliation.admitted_edge_count; - reconciliation.inserted_edge_count += source_reconciliation.inserted_edge_count; - reconciliation.deactivated_edge_count += source_reconciliation.deactivated_edge_count; + let progress = tokio::sync::Mutex::new(startup_progress.take()); + let result = async { + for discovery_source in discovery_sources { + let page_source = CheckpointAssignmentPageSource { + pool, + checkpoint, + discovery_source, + startup_progress: &progress, + }; + let source_reconciliation = + reconcile_discovery_observations_streamed(pool, discovery_source, &page_source) + .await?; + reconciliation.active_edge_count += source_reconciliation.active_edge_count; + reconciliation.admitted_edge_count += source_reconciliation.admitted_edge_count; + reconciliation.inserted_edge_count += source_reconciliation.inserted_edge_count; + reconciliation.deactivated_edge_count += source_reconciliation.deactivated_edge_count; + } + Ok(()) } - Ok(()) + .await; + *startup_progress = progress.into_inner(); + result } pub(super) async fn reconcile_subregistry_discovery_source_through_block( diff --git a/crates/adapters/src/ens_v1_subregistry_discovery/replay.rs b/crates/adapters/src/ens_v1_subregistry_discovery/replay.rs index 0b51a721..917ef312 100644 --- a/crates/adapters/src/ens_v1_subregistry_discovery/replay.rs +++ b/crates/adapters/src/ens_v1_subregistry_discovery/replay.rs @@ -2,7 +2,7 @@ use anyhow::{Context, Result, ensure}; use bigname_storage::acquire_raw_log_staging_read_guard; use sqlx::PgPool; -use crate::checkpoint_context::AdapterCheckpointContext; +use crate::checkpoint_context::{AdapterCheckpointContext, StartupAdapterProgress}; use super::{ DiscoveryEdgeMutation, EnsV1SubregistryDiscoverySyncSummary, ReplayAdapterCheckpointContext, @@ -45,6 +45,7 @@ pub async fn sync_ens_v1_subregistry_discovery_with_replay_checkpoint_and_log_li &checkpoint, max_raw_logs_per_page, None, + None, ) .await } @@ -55,6 +56,7 @@ pub(super) async fn sync_ens_v1_subregistry_discovery_with_checkpoint_context( checkpoint: &AdapterCheckpointContext, max_raw_logs_per_page: usize, progress_log_every_pages: Option, + mut startup_progress: Option<&mut dyn StartupAdapterProgress>, ) -> Result { ensure!( max_raw_logs_per_page > 0, @@ -84,6 +86,7 @@ pub(super) async fn sync_ens_v1_subregistry_discovery_with_checkpoint_context( Some(&checkpoint), checkpoint_page_limit, progress_log_every_pages, + &mut startup_progress, ) .await?; if !repeat_checkpoint_replay { diff --git a/crates/adapters/src/ens_v1_subregistry_discovery/startup.rs b/crates/adapters/src/ens_v1_subregistry_discovery/startup.rs index 19efc84b..e67bc122 100644 --- a/crates/adapters/src/ens_v1_subregistry_discovery/startup.rs +++ b/crates/adapters/src/ens_v1_subregistry_discovery/startup.rs @@ -2,7 +2,7 @@ use anyhow::Result; use sqlx::PgPool; use super::{ - EnsV1SubregistryDiscoverySyncSummary, StartupAdapterCheckpointContext, + EnsV1SubregistryDiscoverySyncSummary, StartupAdapterCheckpointContext, StartupAdapterProgress, checkpoint::SubregistryReplayCheckpoint, loader::RegistryRawLogPosition, replay::sync_ens_v1_subregistry_discovery_with_checkpoint_context, }; @@ -22,6 +22,26 @@ pub async fn sync_ens_v1_subregistry_discovery_with_startup_checkpoint_and_log_l &checkpoint, max_raw_logs_per_page, Some(STARTUP_PROGRESS_LOG_EVERY_PAGES), + None, + ) + .await +} + +pub async fn sync_ens_v1_subregistry_discovery_with_startup_checkpoint_and_log_limit_and_progress( + pool: &PgPool, + chain: &str, + checkpoint: &StartupAdapterCheckpointContext, + max_raw_logs_per_page: usize, + progress: &mut dyn StartupAdapterProgress, +) -> Result { + let checkpoint = checkpoint.adapter_context(pool, chain).await?; + sync_ens_v1_subregistry_discovery_with_checkpoint_context( + pool, + chain, + &checkpoint, + max_raw_logs_per_page, + Some(STARTUP_PROGRESS_LOG_EVERY_PAGES), + Some(progress), ) .await } diff --git a/crates/adapters/src/ens_v1_subregistry_discovery/tests.rs b/crates/adapters/src/ens_v1_subregistry_discovery/tests.rs index d35f72a9..ab6f1261 100644 --- a/crates/adapters/src/ens_v1_subregistry_discovery/tests.rs +++ b/crates/adapters/src/ens_v1_subregistry_discovery/tests.rs @@ -34,6 +34,31 @@ use super::{ static NEXT_TEST_ID: AtomicU64 = AtomicU64::new(1); const TEST_MIGRATOR: sqlx::migrate::Migrator = sqlx::migrate!("../../migrations"); +#[derive(Default)] +struct CountingStartupAdapterProgress { + record_count: usize, + completed_record_count: usize, +} + +impl StartupAdapterProgress for CountingStartupAdapterProgress { + fn record<'a>(&'a mut self, pool: &'a PgPool) -> crate::StartupAdapterProgressFuture<'a> { + self.record_count += 1; + Box::pin(async move { + let status = sqlx::query_scalar::<_, String>( + "SELECT status FROM normalized_replay_adapter_checkpoints \ + WHERE adapter = 'ens_v1_subregistry_discovery' \ + ORDER BY updated_at DESC LIMIT 1", + ) + .fetch_optional(pool) + .await?; + if status.as_deref() == Some("completed") { + self.completed_record_count += 1; + } + Ok(()) + }) + } +} + struct TestDir { path: PathBuf, } @@ -826,13 +851,23 @@ async fn startup_checkpointed_subregistry_matches_uncheckpointed_edges_and_event sync_ens_v1_subregistry_discovery(expected_database.pool(), chain).await?; let startup_checkpoint = StartupAdapterCheckpointContext::new("test", 43)?; - sync_ens_v1_subregistry_discovery_with_startup_checkpoint_and_log_limit( + let mut startup_progress = CountingStartupAdapterProgress::default(); + sync_ens_v1_subregistry_discovery_with_startup_checkpoint_and_log_limit_and_progress( startup_database.pool(), chain, &startup_checkpoint, 1, + &mut startup_progress, ) .await?; + assert!( + startup_progress.record_count > 3, + "checkpoint ingestion and the paged reconciliation/event finalization must repeatedly report startup progress" + ); + assert!( + startup_progress.completed_record_count > 0, + "the completed finalization boundary must report startup progress" + ); assert_eq!( load_subregistry_discovery_outputs(startup_database.pool()).await?, diff --git a/crates/adapters/src/ens_v1_unwrapped_authority.rs b/crates/adapters/src/ens_v1_unwrapped_authority.rs index b024412b..9603559f 100644 --- a/crates/adapters/src/ens_v1_unwrapped_authority.rs +++ b/crates/adapters/src/ens_v1_unwrapped_authority.rs @@ -478,6 +478,7 @@ pub use self::pipeline::{ sync_ens_v1_unwrapped_authority, sync_ens_v1_unwrapped_authority_with_replay_checkpoint_and_log_limit, sync_ens_v1_unwrapped_authority_with_startup_checkpoint_and_log_limit, + sync_ens_v1_unwrapped_authority_with_startup_checkpoint_and_log_limit_and_progress, }; pub use checkpoint::clear_replay_adapter_checkpoints; pub use resolver_profile_reconciliation::{ diff --git a/crates/adapters/src/ens_v1_unwrapped_authority/checkpoint.rs b/crates/adapters/src/ens_v1_unwrapped_authority/checkpoint.rs index 9df3ff2f..d13a6956 100644 --- a/crates/adapters/src/ens_v1_unwrapped_authority/checkpoint.rs +++ b/crates/adapters/src/ens_v1_unwrapped_authority/checkpoint.rs @@ -7,11 +7,19 @@ use bigname_storage::{ }; use futures_util::TryStreamExt; use serde_json::{Value, json}; -use sqlx::{PgPool, Postgres, QueryBuilder, Row}; +use sqlx::{PgPool, Row}; +mod items; +mod payload; mod persistence; mod startup_events; use crate::checkpoint_codec::JsonbCheckpointCodec; +use items::{ + checkpoint_item_rows, checkpoint_pending_observation_delete_keys, delete_checkpoint_items, + insert_checkpoint_items, update_checkpoint_progress, +}; +pub(super) use payload::{decode_item, encode_item}; +use payload::{flushed_events_from_payload, summary_from_payload, summary_payload}; pub use persistence::clear_replay_adapter_checkpoints; use persistence::{delete_checkpoint, load_checkpoint_row}; @@ -588,314 +596,5 @@ impl UnwrappedAuthorityReplayCheckpoint { } } -fn checkpoint_item_rows( - state: &UnwrappedAuthorityReplayCheckpointStateRef<'_>, - delta: &UnwrappedAuthorityReplayCheckpointDelta, -) -> Result> { - let mut rows = Vec::new(); - for key in &delta.history_keys { - if let Some(value) = state.histories.get(key) { - rows.push((ITEM_KIND_HISTORY, key.clone(), encode_item(value)?)); - } - } - for key in &delta.reverse_history_keys { - if let Some(value) = state.reverse_histories.get(key) { - rows.push((ITEM_KIND_REVERSE_HISTORY, key.clone(), encode_item(value)?)); - } - } - for key in &delta.known_name_keys { - if let Some(value) = state.known_names_by_namehash.get(key) { - rows.push((ITEM_KIND_KNOWN_NAME, key.clone(), encode_item(value)?)); - } - } - for key in &delta.known_name_ref_keys { - if let Some(value) = state.known_name_refs_by_namehash.get(key) { - rows.push((ITEM_KIND_KNOWN_NAME_REF, key.clone(), encode_item(value)?)); - } - } - for key in &delta.namehash_labelhash_keys { - if let Some(labelhash) = state.namehash_to_labelhash.get(key) { - rows.push(( - ITEM_KIND_NAMEHASH_LABELHASH, - key.clone(), - CHECKPOINT_CODEC.encode(json!({ "labelhash": labelhash })), - )); - } - } - for key in &delta.pending_observation_keys { - if let Some(observations) = state - .pending_namehash_observations - .get(key) - .filter(|observations| !observations.is_empty()) - { - rows.push(( - ITEM_KIND_PENDING_OBSERVATIONS, - key.clone(), - encode_item(observations.as_slice())?, - )); - } - } - for node in &delta.migrated_nodes { - rows.push(( - ITEM_KIND_MIGRATED_NODE, - node.clone(), - CHECKPOINT_CODEC.encode(json!({ "node": node })), - )); - } - Ok(rows) -} - -fn checkpoint_pending_observation_delete_keys( - state: &UnwrappedAuthorityReplayCheckpointStateRef<'_>, - delta: &UnwrappedAuthorityReplayCheckpointDelta, -) -> Vec { - delta - .pending_observation_keys - .iter() - .filter(|key| { - state - .pending_namehash_observations - .get(*key) - .is_none_or(Vec::is_empty) - }) - .cloned() - .collect() -} - -async fn delete_checkpoint_items( - transaction: &mut sqlx::Transaction<'_, Postgres>, - checkpoint: &UnwrappedAuthorityReplayCheckpoint, - item_kind: &str, - item_keys: &[String], -) -> Result<()> { - for chunk in item_keys.chunks(CHECKPOINT_ITEM_DELETE_BATCH_SIZE) { - if chunk.is_empty() { - continue; - } - sqlx::query( - r#" - DELETE FROM normalized_replay_adapter_checkpoint_items - WHERE deployment_profile = $1 - AND chain_id = $2 - AND cursor_kind = $3 - AND adapter = $4 - AND checkpoint_scope = $5 - AND item_kind = $6 - AND item_key = ANY($7) - "#, - ) - .bind(&checkpoint.context.deployment_profile) - .bind(&checkpoint.chain) - .bind(&checkpoint.context.cursor_kind) - .bind(ADAPTER) - .bind(checkpoint.context.checkpoint_scope) - .bind(item_kind) - .bind(chunk) - .execute(transaction.as_mut()) - .await - .context("failed to delete cleared unwrapped-authority replay checkpoint items")?; - } - Ok(()) -} - -async fn insert_checkpoint_items( - transaction: &mut sqlx::Transaction<'_, Postgres>, - checkpoint: &UnwrappedAuthorityReplayCheckpoint, - item_rows: &[(&'static str, String, Value)], -) -> Result<()> { - for chunk in item_rows.chunks(CHECKPOINT_ITEM_INSERT_BATCH_SIZE) { - if chunk.is_empty() { - continue; - } - let mut builder = QueryBuilder::::new( - r#" - INSERT INTO normalized_replay_adapter_checkpoint_items ( - deployment_profile, - chain_id, - cursor_kind, - adapter, - checkpoint_scope, - item_kind, - item_key, - item_payload - ) - "#, - ); - builder.push_values( - chunk.iter(), - |mut row, (item_kind, item_key, item_payload)| { - row.push_bind(&checkpoint.context.deployment_profile) - .push_bind(&checkpoint.chain) - .push_bind(&checkpoint.context.cursor_kind) - .push_bind(ADAPTER) - .push_bind(checkpoint.context.checkpoint_scope) - .push_bind(*item_kind) - .push_bind(item_key) - .push_bind(item_payload); - }, - ); - builder.push( - r#" - ON CONFLICT ( - deployment_profile, - chain_id, - cursor_kind, - adapter, - checkpoint_scope, - item_kind, - item_key - ) DO UPDATE - SET item_payload = EXCLUDED.item_payload, - updated_at = now() - "#, - ); - builder - .build() - .execute(transaction.as_mut()) - .await - .context("failed to upsert unwrapped-authority replay checkpoint items")?; - } - Ok(()) -} - -async fn update_checkpoint_progress( - transaction: &mut sqlx::Transaction<'_, Postgres>, - checkpoint: &UnwrappedAuthorityReplayCheckpoint, - status: &str, - last_block_number: Option, - scanned_log_count: usize, - matched_log_count: usize, - staged_item_count: usize, - staged_aux_item_count: usize, - state_payload: Value, -) -> Result<()> { - sqlx::query( - r#" - UPDATE normalized_replay_adapter_checkpoints - SET - status = $6, - last_block_number = $7, - last_transaction_index = CASE WHEN $7::BIGINT IS NULL THEN NULL ELSE 0 END, - last_log_index = CASE WHEN $7::BIGINT IS NULL THEN NULL ELSE 0 END, - last_emitting_address = CASE WHEN $7::BIGINT IS NULL THEN NULL ELSE 'block-boundary' END, - staged_item_count = $8, - staged_aux_item_count = $9, - scanned_log_count = $10, - matched_log_count = $11, - state_payload = $12, - raw_log_retention_generation = $13, - raw_log_input_revision = $14, - updated_at = now(), - last_failure_reason = NULL - WHERE deployment_profile = $1 - AND chain_id = $2 - AND cursor_kind = $3 - AND adapter = $4 - AND checkpoint_scope = $5 - "#, - ) - .bind(&checkpoint.context.deployment_profile) - .bind(&checkpoint.chain) - .bind(&checkpoint.context.cursor_kind) - .bind(ADAPTER) - .bind(checkpoint.context.checkpoint_scope) - .bind(status) - .bind(last_block_number) - .bind(i64::try_from(staged_item_count).context("staged item count overflowed i64")?) - .bind(i64::try_from(staged_aux_item_count).context("staged aux item count overflowed i64")?) - .bind(i64::try_from(scanned_log_count).context("scanned log count overflowed i64")?) - .bind(i64::try_from(matched_log_count).context("matched log count overflowed i64")?) - .bind(state_payload) - .bind(checkpoint.raw_log_input_version.retention_generation) - .bind(checkpoint.raw_log_input_version.revision) - .execute(transaction.as_mut()) - .await - .context("failed to update unwrapped-authority replay checkpoint progress")?; - Ok(()) -} - -pub(super) fn encode_item(value: &T) -> Result -where - T: serde::Serialize + ?Sized, -{ - CHECKPOINT_CODEC.encode_serde( - value, - "failed to encode unwrapped-authority checkpoint item", - ) -} - -pub(super) fn decode_item(value: Value, item_kind: &str) -> Result -where - T: serde::de::DeserializeOwned, -{ - CHECKPOINT_CODEC.decode_serde( - value, - "failed to decode unwrapped-authority checkpoint JSONB encoding", - format!("failed to decode unwrapped-authority checkpoint item {item_kind}"), - ) -} - -fn summary_payload(summary: &EnsV1UnwrappedAuthoritySyncSummary) -> Value { - json!({ - "scanned_log_count": summary.scanned_log_count, - "matched_log_count": summary.matched_log_count, - "total_name_surface_count": summary.total_name_surface_count, - "total_resource_count": summary.total_resource_count, - "total_surface_binding_count": summary.total_surface_binding_count, - "total_normalized_event_count": summary.total_normalized_event_count, - "total_normalized_event_inserted_count": summary.total_normalized_event_inserted_count, - "by_kind": summary.by_kind, - }) -} - -fn summary_from_payload(payload: &Value) -> Result { - Ok(EnsV1UnwrappedAuthoritySyncSummary { - scanned_log_count: usize_field(payload, "scanned_log_count")?, - matched_log_count: usize_field(payload, "matched_log_count")?, - total_name_surface_count: usize_field(payload, "total_name_surface_count")?, - total_resource_count: usize_field(payload, "total_resource_count")?, - total_surface_binding_count: usize_field(payload, "total_surface_binding_count")?, - total_normalized_event_count: usize_field(payload, "total_normalized_event_count")?, - total_normalized_event_inserted_count: usize_field( - payload, - "total_normalized_event_inserted_count", - )?, - by_kind: serde_json::from_value( - payload.get("by_kind").cloned().unwrap_or_else(|| json!({})), - ) - .context("checkpoint summary by_kind is invalid")?, - }) -} - -fn flushed_events_from_payload(payload: &Value) -> Result { - Ok(UnwrappedAuthorityReplayFlushedEvents { - total_count: optional_usize_field(payload, "flushed_normalized_event_count")?, - inserted_count: optional_usize_field(payload, "flushed_normalized_event_inserted_count")?, - by_kind: payload - .get("flushed_by_kind") - .cloned() - .map(serde_json::from_value) - .transpose() - .context("checkpoint flushed_by_kind is invalid")? - .unwrap_or_default(), - }) -} - -fn usize_field(payload: &Value, field: &str) -> Result { - let value = payload - .get(field) - .and_then(Value::as_i64) - .with_context(|| format!("checkpoint summary is missing i64 field {field}"))?; - usize::try_from(value) - .with_context(|| format!("checkpoint summary field {field} overflows usize")) -} - -fn optional_usize_field(payload: &Value, field: &str) -> Result { - let Some(value) = payload.get(field).and_then(Value::as_i64) else { - return Ok(0); - }; - usize::try_from(value).with_context(|| format!("checkpoint field {field} overflows usize")) -} - #[cfg(test)] mod tests; diff --git a/crates/adapters/src/ens_v1_unwrapped_authority/checkpoint/items.rs b/crates/adapters/src/ens_v1_unwrapped_authority/checkpoint/items.rs new file mode 100644 index 00000000..295b447c --- /dev/null +++ b/crates/adapters/src/ens_v1_unwrapped_authority/checkpoint/items.rs @@ -0,0 +1,238 @@ +use anyhow::{Context, Result}; +use serde_json::{Value, json}; +use sqlx::{Postgres, QueryBuilder}; + +use super::{ + ADAPTER, CHECKPOINT_CODEC, CHECKPOINT_ITEM_DELETE_BATCH_SIZE, + CHECKPOINT_ITEM_INSERT_BATCH_SIZE, ITEM_KIND_HISTORY, ITEM_KIND_KNOWN_NAME, + ITEM_KIND_KNOWN_NAME_REF, ITEM_KIND_MIGRATED_NODE, ITEM_KIND_NAMEHASH_LABELHASH, + ITEM_KIND_PENDING_OBSERVATIONS, ITEM_KIND_REVERSE_HISTORY, UnwrappedAuthorityReplayCheckpoint, + UnwrappedAuthorityReplayCheckpointDelta, UnwrappedAuthorityReplayCheckpointStateRef, + encode_item, +}; + +pub(super) fn checkpoint_item_rows( + state: &UnwrappedAuthorityReplayCheckpointStateRef<'_>, + delta: &UnwrappedAuthorityReplayCheckpointDelta, +) -> Result> { + let mut rows = Vec::new(); + for key in &delta.history_keys { + if let Some(value) = state.histories.get(key) { + rows.push((ITEM_KIND_HISTORY, key.clone(), encode_item(value)?)); + } + } + for key in &delta.reverse_history_keys { + if let Some(value) = state.reverse_histories.get(key) { + rows.push((ITEM_KIND_REVERSE_HISTORY, key.clone(), encode_item(value)?)); + } + } + for key in &delta.known_name_keys { + if let Some(value) = state.known_names_by_namehash.get(key) { + rows.push((ITEM_KIND_KNOWN_NAME, key.clone(), encode_item(value)?)); + } + } + for key in &delta.known_name_ref_keys { + if let Some(value) = state.known_name_refs_by_namehash.get(key) { + rows.push((ITEM_KIND_KNOWN_NAME_REF, key.clone(), encode_item(value)?)); + } + } + for key in &delta.namehash_labelhash_keys { + if let Some(labelhash) = state.namehash_to_labelhash.get(key) { + rows.push(( + ITEM_KIND_NAMEHASH_LABELHASH, + key.clone(), + CHECKPOINT_CODEC.encode(json!({ "labelhash": labelhash })), + )); + } + } + for key in &delta.pending_observation_keys { + if let Some(observations) = state + .pending_namehash_observations + .get(key) + .filter(|observations| !observations.is_empty()) + { + rows.push(( + ITEM_KIND_PENDING_OBSERVATIONS, + key.clone(), + encode_item(observations.as_slice())?, + )); + } + } + for node in &delta.migrated_nodes { + rows.push(( + ITEM_KIND_MIGRATED_NODE, + node.clone(), + CHECKPOINT_CODEC.encode(json!({ "node": node })), + )); + } + Ok(rows) +} + +pub(super) fn checkpoint_pending_observation_delete_keys( + state: &UnwrappedAuthorityReplayCheckpointStateRef<'_>, + delta: &UnwrappedAuthorityReplayCheckpointDelta, +) -> Vec { + delta + .pending_observation_keys + .iter() + .filter(|key| { + state + .pending_namehash_observations + .get(*key) + .is_none_or(Vec::is_empty) + }) + .cloned() + .collect() +} + +pub(super) async fn delete_checkpoint_items( + transaction: &mut sqlx::Transaction<'_, Postgres>, + checkpoint: &UnwrappedAuthorityReplayCheckpoint, + item_kind: &str, + item_keys: &[String], +) -> Result<()> { + for chunk in item_keys.chunks(CHECKPOINT_ITEM_DELETE_BATCH_SIZE) { + if chunk.is_empty() { + continue; + } + sqlx::query( + r#" + DELETE FROM normalized_replay_adapter_checkpoint_items + WHERE deployment_profile = $1 + AND chain_id = $2 + AND cursor_kind = $3 + AND adapter = $4 + AND checkpoint_scope = $5 + AND item_kind = $6 + AND item_key = ANY($7) + "#, + ) + .bind(&checkpoint.context.deployment_profile) + .bind(&checkpoint.chain) + .bind(&checkpoint.context.cursor_kind) + .bind(ADAPTER) + .bind(checkpoint.context.checkpoint_scope) + .bind(item_kind) + .bind(chunk) + .execute(transaction.as_mut()) + .await + .context("failed to delete cleared unwrapped-authority replay checkpoint items")?; + } + Ok(()) +} + +pub(super) async fn insert_checkpoint_items( + transaction: &mut sqlx::Transaction<'_, Postgres>, + checkpoint: &UnwrappedAuthorityReplayCheckpoint, + item_rows: &[(&'static str, String, Value)], +) -> Result<()> { + for chunk in item_rows.chunks(CHECKPOINT_ITEM_INSERT_BATCH_SIZE) { + if chunk.is_empty() { + continue; + } + let mut builder = QueryBuilder::::new( + r#" + INSERT INTO normalized_replay_adapter_checkpoint_items ( + deployment_profile, + chain_id, + cursor_kind, + adapter, + checkpoint_scope, + item_kind, + item_key, + item_payload + ) + "#, + ); + builder.push_values( + chunk.iter(), + |mut row, (item_kind, item_key, item_payload)| { + row.push_bind(&checkpoint.context.deployment_profile) + .push_bind(&checkpoint.chain) + .push_bind(&checkpoint.context.cursor_kind) + .push_bind(ADAPTER) + .push_bind(checkpoint.context.checkpoint_scope) + .push_bind(*item_kind) + .push_bind(item_key) + .push_bind(item_payload); + }, + ); + builder.push( + r#" + ON CONFLICT ( + deployment_profile, + chain_id, + cursor_kind, + adapter, + checkpoint_scope, + item_kind, + item_key + ) DO UPDATE + SET item_payload = EXCLUDED.item_payload, + updated_at = now() + "#, + ); + builder + .build() + .execute(transaction.as_mut()) + .await + .context("failed to upsert unwrapped-authority replay checkpoint items")?; + } + Ok(()) +} + +pub(super) async fn update_checkpoint_progress( + transaction: &mut sqlx::Transaction<'_, Postgres>, + checkpoint: &UnwrappedAuthorityReplayCheckpoint, + status: &str, + last_block_number: Option, + scanned_log_count: usize, + matched_log_count: usize, + staged_item_count: usize, + staged_aux_item_count: usize, + state_payload: Value, +) -> Result<()> { + sqlx::query( + r#" + UPDATE normalized_replay_adapter_checkpoints + SET + status = $6, + last_block_number = $7, + last_transaction_index = CASE WHEN $7::BIGINT IS NULL THEN NULL ELSE 0 END, + last_log_index = CASE WHEN $7::BIGINT IS NULL THEN NULL ELSE 0 END, + last_emitting_address = CASE WHEN $7::BIGINT IS NULL THEN NULL ELSE 'block-boundary' END, + staged_item_count = $8, + staged_aux_item_count = $9, + scanned_log_count = $10, + matched_log_count = $11, + state_payload = $12, + raw_log_retention_generation = $13, + raw_log_input_revision = $14, + updated_at = now(), + last_failure_reason = NULL + WHERE deployment_profile = $1 + AND chain_id = $2 + AND cursor_kind = $3 + AND adapter = $4 + AND checkpoint_scope = $5 + "#, + ) + .bind(&checkpoint.context.deployment_profile) + .bind(&checkpoint.chain) + .bind(&checkpoint.context.cursor_kind) + .bind(ADAPTER) + .bind(checkpoint.context.checkpoint_scope) + .bind(status) + .bind(last_block_number) + .bind(i64::try_from(staged_item_count).context("staged item count overflowed i64")?) + .bind(i64::try_from(staged_aux_item_count).context("staged aux item count overflowed i64")?) + .bind(i64::try_from(scanned_log_count).context("scanned log count overflowed i64")?) + .bind(i64::try_from(matched_log_count).context("matched log count overflowed i64")?) + .bind(state_payload) + .bind(checkpoint.raw_log_input_version.retention_generation) + .bind(checkpoint.raw_log_input_version.revision) + .execute(transaction.as_mut()) + .await + .context("failed to update unwrapped-authority replay checkpoint progress")?; + Ok(()) +} diff --git a/crates/adapters/src/ens_v1_unwrapped_authority/checkpoint/payload.rs b/crates/adapters/src/ens_v1_unwrapped_authority/checkpoint/payload.rs new file mode 100644 index 00000000..71464d65 --- /dev/null +++ b/crates/adapters/src/ens_v1_unwrapped_authority/checkpoint/payload.rs @@ -0,0 +1,94 @@ +use anyhow::{Context, Result}; +use serde_json::{Value, json}; + +use super::{ + CHECKPOINT_CODEC, EnsV1UnwrappedAuthoritySyncSummary, UnwrappedAuthorityReplayFlushedEvents, +}; + +pub(in crate::ens_v1_unwrapped_authority) fn encode_item(value: &T) -> Result +where + T: serde::Serialize + ?Sized, +{ + CHECKPOINT_CODEC.encode_serde( + value, + "failed to encode unwrapped-authority checkpoint item", + ) +} + +pub(in crate::ens_v1_unwrapped_authority) fn decode_item( + value: Value, + item_kind: &str, +) -> Result +where + T: serde::de::DeserializeOwned, +{ + CHECKPOINT_CODEC.decode_serde( + value, + "failed to decode unwrapped-authority checkpoint JSONB encoding", + format!("failed to decode unwrapped-authority checkpoint item {item_kind}"), + ) +} + +pub(super) fn summary_payload(summary: &EnsV1UnwrappedAuthoritySyncSummary) -> Value { + json!({ + "scanned_log_count": summary.scanned_log_count, + "matched_log_count": summary.matched_log_count, + "total_name_surface_count": summary.total_name_surface_count, + "total_resource_count": summary.total_resource_count, + "total_surface_binding_count": summary.total_surface_binding_count, + "total_normalized_event_count": summary.total_normalized_event_count, + "total_normalized_event_inserted_count": summary.total_normalized_event_inserted_count, + "by_kind": summary.by_kind, + }) +} + +pub(super) fn summary_from_payload(payload: &Value) -> Result { + Ok(EnsV1UnwrappedAuthoritySyncSummary { + scanned_log_count: usize_field(payload, "scanned_log_count")?, + matched_log_count: usize_field(payload, "matched_log_count")?, + total_name_surface_count: usize_field(payload, "total_name_surface_count")?, + total_resource_count: usize_field(payload, "total_resource_count")?, + total_surface_binding_count: usize_field(payload, "total_surface_binding_count")?, + total_normalized_event_count: usize_field(payload, "total_normalized_event_count")?, + total_normalized_event_inserted_count: usize_field( + payload, + "total_normalized_event_inserted_count", + )?, + by_kind: serde_json::from_value( + payload.get("by_kind").cloned().unwrap_or_else(|| json!({})), + ) + .context("checkpoint summary by_kind is invalid")?, + }) +} + +pub(super) fn flushed_events_from_payload( + payload: &Value, +) -> Result { + Ok(UnwrappedAuthorityReplayFlushedEvents { + total_count: optional_usize_field(payload, "flushed_normalized_event_count")?, + inserted_count: optional_usize_field(payload, "flushed_normalized_event_inserted_count")?, + by_kind: payload + .get("flushed_by_kind") + .cloned() + .map(serde_json::from_value) + .transpose() + .context("checkpoint flushed_by_kind is invalid")? + .unwrap_or_default(), + }) +} + +fn usize_field(payload: &Value, field: &str) -> Result { + let value = payload + .get(field) + .and_then(Value::as_i64) + .with_context(|| format!("checkpoint summary is missing i64 field {field}"))?; + usize::try_from(value) + .with_context(|| format!("checkpoint summary field {field} overflows usize")) +} + +fn optional_usize_field(payload: &Value, field: &str) -> Result { + let Some(value) = payload.get(field).and_then(Value::as_i64) else { + return Ok(0); + }; + usize::try_from(value).with_context(|| format!("checkpoint field {field} overflows usize")) +} diff --git a/crates/adapters/src/ens_v1_unwrapped_authority/checkpoint/startup_events.rs b/crates/adapters/src/ens_v1_unwrapped_authority/checkpoint/startup_events.rs index 32b2bb5c..bc4548f6 100644 --- a/crates/adapters/src/ens_v1_unwrapped_authority/checkpoint/startup_events.rs +++ b/crates/adapters/src/ens_v1_unwrapped_authority/checkpoint/startup_events.rs @@ -1,4 +1,5 @@ use super::*; +use crate::checkpoint_context::{StartupAdapterProgress, record_startup_adapter_progress}; use crate::ens_v1_unwrapped_authority::event_persistence::pin_existing_event_manifest_provenance; pub(super) const ITEM_KIND_STARTUP_PENDING_EVENT: &str = "startup_pending_normalized_event"; @@ -57,6 +58,7 @@ impl UnwrappedAuthorityReplayCheckpoint { &mut self, pool: &PgPool, staged_events: &mut UnwrappedAuthorityReplayFlushedEvents, + startup_progress: &mut Option<&mut dyn StartupAdapterProgress>, ) -> Result { ensure!( self.is_startup(), @@ -170,6 +172,7 @@ impl UnwrappedAuthorityReplayCheckpoint { staged_events.inserted_count = next_inserted_count; published_count += events.len(); + record_startup_adapter_progress(pool, startup_progress).await?; } self.flushed_events = staged_events.clone(); Ok(published_count) diff --git a/crates/adapters/src/ens_v1_unwrapped_authority/pipeline.rs b/crates/adapters/src/ens_v1_unwrapped_authority/pipeline.rs index a5d51513..e1c2b9fb 100644 --- a/crates/adapters/src/ens_v1_unwrapped_authority/pipeline.rs +++ b/crates/adapters/src/ens_v1_unwrapped_authority/pipeline.rs @@ -1,6 +1,8 @@ use super::resolver_profile_reconciliation::ResolverProfileReplayContext; use super::*; -use crate::checkpoint_context::AdapterCheckpointContext; +use crate::checkpoint_context::{ + AdapterCheckpointContext, StartupAdapterProgress, record_startup_adapter_progress, +}; use anyhow::ensure; mod apply; @@ -32,6 +34,7 @@ pub(super) async fn sync_ens_v1_unwrapped_authority_with_scope( replay_checkpoint: Option<&AdapterCheckpointContext>, replay_max_raw_logs_per_page: Option, resolver_profile_replay: Option<&mut ResolverProfileReplayContext>, + mut startup_progress: Option<&mut dyn StartupAdapterProgress>, ) -> Result { let source_scope = source_scope.map(normalized_authority_source_scope_targets); let total_started = Instant::now(); @@ -324,6 +327,7 @@ pub(super) async fn sync_ens_v1_unwrapped_authority_with_scope( &flushed_events, ) .await?; + record_startup_adapter_progress(pool, &mut startup_progress).await?; tracing::info!( service = "adapters", adapter = DERIVATION_KIND_ENS_V1_UNWRAPPED_AUTHORITY, @@ -369,6 +373,7 @@ pub(super) async fn sync_ens_v1_unwrapped_authority_with_scope( checkpoint .mark_stream_complete(pool, total_scanned_log_count, matched_log_count) .await?; + record_startup_adapter_progress(pool, &mut startup_progress).await?; } scanned_log_count = total_scanned_log_count; apply_ms = stream_apply_started.elapsed().as_millis(); @@ -533,6 +538,7 @@ pub(super) async fn sync_ens_v1_unwrapped_authority_with_scope( reverse_histories, flushed_events, active_replay_checkpoint: &mut active_replay_checkpoint, + startup_progress, pre_timings: PreMaterializationTimings { active_emitters_ms, raw_log_load_ms, diff --git a/crates/adapters/src/ens_v1_unwrapped_authority/pipeline/entrypoints.rs b/crates/adapters/src/ens_v1_unwrapped_authority/pipeline/entrypoints.rs index 8f72c59f..e1b9b162 100644 --- a/crates/adapters/src/ens_v1_unwrapped_authority/pipeline/entrypoints.rs +++ b/crates/adapters/src/ens_v1_unwrapped_authority/pipeline/entrypoints.rs @@ -1,6 +1,7 @@ use super::*; use crate::checkpoint_context::{ AdapterCheckpointContext, ReplayAdapterCheckpointContext, StartupAdapterCheckpointContext, + StartupAdapterProgress, }; pub async fn sync_ens_v1_unwrapped_authority( @@ -17,6 +18,7 @@ pub async fn sync_ens_v1_unwrapped_authority( None, None, None, + None, ) .await } @@ -32,6 +34,7 @@ pub async fn sync_ens_v1_unwrapped_authority_with_replay_checkpoint_and_log_limi chain, &AdapterCheckpointContext::for_replay(checkpoint), max_raw_logs_per_page, + None, ) .await } @@ -48,6 +51,25 @@ pub async fn sync_ens_v1_unwrapped_authority_with_startup_checkpoint_and_log_lim chain, &checkpoint, max_raw_logs_per_page, + None, + ) + .await +} + +pub async fn sync_ens_v1_unwrapped_authority_with_startup_checkpoint_and_log_limit_and_progress( + pool: &PgPool, + chain: &str, + checkpoint: &StartupAdapterCheckpointContext, + max_raw_logs_per_page: usize, + progress: &mut dyn StartupAdapterProgress, +) -> Result { + let checkpoint = checkpoint.adapter_context(pool, chain).await?; + sync_ens_v1_unwrapped_authority_with_checkpoint_context( + pool, + chain, + &checkpoint, + max_raw_logs_per_page, + Some(progress), ) .await } @@ -57,6 +79,7 @@ async fn sync_ens_v1_unwrapped_authority_with_checkpoint_context( chain: &str, checkpoint: &AdapterCheckpointContext, max_raw_logs_per_page: usize, + startup_progress: Option<&mut dyn StartupAdapterProgress>, ) -> Result { let raw_log_guard = acquire_raw_log_staging_read_guard(pool, chain).await?; let summary = sync_ens_v1_unwrapped_authority_with_scope( @@ -69,6 +92,7 @@ async fn sync_ens_v1_unwrapped_authority_with_checkpoint_context( Some(checkpoint), Some(max_raw_logs_per_page), None, + startup_progress, ) .await?; raw_log_guard.release().await?; @@ -91,6 +115,7 @@ impl EnsV1UnwrappedAuthoritySyncSummary { None, None, None, + None, ) .await } @@ -111,6 +136,7 @@ impl EnsV1UnwrappedAuthoritySyncSummary { None, None, None, + None, ) .await } @@ -132,6 +158,7 @@ impl EnsV1UnwrappedAuthoritySyncSummary { None, None, None, + None, ) .await } diff --git a/crates/adapters/src/ens_v1_unwrapped_authority/pipeline/finalize.rs b/crates/adapters/src/ens_v1_unwrapped_authority/pipeline/finalize.rs index d2f1bf30..d6ef2932 100644 --- a/crates/adapters/src/ens_v1_unwrapped_authority/pipeline/finalize.rs +++ b/crates/adapters/src/ens_v1_unwrapped_authority/pipeline/finalize.rs @@ -20,7 +20,7 @@ pub(super) struct PreMaterializationTimings { pub(super) apply_ms: u128, } -pub(super) struct FinalizeAuthoritySync<'a> { +pub(super) struct FinalizeAuthoritySync<'a, 'progress> { pub(super) pool: &'a PgPool, pub(super) chain: &'a str, pub(super) restrict_to_block_hashes: bool, @@ -36,12 +36,13 @@ pub(super) struct FinalizeAuthoritySync<'a> { pub(super) reverse_histories: BTreeMap, pub(super) flushed_events: UnwrappedAuthorityReplayFlushedEvents, pub(super) active_replay_checkpoint: &'a mut Option, + pub(super) startup_progress: Option<&'progress mut dyn StartupAdapterProgress>, pub(super) pre_timings: PreMaterializationTimings, pub(super) total_started: Instant, } pub(super) async fn finalize_authority_sync( - mut input: FinalizeAuthoritySync<'_>, + mut input: FinalizeAuthoritySync<'_, '_>, ) -> Result { let head_block = input .block_index @@ -84,27 +85,49 @@ pub(super) async fn finalize_authority_sync( &head_ref, input.histories, input.reverse_histories, + &mut input.startup_progress, ) .await?; + record_startup_adapter_progress(input.pool, &mut input.startup_progress).await?; let materialization_ms = materialization_started.elapsed().as_millis(); let normalize_started = Instant::now(); let mut by_kind = input.flushed_events.by_kind.clone(); merge_event_kind_counts(&mut by_kind, count_events_by_kind(&events)); normalize_surface_bindings_for_upsert(&mut bindings)?; + record_startup_adapter_progress(input.pool, &mut input.startup_progress).await?; let normalize_ms = normalize_started.elapsed().as_millis(); let closure_started = Instant::now(); - let closure_count = prepend_existing_open_binding_closures(input.pool, &mut bindings).await?; + let closure_count = prepend_existing_open_binding_closures_with_progress( + input.pool, + &mut bindings, + &mut input.startup_progress, + ) + .await?; let closure_ms = closure_started.elapsed().as_millis(); let binding_closures_started = Instant::now(); if closure_count > 0 { - upsert_surface_bindings_without_snapshots(input.pool, &bindings[..closure_count]).await?; + upsert_binding_batches( + input.pool, + &bindings[..closure_count], + &mut input.startup_progress, + ) + .await?; } let binding_closures_upsert_ms = binding_closures_started.elapsed().as_millis(); - let (binding_overlap_repair_count, binding_overlap_repair_ms) = - close_binding_overlaps(input.pool, &bindings[closure_count..]).await?; + let (binding_overlap_repair_count, binding_overlap_repair_ms) = close_binding_overlaps( + input.pool, + &bindings[closure_count..], + &mut input.startup_progress, + ) + .await?; let bindings_started = Instant::now(); - upsert_surface_bindings_without_snapshots(input.pool, &bindings[closure_count..]).await?; + upsert_binding_batches( + input.pool, + &bindings[closure_count..], + &mut input.startup_progress, + ) + .await?; let bindings_upsert_ms = bindings_started.elapsed().as_millis(); let binding_count = bindings.len(); drop(bindings); @@ -114,14 +137,17 @@ pub(super) async fn finalize_authority_sync( .filter(|checkpoint| checkpoint.is_startup()) { checkpoint - .publish_startup_events(input.pool, &mut input.flushed_events) + .publish_startup_events( + input.pool, + &mut input.flushed_events, + &mut input.startup_progress, + ) .await?; } let normalized_events_started = Instant::now(); let normalized_event_count = events.len(); let event_inserted_count = - event_persistence::upsert_events_preserving_manifest_provenance(input.pool, &mut events) - .await?; + upsert_event_batches(input.pool, &mut events, &mut input.startup_progress).await?; let normalized_events_upsert_ms = normalized_events_started.elapsed().as_millis(); drop(events); @@ -189,6 +215,80 @@ pub(super) async fn finalize_authority_sync( ); if let Some(checkpoint) = input.active_replay_checkpoint.as_mut() { checkpoint.mark_completed(input.pool, &summary).await?; + record_startup_adapter_progress(input.pool, &mut input.startup_progress).await?; } Ok(summary) } + +const FINALIZATION_BINDING_NAME_BATCH_SIZE: usize = 1_000; +const FINALIZATION_EVENT_BATCH_SIZE: usize = 5_000; + +async fn prepend_existing_open_binding_closures_with_progress( + pool: &PgPool, + bindings: &mut Vec, + startup_progress: &mut Option<&mut dyn StartupAdapterProgress>, +) -> Result { + if startup_progress.is_none() { + return prepend_existing_open_binding_closures(pool, bindings).await; + } + + let mut source = std::mem::take(bindings).into_iter().peekable(); + let mut closures = Vec::new(); + let mut incoming = Vec::new(); + while source.peek().is_some() { + let mut batch = Vec::new(); + for _ in 0..FINALIZATION_BINDING_NAME_BATCH_SIZE { + let Some(logical_name_id) = + source.peek().map(|binding| binding.logical_name_id.clone()) + else { + break; + }; + while source + .peek() + .is_some_and(|binding| binding.logical_name_id == logical_name_id) + { + batch.push(source.next().expect("peeked binding must remain available")); + } + } + let closure_count = prepend_existing_open_binding_closures(pool, &mut batch).await?; + incoming.extend(batch.drain(closure_count..)); + closures.extend(batch); + record_startup_adapter_progress(pool, startup_progress).await?; + } + let closure_count = closures.len(); + closures.append(&mut incoming); + *bindings = closures; + Ok(closure_count) +} + +async fn upsert_binding_batches( + pool: &PgPool, + bindings: &[SurfaceBinding], + startup_progress: &mut Option<&mut dyn StartupAdapterProgress>, +) -> Result<()> { + if startup_progress.is_none() { + return upsert_surface_bindings_without_snapshots(pool, bindings).await; + } + for chunk in bindings.chunks(FINALIZATION_BINDING_NAME_BATCH_SIZE) { + upsert_surface_bindings_without_snapshots(pool, chunk).await?; + record_startup_adapter_progress(pool, startup_progress).await?; + } + Ok(()) +} + +async fn upsert_event_batches( + pool: &PgPool, + events: &mut [NormalizedEvent], + startup_progress: &mut Option<&mut dyn StartupAdapterProgress>, +) -> Result { + if startup_progress.is_none() { + return event_persistence::upsert_events_preserving_manifest_provenance(pool, events).await; + } + let mut inserted_count = 0usize; + for chunk in events.chunks_mut(FINALIZATION_EVENT_BATCH_SIZE) { + inserted_count += + event_persistence::upsert_events_preserving_manifest_provenance(pool, chunk).await?; + record_startup_adapter_progress(pool, startup_progress).await?; + } + Ok(inserted_count) +} diff --git a/crates/adapters/src/ens_v1_unwrapped_authority/pipeline/identity.rs b/crates/adapters/src/ens_v1_unwrapped_authority/pipeline/identity.rs index 91833ec3..abf3aed5 100644 --- a/crates/adapters/src/ens_v1_unwrapped_authority/pipeline/identity.rs +++ b/crates/adapters/src/ens_v1_unwrapped_authority/pipeline/identity.rs @@ -93,8 +93,39 @@ pub(super) async fn ensure_binding_authority_identity_rows( pub(super) async fn close_binding_overlaps( pool: &PgPool, bindings: &[SurfaceBinding], + startup_progress: &mut Option<&mut dyn StartupAdapterProgress>, ) -> Result<(usize, u128)> { let started = Instant::now(); - let count = close_weaker_overlapping_existing_surface_bindings(pool, bindings).await?; + if startup_progress.is_none() { + let count = close_weaker_overlapping_existing_surface_bindings(pool, bindings).await?; + return Ok((count, started.elapsed().as_millis())); + } + + let mut count = 0usize; + let mut batch_start = 0usize; + while batch_start < bindings.len() { + let mut batch_end = batch_start; + for _ in 0..1_000 { + let Some(logical_name_id) = bindings + .get(batch_end) + .map(|binding| binding.logical_name_id.as_str()) + else { + break; + }; + while bindings + .get(batch_end) + .is_some_and(|binding| binding.logical_name_id == logical_name_id) + { + batch_end += 1; + } + } + count += close_weaker_overlapping_existing_surface_bindings( + pool, + &bindings[batch_start..batch_end], + ) + .await?; + record_startup_adapter_progress(pool, startup_progress).await?; + batch_start = batch_end; + } Ok((count, started.elapsed().as_millis())) } diff --git a/crates/adapters/src/ens_v1_unwrapped_authority/pipeline/materialize.rs b/crates/adapters/src/ens_v1_unwrapped_authority/pipeline/materialize.rs index 67252bc8..0c8a7d8e 100644 --- a/crates/adapters/src/ens_v1_unwrapped_authority/pipeline/materialize.rs +++ b/crates/adapters/src/ens_v1_unwrapped_authority/pipeline/materialize.rs @@ -12,6 +12,7 @@ pub(super) struct AuthorityMaterialization { } const IDENTITY_MATERIALIZATION_FLUSH_BATCH_SIZE: usize = 10_000; +const MATERIALIZATION_PROGRESS_HISTORY_INTERVAL: usize = 100; pub(super) struct AuthorityIdentityBuffers { token_lineages: Vec, @@ -67,23 +68,32 @@ impl AuthorityIdentityBuffers { } } - async fn flush_if_needed(&mut self, pool: &PgPool) -> Result<()> { + async fn flush_if_needed( + &mut self, + pool: &PgPool, + startup_progress: &mut Option<&mut dyn StartupAdapterProgress>, + ) -> Result<()> { if self.token_lineages.len() >= IDENTITY_MATERIALIZATION_FLUSH_BATCH_SIZE || self.resources.len() >= IDENTITY_MATERIALIZATION_FLUSH_BATCH_SIZE || self.surfaces.len() >= IDENTITY_MATERIALIZATION_FLUSH_BATCH_SIZE { - self.flush(pool).await?; + self.flush(pool, startup_progress).await?; } Ok(()) } - async fn flush(&mut self, pool: &PgPool) -> Result<()> { + async fn flush( + &mut self, + pool: &PgPool, + startup_progress: &mut Option<&mut dyn StartupAdapterProgress>, + ) -> Result<()> { if !self.token_lineages.is_empty() { let started = Instant::now(); upsert_token_lineages_without_snapshots(pool, &self.token_lineages).await?; self.token_lineages_upsert_ms += started.elapsed().as_millis(); self.token_lineage_count += self.token_lineages.len(); self.token_lineages.clear(); + record_startup_adapter_progress(pool, startup_progress).await?; } if !self.resources.is_empty() { let started = Instant::now(); @@ -91,6 +101,7 @@ impl AuthorityIdentityBuffers { self.resources_upsert_ms += started.elapsed().as_millis(); self.resource_count += self.resources.len(); self.resources.clear(); + record_startup_adapter_progress(pool, startup_progress).await?; } if !self.surfaces.is_empty() { let started = Instant::now(); @@ -98,6 +109,7 @@ impl AuthorityIdentityBuffers { self.surfaces_upsert_ms += started.elapsed().as_millis(); self.surface_count += self.surfaces.len(); self.surfaces.clear(); + record_startup_adapter_progress(pool, startup_progress).await?; } Ok(()) } @@ -109,12 +121,18 @@ pub(super) async fn materialize_authority_histories( head_ref: &BoundaryRef, histories: BTreeMap, reverse_histories: BTreeMap, + startup_progress: &mut Option<&mut dyn StartupAdapterProgress>, ) -> Result { let mut identity = AuthorityIdentityBuffers::new(); let mut bindings = Vec::::new(); let mut events = Vec::::new(); - for history in histories.into_values() { + for (history_index, history) in histories.into_values().enumerate() { + if history_index > 0 + && history_index.is_multiple_of(MATERIALIZATION_PROGRESS_HISTORY_INTERVAL) + { + record_startup_adapter_progress(pool, startup_progress).await?; + } let Some(name) = history.name.clone() else { continue; }; @@ -260,12 +278,19 @@ pub(super) async fn materialize_authority_histories( ); } events.extend(finalized.events); - identity.flush_if_needed(pool).await?; + identity.flush_if_needed(pool, startup_progress).await?; } - for history in reverse_histories.into_values() { + record_startup_adapter_progress(pool, startup_progress).await?; + for (history_index, history) in reverse_histories.into_values().enumerate() { + if history_index > 0 + && history_index.is_multiple_of(MATERIALIZATION_PROGRESS_HISTORY_INTERVAL) + { + record_startup_adapter_progress(pool, startup_progress).await?; + } events.extend(history.events); } - identity.flush(pool).await?; + record_startup_adapter_progress(pool, startup_progress).await?; + identity.flush(pool, startup_progress).await?; Ok(AuthorityMaterialization { token_lineage_count: identity.token_lineage_count, diff --git a/crates/adapters/src/ens_v1_unwrapped_authority/resolver_profile_reconciliation.rs b/crates/adapters/src/ens_v1_unwrapped_authority/resolver_profile_reconciliation.rs index c65fdf65..a1c41580 100644 --- a/crates/adapters/src/ens_v1_unwrapped_authority/resolver_profile_reconciliation.rs +++ b/crates/adapters/src/ens_v1_unwrapped_authority/resolver_profile_reconciliation.rs @@ -171,6 +171,7 @@ impl ResolverProfileEventReconciliation { None, None, Some(&mut replay_context), + None, ) .await?; summary.scanned_log_count = replay.scanned_log_count; diff --git a/crates/adapters/src/ens_v1_unwrapped_authority/tests.rs b/crates/adapters/src/ens_v1_unwrapped_authority/tests.rs index 3efa18b8..a5e1d3e9 100644 --- a/crates/adapters/src/ens_v1_unwrapped_authority/tests.rs +++ b/crates/adapters/src/ens_v1_unwrapped_authority/tests.rs @@ -21,6 +21,18 @@ use super::*; static NEXT_TEST_ID: AtomicU64 = AtomicU64::new(0); const BASE_NATIVE_COIN_TYPE: &str = "2147492101"; +#[derive(Default)] +struct CountingStartupAdapterProgress { + record_count: usize, +} + +impl crate::StartupAdapterProgress for CountingStartupAdapterProgress { + fn record<'a>(&'a mut self, _pool: &'a PgPool) -> crate::StartupAdapterProgressFuture<'a> { + self.record_count += 1; + Box::pin(async { Ok(()) }) + } +} + struct TestDatabase { admin_pool: PgPool, pool: PgPool, @@ -4522,13 +4534,20 @@ async fn sync_ens_v1_unwrapped_authority_persists_registrar_identity_rows_idempo assert_eq!(second.total_normalized_event_count, 5); let startup_checkpoint = crate::StartupAdapterCheckpointContext::new("test-startup", 42)?; - let startup = sync_ens_v1_unwrapped_authority_with_startup_checkpoint_and_log_limit( - database.pool(), - "ethereum-mainnet", - &startup_checkpoint, - 100_000, - ) - .await?; + let mut startup_progress = CountingStartupAdapterProgress::default(); + let startup = + sync_ens_v1_unwrapped_authority_with_startup_checkpoint_and_log_limit_and_progress( + database.pool(), + "ethereum-mainnet", + &startup_checkpoint, + 100_000, + &mut startup_progress, + ) + .await?; + assert!( + startup_progress.record_count > 2, + "the checkpoint stream and authority materialization must both report startup progress" + ); assert_eq!(startup.scanned_log_count, second.scanned_log_count); assert_eq!(startup.matched_log_count, second.matched_log_count); assert_eq!( diff --git a/crates/adapters/src/lib.rs b/crates/adapters/src/lib.rs index dad30c63..dfa7f113 100644 --- a/crates/adapters/src/lib.rs +++ b/crates/adapters/src/lib.rs @@ -26,8 +26,8 @@ pub use block_derived_normalized_events::{ sync_block_derived_normalized_events_with_scanned_log_count, }; pub use checkpoint_context::{ - ReplayAdapterCheckpointContext, StartupAdapterCheckpointContext, - clear_startup_adapter_checkpoints, + ReplayAdapterCheckpointContext, StartupAdapterCheckpointContext, StartupAdapterProgress, + StartupAdapterProgressFuture, clear_startup_adapter_checkpoints, }; pub use ens_v1_reverse_claim::{ EnsV1ReverseClaimKindSyncSummary, EnsV1ReverseClaimSyncSummary, sync_ens_v1_reverse_claim, @@ -40,6 +40,7 @@ pub use ens_v1_subregistry_discovery::{ sync_ens_v1_subregistry_discovery_with_replay_checkpoint, sync_ens_v1_subregistry_discovery_with_replay_checkpoint_and_log_limit, sync_ens_v1_subregistry_discovery_with_startup_checkpoint_and_log_limit, + sync_ens_v1_subregistry_discovery_with_startup_checkpoint_and_log_limit_and_progress, }; pub use ens_v1_unwrapped_authority::{ EnsV1TextRecordChange, EnsV1UnwrappedAuthoritySyncSummary, ResolverProfileEventReconciliation, @@ -48,6 +49,7 @@ pub use ens_v1_unwrapped_authority::{ reconcile_resolver_profile_events, sync_ens_v1_unwrapped_authority, sync_ens_v1_unwrapped_authority_with_replay_checkpoint_and_log_limit, sync_ens_v1_unwrapped_authority_with_startup_checkpoint_and_log_limit, + sync_ens_v1_unwrapped_authority_with_startup_checkpoint_and_log_limit_and_progress, }; pub use ens_v2_permissions::{ EnsV2PermissionsKindSyncSummary, EnsV2PermissionsSyncSummary, sync_ens_v2_permissions, diff --git a/crates/execution/src/lib.rs b/crates/execution/src/lib.rs index 5ce2519e..b2adba60 100644 --- a/crates/execution/src/lib.rs +++ b/crates/execution/src/lib.rs @@ -49,7 +49,7 @@ pub use persistence::{ persist_basenames_exact_name_verified_resolution_transport_direct, persist_ens_exact_name_verified_resolution_direct, persist_ens_verified_primary_name, }; -pub use rpc::ChainRpcUrls; +pub use rpc::{ChainRpcUrls, fetch_network_head_block_number}; pub const VERIFIED_RESOLUTION_REQUEST_TYPE: &str = "verified_resolution"; pub const VERIFIED_PRIMARY_NAME_REQUEST_TYPE: &str = "verified_primary_name"; diff --git a/crates/execution/src/rpc.rs b/crates/execution/src/rpc.rs index 02214758..aba31655 100644 --- a/crates/execution/src/rpc.rs +++ b/crates/execution/src/rpc.rs @@ -125,6 +125,40 @@ impl ChainRpcUrls { }); Ok(self) } + + pub fn iter(&self) -> impl Iterator { + self.urls + .iter() + .map(|(chain_id, url)| (chain_id.as_str(), url.as_str())) + } +} + +pub async fn fetch_network_head_block_number(endpoint: &str) -> Result { + let response = JsonRpcHttpClient::new(endpoint)? + .call("eth_blockNumber", Vec::new()) + .await?; + let result = response.result.map_err(|error| { + anyhow::anyhow!( + "eth_blockNumber failed with provider code {:?}: {}", + error.code, + error.message + ) + })?; + parse_quantity_i64(&result).context("eth_blockNumber returned an invalid block quantity") +} + +fn parse_quantity_i64(value: &Value) -> Result { + let quantity = value + .as_str() + .context("JSON-RPC quantity must be a string")?; + let digits = quantity + .strip_prefix("0x") + .context("JSON-RPC quantity must start with 0x")?; + if digits.is_empty() { + bail!("JSON-RPC quantity must contain hexadecimal digits"); + } + let value = u64::from_str_radix(digits, 16).context("JSON-RPC quantity is not hexadecimal")?; + i64::try_from(value).context("JSON-RPC quantity exceeds the supported signed block range") } #[derive(Clone)] @@ -329,4 +363,13 @@ mod tests { server.await??; Ok(()) } + + #[test] + fn network_head_quantity_parser_is_strict_and_bounded() -> Result<()> { + assert_eq!(parse_quantity_i64(&Value::String("0x2a".to_owned()))?, 42); + assert!(parse_quantity_i64(&Value::String("2a".to_owned())).is_err()); + assert!(parse_quantity_i64(&Value::String("0x".to_owned())).is_err()); + assert!(parse_quantity_i64(&Value::String("0x8000000000000000".to_owned())).is_err()); + Ok(()) + } } diff --git a/crates/manifests/src/lib/discovery/reconciliation/streamed.rs b/crates/manifests/src/lib/discovery/reconciliation/streamed.rs index bd0ca20f..a2f66d0b 100644 --- a/crates/manifests/src/lib/discovery/reconciliation/streamed.rs +++ b/crates/manifests/src/lib/discovery/reconciliation/streamed.rs @@ -82,6 +82,13 @@ pub trait DiscoveryObservationPageSource { after_key: Option<&str>, limit: i64, ) -> impl Future>> + Send; + + /// Report a completed bounded reconciliation unit. The default is a + /// no-op; operational callers can use it to keep liveness tied to actual + /// streamed progress without a detached timer. + fn record_progress(&self) -> impl Future> + Send { + async { Ok(()) } + } } #[derive(Clone, Copy, Debug)] @@ -185,12 +192,14 @@ pub(crate) async fn reconcile_discovery_observations_streamed_with_options( Some(discovery_source), ) .await?; - run_streamed_admission_walk(transaction.as_mut(), &admission_state, &options).await?; + source.record_progress().await?; + run_streamed_admission_walk(transaction.as_mut(), &admission_state, &options, source).await?; // Everything below diffs against the pre-mutation edge snapshot, exactly // like the in-memory reconcile computes its whole plan from one // `load_active_reconciled_discovery_edges` read before mutating. materialize_streamed_insert_candidates(transaction.as_mut(), discovery_source).await?; + source.record_progress().await?; let insert_candidate_count = count_temp_rows(transaction.as_mut(), "reconcile_insert_candidates").await?; let deactivation_candidate_count = @@ -233,6 +242,7 @@ pub(crate) async fn reconcile_discovery_observations_streamed_with_options( load_streamed_deactivation_candidates(transaction.as_mut(), discovery_source).await?; let candidate_observations = load_streamed_observations_for_keys(transaction.as_mut(), &deactivation_candidates).await?; + source.record_progress().await?; let direct_terminal_states_by_key = observation_terminal_states(&candidate_observations)?; let observations_by_key = candidate_observations .iter() @@ -270,6 +280,7 @@ pub(crate) async fn reconcile_discovery_observations_streamed_with_options( &mut retained_newer_edge_ids, ) .await?; + source.record_progress().await?; // Chronology rule 2: a desired edge with a newer non-orphaned successor // for the same assignment start is materialized as a closed historical // epoch and the successor is retained. @@ -281,6 +292,7 @@ pub(crate) async fn reconcile_discovery_observations_streamed_with_options( &mut retained_newer_edge_ids, ) .await?; + source.record_progress().await?; let historical_row_ids = historical_edges .iter() .map(|(desired_row_id, _, _)| *desired_row_id) @@ -347,7 +359,11 @@ pub(crate) async fn reconcile_discovery_observations_streamed_with_options( let mut deactivated_edge_count = 0; let mut mutated_chains = BTreeSet::new(); - for candidate in &deactivation_candidates { + for (candidate_index, candidate) in deactivation_candidates.iter().enumerate() { + if candidate_index > 0 && candidate_index.is_multiple_of(options.mutation_batch_size.max(1)) + { + source.record_progress().await?; + } if retained_newer_edge_ids.contains(&candidate.discovery_edge_id) { continue; } @@ -370,6 +386,7 @@ pub(crate) async fn reconcile_discovery_observations_streamed_with_options( deactivated_edge_count += 1; } } + source.record_progress().await?; let mut inserted_edge_count = 0; let mut after_row_id = 0i64; @@ -396,6 +413,7 @@ pub(crate) async fn reconcile_discovery_observations_streamed_with_options( let edge_insert = insert_reconciled_discovery_edges(transaction.as_mut(), &batch).await?; inserted_edge_count += edge_insert.inserted_count + edge_insert.reactivated_count; mutated_chains.extend(batch.iter().map(|edge| edge.chain.clone())); + source.record_progress().await?; } let mut historical_edges = historical_edges @@ -410,6 +428,7 @@ pub(crate) async fn reconcile_discovery_observations_streamed_with_options( .collect::>(); let historical_edge_reconciliation = reconcile_historical_discovery_edges(transaction.as_mut(), &historical_edge_refs).await?; + source.record_progress().await?; inserted_edge_count += historical_edge_reconciliation.inserted_count; if historical_edge_reconciliation.inserted_count > 0 || historical_edge_reconciliation.updated_count > 0 @@ -422,6 +441,7 @@ pub(crate) async fn reconcile_discovery_observations_streamed_with_options( || deactivated_edge_count > 0 { reconcile_active_contract_instance_addresses(transaction.as_mut()).await?; + source.record_progress().await?; } let active_edge_count = load_active_reconciled_discovery_edge_count(transaction.as_mut(), discovery_source).await?; @@ -430,11 +450,13 @@ pub(crate) async fn reconcile_discovery_observations_streamed_with_options( .context("failed to count streamed admitted discovery edges")?; let admission_epoch_bump_count = mutated_chains.len(); bump_discovery_admission_epochs(transaction.as_mut(), &mutated_chains).await?; + source.record_progress().await?; transaction .commit() .await .context("failed to commit streamed discovery-edge reconciliation transaction")?; + source.record_progress().await?; Ok(DiscoveryReconciliationSummary { active_edge_count, diff --git a/crates/manifests/src/lib/discovery/reconciliation/streamed/staging.rs b/crates/manifests/src/lib/discovery/reconciliation/streamed/staging.rs index 25349dd4..0ebd5f69 100644 --- a/crates/manifests/src/lib/discovery/reconciliation/streamed/staging.rs +++ b/crates/manifests/src/lib/discovery/reconciliation/streamed/staging.rs @@ -205,6 +205,7 @@ pub(super) async fn stage_streamed_observations( "failed to stage streamed discovery observations (the page source must yield \ latest-per-key observations with unique observation keys)", )?; + source.record_progress().await?; } } @@ -218,6 +219,7 @@ pub(super) async fn stage_streamed_observations( .await .context("failed to index the streamed reconcile observation temp table")?; analyze_temp_table(&mut *executor, "reconcile_observations").await?; + source.record_progress().await?; Ok(StagedStreamedObservations { staged_observation_count, diff --git a/crates/manifests/src/lib/discovery/reconciliation/streamed/walk_pages.rs b/crates/manifests/src/lib/discovery/reconciliation/streamed/walk_pages.rs index 72d3aafd..890b618f 100644 --- a/crates/manifests/src/lib/discovery/reconciliation/streamed/walk_pages.rs +++ b/crates/manifests/src/lib/discovery/reconciliation/streamed/walk_pages.rs @@ -14,11 +14,11 @@ use super::super::super::provenance::is_zero_address; use super::super::super::types::{AdmittedDiscoveryEdge, ReconciledDiscoveryEdgeSpec}; use super::super::bulk::insert_pending_contract_instance_seeds; use super::super::walk::DiscoveryAdmissionWalk; -use super::StreamedDiscoveryReconciliationOptions; use super::staging::{ STREAMED_OBSERVATION_COLUMNS, STREAMED_OBSERVATION_COLUMNS_QUALIFIED, StreamedObservationRow, analyze_temp_table, stage_streamed_derived_contract_keys, streamed_observation_from_row, }; +use super::{DiscoveryObservationPageSource, StreamedDiscoveryReconciliationOptions}; use crate::normalize_address; /// Fixed-point admission walk over the staged observations. Pass 1 pages the @@ -39,6 +39,7 @@ pub(super) async fn run_streamed_admission_walk( executor: &mut PgConnection, admission_state: &DiscoveryAdmissionState, options: &StreamedDiscoveryReconciliationOptions, + progress: &impl DiscoveryObservationPageSource, ) -> Result<()> { let mut walk = DiscoveryAdmissionWalk::new(admission_state); let mut desired_buffer = Vec::::new(); @@ -78,8 +79,10 @@ pub(super) async fn run_streamed_admission_walk( &mut admitted_buffer, &mut pending_derived_keys, options, + progress, ) .await?; + progress.record_progress().await?; } let derived_observation_page_query = @@ -111,20 +114,26 @@ pub(super) async fn run_streamed_admission_walk( &mut admitted_buffer, &mut pending_derived_keys, options, + progress, ) .await?; + progress.record_progress().await?; } } flush_desired_edge_buffer(&mut *executor, &mut desired_buffer).await?; + progress.record_progress().await?; flush_admitted_edge_buffer(&mut *executor, &mut admitted_buffer).await?; + progress.record_progress().await?; insert_pending_contract_instance_seeds( &mut *executor, &walk.into_sorted_pending_contract_instance_seeds(), ) .await?; + progress.record_progress().await?; analyze_temp_table(&mut *executor, "reconcile_desired_edges").await?; analyze_temp_table(&mut *executor, "reconcile_admitted_edges").await?; + progress.record_progress().await?; Ok(()) } @@ -153,6 +162,7 @@ async fn admit_streamed_observation_page( admitted_buffer: &mut Vec, pending_derived_keys: &mut BTreeSet<(String, String)>, options: &StreamedDiscoveryReconciliationOptions, + progress: &impl DiscoveryObservationPageSource, ) -> Result<()> { // Resolve the page's target addresses through the same query and // first-row-wins fold the full known-address load uses, scoped to one @@ -193,9 +203,11 @@ async fn admit_streamed_observation_page( } if desired_buffer.len() >= options.mutation_batch_size { flush_desired_edge_buffer(&mut *executor, desired_buffer).await?; + progress.record_progress().await?; } if admitted_buffer.len() >= options.mutation_batch_size { flush_admitted_edge_buffer(&mut *executor, admitted_buffer).await?; + progress.record_progress().await?; } } Ok(()) diff --git a/crates/storage/src/identity_facade/mod.rs b/crates/storage/src/identity_facade/mod.rs index 4ea48c59..3d699253 100644 --- a/crates/storage/src/identity_facade/mod.rs +++ b/crates/storage/src/identity_facade/mod.rs @@ -12,7 +12,9 @@ use std::collections::BTreeSet; pub use forward::{load_identity_name_feed_records_by_names, load_identity_records_by_names}; pub use reverse::load_reverse_identity_records; pub use reverse_feed::load_reverse_identity_feed_records; -pub use status::load_indexing_status; +pub use status::{ + PENDING_INVALIDATION_COUNT_CAP, load_expected_status_chain_ids, load_indexing_status, +}; pub use types::{ IdentityAddressRelationRow, IdentityNameCurrentRow, IdentityNameRecordRow, IdentityPrimaryNameSnapshot, IdentityRecordInventoryRow, IndexingStatusChainRow, diff --git a/crates/storage/src/identity_facade/status.rs b/crates/storage/src/identity_facade/status.rs index 8478354d..be3e23bb 100644 --- a/crates/storage/src/identity_facade/status.rs +++ b/crates/storage/src/identity_facade/status.rs @@ -3,6 +3,32 @@ use sqlx::PgPool; use super::{IndexingStatusChainRow, IndexingStatusRead}; +pub const PENDING_INVALIDATION_COUNT_CAP: i64 = 10_000; + +pub async fn load_expected_status_chain_ids(pool: &PgPool) -> Result> { + sqlx::query_scalar( + r#" + SELECT chain_id + FROM ( + SELECT chain_id + FROM chain_checkpoints + UNION + SELECT chain AS chain_id + FROM manifest_versions + WHERE chain IS NOT NULL + AND rollout_status IN ( + 'active'::manifest_rollout_status, + 'shadow'::manifest_rollout_status + ) + ) AS known_chains + ORDER BY chain_id + "#, + ) + .fetch_all(pool) + .await + .context("failed to load expected indexing status chains") +} + pub async fn load_indexing_status(pool: &PgPool) -> Result { let rows = sqlx::query( r#" @@ -155,8 +181,9 @@ pub async fn load_indexing_status(pool: &PgPool) -> Result { }) .collect::>>()?; - let has_unscoped_pending_invalidations = sqlx::query_scalar::<_, bool>( - r#" + let (has_unscoped_pending_invalidations, pending_invalidation_count, dead_letter_count) = + sqlx::query_as::<_, (bool, i64, i64)>( + r#" WITH apply_cursor AS ( SELECT COALESCE(( SELECT last_change_id @@ -170,7 +197,8 @@ pub async fn load_indexing_status(pool: &PgPool) -> Result { CROSS JOIN apply_cursor WHERE change.change_id > apply_cursor.last_change_id ) - SELECT EXISTS ( + SELECT ( + EXISTS ( SELECT 1 FROM projection_invalidations invalidation LEFT JOIN normalized_events event @@ -181,8 +209,8 @@ pub async fn load_indexing_status(pool: &PgPool) -> Result { WHERE event.normalized_event_id IS NULL OR event.chain_id IS NULL OR event.block_number IS NULL - ) - OR EXISTS ( + ) + OR EXISTS ( SELECT 1 FROM unscanned_changes change CROSS JOIN LATERAL ( @@ -193,15 +221,34 @@ pub async fn load_indexing_status(pool: &PgPool) -> Result { ) event WHERE event.chain_id IS NULL OR event.block_number IS NULL - ) + ) + ) AS has_unscoped_pending_invalidations, + ( + SELECT COUNT(*)::BIGINT + FROM ( + SELECT 1 + FROM projection_invalidations + LIMIT $1 + 1 + ) AS bounded_pending_invalidations + ) AS observed_pending_invalidation_count, + ( + SELECT COUNT(*)::BIGINT + FROM projection_invalidation_dead_letters + ) AS dead_letter_count "#, - ) - .fetch_one(pool) - .await - .context("failed to load unscoped indexing invalidation status")?; + ) + .bind(PENDING_INVALIDATION_COUNT_CAP) + .fetch_one(pool) + .await + .context("failed to load unscoped indexing invalidation status")?; + let pending_invalidation_count_capped = + pending_invalidation_count > PENDING_INVALIDATION_COUNT_CAP; Ok(IndexingStatusRead { chains, has_unscoped_pending_invalidations, + pending_invalidation_count: pending_invalidation_count.min(PENDING_INVALIDATION_COUNT_CAP), + pending_invalidation_count_capped, + dead_letter_count, }) } diff --git a/crates/storage/src/identity_facade/types.rs b/crates/storage/src/identity_facade/types.rs index 72ee7610..9f43b10b 100644 --- a/crates/storage/src/identity_facade/types.rs +++ b/crates/storage/src/identity_facade/types.rs @@ -146,6 +146,9 @@ pub struct IdentityPrimaryNameSnapshot { pub struct IndexingStatusRead { pub chains: Vec, pub has_unscoped_pending_invalidations: bool, + pub pending_invalidation_count: i64, + pub pending_invalidation_count_capped: bool, + pub dead_letter_count: i64, } #[derive(Clone, Debug, Eq, PartialEq)] diff --git a/crates/storage/src/lib.rs b/crates/storage/src/lib.rs index f349b080..02b6f23c 100644 --- a/crates/storage/src/lib.rs +++ b/crates/storage/src/lib.rs @@ -32,6 +32,7 @@ mod resolution_support; mod resolver; mod resolver_profile_authority_journal; mod resolver_profile_input_changes; +mod service_heartbeats; mod snapshot_selection; pub mod sql_row; mod stored_lineage_coverage; @@ -167,11 +168,12 @@ pub use identity::{ pub use identity_facade::{ IdentityAddressRelationRow, IdentityNameCurrentRow, IdentityNameRecordRow, IdentityPrimaryNameSnapshot, IdentityRecordInventoryRow, IndexingStatusChainRow, - IndexingStatusRead, ReverseIdentityCursor, ReverseIdentityFeedGroup, ReverseIdentityFeedInput, - ReverseIdentityFeedRecordRow, ReverseIdentityGroup, ReverseIdentityRecordRow, - ReverseIdentityRoles, ReverseIdentityStorageInput, load_identity_name_feed_records_by_names, - load_identity_records_by_names, load_indexing_status, load_reverse_identity_feed_records, - load_reverse_identity_records, + IndexingStatusRead, PENDING_INVALIDATION_COUNT_CAP, ReverseIdentityCursor, + ReverseIdentityFeedGroup, ReverseIdentityFeedInput, ReverseIdentityFeedRecordRow, + ReverseIdentityGroup, ReverseIdentityRecordRow, ReverseIdentityRoles, + ReverseIdentityStorageInput, load_expected_status_chain_ids, + load_identity_name_feed_records_by_names, load_identity_records_by_names, load_indexing_status, + load_reverse_identity_feed_records, load_reverse_identity_records, }; pub use label_preimages::{ LabelPreimage, LabelPreimageImportSummary, backfill_label_preimages_from_existing_facts, @@ -317,6 +319,14 @@ pub use resolver_profile_input_changes::{ load_pending_resolver_profile_input_changes, load_pending_resolver_profile_input_changes_excluding, }; +pub use service_heartbeats::{ + DEFAULT_WORKER_REBUILD_PHASE_MAX_AGE_SECS, INDEXER_SERVICE_NAME, ServiceLoopHeartbeat, + ServiceLoopPhaseHeartbeat, WORKER_SERVICE_NAME, begin_service_loop_phase, + deregister_service_loop, ensure_service_loop_heartbeat_recent, + ensure_service_loop_heartbeat_recent_with_phase, finish_service_loop_phase, + load_preferred_service_loop_heartbeats, load_service_loop_heartbeat, + record_service_loop_heartbeat, register_service_loop, resolve_service_instance_id, +}; pub use snapshot_selection::{ ChainPosition, ChainPositions, SelectedSnapshot, SnapshotAt, SnapshotConsistency, SnapshotPositionRequirement, SnapshotProjectionRead, SnapshotSelectionError, diff --git a/crates/storage/src/service_heartbeats.rs b/crates/storage/src/service_heartbeats.rs new file mode 100644 index 00000000..fff1ca3e --- /dev/null +++ b/crates/storage/src/service_heartbeats.rs @@ -0,0 +1,600 @@ +use std::collections::BTreeSet; + +use anyhow::{Context, Result, bail}; +use sqlx::{PgPool, types::time::OffsetDateTime}; + +pub const INDEXER_SERVICE_NAME: &str = "indexer"; +pub const WORKER_SERVICE_NAME: &str = "worker"; +pub const DEFAULT_WORKER_REBUILD_PHASE_MAX_AGE_SECS: i64 = 43_200; + +const PROCESS_SCOPE_KIND: &str = "process"; +const PROCESS_SCOPE_ID: &str = "process"; +const CHAIN_SCOPE_KIND: &str = "chain"; + +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct ServiceLoopHeartbeat { + pub service_name: String, + pub instance_id: String, + pub started_at: OffsetDateTime, + pub heartbeat_at: OffsetDateTime, + pub age_seconds: i64, + pub active_phase: Option, +} + +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct ServiceLoopPhaseHeartbeat { + pub phase: String, + pub started_at: OffsetDateTime, + pub heartbeat_at: OffsetDateTime, + pub age_seconds: i64, +} + +type ServiceLoopHeartbeatRow = ( + String, + String, + OffsetDateTime, + OffsetDateTime, + i64, + Option, + Option, + Option, + Option, +); + +pub fn resolve_service_instance_id(configured: Option<&str>) -> Result { + let instance_id = match configured { + Some(instance_id) => instance_id.trim().to_owned(), + None => std::env::var("HOSTNAME").unwrap_or_else(|_| "default".to_owned()), + }; + if instance_id.trim().is_empty() { + bail!("heartbeat instance id must not be blank"); + } + Ok(instance_id) +} + +pub async fn register_service_loop( + pool: &PgPool, + service_name: &str, + instance_id: &str, +) -> Result<()> { + validate_identity(service_name, instance_id)?; + + sqlx::query( + r#" + WITH retired_scopes AS ( + DELETE FROM service_loop_heartbeats + WHERE service_name = $1 + AND scope_kind <> 'process' + ), + observed AS ( + SELECT clock_timestamp() AS observed_at + ) + INSERT INTO service_loop_heartbeats ( + service_name, + instance_id, + scope_kind, + scope_id, + started_at, + heartbeat_at + ) + SELECT $1, $2, $3, $4, observed_at, observed_at + FROM observed + ON CONFLICT (service_name, instance_id, scope_kind, scope_id) + DO UPDATE SET + started_at = EXCLUDED.started_at, + heartbeat_at = EXCLUDED.heartbeat_at + "#, + ) + .bind(service_name) + .bind(instance_id) + .bind(PROCESS_SCOPE_KIND) + .bind(PROCESS_SCOPE_ID) + .execute(pool) + .await + .with_context(|| { + format!("failed to register {service_name} service loop heartbeat for {instance_id}") + })?; + + Ok(()) +} + +pub async fn record_service_loop_heartbeat( + pool: &PgPool, + service_name: &str, + instance_id: &str, + chain_ids: &[String], +) -> Result<()> { + validate_identity(service_name, instance_id)?; + if service_name != INDEXER_SERVICE_NAME && !chain_ids.is_empty() { + bail!("only the indexer service may record chain-scoped heartbeats"); + } + + let mut unique_chain_ids = BTreeSet::new(); + for chain_id in chain_ids { + let chain_id = chain_id.trim(); + if chain_id.is_empty() || chain_id == PROCESS_SCOPE_ID { + bail!("heartbeat chain id must be non-blank and must not equal process"); + } + unique_chain_ids.insert(chain_id.to_owned()); + } + + let mut scope_kinds = Vec::with_capacity(unique_chain_ids.len() + 1); + let mut scope_ids = Vec::with_capacity(unique_chain_ids.len() + 1); + scope_kinds.push(PROCESS_SCOPE_KIND.to_owned()); + scope_ids.push(PROCESS_SCOPE_ID.to_owned()); + for chain_id in unique_chain_ids { + scope_kinds.push(CHAIN_SCOPE_KIND.to_owned()); + scope_ids.push(chain_id); + } + + let recorded = sqlx::query( + r#" + WITH registered_process AS MATERIALIZED ( + /* service_loop_heartbeat_registration_fence */ SELECT scope_id + FROM service_loop_heartbeats + WHERE service_name = $1 + AND instance_id = $2 + AND scope_kind = 'process' + AND scope_id = 'process' + FOR UPDATE + ), + retired_phases AS ( + DELETE FROM service_loop_heartbeats + WHERE service_name = $1 + AND instance_id = $2 + AND scope_kind = 'phase' + AND EXISTS (SELECT 1 FROM registered_process) + ), + observed AS ( + SELECT clock_timestamp() AS observed_at + ) + INSERT INTO service_loop_heartbeats ( + service_name, + instance_id, + scope_kind, + scope_id, + started_at, + heartbeat_at + ) + SELECT + $1, + $2, + scope.scope_kind, + scope.scope_id, + observed.observed_at, + observed.observed_at + FROM UNNEST($3::TEXT[], $4::TEXT[]) AS scope(scope_kind, scope_id) + CROSS JOIN observed + CROSS JOIN registered_process + ON CONFLICT (service_name, instance_id, scope_kind, scope_id) + DO UPDATE SET heartbeat_at = EXCLUDED.heartbeat_at + "#, + ) + .bind(service_name) + .bind(instance_id) + .bind(&scope_kinds) + .bind(&scope_ids) + .execute(pool) + .await + .with_context(|| { + format!("failed to record {service_name} service loop heartbeat for {instance_id}") + })?; + if recorded.rows_affected() == 0 { + bail!("{service_name} service loop heartbeat for {instance_id} is not registered"); + } + + Ok(()) +} + +pub async fn begin_service_loop_phase( + pool: &PgPool, + service_name: &str, + instance_id: &str, + phase: &str, +) -> Result<()> { + validate_identity(service_name, instance_id)?; + validate_phase(service_name, phase)?; + + let recorded = sqlx::query( + r#" + WITH registered_process AS MATERIALIZED ( + /* begin_service_loop_phase_registration_fence */ SELECT scope_id + FROM service_loop_heartbeats + WHERE service_name = $1 + AND instance_id = $2 + AND scope_kind = 'process' + AND scope_id = 'process' + FOR UPDATE + ), + retired_phases AS ( + DELETE FROM service_loop_heartbeats + WHERE service_name = $1 + AND instance_id = $2 + AND scope_kind = 'phase' + AND EXISTS (SELECT 1 FROM registered_process) + ), + observed AS ( + SELECT clock_timestamp() AS observed_at + ), + process_heartbeat AS ( + UPDATE service_loop_heartbeats + SET heartbeat_at = observed.observed_at + FROM observed + WHERE service_name = $1 + AND instance_id = $2 + AND scope_kind = 'process' + AND scope_id = 'process' + AND EXISTS (SELECT 1 FROM registered_process) + RETURNING service_loop_heartbeats.scope_id + ) + INSERT INTO service_loop_heartbeats ( + service_name, + instance_id, + scope_kind, + scope_id, + started_at, + heartbeat_at + ) + SELECT $1, $2, 'phase', $3, observed_at, observed_at + FROM observed + CROSS JOIN process_heartbeat + ON CONFLICT (service_name, instance_id, scope_kind, scope_id) + DO UPDATE SET + started_at = EXCLUDED.started_at, + heartbeat_at = EXCLUDED.heartbeat_at + "#, + ) + .bind(service_name) + .bind(instance_id) + .bind(phase.trim()) + .execute(pool) + .await + .with_context(|| { + format!( + "failed to begin {service_name} service loop phase {} for {instance_id}", + phase.trim() + ) + })?; + if recorded.rows_affected() == 0 { + bail!("{service_name} service loop heartbeat for {instance_id} is not registered"); + } + + Ok(()) +} + +pub async fn finish_service_loop_phase( + pool: &PgPool, + service_name: &str, + instance_id: &str, + phase: &str, +) -> Result<()> { + validate_identity(service_name, instance_id)?; + validate_phase(service_name, phase)?; + + sqlx::query( + r#" + WITH retired_phase AS ( + DELETE FROM service_loop_heartbeats + WHERE service_name = $1 + AND instance_id = $2 + AND scope_kind = 'phase' + AND scope_id = $3 + ) + UPDATE service_loop_heartbeats + SET heartbeat_at = clock_timestamp() + WHERE service_name = $1 + AND instance_id = $2 + AND scope_kind = 'process' + AND scope_id = 'process' + "#, + ) + .bind(service_name) + .bind(instance_id) + .bind(phase.trim()) + .execute(pool) + .await + .with_context(|| { + format!( + "failed to finish {service_name} service loop phase {} for {instance_id}", + phase.trim() + ) + })?; + + Ok(()) +} + +pub async fn deregister_service_loop( + pool: &PgPool, + service_name: &str, + instance_id: &str, +) -> Result<()> { + validate_identity(service_name, instance_id)?; + if service_name != WORKER_SERVICE_NAME { + bail!("only the worker service may deregister its service loop"); + } + + let mut transaction = pool + .begin() + .await + .with_context(|| format!("failed to begin {service_name} loop deregistration"))?; + sqlx::query("DELETE FROM service_loop_heartbeats WHERE service_name = $1 AND instance_id = $2 AND scope_kind = 'process' AND scope_id = 'process'") + .bind(service_name) + .bind(instance_id) + .execute(&mut *transaction) + .await + .with_context(|| { + format!("failed to fence {service_name} service loop writers for {instance_id}") + })?; + sqlx::query("DELETE FROM service_loop_heartbeats WHERE service_name = $1 AND instance_id = $2") + .bind(service_name) + .bind(instance_id) + .execute(&mut *transaction) + .await + .with_context(|| { + format!("failed to clear {service_name} service loop rows for {instance_id}") + })?; + transaction + .commit() + .await + .with_context(|| format!("failed to commit {service_name} loop deregistration"))?; + + Ok(()) +} + +pub async fn load_service_loop_heartbeat( + pool: &PgPool, + service_name: &str, + instance_id: &str, +) -> Result> { + validate_identity(service_name, instance_id)?; + + let row = sqlx::query_as::<_, ServiceLoopHeartbeatRow>( + r#" + SELECT + process.service_name, + process.instance_id, + process.started_at, + process.heartbeat_at, + GREATEST( + FLOOR(EXTRACT(EPOCH FROM (clock_timestamp() - process.heartbeat_at)))::BIGINT, + 0 + ) AS age_seconds, + phase.scope_id AS phase, + phase.started_at AS phase_started_at, + phase.heartbeat_at AS phase_heartbeat_at, + CASE + WHEN phase.heartbeat_at IS NULL THEN NULL + ELSE GREATEST( + FLOOR(EXTRACT(EPOCH FROM (clock_timestamp() - phase.heartbeat_at)))::BIGINT, + 0 + ) + END AS phase_age_seconds + FROM service_loop_heartbeats AS process + LEFT JOIN LATERAL ( + SELECT scope_id, started_at, heartbeat_at + FROM service_loop_heartbeats + WHERE service_name = process.service_name + AND instance_id = process.instance_id + AND scope_kind = 'phase' + ORDER BY heartbeat_at DESC, scope_id + LIMIT 1 + ) AS phase ON TRUE + WHERE process.service_name = $1 + AND process.instance_id = $2 + AND process.scope_kind = $3 + AND process.scope_id = $4 + "#, + ) + .bind(service_name) + .bind(instance_id) + .bind(PROCESS_SCOPE_KIND) + .bind(PROCESS_SCOPE_ID) + .fetch_optional(pool) + .await + .with_context(|| { + format!("failed to load {service_name} service loop heartbeat for {instance_id}") + })?; + + Ok(row.map(heartbeat_from_row)) +} + +pub async fn ensure_service_loop_heartbeat_recent( + pool: &PgPool, + service_name: &str, + instance_id: &str, + max_age_seconds: i64, +) -> Result { + ensure_service_loop_heartbeat_recent_with_phase( + pool, + service_name, + instance_id, + max_age_seconds, + max_age_seconds, + ) + .await +} + +pub async fn ensure_service_loop_heartbeat_recent_with_phase( + pool: &PgPool, + service_name: &str, + instance_id: &str, + max_age_seconds: i64, + phase_max_age_seconds: i64, +) -> Result { + if max_age_seconds <= 0 { + bail!("heartbeat maximum age must be greater than zero seconds"); + } + if phase_max_age_seconds <= 0 { + bail!("heartbeat phase maximum age must be greater than zero seconds"); + } + + let heartbeat = load_service_loop_heartbeat(pool, service_name, instance_id) + .await? + .with_context(|| { + format!( + "{service_name} loop heartbeat was not found for instance {instance_id}; the process loop never started" + ) + })?; + if let Some(phase) = heartbeat.active_phase.as_ref() { + if phase.age_seconds > phase_max_age_seconds { + bail!( + "{service_name} loop phase {} for instance {instance_id} is stale ({} seconds old; maximum {}); the phase stopped or wedged", + phase.phase, + phase.age_seconds, + phase_max_age_seconds + ); + } + } else if heartbeat.age_seconds > max_age_seconds { + bail!( + "{service_name} loop heartbeat for instance {instance_id} is stale ({} seconds old; maximum {}); the process loop stopped or wedged", + heartbeat.age_seconds, + max_age_seconds + ); + } + + Ok(heartbeat) +} + +pub async fn load_preferred_service_loop_heartbeats( + pool: &PgPool, + service_names: &[&str], + max_age_seconds: i64, + phase_max_age_seconds: i64, +) -> Result> { + for service_name in service_names { + validate_service_name(service_name)?; + } + if max_age_seconds <= 0 { + bail!("heartbeat maximum age must be greater than zero seconds"); + } + if phase_max_age_seconds <= 0 { + bail!("heartbeat phase maximum age must be greater than zero seconds"); + } + + let rows = sqlx::query_as::<_, ServiceLoopHeartbeatRow>( + r#" + WITH candidate_heartbeats AS ( + SELECT + process.service_name, + process.instance_id, + process.started_at AS process_started_at, + process.heartbeat_at AS process_heartbeat_at, + GREATEST( + FLOOR(EXTRACT(EPOCH FROM (clock_timestamp() - process.heartbeat_at)))::BIGINT, + 0 + ) AS age_seconds, + phase.scope_id AS phase, + phase.started_at AS phase_started_at, + phase.heartbeat_at AS phase_heartbeat_at, + CASE + WHEN phase.heartbeat_at IS NULL THEN NULL + ELSE GREATEST( + FLOOR(EXTRACT(EPOCH FROM (clock_timestamp() - phase.heartbeat_at)))::BIGINT, + 0 + ) + END AS phase_age_seconds + FROM service_loop_heartbeats AS process + LEFT JOIN LATERAL ( + SELECT scope_id, started_at, heartbeat_at + FROM service_loop_heartbeats + WHERE service_name = process.service_name + AND instance_id = process.instance_id + AND scope_kind = 'phase' + ORDER BY heartbeat_at DESC, scope_id + LIMIT 1 + ) AS phase ON TRUE + WHERE process.service_name = ANY($1::TEXT[]) + AND process.scope_kind = $2 + AND process.scope_id = $3 + ), + ranked_heartbeats AS ( + SELECT + candidate_heartbeats.*, + ROW_NUMBER() OVER ( + PARTITION BY service_name + ORDER BY + CASE + WHEN phase_heartbeat_at IS NOT NULL + THEN phase_age_seconds <= $5 + ELSE age_seconds <= $4 + END DESC, + process_heartbeat_at DESC, + instance_id + ) AS preference + FROM candidate_heartbeats + ) + SELECT + service_name, + instance_id, + process_started_at, + process_heartbeat_at, + age_seconds, + phase, + phase_started_at, + phase_heartbeat_at, + phase_age_seconds + FROM ranked_heartbeats + WHERE preference = 1 + ORDER BY service_name + "#, + ) + .bind(service_names) + .bind(PROCESS_SCOPE_KIND) + .bind(PROCESS_SCOPE_ID) + .bind(max_age_seconds) + .bind(phase_max_age_seconds) + .fetch_all(pool) + .await + .context("failed to load preferred service loop heartbeats")?; + + Ok(rows.into_iter().map(heartbeat_from_row).collect()) +} + +fn heartbeat_from_row(row: ServiceLoopHeartbeatRow) -> ServiceLoopHeartbeat { + let active_phase = match (row.5, row.6, row.7, row.8) { + (Some(phase), Some(started_at), Some(heartbeat_at), Some(age_seconds)) => { + Some(ServiceLoopPhaseHeartbeat { + phase, + started_at, + heartbeat_at, + age_seconds, + }) + } + (None, None, None, None) => None, + _ => unreachable!("phase heartbeat columns must all be null or all be present"), + }; + ServiceLoopHeartbeat { + service_name: row.0, + instance_id: row.1, + started_at: row.2, + heartbeat_at: row.3, + age_seconds: row.4, + active_phase, + } +} + +fn validate_identity(service_name: &str, instance_id: &str) -> Result<()> { + validate_service_name(service_name)?; + if instance_id.trim().is_empty() { + bail!("heartbeat instance id must not be blank"); + } + Ok(()) +} + +fn validate_service_name(service_name: &str) -> Result<()> { + if !matches!(service_name, INDEXER_SERVICE_NAME | WORKER_SERVICE_NAME) { + bail!("unsupported heartbeat service name {service_name}"); + } + Ok(()) +} + +fn validate_phase(service_name: &str, phase: &str) -> Result<()> { + if service_name != WORKER_SERVICE_NAME { + bail!("only the worker service may register phase-scoped heartbeats"); + } + let phase = phase.trim(); + if phase.is_empty() || phase == PROCESS_SCOPE_ID { + bail!("heartbeat phase must be non-blank and must not equal process"); + } + Ok(()) +} diff --git a/docker-compose.server.yml b/docker-compose.server.yml index a7492781..4dbe9cac 100644 --- a/docker-compose.server.yml +++ b/docker-compose.server.yml @@ -81,6 +81,13 @@ services: <<: *bigname-env BIGNAME_API_BIND_ADDR: ${BIGNAME_API_BIND_ADDR:-0.0.0.0:3000} BIGNAME_API_CHAIN_RPC_URLS: ${BIGNAME_API_CHAIN_RPC_URLS:-} + BIGNAME_API_HEARTBEAT_MAX_AGE_SECS: ${BIGNAME_API_HEARTBEAT_MAX_AGE_SECS:-20} + BIGNAME_API_WORKER_REBUILD_PHASE_MAX_AGE_SECS: ${BIGNAME_API_WORKER_REBUILD_PHASE_MAX_AGE_SECS:-43200} + BIGNAME_API_STATUS_PROVIDER_TIMEOUT_MS: ${BIGNAME_API_STATUS_PROVIDER_TIMEOUT_MS:-750} + BIGNAME_API_STATUS_PROVIDER_REFRESH_SECS: ${BIGNAME_API_STATUS_PROVIDER_REFRESH_SECS:-5} + BIGNAME_API_STATUS_PROVIDER_CACHE_TTL_SECS: ${BIGNAME_API_STATUS_PROVIDER_CACHE_TTL_SECS:-30} + BIGNAME_API_STATUS_MAX_BLOCK_LAG: ${BIGNAME_API_STATUS_MAX_BLOCK_LAG:-30} + BIGNAME_API_STATUS_MAX_LAG_SECS: ${BIGNAME_API_STATUS_MAX_LAG_SECS:-60} BIGNAME_API_IDENTITY_BATCH_LIMIT: ${BIGNAME_API_IDENTITY_BATCH_LIMIT:-1000} BIGNAME_API_LOOKUP_BATCH_LIMIT: ${BIGNAME_API_LOOKUP_BATCH_LIMIT:-1000} BIGNAME_API_REQUEST_TIMEOUT_MS: ${BIGNAME_API_REQUEST_TIMEOUT_MS:-30000} @@ -97,7 +104,7 @@ services: healthcheck: test: - CMD-SHELL - - curl -fsS "http://127.0.0.1:$${BIGNAME_API_BIND_ADDR##*:}/healthz" | grep -q '"status":"ready"' + - curl -fsS "http://127.0.0.1:$${BIGNAME_API_BIND_ADDR##*:}/healthz" | grep -q '"api_status":"ready"' interval: 10s timeout: 5s retries: 12 @@ -113,8 +120,10 @@ services: condition: service_completed_successfully environment: <<: *bigname-env + BIGNAME_HEARTBEAT_INSTANCE_ID: ${BIGNAME_INDEXER_HEARTBEAT_INSTANCE_ID:-indexer} BIGNAME_INDEXER_MANIFESTS_ROOT: ${BIGNAME_INDEXER_MANIFESTS_ROOT:-/app/manifests/mainnet} BIGNAME_INDEXER_POLL_INTERVAL_SECS: ${BIGNAME_INDEXER_POLL_INTERVAL_SECS:-5} + BIGNAME_INDEXER_HEARTBEAT_MAX_AGE_SECS: ${BIGNAME_INDEXER_HEARTBEAT_MAX_AGE_SECS:-20} BIGNAME_INDEXER_RAW_CODE_BASELINE_MAX_ADDRESSES_PER_TICK: ${BIGNAME_INDEXER_RAW_CODE_BASELINE_MAX_ADDRESSES_PER_TICK:-2048} BIGNAME_INDEXER_HASH_PINNED_BACKFILL_CHUNK_BLOCKS: ${BIGNAME_INDEXER_HASH_PINNED_BACKFILL_CHUNK_BLOCKS:-1024} BIGNAME_INDEXER_HASH_PINNED_BACKFILL_MAX_LOGS_PER_PUSH: ${BIGNAME_INDEXER_HASH_PINNED_BACKFILL_MAX_LOGS_PER_PUSH:-10000} @@ -183,7 +192,10 @@ services: condition: service_completed_successfully environment: <<: *bigname-env + BIGNAME_HEARTBEAT_INSTANCE_ID: ${BIGNAME_WORKER_HEARTBEAT_INSTANCE_ID:-worker} BIGNAME_WORKER_POLL_INTERVAL_SECS: ${BIGNAME_WORKER_POLL_INTERVAL_SECS:-5} + BIGNAME_WORKER_HEARTBEAT_MAX_AGE_SECS: ${BIGNAME_WORKER_HEARTBEAT_MAX_AGE_SECS:-20} + BIGNAME_WORKER_REBUILD_PHASE_MAX_AGE_SECS: ${BIGNAME_WORKER_REBUILD_PHASE_MAX_AGE_SECS:-43200} BIGNAME_WORKER_CHAIN_RPC_URLS: ${BIGNAME_WORKER_CHAIN_RPC_URLS:-} BIGNAME_WORKER_TEXT_HYDRATION_MULTICALL3_ADDRESS: ${BIGNAME_WORKER_TEXT_HYDRATION_MULTICALL3_ADDRESS:-0xcA11bde05977b3631167028862bE2a173976CA11} BIGNAME_WORKER_TEXT_HYDRATION_BATCH_SIZE: ${BIGNAME_WORKER_TEXT_HYDRATION_BATCH_SIZE:-250} diff --git a/docs/adrs/0006-api-v2-product-surface.md b/docs/adrs/0006-api-v2-product-surface.md index 6ab5e49f..6224dc37 100644 --- a/docs/adrs/0006-api-v2-product-surface.md +++ b/docs/adrs/0006-api-v2-product-surface.md @@ -181,7 +181,7 @@ Tier 1 — lookup primitives: | Route | Purpose | | --- | --- | | `POST /v2/lookup` | Batched forward (name) and reverse (address + coin type) resolution. `profile=feed` is the latency path; `profile=detail` returns full records. Replaces `POST /v1/identity:lookup`. | -| `GET /v2/status` | Per-chain indexing readiness: `chains: {: {latest_block, indexed_block, safe_block, finalized_block, lag_blocks, lag_seconds, status}}`. `status` here is the ops vocabulary `ready\|degraded\|stale` — the one non-result status enum, scoped to this route. | +| `GET /v2/status` | Per-chain indexing readiness: root `pending_invalidation_count`, `pending_invalidation_count_capped`, and `dead_letter_count` fields plus `chains: {: {latest_block, indexed_block, safe_block, finalized_block, lag_blocks, lag_seconds, network_block, network_head_observed_at, network_head_age_seconds, network_head_status, ingestion_lag_blocks, ingestion_lag_seconds, status}}`. The pending count is exact through 10,000; the capped marker means the bounded query observed at least 10,001 live rows. The provider fields are a timeout-bounded asynchronously refreshed cache comparison; the request performs no provider I/O. `status` here is the ops vocabulary `ready\|degraded\|stale` — the one non-result status enum, scoped to this route. | Tier 2 — product reads: @@ -212,8 +212,11 @@ Tier 3 — diagnostics (the only routes carrying pipeline vocabulary): | `GET /v2/diagnostics/namespaces/{namespace}/manifests` | Active manifest versions, source families, deployment epochs, capability flags. Replaces `/v1/manifests/{namespace}`. | | `GET /v2/diagnostics/events` | Raw normalized-event rows: upstream event kinds, event identity, full provenance. Same filters as `/v2/events`. This is the home of `v1` history `view=full`, resolving ADR 0003's open history full-view decision: the full row shape survives as a diagnostics contract, not a product one. | -`GET /healthz`, `GET /`, `GET /docs`, and `GET /openapi.json` remain non-contract -helpers. +`GET /healthz` remains the unversioned operator health contract outside the +versioned product routes. Its HTTP status and `api_status` are API-local +process/database readiness; aggregate `status` and `loops` retain indexer and +worker liveness evidence. `GET /`, `GET /docs`, and `GET /openapi.json` remain +non-contract helpers. Deleted from the public catalog (capability absorbed as noted): the `profiles/` prefix, `/v1/coverage/*` and `/v1/explain/*` (moved under diagnostics), diff --git a/docs/api-v1-routes.md b/docs/api-v1-routes.md index 06340018..b3a0038f 100644 --- a/docs/api-v1-routes.md +++ b/docs/api-v1-routes.md @@ -10,7 +10,7 @@ Use the route groups as integration guidance, not just documentation order: | --- | --- | --- | | Native slim identity | `POST /v1/identity:lookup`, `GET /v1/status` | Partner-1 feed/profile reads and shadow comparison. Use `profile=feed` for the under-10 ms p95 feed target. | | Canonical product reads | `/v1/names*`, `/v1/profiles/names/*`, `/v1/addresses/{address}/names`, `/v1/primary-names*`, `/v1/resources/{resource_id}/permissions`, `/v1/events` | New app, explorer, and public API integrations that want bigname-native semantics. | -| Metadata/control plane | `/v1/namespaces/*`, `/v1/manifests/*` | Namespace and manifest introspection. | +| Metadata/control plane | `/v1/namespaces/*`, `/v1/manifests/*`, `/healthz` | Namespace, manifest, API/database readiness, and indexer/worker loop-liveness introspection. | | Diagnostics/provenance | `/v1/coverage/*`, `/v1/explain/*` | Completeness, freshness, derivation, persisted execution, and audit detail. | | Specialist adjuncts | `/v1/roles`, `/v1/names/*/roles`, `/v1/resources/lookup`, `/v1/history/*`, `/v1/resolvers/*/overview` | Supported surfaces for specialist workflows; prefer canonical product reads for new integrations when they fit. | @@ -309,7 +309,8 @@ Rules: ## `GET /v1/status` -Projection/indexing readiness and chain lag. This is not `/healthz`; `/healthz` reports process and database readiness. +Projection/indexing readiness and chain lag. This is not `/healthz`; `/healthz` +reports API/database readiness and indexer/worker loop liveness. Response: @@ -317,6 +318,9 @@ Response: { "data": { "status": "ready", + "pending_invalidation_count": 0, + "pending_invalidation_count_capped": false, + "dead_letter_count": 0, "chains": { "ethereum-mainnet": { "canonical_block": 0, @@ -325,7 +329,13 @@ Response: "latest_projected_block": 0, "latest_projected_timestamp": null, "projection_lag_blocks": 0, - "projection_lag_seconds": null + "projection_lag_seconds": null, + "network_block": 0, + "network_head_observed_at": "2026-07-21T12:00:00Z", + "network_head_age_seconds": 1, + "network_head_status": "fresh", + "ingestion_lag_blocks": 0, + "ingestion_lag_seconds": 0 } } } @@ -334,6 +344,49 @@ Response: Uses active/shadow `manifest_versions` to include chains expected by the loaded profile, plus `chain_checkpoints`, retained `chain_lineage`, `projection_normalized_event_changes`, `projection_apply_cursors`, and `projection_invalidations` where available. Fields stay `null` when the deployment has not yet retained the corresponding operational metadata. If no chain readiness data exists for an expected chain, or if pending direct invalidations cannot be tied to a normalized-event chain position, `status` is `degraded`. If any expected chain has unapplied normalized-event changes beyond the projection-apply cursor, `status` is `stale` and the lag fields identify the affected chain. +`pending_invalidation_count` is an exact live [projection](glossary.md) +invalidation-queue row count through 10,000. The status query reads at most +10,001 queue rows: if it observes more than the reported cap, the response is +`pending_invalidation_count=10000` and +`pending_invalidation_count_capped=true`; otherwise the boolean is `false` and +the integer is exact. `dead_letter_count` is the number of terminal +invalidation failures retained for operator inspection. These numeric fields +preserve the evidence that readiness previously folded into boolean state; +dead letters are informational and do not by themselves change readiness. + +The API also compares each stored canonical head with a cached provider +`eth_blockNumber` observation. `network_head_status` is `fresh`, `stale`, +`unavailable`, `pending`, or `unconfigured`. A successful observation supplies +`network_block`, `network_head_observed_at`, and +`network_head_age_seconds`. When the network is ahead, +`ingestion_lag_blocks` is the block difference and `ingestion_lag_seconds` is +the difference between the provider observation time and the stored canonical +block timestamp. A fresh observation changes chain readiness to `stale` when +either lag exceeds `BIGNAME_API_STATUS_MAX_BLOCK_LAG` (default 30, so the +fastest configured two-second chain tolerates ordinary poll and provider skew) or +`BIGNAME_API_STATUS_MAX_LAG_SECS` (default 60). A missing, failed, +not-yet-completed, or cache-expired provider observation changes readiness to +`degraded`, with its reason retained in `network_head_status`. When a refresh +fails after an earlier success, `network_head_status` becomes `unavailable` +immediately while the prior head, observation time, age, and lag values remain +available as cached evidence; the next successful refresh replaces them. + +Provider access is never performed by the status request. A background task +refreshes all configured `BIGNAME_API_CHAIN_RPC_URLS` concurrently every +`BIGNAME_API_STATUS_PROVIDER_REFRESH_SECS` (default 5), bounds each call by +`BIGNAME_API_STATUS_PROVIDER_TIMEOUT_MS` (default 750), and serves the most +recent successful evidence for at most +`BIGNAME_API_STATUS_PROVIDER_CACHE_TTL_SECS` (default 30). A failed latest +attempt is never presented as fresh even while that evidence remains visible. +Status requests only read that in-memory cache, so a slow provider cannot hold +the route open. The mapping is load-bearing for status readiness: API startup +compares it with every expected status chain, emits a warning naming omissions, +and leaves each omitted chain `unconfigured` and aggregate readiness +`degraded` when its projection is current. If that chain also has positive +`lag_blocks`, the known projection lag takes precedence and aggregate readiness +is `stale`; `network_head_status` still reports the omitted provider as +`unconfigured`. + ## `GET /v1/names` Compact app-facing collection: exact lookup, address-owned lists, owner/registrant/effective-controller relations, name search, suggestions. diff --git a/docs/api-v1.md b/docs/api-v1.md index 7ac749a1..72bb41a0 100644 --- a/docs/api-v1.md +++ b/docs/api-v1.md @@ -358,7 +358,7 @@ The actual published routes are listed below. Per-route semantics are in [`api-v | Route | Purpose | | --- | --- | | `POST /v1/identity:lookup` | Native slim identity lookup. `profile=feed` is the partner-1 latency path; `profile=detail` is profile aggregation. | -| `GET /v1/status` | Public projection/indexing readiness by chain. | +| `GET /v1/status` | Public projection/indexing readiness by chain, including cached network-head freshness and invalidation/dead-letter counts. | ### Canonical product reads @@ -380,7 +380,7 @@ The actual published routes are listed below. Per-route semantics are in [`api-v | --- | --- | | `GET /v1/namespaces/{namespace}` | Namespace metadata. | | `GET /v1/manifests/{namespace}` | Active manifest versions and capabilities. | -| `GET /healthz` | Process and database readiness check. Not part of the `v1` contract. | +| `GET /healthz` | API-local process/database readiness (`api_status` and HTTP status) plus aggregate indexer/worker loop-liveness evidence (`status` and `loops`). Unversioned operator contract; not part of the `v1` contract. | ### Diagnostics and provenance diff --git a/docs/api-v1.openapi.json b/docs/api-v1.openapi.json index 50018e38..9d43364b 100644 --- a/docs/api-v1.openapi.json +++ b/docs/api-v1.openapi.json @@ -800,6 +800,75 @@ ], "type": "object" }, + "HealthLoop": { + "properties": { + "heartbeat_age_seconds": { + "minimum": 0, + "type": [ + "integer", + "null" + ] + }, + "heartbeat_at": { + "format": "date-time", + "type": [ + "string", + "null" + ] + }, + "max_age_seconds": { + "minimum": 1, + "type": "integer" + }, + "phase": { + "description": "Named worker rebuild phase when the loop is inside a monolithic operation governed by its separately configurable maximum age; null for normal loop progress.", + "type": [ + "string", + "null" + ] + }, + "started_at": { + "format": "date-time", + "type": [ + "string", + "null" + ] + }, + "status": { + "enum": [ + "running", + "stale", + "not_started", + "unavailable" + ], + "type": "string" + } + }, + "required": [ + "status", + "phase", + "started_at", + "heartbeat_at", + "heartbeat_age_seconds", + "max_age_seconds" + ], + "type": "object" + }, + "HealthLoops": { + "properties": { + "indexer": { + "$ref": "#/components/schemas/HealthLoop" + }, + "worker": { + "$ref": "#/components/schemas/HealthLoop" + } + }, + "required": [ + "indexer", + "worker" + ], + "type": "object" + }, "HealthProcess": { "properties": { "status": { @@ -825,12 +894,23 @@ }, "HealthResponse": { "properties": { + "api_status": { + "description": "API-local readiness: ready when this process is serving and its database reachability query succeeds; independent of indexer and worker loop state.", + "enum": [ + "ready", + "degraded" + ], + "type": "string" + }, "database": { "$ref": "#/components/schemas/HealthDatabase" }, "identity": { "$ref": "#/components/schemas/HealthIdentity" }, + "loops": { + "$ref": "#/components/schemas/HealthLoops" + }, "process": { "$ref": "#/components/schemas/HealthProcess" }, @@ -838,6 +918,11 @@ "type": "string" }, "status": { + "description": "Aggregate database, indexer-loop, and worker-loop health for status consumers.", + "enum": [ + "ready", + "degraded" + ], "type": "string" } }, @@ -845,8 +930,10 @@ "service", "identity", "status", + "api_status", "process", - "database" + "database", + "loops" ], "type": "object" }, @@ -1103,6 +1190,20 @@ "null" ] }, + "ingestion_lag_blocks": { + "minimum": 0, + "type": [ + "integer", + "null" + ] + }, + "ingestion_lag_seconds": { + "minimum": 0, + "type": [ + "integer", + "null" + ] + }, "latest_projected_block": { "type": [ "integer", @@ -1116,6 +1217,37 @@ "null" ] }, + "network_block": { + "minimum": 0, + "type": [ + "integer", + "null" + ] + }, + "network_head_age_seconds": { + "minimum": 0, + "type": [ + "integer", + "null" + ] + }, + "network_head_observed_at": { + "format": "date-time", + "type": [ + "string", + "null" + ] + }, + "network_head_status": { + "enum": [ + "fresh", + "stale", + "unavailable", + "pending", + "unconfigured" + ], + "type": "string" + }, "projection_lag_blocks": { "type": [ "integer", @@ -1142,12 +1274,32 @@ "latest_projected_block", "latest_projected_timestamp", "projection_lag_blocks", - "projection_lag_seconds" + "projection_lag_seconds", + "network_block", + "network_head_observed_at", + "network_head_age_seconds", + "network_head_status", + "ingestion_lag_blocks", + "ingestion_lag_seconds" ], "type": "object" }, "type": "object" }, + "dead_letter_count": { + "minimum": 0, + "type": "integer" + }, + "pending_invalidation_count": { + "description": "Exact live queue row count when pending_invalidation_count_capped is false. When capped is true, this field equals the cap and the queue contains at least one additional row.", + "maximum": 10000, + "minimum": 0, + "type": "integer" + }, + "pending_invalidation_count_capped": { + "description": "True when the bounded status query stopped after observing more rows than pending_invalidation_count reports.", + "type": "boolean" + }, "status": { "enum": [ "ready", @@ -1159,6 +1311,9 @@ }, "required": [ "status", + "pending_invalidation_count", + "pending_invalidation_count_capped", + "dead_letter_count", "chains" ], "type": "object" diff --git a/docs/api-v2-routes.md b/docs/api-v2-routes.md index ddcdfef3..291b0b41 100644 --- a/docs/api-v2-routes.md +++ b/docs/api-v2-routes.md @@ -8,7 +8,8 @@ error shape live in [`api-v2.md`](api-v2.md). Routes below use the `/v2` development prefix. At the switch, the prefix becomes `/v1`; no permanent public `/v2` prefix ships. -`GET /healthz`, `GET /`, `GET /docs`, and `GET /openapi.json` remain +`GET /healthz` remains the unversioned operator health contract outside the +versioned product routes. `GET /`, `GET /docs`, and `GET /openapi.json` remain non-contract helpers. ## Shared Route Rules @@ -79,7 +80,10 @@ Field ownership: names collection inside one resolver overview object. - Ops status containers are route-local: `/v2/status` owns `chains`, `latest_block`, `indexed_block`, `safe_block`, `finalized_block`, - `lag_blocks`, and `lag_seconds`. + `lag_blocks`, `lag_seconds`, `pending_invalidation_count`, + `pending_invalidation_count_capped`, `dead_letter_count`, `network_block`, `network_head_observed_at`, + `network_head_age_seconds`, `network_head_status`, + `ingestion_lag_blocks`, and `ingestion_lag_seconds`. - Diagnostic-only field names are route-local to diagnostics unless they are already dictionary fields. Diagnostics may use pipeline vocabulary because their tier is explicitly separate from product reads. @@ -136,12 +140,28 @@ Field ownership: - Purpose: per-chain indexing readiness. - Request parameters: none. - Response shape: `data.status` plus `data.chains`, keyed by `chain_id`. - Each chain entry carries `latest_block`, `indexed_block`, `safe_block`, - `finalized_block`, `lag_blocks`, `lag_seconds`, and route-local ops - `status`. + `data.pending_invalidation_count` reports live queued work exactly through + 10,000. `data.pending_invalidation_count_capped=true` means the bounded query + observed at least 10,001 rows and reports 10,000 instead of scanning the + remaining queue. `data.dead_letter_count` reports terminal invalidation failures. Each chain + entry carries `latest_block`, `indexed_block`, `safe_block`, + `finalized_block`, `lag_blocks`, `lag_seconds`, `network_block`, + `network_head_observed_at`, `network_head_age_seconds`, + `network_head_status`, `ingestion_lag_blocks`, `ingestion_lag_seconds`, and + route-local ops `status`. - Pagination behavior: none. - Status semantics: route-local ops `status` is `ready`, `degraded`, or - `stale`. This is the only non-result `status` enum in `v2`. + `stale`. This is the only non-result `status` enum in `v2`. Projection lag + or a fresh provider comparison beyond either configured ingestion-lag + threshold is `stale`. Missing stored readiness or a provider observation + whose `network_head_status` is `stale`, `unavailable`, `pending`, or + `unconfigured` is `degraded` when its projection is current. Positive + projection lag takes precedence and is `stale`; `network_head_status` still + reports the provider state. The provider head is refreshed asynchronously + under a timeout and cache TTL; this route reads only the cache and never + waits for provider I/O. If the latest refresh fails after a successful one, + `network_head_status` becomes `unavailable` immediately while the last head, + observation time, age, and lag values remain as cached evidence. - Replaces (v1): `GET /v1/status`. ## Tier 2: Product Reads diff --git a/docs/api-v2.md b/docs/api-v2.md index ecd9635b..096560aa 100644 --- a/docs/api-v2.md +++ b/docs/api-v2.md @@ -229,7 +229,13 @@ Lookup primitives serve the partner latency path and current indexing status: The lookup route uses the common record shape and in-band per-result statuses. `GET /v2/status` is the only route with the ops status vocabulary -`ready`, `degraded`, `stale`. +`ready`, `degraded`, `stale`. It exposes the live invalidation count exactly +through 10,000, marks larger queues with +`pending_invalidation_count_capped=true`, and reports the dead-letter count plus +cached network-head comparison evidence. Provider refresh runs +asynchronously under a timeout and cache TTL, so the route never waits for a +provider. A failed latest refresh degrades readiness immediately while keeping +the last successful head comparison visible as cached evidence. ### Tier 2: Product Reads diff --git a/docs/consumer-capabilities.md b/docs/consumer-capabilities.md index efa176a0..047bd4bc 100644 --- a/docs/consumer-capabilities.md +++ b/docs/consumer-capabilities.md @@ -10,7 +10,7 @@ Use these sets when choosing a public route: | --- | --- | --- | | Native slim identity | `POST /v1/identity:lookup`, `GET /v1/status` | partner-1 style feeds, profile aggregation, and shadow comparison. Feed rendering should use `profile=feed`, which is backed by compact count/identity [sidecars](glossary.md). | | Canonical product reads | `/v1/names*`, `/v1/profiles/names/*`, `/v1/addresses/{address}/names`, `/v1/primary-names*`, `/v1/resources/{resource_id}/permissions`, `/v1/events` | first-party app, explorer, and public API integrations that want the bigname contract. | -| Metadata/control plane | `/v1/namespaces/*`, `/v1/manifests/*` | manifest and namespace introspection. | +| Metadata/control plane | `/v1/namespaces/*`, `/v1/manifests/*`, `/healthz` | manifest and namespace reads; API-local process/database readiness separated from aggregate indexer/worker loop-liveness introspection. | | Diagnostics/provenance | `/v1/coverage/*`, `/v1/explain/*` | debugging completeness, support, derivation, persisted execution, and audit paths. | | Specialist adjuncts | `/v1/roles`, `/v1/names/*/roles`, `/v1/resources/lookup`, `/v1/history/*`, `/v1/resolvers/*/overview` | supported routes for specialist views and narrow adjuncts. Prefer the canonical product reads above when they satisfy the use case. | diff --git a/docs/deployment.md b/docs/deployment.md index 1073279f..fad946fb 100644 --- a/docs/deployment.md +++ b/docs/deployment.md @@ -32,7 +32,13 @@ docker run --rm ghcr.io/ensdomains/bigname:latest bigname-worker inspect watch-p 1. Install Docker and Docker Compose. 2. Copy `.env.server.example` to `.env.server` and change the placeholder passwords. 3. Set `BIGNAME_IMAGE` to the image tag to run. -4. Start the stack: +4. Set `BIGNAME_API_CHAIN_RPC_URLS` with one `=` entry for every + chain expected by an active/shadow manifest or stored checkpoint. This + variable is load-bearing for `/v1/status` and `/v2/status` readiness. The + API starts and serves local readiness when entries are absent, but logs a + `WARN` naming every missing chain and keeps those status chains fail-closed + as `degraded`. +5. Start the stack: ```sh docker compose --env-file .env.server -f docker-compose.server.yml up -d @@ -46,12 +52,37 @@ deployments normally set it to `127.0.0.1` and expose traffic through the Caddy override documented in `docs/production.md`. The indexer and worker healthcheck commands verify that applied database -migrations exactly match the migration set compiled into the running binary. -They fail closed for missing, failed, checksum-mismatched, or newer unknown -migrations. During rolling upgrades, running `migrate` before recreating old +migrations exactly match the migration set compiled into the running binary +and that the checked process instance's main-loop heartbeat is recent. They +fail closed for missing, failed, checksum-mismatched, or newer unknown +migrations, for a loop that never registered, and for a loop whose heartbeat +exceeds the service-specific maximum age. The default maximum is 20 seconds; +set `BIGNAME_INDEXER_HEARTBEAT_MAX_AGE_SECS` and +`BIGNAME_WORKER_HEARTBEAT_MAX_AGE_SECS` in proportion to custom poll +intervals. Worker rebuild operations with no safe inner batch boundary use a +named phase row and the independently tunable +`BIGNAME_WORKER_REBUILD_PHASE_MAX_AGE_SECS` (default 43,200); set the matching +API interpretation with `BIGNAME_API_WORKER_REBUILD_PHASE_MAX_AGE_SECS`. +`docker-compose.server.yml` maps stable per-service instance IDs from +`BIGNAME_INDEXER_HEARTBEAT_INSTANCE_ID` and +`BIGNAME_WORKER_HEARTBEAT_INSTANCE_ID`, defaulting to `indexer` and `worker`. +This lets a recreated single-writer service retire unfinished non-process +heartbeat rows from the prior container during registration. +During rolling upgrades, running `migrate` before recreating old service containers can therefore mark those old indexer/worker containers -unhealthy until they are replaced with the matching image; treat that as version -skew, not evidence that PostgreSQL is down. +unhealthy until they are replaced with the matching image; treat that as +version skew, not evidence that PostgreSQL is down. + +The API `/healthz` HTTP status and `api_status` field cover only the serving API +process and its `SELECT 1` database probe. Its aggregate `status` and `loops` +object still require recent indexer and worker evidence, using +`BIGNAME_API_HEARTBEAT_MAX_AGE_SECS` (default 20), so a planned indexer restart +or long worker phase stays visible without making the API container or public +edge unhealthy. The status routes use the API chain RPC mapping for an +asynchronous cached network-head probe. Tune its provider timeout, refresh +interval, cache TTL, and ingestion block/time limits with the +`BIGNAME_API_STATUS_PROVIDER_*` and `BIGNAME_API_STATUS_MAX_*` variables in +`.env.server.example`; the default maximum block lag is 30. ### Resolver-profile replay after an upgrade or compaction @@ -79,8 +110,9 @@ worker-rolling-compatible upgrade. Deployment automation must drain public API traffic, stop every old worker, and confirm no old worker process remains before starting any worker from the new image. Start one new worker, wait until every current projection family has a marker for the new replay version and -`projection_invalidations` is empty, and only then start or undrain the API and -the remaining new workers. Indexers may continue ingesting while workers are +`projection_invalidations` is empty, and only then start or undrain the API. +The supported deployment has one active worker; do not overlap old and new +workers during this handoff. Indexers may continue ingesting while workers are stopped; their durable changes will be consumed after replay handoff. Replay version 9 is such an upgrade: it forces the full permission cutover that @@ -153,9 +185,12 @@ If `BIGNAME_INDEXER_CHAIN_RPC_URLS` is unset, the indexer still syncs manifest/watch state, but provider-backed live ingestion remains idle. Current bootstrap RPC support accepts `http://` and `https://` endpoints. -The API service also needs its own Ethereum JSON-RPC provider for live ENS -verified resolution and the ENS/60 primary-name on-demand reverse/forward RPC -fallback, configured as +The API service uses its own JSON-RPC mapping both for indexing-status +network-head readiness and for live execution. Every chain expected by the +status chain set needs an entry; startup logs a loud warning with the missing +chain names, and their status remains fail-closed. Ethereum mainnet is also +required for live ENS verified resolution and the ENS/60 primary-name +on-demand reverse/forward RPC fallback, configured as `BIGNAME_API_CHAIN_RPC_URLS=ethereum-mainnet=`. `GET /v1/profiles/names/{name}` in `mode=verified|both`, `GET /v1/names/{namespace}/{name}/records` when it needs verified values, `GET /v2/names/{name}?source=verified`, and @@ -598,8 +633,16 @@ Hash-pinned backfill execution batches each reserved range into `BIGNAME_INDEXER_HASH_PINNED_BACKFILL_CHUNK_BLOCKS`-sized chunks. The default server profile uses `1024` blocks. Larger chunks reduce checkpoint churn and RPC round trips during long historical bootstrap, while also increasing the amount -of range work retried after a failed chunk. Raw-only sparse backfill also caps -each materialized push with +of range work retried after a failed chunk. During automatic startup only, each +configured chunk is executed as progress units of at most 32 blocks and the +indexer heartbeat advances after each completed unit; manual backfills retain +the configured chunk as their execution unit. The startup adapter pass then +advances that same heartbeat after checkpoint stream pages and bounded +discovery, identity, binding, and normalized-event finalization batches, so a +large materialization stays live without a free-running timer masking a stuck +operation. Live manifest and discovery refresh adapter passes use the same +checkpoint-page callbacks and family-boundary beats. Raw-only sparse backfill +also caps each materialized push with `BIGNAME_INDEXER_HASH_PINNED_BACKFILL_MAX_LOGS_PER_PUSH` so dense log spans are split before transaction and receipt fetch/persist work. The older `BIGNAME_INDEXER_HASH_PINNED_BACKFILL_MAX_LOGS_PER_RANGE` name is still accepted diff --git a/docs/development.md b/docs/development.md index f050fcba..c6299f83 100644 --- a/docs/development.md +++ b/docs/development.md @@ -95,9 +95,11 @@ execution uses the selected exact-name snapshot: no `at` and no checkpoint, and the API call targets that selected block rather than provider latest. -Configure `BIGNAME_API_CHAIN_RPC_URLS=ethereum-mainnet=` for the API -process before relying on live ENS verified resolution or the ENS/60 -primary-name on-demand reverse/forward RPC fallback. This is separate from +Configure `BIGNAME_API_CHAIN_RPC_URLS` for every chain expected by the status +chain set before relying on `/v1/status` or `/v2/status`; startup warns with +the exact missing chain names and their readiness stays fail-closed. Include +`ethereum-mainnet=` before relying on live ENS verified resolution or +the ENS/60 primary-name on-demand reverse/forward RPC fallback. This is separate from `BIGNAME_INDEXER_CHAIN_RPC_URLS`, which feeds indexer intake and checkpoint state only. If the API Ethereum provider is not configured, supported live ENS verified selectors fail closed instead of falling back to declared record cache: @@ -135,11 +137,12 @@ The API process exposes `GET /healthz` on the same bind address as `cargo api -- serve` and `./scripts/dev-up`. The default local address is `http://127.0.0.1:3000/healthz`. -`/healthz` is an unversioned operator endpoint. The production compose probe +`/healthz` is an unversioned operator contract endpoint. The production compose probe connects to `127.0.0.1` inside the API container even though the process listens on its configured bind address (`0.0.0.0:3000` by default in compose); the public Caddy edge does not expose it. It is not part of the versioned `/v1` read -API and should not be treated as a consumer compatibility surface. +API and should not be treated as a consumer compatibility surface. Its frozen +response may grow additively. The response's `identity` object describes the binary's compatibility inputs: `version` is the Cargo package version; `build_sha` is the compile-time @@ -152,17 +155,67 @@ compatibility version for the published permissions read model. These are binary identity values, not a report of the database's applied migration or replay progress. -The endpoint separates process readiness from database readiness: +The endpoint separates API-process readiness, database readiness, and +indexer/worker main-loop liveness: -- Healthy database: `200 OK`, top-level `status` is `ready`, +- Healthy database and recent indexer and worker loop heartbeats: `200 OK`, + top-level `api_status` and aggregate `status` are `ready`, `process.status` is `running`, `database.status` is `reachable`, `database.reachable` is `true`, `database.check` is `select_1`, and - `database.error` is `null`. + `database.error` is `null`. `loops.indexer.status` and + `loops.worker.status` are `running` and each includes `started_at`, + `heartbeat_at`, `heartbeat_age_seconds`, and `max_age_seconds`. - Unreachable database or pool: `503 Service Unavailable`, top-level `status` - is `degraded`, `process.status` remains `running`, `database.status` is + and `api_status` are `degraded`, `process.status` remains `running`, `database.status` is `unreachable`, `database.reachable` is `false`, `database.check` remains - `select_1`, and `database.error` is `database readiness query failed`. + `select_1`, `database.error` is `database readiness query failed`, and both + loop statuses are `unavailable`. +- Reachable database with a missing or old loop heartbeat: `200 OK` with + `api_status=ready` and aggregate `status=degraded`. This keeps API container + and public-edge readiness local to the serving process and database while + retaining the indexer/worker failure in the payload. A missing row is + `not_started`; a row older than + `BIGNAME_API_HEARTBEAT_MAX_AGE_SECS` is `stale`. The default maximum age is + 20 seconds, four times the default five-second indexer and worker loop + intervals. Database reachability is checked with `SELECT 1` through the configured -PostgreSQL pool. A degraded response means the API process handled the request, -but the configured database pool could not satisfy the readiness query. +PostgreSQL pool. `api_status` is API-local readiness: because the handler is +serving by definition, it is `ready` exactly when that query succeeds, and the +HTTP status follows this field. Aggregate `status` additionally requires both +service loops and may therefore be `degraded` in an HTTP 200 response. +The API prefers the retained service instance with a currently healthy normal +heartbeat or long-operation phase, using the newest stale evidence only when +none is healthy. A deployment runs one active writer for each service; process +rows retained across a restart make that selection robust during the handoff +without authorizing concurrent workers. The indexer and worker `healthcheck` +subcommands instead validate the process row for their own +`BIGNAME_HEARTBEAT_INSTANCE_ID`. Local binaries fall +back to `HOSTNAME`; the server compose file pins stable `indexer` and `worker` +identities so a recreated container refreshes the same process row. The checks +fail when the row is absent or older than the service-specific +`BIGNAME_INDEXER_HEARTBEAT_MAX_AGE_SECS` or +`BIGNAME_WORKER_HEARTBEAT_MAX_AGE_SECS` limit. This distinguishes a loop that +never registered from one that registered and then stopped advancing. Worker +bootstrap replay and projection apply refresh the process row only at actual +progress boundaries; the indexer registers before startup bootstrap and +refreshes after completed hash-pinned progress units of at most 32 blocks +inside the configured checkpoint chunk, then after completed startup adapter +checkpoint stream pages and bounded discovery, identity, binding, and +normalized-event finalization batches. Live manifest and discovery refresh +adapter passes reuse those checkpoint-page callbacks and family-boundary beats. +A free-running heartbeat task does not +keep a stuck operation healthy. Worker rebuilds refresh at their +existing projection batch boundaries. A monolithic worker SQL or hydration +operation instead sets `loops.worker.phase` and uses +`BIGNAME_API_WORKER_REBUILD_PHASE_MAX_AGE_SECS` (default 43,200) in the API and +`BIGNAME_WORKER_REBUILD_PHASE_MAX_AGE_SECS` in the worker container check. The +named phase is removed when normal bounded progress resumes; graceful shutdown +deregisters the instance, fencing further phase writes and removing all of its +heartbeat rows before exit. Registering a new single-writer worker retires a +dead predecessor's phase. If the process exits without either cleanup path and +no replacement registers, the retained phase becomes `stale` at that separate +limit. +Failure to persist the phase start aborts that rebuild attempt before its +monolithic operation begins; ordinary bounded-progress beat failures still +warn and retry at the next progress boundary. diff --git a/docs/runbooks/release.md b/docs/runbooks/release.md index 8e5abd44..0d517546 100644 --- a/docs/runbooks/release.md +++ b/docs/runbooks/release.md @@ -6,8 +6,8 @@ the configured PostgreSQL database, local pinned upstream refs, generated OpenAP artifact consistency, and the conformance ownership table for published OpenAPI paths, runs focused reorg chaos, capability, and resolver-profile conformance guards, runs the live manifest-drift audit with worker-owned alert observation -persistence, inspects the runtime [watch plan](../glossary.md), and checks the API process -readiness endpoint from a prebuilt local binary. It then validates the public +persistence, inspects the runtime [watch plan](../glossary.md), and checks the +API health contract from a prebuilt local binary. It then validates the public edge configuration and checks the allowlist through an ephemeral Caddy container in front of that API process. It does not deploy, contact external RPC providers, contact GitHub or Fly, or validate a remote production target. @@ -43,7 +43,7 @@ scripts/release-smoke --help reorg chaos and dynamic resolver-profile conformance guards need a local PostgreSQL server where they can create, migrate, and drop temporary test databases; migrations, the manifest-drift audit and inspection path, runtime - watch-plan inspection, and readiness all use the configured database even + watch-plan inspection, and API health all use the configured database even when `--no-network` is passed. - The checked-in migration that creates `manifest_alert_observations` must have run before manifest-drift smoke checks can persist or read alert observations. @@ -72,9 +72,9 @@ scripts/release-smoke --help `http://127.0.0.1:3001`; set `BIGNAME_SMOKE_PUBLIC_EDGE_URL` when that port is already in use. The ephemeral Caddy container uses host networking and no named volumes, then is removed when the check exits. -- The readiness check builds `bigname-api` before starting the probe window, +- The health-contract check builds `bigname-api` before starting the probe window, then runs the compiled binary from Cargo's local target directory directly. - Slow local compilation therefore fails or completes before readiness polling + Slow local compilation therefore fails or completes before health polling begins; the 30 one-second probes measure server startup and health only. - For `--no-network`, Cargo dependencies and the selected Caddy image must already be cached locally. @@ -121,11 +121,16 @@ environment variables above take precedence over values loaded from `.env`. 10. Runs `cargo run --locked -p bigname-worker -- inspect watch-plan --json` against the configured database as a read-only runtime watch-plan inspection. 11. Runs `cargo build --locked -p bigname-api --bin bigname-api` so API compile - time is outside the readiness probe window. + time is outside the health probe window. 12. Starts the compiled `bigname-api serve --bind-addr ` binary directly from Cargo's local target - directory and probes `/healthz` until it returns `200` with - `"status":"ready"`. + directory and probes `/healthz`. A database with live indexer and worker + loops must return `200` with `"status":"ready"`. The standalone smoke API + has no service loops of its own, so it also accepts `200` with + `"api_status":"ready"`, aggregate `"status":"degraded"`, a running API + process, a reachable database, and each loop either `running` or + `not_started`. `not_started` is the explicit standalone exception. `stale` + proves a heartbeat row exists and fails the gate, as does `unavailable`. 13. Validates `docker/caddy/Caddyfile`, starts an ephemeral Caddy container in front of that API, and runs `scripts/public-edge-smoke`. The check proves the mounted v2 routes and GraphiQL succeed on the internal listener but @@ -195,7 +200,8 @@ A passing gate means: from the configured local database; - the API binary builds locally from this revision; - the API process can start from that built binary; and -- the process readiness endpoint reports ready against that database; +- `/healthz` reports either full readiness with live service loops or the + documented API-local-ready standalone state caused only by absent loops; - the repository Caddyfile validates and starts in an ephemeral container; and - allowed public-edge requests reach the API while v2, GraphiQL, and unknown paths return the expected public `404` responses. @@ -244,11 +250,12 @@ Any non-zero exit blocks the release candidate until triaged. until database reachability, manifest/discovery state, or the inspection command failure is triaged. - API prebuild failure: the local `bigname-api` binary could not be built before - readiness probing. Do not promote until the compile failure or missing offline + health probing. Do not promote until the compile failure or missing offline cache is triaged. -- Readiness failure: the API did not stay up or `/healthz` did not report - ready after the binary was built and started directly. Do not promote until - the API logs and database reachability explain the failure. +- Health-contract failure: the API did not stay up, the database was + unreachable, loop liveness could not be read, or `/healthz` returned neither + full readiness nor the documented standalone degraded state. Do not promote + until the API logs and database/loop evidence explain the failure. - Public-edge failure: the Caddyfile did not validate, the ephemeral Caddy container did not start, an admitted helper, v1 REST, GraphQL, or preflight request failed, or a denied v2, GraphiQL, or unknown-path request did not @@ -289,13 +296,13 @@ checks while adding the local pinned upstream-ref check, focused reorg chaos conformance guard, no-Postgres OpenAPI conformance-owner guard, focused capability cutover evidence guard, focused dynamic resolver-profile conformance guard, live manifest-drift audit with worker-owned alert persistence, runtime -watch-plan inspection, local API prebuild plus readiness, and the public-edge +watch-plan inspection, local API prebuild plus health-contract checks, and the public-edge allowlist check. CI pulls `caddy:2-alpine` before entering the no-network smoke phase; the gate then uses that cached image with pulling disabled. It uses loopback-only smoke URLs, offline Cargo execution, the checked-out `.refs/` state, and the configured local PostgreSQL server/database for reorg chaos and dynamic resolver-profile temporary databases, migrations, manifest-drift audit, -watch-plan inspection, API prebuild, readiness, and the internal side of the +watch-plan inspection, API prebuild, health-contract checks, and the internal side of the edge comparison. A CI failure has the same release-blocking meaning as a local non-zero exit, except that missing cached dependencies are a CI environment issue rather than a product regression. diff --git a/docs/runbooks/rollback.md b/docs/runbooks/rollback.md index e8624362..30981d56 100644 --- a/docs/runbooks/rollback.md +++ b/docs/runbooks/rollback.md @@ -6,8 +6,9 @@ configured PostgreSQL database, local pinned upstream refs, generated OpenAPI artifact consistency, migration idempotence, the conformance ownership table for published OpenAPI paths, runs focused reorg chaos, capability, and resolver-profile conformance guards, runs the live manifest-drift audit with -worker-owned alert observation persistence, inspects the runtime [watch plan](../glossary.md), and -checks the API process readiness endpoint from a prebuilt local binary. It does +worker-owned alert observation persistence, inspects the runtime [watch +plan](../glossary.md), and checks the API health contract from a prebuilt local +binary. It does not perform the production rollback, deploy, contact external RPC providers, contact GitHub or Fly, or validate a remote production target. @@ -41,7 +42,7 @@ scripts/rollback-smoke --help reorg chaos and dynamic resolver-profile conformance guards need a local PostgreSQL server where they can create, migrate, and drop temporary test databases; migrations, the manifest-drift audit and inspection path, runtime - watch-plan inspection, and readiness all use the configured database even + watch-plan inspection, and API health all use the configured database even when `--no-network` is passed. - The checked-in migration that creates `manifest_alert_observations` must have run before manifest-drift smoke checks can persist or read alert observations. @@ -53,9 +54,9 @@ scripts/rollback-smoke --help - `BIGNAME_SMOKE_API_HEALTH_URL` is reachable from the operator host. By default it is derived from `BIGNAME_SMOKE_API_BIND_ADDR` as `http:///healthz`. -- The readiness check builds `bigname-api` before starting the probe window, +- The health-contract check builds `bigname-api` before starting the probe window, then runs the compiled binary from Cargo's local target directory directly. - Slow local compilation therefore fails or completes before readiness polling + Slow local compilation therefore fails or completes before health polling begins; the 30 one-second probes measure server startup and health only. - For `--no-network`, Cargo dependencies must already be cached locally. @@ -102,11 +103,16 @@ The script loads `.env` when it exists, then uses the environment values above. 11. Runs `cargo run --locked -p bigname-worker -- inspect watch-plan --json` against the configured database as a read-only runtime watch-plan inspection. 12. Runs `cargo build --locked -p bigname-api --bin bigname-api` so API compile - time is outside the readiness probe window. + time is outside the health probe window. 13. Starts the compiled `bigname-api serve --bind-addr ` binary directly from Cargo's local target - directory and probes `/healthz` until it returns `200` with - `"status":"ready"`. + directory and probes `/healthz`. A database with live indexer and worker + loops must return `200` with `"status":"ready"`. The standalone smoke API + also accepts `200` only when `api_status` is `ready`, aggregate `status` is + `degraded`, the API process is running, the database is reachable, and each + loop is `running` or `not_started`. `not_started` is the explicit standalone + exception. A `stale` loop has a persisted heartbeat row and fails the gate, + as does `unavailable`. With `--no-network`, the script also sets `CARGO_NET_OFFLINE=true`, passes `--offline` to Cargo, and rejects non-loopback smoke bind or health URLs. The @@ -162,7 +168,8 @@ A passing gate means: from the configured local database; - the API binary builds locally from the rollback checkout; - the API process can start from that built binary; and -- the unversioned readiness endpoint reports ready against that database. +- `/healthz` reports either full readiness with live service loops or the + documented API-local-ready standalone state caused only by absent loops. ## Failure Criteria @@ -213,12 +220,13 @@ Any non-zero exit blocks automatic rollback promotion until triaged. the rollback checkout until database reachability, manifest/discovery state, or the inspection command failure is triaged. - API prebuild failure: the local `bigname-api` binary could not be built before - readiness probing. Do not promote the rollback checkout until the compile + health probing. Do not promote the rollback checkout until the compile failure or missing offline cache is triaged. -- Readiness failure: the rollback API did not stay up or `/healthz` did not - report ready after the binary was built and started directly. Do not treat the - rollback as service-restoring until the API logs and database reachability - explain the failure. +- Health-contract failure: the rollback API did not stay up, the database was + unreachable, loop liveness could not be read, or `/healthz` returned neither + full readiness nor the documented standalone degraded state. Do not treat + the rollback as service-restoring until the API logs and database/loop + evidence explain the failure. - No-network failure: the gate was not fully local, the bind or health URL was not loopback, or Cargo could not build from its local cache. Fix the operator environment before treating it as a rollback-candidate failure. @@ -249,7 +257,7 @@ state that represent the rolled-back service when local access is available. A passing local gate is not a substitute for production health checks; it confirms only the local migration, artifact, pinned-ref, reorg chaos, conformance-owner, capability-cutover, dynamic resolver-profile, manifest-drift audit persistence, -watch-plan inspection, API prebuild, and readiness behaviors covered above. +watch-plan inspection, API prebuild, and health-contract behaviors covered above. Do not use this gate as proof of external integration health. It intentionally does not exercise deploy commands, external RPC, GitHub, Fly, or remote @@ -269,10 +277,10 @@ conformance guard, double migration idempotence check, the no-Postgres OpenAPI conformance-owner guard, focused capability cutover evidence guard, focused dynamic resolver-profile conformance guard, live manifest-drift audit with worker-owned alert persistence, runtime watch-plan inspection, and local API -prebuild plus readiness. It uses loopback-only smoke URLs, offline Cargo +prebuild plus health-contract checks. It uses loopback-only smoke URLs, offline Cargo execution, the checked-out `.refs/` state, and the configured local PostgreSQL server/database for reorg chaos and dynamic resolver-profile temporary databases, migrations, manifest-drift audit, watch-plan inspection, API -prebuild, and readiness. A CI failure has the same rollback-blocking meaning as +prebuild, and health-contract checks. A CI failure has the same rollback-blocking meaning as a local non-zero exit, except that missing cached dependencies are a CI environment issue rather than a product regression. diff --git a/docs/storage.md b/docs/storage.md index 61be8a62..8bd98ba5 100644 --- a/docs/storage.md +++ b/docs/storage.md @@ -487,10 +487,70 @@ For ENSv2, `resource_id` keys by `(chain_id, registry_contract_instance_id, upst | `current_projection_replay_status` | projection workers; ratified storage correction tooling may clear affected markers when it deletes projection rows | durable operational completion markers for bootstrap/full all-current projection replay | | `projection_normalized_event_changes` | normalized-event storage trigger; projection workers consume | append-only downstream change log for normalized-event inserts and canonicality-state updates, consumed through finite, bounded-wait complete-prefix captures | | `projection_apply_cursors`, `projection_invalidations`, `projection_invalidation_dead_letters` | projection workers; storage trigger for projection-relevant `surface_bindings` repairs; bounded normalized-event adapter repair invalidations | durable projection apply watermarks, live key-scoped projection invalidation queue, and terminal operator-visible dead-letter records | +| `service_loop_heartbeats` | indexer and worker main loops | durable per-process loop liveness, per-chain indexer loop liveness, and named worker long-operation phases; operational health evidence only, not chain progress or API read-model data | | `execution_*` | execution workers; API on-demand verified-resolution cache misses for documented product routes; synchronous indexer/reorg repair for orphan-block cache outcome deletes only | durable traces and steps, normal `execution_cache_outcomes` writes, invalidation records | The API process is otherwise read-only against storage. +`service_loop_heartbeats` identifies a service instance by `service_name` and +`instance_id`. Registering the process-scoped row retires every same-service +non-process row before it resets `started_at`; stale chain scopes and a prior +worker's unfinished phase therefore cannot survive a single-writer service +handoff. Process rows remain available to rank instances during that handoff. +The supported deployment has one active writer for each service. Each main-loop tick +advances `heartbeat_at` for its process row. The indexer registers this row +immediately after opening its database pool, before startup bootstrap, and +advances the process plus deduplicated chain rows after completed hash-pinned +bootstrap progress units of at most 32 blocks and after completed startup +adapter checkpoint stream pages and bounded discovery, identity, binding, and +normalized-event finalization batches. Live manifest and discovery refreshes +reuse those checkpoint-page progress callbacks and family-boundary beats. This +does not change the configured 1,024-block default checkpoint boundary for +non-startup backfills. The worker +advances the process row after bounded +[projection](glossary.md) rebuild batches and projection-apply units. This +keeps long, actively progressing work live without using a detached timer that +could mask a stuck operation. A missing +process-scoped row therefore means that instance's loop never registered or +gracefully deregistered, while a present row older than the configured maximum +age means the loop stopped or wedged after starting. For each service, the API +prefers an instance whose normal heartbeat or active long-operation phase is +within its configured age, then falls back to the newest stale evidence when +none is healthy. One live instance can therefore satisfy shared readiness +without being hidden by a newer retained instance that stopped. Each +container's `healthcheck` subcommand reads its own instance row, so another +instance cannot hide a stopped process. These rows are mutable operational signals. They are +not raw facts, [replay](glossary.md) checkpoints, chain checkpoints, or +projection freshness evidence. + +Full worker rebuild heartbeat routes are explicit: + +| Rebuild step | Bounded progress heartbeat | Named monolithic phases | +| --- | --- | --- | +| `name_current` | each completed name task; staged writes remain in 2,000-row batches | `name_current.load_inputs`, `name_current.publish` | +| `children_current` | each completed declared-child source; staged writes remain in 2,000-row batches | `children_current.count_existing`, `children_current.publish`, `children_current.count_published_parents` | +| `permissions_current` | each completed resource task; staged writes remain in 2,000-row batches | `permissions_current.count_existing`, `permissions_current.publish` | +| `record_inventory_current` | each completed resource task, staged in 500-row batches; text hydration also beats after each bounded 500-row page | `record_inventory_current.count_existing`, `record_inventory_current.publish` | +| `resolver_current` | each completed resolver target; staged writes remain in 1,000-row batches | `resolver_current.load_profile`, `resolver_current.load_targets`, `resolver_current.count_existing`, `resolver_current.publish` | +| `address_names_current` | each completed surface binding; staged writes remain in 2,000-row batches | `address_names_current.prepare`, `address_names_current.publish`, `address_names_current.count_published_addresses` | +| `primary_names_current` | each streamed tuple; legacy hydration beats during 1,000-candidate planning, configured provider batches, resolver-edge batches, and 1,000-row upserts | `primary_names_current.count_existing`, `primary_names_current.invalidate_execution_cache`, `primary_names_current.publish`, `primary_names_current.legacy_hydration.load_reverse_claim_candidates`, `primary_names_current.legacy_hydration.load_resolver_edge_candidates` | + +A named phase is a distinct `scope_kind='phase'` row for the worker instance. +Its `heartbeat_at` is the phase start rather than a free-running timer, so a +crash or wedge still ages out. Worker and API checks use the separately +environment-tunable phase maximum (default 43,200 seconds) only while that row +exists; completing the phase removes it and refreshes ordinary process +evidence. Graceful worker shutdown first deletes its process row as a write +fence, then deletes the instance's remaining heartbeat rows; a new same-service +registration also clears phases left by a predecessor that exited without +running the hook. Ordinary worker heartbeat-write failures warn and +continue so a transient database write failure degrades liveness evidence and +remains due for retry, rather than converting the database blip into worker +restart churn. A +named phase is different: if its start marker cannot be persisted, the current +rebuild attempt fails before starting the monolithic work, so the worker never +runs a many-hour operation without the evidence used to interpret it. + Within `execution_*`, the API may write traces, steps, and normal `execution_cache_outcomes` only for documented on-demand verified-resolution product routes when a selected-snapshot cache miss is live-executed and diff --git a/migrations/20260721130000_service_loop_heartbeats.sql b/migrations/20260721130000_service_loop_heartbeats.sql new file mode 100644 index 00000000..39bf0d0a --- /dev/null +++ b/migrations/20260721130000_service_loop_heartbeats.sql @@ -0,0 +1,25 @@ +CREATE TABLE service_loop_heartbeats ( + service_name TEXT NOT NULL, + instance_id TEXT NOT NULL, + scope_kind TEXT NOT NULL, + scope_id TEXT NOT NULL, + started_at TIMESTAMPTZ NOT NULL, + heartbeat_at TIMESTAMPTZ NOT NULL, + PRIMARY KEY (service_name, instance_id, scope_kind, scope_id), + CHECK (service_name IN ('indexer', 'worker')), + CHECK (btrim(instance_id) <> ''), + CHECK ( + (scope_kind = 'process' AND scope_id = 'process') + OR ( + service_name = 'indexer' + AND scope_kind = 'chain' + AND btrim(scope_id) <> '' + AND scope_id <> 'process' + ) + ), + CHECK (heartbeat_at >= started_at) +); + +CREATE INDEX service_loop_heartbeats_latest_process_idx + ON service_loop_heartbeats (service_name, heartbeat_at DESC, instance_id) + WHERE scope_kind = 'process' AND scope_id = 'process'; diff --git a/migrations/20260721140000_service_loop_phase_heartbeats.sql b/migrations/20260721140000_service_loop_phase_heartbeats.sql new file mode 100644 index 00000000..4349647d --- /dev/null +++ b/migrations/20260721140000_service_loop_phase_heartbeats.sql @@ -0,0 +1,44 @@ +DO $$ +DECLARE + existing_scope_constraint TEXT; +BEGIN + SELECT constraint_row.conname + INTO existing_scope_constraint + FROM pg_constraint AS constraint_row + WHERE constraint_row.conrelid = 'service_loop_heartbeats'::regclass + AND constraint_row.contype = 'c' + AND pg_get_constraintdef(constraint_row.oid) LIKE '%scope_kind%' + ORDER BY constraint_row.conname + LIMIT 1; + + IF existing_scope_constraint IS NULL THEN + RAISE EXCEPTION 'service_loop_heartbeats scope constraint was not found'; + END IF; + + EXECUTE format( + 'ALTER TABLE service_loop_heartbeats DROP CONSTRAINT %I', + existing_scope_constraint + ); +END +$$; + +ALTER TABLE service_loop_heartbeats + ADD CONSTRAINT service_loop_heartbeats_scope_check CHECK ( + (scope_kind = 'process' AND scope_id = 'process') + OR ( + service_name = 'indexer' + AND scope_kind = 'chain' + AND btrim(scope_id) <> '' + AND scope_id <> 'process' + ) + OR ( + service_name = 'worker' + AND scope_kind = 'phase' + AND btrim(scope_id) <> '' + AND scope_id <> 'process' + ) + ); + +CREATE INDEX service_loop_heartbeats_active_phase_idx + ON service_loop_heartbeats (service_name, instance_id, heartbeat_at DESC) + WHERE scope_kind = 'phase'; diff --git a/scripts/public-edge-smoke b/scripts/public-edge-smoke index fb7dec06..f6c25374 100755 --- a/scripts/public-edge-smoke +++ b/scripts/public-edge-smoke @@ -25,6 +25,8 @@ Environment: BIGNAME_SMOKE_CADDY_IMAGE Caddy image, default caddy:2-alpine. BIGNAME_SMOKE_INTERNAL_REQUEST_TIMEOUT_SECS Internal request timeout, default 60. + BIGNAME_SMOKE_EXPECTED_HEALTH_STATUS + Expected /healthz status, 200 (default) or 503. USAGE } @@ -87,6 +89,7 @@ internal_api_url="${BIGNAME_SMOKE_INTERNAL_API_URL:-http://127.0.0.1:3000}" public_edge_url="${BIGNAME_SMOKE_PUBLIC_EDGE_URL:-http://127.0.0.1:3001}" caddy_image="${BIGNAME_SMOKE_CADDY_IMAGE:-caddy:2-alpine}" internal_request_timeout_secs="${BIGNAME_SMOKE_INTERNAL_REQUEST_TIMEOUT_SECS:-60}" +expected_health_status="${BIGNAME_SMOKE_EXPECTED_HEALTH_STATUS:-200}" internal_api_url="${internal_api_url%/}" public_edge_url="${public_edge_url%/}" smoke_tmp="$(mktemp -d -t bigname-public-edge-smoke.XXXXXX)" @@ -97,6 +100,11 @@ if ! [[ "$internal_request_timeout_secs" =~ ^[1-9][0-9]{0,3}$ ]] || fail "BIGNAME_SMOKE_INTERNAL_REQUEST_TIMEOUT_SECS must be an integer from 1 to 3600" fi +case "$expected_health_status" in + 200|503) ;; + *) fail "BIGNAME_SMOKE_EXPECTED_HEALTH_STATUS must be 200 or 503" ;; +esac + loopback_http_port() { local url="$1" local port @@ -261,7 +269,7 @@ unknown_path='/public-edge-smoke-unknown' manager_origin='https://app.ens.dev' log "proving denied API routes exist on the internal listener" -assert_status internal-health 200 GET "${internal_api_url}/healthz" '' +assert_status internal-health "$expected_health_status" GET "${internal_api_url}/healthz" '' assert_status internal-v2-status 200 GET "${internal_api_url}/v2/status" '' assert_status internal-v2-lookup 200 POST "${internal_api_url}/v2/lookup" "$v2_lookup_body" assert_status internal-v2-diagnostic 200 GET "${internal_api_url}${diagnostic_path}" '' diff --git a/scripts/release-smoke b/scripts/release-smoke index 46b370ea..c0323c5c 100755 --- a/scripts/release-smoke +++ b/scripts/release-smoke @@ -20,7 +20,7 @@ Runs the local release smoke gate: - live manifest-drift audit - runtime watch-plan inspection - prebuilt local API binary - - local API /healthz readiness + - local API /healthz contract (ready with live loops, honestly degraded without them) - public-edge allowlist through an ephemeral Caddy container Environment: @@ -142,10 +142,11 @@ assert_no_network_mode() { ;; esac - log "no-network mode enabled: cargo is offline and readiness uses loopback only" + log "no-network mode enabled: cargo is offline and API health uses loopback only" } api_pid="" +health_http_status="" target_dir="${CARGO_TARGET_DIR:-target}" if [ -n "${CARGO_BUILD_TARGET:-}" ]; then api_bin="${target_dir}/${CARGO_BUILD_TARGET}/debug/bigname-api" @@ -212,7 +213,7 @@ run_watch_plan_inspection_gate() { } build_api_for_readiness() { - log "building API binary for readiness probe" + log "building API binary for health probe" cargo build "${cargo_args[@]}" -p bigname-api --bin bigname-api [ -x "$api_bin" ] || fail "expected built API binary at ${api_bin}" } @@ -220,7 +221,7 @@ build_api_for_readiness() { run_readiness_check() { require_command curl - log "starting API for readiness probe on ${BIGNAME_SMOKE_API_BIND_ADDR}" + log "starting API for health probe on ${BIGNAME_SMOKE_API_BIND_ADDR}" "$api_bin" serve \ --bind-addr "$BIGNAME_SMOKE_API_BIND_ADDR" \ > "$api_log_tmp" 2>&1 & @@ -229,7 +230,7 @@ run_readiness_check() { for _attempt in $(seq 1 30); do if ! kill -0 "$api_pid" 2>/dev/null; then sed -n '1,160p' "$api_log_tmp" >&2 || true - fail "API exited before readiness probe completed" + fail "API exited before health probe completed" fi http_status="$( @@ -240,7 +241,22 @@ run_readiness_check() { )" if [ "$http_status" = "200" ] && grep -q '"status":"ready"' "$health_body_tmp"; then - log "readiness probe passed" + health_http_status="$http_status" + log "full readiness probe passed" + return 0 + fi + + # A standalone smoke database has no indexer/worker processes, so not_started is admissible. + # A stale status proves a heartbeat row exists and must fail instead of masking a wedged loop. + if [ "$http_status" = "200" ] \ + && grep -q '"status":"degraded"' "$health_body_tmp" \ + && grep -q '"api_status":"ready"' "$health_body_tmp" \ + && grep -q '"process":{"status":"running"}' "$health_body_tmp" \ + && grep -q '"database":{"status":"reachable","reachable":true' "$health_body_tmp" \ + && grep -Eq '"indexer":{"status":"(running|not_started)"' "$health_body_tmp" \ + && grep -Eq '"worker":{"status":"(running|not_started)"' "$health_body_tmp"; then + health_http_status="$http_status" + log "standalone health contract passed; absent service loops are explicitly not_started" return 0 fi @@ -249,7 +265,7 @@ run_readiness_check() { sed -n '1,160p' "$api_log_tmp" >&2 || true sed -n '1,80p' "$health_body_tmp" >&2 || true - fail "API readiness probe failed at ${BIGNAME_SMOKE_API_HEALTH_URL}" + fail "API health probe failed at ${BIGNAME_SMOKE_API_HEALTH_URL}" } run_public_edge_check() { @@ -262,11 +278,13 @@ run_public_edge_check() { log "checking the public-edge allowlist through Caddy" BIGNAME_SMOKE_INTERNAL_API_URL="$BIGNAME_SMOKE_INTERNAL_API_URL" \ BIGNAME_SMOKE_INTERNAL_REQUEST_TIMEOUT_SECS="$BIGNAME_SMOKE_INTERNAL_REQUEST_TIMEOUT_SECS" \ + BIGNAME_SMOKE_EXPECTED_HEALTH_STATUS="$health_http_status" \ scripts/public-edge-smoke "${edge_args[@]}" } require_command cargo require_command diff +require_command grep require_command seq assert_no_network_mode run_pinned_refs_check diff --git a/scripts/rollback-smoke b/scripts/rollback-smoke index bca020fe..0287d8e8 100755 --- a/scripts/rollback-smoke +++ b/scripts/rollback-smoke @@ -20,7 +20,7 @@ Runs the local rollback smoke gate: - live manifest-drift audit - runtime watch-plan inspection - prebuilt local API binary - - local API /healthz readiness + - local API /healthz contract (ready with live loops, honestly degraded without them) Environment: BIGNAME_DATABASE_URL or DATABASE_URL PostgreSQL connection URL. @@ -114,7 +114,7 @@ assert_no_network_mode() { ;; esac - log "no-network mode enabled: cargo is offline and readiness uses loopback only" + log "no-network mode enabled: cargo is offline and API health uses loopback only" } api_pid="" @@ -184,7 +184,7 @@ run_watch_plan_inspection_gate() { } build_api_for_readiness() { - log "building API binary for readiness probe" + log "building API binary for health probe" cargo build "${cargo_args[@]}" -p bigname-api --bin bigname-api [ -x "$api_bin" ] || fail "expected built API binary at ${api_bin}" } @@ -192,7 +192,7 @@ build_api_for_readiness() { run_readiness_check() { require_command curl - log "starting API for readiness probe on ${BIGNAME_SMOKE_API_BIND_ADDR}" + log "starting API for health probe on ${BIGNAME_SMOKE_API_BIND_ADDR}" "$api_bin" serve \ --bind-addr "$BIGNAME_SMOKE_API_BIND_ADDR" \ > "$api_log_tmp" 2>&1 & @@ -201,7 +201,7 @@ run_readiness_check() { for _attempt in $(seq 1 30); do if ! kill -0 "$api_pid" 2>/dev/null; then sed -n '1,160p' "$api_log_tmp" >&2 || true - fail "API exited before readiness probe completed" + fail "API exited before health probe completed" fi http_status="$( @@ -212,7 +212,20 @@ run_readiness_check() { )" if [ "$http_status" = "200" ] && grep -q '"status":"ready"' "$health_body_tmp"; then - log "readiness probe passed" + log "full readiness probe passed" + return 0 + fi + + # A standalone smoke database has no indexer/worker processes, so not_started is admissible. + # A stale status proves a heartbeat row exists and must fail instead of masking a wedged loop. + if [ "$http_status" = "200" ] \ + && grep -q '"status":"degraded"' "$health_body_tmp" \ + && grep -q '"api_status":"ready"' "$health_body_tmp" \ + && grep -q '"process":{"status":"running"}' "$health_body_tmp" \ + && grep -q '"database":{"status":"reachable","reachable":true' "$health_body_tmp" \ + && grep -Eq '"indexer":{"status":"(running|not_started)"' "$health_body_tmp" \ + && grep -Eq '"worker":{"status":"(running|not_started)"' "$health_body_tmp"; then + log "standalone health contract passed; absent service loops are explicitly not_started" return 0 fi @@ -221,11 +234,12 @@ run_readiness_check() { sed -n '1,160p' "$api_log_tmp" >&2 || true sed -n '1,80p' "$health_body_tmp" >&2 || true - fail "API readiness probe failed at ${BIGNAME_SMOKE_API_HEALTH_URL}" + fail "API health probe failed at ${BIGNAME_SMOKE_API_HEALTH_URL}" } require_command cargo require_command diff +require_command grep require_command seq assert_no_network_mode run_pinned_refs_check diff --git a/scripts/rust-file-size-baseline.toml b/scripts/rust-file-size-baseline.toml index 99323735..e5fd36b0 100644 --- a/scripts/rust-file-size-baseline.toml +++ b/scripts/rust-file-size-baseline.toml @@ -25,17 +25,6 @@ path = "apps/indexer/src/main/backfill/reservation_execution/coinbase_sql.rs" loc = 790 justification = "Phase 7 allowlist for an oversized production file; future changes should split or shrink it." -[[files]] -path = "crates/adapters/src/ens_v1_subregistry_discovery/checkpoint.rs" -loc = 676 -justification = "Phase 7 allowlist for an oversized production file; future changes should split or shrink it." - -[[files]] -path = "crates/adapters/src/ens_v1_unwrapped_authority/checkpoint.rs" -loc = 901 -justification = "Phase 7 allowlist for an oversized production file; future changes should split or shrink it." -review_justification = "Explicit review accepted this file above 900 LOC; future changes should split or shrink it." - [[files]] path = "crates/adapters/src/ens_v1_unwrapped_authority/loading/active_emitters.rs" loc = 892 diff --git a/tests/conformance/README.md b/tests/conformance/README.md index 02e700d0..0f58bfc2 100644 --- a/tests/conformance/README.md +++ b/tests/conformance/README.md @@ -47,7 +47,8 @@ cargo test --manifest-path tests/conformance/Cargo.toml openapi --locked This no-Postgres guard reads `docs/api-v1.openapi.json` and fails if a published path lacks either a conformance harness owner or an explicit out-of-scope reason; -the unversioned `/healthz` endpoint remains out of scope. +the unversioned `/healthz` operator contract remains outside the published path +set and is frozen by the API health/OpenAPI-component tests instead. Focused backfilled-data consumer conformance job, from the repository root: diff --git a/tests/conformance/build.rs b/tests/conformance/build.rs index c59ad033..8c5d1ee9 100644 --- a/tests/conformance/build.rs +++ b/tests/conformance/build.rs @@ -577,6 +577,8 @@ fn rewrite_openapi_components(source: &str) -> String { ); schemas.insert("HealthProcess".to_owned(), health_process_schema()); schemas.insert("HealthDatabase".to_owned(), health_database_schema()); + schemas.insert("HealthLoop".to_owned(), health_loop_schema()); + schemas.insert("HealthLoops".to_owned(), health_loops_schema()); schemas.insert( "HealthResponse".to_owned(), health_response_schema(), diff --git a/tests/conformance/src/conformance/harness.rs b/tests/conformance/src/conformance/harness.rs index 29b5d1c3..50a91682 100644 --- a/tests/conformance/src/conformance/harness.rs +++ b/tests/conformance/src/conformance/harness.rs @@ -373,10 +373,10 @@ } fn app_state(&self) -> AppState { - AppState { - pool: self.pool.clone(), - chain_rpc_urls: bigname_execution::ChainRpcUrls::default(), - } + AppState::new( + self.pool.clone(), + bigname_execution::ChainRpcUrls::default(), + ) } async fn insert_manifest( diff --git a/tests/e2e/README.md b/tests/e2e/README.md index fbbfbd1b..f52d53a8 100644 --- a/tests/e2e/README.md +++ b/tests/e2e/README.md @@ -117,11 +117,12 @@ suite only at an isolated test PostgreSQL server. `register_eth_name::live_worker_applies_registration_and_renewal_while_api_serves` smoke instead keeps the production `worker run` loop active with the indexer and API, proving bootstrap handoff and continuous projection apply - for one registration/renewal path. Execution-plane scenarios start the API - with `--chain-rpc-url ethereum-mainnet=` so on-demand verified - resolution executes against the selected stored snapshot. Backfill helpers - exercise raw-fact-to-normalized-event and projection rebuild boundaries - where canonical checkpoints intentionally make API reads unavailable. + for one registration/renewal path. Every process-backed API session sets + `BIGNAME_API_CHAIN_RPC_URLS` to `chain=` for each served chain, so + status freshness and on-demand verified resolution use the same local + providers as intake. Backfill helpers exercise raw-fact-to-normalized-event + and projection rebuild boundaries where canonical checkpoints intentionally + make API reads unavailable. 5. **Assertions** — each scenario asserts the validation layers material to its claim. Many cover persisted raw logs, canonical normalized events, rebuilt projections, and public HTTP output; the verified-resolution diff --git a/tests/e2e/src/harness/pipeline.rs b/tests/e2e/src/harness/pipeline.rs index bd071416..a420c876 100644 --- a/tests/e2e/src/harness/pipeline.rs +++ b/tests/e2e/src/harness/pipeline.rs @@ -1259,15 +1259,14 @@ pub struct ApiServer { } impl ApiServer { - pub async fn start(repo_root: &Path, database_url: &str) -> Result { - Self::start_with_chain_rpc_urls(repo_root, database_url, &[]).await - } - - pub async fn start_with_chain_rpc_urls( + pub async fn start( repo_root: &Path, database_url: &str, chain_rpc_urls: &[ChainRpcUrl<'_>], ) -> Result { + if chain_rpc_urls.is_empty() { + bail!("e2e API startup requires at least one chain RPC URL"); + } let api = &pipeline_binaries(repo_root).await?.api; let ready_timeout_secs = ready_timeout_secs()?; let deadline = deadline_after(ready_timeout_secs, "API readiness")?; @@ -1331,10 +1330,10 @@ impl ApiServer { "--database-url", database_url, ]); - if !chain_rpc_urls.is_empty() { - let chain_rpc_urls = format_chain_rpc_urls(chain_rpc_urls); - command.args(["--chain-rpc-url", &chain_rpc_urls]); - } + command.env( + "BIGNAME_API_CHAIN_RPC_URLS", + format_chain_rpc_urls(chain_rpc_urls), + ); let child = command .kill_on_drop(true) .spawn() diff --git a/tests/e2e/src/scenarios/catchup_equivalence.rs b/tests/e2e/src/scenarios/catchup_equivalence.rs index bbd871aa..ad8e923a 100644 --- a/tests/e2e/src/scenarios/catchup_equivalence.rs +++ b/tests/e2e/src/scenarios/catchup_equivalence.rs @@ -344,7 +344,7 @@ async fn live_ingest( .await?; live_session.stop().await?; pipeline::worker_replay_all_current_projections(&root, &db.url).await?; - support::serve_existing_db(db, scratch).await + support::serve_existing_db(db, scratch, anvil).await } async fn automatic_catchup( @@ -387,7 +387,7 @@ async fn automatic_catchup( .await?; session.stop().await?; pipeline::worker_replay_all_current_projections(&root, &db.url).await?; - support::serve_existing_db(db, scratch).await + support::serve_existing_db(db, scratch, anvil).await } #[tokio::test] diff --git a/tests/e2e/src/scenarios/cross_protocol.rs b/tests/e2e/src/scenarios/cross_protocol.rs index 1e72acd6..84a0b95e 100644 --- a/tests/e2e/src/scenarios/cross_protocol.rs +++ b/tests/e2e/src/scenarios/cross_protocol.rs @@ -548,7 +548,7 @@ async fn base_reorg_leaves_ethereum_canonicality_untouched() -> Result<()> { eth_checkpoint_after, eth_checkpoint_before, "the ethereum checkpoint must not move during a Base-only reorg" ); - let api = pipeline::ApiServer::start(&root, &db.url).await?; + let api = pipeline::ApiServer::start(&root, &db.url, &chain_rpc_urls).await?; let (status, steady) = api.get_json("/v1/names/ens/steady.eth").await?; assert_eq!(status, 200, "the ethereum name must still serve: {steady}"); assert_eq!( diff --git a/tests/e2e/src/scenarios/perturbations.rs b/tests/e2e/src/scenarios/perturbations.rs index 74173261..d7a5f587 100644 --- a/tests/e2e/src/scenarios/perturbations.rs +++ b/tests/e2e/src/scenarios/perturbations.rs @@ -294,7 +294,7 @@ async fn rich_chain_live_reorg_converges_to_winning_branch() -> Result<()> { "losing block {losing_hash} should retain orphaned raw logs" ); - let reorg_run = support::serve_existing_db(db, scratch).await?; + let reorg_run = support::serve_existing_db(db, scratch, &anvil).await?; assert_exact_resolver(&reorg_run, replacement_resolver.address).await?; let reorg_snapshots = chain_snapshots(&reorg_run, &chain).await?; diff --git a/tests/e2e/src/scenarios/provider_faults.rs b/tests/e2e/src/scenarios/provider_faults.rs index 9aeb3e99..bddb07af 100644 --- a/tests/e2e/src/scenarios/provider_faults.rs +++ b/tests/e2e/src/scenarios/provider_faults.rs @@ -525,7 +525,7 @@ async fn silently_short_logs_are_contained_until_refetch_then_match_control() -> "the correct refetch did not retain both formerly omitted logs" ); pipeline::worker_replay_all_current_projections(&repo_root(), &db.url).await?; - let faulted = support::serve_existing_db(db, scratch).await?; + let faulted = support::serve_existing_db(db, scratch, &anvil).await?; let faulted_snapshots = support::route_snapshots(&faulted, &fixture.subjects()).await?; let control = @@ -646,7 +646,7 @@ async fn transient_provider_faults_and_partial_receipts_recover_to_control() -> "recovered live poll did not retain the target receipt" ); pipeline::worker_replay_all_current_projections(&repo_root(), &db.url).await?; - let faulted = support::serve_existing_db(db, scratch).await?; + let faulted = support::serve_existing_db(db, scratch, &anvil).await?; let subjects = perturb::RouteSnapshotSubjects::new( [name], [format!("{owner:#x}"), format!("{record_target:#x}")], diff --git a/tests/e2e/src/scenarios/register_eth_name.rs b/tests/e2e/src/scenarios/register_eth_name.rs index 94c0af4c..537097da 100644 --- a/tests/e2e/src/scenarios/register_eth_name.rs +++ b/tests/e2e/src/scenarios/register_eth_name.rs @@ -166,7 +166,8 @@ async fn live_worker_applies_registration_and_renewal_while_api_serves() -> Resu WHERE cursor_name = 'normalized_events_to_projection_invalidations')", ) .await?; - let api = pipeline::ApiServer::start(&root, &db.url).await?; + let chain_rpc_urls = [("ethereum-mainnet", anvil.url.as_str())]; + let api = pipeline::ApiServer::start(&root, &db.url, &chain_rpc_urls).await?; let user = rpc.accounts().await?[1]; ens_v1::register_eth_name( diff --git a/tests/e2e/src/scenarios/resolver_records.rs b/tests/e2e/src/scenarios/resolver_records.rs index f4aa7a22..8cdcf3f7 100644 --- a/tests/e2e/src/scenarios/resolver_records.rs +++ b/tests/e2e/src/scenarios/resolver_records.rs @@ -673,7 +673,7 @@ async fn live_code_hash_profile_transition_orphans_and_reactivates_records() -> worker .wait_for_sql(&db.pool, &initial_projection_ready) .await?; - let api = pipeline::ApiServer::start(&root, &db.url).await?; + let api = pipeline::ApiServer::start(&root, &db.url, &chain_rpc_urls).await?; let initial_exact = exact_name(&api, "ens", NAME).await?; assert_resolver(&initial_exact, resolver.address); diff --git a/tests/e2e/src/scenarios/reverse_primary.rs b/tests/e2e/src/scenarios/reverse_primary.rs index 0a5a6956..fe256157 100644 --- a/tests/e2e/src/scenarios/reverse_primary.rs +++ b/tests/e2e/src/scenarios/reverse_primary.rs @@ -26,8 +26,8 @@ fn assert_declared_success(body: &Value, expected_name: &str) { ); } -/// Reverse claims are declared candidates. With no execution RPC configured, -/// `mode=declared` omits verified state and `mode=both` keeps verification +/// Reverse claims are declared candidates. `mode=declared` omits verified +/// state, while `mode=both` keeps a claim without matching forward resolution /// separated as `not_found`. #[tokio::test] async fn reverse_claim_set_changed_then_cleared_tracks_declared_candidate() -> Result<()> { @@ -65,7 +65,7 @@ async fn reverse_claim_set_changed_then_cleared_tracks_declared_candidate() -> R assert_eq!( pointer(&both, "/verified_state/verified_primary_name/status"), "not_found", - "without execution readback, verified primary state should remain absent/not_found; body: {both}" + "a reverse claim without matching forward resolution should remain absent/not_found; body: {both}" ); first.db.cleanup().await?; diff --git a/tests/e2e/src/scenarios/support.rs b/tests/e2e/src/scenarios/support.rs index 013fc6f7..1e47ea33 100644 --- a/tests/e2e/src/scenarios/support.rs +++ b/tests/e2e/src/scenarios/support.rs @@ -30,7 +30,6 @@ async fn ingest_local_chains( chains: &[LocalChain<'_>], mine_margin: bool, ready_sql: Option<&str>, - serve_with_chain_rpc_urls: bool, generate_profile: F, ) -> Result where @@ -66,11 +65,7 @@ where ) .await?; pipeline::worker_replay_all_current_projections(&repo_root, &db.url).await?; - let api = if serve_with_chain_rpc_urls { - pipeline::ApiServer::start_with_chain_rpc_urls(&repo_root, &db.url, &chain_rpc_urls).await? - } else { - pipeline::ApiServer::start(&repo_root, &db.url).await? - }; + let api = pipeline::ApiServer::start(&repo_root, &db.url, &chain_rpc_urls).await?; Ok(PipelineRun { db, api, @@ -189,7 +184,7 @@ pub async fn ingest_and_serve( anvil, id: "ethereum-mainnet", }]; - ingest_local_chains(&chains, true, ready_sql, false, |scratch, repo_root| { + ingest_local_chains(&chains, true, ready_sql, |scratch, repo_root| { manifests::generate_local_profile(scratch, repo_root, &deployment.manifest_targets()) }) .await @@ -208,7 +203,7 @@ pub async fn ingest_at_current_head( anvil, id: "ethereum-mainnet", }]; - ingest_local_chains(&chains, false, ready_sql, false, |scratch, repo_root| { + ingest_local_chains(&chains, false, ready_sql, |scratch, repo_root| { manifests::generate_local_profile(scratch, repo_root, &deployment.manifest_targets()) }) .await @@ -227,7 +222,7 @@ pub async fn ingest_and_serve_with_ens_execution( anvil, id: "ethereum-mainnet", }]; - ingest_local_chains(&chains, true, ready_sql, true, |scratch, repo_root| { + ingest_local_chains(&chains, true, ready_sql, |scratch, repo_root| { let mut targets = deployment.manifest_targets(); targets.insert( "universal_resolver", @@ -247,7 +242,7 @@ pub async fn ingest_basenames_and_serve( anvil: base_anvil, id: "base-mainnet", }]; - ingest_local_chains(&chains, true, ready_sql, false, |scratch, repo_root| { + ingest_local_chains(&chains, true, ready_sql, |scratch, repo_root| { manifests::generate_local_basenames_profile( scratch, repo_root, @@ -266,7 +261,7 @@ pub async fn ingest_basenames_at_current_head( anvil: base_anvil, id: "base-mainnet", }]; - ingest_local_chains(&chains, false, ready_sql, false, |scratch, repo_root| { + ingest_local_chains(&chains, false, ready_sql, |scratch, repo_root| { manifests::generate_local_basenames_profile( scratch, repo_root, @@ -285,7 +280,7 @@ pub async fn ingest_ens_v2_sepolia_and_serve( anvil: sepolia_anvil, id: "ethereum-sepolia", }]; - ingest_local_chains(&chains, true, ready_sql, false, |scratch, repo_root| { + ingest_local_chains(&chains, true, ready_sql, |scratch, repo_root| { manifests::generate_local_sepolia_profile( scratch, repo_root, @@ -316,7 +311,7 @@ pub async fn ingest_mainnet_composed_and_serve( id: "base-mainnet", }, ]; - ingest_local_chains(&chains, true, ready_sql, false, |scratch, repo_root| { + ingest_local_chains(&chains, true, ready_sql, |scratch, repo_root| { manifests::generate_local_mainnet_composed_profile( scratch, repo_root, @@ -414,7 +409,8 @@ where ) .await?; pipeline::worker_replay_all_current_projections(&repo_root, &db.url).await?; - let api = pipeline::ApiServer::start(&repo_root, &db.url).await?; + let chain_rpc_urls = [("ethereum-mainnet", anvil.url.as_str())]; + let api = pipeline::ApiServer::start(&repo_root, &db.url, &chain_rpc_urls).await?; Ok(PipelineRun { db, api, @@ -441,8 +437,13 @@ pub async fn backfill_normalized_events( .await } -pub async fn serve_existing_db(db: HarnessDb, scratch: TempDir) -> Result { - let api = pipeline::ApiServer::start(&repo_root(), &db.url).await?; +pub async fn serve_existing_db( + db: HarnessDb, + scratch: TempDir, + anvil: &Anvil, +) -> Result { + let chain_rpc_urls = [("ethereum-mainnet", anvil.url.as_str())]; + let api = pipeline::ApiServer::start(&repo_root(), &db.url, &chain_rpc_urls).await?; Ok(PipelineRun { db, api,