diff --git a/README.md b/README.md index 30cb270..ba433c7 100644 --- a/README.md +++ b/README.md @@ -1574,10 +1574,13 @@ jamtart/ │ └── bin/ │ └── tart-dash.rs # Terminal UI dashboard ├── migrations/ -│ ├── 001_postgres_schema.sql # Initial schema -│ ├── 002_performance_indexes.sql # Performance indexes -│ ├── 003_frontend_analytics_indexes.sql # Analytics indexes -│ └── 004_frontend_search_indexes.sql # Search indexes +│ ├── 001_core.sql # Extensions, raw events hypertable, nodes, event_types +│ ├── 002_count_tables.sql # 14 per-group count tables (all 115 event types) +│ ├── 003_count_aggregates.sql # _1m/_1h continuous aggregates over count tables +│ ├── 004_union_views.sql # all_event_stats_30s/1m/1h + all_core_stats_1m +│ ├── 005_node_stats_services.sql # node_stats + event_services + their aggregates +│ ├── 006_trackers.sql # convergence, wp_tracking, DA stats & latency hists +│ └── 007_onchain.sql # on-chain core/service/validator stats ├── tests/ │ ├── types_tests.rs # Type system tests │ ├── events_tests.rs # Event encoding tests diff --git a/migrations/002_event_types.sql b/migrations/001_core.sql similarity index 51% rename from migrations/002_event_types.sql rename to migrations/001_core.sql index 172e2f5..43960ef 100644 --- a/migrations/002_event_types.sql +++ b/migrations/001_core.sql @@ -1,7 +1,110 @@ +-- TART baseline schema (squashed from historical migrations 001-024). +-- +-- The old migration history created continuous aggregates with live refresh +-- policies and later DROPped them (006, 015, 020). On a fresh database the +-- policy jobs start running mid-sequence and DROP MATERIALIZED VIEW ... CASCADE +-- deadlocks against the TimescaleDB job scheduler. This baseline creates only +-- the final schema, so no aggregate is ever dropped during migration. +-- +-- RULE FOR FUTURE MIGRATIONS: never DROP a continuous aggregate (or a table +-- with policy jobs) that an EARLIER migration created — on a fresh database +-- its background job may already be running and the DROP can deadlock with +-- the scheduler. If a drop is unavoidable, remove the policies first and +-- accept that the race still exists; prefer additive changes. +-- +-- Column order note: several tables list columns in "historical" order +-- (original columns first, later ALTER TABLE ADD COLUMNs last) so that a +-- fresh database is catalog-identical to one that replayed the old history. + +-- Enable extensions +CREATE EXTENSION IF NOT EXISTS timescaledb; +CREATE EXTENSION IF NOT EXISTS pg_stat_statements; + +-- ============================================================ +-- Nodes table (regular PostgreSQL table - low cardinality, ~1024 rows) +-- ============================================================ +CREATE TABLE IF NOT EXISTS nodes ( + node_id TEXT PRIMARY KEY, + peer_id TEXT NOT NULL, + implementation_name TEXT NOT NULL, + implementation_version TEXT NOT NULL, + node_info JSONB NOT NULL, + connected_at TIMESTAMPTZ NOT NULL, + disconnected_at TIMESTAMPTZ, + last_seen_at TIMESTAMPTZ NOT NULL, + is_connected BOOLEAN DEFAULT true, + event_count BIGINT DEFAULT 0, + total_events BIGINT DEFAULT 0, + address TEXT, + created_at TIMESTAMPTZ DEFAULT CURRENT_TIMESTAMP +); + +CREATE INDEX IF NOT EXISTS idx_nodes_connected ON nodes(is_connected, last_seen_at DESC) WHERE is_connected = true; +CREATE INDEX IF NOT EXISTS idx_nodes_last_seen ON nodes(last_seen_at DESC); + +-- ============================================================ +-- Raw events hypertable: 1h browsing store with hot columns. +-- All 115 event types are written here; aggregation lives in count tables. +-- Historically created as 'events' and renamed; index names keep the +-- original idx_events_* prefix on purpose. +-- ============================================================ +CREATE TABLE IF NOT EXISTS ingested_raw_events ( + timestamp TIMESTAMPTZ NOT NULL, + node_id TEXT NOT NULL, + event_id BIGINT NOT NULL, + event_type SMALLINT NOT NULL, + data JSONB NOT NULL, + created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + -- Hot columns: frequently-queried JSONB fields promoted to real columns + slot INT, + core SMALLINT, + submission_id BIGINT +); + +-- Convert to hypertable with 1-hour chunks +SELECT create_hypertable('ingested_raw_events', 'timestamp', + chunk_time_interval => INTERVAL '1 hour', + create_default_indexes => FALSE, + if_not_exists => TRUE +); + +-- Space partitioning on node_id (32 hash buckets for write distribution) +SELECT add_dimension('ingested_raw_events', by_hash('node_id', 32), if_not_exists => TRUE); + +-- Minimal indexes (each index costs write throughput at 3M events/s) +CREATE INDEX IF NOT EXISTS idx_events_node_time ON ingested_raw_events (node_id, timestamp DESC); +CREATE INDEX IF NOT EXISTS idx_events_type_time ON ingested_raw_events (event_type, timestamp DESC); +CREATE INDEX IF NOT EXISTS idx_events_slot ON ingested_raw_events (slot, timestamp DESC) WHERE slot IS NOT NULL; +CREATE INDEX IF NOT EXISTS idx_events_core ON ingested_raw_events (core, timestamp DESC) WHERE core IS NOT NULL; +CREATE INDEX IF NOT EXISTS idx_events_submission_id ON ingested_raw_events (node_id, submission_id, timestamp DESC) WHERE submission_id IS NOT NULL; + +-- Pure browsing store — aggressive 1h retention, checked every 5 minutes +SELECT add_retention_policy('ingested_raw_events', INTERVAL '1 hour', schedule_interval => INTERVAL '5 minutes'); + +-- VIEW alias: legacy endpoints reference 'events'. +-- Created BEFORE the wp_hash column is added so the view's column list +-- (SELECT * expands at creation time) matches the historical schema. +CREATE VIEW events AS SELECT * FROM ingested_raw_events; + +-- wp_hash hot column (added after the view on purpose — see above) +ALTER TABLE ingested_raw_events ADD COLUMN IF NOT EXISTS wp_hash BYTEA; +CREATE INDEX IF NOT EXISTS idx_ire_wp_hash + ON ingested_raw_events (wp_hash, timestamp DESC) WHERE wp_hash IS NOT NULL; + +-- ============================================================ +-- Stats cache table for pre-computed aggregations +-- ============================================================ +CREATE TABLE IF NOT EXISTS stats_cache ( + key TEXT PRIMARY KEY, + value JSONB NOT NULL, + updated_at TIMESTAMPTZ DEFAULT CURRENT_TIMESTAMP +); + +-- ============================================================ -- Event type lookup table for human-readable names and grouping. -- TODO: Generate this table automatically from src/events.rs definitions. -- For now, this is a hardcoded list matching the JIP-3 telemetry protocol event types. - +-- ============================================================ CREATE TABLE IF NOT EXISTS event_types ( id SMALLINT PRIMARY KEY, name TEXT NOT NULL, @@ -134,18 +237,3 @@ INSERT INTO event_types (id, name, group_name) VALUES (198, 'PreimageTransferred', 'preimages'), (199, 'PreimageDiscarded', 'preimages') ON CONFLICT (id) DO UPDATE SET name = EXCLUDED.name, group_name = EXCLUDED.group_name; - --- Convenience view: events pre-joined with type names and groups. --- Use this in Grafana panels instead of raw events + JOIN. -CREATE OR REPLACE VIEW events_view AS -SELECT - e.timestamp, - e.node_id, - e.event_id, - e.event_type, - et.name AS event_name, - et.group_name AS event_group, - e.data, - e.created_at -FROM events e -LEFT JOIN event_types et ON e.event_type = et.id; diff --git a/migrations/001_timescaledb_schema.sql b/migrations/001_timescaledb_schema.sql deleted file mode 100644 index a0601e7..0000000 --- a/migrations/001_timescaledb_schema.sql +++ /dev/null @@ -1,126 +0,0 @@ --- no transaction --- TimescaleDB schema for TART Backend --- Designed for high-throughput event ingestion: 3M events/sec from 1024+ nodes --- Features: automatic chunking, continuous aggregates, compression, retention policies - --- Enable extensions -CREATE EXTENSION IF NOT EXISTS timescaledb; -CREATE EXTENSION IF NOT EXISTS pg_stat_statements; - --- Nodes table (regular PostgreSQL table - low cardinality, ~1024 rows) -CREATE TABLE IF NOT EXISTS nodes ( - node_id TEXT PRIMARY KEY, - peer_id TEXT NOT NULL, - implementation_name TEXT NOT NULL, - implementation_version TEXT NOT NULL, - node_info JSONB NOT NULL, - connected_at TIMESTAMPTZ NOT NULL, - disconnected_at TIMESTAMPTZ, - last_seen_at TIMESTAMPTZ NOT NULL, - is_connected BOOLEAN DEFAULT true, - event_count BIGINT DEFAULT 0, - total_events BIGINT DEFAULT 0, - address TEXT, - created_at TIMESTAMPTZ DEFAULT CURRENT_TIMESTAMP -); - -CREATE INDEX IF NOT EXISTS idx_nodes_connected ON nodes(is_connected, last_seen_at DESC) WHERE is_connected = true; -CREATE INDEX IF NOT EXISTS idx_nodes_last_seen ON nodes(last_seen_at DESC); - --- Events hypertable (main time-series data) --- --- Schema differences from the original PostgreSQL schema (v0.2.0): --- id BIGSERIAL PRIMARY KEY → event_id BIGINT (no PK) --- Hypertables don't support BIGSERIAL PKs; unique constraints require --- the partition column. At 3M events/s dedup is too expensive anyway. --- created_at TIMESTAMPTZ (kept, same as v0.2.0) --- timestamp TIMESTAMPTZ (kept, same name as v0.2.0) --- Now used as hypertable partition key (1-hour chunks). --- event_type INTEGER → event_type SMALLINT --- 130 event types fit in 2 bytes; saves ~2GB/day at full throughput. --- -CREATE TABLE IF NOT EXISTS events ( - timestamp TIMESTAMPTZ NOT NULL, - node_id TEXT NOT NULL, - event_id BIGINT NOT NULL, - event_type SMALLINT NOT NULL, - data JSONB NOT NULL, - created_at TIMESTAMPTZ NOT NULL DEFAULT NOW() -); - --- Convert to hypertable with 1-hour chunks -SELECT create_hypertable('events', 'timestamp', - chunk_time_interval => INTERVAL '1 hour', - create_default_indexes => FALSE, - if_not_exists => TRUE -); - --- Space partitioning on node_id (32 hash buckets for write distribution) -SELECT add_dimension('events', by_hash('node_id', 32), if_not_exists => TRUE); - --- Minimal indexes (each index costs write throughput at 3M events/s) -CREATE INDEX IF NOT EXISTS idx_events_node_time ON events (node_id, timestamp DESC); -CREATE INDEX IF NOT EXISTS idx_events_type_time ON events (event_type, timestamp DESC); - --- Stats cache table for pre-computed aggregations -CREATE TABLE IF NOT EXISTS stats_cache ( - key TEXT PRIMARY KEY, - value JSONB NOT NULL, - updated_at TIMESTAMPTZ DEFAULT CURRENT_TIMESTAMP -); - --- 1-minute continuous aggregate -CREATE MATERIALIZED VIEW IF NOT EXISTS event_stats_1m -WITH (timescaledb.continuous) AS -SELECT - time_bucket('1 minute', timestamp) AS bucket, - node_id, - event_type, - COUNT(*) AS event_count, - MIN(timestamp) AS first_event, - MAX(timestamp) AS last_event -FROM events -GROUP BY bucket, node_id, event_type -WITH NO DATA; - -SELECT add_continuous_aggregate_policy('event_stats_1m', - start_offset => INTERVAL '10 minutes', - end_offset => INTERVAL '2 minutes', - schedule_interval => INTERVAL '2 minutes', - if_not_exists => TRUE -); - --- 1-hour continuous aggregate (hierarchical, from 1-minute) -CREATE MATERIALIZED VIEW IF NOT EXISTS event_stats_1h -WITH (timescaledb.continuous) AS -SELECT - time_bucket('1 hour', bucket) AS bucket, - node_id, - event_type, - SUM(event_count) AS event_count, - MIN(first_event) AS first_event, - MAX(last_event) AS last_event -FROM event_stats_1m -GROUP BY 1, node_id, event_type -WITH NO DATA; - -SELECT add_continuous_aggregate_policy('event_stats_1h', - start_offset => INTERVAL '4 hours', - end_offset => INTERVAL '1 hour', - schedule_interval => INTERVAL '1 hour', - if_not_exists => TRUE -); - --- Compression policy (compress chunks older than 2 hours) -ALTER TABLE events SET ( - timescaledb.compress, - timescaledb.compress_segmentby = 'node_id, event_type', - timescaledb.compress_orderby = 'timestamp DESC' -); - -SELECT add_compression_policy('events', INTERVAL '2 hours', if_not_exists => TRUE); - --- Retention policies -SELECT add_retention_policy('events', INTERVAL '7 days', if_not_exists => TRUE); -SELECT add_retention_policy('event_stats_1m', INTERVAL '30 days', if_not_exists => TRUE); -SELECT add_retention_policy('event_stats_1h', INTERVAL '365 days', if_not_exists => TRUE); diff --git a/migrations/002_count_tables.sql b/migrations/002_count_tables.sql new file mode 100644 index 0000000..54933df --- /dev/null +++ b/migrations/002_count_tables.sql @@ -0,0 +1,332 @@ +-- Per-protocol-group count tables for pre-aggregated events (all 115 types). +-- Events are counted in-memory (DashMap) and flushed every 5s via COPY BINARY. +-- Append-only: multiple rows per logical key are correct because all query paths +-- do SUM(event_count) GROUP BY ... + +-- ============================================================ +-- 1. status_counts (types 0, 10-13) +-- Dropped, Status, BestBlockChanged, FinalizedBlockChanged, SyncStatusChanged +-- ============================================================ +CREATE TABLE status_counts ( + bucket TIMESTAMPTZ NOT NULL, + node_id TEXT NOT NULL, + event_type SMALLINT NOT NULL, + event_count BIGINT NOT NULL, + slot INT +); +SELECT create_hypertable('status_counts', 'bucket', chunk_time_interval => INTERVAL '1 day'); +ALTER TABLE status_counts ADD CHECK (event_type = 0 OR event_type BETWEEN 10 AND 13); +CREATE INDEX ON status_counts (node_id, event_type, bucket DESC); +-- sync_timeline queries status_counts by event_type without node_id +CREATE INDEX IF NOT EXISTS idx_status_counts_et + ON status_counts (event_type, bucket DESC) WHERE slot IS NOT NULL; +ALTER TABLE status_counts SET ( + timescaledb.compress, + timescaledb.compress_segmentby = 'node_id, event_type', + timescaledb.compress_orderby = 'bucket DESC' +); +SELECT add_compression_policy('status_counts', compress_after => INTERVAL '2 hours'); +SELECT add_retention_policy('status_counts', INTERVAL '3 days'); + +-- ============================================================ +-- 2. connection_counts (types 20-28) +-- ConnectionRefused through PeerMisbehaved +-- ============================================================ +CREATE TABLE connection_counts ( + bucket TIMESTAMPTZ NOT NULL, + node_id TEXT NOT NULL, + event_type SMALLINT NOT NULL, + event_count BIGINT NOT NULL, + reason TEXT +); +SELECT create_hypertable('connection_counts', 'bucket', chunk_time_interval => INTERVAL '1 day'); +ALTER TABLE connection_counts ADD CHECK (event_type BETWEEN 20 AND 28); +CREATE INDEX ON connection_counts (node_id, event_type, bucket DESC); +-- connections_timeline queries connection_counts by event_type without node_id +CREATE INDEX IF NOT EXISTS idx_connection_counts_et + ON connection_counts (event_type, bucket DESC); +ALTER TABLE connection_counts SET ( + timescaledb.compress, + timescaledb.compress_segmentby = 'node_id, event_type', + timescaledb.compress_orderby = 'bucket DESC' +); +SELECT add_compression_policy('connection_counts', compress_after => INTERVAL '2 hours'); +SELECT add_retention_policy('connection_counts', INTERVAL '3 days'); + +-- ============================================================ +-- 3. block_counts (types 40-47) +-- Authoring through BlockExecuted +-- ============================================================ +CREATE TABLE block_counts ( + bucket TIMESTAMPTZ NOT NULL, + node_id TEXT NOT NULL, + event_type SMALLINT NOT NULL, + event_count BIGINT NOT NULL, + slot INT, + reason TEXT +); +SELECT create_hypertable('block_counts', 'bucket', chunk_time_interval => INTERVAL '1 day'); +ALTER TABLE block_counts ADD CHECK (event_type BETWEEN 40 AND 47); +CREATE INDEX ON block_counts (node_id, event_type, bucket DESC); +ALTER TABLE block_counts SET ( + timescaledb.compress, + timescaledb.compress_segmentby = 'node_id, event_type', + timescaledb.compress_orderby = 'bucket DESC' +); +SELECT add_compression_policy('block_counts', compress_after => INTERVAL '2 hours'); +SELECT add_retention_policy('block_counts', INTERVAL '3 days'); + +-- ============================================================ +-- 4. ticket_low_counts (types 80-82) +-- GeneratingTickets, TicketGenerationFailed, TicketsGenerated +-- ============================================================ +CREATE TABLE ticket_low_counts ( + bucket TIMESTAMPTZ NOT NULL, + node_id TEXT NOT NULL, + event_type SMALLINT NOT NULL, + event_count BIGINT NOT NULL, + reason TEXT +); +SELECT create_hypertable('ticket_low_counts', 'bucket', chunk_time_interval => INTERVAL '1 day'); +ALTER TABLE ticket_low_counts ADD CHECK (event_type BETWEEN 80 AND 82); +CREATE INDEX ON ticket_low_counts (node_id, event_type, bucket DESC); +ALTER TABLE ticket_low_counts SET ( + timescaledb.compress, + timescaledb.compress_segmentby = 'node_id, event_type', + timescaledb.compress_orderby = 'bucket DESC' +); +SELECT add_compression_policy('ticket_low_counts', compress_after => INTERVAL '2 hours'); +SELECT add_retention_policy('ticket_low_counts', INTERVAL '3 days'); + +-- ============================================================ +-- 5. wp_pipeline_counts (types 90-105) +-- WorkPackageSubmission through GuaranteeBuilt +-- Core is nullable — enrichment may fail for types 90, 91, 103 +-- ============================================================ +CREATE TABLE wp_pipeline_counts ( + bucket TIMESTAMPTZ NOT NULL, + node_id TEXT NOT NULL, + event_type SMALLINT NOT NULL, + event_count BIGINT NOT NULL, + core SMALLINT, + reason TEXT +); +SELECT create_hypertable('wp_pipeline_counts', 'bucket', chunk_time_interval => INTERVAL '1 day'); +ALTER TABLE wp_pipeline_counts ADD CHECK (event_type BETWEEN 90 AND 105); +CREATE INDEX ON wp_pipeline_counts (node_id, event_type, bucket DESC); +-- all_core_stats_1m queries raw tables with core filter (partial: many rows have NULL core) +CREATE INDEX IF NOT EXISTS idx_wp_pipeline_counts_core + ON wp_pipeline_counts (core, event_type, bucket DESC) WHERE core IS NOT NULL; +ALTER TABLE wp_pipeline_counts SET ( + timescaledb.compress, + timescaledb.compress_segmentby = 'node_id, event_type', + timescaledb.compress_orderby = 'bucket DESC' +); +SELECT add_compression_policy('wp_pipeline_counts', compress_after => INTERVAL '2 hours'); +SELECT add_retention_policy('wp_pipeline_counts', INTERVAL '3 days'); + +-- ============================================================ +-- 6. block_distribution_counts (types 60-68) +-- ============================================================ +CREATE TABLE block_distribution_counts ( + bucket TIMESTAMPTZ NOT NULL, + node_id TEXT NOT NULL, + event_type SMALLINT NOT NULL, + event_count BIGINT NOT NULL, + slot INT, + reason TEXT +); +SELECT create_hypertable('block_distribution_counts', 'bucket', chunk_time_interval => INTERVAL '1 day'); +ALTER TABLE block_distribution_counts ADD CHECK (event_type BETWEEN 60 AND 68); +CREATE INDEX ON block_distribution_counts (node_id, event_type, bucket DESC); +ALTER TABLE block_distribution_counts SET ( + timescaledb.compress, + timescaledb.compress_segmentby = 'node_id, event_type', + timescaledb.compress_orderby = 'bucket DESC' +); +SELECT add_compression_policy('block_distribution_counts', compress_after => INTERVAL '2 hours'); +SELECT add_retention_policy('block_distribution_counts', INTERVAL '3 days'); + +-- ============================================================ +-- 7. ticket_counts (types 83-84) +-- ============================================================ +CREATE TABLE ticket_counts ( + bucket TIMESTAMPTZ NOT NULL, + node_id TEXT NOT NULL, + event_type SMALLINT NOT NULL, + event_count BIGINT NOT NULL, + reason TEXT, + from_proxy BOOLEAN, + epoch INT +); +SELECT create_hypertable('ticket_counts', 'bucket', chunk_time_interval => INTERVAL '1 day'); +ALTER TABLE ticket_counts ADD CHECK (event_type BETWEEN 83 AND 84); +CREATE INDEX ON ticket_counts (node_id, event_type, bucket DESC); +ALTER TABLE ticket_counts SET ( + timescaledb.compress, + timescaledb.compress_segmentby = 'node_id, event_type', + timescaledb.compress_orderby = 'bucket DESC' +); +SELECT add_compression_policy('ticket_counts', compress_after => INTERVAL '2 hours'); +SELECT add_retention_policy('ticket_counts', INTERVAL '3 days'); + +-- ============================================================ +-- 8. guarantee_sending_counts (types 106-109) +-- ============================================================ +CREATE TABLE guarantee_sending_counts ( + bucket TIMESTAMPTZ NOT NULL, + node_id TEXT NOT NULL, + event_type SMALLINT NOT NULL, + event_count BIGINT NOT NULL, + core SMALLINT, + reason TEXT +); +SELECT create_hypertable('guarantee_sending_counts', 'bucket', chunk_time_interval => INTERVAL '1 day'); +ALTER TABLE guarantee_sending_counts ADD CHECK (event_type BETWEEN 106 AND 109); +CREATE INDEX ON guarantee_sending_counts (node_id, event_type, bucket DESC); +-- all_core_stats_1m queries raw tables with core filter (partial: many rows have NULL core) +CREATE INDEX IF NOT EXISTS idx_guarantee_sending_counts_core + ON guarantee_sending_counts (core, event_type, bucket DESC) WHERE core IS NOT NULL; +ALTER TABLE guarantee_sending_counts SET ( + timescaledb.compress, + timescaledb.compress_segmentby = 'node_id, event_type', + timescaledb.compress_orderby = 'bucket DESC' +); +SELECT add_compression_policy('guarantee_sending_counts', compress_after => INTERVAL '2 hours'); +SELECT add_retention_policy('guarantee_sending_counts', INTERVAL '3 days'); + +-- ============================================================ +-- 9. guarantee_receiving_counts (types 110-113) +-- ============================================================ +CREATE TABLE guarantee_receiving_counts ( + bucket TIMESTAMPTZ NOT NULL, + node_id TEXT NOT NULL, + event_type SMALLINT NOT NULL, + event_count BIGINT NOT NULL, + slot INT, + reason TEXT +); +SELECT create_hypertable('guarantee_receiving_counts', 'bucket', chunk_time_interval => INTERVAL '1 day'); +ALTER TABLE guarantee_receiving_counts ADD CHECK (event_type BETWEEN 110 AND 113); +CREATE INDEX ON guarantee_receiving_counts (node_id, event_type, bucket DESC); +-- Partial index for /guarantee-discards endpoint (no node_id leading column) +CREATE INDEX ON guarantee_receiving_counts (event_type, bucket DESC) WHERE reason IS NOT NULL; +ALTER TABLE guarantee_receiving_counts SET ( + timescaledb.compress, + timescaledb.compress_segmentby = 'node_id, event_type', + timescaledb.compress_orderby = 'bucket DESC' +); +SELECT add_compression_policy('guarantee_receiving_counts', compress_after => INTERVAL '2 hours'); +SELECT add_retention_policy('guarantee_receiving_counts', INTERVAL '3 days'); + +-- ============================================================ +-- 10. shard_counts (types 120-125) +-- ============================================================ +CREATE TABLE shard_counts ( + bucket TIMESTAMPTZ NOT NULL, + node_id TEXT NOT NULL, + event_type SMALLINT NOT NULL, + event_count BIGINT NOT NULL, + reason TEXT +); +SELECT create_hypertable('shard_counts', 'bucket', chunk_time_interval => INTERVAL '1 day'); +ALTER TABLE shard_counts ADD CHECK (event_type BETWEEN 120 AND 125); +CREATE INDEX ON shard_counts (node_id, event_type, bucket DESC); +ALTER TABLE shard_counts SET ( + timescaledb.compress, + timescaledb.compress_segmentby = 'node_id, event_type', + timescaledb.compress_orderby = 'bucket DESC' +); +SELECT add_compression_policy('shard_counts', compress_after => INTERVAL '2 hours'); +SELECT add_retention_policy('shard_counts', INTERVAL '3 days'); + +-- ============================================================ +-- 11. assurance_counts (types 126-131) +-- ============================================================ +CREATE TABLE assurance_counts ( + bucket TIMESTAMPTZ NOT NULL, + node_id TEXT NOT NULL, + event_type SMALLINT NOT NULL, + event_count BIGINT NOT NULL, + reason TEXT +); +SELECT create_hypertable('assurance_counts', 'bucket', chunk_time_interval => INTERVAL '1 day'); +ALTER TABLE assurance_counts ADD CHECK (event_type BETWEEN 126 AND 131); +CREATE INDEX ON assurance_counts (node_id, event_type, bucket DESC); +ALTER TABLE assurance_counts SET ( + timescaledb.compress, + timescaledb.compress_segmentby = 'node_id, event_type', + timescaledb.compress_orderby = 'bucket DESC' +); +SELECT add_compression_policy('assurance_counts', compress_after => INTERVAL '2 hours'); +SELECT add_retention_policy('assurance_counts', INTERVAL '3 days'); + +-- ============================================================ +-- 12. bundle_counts (types 140-153) +-- ============================================================ +CREATE TABLE bundle_counts ( + bucket TIMESTAMPTZ NOT NULL, + node_id TEXT NOT NULL, + event_type SMALLINT NOT NULL, + event_count BIGINT NOT NULL, + reason TEXT, + kind SMALLINT +); +SELECT create_hypertable('bundle_counts', 'bucket', chunk_time_interval => INTERVAL '1 day'); +ALTER TABLE bundle_counts ADD CHECK (event_type BETWEEN 140 AND 153); +CREATE INDEX ON bundle_counts (node_id, event_type, bucket DESC); +ALTER TABLE bundle_counts SET ( + timescaledb.compress, + timescaledb.compress_segmentby = 'node_id, event_type', + timescaledb.compress_orderby = 'bucket DESC' +); +SELECT add_compression_policy('bundle_counts', compress_after => INTERVAL '2 hours'); +SELECT add_retention_policy('bundle_counts', INTERVAL '3 days'); + +-- ============================================================ +-- 13. segment_counts (types 160-178) +-- ============================================================ +CREATE TABLE segment_counts ( + bucket TIMESTAMPTZ NOT NULL, + node_id TEXT NOT NULL, + event_type SMALLINT NOT NULL, + event_count BIGINT NOT NULL, + core SMALLINT, + reason TEXT, + kind SMALLINT +); +SELECT create_hypertable('segment_counts', 'bucket', chunk_time_interval => INTERVAL '1 day'); +ALTER TABLE segment_counts ADD CHECK (event_type BETWEEN 160 AND 178); +CREATE INDEX ON segment_counts (node_id, event_type, bucket DESC); +-- all_core_stats_1m queries raw tables with core filter (partial: many rows have NULL core) +CREATE INDEX IF NOT EXISTS idx_segment_counts_core + ON segment_counts (core, event_type, bucket DESC) WHERE core IS NOT NULL; +ALTER TABLE segment_counts SET ( + timescaledb.compress, + timescaledb.compress_segmentby = 'node_id, event_type', + timescaledb.compress_orderby = 'bucket DESC' +); +SELECT add_compression_policy('segment_counts', compress_after => INTERVAL '2 hours'); +SELECT add_retention_policy('segment_counts', INTERVAL '3 days'); + +-- ============================================================ +-- 14. preimage_counts (types 190-199) +-- ============================================================ +CREATE TABLE preimage_counts ( + bucket TIMESTAMPTZ NOT NULL, + node_id TEXT NOT NULL, + event_type SMALLINT NOT NULL, + event_count BIGINT NOT NULL, + reason TEXT, + service_id INT +); +SELECT create_hypertable('preimage_counts', 'bucket', chunk_time_interval => INTERVAL '1 day'); +ALTER TABLE preimage_counts ADD CHECK (event_type BETWEEN 190 AND 199); +CREATE INDEX ON preimage_counts (node_id, event_type, bucket DESC); +ALTER TABLE preimage_counts SET ( + timescaledb.compress, + timescaledb.compress_segmentby = 'node_id, event_type', + timescaledb.compress_orderby = 'bucket DESC' +); +SELECT add_compression_policy('preimage_counts', compress_after => INTERVAL '2 hours'); +SELECT add_retention_policy('preimage_counts', INTERVAL '3 days'); diff --git a/migrations/003_count_aggregates.sql b/migrations/003_count_aggregates.sql new file mode 100644 index 0000000..04eecb6 --- /dev/null +++ b/migrations/003_count_aggregates.sql @@ -0,0 +1,595 @@ +-- Hierarchical continuous aggregates over the count tables: +-- _1m (from raw, 30d retention) and _1h (from _1m, 365d retention). +-- guarantee_sending and segment aggregates keep the `core` dimension. + +-- ============================================================ +-- block_distribution_counts +-- ============================================================ +CREATE MATERIALIZED VIEW block_distribution_counts_1m +WITH (timescaledb.continuous) AS +SELECT + time_bucket('1 minute', bucket) AS bucket, + node_id, event_type, + SUM(event_count) AS event_count +FROM block_distribution_counts +GROUP BY 1, node_id, event_type +WITH NO DATA; + +SELECT add_continuous_aggregate_policy('block_distribution_counts_1m', + start_offset => INTERVAL '10 minutes', + end_offset => INTERVAL '2 minutes', + schedule_interval => INTERVAL '2 minutes'); +SELECT add_retention_policy('block_distribution_counts_1m', INTERVAL '30 days'); + +CREATE MATERIALIZED VIEW block_distribution_counts_1h +WITH (timescaledb.continuous) AS +SELECT + time_bucket('1 hour', bucket) AS bucket, + node_id, event_type, + SUM(event_count) AS event_count +FROM block_distribution_counts_1m +GROUP BY 1, node_id, event_type +WITH NO DATA; + +SELECT add_continuous_aggregate_policy('block_distribution_counts_1h', + start_offset => INTERVAL '4 hours', + end_offset => INTERVAL '1 hour', + schedule_interval => INTERVAL '1 hour'); +SELECT add_retention_policy('block_distribution_counts_1h', INTERVAL '365 days'); + +-- ============================================================ +-- ticket_counts +-- ============================================================ +CREATE MATERIALIZED VIEW ticket_counts_1m +WITH (timescaledb.continuous) AS +SELECT + time_bucket('1 minute', bucket) AS bucket, + node_id, event_type, + SUM(event_count) AS event_count +FROM ticket_counts +GROUP BY 1, node_id, event_type +WITH NO DATA; + +SELECT add_continuous_aggregate_policy('ticket_counts_1m', + start_offset => INTERVAL '10 minutes', + end_offset => INTERVAL '2 minutes', + schedule_interval => INTERVAL '2 minutes'); +SELECT add_retention_policy('ticket_counts_1m', INTERVAL '30 days'); + +CREATE MATERIALIZED VIEW ticket_counts_1h +WITH (timescaledb.continuous) AS +SELECT + time_bucket('1 hour', bucket) AS bucket, + node_id, event_type, + SUM(event_count) AS event_count +FROM ticket_counts_1m +GROUP BY 1, node_id, event_type +WITH NO DATA; + +SELECT add_continuous_aggregate_policy('ticket_counts_1h', + start_offset => INTERVAL '4 hours', + end_offset => INTERVAL '1 hour', + schedule_interval => INTERVAL '1 hour'); +SELECT add_retention_policy('ticket_counts_1h', INTERVAL '365 days'); + +-- ============================================================ +-- guarantee_sending_counts +-- ============================================================ +CREATE MATERIALIZED VIEW guarantee_sending_counts_1m +WITH (timescaledb.continuous) AS +SELECT + time_bucket('1 minute', bucket) AS bucket, + node_id, event_type, core, + SUM(event_count) AS event_count +FROM guarantee_sending_counts +GROUP BY 1, node_id, event_type, core +WITH NO DATA; + +SELECT add_continuous_aggregate_policy('guarantee_sending_counts_1m', + start_offset => INTERVAL '10 minutes', + end_offset => INTERVAL '2 minutes', + schedule_interval => INTERVAL '2 minutes'); +SELECT add_retention_policy('guarantee_sending_counts_1m', INTERVAL '30 days'); + +CREATE MATERIALIZED VIEW guarantee_sending_counts_1h +WITH (timescaledb.continuous) AS +SELECT + time_bucket('1 hour', bucket) AS bucket, + node_id, event_type, core, + SUM(event_count) AS event_count +FROM guarantee_sending_counts_1m +GROUP BY 1, node_id, event_type, core +WITH NO DATA; + +SELECT add_continuous_aggregate_policy('guarantee_sending_counts_1h', + start_offset => INTERVAL '4 hours', + end_offset => INTERVAL '1 hour', + schedule_interval => INTERVAL '1 hour'); +SELECT add_retention_policy('guarantee_sending_counts_1h', INTERVAL '365 days'); + +-- ============================================================ +-- guarantee_receiving_counts +-- ============================================================ +CREATE MATERIALIZED VIEW guarantee_receiving_counts_1m +WITH (timescaledb.continuous) AS +SELECT + time_bucket('1 minute', bucket) AS bucket, + node_id, event_type, + SUM(event_count) AS event_count +FROM guarantee_receiving_counts +GROUP BY 1, node_id, event_type +WITH NO DATA; + +SELECT add_continuous_aggregate_policy('guarantee_receiving_counts_1m', + start_offset => INTERVAL '10 minutes', + end_offset => INTERVAL '2 minutes', + schedule_interval => INTERVAL '2 minutes'); +SELECT add_retention_policy('guarantee_receiving_counts_1m', INTERVAL '30 days'); + +CREATE MATERIALIZED VIEW guarantee_receiving_counts_1h +WITH (timescaledb.continuous) AS +SELECT + time_bucket('1 hour', bucket) AS bucket, + node_id, event_type, + SUM(event_count) AS event_count +FROM guarantee_receiving_counts_1m +GROUP BY 1, node_id, event_type +WITH NO DATA; + +SELECT add_continuous_aggregate_policy('guarantee_receiving_counts_1h', + start_offset => INTERVAL '4 hours', + end_offset => INTERVAL '1 hour', + schedule_interval => INTERVAL '1 hour'); +SELECT add_retention_policy('guarantee_receiving_counts_1h', INTERVAL '365 days'); + +-- ============================================================ +-- shard_counts +-- ============================================================ +CREATE MATERIALIZED VIEW shard_counts_1m +WITH (timescaledb.continuous) AS +SELECT + time_bucket('1 minute', bucket) AS bucket, + node_id, event_type, + SUM(event_count) AS event_count +FROM shard_counts +GROUP BY 1, node_id, event_type +WITH NO DATA; + +SELECT add_continuous_aggregate_policy('shard_counts_1m', + start_offset => INTERVAL '10 minutes', + end_offset => INTERVAL '2 minutes', + schedule_interval => INTERVAL '2 minutes'); +SELECT add_retention_policy('shard_counts_1m', INTERVAL '30 days'); + +CREATE MATERIALIZED VIEW shard_counts_1h +WITH (timescaledb.continuous) AS +SELECT + time_bucket('1 hour', bucket) AS bucket, + node_id, event_type, + SUM(event_count) AS event_count +FROM shard_counts_1m +GROUP BY 1, node_id, event_type +WITH NO DATA; + +SELECT add_continuous_aggregate_policy('shard_counts_1h', + start_offset => INTERVAL '4 hours', + end_offset => INTERVAL '1 hour', + schedule_interval => INTERVAL '1 hour'); +SELECT add_retention_policy('shard_counts_1h', INTERVAL '365 days'); + +-- ============================================================ +-- assurance_counts +-- ============================================================ +CREATE MATERIALIZED VIEW assurance_counts_1m +WITH (timescaledb.continuous) AS +SELECT + time_bucket('1 minute', bucket) AS bucket, + node_id, event_type, + SUM(event_count) AS event_count +FROM assurance_counts +GROUP BY 1, node_id, event_type +WITH NO DATA; + +SELECT add_continuous_aggregate_policy('assurance_counts_1m', + start_offset => INTERVAL '10 minutes', + end_offset => INTERVAL '2 minutes', + schedule_interval => INTERVAL '2 minutes'); +SELECT add_retention_policy('assurance_counts_1m', INTERVAL '30 days'); + +CREATE MATERIALIZED VIEW assurance_counts_1h +WITH (timescaledb.continuous) AS +SELECT + time_bucket('1 hour', bucket) AS bucket, + node_id, event_type, + SUM(event_count) AS event_count +FROM assurance_counts_1m +GROUP BY 1, node_id, event_type +WITH NO DATA; + +SELECT add_continuous_aggregate_policy('assurance_counts_1h', + start_offset => INTERVAL '4 hours', + end_offset => INTERVAL '1 hour', + schedule_interval => INTERVAL '1 hour'); +SELECT add_retention_policy('assurance_counts_1h', INTERVAL '365 days'); + +-- ============================================================ +-- bundle_counts +-- ============================================================ +CREATE MATERIALIZED VIEW bundle_counts_1m +WITH (timescaledb.continuous) AS +SELECT + time_bucket('1 minute', bucket) AS bucket, + node_id, event_type, + SUM(event_count) AS event_count +FROM bundle_counts +GROUP BY 1, node_id, event_type +WITH NO DATA; + +SELECT add_continuous_aggregate_policy('bundle_counts_1m', + start_offset => INTERVAL '10 minutes', + end_offset => INTERVAL '2 minutes', + schedule_interval => INTERVAL '2 minutes'); +SELECT add_retention_policy('bundle_counts_1m', INTERVAL '30 days'); + +CREATE MATERIALIZED VIEW bundle_counts_1h +WITH (timescaledb.continuous) AS +SELECT + time_bucket('1 hour', bucket) AS bucket, + node_id, event_type, + SUM(event_count) AS event_count +FROM bundle_counts_1m +GROUP BY 1, node_id, event_type +WITH NO DATA; + +SELECT add_continuous_aggregate_policy('bundle_counts_1h', + start_offset => INTERVAL '4 hours', + end_offset => INTERVAL '1 hour', + schedule_interval => INTERVAL '1 hour'); +SELECT add_retention_policy('bundle_counts_1h', INTERVAL '365 days'); + +-- ============================================================ +-- segment_counts +-- ============================================================ +CREATE MATERIALIZED VIEW segment_counts_1m +WITH (timescaledb.continuous) AS +SELECT + time_bucket('1 minute', bucket) AS bucket, + node_id, event_type, core, + SUM(event_count) AS event_count +FROM segment_counts +GROUP BY 1, node_id, event_type, core +WITH NO DATA; + +SELECT add_continuous_aggregate_policy('segment_counts_1m', + start_offset => INTERVAL '10 minutes', + end_offset => INTERVAL '2 minutes', + schedule_interval => INTERVAL '2 minutes'); +SELECT add_retention_policy('segment_counts_1m', INTERVAL '30 days'); + +CREATE MATERIALIZED VIEW segment_counts_1h +WITH (timescaledb.continuous) AS +SELECT + time_bucket('1 hour', bucket) AS bucket, + node_id, event_type, core, + SUM(event_count) AS event_count +FROM segment_counts_1m +GROUP BY 1, node_id, event_type, core +WITH NO DATA; + +SELECT add_continuous_aggregate_policy('segment_counts_1h', + start_offset => INTERVAL '4 hours', + end_offset => INTERVAL '1 hour', + schedule_interval => INTERVAL '1 hour'); +SELECT add_retention_policy('segment_counts_1h', INTERVAL '365 days'); + +-- ============================================================ +-- preimage_counts +-- ============================================================ +CREATE MATERIALIZED VIEW preimage_counts_1m +WITH (timescaledb.continuous) AS +SELECT + time_bucket('1 minute', bucket) AS bucket, + node_id, event_type, + SUM(event_count) AS event_count +FROM preimage_counts +GROUP BY 1, node_id, event_type +WITH NO DATA; + +SELECT add_continuous_aggregate_policy('preimage_counts_1m', + start_offset => INTERVAL '10 minutes', + end_offset => INTERVAL '2 minutes', + schedule_interval => INTERVAL '2 minutes'); +SELECT add_retention_policy('preimage_counts_1m', INTERVAL '30 days'); + +CREATE MATERIALIZED VIEW preimage_counts_1h +WITH (timescaledb.continuous) AS +SELECT + time_bucket('1 hour', bucket) AS bucket, + node_id, event_type, + SUM(event_count) AS event_count +FROM preimage_counts_1m +GROUP BY 1, node_id, event_type +WITH NO DATA; + +SELECT add_continuous_aggregate_policy('preimage_counts_1h', + start_offset => INTERVAL '4 hours', + end_offset => INTERVAL '1 hour', + schedule_interval => INTERVAL '1 hour'); +SELECT add_retention_policy('preimage_counts_1h', INTERVAL '365 days'); + +-- ============================================================ +-- status_counts +-- ============================================================ +CREATE MATERIALIZED VIEW status_counts_1m +WITH (timescaledb.continuous) AS +SELECT + time_bucket('1 minute', bucket) AS bucket, + node_id, event_type, + SUM(event_count) AS event_count +FROM status_counts +GROUP BY 1, node_id, event_type +WITH NO DATA; + +SELECT add_continuous_aggregate_policy('status_counts_1m', + start_offset => INTERVAL '10 minutes', + end_offset => INTERVAL '2 minutes', + schedule_interval => INTERVAL '2 minutes'); +SELECT add_retention_policy('status_counts_1m', INTERVAL '30 days'); + +CREATE MATERIALIZED VIEW status_counts_1h +WITH (timescaledb.continuous) AS +SELECT + time_bucket('1 hour', bucket) AS bucket, + node_id, event_type, + SUM(event_count) AS event_count +FROM status_counts_1m +GROUP BY 1, node_id, event_type +WITH NO DATA; + +SELECT add_continuous_aggregate_policy('status_counts_1h', + start_offset => INTERVAL '4 hours', + end_offset => INTERVAL '1 hour', + schedule_interval => INTERVAL '1 hour'); +SELECT add_retention_policy('status_counts_1h', INTERVAL '365 days'); + +-- ============================================================ +-- connection_counts +-- ============================================================ +CREATE MATERIALIZED VIEW connection_counts_1m +WITH (timescaledb.continuous) AS +SELECT + time_bucket('1 minute', bucket) AS bucket, + node_id, event_type, + SUM(event_count) AS event_count +FROM connection_counts +GROUP BY 1, node_id, event_type +WITH NO DATA; + +SELECT add_continuous_aggregate_policy('connection_counts_1m', + start_offset => INTERVAL '10 minutes', + end_offset => INTERVAL '2 minutes', + schedule_interval => INTERVAL '2 minutes'); +SELECT add_retention_policy('connection_counts_1m', INTERVAL '30 days'); + +CREATE MATERIALIZED VIEW connection_counts_1h +WITH (timescaledb.continuous) AS +SELECT + time_bucket('1 hour', bucket) AS bucket, + node_id, event_type, + SUM(event_count) AS event_count +FROM connection_counts_1m +GROUP BY 1, node_id, event_type +WITH NO DATA; + +SELECT add_continuous_aggregate_policy('connection_counts_1h', + start_offset => INTERVAL '4 hours', + end_offset => INTERVAL '1 hour', + schedule_interval => INTERVAL '1 hour'); +SELECT add_retention_policy('connection_counts_1h', INTERVAL '365 days'); + +-- ============================================================ +-- block_counts +-- ============================================================ +CREATE MATERIALIZED VIEW block_counts_1m +WITH (timescaledb.continuous) AS +SELECT + time_bucket('1 minute', bucket) AS bucket, + node_id, event_type, + SUM(event_count) AS event_count +FROM block_counts +GROUP BY 1, node_id, event_type +WITH NO DATA; + +SELECT add_continuous_aggregate_policy('block_counts_1m', + start_offset => INTERVAL '10 minutes', + end_offset => INTERVAL '2 minutes', + schedule_interval => INTERVAL '2 minutes'); +SELECT add_retention_policy('block_counts_1m', INTERVAL '30 days'); + +CREATE MATERIALIZED VIEW block_counts_1h +WITH (timescaledb.continuous) AS +SELECT + time_bucket('1 hour', bucket) AS bucket, + node_id, event_type, + SUM(event_count) AS event_count +FROM block_counts_1m +GROUP BY 1, node_id, event_type +WITH NO DATA; + +SELECT add_continuous_aggregate_policy('block_counts_1h', + start_offset => INTERVAL '4 hours', + end_offset => INTERVAL '1 hour', + schedule_interval => INTERVAL '1 hour'); +SELECT add_retention_policy('block_counts_1h', INTERVAL '365 days'); + +-- ============================================================ +-- ticket_low_counts +-- ============================================================ +CREATE MATERIALIZED VIEW ticket_low_counts_1m +WITH (timescaledb.continuous) AS +SELECT + time_bucket('1 minute', bucket) AS bucket, + node_id, event_type, + SUM(event_count) AS event_count +FROM ticket_low_counts +GROUP BY 1, node_id, event_type +WITH NO DATA; + +SELECT add_continuous_aggregate_policy('ticket_low_counts_1m', + start_offset => INTERVAL '10 minutes', + end_offset => INTERVAL '2 minutes', + schedule_interval => INTERVAL '2 minutes'); +SELECT add_retention_policy('ticket_low_counts_1m', INTERVAL '30 days'); + +CREATE MATERIALIZED VIEW ticket_low_counts_1h +WITH (timescaledb.continuous) AS +SELECT + time_bucket('1 hour', bucket) AS bucket, + node_id, event_type, + SUM(event_count) AS event_count +FROM ticket_low_counts_1m +GROUP BY 1, node_id, event_type +WITH NO DATA; + +SELECT add_continuous_aggregate_policy('ticket_low_counts_1h', + start_offset => INTERVAL '4 hours', + end_offset => INTERVAL '1 hour', + schedule_interval => INTERVAL '1 hour'); +SELECT add_retention_policy('ticket_low_counts_1h', INTERVAL '365 days'); + +-- ============================================================ +-- wp_pipeline_counts +-- ============================================================ +CREATE MATERIALIZED VIEW wp_pipeline_counts_1m +WITH (timescaledb.continuous) AS +SELECT + time_bucket('1 minute', bucket) AS bucket, + node_id, event_type, + SUM(event_count) AS event_count +FROM wp_pipeline_counts +GROUP BY 1, node_id, event_type +WITH NO DATA; + +SELECT add_continuous_aggregate_policy('wp_pipeline_counts_1m', + start_offset => INTERVAL '10 minutes', + end_offset => INTERVAL '2 minutes', + schedule_interval => INTERVAL '2 minutes'); +SELECT add_retention_policy('wp_pipeline_counts_1m', INTERVAL '30 days'); + +CREATE MATERIALIZED VIEW wp_pipeline_counts_1h +WITH (timescaledb.continuous) AS +SELECT + time_bucket('1 hour', bucket) AS bucket, + node_id, event_type, + SUM(event_count) AS event_count +FROM wp_pipeline_counts_1m +GROUP BY 1, node_id, event_type +WITH NO DATA; + +SELECT add_continuous_aggregate_policy('wp_pipeline_counts_1h', + start_offset => INTERVAL '4 hours', + end_offset => INTERVAL '1 hour', + schedule_interval => INTERVAL '1 hour'); +SELECT add_retention_policy('wp_pipeline_counts_1h', INTERVAL '365 days'); + +-- ============================================================ +-- Real-time aggregation (materialized_only = false) on all 28 aggregates: +-- the query planner appends a live tail scan over the un-materialized +-- window (last 2-4 min), so recent data is visible in Grafana panels and +-- every branch of the all_event_stats_* UNION views has the same freshness. +-- +-- PERFORMANCE WARNING (1024-validator networks): the tail scan reads the +-- raw count table for the un-materialized window on every query. If +-- aggregate queries become slow, this setting is the first thing to check. +-- To revert a single aggregate: +-- ALTER MATERIALIZED VIEW SET (timescaledb.materialized_only = true); +-- ============================================================ +ALTER MATERIALIZED VIEW status_counts_1m SET (timescaledb.materialized_only = false); +ALTER MATERIALIZED VIEW status_counts_1h SET (timescaledb.materialized_only = false); +ALTER MATERIALIZED VIEW connection_counts_1m SET (timescaledb.materialized_only = false); +ALTER MATERIALIZED VIEW connection_counts_1h SET (timescaledb.materialized_only = false); +ALTER MATERIALIZED VIEW block_counts_1m SET (timescaledb.materialized_only = false); +ALTER MATERIALIZED VIEW block_counts_1h SET (timescaledb.materialized_only = false); +ALTER MATERIALIZED VIEW ticket_low_counts_1m SET (timescaledb.materialized_only = false); +ALTER MATERIALIZED VIEW ticket_low_counts_1h SET (timescaledb.materialized_only = false); +ALTER MATERIALIZED VIEW wp_pipeline_counts_1m SET (timescaledb.materialized_only = false); +ALTER MATERIALIZED VIEW wp_pipeline_counts_1h SET (timescaledb.materialized_only = false); +ALTER MATERIALIZED VIEW block_distribution_counts_1m SET (timescaledb.materialized_only = false); +ALTER MATERIALIZED VIEW block_distribution_counts_1h SET (timescaledb.materialized_only = false); +ALTER MATERIALIZED VIEW ticket_counts_1m SET (timescaledb.materialized_only = false); +ALTER MATERIALIZED VIEW ticket_counts_1h SET (timescaledb.materialized_only = false); +ALTER MATERIALIZED VIEW guarantee_sending_counts_1m SET (timescaledb.materialized_only = false); +ALTER MATERIALIZED VIEW guarantee_sending_counts_1h SET (timescaledb.materialized_only = false); +ALTER MATERIALIZED VIEW guarantee_receiving_counts_1m SET (timescaledb.materialized_only = false); +ALTER MATERIALIZED VIEW guarantee_receiving_counts_1h SET (timescaledb.materialized_only = false); +ALTER MATERIALIZED VIEW shard_counts_1m SET (timescaledb.materialized_only = false); +ALTER MATERIALIZED VIEW shard_counts_1h SET (timescaledb.materialized_only = false); +ALTER MATERIALIZED VIEW assurance_counts_1m SET (timescaledb.materialized_only = false); +ALTER MATERIALIZED VIEW assurance_counts_1h SET (timescaledb.materialized_only = false); +ALTER MATERIALIZED VIEW bundle_counts_1m SET (timescaledb.materialized_only = false); +ALTER MATERIALIZED VIEW bundle_counts_1h SET (timescaledb.materialized_only = false); +ALTER MATERIALIZED VIEW segment_counts_1m SET (timescaledb.materialized_only = false); +ALTER MATERIALIZED VIEW segment_counts_1h SET (timescaledb.materialized_only = false); +ALTER MATERIALIZED VIEW preimage_counts_1m SET (timescaledb.materialized_only = false); +ALTER MATERIALIZED VIEW preimage_counts_1h SET (timescaledb.materialized_only = false); + +-- ============================================================ +-- (event_type, bucket DESC) indexes: aggregate queries filter by event_type +-- after bucket range narrowing; without these they sequential-scan. +-- ============================================================ +CREATE INDEX IF NOT EXISTS idx_block_distribution_counts_1m_et + ON block_distribution_counts_1m (event_type, bucket DESC); +CREATE INDEX IF NOT EXISTS idx_block_distribution_counts_1h_et + ON block_distribution_counts_1h (event_type, bucket DESC); +CREATE INDEX IF NOT EXISTS idx_ticket_counts_1m_et + ON ticket_counts_1m (event_type, bucket DESC); +CREATE INDEX IF NOT EXISTS idx_ticket_counts_1h_et + ON ticket_counts_1h (event_type, bucket DESC); +CREATE INDEX IF NOT EXISTS idx_guarantee_sending_counts_1m_et + ON guarantee_sending_counts_1m (event_type, bucket DESC); +CREATE INDEX IF NOT EXISTS idx_guarantee_sending_counts_1h_et + ON guarantee_sending_counts_1h (event_type, bucket DESC); +CREATE INDEX IF NOT EXISTS idx_guarantee_receiving_counts_1m_et + ON guarantee_receiving_counts_1m (event_type, bucket DESC); +CREATE INDEX IF NOT EXISTS idx_guarantee_receiving_counts_1h_et + ON guarantee_receiving_counts_1h (event_type, bucket DESC); +CREATE INDEX IF NOT EXISTS idx_shard_counts_1m_et + ON shard_counts_1m (event_type, bucket DESC); +CREATE INDEX IF NOT EXISTS idx_shard_counts_1h_et + ON shard_counts_1h (event_type, bucket DESC); +CREATE INDEX IF NOT EXISTS idx_assurance_counts_1m_et + ON assurance_counts_1m (event_type, bucket DESC); +CREATE INDEX IF NOT EXISTS idx_assurance_counts_1h_et + ON assurance_counts_1h (event_type, bucket DESC); +CREATE INDEX IF NOT EXISTS idx_bundle_counts_1m_et + ON bundle_counts_1m (event_type, bucket DESC); +CREATE INDEX IF NOT EXISTS idx_bundle_counts_1h_et + ON bundle_counts_1h (event_type, bucket DESC); +CREATE INDEX IF NOT EXISTS idx_segment_counts_1m_et + ON segment_counts_1m (event_type, bucket DESC); +CREATE INDEX IF NOT EXISTS idx_segment_counts_1h_et + ON segment_counts_1h (event_type, bucket DESC); +CREATE INDEX IF NOT EXISTS idx_preimage_counts_1m_et + ON preimage_counts_1m (event_type, bucket DESC); +CREATE INDEX IF NOT EXISTS idx_preimage_counts_1h_et + ON preimage_counts_1h (event_type, bucket DESC); +CREATE INDEX IF NOT EXISTS idx_status_counts_1m_et + ON status_counts_1m (event_type, bucket DESC); +CREATE INDEX IF NOT EXISTS idx_status_counts_1h_et + ON status_counts_1h (event_type, bucket DESC); +CREATE INDEX IF NOT EXISTS idx_connection_counts_1m_et + ON connection_counts_1m (event_type, bucket DESC); +CREATE INDEX IF NOT EXISTS idx_connection_counts_1h_et + ON connection_counts_1h (event_type, bucket DESC); +CREATE INDEX IF NOT EXISTS idx_block_counts_1m_et + ON block_counts_1m (event_type, bucket DESC); +CREATE INDEX IF NOT EXISTS idx_block_counts_1h_et + ON block_counts_1h (event_type, bucket DESC); +CREATE INDEX IF NOT EXISTS idx_ticket_low_counts_1m_et + ON ticket_low_counts_1m (event_type, bucket DESC); +CREATE INDEX IF NOT EXISTS idx_ticket_low_counts_1h_et + ON ticket_low_counts_1h (event_type, bucket DESC); +CREATE INDEX IF NOT EXISTS idx_wp_pipeline_counts_1m_et + ON wp_pipeline_counts_1m (event_type, bucket DESC); +CREATE INDEX IF NOT EXISTS idx_wp_pipeline_counts_1h_et + ON wp_pipeline_counts_1h (event_type, bucket DESC); diff --git a/migrations/003_realtime_aggregates.sql b/migrations/003_realtime_aggregates.sql deleted file mode 100644 index 3989122..0000000 --- a/migrations/003_realtime_aggregates.sql +++ /dev/null @@ -1,14 +0,0 @@ --- Tighten continuous aggregate refresh policy. --- 30s end_offset + 30s schedule_interval ensures data is materialized quickly. --- We keep materialized_only=true (the default when views were created WITH NO DATA) --- because real-time mode (materialized_only=false) forces every aggregate query to --- also scan unmaterialized raw events, which is catastrophic under high write load. --- A 30-60s lag in aggregate data is acceptable for dashboard analytics. - -SELECT remove_continuous_aggregate_policy('event_stats_1m', if_exists => TRUE); -SELECT add_continuous_aggregate_policy('event_stats_1m', - start_offset => INTERVAL '10 minutes', - end_offset => INTERVAL '30 seconds', - schedule_interval => INTERVAL '30 seconds', - if_not_exists => TRUE -); diff --git a/migrations/004_hot_columns.sql b/migrations/004_hot_columns.sql deleted file mode 100644 index 45d2f5a..0000000 --- a/migrations/004_hot_columns.sql +++ /dev/null @@ -1,11 +0,0 @@ --- Hot columns: promote frequently-queried JSONB fields to top-level columns --- Nullable, no default — instant ALTER. NULL for existing rows, populated going forward. --- 7-day retention ages out NULLs. - -ALTER TABLE events ADD COLUMN IF NOT EXISTS slot INT; -ALTER TABLE events ADD COLUMN IF NOT EXISTS core SMALLINT; -ALTER TABLE events ADD COLUMN IF NOT EXISTS submission_id BIGINT; - -CREATE INDEX IF NOT EXISTS idx_events_slot ON events (slot, timestamp DESC) WHERE slot IS NOT NULL; -CREATE INDEX IF NOT EXISTS idx_events_core ON events (core, timestamp DESC) WHERE core IS NOT NULL; -CREATE INDEX IF NOT EXISTS idx_events_submission_id ON events (node_id, submission_id, timestamp DESC) WHERE submission_id IS NOT NULL; diff --git a/migrations/004_union_views.sql b/migrations/004_union_views.sql new file mode 100644 index 0000000..147fc6b --- /dev/null +++ b/migrations/004_union_views.sql @@ -0,0 +1,65 @@ +-- UNION views: transparent query interface over the 14 count-table groups. +-- Grafana endpoints auto-select the tier (30s raw / 1m / 1h) by time range. + +-- 30s: raw count tables +CREATE VIEW all_event_stats_30s AS + SELECT bucket, node_id, event_type, event_count FROM status_counts + UNION ALL SELECT bucket, node_id, event_type, event_count FROM connection_counts + UNION ALL SELECT bucket, node_id, event_type, event_count FROM block_counts + UNION ALL SELECT bucket, node_id, event_type, event_count FROM ticket_low_counts + UNION ALL SELECT bucket, node_id, event_type, event_count FROM wp_pipeline_counts + UNION ALL SELECT bucket, node_id, event_type, event_count FROM block_distribution_counts + UNION ALL SELECT bucket, node_id, event_type, event_count FROM ticket_counts + UNION ALL SELECT bucket, node_id, event_type, event_count FROM guarantee_sending_counts + UNION ALL SELECT bucket, node_id, event_type, event_count FROM guarantee_receiving_counts + UNION ALL SELECT bucket, node_id, event_type, event_count FROM shard_counts + UNION ALL SELECT bucket, node_id, event_type, event_count FROM assurance_counts + UNION ALL SELECT bucket, node_id, event_type, event_count FROM bundle_counts + UNION ALL SELECT bucket, node_id, event_type, event_count FROM segment_counts + UNION ALL SELECT bucket, node_id, event_type, event_count FROM preimage_counts; + +-- 1m: aggregated continuous aggregates +CREATE VIEW all_event_stats_1m AS + SELECT bucket, node_id, event_type, event_count FROM status_counts_1m + UNION ALL SELECT bucket, node_id, event_type, event_count FROM connection_counts_1m + UNION ALL SELECT bucket, node_id, event_type, event_count FROM block_counts_1m + UNION ALL SELECT bucket, node_id, event_type, event_count FROM ticket_low_counts_1m + UNION ALL SELECT bucket, node_id, event_type, event_count FROM wp_pipeline_counts_1m + UNION ALL SELECT bucket, node_id, event_type, event_count FROM block_distribution_counts_1m + UNION ALL SELECT bucket, node_id, event_type, event_count FROM ticket_counts_1m + UNION ALL SELECT bucket, node_id, event_type, event_count FROM guarantee_sending_counts_1m + UNION ALL SELECT bucket, node_id, event_type, event_count FROM guarantee_receiving_counts_1m + UNION ALL SELECT bucket, node_id, event_type, event_count FROM shard_counts_1m + UNION ALL SELECT bucket, node_id, event_type, event_count FROM assurance_counts_1m + UNION ALL SELECT bucket, node_id, event_type, event_count FROM bundle_counts_1m + UNION ALL SELECT bucket, node_id, event_type, event_count FROM segment_counts_1m + UNION ALL SELECT bucket, node_id, event_type, event_count FROM preimage_counts_1m; + +-- 1h: aggregated continuous aggregates +CREATE VIEW all_event_stats_1h AS + SELECT bucket, node_id, event_type, event_count FROM status_counts_1h + UNION ALL SELECT bucket, node_id, event_type, event_count FROM connection_counts_1h + UNION ALL SELECT bucket, node_id, event_type, event_count FROM block_counts_1h + UNION ALL SELECT bucket, node_id, event_type, event_count FROM ticket_low_counts_1h + UNION ALL SELECT bucket, node_id, event_type, event_count FROM wp_pipeline_counts_1h + UNION ALL SELECT bucket, node_id, event_type, event_count FROM block_distribution_counts_1h + UNION ALL SELECT bucket, node_id, event_type, event_count FROM ticket_counts_1h + UNION ALL SELECT bucket, node_id, event_type, event_count FROM guarantee_sending_counts_1h + UNION ALL SELECT bucket, node_id, event_type, event_count FROM guarantee_receiving_counts_1h + UNION ALL SELECT bucket, node_id, event_type, event_count FROM shard_counts_1h + UNION ALL SELECT bucket, node_id, event_type, event_count FROM assurance_counts_1h + UNION ALL SELECT bucket, node_id, event_type, event_count FROM bundle_counts_1h + UNION ALL SELECT bucket, node_id, event_type, event_count FROM segment_counts_1h + UNION ALL SELECT bucket, node_id, event_type, event_count FROM preimage_counts_1h; + +-- Core-aware UNION view (for timeseries?group_by=core and core=X filter) +-- Only tables with a core column participate. +-- Uses raw count tables (not _1m aggregates) because the continuous aggregates +-- with node_id GROUP BY would double-count; the raw tables carry core directly. +CREATE VIEW all_core_stats_1m AS + SELECT bucket, event_type, core, event_count + FROM guarantee_sending_counts WHERE core IS NOT NULL + UNION ALL SELECT bucket, event_type, core, event_count + FROM segment_counts WHERE core IS NOT NULL + UNION ALL SELECT bucket, event_type, core, event_count + FROM wp_pipeline_counts WHERE core IS NOT NULL; diff --git a/migrations/005_30s_aggregate.sql b/migrations/005_30s_aggregate.sql deleted file mode 100644 index ae25230..0000000 --- a/migrations/005_30s_aggregate.sql +++ /dev/null @@ -1,22 +0,0 @@ --- 30-second continuous aggregate: finest granularity for debugging --- Only this aggregate scans raw events. 1m and 1h are hierarchical (from 30s/1m). - -CREATE MATERIALIZED VIEW IF NOT EXISTS event_stats_30s -WITH (timescaledb.continuous) AS -SELECT - time_bucket('30 seconds', timestamp) AS bucket, - node_id, event_type, - COUNT(*) AS event_count, - MIN(timestamp) AS first_event, - MAX(timestamp) AS last_event -FROM events -GROUP BY bucket, node_id, event_type -WITH NO DATA; - -SELECT add_continuous_aggregate_policy('event_stats_30s', - start_offset => INTERVAL '5 minutes', - end_offset => INTERVAL '1 minute', - schedule_interval => INTERVAL '1 minute', - if_not_exists => TRUE); - -SELECT add_retention_policy('event_stats_30s', INTERVAL '3 days', if_not_exists => TRUE); diff --git a/migrations/005_node_stats_services.sql b/migrations/005_node_stats_services.sql new file mode 100644 index 0000000..bf9c88f --- /dev/null +++ b/migrations/005_node_stats_services.sql @@ -0,0 +1,146 @@ +-- Node stats (Status event extraction) and per-service junction table, +-- each with a 1-minute continuous aggregate. + +-- ============================================================ +-- node_stats: extracts Status (event 10) fields at ingestion time. +-- Avoids JSONB queries for peer counts, DA storage, preimages. +-- ~512 rows/sec (Status fires every 2s, 1,023 nodes). Tiny rows (~50 bytes). +-- ============================================================ +CREATE TABLE IF NOT EXISTS node_stats ( + timestamp TIMESTAMPTZ NOT NULL, + node_id TEXT NOT NULL, + num_peers INT NOT NULL, + num_val_peers INT NOT NULL, + num_sync_peers INT NOT NULL, + num_shards INT NOT NULL, + shards_size BIGINT NOT NULL, + num_preimages INT NOT NULL, + preimages_size INT NOT NULL, + -- Guarantee pool: scalar summaries only (array dropped) + min_guarantees SMALLINT NOT NULL, + max_guarantees SMALLINT NOT NULL, + avg_guarantees REAL NOT NULL, + zero_guarantee_cores SMALLINT NOT NULL +); + +SELECT create_hypertable('node_stats', 'timestamp', + chunk_time_interval => INTERVAL '1 hour', + create_default_indexes => FALSE, + if_not_exists => TRUE +); + +CREATE INDEX IF NOT EXISTS idx_node_stats_node_time ON node_stats (node_id, timestamp DESC); +CREATE INDEX IF NOT EXISTS idx_node_stats_time ON node_stats (timestamp DESC); + +ALTER TABLE node_stats SET ( + timescaledb.compress, + timescaledb.compress_segmentby = 'node_id', + timescaledb.compress_orderby = 'timestamp DESC' +); + +SELECT add_compression_policy('node_stats', INTERVAL '2 hours', if_not_exists => TRUE); +SELECT add_retention_policy('node_stats', INTERVAL '7 days', if_not_exists => TRUE); + +-- Node stats aggregate: AVG/MIN/MAX per node per minute. +-- For longer time ranges and network-wide views. +CREATE MATERIALIZED VIEW IF NOT EXISTS node_stats_1m +WITH (timescaledb.continuous) AS +SELECT + time_bucket('1 minute', timestamp) AS bucket, + node_id, + AVG(num_peers)::INT AS avg_peers, + MIN(num_peers) AS min_peers, + MAX(num_peers) AS max_peers, + AVG(num_val_peers)::INT AS avg_val_peers, + MIN(num_val_peers) AS min_val_peers, + MAX(num_val_peers) AS max_val_peers, + AVG(num_sync_peers)::INT AS avg_sync_peers, + MIN(num_sync_peers) AS min_sync_peers, + MAX(num_sync_peers) AS max_sync_peers, + AVG(num_shards)::INT AS avg_shards, + MIN(num_shards) AS min_shards, + MAX(num_shards) AS max_shards, + AVG(shards_size)::BIGINT AS avg_shards_size, + MAX(shards_size) AS max_shards_size, + AVG(num_preimages)::INT AS avg_preimages, + MAX(num_preimages) AS max_preimages, + AVG(preimages_size)::INT AS avg_preimages_size, + MAX(preimages_size) AS max_preimages_size, + -- Guarantee pool scalars (from pre-computed columns) + AVG(avg_guarantees) AS avg_guarantees, + MIN(min_guarantees) AS min_guarantees, + MAX(max_guarantees) AS max_guarantees, + MAX(zero_guarantee_cores) AS max_zero_guarantee_cores, + COUNT(*) AS status_count +FROM node_stats +GROUP BY bucket, node_id +WITH NO DATA; + +SELECT add_continuous_aggregate_policy('node_stats_1m', + start_offset => INTERVAL '10 minutes', + end_offset => INTERVAL '2 minutes', + schedule_interval => INTERVAL '2 minutes', + if_not_exists => TRUE); + +SELECT add_retention_policy('node_stats_1m', INTERVAL '30 days', if_not_exists => TRUE); + +ALTER MATERIALIZED VIEW node_stats_1m SET (timescaledb.materialized_only = false); + +-- node_stats_1m: per-node drill-down queries (1024 nodes, huge selectivity gain) +CREATE INDEX IF NOT EXISTS idx_node_stats_1m_node + ON node_stats_1m (node_id, bucket DESC); + +-- ============================================================ +-- event_services: service junction table, one row per service per event. +-- Only low-volume pipeline events written (~13.5K rows/slot, ~2.3K rows/sec). +-- gas_used populated for gas-bearing events (Authorized=95, Refined=101, BlockExecuted=47). +-- elapsed_ns/load_ns: execution timing for types 47, 95, 101. +-- ============================================================ +CREATE TABLE IF NOT EXISTS event_services ( + timestamp TIMESTAMPTZ NOT NULL, + node_id TEXT NOT NULL, + event_type SMALLINT NOT NULL, + service_id INT NOT NULL, + gas_used BIGINT, -- NULL for events without gas data + elapsed_ns BIGINT, -- total wall-clock execution time (from ExecCost.ns) + load_ns BIGINT -- PVM code loading/compilation time +); + +SELECT create_hypertable('event_services', 'timestamp', + chunk_time_interval => INTERVAL '1 hour', + create_default_indexes => FALSE, + if_not_exists => TRUE +); + +CREATE INDEX IF NOT EXISTS idx_event_services_service ON event_services (service_id, timestamp DESC); +CREATE INDEX IF NOT EXISTS idx_event_services_time ON event_services (timestamp DESC); + +-- Retention matches raw events' historical value (no compression on purpose) +SELECT add_retention_policy('event_services', INTERVAL '7 days', if_not_exists => TRUE); + +-- Service stats aggregate: per-service event counts and gas totals per minute. +CREATE MATERIALIZED VIEW IF NOT EXISTS service_stats_1m +WITH (timescaledb.continuous) AS +SELECT + time_bucket('1 minute', timestamp) AS bucket, + service_id, + event_type, + COUNT(*) AS event_count, + SUM(gas_used) AS total_gas +FROM event_services +GROUP BY bucket, service_id, event_type +WITH NO DATA; + +SELECT add_continuous_aggregate_policy('service_stats_1m', + start_offset => INTERVAL '10 minutes', + end_offset => INTERVAL '2 minutes', + schedule_interval => INTERVAL '2 minutes', + if_not_exists => TRUE); + +SELECT add_retention_policy('service_stats_1m', INTERVAL '30 days', if_not_exists => TRUE); + +ALTER MATERIALIZED VIEW service_stats_1m SET (timescaledb.materialized_only = false); + +-- service_stats_1m: queries filter by service_id +CREATE INDEX IF NOT EXISTS idx_service_stats_1m_svc + ON service_stats_1m (service_id, bucket DESC); diff --git a/migrations/006_hierarchical_1m.sql b/migrations/006_hierarchical_1m.sql deleted file mode 100644 index 93a92d7..0000000 --- a/migrations/006_hierarchical_1m.sql +++ /dev/null @@ -1,50 +0,0 @@ --- Rebuild event_stats_1m as hierarchical aggregate FROM event_stats_30s --- and event_stats_1h FROM event_stats_1m. --- This eliminates the double raw-event scan (previously both 1m and 30s scanned raw). --- Chain: raw events -> 30s -> 1m -> 1h - --- Drop existing aggregates (1h depends on 1m, so drop 1h first) -DROP MATERIALIZED VIEW IF EXISTS event_stats_1h CASCADE; -DROP MATERIALIZED VIEW IF EXISTS event_stats_1m CASCADE; - --- Recreate 1m from 30s (hierarchical) -CREATE MATERIALIZED VIEW event_stats_1m -WITH (timescaledb.continuous) AS -SELECT - time_bucket('1 minute', bucket) AS bucket, - node_id, event_type, - SUM(event_count) AS event_count, - MIN(first_event) AS first_event, - MAX(last_event) AS last_event -FROM event_stats_30s -GROUP BY 1, node_id, event_type -WITH NO DATA; - -SELECT add_continuous_aggregate_policy('event_stats_1m', - start_offset => INTERVAL '10 minutes', - end_offset => INTERVAL '2 minutes', - schedule_interval => INTERVAL '2 minutes', - if_not_exists => TRUE); - -SELECT add_retention_policy('event_stats_1m', INTERVAL '30 days', if_not_exists => TRUE); - --- Recreate 1h from 1m (hierarchical, unchanged logic) -CREATE MATERIALIZED VIEW event_stats_1h -WITH (timescaledb.continuous) AS -SELECT - time_bucket('1 hour', bucket) AS bucket, - node_id, event_type, - SUM(event_count) AS event_count, - MIN(first_event) AS first_event, - MAX(last_event) AS last_event -FROM event_stats_1m -GROUP BY 1, node_id, event_type -WITH NO DATA; - -SELECT add_continuous_aggregate_policy('event_stats_1h', - start_offset => INTERVAL '4 hours', - end_offset => INTERVAL '1 hour', - schedule_interval => INTERVAL '1 hour', - if_not_exists => TRUE); - -SELECT add_retention_policy('event_stats_1h', INTERVAL '365 days', if_not_exists => TRUE); diff --git a/migrations/006_trackers.sql b/migrations/006_trackers.sql new file mode 100644 index 0000000..cb33a82 --- /dev/null +++ b/migrations/006_trackers.sql @@ -0,0 +1,405 @@ +-- Tracker-populated tables: slot/guarantee/assurance convergence, work-package +-- pipeline tracking, and DA operational stats + latency histograms. +-- +-- Histogram columns use CONVERGENCE_BOUNDS (23 buckets, 0ms-120s): +-- [0,2) [2,5) [5,10) [10,15) [15,20) [20,30) [30,50) [50,75) [75,100) +-- [100,150) [150,250) [250,500) [500,1k) [1k,2k) [2k,5k) [5k,10k) +-- [10k,15k) [15k,20k) [20k,25k) [25k,30k) [30k,60k) [60k,120k) [120k,+inf) + +-- ============================================================ +-- Slot convergence: pre-computed per-slot block propagation stats. +-- Populated at ingestion time by SlotTracker — no raw event scans needed. +-- ~4 rows per slot. Regular table (not hypertable). +-- ============================================================ +CREATE TABLE IF NOT EXISTS slot_convergence ( + slot INT NOT NULL, + event_type SMALLINT NOT NULL, + node_count SMALLINT NOT NULL, + p50_ms INT NOT NULL, + p99_ms INT NOT NULL, + p100_ms INT NOT NULL, + authored_at TIMESTAMPTZ NOT NULL, + p75_ms INT, + p95_ms INT, + PRIMARY KEY (slot, event_type) +); + +CREATE INDEX IF NOT EXISTS idx_slot_convergence_time ON slot_convergence (authored_at DESC); + +-- ============================================================ +-- Work package tracking: unique WP counting and pipeline funnel. +-- Regular table (NOT hypertable) — wp_hash is the true unique key. +-- ============================================================ +CREATE TABLE IF NOT EXISTS wp_tracking ( + wp_hash BYTEA PRIMARY KEY, + first_seen TIMESTAMPTZ NOT NULL, + last_updated TIMESTAMPTZ NOT NULL, + core SMALLINT NOT NULL, + service_ids INT[] NOT NULL, + -- Pipeline stage timestamps (NULL = not reached yet) + received_at TIMESTAMPTZ, + authorized_at TIMESTAMPTZ, + refined_at TIMESTAMPTZ, + report_built_at TIMESTAMPTZ, + guarantee_built_at TIMESTAMPTZ, + distributed_at TIMESTAMPTZ, + failed_at TIMESTAMPTZ, + -- Counts + received_by SMALLINT DEFAULT 0, + guaranteed_by SMALLINT DEFAULT 0, + -- Pipeline stage as explicit ordinal (NOT event_type number) + -- 0=received, 1=authorized, 2=refined, 3=report_built, 4=guarantee_built, 5=distributed + stage SMALLINT NOT NULL, + -- node_id: which node first received this WP (from WorkPackageReceived event) + node_id TEXT, + -- refine_gas_used: total gas from Refined event (SUM of costs[].total.gas_used) + refine_gas_used BIGINT, + -- failure_reason: from WorkPackageFailed event reason field + failure_reason TEXT, + -- discard_reason: from GuaranteeDiscarded event via guarantee_convergence wp_hash mapping + discard_reason TEXT +); + +CREATE INDEX IF NOT EXISTS idx_wp_tracking_time ON wp_tracking (first_seen DESC); +CREATE INDEX IF NOT EXISTS idx_wp_tracking_core ON wp_tracking (core, first_seen DESC); +CREATE INDEX IF NOT EXISTS idx_wp_tracking_stage ON wp_tracking (stage, first_seen DESC); +-- Partial index for wp-active queries: only rows that haven't completed or failed +CREATE INDEX IF NOT EXISTS idx_wp_tracking_active + ON wp_tracking (first_seen DESC) + WHERE distributed_at IS NULL AND failed_at IS NULL; + +-- ============================================================ +-- Guarantee convergence (per work_report_hash). +-- Measures: GuaranteeBuilt(105) -> GuaranteeReceived(112) propagation latency. +-- ============================================================ +CREATE TABLE IF NOT EXISTS guarantee_convergence ( + work_report_hash BYTEA NOT NULL PRIMARY KEY, + slot INT NOT NULL, + core SMALLINT, -- nullable: NULL when guarantor not connected to telemetry + wp_hash BYTEA, + node_count SMALLINT NOT NULL, + p50_ms INT NOT NULL, + p75_ms INT, + p95_ms INT, + p99_ms INT NOT NULL, + p100_ms INT NOT NULL, + built_at TIMESTAMPTZ NOT NULL, + -- builder_node_id: for per-guarantor analysis + builder_node_id TEXT, + h_0_2 INT DEFAULT 0, + h_2_5 INT DEFAULT 0, + h_5_10 INT DEFAULT 0, + h_10_15 INT DEFAULT 0, + h_15_20 INT DEFAULT 0, + h_20_30 INT DEFAULT 0, + h_30_50 INT DEFAULT 0, + h_50_75 INT DEFAULT 0, + h_75_100 INT DEFAULT 0, + h_100_150 INT DEFAULT 0, + h_150_250 INT DEFAULT 0, + h_250_500 INT DEFAULT 0, + h_500_1000 INT DEFAULT 0, + h_1000_2000 INT DEFAULT 0, + h_2000_5000 INT DEFAULT 0, + h_5000_10000 INT DEFAULT 0, + h_10000_15000 INT DEFAULT 0, + h_15000_20000 INT DEFAULT 0, + h_20000_25000 INT DEFAULT 0, + h_25000_30000 INT DEFAULT 0, + h_30000_60000 INT DEFAULT 0, + h_60000_120000 INT DEFAULT 0, + h_120000_plus INT DEFAULT 0, + hist_total INT DEFAULT 0 +); + +CREATE INDEX IF NOT EXISTS idx_guarantee_convergence_time + ON guarantee_convergence (built_at DESC); +CREATE INDEX IF NOT EXISTS idx_guarantee_convergence_core + ON guarantee_convergence (core, built_at DESC); +CREATE INDEX IF NOT EXISTS idx_guarantee_convergence_wp + ON guarantee_convergence (wp_hash, built_at DESC); + +-- ============================================================ +-- Guarantee convergence per-slot summary. +-- ============================================================ +CREATE TABLE IF NOT EXISTS guarantee_convergence_slots ( + slot INT NOT NULL PRIMARY KEY, + slot_timestamp TIMESTAMPTZ, + guarantee_count SMALLINT NOT NULL, + node_count SMALLINT NOT NULL, + p50_ms INT, + p75_ms INT, + p95_ms INT, + p99_ms INT, + p100_ms INT, + built_at TIMESTAMPTZ NOT NULL +); + +CREATE INDEX IF NOT EXISTS idx_guarantee_conv_slots_time + ON guarantee_convergence_slots (built_at DESC); + +-- ============================================================ +-- Assurance convergence per-anchor summary. +-- Measures: DistributingAssurance(126) -> AssuranceReceived(131) per sender. +-- ============================================================ +CREATE TABLE IF NOT EXISTS assurance_convergence ( + anchor BYTEA NOT NULL PRIMARY KEY, + slot INT, + slot_timestamp TIMESTAMPTZ, + sender_count SMALLINT NOT NULL, + receiver_count INT NOT NULL, + -- Reception convergence (distribution->reception deltas, clamped to >= 0) + p50_ms INT NOT NULL, + p75_ms INT, + p95_ms INT, + p99_ms INT NOT NULL, + p100_ms INT NOT NULL, + -- Distribution start spread (relative to first distributor) + dist_start_p50_ms INT, + dist_start_p95_ms INT, + dist_start_p99_ms INT, + dist_start_p100_ms INT, + first_distributed_at TIMESTAMPTZ, + last_distributed_at TIMESTAMPTZ, + h_0_2 INT DEFAULT 0, + h_2_5 INT DEFAULT 0, + h_5_10 INT DEFAULT 0, + h_10_15 INT DEFAULT 0, + h_15_20 INT DEFAULT 0, + h_20_30 INT DEFAULT 0, + h_30_50 INT DEFAULT 0, + h_50_75 INT DEFAULT 0, + h_75_100 INT DEFAULT 0, + h_100_150 INT DEFAULT 0, + h_150_250 INT DEFAULT 0, + h_250_500 INT DEFAULT 0, + h_500_1000 INT DEFAULT 0, + h_1000_2000 INT DEFAULT 0, + h_2000_5000 INT DEFAULT 0, + h_5000_10000 INT DEFAULT 0, + h_10000_15000 INT DEFAULT 0, + h_15000_20000 INT DEFAULT 0, + h_20000_25000 INT DEFAULT 0, + h_25000_30000 INT DEFAULT 0, + h_30000_60000 INT DEFAULT 0, + h_60000_120000 INT DEFAULT 0, + h_120000_plus INT DEFAULT 0, + hist_total INT DEFAULT 0 +); + +CREATE INDEX IF NOT EXISTS idx_assurance_convergence_slot + ON assurance_convergence (slot DESC); +-- queries filter by first_distributed_at +CREATE INDEX IF NOT EXISTS idx_assurance_convergence_time + ON assurance_convergence (first_distributed_at); + +-- ============================================================ +-- Assurance convergence per-sender detail (hypertable, default chunking). +-- INSERT-only (no unique constraint — cross-chunk uniqueness impractical). +-- ============================================================ +CREATE TABLE IF NOT EXISTS assurance_convergence_senders ( + distributed_at TIMESTAMPTZ NOT NULL, + anchor BYTEA NOT NULL, + sender_node_id TEXT NOT NULL, + node_count SMALLINT NOT NULL, + p50_ms INT NOT NULL, + p75_ms INT, + p95_ms INT, + p99_ms INT NOT NULL, + p100_ms INT NOT NULL, + h_0_2 INT DEFAULT 0, + h_2_5 INT DEFAULT 0, + h_5_10 INT DEFAULT 0, + h_10_15 INT DEFAULT 0, + h_15_20 INT DEFAULT 0, + h_20_30 INT DEFAULT 0, + h_30_50 INT DEFAULT 0, + h_50_75 INT DEFAULT 0, + h_75_100 INT DEFAULT 0, + h_100_150 INT DEFAULT 0, + h_150_250 INT DEFAULT 0, + h_250_500 INT DEFAULT 0, + h_500_1000 INT DEFAULT 0, + h_1000_2000 INT DEFAULT 0, + h_2000_5000 INT DEFAULT 0, + h_5000_10000 INT DEFAULT 0, + h_10000_15000 INT DEFAULT 0, + h_15000_20000 INT DEFAULT 0, + h_20000_25000 INT DEFAULT 0, + h_25000_30000 INT DEFAULT 0, + h_30000_60000 INT DEFAULT 0, + h_60000_120000 INT DEFAULT 0, + h_120000_plus INT DEFAULT 0, + hist_total INT DEFAULT 0 +); + +SELECT create_hypertable('assurance_convergence_senders', 'distributed_at', if_not_exists => TRUE); + +CREATE INDEX IF NOT EXISTS idx_assurance_conv_senders_node + ON assurance_convergence_senders (sender_node_id, distributed_at DESC); + +-- ============================================================ +-- DA node stats: per-node shard event counts, latency averages, shard inventory. +-- Populated by da_tracker flush every 10s (hypertable, default chunking). +-- ============================================================ +CREATE TABLE IF NOT EXISTS da_node_stats ( + ts TIMESTAMPTZ NOT NULL, + node_id TEXT NOT NULL, + shard_requests_sent INT DEFAULT 0, + shard_requests_received INT DEFAULT 0, + shard_sent_confirmed INT DEFAULT 0, + shard_received_confirmed INT DEFAULT 0, + shards_transferred INT DEFAULT 0, + shard_failures INT DEFAULT 0, + preimage_ann_failures INT DEFAULT 0, + preimages_announced INT DEFAULT 0, + preimages_forgotten INT DEFAULT 0, + assurer_avg_latency_ms REAL, + assurer_latency_samples INT DEFAULT 0, + guarantor_avg_latency_ms REAL, + guarantor_latency_samples INT DEFAULT 0, + active_shards INT DEFAULT 0 +); + +SELECT create_hypertable('da_node_stats', 'ts', if_not_exists => TRUE); + +CREATE INDEX IF NOT EXISTS idx_da_node_stats_node + ON da_node_stats (node_id, ts DESC); + +-- ============================================================ +-- Shard latency histogram. 14 buckets (ms): [0,1) [1,2) [2,5) [5,10) [10,25) +-- [25,50) [50,100) [100,250) [250,500) [500,1000) [1000,2000) [2000,3000) +-- [3000,5000) [5000,inf). Side: 0=assurer (120->125), 1=guarantor (121->124). +-- (Hypertable, default chunking; deliberately no secondary index.) +-- ============================================================ +CREATE TABLE IF NOT EXISTS shard_latency_hist ( + ts TIMESTAMPTZ NOT NULL, + node_id TEXT NOT NULL, + side SMALLINT NOT NULL, + b_0_1 INT DEFAULT 0, + b_1_2 INT DEFAULT 0, + b_2_5 INT DEFAULT 0, + b_5_10 INT DEFAULT 0, + b_10_25 INT DEFAULT 0, + b_25_50 INT DEFAULT 0, + b_50_100 INT DEFAULT 0, + b_100_250 INT DEFAULT 0, + b_250_500 INT DEFAULT 0, + b_500_1000 INT DEFAULT 0, + b_1000_2000 INT DEFAULT 0, + b_2000_3000 INT DEFAULT 0, + b_3000_5000 INT DEFAULT 0, + b_5000_plus INT DEFAULT 0, + total_count INT DEFAULT 0, + failed_count INT DEFAULT 0 +); + +SELECT create_hypertable('shard_latency_hist', 'ts', if_not_exists => TRUE); + +-- ============================================================ +-- bundle_latency_hist: side 0=shard_req(140->145), 1=shard_resp(141->145), 2=full_req(148->153), 3=full_resp(149->153), 4=reconstruct(146->147), 5=e2e(140->147) +-- ============================================================ +CREATE TABLE IF NOT EXISTS bundle_latency_hist ( + ts TIMESTAMPTZ NOT NULL, + node_id TEXT NOT NULL, + side SMALLINT NOT NULL, + h_0_2 INT DEFAULT 0, + h_2_5 INT DEFAULT 0, + h_5_10 INT DEFAULT 0, + h_10_15 INT DEFAULT 0, + h_15_20 INT DEFAULT 0, + h_20_30 INT DEFAULT 0, + h_30_50 INT DEFAULT 0, + h_50_75 INT DEFAULT 0, + h_75_100 INT DEFAULT 0, + h_100_150 INT DEFAULT 0, + h_150_250 INT DEFAULT 0, + h_250_500 INT DEFAULT 0, + h_500_1000 INT DEFAULT 0, + h_1000_2000 INT DEFAULT 0, + h_2000_5000 INT DEFAULT 0, + h_5000_10000 INT DEFAULT 0, + h_10000_15000 INT DEFAULT 0, + h_15000_20000 INT DEFAULT 0, + h_20000_25000 INT DEFAULT 0, + h_25000_30000 INT DEFAULT 0, + h_30000_60000 INT DEFAULT 0, + h_60000_120000 INT DEFAULT 0, + h_120000_plus INT DEFAULT 0, + total_count INT DEFAULT 0, + failed_count INT DEFAULT 0 +); +SELECT create_hypertable('bundle_latency_hist', 'ts', chunk_time_interval => INTERVAL '1 hour', if_not_exists => TRUE); +CREATE INDEX IF NOT EXISTS idx_bundle_latency_hist_node ON bundle_latency_hist (node_id, ts DESC); + +-- ============================================================ +-- segment_latency_hist: side 0=shard_req(162->167), 1=shard_resp(163->167), 2=full_req(173->178), 3=full_resp(174->178), 4=reconstruct(168->170) +-- ============================================================ +CREATE TABLE IF NOT EXISTS segment_latency_hist ( + ts TIMESTAMPTZ NOT NULL, + node_id TEXT NOT NULL, + side SMALLINT NOT NULL, + h_0_2 INT DEFAULT 0, + h_2_5 INT DEFAULT 0, + h_5_10 INT DEFAULT 0, + h_10_15 INT DEFAULT 0, + h_15_20 INT DEFAULT 0, + h_20_30 INT DEFAULT 0, + h_30_50 INT DEFAULT 0, + h_50_75 INT DEFAULT 0, + h_75_100 INT DEFAULT 0, + h_100_150 INT DEFAULT 0, + h_150_250 INT DEFAULT 0, + h_250_500 INT DEFAULT 0, + h_500_1000 INT DEFAULT 0, + h_1000_2000 INT DEFAULT 0, + h_2000_5000 INT DEFAULT 0, + h_5000_10000 INT DEFAULT 0, + h_10000_15000 INT DEFAULT 0, + h_15000_20000 INT DEFAULT 0, + h_20000_25000 INT DEFAULT 0, + h_25000_30000 INT DEFAULT 0, + h_30000_60000 INT DEFAULT 0, + h_60000_120000 INT DEFAULT 0, + h_120000_plus INT DEFAULT 0, + total_count INT DEFAULT 0, + failed_count INT DEFAULT 0 +); +SELECT create_hypertable('segment_latency_hist', 'ts', chunk_time_interval => INTERVAL '1 hour', if_not_exists => TRUE); +CREATE INDEX IF NOT EXISTS idx_segment_latency_hist_node ON segment_latency_hist (node_id, ts DESC); + +-- ============================================================ +-- preimage_latency_hist: side 0=req(193->198), 1=resp(194->198) +-- ============================================================ +CREATE TABLE IF NOT EXISTS preimage_latency_hist ( + ts TIMESTAMPTZ NOT NULL, + node_id TEXT NOT NULL, + side SMALLINT NOT NULL, + h_0_2 INT DEFAULT 0, + h_2_5 INT DEFAULT 0, + h_5_10 INT DEFAULT 0, + h_10_15 INT DEFAULT 0, + h_15_20 INT DEFAULT 0, + h_20_30 INT DEFAULT 0, + h_30_50 INT DEFAULT 0, + h_50_75 INT DEFAULT 0, + h_75_100 INT DEFAULT 0, + h_100_150 INT DEFAULT 0, + h_150_250 INT DEFAULT 0, + h_250_500 INT DEFAULT 0, + h_500_1000 INT DEFAULT 0, + h_1000_2000 INT DEFAULT 0, + h_2000_5000 INT DEFAULT 0, + h_5000_10000 INT DEFAULT 0, + h_10000_15000 INT DEFAULT 0, + h_15000_20000 INT DEFAULT 0, + h_20000_25000 INT DEFAULT 0, + h_25000_30000 INT DEFAULT 0, + h_30000_60000 INT DEFAULT 0, + h_60000_120000 INT DEFAULT 0, + h_120000_plus INT DEFAULT 0, + total_count INT DEFAULT 0, + failed_count INT DEFAULT 0 +); +SELECT create_hypertable('preimage_latency_hist', 'ts', chunk_time_interval => INTERVAL '1 hour', if_not_exists => TRUE); +CREATE INDEX IF NOT EXISTS idx_preimage_latency_hist_node ON preimage_latency_hist (node_id, ts DESC); diff --git a/migrations/007_event_services.sql b/migrations/007_event_services.sql deleted file mode 100644 index ffc8637..0000000 --- a/migrations/007_event_services.sql +++ /dev/null @@ -1,23 +0,0 @@ --- Service junction table: one row per service per event. --- Only low-volume pipeline events written (~13.5K rows/slot, ~2.3K rows/sec). --- gas_used populated for gas-bearing events (Authorized=95, Refined=101, BlockExecuted=47). - -CREATE TABLE IF NOT EXISTS event_services ( - timestamp TIMESTAMPTZ NOT NULL, - node_id TEXT NOT NULL, - event_type SMALLINT NOT NULL, - service_id INT NOT NULL, - gas_used BIGINT -- NULL for events without gas data -); - -SELECT create_hypertable('event_services', 'timestamp', - chunk_time_interval => INTERVAL '1 hour', - create_default_indexes => FALSE, - if_not_exists => TRUE -); - -CREATE INDEX IF NOT EXISTS idx_event_services_service ON event_services (service_id, timestamp DESC); -CREATE INDEX IF NOT EXISTS idx_event_services_time ON event_services (timestamp DESC); - --- Retention matches raw events -SELECT add_retention_policy('event_services', INTERVAL '7 days', if_not_exists => TRUE); diff --git a/migrations/014_onchain_stats.sql b/migrations/007_onchain.sql similarity index 85% rename from migrations/014_onchain_stats.sql rename to migrations/007_onchain.sql index cf723d9..103e0f7 100644 --- a/migrations/014_onchain_stats.sql +++ b/migrations/007_onchain.sql @@ -105,19 +105,3 @@ CREATE TABLE onchain_finalization ( updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW() ); INSERT INTO onchain_finalization DEFAULT VALUES; - --- TODO: No continuous aggregates yet. --- --- On-chain stats produce ~1,400 rows/block (341 cores + ~50 services + --- 1024 validators). A 24h query scans ~14,400 rows per entity — fast for --- Postgres even without pre-computation. Compare to telemetry at 3M events/s --- where aggregates are essential. Also, different fields need different --- aggregation functions (SUM for gas, AVG for popularity, MAX for cumulative --- validators), so we want to see real usage patterns before committing to a --- schema. --- --- If Grafana queries get slow for large time ranges (>7d), add: --- onchain_core_stats_1m (SUM gas/da/imports/exports/bundle, AVG popularity) --- onchain_service_stats_1m (SUM all fields) --- onchain_validator_stats_1m (MAX all fields — epoch-cumulative) --- Non-breaking change: swap which table the query reads from. diff --git a/migrations/008_service_aggregate.sql b/migrations/008_service_aggregate.sql deleted file mode 100644 index 5768506..0000000 --- a/migrations/008_service_aggregate.sql +++ /dev/null @@ -1,21 +0,0 @@ --- Service stats aggregate: per-service event counts and gas totals per minute. - -CREATE MATERIALIZED VIEW IF NOT EXISTS service_stats_1m -WITH (timescaledb.continuous) AS -SELECT - time_bucket('1 minute', timestamp) AS bucket, - service_id, - event_type, - COUNT(*) AS event_count, - SUM(gas_used) AS total_gas -FROM event_services -GROUP BY bucket, service_id, event_type -WITH NO DATA; - -SELECT add_continuous_aggregate_policy('service_stats_1m', - start_offset => INTERVAL '10 minutes', - end_offset => INTERVAL '2 minutes', - schedule_interval => INTERVAL '2 minutes', - if_not_exists => TRUE); - -SELECT add_retention_policy('service_stats_1m', INTERVAL '30 days', if_not_exists => TRUE); diff --git a/migrations/009_core_aggregate.sql b/migrations/009_core_aggregate.sql deleted file mode 100644 index e16c7d2..0000000 --- a/migrations/009_core_aggregate.sql +++ /dev/null @@ -1,21 +0,0 @@ --- Core stats aggregate: per-core event counts per minute. --- Depends on hot column `core` from migration 004. - -CREATE MATERIALIZED VIEW IF NOT EXISTS core_stats_1m -WITH (timescaledb.continuous) AS -SELECT - time_bucket('1 minute', timestamp) AS bucket, - core, event_type, - COUNT(*) AS event_count -FROM events -WHERE core IS NOT NULL -GROUP BY bucket, core, event_type -WITH NO DATA; - -SELECT add_continuous_aggregate_policy('core_stats_1m', - start_offset => INTERVAL '10 minutes', - end_offset => INTERVAL '2 minutes', - schedule_interval => INTERVAL '2 minutes', - if_not_exists => TRUE); - -SELECT add_retention_policy('core_stats_1m', INTERVAL '30 days', if_not_exists => TRUE); diff --git a/migrations/010_node_stats.sql b/migrations/010_node_stats.sql deleted file mode 100644 index 3f5f3db..0000000 --- a/migrations/010_node_stats.sql +++ /dev/null @@ -1,39 +0,0 @@ --- Node stats table: extracts Status (event 10) fields at ingestion time. --- Avoids JSONB queries for peer counts, DA storage, preimages. --- ~512 rows/sec (Status fires every 2s, 1,023 nodes). Tiny rows (~50 bytes). - -CREATE TABLE IF NOT EXISTS node_stats ( - timestamp TIMESTAMPTZ NOT NULL, - node_id TEXT NOT NULL, - num_peers INT NOT NULL, - num_val_peers INT NOT NULL, - num_sync_peers INT NOT NULL, - num_shards INT NOT NULL, - shards_size BIGINT NOT NULL, - num_preimages INT NOT NULL, - preimages_size INT NOT NULL, - -- Guarantee pool: scalar summaries only (array dropped) - min_guarantees SMALLINT NOT NULL, - max_guarantees SMALLINT NOT NULL, - avg_guarantees REAL NOT NULL, - zero_guarantee_cores SMALLINT NOT NULL -); - -SELECT create_hypertable('node_stats', 'timestamp', - chunk_time_interval => INTERVAL '1 hour', - create_default_indexes => FALSE, - if_not_exists => TRUE -); - -CREATE INDEX IF NOT EXISTS idx_node_stats_node_time ON node_stats (node_id, timestamp DESC); -CREATE INDEX IF NOT EXISTS idx_node_stats_time ON node_stats (timestamp DESC); - --- Compression -ALTER TABLE node_stats SET ( - timescaledb.compress, - timescaledb.compress_segmentby = 'node_id', - timescaledb.compress_orderby = 'timestamp DESC' -); - -SELECT add_compression_policy('node_stats', INTERVAL '2 hours', if_not_exists => TRUE); -SELECT add_retention_policy('node_stats', INTERVAL '7 days', if_not_exists => TRUE); diff --git a/migrations/011_node_stats_aggregate.sql b/migrations/011_node_stats_aggregate.sql deleted file mode 100644 index 7539d6c..0000000 --- a/migrations/011_node_stats_aggregate.sql +++ /dev/null @@ -1,43 +0,0 @@ --- Node stats aggregate: AVG/MIN/MAX per node per minute. --- For longer time ranges and network-wide views. - -CREATE MATERIALIZED VIEW IF NOT EXISTS node_stats_1m -WITH (timescaledb.continuous) AS -SELECT - time_bucket('1 minute', timestamp) AS bucket, - node_id, - AVG(num_peers)::INT AS avg_peers, - MIN(num_peers) AS min_peers, - MAX(num_peers) AS max_peers, - AVG(num_val_peers)::INT AS avg_val_peers, - MIN(num_val_peers) AS min_val_peers, - MAX(num_val_peers) AS max_val_peers, - AVG(num_sync_peers)::INT AS avg_sync_peers, - MIN(num_sync_peers) AS min_sync_peers, - MAX(num_sync_peers) AS max_sync_peers, - AVG(num_shards)::INT AS avg_shards, - MIN(num_shards) AS min_shards, - MAX(num_shards) AS max_shards, - AVG(shards_size)::BIGINT AS avg_shards_size, - MAX(shards_size) AS max_shards_size, - AVG(num_preimages)::INT AS avg_preimages, - MAX(num_preimages) AS max_preimages, - AVG(preimages_size)::INT AS avg_preimages_size, - MAX(preimages_size) AS max_preimages_size, - -- Guarantee pool scalars (from pre-computed columns) - AVG(avg_guarantees) AS avg_guarantees, - MIN(min_guarantees) AS min_guarantees, - MAX(max_guarantees) AS max_guarantees, - MAX(zero_guarantee_cores) AS max_zero_guarantee_cores, - COUNT(*) AS status_count -FROM node_stats -GROUP BY bucket, node_id -WITH NO DATA; - -SELECT add_continuous_aggregate_policy('node_stats_1m', - start_offset => INTERVAL '10 minutes', - end_offset => INTERVAL '2 minutes', - schedule_interval => INTERVAL '2 minutes', - if_not_exists => TRUE); - -SELECT add_retention_policy('node_stats_1m', INTERVAL '30 days', if_not_exists => TRUE); diff --git a/migrations/012_slot_convergence.sql b/migrations/012_slot_convergence.sql deleted file mode 100644 index c1535ed..0000000 --- a/migrations/012_slot_convergence.sql +++ /dev/null @@ -1,17 +0,0 @@ --- Slot convergence: pre-computed per-slot block propagation stats. --- Populated at ingestion time by SlotTracker — no raw event scans needed. --- ~4 rows per slot (authored, announced, imported, executed). --- At 7-day retention: ~400K rows. Regular table (not hypertable). - -CREATE TABLE IF NOT EXISTS slot_convergence ( - slot INT NOT NULL, - event_type SMALLINT NOT NULL, - node_count SMALLINT NOT NULL, - p50_ms INT NOT NULL, - p99_ms INT NOT NULL, - p100_ms INT NOT NULL, - authored_at TIMESTAMPTZ NOT NULL, - PRIMARY KEY (slot, event_type) -); - -CREATE INDEX IF NOT EXISTS idx_slot_convergence_time ON slot_convergence (authored_at DESC); diff --git a/migrations/013_wp_tracking.sql b/migrations/013_wp_tracking.sql deleted file mode 100644 index 16f5b80..0000000 --- a/migrations/013_wp_tracking.sql +++ /dev/null @@ -1,29 +0,0 @@ --- Work package tracking: unique WP counting and pipeline funnel. --- Regular table (NOT hypertable) — wp_hash is the true unique key. --- At ~57 rows/sec with 7-day retention: ~34M rows max. - -CREATE TABLE IF NOT EXISTS wp_tracking ( - wp_hash BYTEA PRIMARY KEY, - first_seen TIMESTAMPTZ NOT NULL, - last_updated TIMESTAMPTZ NOT NULL, - core SMALLINT NOT NULL, - service_ids INT[] NOT NULL, - -- Pipeline stage timestamps (NULL = not reached yet) - received_at TIMESTAMPTZ, - authorized_at TIMESTAMPTZ, - refined_at TIMESTAMPTZ, - report_built_at TIMESTAMPTZ, - guarantee_built_at TIMESTAMPTZ, - distributed_at TIMESTAMPTZ, - failed_at TIMESTAMPTZ, - -- Counts - received_by SMALLINT DEFAULT 0, - guaranteed_by SMALLINT DEFAULT 0, - -- Pipeline stage as explicit ordinal (NOT event_type number) - -- 0=received, 1=authorized, 2=refined, 3=report_built, 4=guarantee_built, 5=distributed - stage SMALLINT NOT NULL -); - -CREATE INDEX IF NOT EXISTS idx_wp_tracking_time ON wp_tracking (first_seen DESC); -CREATE INDEX IF NOT EXISTS idx_wp_tracking_core ON wp_tracking (core, first_seen DESC); -CREATE INDEX IF NOT EXISTS idx_wp_tracking_stage ON wp_tracking (stage, first_seen DESC); diff --git a/migrations/015_rename_events.sql b/migrations/015_rename_events.sql deleted file mode 100644 index 32eb34a..0000000 --- a/migrations/015_rename_events.sql +++ /dev/null @@ -1,111 +0,0 @@ --- Rename raw events hypertable: events → ingested_raw_events. --- Pre-aggregated event types will be written to per-group count tables (migration 016). --- The VIEW alias preserves backward compatibility for sqlx migrations that reference 'events'. --- --- DB will be dropped and re-created, so no data migration needed. - --- 1. Drop continuous aggregates that reference 'events' directly. --- (event_stats_1m/1h are hierarchical from 30s, but core_stats_1m scans 'events' directly) -DROP MATERIALIZED VIEW IF EXISTS event_stats_1h CASCADE; -DROP MATERIALIZED VIEW IF EXISTS event_stats_1m CASCADE; -DROP MATERIALIZED VIEW IF EXISTS event_stats_30s CASCADE; -DROP MATERIALIZED VIEW IF EXISTS core_stats_1m CASCADE; - --- Also drop the events_view (from migration 002) that joins events with event_types -DROP VIEW IF EXISTS events_view CASCADE; - --- 2. Rename the hypertable -ALTER TABLE events RENAME TO ingested_raw_events; - --- 3. Create VIEW alias for backward compatibility --- (earlier migrations reference 'events' — VIEW satisfies those references) -CREATE VIEW events AS SELECT * FROM ingested_raw_events; - --- 4. Recreate event_stats_30s on ingested_raw_events -CREATE MATERIALIZED VIEW event_stats_30s -WITH (timescaledb.continuous) AS -SELECT - time_bucket('30 seconds', timestamp) AS bucket, - node_id, event_type, - COUNT(*) AS event_count, - MIN(timestamp) AS first_event, - MAX(timestamp) AS last_event -FROM ingested_raw_events -GROUP BY bucket, node_id, event_type -WITH NO DATA; - -SELECT add_continuous_aggregate_policy('event_stats_30s', - start_offset => INTERVAL '5 minutes', - end_offset => INTERVAL '1 minute', - schedule_interval => INTERVAL '1 minute', - if_not_exists => TRUE); - -SELECT add_retention_policy('event_stats_30s', INTERVAL '3 days', if_not_exists => TRUE); - --- 5. Recreate event_stats_1m (hierarchical from 30s) -CREATE MATERIALIZED VIEW event_stats_1m -WITH (timescaledb.continuous) AS -SELECT - time_bucket('1 minute', bucket) AS bucket, - node_id, event_type, - SUM(event_count) AS event_count, - MIN(first_event) AS first_event, - MAX(last_event) AS last_event -FROM event_stats_30s -GROUP BY 1, node_id, event_type -WITH NO DATA; - -SELECT add_continuous_aggregate_policy('event_stats_1m', - start_offset => INTERVAL '10 minutes', - end_offset => INTERVAL '2 minutes', - schedule_interval => INTERVAL '2 minutes', - if_not_exists => TRUE); - -SELECT add_retention_policy('event_stats_1m', INTERVAL '30 days', if_not_exists => TRUE); - --- 6. Recreate event_stats_1h (hierarchical from 1m) -CREATE MATERIALIZED VIEW event_stats_1h -WITH (timescaledb.continuous) AS -SELECT - time_bucket('1 hour', bucket) AS bucket, - node_id, event_type, - SUM(event_count) AS event_count, - MIN(first_event) AS first_event, - MAX(last_event) AS last_event -FROM event_stats_1m -GROUP BY 1, node_id, event_type -WITH NO DATA; - -SELECT add_continuous_aggregate_policy('event_stats_1h', - start_offset => INTERVAL '4 hours', - end_offset => INTERVAL '1 hour', - schedule_interval => INTERVAL '1 hour', - if_not_exists => TRUE); - -SELECT add_retention_policy('event_stats_1h', INTERVAL '365 days', if_not_exists => TRUE); - --- 7. Recreate core_stats_1m on ingested_raw_events -CREATE MATERIALIZED VIEW core_stats_1m -WITH (timescaledb.continuous) AS -SELECT - time_bucket('1 minute', timestamp) AS bucket, - core, event_type, - COUNT(*) AS event_count -FROM ingested_raw_events -WHERE core IS NOT NULL -GROUP BY bucket, core, event_type -WITH NO DATA; - -SELECT add_continuous_aggregate_policy('core_stats_1m', - start_offset => INTERVAL '10 minutes', - end_offset => INTERVAL '2 minutes', - schedule_interval => INTERVAL '2 minutes', - if_not_exists => TRUE); - -SELECT add_retention_policy('core_stats_1m', INTERVAL '30 days', if_not_exists => TRUE); - --- 8. Recreate events_view (from migration 002) -CREATE VIEW events_view AS - SELECT e.*, et.name AS event_type_name, et.group_name - FROM ingested_raw_events e - LEFT JOIN event_types et ON e.event_type = et.id; diff --git a/migrations/016_count_tables.sql b/migrations/016_count_tables.sql deleted file mode 100644 index a01643a..0000000 --- a/migrations/016_count_tables.sql +++ /dev/null @@ -1,544 +0,0 @@ --- Per-protocol-group count tables for pre-aggregated high-volume events. --- Events are counted in-memory (DashMap) and flushed every 5s via COPY BINARY. --- Append-only: multiple rows per logical key are correct because all query paths --- do SUM(event_count) GROUP BY ... - --- ============================================================ --- 1. block_distribution_counts (types 60-68) --- ============================================================ -CREATE TABLE block_distribution_counts ( - bucket TIMESTAMPTZ NOT NULL, - node_id TEXT NOT NULL, - event_type SMALLINT NOT NULL, - event_count BIGINT NOT NULL, - slot INT, - reason TEXT -); -SELECT create_hypertable('block_distribution_counts', 'bucket', chunk_time_interval => INTERVAL '1 day'); -ALTER TABLE block_distribution_counts ADD CHECK (event_type BETWEEN 60 AND 68); -CREATE INDEX ON block_distribution_counts (node_id, event_type, bucket DESC); -ALTER TABLE block_distribution_counts SET ( - timescaledb.compress, - timescaledb.compress_segmentby = 'node_id, event_type', - timescaledb.compress_orderby = 'bucket DESC' -); -SELECT add_compression_policy('block_distribution_counts', compress_after => INTERVAL '2 hours'); -SELECT add_retention_policy('block_distribution_counts', INTERVAL '3 days'); - -CREATE MATERIALIZED VIEW block_distribution_counts_1m -WITH (timescaledb.continuous) AS -SELECT - time_bucket('1 minute', bucket) AS bucket, - node_id, event_type, - SUM(event_count) AS event_count -FROM block_distribution_counts -GROUP BY 1, node_id, event_type -WITH NO DATA; - -SELECT add_continuous_aggregate_policy('block_distribution_counts_1m', - start_offset => INTERVAL '10 minutes', - end_offset => INTERVAL '2 minutes', - schedule_interval => INTERVAL '2 minutes'); -SELECT add_retention_policy('block_distribution_counts_1m', INTERVAL '30 days'); - -CREATE MATERIALIZED VIEW block_distribution_counts_1h -WITH (timescaledb.continuous) AS -SELECT - time_bucket('1 hour', bucket) AS bucket, - node_id, event_type, - SUM(event_count) AS event_count -FROM block_distribution_counts_1m -GROUP BY 1, node_id, event_type -WITH NO DATA; - -SELECT add_continuous_aggregate_policy('block_distribution_counts_1h', - start_offset => INTERVAL '4 hours', - end_offset => INTERVAL '1 hour', - schedule_interval => INTERVAL '1 hour'); -SELECT add_retention_policy('block_distribution_counts_1h', INTERVAL '365 days'); - --- ============================================================ --- 2. ticket_counts (types 83-84) --- ============================================================ -CREATE TABLE ticket_counts ( - bucket TIMESTAMPTZ NOT NULL, - node_id TEXT NOT NULL, - event_type SMALLINT NOT NULL, - event_count BIGINT NOT NULL, - reason TEXT, - from_proxy BOOLEAN, - epoch INT -); -SELECT create_hypertable('ticket_counts', 'bucket', chunk_time_interval => INTERVAL '1 day'); -ALTER TABLE ticket_counts ADD CHECK (event_type BETWEEN 83 AND 84); -CREATE INDEX ON ticket_counts (node_id, event_type, bucket DESC); -ALTER TABLE ticket_counts SET ( - timescaledb.compress, - timescaledb.compress_segmentby = 'node_id, event_type', - timescaledb.compress_orderby = 'bucket DESC' -); -SELECT add_compression_policy('ticket_counts', compress_after => INTERVAL '2 hours'); -SELECT add_retention_policy('ticket_counts', INTERVAL '3 days'); - -CREATE MATERIALIZED VIEW ticket_counts_1m -WITH (timescaledb.continuous) AS -SELECT - time_bucket('1 minute', bucket) AS bucket, - node_id, event_type, - SUM(event_count) AS event_count -FROM ticket_counts -GROUP BY 1, node_id, event_type -WITH NO DATA; - -SELECT add_continuous_aggregate_policy('ticket_counts_1m', - start_offset => INTERVAL '10 minutes', - end_offset => INTERVAL '2 minutes', - schedule_interval => INTERVAL '2 minutes'); -SELECT add_retention_policy('ticket_counts_1m', INTERVAL '30 days'); - -CREATE MATERIALIZED VIEW ticket_counts_1h -WITH (timescaledb.continuous) AS -SELECT - time_bucket('1 hour', bucket) AS bucket, - node_id, event_type, - SUM(event_count) AS event_count -FROM ticket_counts_1m -GROUP BY 1, node_id, event_type -WITH NO DATA; - -SELECT add_continuous_aggregate_policy('ticket_counts_1h', - start_offset => INTERVAL '4 hours', - end_offset => INTERVAL '1 hour', - schedule_interval => INTERVAL '1 hour'); -SELECT add_retention_policy('ticket_counts_1h', INTERVAL '365 days'); - --- ============================================================ --- 3. guarantee_sending_counts (types 106-108) --- ============================================================ -CREATE TABLE guarantee_sending_counts ( - bucket TIMESTAMPTZ NOT NULL, - node_id TEXT NOT NULL, - event_type SMALLINT NOT NULL, - event_count BIGINT NOT NULL, - core SMALLINT, - reason TEXT -); -SELECT create_hypertable('guarantee_sending_counts', 'bucket', chunk_time_interval => INTERVAL '1 day'); -ALTER TABLE guarantee_sending_counts ADD CHECK (event_type BETWEEN 106 AND 108); -CREATE INDEX ON guarantee_sending_counts (node_id, event_type, bucket DESC); -ALTER TABLE guarantee_sending_counts SET ( - timescaledb.compress, - timescaledb.compress_segmentby = 'node_id, event_type', - timescaledb.compress_orderby = 'bucket DESC' -); -SELECT add_compression_policy('guarantee_sending_counts', compress_after => INTERVAL '2 hours'); -SELECT add_retention_policy('guarantee_sending_counts', INTERVAL '3 days'); - -CREATE MATERIALIZED VIEW guarantee_sending_counts_1m -WITH (timescaledb.continuous) AS -SELECT - time_bucket('1 minute', bucket) AS bucket, - node_id, event_type, core, - SUM(event_count) AS event_count -FROM guarantee_sending_counts -GROUP BY 1, node_id, event_type, core -WITH NO DATA; - -SELECT add_continuous_aggregate_policy('guarantee_sending_counts_1m', - start_offset => INTERVAL '10 minutes', - end_offset => INTERVAL '2 minutes', - schedule_interval => INTERVAL '2 minutes'); -SELECT add_retention_policy('guarantee_sending_counts_1m', INTERVAL '30 days'); - -CREATE MATERIALIZED VIEW guarantee_sending_counts_1h -WITH (timescaledb.continuous) AS -SELECT - time_bucket('1 hour', bucket) AS bucket, - node_id, event_type, core, - SUM(event_count) AS event_count -FROM guarantee_sending_counts_1m -GROUP BY 1, node_id, event_type, core -WITH NO DATA; - -SELECT add_continuous_aggregate_policy('guarantee_sending_counts_1h', - start_offset => INTERVAL '4 hours', - end_offset => INTERVAL '1 hour', - schedule_interval => INTERVAL '1 hour'); -SELECT add_retention_policy('guarantee_sending_counts_1h', INTERVAL '365 days'); - --- ============================================================ --- 4. guarantee_receiving_counts (types 110-113) --- ============================================================ -CREATE TABLE guarantee_receiving_counts ( - bucket TIMESTAMPTZ NOT NULL, - node_id TEXT NOT NULL, - event_type SMALLINT NOT NULL, - event_count BIGINT NOT NULL, - slot INT, - reason TEXT -); -SELECT create_hypertable('guarantee_receiving_counts', 'bucket', chunk_time_interval => INTERVAL '1 day'); -ALTER TABLE guarantee_receiving_counts ADD CHECK (event_type BETWEEN 110 AND 113); -CREATE INDEX ON guarantee_receiving_counts (node_id, event_type, bucket DESC); --- Partial index for /guarantee-discards endpoint (no node_id leading column) -CREATE INDEX ON guarantee_receiving_counts (event_type, bucket DESC) WHERE reason IS NOT NULL; -ALTER TABLE guarantee_receiving_counts SET ( - timescaledb.compress, - timescaledb.compress_segmentby = 'node_id, event_type', - timescaledb.compress_orderby = 'bucket DESC' -); -SELECT add_compression_policy('guarantee_receiving_counts', compress_after => INTERVAL '2 hours'); -SELECT add_retention_policy('guarantee_receiving_counts', INTERVAL '3 days'); - -CREATE MATERIALIZED VIEW guarantee_receiving_counts_1m -WITH (timescaledb.continuous) AS -SELECT - time_bucket('1 minute', bucket) AS bucket, - node_id, event_type, - SUM(event_count) AS event_count -FROM guarantee_receiving_counts -GROUP BY 1, node_id, event_type -WITH NO DATA; - -SELECT add_continuous_aggregate_policy('guarantee_receiving_counts_1m', - start_offset => INTERVAL '10 minutes', - end_offset => INTERVAL '2 minutes', - schedule_interval => INTERVAL '2 minutes'); -SELECT add_retention_policy('guarantee_receiving_counts_1m', INTERVAL '30 days'); - -CREATE MATERIALIZED VIEW guarantee_receiving_counts_1h -WITH (timescaledb.continuous) AS -SELECT - time_bucket('1 hour', bucket) AS bucket, - node_id, event_type, - SUM(event_count) AS event_count -FROM guarantee_receiving_counts_1m -GROUP BY 1, node_id, event_type -WITH NO DATA; - -SELECT add_continuous_aggregate_policy('guarantee_receiving_counts_1h', - start_offset => INTERVAL '4 hours', - end_offset => INTERVAL '1 hour', - schedule_interval => INTERVAL '1 hour'); -SELECT add_retention_policy('guarantee_receiving_counts_1h', INTERVAL '365 days'); - --- ============================================================ --- 5. shard_counts (types 120-125) --- ============================================================ -CREATE TABLE shard_counts ( - bucket TIMESTAMPTZ NOT NULL, - node_id TEXT NOT NULL, - event_type SMALLINT NOT NULL, - event_count BIGINT NOT NULL, - reason TEXT -); -SELECT create_hypertable('shard_counts', 'bucket', chunk_time_interval => INTERVAL '1 day'); -ALTER TABLE shard_counts ADD CHECK (event_type BETWEEN 120 AND 125); -CREATE INDEX ON shard_counts (node_id, event_type, bucket DESC); -ALTER TABLE shard_counts SET ( - timescaledb.compress, - timescaledb.compress_segmentby = 'node_id, event_type', - timescaledb.compress_orderby = 'bucket DESC' -); -SELECT add_compression_policy('shard_counts', compress_after => INTERVAL '2 hours'); -SELECT add_retention_policy('shard_counts', INTERVAL '3 days'); - -CREATE MATERIALIZED VIEW shard_counts_1m -WITH (timescaledb.continuous) AS -SELECT - time_bucket('1 minute', bucket) AS bucket, - node_id, event_type, - SUM(event_count) AS event_count -FROM shard_counts -GROUP BY 1, node_id, event_type -WITH NO DATA; - -SELECT add_continuous_aggregate_policy('shard_counts_1m', - start_offset => INTERVAL '10 minutes', - end_offset => INTERVAL '2 minutes', - schedule_interval => INTERVAL '2 minutes'); -SELECT add_retention_policy('shard_counts_1m', INTERVAL '30 days'); - -CREATE MATERIALIZED VIEW shard_counts_1h -WITH (timescaledb.continuous) AS -SELECT - time_bucket('1 hour', bucket) AS bucket, - node_id, event_type, - SUM(event_count) AS event_count -FROM shard_counts_1m -GROUP BY 1, node_id, event_type -WITH NO DATA; - -SELECT add_continuous_aggregate_policy('shard_counts_1h', - start_offset => INTERVAL '4 hours', - end_offset => INTERVAL '1 hour', - schedule_interval => INTERVAL '1 hour'); -SELECT add_retention_policy('shard_counts_1h', INTERVAL '365 days'); - --- ============================================================ --- 6. assurance_counts (types 126-131) --- ============================================================ -CREATE TABLE assurance_counts ( - bucket TIMESTAMPTZ NOT NULL, - node_id TEXT NOT NULL, - event_type SMALLINT NOT NULL, - event_count BIGINT NOT NULL, - reason TEXT -); -SELECT create_hypertable('assurance_counts', 'bucket', chunk_time_interval => INTERVAL '1 day'); -ALTER TABLE assurance_counts ADD CHECK (event_type BETWEEN 126 AND 131); -CREATE INDEX ON assurance_counts (node_id, event_type, bucket DESC); -ALTER TABLE assurance_counts SET ( - timescaledb.compress, - timescaledb.compress_segmentby = 'node_id, event_type', - timescaledb.compress_orderby = 'bucket DESC' -); -SELECT add_compression_policy('assurance_counts', compress_after => INTERVAL '2 hours'); -SELECT add_retention_policy('assurance_counts', INTERVAL '3 days'); - -CREATE MATERIALIZED VIEW assurance_counts_1m -WITH (timescaledb.continuous) AS -SELECT - time_bucket('1 minute', bucket) AS bucket, - node_id, event_type, - SUM(event_count) AS event_count -FROM assurance_counts -GROUP BY 1, node_id, event_type -WITH NO DATA; - -SELECT add_continuous_aggregate_policy('assurance_counts_1m', - start_offset => INTERVAL '10 minutes', - end_offset => INTERVAL '2 minutes', - schedule_interval => INTERVAL '2 minutes'); -SELECT add_retention_policy('assurance_counts_1m', INTERVAL '30 days'); - -CREATE MATERIALIZED VIEW assurance_counts_1h -WITH (timescaledb.continuous) AS -SELECT - time_bucket('1 hour', bucket) AS bucket, - node_id, event_type, - SUM(event_count) AS event_count -FROM assurance_counts_1m -GROUP BY 1, node_id, event_type -WITH NO DATA; - -SELECT add_continuous_aggregate_policy('assurance_counts_1h', - start_offset => INTERVAL '4 hours', - end_offset => INTERVAL '1 hour', - schedule_interval => INTERVAL '1 hour'); -SELECT add_retention_policy('assurance_counts_1h', INTERVAL '365 days'); - --- ============================================================ --- 7. bundle_counts (types 140-153) --- ============================================================ -CREATE TABLE bundle_counts ( - bucket TIMESTAMPTZ NOT NULL, - node_id TEXT NOT NULL, - event_type SMALLINT NOT NULL, - event_count BIGINT NOT NULL, - reason TEXT, - kind SMALLINT -); -SELECT create_hypertable('bundle_counts', 'bucket', chunk_time_interval => INTERVAL '1 day'); -ALTER TABLE bundle_counts ADD CHECK (event_type BETWEEN 140 AND 153); -CREATE INDEX ON bundle_counts (node_id, event_type, bucket DESC); -ALTER TABLE bundle_counts SET ( - timescaledb.compress, - timescaledb.compress_segmentby = 'node_id, event_type', - timescaledb.compress_orderby = 'bucket DESC' -); -SELECT add_compression_policy('bundle_counts', compress_after => INTERVAL '2 hours'); -SELECT add_retention_policy('bundle_counts', INTERVAL '3 days'); - -CREATE MATERIALIZED VIEW bundle_counts_1m -WITH (timescaledb.continuous) AS -SELECT - time_bucket('1 minute', bucket) AS bucket, - node_id, event_type, - SUM(event_count) AS event_count -FROM bundle_counts -GROUP BY 1, node_id, event_type -WITH NO DATA; - -SELECT add_continuous_aggregate_policy('bundle_counts_1m', - start_offset => INTERVAL '10 minutes', - end_offset => INTERVAL '2 minutes', - schedule_interval => INTERVAL '2 minutes'); -SELECT add_retention_policy('bundle_counts_1m', INTERVAL '30 days'); - -CREATE MATERIALIZED VIEW bundle_counts_1h -WITH (timescaledb.continuous) AS -SELECT - time_bucket('1 hour', bucket) AS bucket, - node_id, event_type, - SUM(event_count) AS event_count -FROM bundle_counts_1m -GROUP BY 1, node_id, event_type -WITH NO DATA; - -SELECT add_continuous_aggregate_policy('bundle_counts_1h', - start_offset => INTERVAL '4 hours', - end_offset => INTERVAL '1 hour', - schedule_interval => INTERVAL '1 hour'); -SELECT add_retention_policy('bundle_counts_1h', INTERVAL '365 days'); - --- ============================================================ --- 8. segment_counts (types 160-178) --- ============================================================ -CREATE TABLE segment_counts ( - bucket TIMESTAMPTZ NOT NULL, - node_id TEXT NOT NULL, - event_type SMALLINT NOT NULL, - event_count BIGINT NOT NULL, - core SMALLINT, - reason TEXT, - kind SMALLINT -); -SELECT create_hypertable('segment_counts', 'bucket', chunk_time_interval => INTERVAL '1 day'); -ALTER TABLE segment_counts ADD CHECK (event_type BETWEEN 160 AND 178); -CREATE INDEX ON segment_counts (node_id, event_type, bucket DESC); -ALTER TABLE segment_counts SET ( - timescaledb.compress, - timescaledb.compress_segmentby = 'node_id, event_type', - timescaledb.compress_orderby = 'bucket DESC' -); -SELECT add_compression_policy('segment_counts', compress_after => INTERVAL '2 hours'); -SELECT add_retention_policy('segment_counts', INTERVAL '3 days'); - -CREATE MATERIALIZED VIEW segment_counts_1m -WITH (timescaledb.continuous) AS -SELECT - time_bucket('1 minute', bucket) AS bucket, - node_id, event_type, core, - SUM(event_count) AS event_count -FROM segment_counts -GROUP BY 1, node_id, event_type, core -WITH NO DATA; - -SELECT add_continuous_aggregate_policy('segment_counts_1m', - start_offset => INTERVAL '10 minutes', - end_offset => INTERVAL '2 minutes', - schedule_interval => INTERVAL '2 minutes'); -SELECT add_retention_policy('segment_counts_1m', INTERVAL '30 days'); - -CREATE MATERIALIZED VIEW segment_counts_1h -WITH (timescaledb.continuous) AS -SELECT - time_bucket('1 hour', bucket) AS bucket, - node_id, event_type, core, - SUM(event_count) AS event_count -FROM segment_counts_1m -GROUP BY 1, node_id, event_type, core -WITH NO DATA; - -SELECT add_continuous_aggregate_policy('segment_counts_1h', - start_offset => INTERVAL '4 hours', - end_offset => INTERVAL '1 hour', - schedule_interval => INTERVAL '1 hour'); -SELECT add_retention_policy('segment_counts_1h', INTERVAL '365 days'); - --- ============================================================ --- 9. preimage_counts (types 190-199) --- ============================================================ -CREATE TABLE preimage_counts ( - bucket TIMESTAMPTZ NOT NULL, - node_id TEXT NOT NULL, - event_type SMALLINT NOT NULL, - event_count BIGINT NOT NULL, - reason TEXT, - service_id INT -); -SELECT create_hypertable('preimage_counts', 'bucket', chunk_time_interval => INTERVAL '1 day'); -ALTER TABLE preimage_counts ADD CHECK (event_type BETWEEN 190 AND 199); -CREATE INDEX ON preimage_counts (node_id, event_type, bucket DESC); -ALTER TABLE preimage_counts SET ( - timescaledb.compress, - timescaledb.compress_segmentby = 'node_id, event_type', - timescaledb.compress_orderby = 'bucket DESC' -); -SELECT add_compression_policy('preimage_counts', compress_after => INTERVAL '2 hours'); -SELECT add_retention_policy('preimage_counts', INTERVAL '3 days'); - -CREATE MATERIALIZED VIEW preimage_counts_1m -WITH (timescaledb.continuous) AS -SELECT - time_bucket('1 minute', bucket) AS bucket, - node_id, event_type, - SUM(event_count) AS event_count -FROM preimage_counts -GROUP BY 1, node_id, event_type -WITH NO DATA; - -SELECT add_continuous_aggregate_policy('preimage_counts_1m', - start_offset => INTERVAL '10 minutes', - end_offset => INTERVAL '2 minutes', - schedule_interval => INTERVAL '2 minutes'); -SELECT add_retention_policy('preimage_counts_1m', INTERVAL '30 days'); - -CREATE MATERIALIZED VIEW preimage_counts_1h -WITH (timescaledb.continuous) AS -SELECT - time_bucket('1 hour', bucket) AS bucket, - node_id, event_type, - SUM(event_count) AS event_count -FROM preimage_counts_1m -GROUP BY 1, node_id, event_type -WITH NO DATA; - -SELECT add_continuous_aggregate_policy('preimage_counts_1h', - start_offset => INTERVAL '4 hours', - end_offset => INTERVAL '1 hour', - schedule_interval => INTERVAL '1 hour'); -SELECT add_retention_policy('preimage_counts_1h', INTERVAL '365 days'); - --- ============================================================ --- UNION views: transparent query interface combining raw + pre-aggregated --- ============================================================ - --- 30s: raw event_stats_30s + raw count tables -CREATE VIEW all_event_stats_30s AS - SELECT bucket, node_id, event_type, event_count FROM event_stats_30s - UNION ALL SELECT bucket, node_id, event_type, event_count FROM block_distribution_counts - UNION ALL SELECT bucket, node_id, event_type, event_count FROM ticket_counts - UNION ALL SELECT bucket, node_id, event_type, event_count FROM guarantee_sending_counts - UNION ALL SELECT bucket, node_id, event_type, event_count FROM guarantee_receiving_counts - UNION ALL SELECT bucket, node_id, event_type, event_count FROM shard_counts - UNION ALL SELECT bucket, node_id, event_type, event_count FROM assurance_counts - UNION ALL SELECT bucket, node_id, event_type, event_count FROM bundle_counts - UNION ALL SELECT bucket, node_id, event_type, event_count FROM segment_counts - UNION ALL SELECT bucket, node_id, event_type, event_count FROM preimage_counts; - --- 1m: aggregated -CREATE VIEW all_event_stats_1m AS - SELECT bucket, node_id, event_type, event_count FROM event_stats_1m - UNION ALL SELECT bucket, node_id, event_type, event_count FROM block_distribution_counts_1m - UNION ALL SELECT bucket, node_id, event_type, event_count FROM ticket_counts_1m - UNION ALL SELECT bucket, node_id, event_type, event_count FROM guarantee_sending_counts_1m - UNION ALL SELECT bucket, node_id, event_type, event_count FROM guarantee_receiving_counts_1m - UNION ALL SELECT bucket, node_id, event_type, event_count FROM shard_counts_1m - UNION ALL SELECT bucket, node_id, event_type, event_count FROM assurance_counts_1m - UNION ALL SELECT bucket, node_id, event_type, event_count FROM bundle_counts_1m - UNION ALL SELECT bucket, node_id, event_type, event_count FROM segment_counts_1m - UNION ALL SELECT bucket, node_id, event_type, event_count FROM preimage_counts_1m; - --- 1h: aggregated -CREATE VIEW all_event_stats_1h AS - SELECT bucket, node_id, event_type, event_count FROM event_stats_1h - UNION ALL SELECT bucket, node_id, event_type, event_count FROM block_distribution_counts_1h - UNION ALL SELECT bucket, node_id, event_type, event_count FROM ticket_counts_1h - UNION ALL SELECT bucket, node_id, event_type, event_count FROM guarantee_sending_counts_1h - UNION ALL SELECT bucket, node_id, event_type, event_count FROM guarantee_receiving_counts_1h - UNION ALL SELECT bucket, node_id, event_type, event_count FROM shard_counts_1h - UNION ALL SELECT bucket, node_id, event_type, event_count FROM assurance_counts_1h - UNION ALL SELECT bucket, node_id, event_type, event_count FROM bundle_counts_1h - UNION ALL SELECT bucket, node_id, event_type, event_count FROM segment_counts_1h - UNION ALL SELECT bucket, node_id, event_type, event_count FROM preimage_counts_1h; - --- Core-aware UNION view (for timeseries?group_by=core and core=X filter) --- core_stats_1m has no node_id column, so we project without it. -CREATE VIEW all_core_stats_1m AS - SELECT bucket, event_type, core, event_count FROM core_stats_1m - UNION ALL SELECT bucket, event_type, core, event_count - FROM guarantee_sending_counts_1m WHERE core IS NOT NULL - UNION ALL SELECT bucket, event_type, core, event_count - FROM segment_counts_1m WHERE core IS NOT NULL; diff --git a/migrations/017_realtime_aggregates.sql b/migrations/017_realtime_aggregates.sql deleted file mode 100644 index d06574c..0000000 --- a/migrations/017_realtime_aggregates.sql +++ /dev/null @@ -1,42 +0,0 @@ --- Enable real-time aggregation on all continuous aggregates. --- --- With materialized_only = false, TimescaleDB appends a live tail scan on the --- source table for the un-materialized time window (last 2-4 minutes). This --- eliminates the gap where recent data is invisible in Grafana panels. --- --- PERFORMANCE WARNING (1024-validator networks): --- If aggregate queries become slow, this setting is the first thing to check. --- The tail scan reads raw data for the un-materialized window on every query. --- Post count-table refactoring the cost is low (ingested_raw_events only has --- low-volume event types), but under extreme load it may add latency. --- --- To revert a single aggregate: --- ALTER MATERIALIZED VIEW SET (timescaledb.materialized_only = true); - --- Original aggregates (from migrations 005/006/009/008/011/015) -ALTER MATERIALIZED VIEW event_stats_30s SET (timescaledb.materialized_only = false); -ALTER MATERIALIZED VIEW event_stats_1m SET (timescaledb.materialized_only = false); -ALTER MATERIALIZED VIEW event_stats_1h SET (timescaledb.materialized_only = false); -ALTER MATERIALIZED VIEW core_stats_1m SET (timescaledb.materialized_only = false); -ALTER MATERIALIZED VIEW service_stats_1m SET (timescaledb.materialized_only = false); -ALTER MATERIALIZED VIEW node_stats_1m SET (timescaledb.materialized_only = false); - --- Count table aggregates (from migration 016) — 9 groups × 2 tiers -ALTER MATERIALIZED VIEW block_distribution_counts_1m SET (timescaledb.materialized_only = false); -ALTER MATERIALIZED VIEW block_distribution_counts_1h SET (timescaledb.materialized_only = false); -ALTER MATERIALIZED VIEW ticket_counts_1m SET (timescaledb.materialized_only = false); -ALTER MATERIALIZED VIEW ticket_counts_1h SET (timescaledb.materialized_only = false); -ALTER MATERIALIZED VIEW guarantee_sending_counts_1m SET (timescaledb.materialized_only = false); -ALTER MATERIALIZED VIEW guarantee_sending_counts_1h SET (timescaledb.materialized_only = false); -ALTER MATERIALIZED VIEW guarantee_receiving_counts_1m SET (timescaledb.materialized_only = false); -ALTER MATERIALIZED VIEW guarantee_receiving_counts_1h SET (timescaledb.materialized_only = false); -ALTER MATERIALIZED VIEW shard_counts_1m SET (timescaledb.materialized_only = false); -ALTER MATERIALIZED VIEW shard_counts_1h SET (timescaledb.materialized_only = false); -ALTER MATERIALIZED VIEW assurance_counts_1m SET (timescaledb.materialized_only = false); -ALTER MATERIALIZED VIEW assurance_counts_1h SET (timescaledb.materialized_only = false); -ALTER MATERIALIZED VIEW bundle_counts_1m SET (timescaledb.materialized_only = false); -ALTER MATERIALIZED VIEW bundle_counts_1h SET (timescaledb.materialized_only = false); -ALTER MATERIALIZED VIEW segment_counts_1m SET (timescaledb.materialized_only = false); -ALTER MATERIALIZED VIEW segment_counts_1h SET (timescaledb.materialized_only = false); -ALTER MATERIALIZED VIEW preimage_counts_1m SET (timescaledb.materialized_only = false); -ALTER MATERIALIZED VIEW preimage_counts_1h SET (timescaledb.materialized_only = false); diff --git a/migrations/018_convergence_tables.sql b/migrations/018_convergence_tables.sql deleted file mode 100644 index 403e85f..0000000 --- a/migrations/018_convergence_tables.sql +++ /dev/null @@ -1,156 +0,0 @@ --- Migration 018: Convergence tables + expanded percentiles --- --- Adds guarantee convergence tracking (per work_report_hash + per-slot summary) --- and expands existing slot_convergence with p75/p95 percentile columns. - --- ── Expanded percentiles for existing slot_convergence ────────────────── -ALTER TABLE slot_convergence ADD COLUMN IF NOT EXISTS p75_ms INT; -ALTER TABLE slot_convergence ADD COLUMN IF NOT EXISTS p95_ms INT; - --- ── Guarantee convergence (per work_report_hash) ──────────────────────── --- One row per guarantee. Populated by convergence_tracker flush. --- Measures: GuaranteeBuilt(105) → GuaranteeReceived(112) propagation latency. -CREATE TABLE IF NOT EXISTS guarantee_convergence ( - work_report_hash BYTEA NOT NULL PRIMARY KEY, - slot INT NOT NULL, - core SMALLINT, -- nullable: NULL when guarantor not connected to telemetry - wp_hash BYTEA, - node_count SMALLINT NOT NULL, - p50_ms INT NOT NULL, - p75_ms INT, - p95_ms INT, - p99_ms INT NOT NULL, - p100_ms INT NOT NULL, - built_at TIMESTAMPTZ NOT NULL -); - -CREATE INDEX IF NOT EXISTS idx_guarantee_convergence_time - ON guarantee_convergence (built_at DESC); -CREATE INDEX IF NOT EXISTS idx_guarantee_convergence_core - ON guarantee_convergence (core, built_at DESC); -CREATE INDEX IF NOT EXISTS idx_guarantee_convergence_wp - ON guarantee_convergence (wp_hash, built_at DESC); - --- ── Guarantee convergence per-slot summary ────────────────────────────── --- One row per slot. Aggregates all guarantees in the slot. --- Used by /guarantee-convergence overview endpoint. -CREATE TABLE IF NOT EXISTS guarantee_convergence_slots ( - slot INT NOT NULL PRIMARY KEY, - slot_timestamp TIMESTAMPTZ, - guarantee_count SMALLINT NOT NULL, - node_count SMALLINT NOT NULL, - p50_ms INT, - p75_ms INT, - p95_ms INT, - p99_ms INT, - p100_ms INT, - built_at TIMESTAMPTZ NOT NULL -); - -CREATE INDEX IF NOT EXISTS idx_guarantee_conv_slots_time - ON guarantee_convergence_slots (built_at DESC); - --- ── Assurance convergence per-anchor summary ──────────────────────────── --- One row per block anchor. Aggregates all senders' assurance propagation. --- Measures: DistributingAssurance(126) → AssuranceReceived(131) per sender. --- Also tracks distribution start spread (how quickly validators begin distributing). -CREATE TABLE IF NOT EXISTS assurance_convergence ( - anchor BYTEA NOT NULL PRIMARY KEY, - slot INT, - slot_timestamp TIMESTAMPTZ, - sender_count SMALLINT NOT NULL, - receiver_count INT NOT NULL, - -- Reception convergence (distribution→reception deltas, clamped to >= 0) - p50_ms INT NOT NULL, - p75_ms INT, - p95_ms INT, - p99_ms INT NOT NULL, - p100_ms INT NOT NULL, - -- Distribution start spread (relative to first distributor) - dist_start_p50_ms INT, - dist_start_p95_ms INT, - dist_start_p99_ms INT, - dist_start_p100_ms INT, - first_distributed_at TIMESTAMPTZ, - last_distributed_at TIMESTAMPTZ -); - -CREATE INDEX IF NOT EXISTS idx_assurance_convergence_slot - ON assurance_convergence (slot DESC); - --- ── Assurance convergence per-sender detail ───────────────────────────── --- For debugging individual node assurance propagation. --- Hypertable: ~1023 senders × ~14.4k anchors/day ≈ ~14.7M rows/day at full load. --- INSERT-only (no unique constraint — cross-chunk uniqueness impractical on hypertables). -CREATE TABLE IF NOT EXISTS assurance_convergence_senders ( - distributed_at TIMESTAMPTZ NOT NULL, - anchor BYTEA NOT NULL, - sender_node_id TEXT NOT NULL, - node_count SMALLINT NOT NULL, - p50_ms INT NOT NULL, - p75_ms INT, - p95_ms INT, - p99_ms INT NOT NULL, - p100_ms INT NOT NULL -); - -SELECT create_hypertable('assurance_convergence_senders', 'distributed_at', if_not_exists => TRUE); - -CREATE INDEX IF NOT EXISTS idx_assurance_conv_senders_node - ON assurance_convergence_senders (sender_node_id, distributed_at DESC); - --- ── DA node stats ─────────────────────────────────────────────────────── --- Per-node DA operational stats: shard event counts, latency averages, shard inventory. --- Populated by da_tracker flush every 10s. One row per active node per flush. -CREATE TABLE IF NOT EXISTS da_node_stats ( - ts TIMESTAMPTZ NOT NULL, - node_id TEXT NOT NULL, - shard_requests_sent INT DEFAULT 0, - shard_requests_received INT DEFAULT 0, - shard_sent_confirmed INT DEFAULT 0, - shard_received_confirmed INT DEFAULT 0, - shards_transferred INT DEFAULT 0, - shard_failures INT DEFAULT 0, - preimage_ann_failures INT DEFAULT 0, - preimages_announced INT DEFAULT 0, - preimages_forgotten INT DEFAULT 0, - assurer_avg_latency_ms REAL, - assurer_latency_samples INT DEFAULT 0, - guarantor_avg_latency_ms REAL, - guarantor_latency_samples INT DEFAULT 0, - active_shards INT DEFAULT 0 -); - -SELECT create_hypertable('da_node_stats', 'ts', if_not_exists => TRUE); - -CREATE INDEX IF NOT EXISTS idx_da_node_stats_node - ON da_node_stats (node_id, ts DESC); - --- ── Shard latency histogram ───────────────────────────────────────────── --- Latency distribution for shard requests. 14 buckets (ms): [0,1), [1,2), [2,5), [5,10), --- [10,25), [25,50), [50,100), [100,250), [250,500), [500,1000), [1000,2000), [2000,3000), --- [3000,5000), [5000,∞). Side: 0=assurer (120→125), 1=guarantor (121→124). --- Histograms are mergeable: SUM bucket columns across nodes/time for combined distribution. -CREATE TABLE IF NOT EXISTS shard_latency_hist ( - ts TIMESTAMPTZ NOT NULL, - node_id TEXT NOT NULL, - side SMALLINT NOT NULL, - b_0_1 INT DEFAULT 0, - b_1_2 INT DEFAULT 0, - b_2_5 INT DEFAULT 0, - b_5_10 INT DEFAULT 0, - b_10_25 INT DEFAULT 0, - b_25_50 INT DEFAULT 0, - b_50_100 INT DEFAULT 0, - b_100_250 INT DEFAULT 0, - b_250_500 INT DEFAULT 0, - b_500_1000 INT DEFAULT 0, - b_1000_2000 INT DEFAULT 0, - b_2000_3000 INT DEFAULT 0, - b_3000_5000 INT DEFAULT 0, - b_5000_plus INT DEFAULT 0, - total_count INT DEFAULT 0, - failed_count INT DEFAULT 0 -); - -SELECT create_hypertable('shard_latency_hist', 'ts', if_not_exists => TRUE); diff --git a/migrations/019_convergence_histograms.sql b/migrations/019_convergence_histograms.sql deleted file mode 100644 index 2d52a31..0000000 --- a/migrations/019_convergence_histograms.sql +++ /dev/null @@ -1,89 +0,0 @@ --- Migration 019: Add convergence histograms for precise interval aggregation --- --- Adds 23-bucket latency histograms to convergence tables, enabling mergeable --- time-bucket aggregation. Buckets (ms): --- [0,2) [2,5) [5,10) [10,15) [15,20) [20,30) [30,50) [50,75) [75,100) --- [100,150) [150,250) [250,500) [500,1000) [1000,2000) [2000,5000) --- [5000,10000) [10000,15000) [15000,20000) [20000,25000) [25000,30000) --- [30000,60000) [60000,120000) [120000,∞) --- --- Also adds builder_node_id to guarantee_convergence for per-guarantor analysis. - --- ── guarantee_convergence ───────────────────────────────────────────── -ALTER TABLE guarantee_convergence ADD COLUMN IF NOT EXISTS builder_node_id TEXT; -ALTER TABLE guarantee_convergence ADD COLUMN IF NOT EXISTS h_0_2 INT DEFAULT 0; -ALTER TABLE guarantee_convergence ADD COLUMN IF NOT EXISTS h_2_5 INT DEFAULT 0; -ALTER TABLE guarantee_convergence ADD COLUMN IF NOT EXISTS h_5_10 INT DEFAULT 0; -ALTER TABLE guarantee_convergence ADD COLUMN IF NOT EXISTS h_10_15 INT DEFAULT 0; -ALTER TABLE guarantee_convergence ADD COLUMN IF NOT EXISTS h_15_20 INT DEFAULT 0; -ALTER TABLE guarantee_convergence ADD COLUMN IF NOT EXISTS h_20_30 INT DEFAULT 0; -ALTER TABLE guarantee_convergence ADD COLUMN IF NOT EXISTS h_30_50 INT DEFAULT 0; -ALTER TABLE guarantee_convergence ADD COLUMN IF NOT EXISTS h_50_75 INT DEFAULT 0; -ALTER TABLE guarantee_convergence ADD COLUMN IF NOT EXISTS h_75_100 INT DEFAULT 0; -ALTER TABLE guarantee_convergence ADD COLUMN IF NOT EXISTS h_100_150 INT DEFAULT 0; -ALTER TABLE guarantee_convergence ADD COLUMN IF NOT EXISTS h_150_250 INT DEFAULT 0; -ALTER TABLE guarantee_convergence ADD COLUMN IF NOT EXISTS h_250_500 INT DEFAULT 0; -ALTER TABLE guarantee_convergence ADD COLUMN IF NOT EXISTS h_500_1000 INT DEFAULT 0; -ALTER TABLE guarantee_convergence ADD COLUMN IF NOT EXISTS h_1000_2000 INT DEFAULT 0; -ALTER TABLE guarantee_convergence ADD COLUMN IF NOT EXISTS h_2000_5000 INT DEFAULT 0; -ALTER TABLE guarantee_convergence ADD COLUMN IF NOT EXISTS h_5000_10000 INT DEFAULT 0; -ALTER TABLE guarantee_convergence ADD COLUMN IF NOT EXISTS h_10000_15000 INT DEFAULT 0; -ALTER TABLE guarantee_convergence ADD COLUMN IF NOT EXISTS h_15000_20000 INT DEFAULT 0; -ALTER TABLE guarantee_convergence ADD COLUMN IF NOT EXISTS h_20000_25000 INT DEFAULT 0; -ALTER TABLE guarantee_convergence ADD COLUMN IF NOT EXISTS h_25000_30000 INT DEFAULT 0; -ALTER TABLE guarantee_convergence ADD COLUMN IF NOT EXISTS h_30000_60000 INT DEFAULT 0; -ALTER TABLE guarantee_convergence ADD COLUMN IF NOT EXISTS h_60000_120000 INT DEFAULT 0; -ALTER TABLE guarantee_convergence ADD COLUMN IF NOT EXISTS h_120000_plus INT DEFAULT 0; -ALTER TABLE guarantee_convergence ADD COLUMN IF NOT EXISTS hist_total INT DEFAULT 0; - --- ── assurance_convergence ───────────────────────────────────────────── -ALTER TABLE assurance_convergence ADD COLUMN IF NOT EXISTS h_0_2 INT DEFAULT 0; -ALTER TABLE assurance_convergence ADD COLUMN IF NOT EXISTS h_2_5 INT DEFAULT 0; -ALTER TABLE assurance_convergence ADD COLUMN IF NOT EXISTS h_5_10 INT DEFAULT 0; -ALTER TABLE assurance_convergence ADD COLUMN IF NOT EXISTS h_10_15 INT DEFAULT 0; -ALTER TABLE assurance_convergence ADD COLUMN IF NOT EXISTS h_15_20 INT DEFAULT 0; -ALTER TABLE assurance_convergence ADD COLUMN IF NOT EXISTS h_20_30 INT DEFAULT 0; -ALTER TABLE assurance_convergence ADD COLUMN IF NOT EXISTS h_30_50 INT DEFAULT 0; -ALTER TABLE assurance_convergence ADD COLUMN IF NOT EXISTS h_50_75 INT DEFAULT 0; -ALTER TABLE assurance_convergence ADD COLUMN IF NOT EXISTS h_75_100 INT DEFAULT 0; -ALTER TABLE assurance_convergence ADD COLUMN IF NOT EXISTS h_100_150 INT DEFAULT 0; -ALTER TABLE assurance_convergence ADD COLUMN IF NOT EXISTS h_150_250 INT DEFAULT 0; -ALTER TABLE assurance_convergence ADD COLUMN IF NOT EXISTS h_250_500 INT DEFAULT 0; -ALTER TABLE assurance_convergence ADD COLUMN IF NOT EXISTS h_500_1000 INT DEFAULT 0; -ALTER TABLE assurance_convergence ADD COLUMN IF NOT EXISTS h_1000_2000 INT DEFAULT 0; -ALTER TABLE assurance_convergence ADD COLUMN IF NOT EXISTS h_2000_5000 INT DEFAULT 0; -ALTER TABLE assurance_convergence ADD COLUMN IF NOT EXISTS h_5000_10000 INT DEFAULT 0; -ALTER TABLE assurance_convergence ADD COLUMN IF NOT EXISTS h_10000_15000 INT DEFAULT 0; -ALTER TABLE assurance_convergence ADD COLUMN IF NOT EXISTS h_15000_20000 INT DEFAULT 0; -ALTER TABLE assurance_convergence ADD COLUMN IF NOT EXISTS h_20000_25000 INT DEFAULT 0; -ALTER TABLE assurance_convergence ADD COLUMN IF NOT EXISTS h_25000_30000 INT DEFAULT 0; -ALTER TABLE assurance_convergence ADD COLUMN IF NOT EXISTS h_30000_60000 INT DEFAULT 0; -ALTER TABLE assurance_convergence ADD COLUMN IF NOT EXISTS h_60000_120000 INT DEFAULT 0; -ALTER TABLE assurance_convergence ADD COLUMN IF NOT EXISTS h_120000_plus INT DEFAULT 0; -ALTER TABLE assurance_convergence ADD COLUMN IF NOT EXISTS hist_total INT DEFAULT 0; - --- ── assurance_convergence_senders ───────────────────────────────────── -ALTER TABLE assurance_convergence_senders ADD COLUMN IF NOT EXISTS h_0_2 INT DEFAULT 0; -ALTER TABLE assurance_convergence_senders ADD COLUMN IF NOT EXISTS h_2_5 INT DEFAULT 0; -ALTER TABLE assurance_convergence_senders ADD COLUMN IF NOT EXISTS h_5_10 INT DEFAULT 0; -ALTER TABLE assurance_convergence_senders ADD COLUMN IF NOT EXISTS h_10_15 INT DEFAULT 0; -ALTER TABLE assurance_convergence_senders ADD COLUMN IF NOT EXISTS h_15_20 INT DEFAULT 0; -ALTER TABLE assurance_convergence_senders ADD COLUMN IF NOT EXISTS h_20_30 INT DEFAULT 0; -ALTER TABLE assurance_convergence_senders ADD COLUMN IF NOT EXISTS h_30_50 INT DEFAULT 0; -ALTER TABLE assurance_convergence_senders ADD COLUMN IF NOT EXISTS h_50_75 INT DEFAULT 0; -ALTER TABLE assurance_convergence_senders ADD COLUMN IF NOT EXISTS h_75_100 INT DEFAULT 0; -ALTER TABLE assurance_convergence_senders ADD COLUMN IF NOT EXISTS h_100_150 INT DEFAULT 0; -ALTER TABLE assurance_convergence_senders ADD COLUMN IF NOT EXISTS h_150_250 INT DEFAULT 0; -ALTER TABLE assurance_convergence_senders ADD COLUMN IF NOT EXISTS h_250_500 INT DEFAULT 0; -ALTER TABLE assurance_convergence_senders ADD COLUMN IF NOT EXISTS h_500_1000 INT DEFAULT 0; -ALTER TABLE assurance_convergence_senders ADD COLUMN IF NOT EXISTS h_1000_2000 INT DEFAULT 0; -ALTER TABLE assurance_convergence_senders ADD COLUMN IF NOT EXISTS h_2000_5000 INT DEFAULT 0; -ALTER TABLE assurance_convergence_senders ADD COLUMN IF NOT EXISTS h_5000_10000 INT DEFAULT 0; -ALTER TABLE assurance_convergence_senders ADD COLUMN IF NOT EXISTS h_10000_15000 INT DEFAULT 0; -ALTER TABLE assurance_convergence_senders ADD COLUMN IF NOT EXISTS h_15000_20000 INT DEFAULT 0; -ALTER TABLE assurance_convergence_senders ADD COLUMN IF NOT EXISTS h_20000_25000 INT DEFAULT 0; -ALTER TABLE assurance_convergence_senders ADD COLUMN IF NOT EXISTS h_25000_30000 INT DEFAULT 0; -ALTER TABLE assurance_convergence_senders ADD COLUMN IF NOT EXISTS h_30000_60000 INT DEFAULT 0; -ALTER TABLE assurance_convergence_senders ADD COLUMN IF NOT EXISTS h_60000_120000 INT DEFAULT 0; -ALTER TABLE assurance_convergence_senders ADD COLUMN IF NOT EXISTS h_120000_plus INT DEFAULT 0; -ALTER TABLE assurance_convergence_senders ADD COLUMN IF NOT EXISTS hist_total INT DEFAULT 0; diff --git a/migrations/020_unified_event_architecture.sql b/migrations/020_unified_event_architecture.sql deleted file mode 100644 index 482dec2..0000000 --- a/migrations/020_unified_event_architecture.sql +++ /dev/null @@ -1,400 +0,0 @@ --- Migration 020: Unified event architecture. --- --- Expands count tables to cover ALL 115 event types, adds wp_hash hot column --- to ingested_raw_events, rebuilds UNION views to reference only count tables, --- drops old continuous aggregates, and sets 1h retention on raw events. --- --- After this migration: --- - All 115 types → count tables (long-term aggregation, single source) --- - All 115 types → ingested_raw_events (1h browsing store, hot columns) --- - event_stats_30s/1m/1h and core_stats_1m → DROPPED --- - UNION views → rebuilt with 14 count table branches (no continuous aggregate branches) - --- ============================================================ --- 1. status_counts (types 0, 10-13) --- Dropped, Status, BestBlockChanged, FinalizedBlockChanged, SyncStatusChanged --- ============================================================ -CREATE TABLE status_counts ( - bucket TIMESTAMPTZ NOT NULL, - node_id TEXT NOT NULL, - event_type SMALLINT NOT NULL, - event_count BIGINT NOT NULL, - slot INT -); -SELECT create_hypertable('status_counts', 'bucket', chunk_time_interval => INTERVAL '1 day'); -ALTER TABLE status_counts ADD CHECK (event_type = 0 OR event_type BETWEEN 10 AND 13); -CREATE INDEX ON status_counts (node_id, event_type, bucket DESC); -ALTER TABLE status_counts SET ( - timescaledb.compress, - timescaledb.compress_segmentby = 'node_id, event_type', - timescaledb.compress_orderby = 'bucket DESC' -); -SELECT add_compression_policy('status_counts', compress_after => INTERVAL '2 hours'); -SELECT add_retention_policy('status_counts', INTERVAL '3 days'); - -CREATE MATERIALIZED VIEW status_counts_1m -WITH (timescaledb.continuous) AS -SELECT - time_bucket('1 minute', bucket) AS bucket, - node_id, event_type, - SUM(event_count) AS event_count -FROM status_counts -GROUP BY 1, node_id, event_type -WITH NO DATA; - -SELECT add_continuous_aggregate_policy('status_counts_1m', - start_offset => INTERVAL '10 minutes', - end_offset => INTERVAL '2 minutes', - schedule_interval => INTERVAL '2 minutes'); -SELECT add_retention_policy('status_counts_1m', INTERVAL '30 days'); - -CREATE MATERIALIZED VIEW status_counts_1h -WITH (timescaledb.continuous) AS -SELECT - time_bucket('1 hour', bucket) AS bucket, - node_id, event_type, - SUM(event_count) AS event_count -FROM status_counts_1m -GROUP BY 1, node_id, event_type -WITH NO DATA; - -SELECT add_continuous_aggregate_policy('status_counts_1h', - start_offset => INTERVAL '4 hours', - end_offset => INTERVAL '1 hour', - schedule_interval => INTERVAL '1 hour'); -SELECT add_retention_policy('status_counts_1h', INTERVAL '365 days'); - --- ============================================================ --- 2. connection_counts (types 20-28) --- ConnectionRefused through PeerMisbehaved --- ============================================================ -CREATE TABLE connection_counts ( - bucket TIMESTAMPTZ NOT NULL, - node_id TEXT NOT NULL, - event_type SMALLINT NOT NULL, - event_count BIGINT NOT NULL, - reason TEXT -); -SELECT create_hypertable('connection_counts', 'bucket', chunk_time_interval => INTERVAL '1 day'); -ALTER TABLE connection_counts ADD CHECK (event_type BETWEEN 20 AND 28); -CREATE INDEX ON connection_counts (node_id, event_type, bucket DESC); -ALTER TABLE connection_counts SET ( - timescaledb.compress, - timescaledb.compress_segmentby = 'node_id, event_type', - timescaledb.compress_orderby = 'bucket DESC' -); -SELECT add_compression_policy('connection_counts', compress_after => INTERVAL '2 hours'); -SELECT add_retention_policy('connection_counts', INTERVAL '3 days'); - -CREATE MATERIALIZED VIEW connection_counts_1m -WITH (timescaledb.continuous) AS -SELECT - time_bucket('1 minute', bucket) AS bucket, - node_id, event_type, - SUM(event_count) AS event_count -FROM connection_counts -GROUP BY 1, node_id, event_type -WITH NO DATA; - -SELECT add_continuous_aggregate_policy('connection_counts_1m', - start_offset => INTERVAL '10 minutes', - end_offset => INTERVAL '2 minutes', - schedule_interval => INTERVAL '2 minutes'); -SELECT add_retention_policy('connection_counts_1m', INTERVAL '30 days'); - -CREATE MATERIALIZED VIEW connection_counts_1h -WITH (timescaledb.continuous) AS -SELECT - time_bucket('1 hour', bucket) AS bucket, - node_id, event_type, - SUM(event_count) AS event_count -FROM connection_counts_1m -GROUP BY 1, node_id, event_type -WITH NO DATA; - -SELECT add_continuous_aggregate_policy('connection_counts_1h', - start_offset => INTERVAL '4 hours', - end_offset => INTERVAL '1 hour', - schedule_interval => INTERVAL '1 hour'); -SELECT add_retention_policy('connection_counts_1h', INTERVAL '365 days'); - --- ============================================================ --- 3. block_counts (types 40-47) --- Authoring through BlockExecuted --- ============================================================ -CREATE TABLE block_counts ( - bucket TIMESTAMPTZ NOT NULL, - node_id TEXT NOT NULL, - event_type SMALLINT NOT NULL, - event_count BIGINT NOT NULL, - slot INT, - reason TEXT -); -SELECT create_hypertable('block_counts', 'bucket', chunk_time_interval => INTERVAL '1 day'); -ALTER TABLE block_counts ADD CHECK (event_type BETWEEN 40 AND 47); -CREATE INDEX ON block_counts (node_id, event_type, bucket DESC); -ALTER TABLE block_counts SET ( - timescaledb.compress, - timescaledb.compress_segmentby = 'node_id, event_type', - timescaledb.compress_orderby = 'bucket DESC' -); -SELECT add_compression_policy('block_counts', compress_after => INTERVAL '2 hours'); -SELECT add_retention_policy('block_counts', INTERVAL '3 days'); - -CREATE MATERIALIZED VIEW block_counts_1m -WITH (timescaledb.continuous) AS -SELECT - time_bucket('1 minute', bucket) AS bucket, - node_id, event_type, - SUM(event_count) AS event_count -FROM block_counts -GROUP BY 1, node_id, event_type -WITH NO DATA; - -SELECT add_continuous_aggregate_policy('block_counts_1m', - start_offset => INTERVAL '10 minutes', - end_offset => INTERVAL '2 minutes', - schedule_interval => INTERVAL '2 minutes'); -SELECT add_retention_policy('block_counts_1m', INTERVAL '30 days'); - -CREATE MATERIALIZED VIEW block_counts_1h -WITH (timescaledb.continuous) AS -SELECT - time_bucket('1 hour', bucket) AS bucket, - node_id, event_type, - SUM(event_count) AS event_count -FROM block_counts_1m -GROUP BY 1, node_id, event_type -WITH NO DATA; - -SELECT add_continuous_aggregate_policy('block_counts_1h', - start_offset => INTERVAL '4 hours', - end_offset => INTERVAL '1 hour', - schedule_interval => INTERVAL '1 hour'); -SELECT add_retention_policy('block_counts_1h', INTERVAL '365 days'); - --- ============================================================ --- 4. ticket_low_counts (types 80-82) --- GeneratingTickets, TicketGenerationFailed, TicketsGenerated --- ============================================================ -CREATE TABLE ticket_low_counts ( - bucket TIMESTAMPTZ NOT NULL, - node_id TEXT NOT NULL, - event_type SMALLINT NOT NULL, - event_count BIGINT NOT NULL, - reason TEXT -); -SELECT create_hypertable('ticket_low_counts', 'bucket', chunk_time_interval => INTERVAL '1 day'); -ALTER TABLE ticket_low_counts ADD CHECK (event_type BETWEEN 80 AND 82); -CREATE INDEX ON ticket_low_counts (node_id, event_type, bucket DESC); -ALTER TABLE ticket_low_counts SET ( - timescaledb.compress, - timescaledb.compress_segmentby = 'node_id, event_type', - timescaledb.compress_orderby = 'bucket DESC' -); -SELECT add_compression_policy('ticket_low_counts', compress_after => INTERVAL '2 hours'); -SELECT add_retention_policy('ticket_low_counts', INTERVAL '3 days'); - -CREATE MATERIALIZED VIEW ticket_low_counts_1m -WITH (timescaledb.continuous) AS -SELECT - time_bucket('1 minute', bucket) AS bucket, - node_id, event_type, - SUM(event_count) AS event_count -FROM ticket_low_counts -GROUP BY 1, node_id, event_type -WITH NO DATA; - -SELECT add_continuous_aggregate_policy('ticket_low_counts_1m', - start_offset => INTERVAL '10 minutes', - end_offset => INTERVAL '2 minutes', - schedule_interval => INTERVAL '2 minutes'); -SELECT add_retention_policy('ticket_low_counts_1m', INTERVAL '30 days'); - -CREATE MATERIALIZED VIEW ticket_low_counts_1h -WITH (timescaledb.continuous) AS -SELECT - time_bucket('1 hour', bucket) AS bucket, - node_id, event_type, - SUM(event_count) AS event_count -FROM ticket_low_counts_1m -GROUP BY 1, node_id, event_type -WITH NO DATA; - -SELECT add_continuous_aggregate_policy('ticket_low_counts_1h', - start_offset => INTERVAL '4 hours', - end_offset => INTERVAL '1 hour', - schedule_interval => INTERVAL '1 hour'); -SELECT add_retention_policy('ticket_low_counts_1h', INTERVAL '365 days'); - --- ============================================================ --- 5. wp_pipeline_counts (types 90-105) --- WorkPackageSubmission through GuaranteeBuilt --- Core is nullable — enrichment may fail for types 90, 91, 103 --- Types 106-109 go to guarantee_sending_counts (CHECK extended below) --- ============================================================ -CREATE TABLE wp_pipeline_counts ( - bucket TIMESTAMPTZ NOT NULL, - node_id TEXT NOT NULL, - event_type SMALLINT NOT NULL, - event_count BIGINT NOT NULL, - core SMALLINT, - reason TEXT -); -SELECT create_hypertable('wp_pipeline_counts', 'bucket', chunk_time_interval => INTERVAL '1 day'); -ALTER TABLE wp_pipeline_counts ADD CHECK (event_type BETWEEN 90 AND 105); -CREATE INDEX ON wp_pipeline_counts (node_id, event_type, bucket DESC); -ALTER TABLE wp_pipeline_counts SET ( - timescaledb.compress, - timescaledb.compress_segmentby = 'node_id, event_type', - timescaledb.compress_orderby = 'bucket DESC' -); -SELECT add_compression_policy('wp_pipeline_counts', compress_after => INTERVAL '2 hours'); -SELECT add_retention_policy('wp_pipeline_counts', INTERVAL '3 days'); - -CREATE MATERIALIZED VIEW wp_pipeline_counts_1m -WITH (timescaledb.continuous) AS -SELECT - time_bucket('1 minute', bucket) AS bucket, - node_id, event_type, - SUM(event_count) AS event_count -FROM wp_pipeline_counts -GROUP BY 1, node_id, event_type -WITH NO DATA; - -SELECT add_continuous_aggregate_policy('wp_pipeline_counts_1m', - start_offset => INTERVAL '10 minutes', - end_offset => INTERVAL '2 minutes', - schedule_interval => INTERVAL '2 minutes'); -SELECT add_retention_policy('wp_pipeline_counts_1m', INTERVAL '30 days'); - -CREATE MATERIALIZED VIEW wp_pipeline_counts_1h -WITH (timescaledb.continuous) AS -SELECT - time_bucket('1 hour', bucket) AS bucket, - node_id, event_type, - SUM(event_count) AS event_count -FROM wp_pipeline_counts_1m -GROUP BY 1, node_id, event_type -WITH NO DATA; - -SELECT add_continuous_aggregate_policy('wp_pipeline_counts_1h', - start_offset => INTERVAL '4 hours', - end_offset => INTERVAL '1 hour', - schedule_interval => INTERVAL '1 hour'); -SELECT add_retention_policy('wp_pipeline_counts_1h', INTERVAL '365 days'); - --- ============================================================ --- 5b. Extend guarantee_sending_counts to include type 109 (GuaranteesDistributed) --- Previously type 109 was not pre-aggregated; now it goes to guarantee_sending_counts. --- ============================================================ -ALTER TABLE guarantee_sending_counts DROP CONSTRAINT IF EXISTS guarantee_sending_counts_event_type_check; -ALTER TABLE guarantee_sending_counts ADD CHECK (event_type BETWEEN 106 AND 109); - --- ============================================================ --- 6. wp_hash hot column on ingested_raw_events --- Enables /grafana/wp/{hash} journey drilldown without JSONB chains --- ============================================================ -ALTER TABLE ingested_raw_events ADD COLUMN IF NOT EXISTS wp_hash BYTEA; -CREATE INDEX IF NOT EXISTS idx_ire_wp_hash - ON ingested_raw_events (wp_hash, timestamp DESC) WHERE wp_hash IS NOT NULL; - --- ============================================================ --- 7. Drop old UNION views (must drop before dropping underlying aggregates) --- ============================================================ -DROP VIEW IF EXISTS all_event_stats_30s CASCADE; -DROP VIEW IF EXISTS all_event_stats_1m CASCADE; -DROP VIEW IF EXISTS all_event_stats_1h CASCADE; -DROP VIEW IF EXISTS all_core_stats_1m CASCADE; - --- ============================================================ --- 8. Drop old continuous aggregates --- Count tables are now the single aggregation source. --- ============================================================ --- Must drop hierarchical aggregates top-down (1h depends on 1m, 1m depends on 30s) -DROP MATERIALIZED VIEW IF EXISTS event_stats_1h CASCADE; -DROP MATERIALIZED VIEW IF EXISTS event_stats_1m CASCADE; -DROP MATERIALIZED VIEW IF EXISTS event_stats_30s CASCADE; -DROP MATERIALIZED VIEW IF EXISTS core_stats_1m CASCADE; - --- ============================================================ --- 9. Rebuild UNION views — count tables only (14 branches) --- ============================================================ - --- 30s: raw count tables -CREATE VIEW all_event_stats_30s AS - SELECT bucket, node_id, event_type, event_count FROM status_counts - UNION ALL SELECT bucket, node_id, event_type, event_count FROM connection_counts - UNION ALL SELECT bucket, node_id, event_type, event_count FROM block_counts - UNION ALL SELECT bucket, node_id, event_type, event_count FROM ticket_low_counts - UNION ALL SELECT bucket, node_id, event_type, event_count FROM wp_pipeline_counts - UNION ALL SELECT bucket, node_id, event_type, event_count FROM block_distribution_counts - UNION ALL SELECT bucket, node_id, event_type, event_count FROM ticket_counts - UNION ALL SELECT bucket, node_id, event_type, event_count FROM guarantee_sending_counts - UNION ALL SELECT bucket, node_id, event_type, event_count FROM guarantee_receiving_counts - UNION ALL SELECT bucket, node_id, event_type, event_count FROM shard_counts - UNION ALL SELECT bucket, node_id, event_type, event_count FROM assurance_counts - UNION ALL SELECT bucket, node_id, event_type, event_count FROM bundle_counts - UNION ALL SELECT bucket, node_id, event_type, event_count FROM segment_counts - UNION ALL SELECT bucket, node_id, event_type, event_count FROM preimage_counts; - --- 1m: aggregated continuous aggregates -CREATE VIEW all_event_stats_1m AS - SELECT bucket, node_id, event_type, event_count FROM status_counts_1m - UNION ALL SELECT bucket, node_id, event_type, event_count FROM connection_counts_1m - UNION ALL SELECT bucket, node_id, event_type, event_count FROM block_counts_1m - UNION ALL SELECT bucket, node_id, event_type, event_count FROM ticket_low_counts_1m - UNION ALL SELECT bucket, node_id, event_type, event_count FROM wp_pipeline_counts_1m - UNION ALL SELECT bucket, node_id, event_type, event_count FROM block_distribution_counts_1m - UNION ALL SELECT bucket, node_id, event_type, event_count FROM ticket_counts_1m - UNION ALL SELECT bucket, node_id, event_type, event_count FROM guarantee_sending_counts_1m - UNION ALL SELECT bucket, node_id, event_type, event_count FROM guarantee_receiving_counts_1m - UNION ALL SELECT bucket, node_id, event_type, event_count FROM shard_counts_1m - UNION ALL SELECT bucket, node_id, event_type, event_count FROM assurance_counts_1m - UNION ALL SELECT bucket, node_id, event_type, event_count FROM bundle_counts_1m - UNION ALL SELECT bucket, node_id, event_type, event_count FROM segment_counts_1m - UNION ALL SELECT bucket, node_id, event_type, event_count FROM preimage_counts_1m; - --- 1h: aggregated continuous aggregates -CREATE VIEW all_event_stats_1h AS - SELECT bucket, node_id, event_type, event_count FROM status_counts_1h - UNION ALL SELECT bucket, node_id, event_type, event_count FROM connection_counts_1h - UNION ALL SELECT bucket, node_id, event_type, event_count FROM block_counts_1h - UNION ALL SELECT bucket, node_id, event_type, event_count FROM ticket_low_counts_1h - UNION ALL SELECT bucket, node_id, event_type, event_count FROM wp_pipeline_counts_1h - UNION ALL SELECT bucket, node_id, event_type, event_count FROM block_distribution_counts_1h - UNION ALL SELECT bucket, node_id, event_type, event_count FROM ticket_counts_1h - UNION ALL SELECT bucket, node_id, event_type, event_count FROM guarantee_sending_counts_1h - UNION ALL SELECT bucket, node_id, event_type, event_count FROM guarantee_receiving_counts_1h - UNION ALL SELECT bucket, node_id, event_type, event_count FROM shard_counts_1h - UNION ALL SELECT bucket, node_id, event_type, event_count FROM assurance_counts_1h - UNION ALL SELECT bucket, node_id, event_type, event_count FROM bundle_counts_1h - UNION ALL SELECT bucket, node_id, event_type, event_count FROM segment_counts_1h - UNION ALL SELECT bucket, node_id, event_type, event_count FROM preimage_counts_1h; - --- Core-aware UNION view (for timeseries?group_by=core and core=X filter) --- Only tables with a core column participate. --- Uses raw count tables (not _1m aggregates) because the continuous aggregates --- drop the core dimension (they GROUP BY node_id, event_type only). -CREATE VIEW all_core_stats_1m AS - SELECT bucket, event_type, core, event_count - FROM guarantee_sending_counts WHERE core IS NOT NULL - UNION ALL SELECT bucket, event_type, core, event_count - FROM segment_counts WHERE core IS NOT NULL - UNION ALL SELECT bucket, event_type, core, event_count - FROM wp_pipeline_counts WHERE core IS NOT NULL; - --- ============================================================ --- 10. Set 1h retention on ingested_raw_events --- Table is now a pure browsing store — no aggregation depends on it. --- ============================================================ -SELECT remove_retention_policy('ingested_raw_events', if_exists => TRUE); -SELECT add_retention_policy('ingested_raw_events', INTERVAL '1 hour', schedule_interval => INTERVAL '5 minutes'); - --- ============================================================ --- 11. Drop the backward-compatibility VIEW alias for 'events' --- (created in migration 015, no longer needed) --- ============================================================ -DROP VIEW IF EXISTS events_view CASCADE; --- Keep the 'events' VIEW alias for now — legacy endpoints still reference it --- until they are removed in later phases. diff --git a/migrations/021_wp_tracking_enhancements.sql b/migrations/021_wp_tracking_enhancements.sql deleted file mode 100644 index 7a27b72..0000000 --- a/migrations/021_wp_tracking_enhancements.sql +++ /dev/null @@ -1,21 +0,0 @@ --- Migration 021: Enhance wp_tracking with node_id, gas, failure/discard reasons. --- --- Adds columns needed by /grafana/wp-active, /grafana/wp/{hash}, and --- /grafana/cores/:core_id (extended WP list). --- --- node_id: which node first received this WP (from WorkPackageReceived event) --- refine_gas_used: total gas from Refined event (SUM of costs[].total.gas_used) --- failure_reason: from WorkPackageFailed event reason field --- discard_reason: from GuaranteeDiscarded event via guarantee_convergence wp_hash mapping - -ALTER TABLE wp_tracking ADD COLUMN IF NOT EXISTS node_id TEXT; -ALTER TABLE wp_tracking ADD COLUMN IF NOT EXISTS refine_gas_used BIGINT; -ALTER TABLE wp_tracking ADD COLUMN IF NOT EXISTS failure_reason TEXT; -ALTER TABLE wp_tracking ADD COLUMN IF NOT EXISTS discard_reason TEXT; - --- Partial index for wp-active queries: matches the exact WHERE clause --- of "in-flight WPs" queries. Only indexes rows that haven't completed --- or failed — keeps the index small and fast. -CREATE INDEX IF NOT EXISTS idx_wp_tracking_active - ON wp_tracking (first_seen DESC) - WHERE distributed_at IS NULL AND failed_at IS NULL; diff --git a/migrations/022_event_services_timing.sql b/migrations/022_event_services_timing.sql deleted file mode 100644 index 5de8198..0000000 --- a/migrations/022_event_services_timing.sql +++ /dev/null @@ -1,7 +0,0 @@ --- Add execution timing columns to event_services. --- elapsed_ns: total wall-clock execution time (from ExecCost.ns) --- load_ns: PVM code loading/compilation time --- Populated at ingestion for types 47 (BlockExecuted), 95 (Authorized), 101 (Refined). - -ALTER TABLE event_services ADD COLUMN IF NOT EXISTS elapsed_ns BIGINT; -ALTER TABLE event_services ADD COLUMN IF NOT EXISTS load_ns BIGINT; diff --git a/migrations/023_aggregate_indexes.sql b/migrations/023_aggregate_indexes.sql deleted file mode 100644 index 6ffa323..0000000 --- a/migrations/023_aggregate_indexes.sql +++ /dev/null @@ -1,106 +0,0 @@ --- Add indexes on continuous aggregates and convergence tables to speed up --- Grafana endpoint queries. Continuous aggregates only had TimescaleDB's --- default bucket index — queries filtering by event_type, service_id, or --- node_id were doing sequential scans after bucket range narrowing. - --- ============================================================ --- Tier 1: _1m continuous aggregates — (event_type, bucket DESC) --- Speeds up ~15 Grafana endpoints that query all_event_stats_1m --- with: WHERE bucket >= $1 AND bucket < $2 AND event_type = ANY($3) --- ============================================================ - -CREATE INDEX IF NOT EXISTS idx_status_counts_1m_et - ON status_counts_1m (event_type, bucket DESC); -CREATE INDEX IF NOT EXISTS idx_connection_counts_1m_et - ON connection_counts_1m (event_type, bucket DESC); -CREATE INDEX IF NOT EXISTS idx_block_counts_1m_et - ON block_counts_1m (event_type, bucket DESC); -CREATE INDEX IF NOT EXISTS idx_ticket_low_counts_1m_et - ON ticket_low_counts_1m (event_type, bucket DESC); -CREATE INDEX IF NOT EXISTS idx_wp_pipeline_counts_1m_et - ON wp_pipeline_counts_1m (event_type, bucket DESC); -CREATE INDEX IF NOT EXISTS idx_block_distribution_counts_1m_et - ON block_distribution_counts_1m (event_type, bucket DESC); -CREATE INDEX IF NOT EXISTS idx_ticket_counts_1m_et - ON ticket_counts_1m (event_type, bucket DESC); -CREATE INDEX IF NOT EXISTS idx_guarantee_sending_counts_1m_et - ON guarantee_sending_counts_1m (event_type, bucket DESC); -CREATE INDEX IF NOT EXISTS idx_guarantee_receiving_counts_1m_et - ON guarantee_receiving_counts_1m (event_type, bucket DESC); -CREATE INDEX IF NOT EXISTS idx_shard_counts_1m_et - ON shard_counts_1m (event_type, bucket DESC); -CREATE INDEX IF NOT EXISTS idx_assurance_counts_1m_et - ON assurance_counts_1m (event_type, bucket DESC); -CREATE INDEX IF NOT EXISTS idx_bundle_counts_1m_et - ON bundle_counts_1m (event_type, bucket DESC); -CREATE INDEX IF NOT EXISTS idx_segment_counts_1m_et - ON segment_counts_1m (event_type, bucket DESC); -CREATE INDEX IF NOT EXISTS idx_preimage_counts_1m_et - ON preimage_counts_1m (event_type, bucket DESC); - --- assurance_convergence: only had (slot DESC), but queries filter by first_distributed_at -CREATE INDEX IF NOT EXISTS idx_assurance_convergence_time - ON assurance_convergence (first_distributed_at); - --- ============================================================ --- Tier 2: _1h continuous aggregates — (event_type, bucket DESC) --- Same pattern for all_event_stats_1h (time ranges > 30 days) --- ============================================================ - -CREATE INDEX IF NOT EXISTS idx_status_counts_1h_et - ON status_counts_1h (event_type, bucket DESC); -CREATE INDEX IF NOT EXISTS idx_connection_counts_1h_et - ON connection_counts_1h (event_type, bucket DESC); -CREATE INDEX IF NOT EXISTS idx_block_counts_1h_et - ON block_counts_1h (event_type, bucket DESC); -CREATE INDEX IF NOT EXISTS idx_ticket_low_counts_1h_et - ON ticket_low_counts_1h (event_type, bucket DESC); -CREATE INDEX IF NOT EXISTS idx_wp_pipeline_counts_1h_et - ON wp_pipeline_counts_1h (event_type, bucket DESC); -CREATE INDEX IF NOT EXISTS idx_block_distribution_counts_1h_et - ON block_distribution_counts_1h (event_type, bucket DESC); -CREATE INDEX IF NOT EXISTS idx_ticket_counts_1h_et - ON ticket_counts_1h (event_type, bucket DESC); -CREATE INDEX IF NOT EXISTS idx_guarantee_sending_counts_1h_et - ON guarantee_sending_counts_1h (event_type, bucket DESC); -CREATE INDEX IF NOT EXISTS idx_guarantee_receiving_counts_1h_et - ON guarantee_receiving_counts_1h (event_type, bucket DESC); -CREATE INDEX IF NOT EXISTS idx_shard_counts_1h_et - ON shard_counts_1h (event_type, bucket DESC); -CREATE INDEX IF NOT EXISTS idx_assurance_counts_1h_et - ON assurance_counts_1h (event_type, bucket DESC); -CREATE INDEX IF NOT EXISTS idx_bundle_counts_1h_et - ON bundle_counts_1h (event_type, bucket DESC); -CREATE INDEX IF NOT EXISTS idx_segment_counts_1h_et - ON segment_counts_1h (event_type, bucket DESC); -CREATE INDEX IF NOT EXISTS idx_preimage_counts_1h_et - ON preimage_counts_1h (event_type, bucket DESC); - --- service_stats_1m: queries filter by service_id -CREATE INDEX IF NOT EXISTS idx_service_stats_1m_svc - ON service_stats_1m (service_id, bucket DESC); - --- node_stats_1m: per-node drill-down queries (1024 nodes, huge selectivity gain) -CREATE INDEX IF NOT EXISTS idx_node_stats_1m_node - ON node_stats_1m (node_id, bucket DESC); - --- ============================================================ --- Tier 3: Raw 30s count tables — write cost tradeoff --- These receive COPY BINARY every 5s but have 3-day retention. --- ============================================================ - --- all_core_stats_1m queries raw tables with core filter (partial: many rows have NULL core) -CREATE INDEX IF NOT EXISTS idx_guarantee_sending_counts_core - ON guarantee_sending_counts (core, event_type, bucket DESC) WHERE core IS NOT NULL; -CREATE INDEX IF NOT EXISTS idx_segment_counts_core - ON segment_counts (core, event_type, bucket DESC) WHERE core IS NOT NULL; -CREATE INDEX IF NOT EXISTS idx_wp_pipeline_counts_core - ON wp_pipeline_counts (core, event_type, bucket DESC) WHERE core IS NOT NULL; - --- sync_timeline queries status_counts by event_type without node_id -CREATE INDEX IF NOT EXISTS idx_status_counts_et - ON status_counts (event_type, bucket DESC) WHERE slot IS NOT NULL; - --- connections_timeline queries connection_counts by event_type without node_id -CREATE INDEX IF NOT EXISTS idx_connection_counts_et - ON connection_counts (event_type, bucket DESC); diff --git a/migrations/024_da_latency_tables.sql b/migrations/024_da_latency_tables.sql deleted file mode 100644 index 7ef9660..0000000 --- a/migrations/024_da_latency_tables.sql +++ /dev/null @@ -1,118 +0,0 @@ --- Migration 024: DA latency histogram tables for bundle reconstruction, --- segment fetching, and preimage transfers. --- --- Uses CONVERGENCE_BOUNDS (23 buckets, 0ms–120s): --- [0,2) [2,5) [5,10) [10,15) [15,20) [20,30) [30,50) [50,75) [75,100) --- [100,150) [150,250) [250,500) [500,1k) [1k,2k) [2k,5k) [5k,10k) --- [10k,15k) [15k,20k) [20k,25k) [25k,30k) [30k,60k) [60k,120k) [120k,+inf) --- --- Side encoding per table: --- bundle_latency_hist: 0=shard_req(140->145), 1=shard_resp(141->145), --- 2=full_req(148->153), 3=full_resp(149->153), --- 4=reconstruct(146->147), 5=e2e(140->147) --- segment_latency_hist: 0=shard_req(162->167), 1=shard_resp(163->167), --- 2=full_req(173->178), 3=full_resp(174->178), --- 4=reconstruct(168->170) --- preimage_latency_hist: 0=req(193->198), 1=resp(194->198) - --- ── Bundle reconstruction latency ──────────────────────────────────── -CREATE TABLE IF NOT EXISTS bundle_latency_hist ( - ts TIMESTAMPTZ NOT NULL, - node_id TEXT NOT NULL, - side SMALLINT NOT NULL, - h_0_2 INT DEFAULT 0, - h_2_5 INT DEFAULT 0, - h_5_10 INT DEFAULT 0, - h_10_15 INT DEFAULT 0, - h_15_20 INT DEFAULT 0, - h_20_30 INT DEFAULT 0, - h_30_50 INT DEFAULT 0, - h_50_75 INT DEFAULT 0, - h_75_100 INT DEFAULT 0, - h_100_150 INT DEFAULT 0, - h_150_250 INT DEFAULT 0, - h_250_500 INT DEFAULT 0, - h_500_1000 INT DEFAULT 0, - h_1000_2000 INT DEFAULT 0, - h_2000_5000 INT DEFAULT 0, - h_5000_10000 INT DEFAULT 0, - h_10000_15000 INT DEFAULT 0, - h_15000_20000 INT DEFAULT 0, - h_20000_25000 INT DEFAULT 0, - h_25000_30000 INT DEFAULT 0, - h_30000_60000 INT DEFAULT 0, - h_60000_120000 INT DEFAULT 0, - h_120000_plus INT DEFAULT 0, - total_count INT DEFAULT 0, - failed_count INT DEFAULT 0 -); -SELECT create_hypertable('bundle_latency_hist', 'ts', chunk_time_interval => INTERVAL '1 hour', if_not_exists => TRUE); -CREATE INDEX IF NOT EXISTS idx_bundle_latency_hist_node ON bundle_latency_hist (node_id, ts DESC); - --- ── Segment fetching latency ───────────────────────────────────────── -CREATE TABLE IF NOT EXISTS segment_latency_hist ( - ts TIMESTAMPTZ NOT NULL, - node_id TEXT NOT NULL, - side SMALLINT NOT NULL, - h_0_2 INT DEFAULT 0, - h_2_5 INT DEFAULT 0, - h_5_10 INT DEFAULT 0, - h_10_15 INT DEFAULT 0, - h_15_20 INT DEFAULT 0, - h_20_30 INT DEFAULT 0, - h_30_50 INT DEFAULT 0, - h_50_75 INT DEFAULT 0, - h_75_100 INT DEFAULT 0, - h_100_150 INT DEFAULT 0, - h_150_250 INT DEFAULT 0, - h_250_500 INT DEFAULT 0, - h_500_1000 INT DEFAULT 0, - h_1000_2000 INT DEFAULT 0, - h_2000_5000 INT DEFAULT 0, - h_5000_10000 INT DEFAULT 0, - h_10000_15000 INT DEFAULT 0, - h_15000_20000 INT DEFAULT 0, - h_20000_25000 INT DEFAULT 0, - h_25000_30000 INT DEFAULT 0, - h_30000_60000 INT DEFAULT 0, - h_60000_120000 INT DEFAULT 0, - h_120000_plus INT DEFAULT 0, - total_count INT DEFAULT 0, - failed_count INT DEFAULT 0 -); -SELECT create_hypertable('segment_latency_hist', 'ts', chunk_time_interval => INTERVAL '1 hour', if_not_exists => TRUE); -CREATE INDEX IF NOT EXISTS idx_segment_latency_hist_node ON segment_latency_hist (node_id, ts DESC); - --- ── Preimage transfer latency ──────────────────────────────────────── -CREATE TABLE IF NOT EXISTS preimage_latency_hist ( - ts TIMESTAMPTZ NOT NULL, - node_id TEXT NOT NULL, - side SMALLINT NOT NULL, - h_0_2 INT DEFAULT 0, - h_2_5 INT DEFAULT 0, - h_5_10 INT DEFAULT 0, - h_10_15 INT DEFAULT 0, - h_15_20 INT DEFAULT 0, - h_20_30 INT DEFAULT 0, - h_30_50 INT DEFAULT 0, - h_50_75 INT DEFAULT 0, - h_75_100 INT DEFAULT 0, - h_100_150 INT DEFAULT 0, - h_150_250 INT DEFAULT 0, - h_250_500 INT DEFAULT 0, - h_500_1000 INT DEFAULT 0, - h_1000_2000 INT DEFAULT 0, - h_2000_5000 INT DEFAULT 0, - h_5000_10000 INT DEFAULT 0, - h_10000_15000 INT DEFAULT 0, - h_15000_20000 INT DEFAULT 0, - h_20000_25000 INT DEFAULT 0, - h_25000_30000 INT DEFAULT 0, - h_30000_60000 INT DEFAULT 0, - h_60000_120000 INT DEFAULT 0, - h_120000_plus INT DEFAULT 0, - total_count INT DEFAULT 0, - failed_count INT DEFAULT 0 -); -SELECT create_hypertable('preimage_latency_hist', 'ts', chunk_time_interval => INTERVAL '1 hour', if_not_exists => TRUE); -CREATE INDEX IF NOT EXISTS idx_preimage_latency_hist_node ON preimage_latency_hist (node_id, ts DESC); diff --git a/src/store.rs b/src/store.rs index e4231a5..33e4776 100644 --- a/src/store.rs +++ b/src/store.rs @@ -9,7 +9,7 @@ use sqlx::{postgres::PgPoolOptions, Executor, PgPool, Row}; use std::collections::HashMap; use std::sync::Arc; use std::time::Duration; -use tracing::{info, warn}; +use tracing::info; /// Shared string type for node IDs (matches batch_writer::NodeId). type NodeId = Arc; @@ -64,19 +64,6 @@ pub type NodeStatsRow<'a> = ( i16, ); -/// True if a migration failed as a deadlock victim (SQLSTATE 40P01), e.g. by -/// colliding with a TimescaleDB background policy job. -fn is_deadlock(e: &sqlx::migrate::MigrateError) -> bool { - use sqlx::migrate::MigrateError; - match e { - MigrateError::Execute(sqlx::Error::Database(db_err)) - | MigrateError::ExecuteMigration(sqlx::Error::Database(db_err), _) => { - db_err.code().as_deref() == Some("40P01") - } - _ => false, - } -} - pub struct EventStore { pool: PgPool, // read pool — API queries + cache warmer write_pool: PgPool, // write pool — batch writer, node updates @@ -125,25 +112,8 @@ impl EventStore { info!("Write pool connected (200 conns, no statement_timeout)"); - // Run migrations (using write pool — no timeout constraint). - // Retry on deadlock: TimescaleDB background jobs (continuous aggregate - // refresh/retention policies created by earlier migrations) can collide - // with DROP MATERIALIZED VIEW in later migrations on a fresh database. - let mut attempt = 0; - loop { - match sqlx::migrate!("./migrations").run(&write_pool).await { - Ok(()) => break, - Err(e) if attempt < 5 && is_deadlock(&e) => { - attempt += 1; - warn!( - "Migration deadlocked with a TimescaleDB background job, retrying ({}/5): {}", - attempt, e - ); - tokio::time::sleep(Duration::from_secs(2)).await; - } - Err(e) => return Err(e.into()), - } - } + // Run migrations (using write pool — no timeout constraint) + sqlx::migrate!("./migrations").run(&write_pool).await?; info!("Migrations applied successfully"); diff --git a/tests/README.md b/tests/README.md index 19be88d..f7f7d38 100644 --- a/tests/README.md +++ b/tests/README.md @@ -37,6 +37,9 @@ docker-compose up -d postgres # Create test database and run migrations cargo sqlx database create cargo sqlx migrate run +# NOTE: if your tart_test predates the migration squash (2026-08), drop it once +# first (`cargo sqlx database drop -y && cargo sqlx database create`) — the old +# _sqlx_migrations ledger doesn't match the squashed migration set. # Run integration tests SERIALLY cargo test --test api_tests --test integration_tests --test optimized_server_tests -- --test-threads=1