From d5c44c3fce999a2948c9dbfed2170479ec407bcb Mon Sep 17 00:00:00 2001 From: JivusAyrus Date: Tue, 25 Aug 2026 12:45:17 +0530 Subject: [PATCH 1/3] feat: add migration scripts for otel_spans_by_trace table and materialized view --- .../20260707090831_otel_spans_by_trace.sql | 25 ++++++++++++++ .../20260707090832_otel_spans_by_trace_mv.sql | 34 +++++++++++++++++++ 2 files changed, 59 insertions(+) create mode 100644 controlplane/clickhouse/migrations/20260707090831_otel_spans_by_trace.sql create mode 100644 controlplane/clickhouse/migrations/20260707090832_otel_spans_by_trace_mv.sql diff --git a/controlplane/clickhouse/migrations/20260707090831_otel_spans_by_trace.sql b/controlplane/clickhouse/migrations/20260707090831_otel_spans_by_trace.sql new file mode 100644 index 0000000000..e6590e5493 --- /dev/null +++ b/controlplane/clickhouse/migrations/20260707090831_otel_spans_by_trace.sql @@ -0,0 +1,25 @@ +-- migrate:up + +CREATE TABLE IF NOT EXISTS otel_spans_by_trace ( + FederatedGraphID String CODEC (ZSTD(3)), + OrganizationID LowCardinality(String) CODEC (ZSTD(3)), + TraceId String CODEC (ZSTD(3)), + Timestamp DateTime64(9) CODEC (Delta(8), ZSTD(3)), + SpanId String CODEC (ZSTD(3)), + ParentSpanId String CODEC (ZSTD(3)), + SpanName LowCardinality(String) CODEC (ZSTD(3)), + SpanKind LowCardinality(String) CODEC (ZSTD(3)), + ServiceName LowCardinality(String) CODEC (ZSTD(3)), + Duration Int64 CODEC (ZSTD(3)), + StatusCode LowCardinality(String) CODEC (ZSTD(3)), + StatusMessage String CODEC (ZSTD(3)), + ScopeName String CODEC (ZSTD(3)), + SpanAttributes Map(LowCardinality(String), String) CODEC (ZSTD(3)) +) ENGINE = MergeTree +PARTITION BY toDate(Timestamp) +ORDER BY (FederatedGraphID, OrganizationID, TraceId, Timestamp, SpanId) +TTL toDateTime(Timestamp) + toIntervalDay(30) SETTINGS index_granularity = 8192, ttl_only_drop_parts = 1; + +-- migrate:down + +DROP TABLE IF EXISTS cosmo.otel_spans_by_trace diff --git a/controlplane/clickhouse/migrations/20260707090832_otel_spans_by_trace_mv.sql b/controlplane/clickhouse/migrations/20260707090832_otel_spans_by_trace_mv.sql new file mode 100644 index 0000000000..30bbcaa739 --- /dev/null +++ b/controlplane/clickhouse/migrations/20260707090832_otel_spans_by_trace_mv.sql @@ -0,0 +1,34 @@ +-- migrate:up + +-- Feeds otel_spans_by_trace from otel_traces. This is the ONE trace MV that keeps every span +-- (no wg.router.root_span filter) — GetTrace needs the whole tree, not just the request row. +-- Spans are scoped to a graph/org by requiring wg.federated_graph.id (all cosmo spans carry it; +-- this matches the tenant filter GetTrace already applies and keeps non-cosmo spans out). A +-- TO-table MV's order is governed by the target, so no trailing ORDER BY. No Timestamp cut-line: +-- ingests from creation forward; history is backfilled (see +-- rfc/traces-analytics-performance/backfill_otel_spans_by_trace.sh). + +CREATE MATERIALIZED VIEW IF NOT EXISTS cosmo.otel_spans_by_trace_mv TO cosmo.otel_spans_by_trace AS +SELECT + SpanAttributes ['wg.federated_graph.id'] as FederatedGraphID, + toLowCardinality(SpanAttributes ['wg.organization.id']) as OrganizationID, + TraceId, + Timestamp, + SpanId, + ParentSpanId, + SpanName, + SpanKind, + ServiceName, + Duration, + StatusCode, + StatusMessage, + ScopeName, + SpanAttributes +FROM + cosmo.otel_traces +WHERE + SpanAttributes ['wg.federated_graph.id'] != ''; + +-- migrate:down + +DROP VIEW IF EXISTS cosmo.otel_spans_by_trace_mv From 5531b30829637962a4eba563d2840a5840348f9c Mon Sep 17 00:00:00 2001 From: JivusAyrus Date: Tue, 25 Aug 2026 13:01:14 +0530 Subject: [PATCH 2/3] refactor: remove outdated comments from otel_spans_by_trace materialized view migration --- .../migrations/20260707090832_otel_spans_by_trace_mv.sql | 8 -------- 1 file changed, 8 deletions(-) diff --git a/controlplane/clickhouse/migrations/20260707090832_otel_spans_by_trace_mv.sql b/controlplane/clickhouse/migrations/20260707090832_otel_spans_by_trace_mv.sql index 30bbcaa739..a94d553a73 100644 --- a/controlplane/clickhouse/migrations/20260707090832_otel_spans_by_trace_mv.sql +++ b/controlplane/clickhouse/migrations/20260707090832_otel_spans_by_trace_mv.sql @@ -1,13 +1,5 @@ -- migrate:up --- Feeds otel_spans_by_trace from otel_traces. This is the ONE trace MV that keeps every span --- (no wg.router.root_span filter) — GetTrace needs the whole tree, not just the request row. --- Spans are scoped to a graph/org by requiring wg.federated_graph.id (all cosmo spans carry it; --- this matches the tenant filter GetTrace already applies and keeps non-cosmo spans out). A --- TO-table MV's order is governed by the target, so no trailing ORDER BY. No Timestamp cut-line: --- ingests from creation forward; history is backfilled (see --- rfc/traces-analytics-performance/backfill_otel_spans_by_trace.sh). - CREATE MATERIALIZED VIEW IF NOT EXISTS cosmo.otel_spans_by_trace_mv TO cosmo.otel_spans_by_trace AS SELECT SpanAttributes ['wg.federated_graph.id'] as FederatedGraphID, From a4f9638ded02840848a64eee6d2ac45a4e371159 Mon Sep 17 00:00:00 2001 From: JivusAyrus Date: Mon, 31 Aug 2026 13:13:27 +0530 Subject: [PATCH 3/3] fix: specify schema for otel_spans_by_trace table creation --- .../migrations/20260707090831_otel_spans_by_trace.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/controlplane/clickhouse/migrations/20260707090831_otel_spans_by_trace.sql b/controlplane/clickhouse/migrations/20260707090831_otel_spans_by_trace.sql index e6590e5493..16c0524d90 100644 --- a/controlplane/clickhouse/migrations/20260707090831_otel_spans_by_trace.sql +++ b/controlplane/clickhouse/migrations/20260707090831_otel_spans_by_trace.sql @@ -1,6 +1,6 @@ -- migrate:up -CREATE TABLE IF NOT EXISTS otel_spans_by_trace ( +CREATE TABLE IF NOT EXISTS cosmo.otel_spans_by_trace ( FederatedGraphID String CODEC (ZSTD(3)), OrganizationID LowCardinality(String) CODEC (ZSTD(3)), TraceId String CODEC (ZSTD(3)),