diff --git a/CHANGELOG.md b/CHANGELOG.md index 94bcb79992b..487befd7b33 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,7 @@ * [CHANGE] Removed the following deprecated config: `-querier.filter-queryables-enabled`, `-query-frontend.cache-samples-processed-stats`, `-ingest-storage.kafka.write-clients`, `-blocks-storage.tsdb.head-postings-for-matchers-cache-size`, `-blocks-storage.tsdb.block-postings-for-matchers-cache-size`. #16352 * [CHANGE] The `bucket` label of the `thanos_objstore_bucket_*` metrics, previously always empty, is now set to the name of the bucket the metrics refer to. This lets a component that accesses more than one bucket report each of them separately. The `thanos_store_bucket_cache_*` and `cortex_bucket_index_load*` metrics gained a `bucket` label carrying the same bucket name, for the same reason. #16265 * [CHANGE] Compactor: Stabilize `-compactor.first-level-compaction-skip-future-max-time` to `true` and `-compactor.first-level-compaction-ooo-wait-period` to 5 minutes, both of which have been shown to improve batching during the split phase and reduce the total volume of L2 blocks in deployments with lots of out-of-order writes. #16464 +* [CHANGE] Distributor: Remote Write 2.0's `created_timestamp` field on `TimeSeries` has moved to a `start_timestamp` field on each `Sample` and `Histogram`, matching the upstream Remote Write 2.0 specification. The old per-series field is now reserved, but is still marshalled (derived from the first sample's or histogram's start timestamp) for compatibility with not-yet-upgraded internal components during a rolling upgrade. #16475 * [CHANGE] Querier, Store-gateway: Remove support for opaque GRPC hint types between queriers and store-gateways. Note that this change requires upgrading from Mimir 3.2. See associated release notes for more information. #16481 * [ENHANCEMENT] Compactor: Add the experimental `-compactor.block-health-validation-concurrency` option to limit how many blocks are validated concurrently within a compaction job. #16269 * [ENHANCEMENT] Query-frontend: Improve the stability of cardinality estimates and therefore sharding factors for queries when running splitting and caching inside MQE is enabled, or range vector splitting is enabled. #16274 #16301 #16305 #16311 diff --git a/integration/distributor_test.go b/integration/distributor_test.go index 29ead279852..ecbdde235ca 100644 --- a/integration/distributor_test.go +++ b/integration/distributor_test.go @@ -1140,6 +1140,118 @@ func TestDistributor_RW2_RC3_CreatedTimestamp(t *testing.T) { require.Equal(t, want.String(), got.String()) } +// TestDistributor_RW2_StartTimestamp checks that Remote Write 2.0's per-sample start_timestamp +// (Sample.StartTimestamp), unlike the older per-series created_timestamp, lets each sample in a +// batch carry its own start timestamp, and that each distinct one independently triggers its own +// zero sample. +func TestDistributor_RW2_StartTimestamp(t *testing.T) { + s, err := e2e.NewScenario(networkName) + require.NoError(t, err) + defer s.Close() + + previousRuntimeConfig := "" + require.NoError(t, writeFileToSharedDir(s, "runtime.yaml", []byte(previousRuntimeConfig))) + + // Start dependencies. + consul := e2edb.NewConsul() + minio := e2edb.NewMinio(9000, blocksBucketName) + require.NoError(t, s.StartAndWaitReady(consul, minio)) + + baseFlags := map[string]string{ + "-distributor.ingestion-tenant-shard-size": "0", + "-ingester.ring.heartbeat-period": "1s", + "-distributor.ha-tracker.enable": "true", + "-distributor.ha-tracker.enable-for-all-users": "true", + "-distributor.ha-tracker.store": "consul", + "-distributor.ha-tracker.consul.hostname": consul.NetworkHTTPEndpoint(), + "-distributor.ha-tracker.prefix": "prom_ha/", + "-timeseries-unmarshal-caching-optimization-enabled": strconv.FormatBool(true), + } + + flags := mergeFlags( + BlocksStorageFlags(), + BlocksStorageS3Flags(), + baseFlags, + ) + + // We want only distributor to be reloading runtime config. + distributorFlags := mergeFlags(flags, map[string]string{ + "-runtime-config.file": filepath.Join(e2e.ContainerSharedDir, "runtime.yaml"), + "-runtime-config.reload-period": "100ms", + // Set non-zero default for number of exemplars. That way our values used in the test (0 and 100) will show up in runtime config diff. + "-ingester.max-global-exemplars-per-user": "3", + }) + + // Ingester will not reload runtime config. + ingesterFlags := mergeFlags(flags, map[string]string{ + // Ingester will always see exemplars enabled. We do this to avoid waiting for ingester to apply new setting to TSDB. + "-ingester.max-global-exemplars-per-user": "100", + }) + + // Start Mimir components. + distributor := e2emimir.NewDistributor("distributor", consul.NetworkHTTPEndpoint(), distributorFlags) + ingester := e2emimir.NewIngester("ingester", consul.NetworkHTTPEndpoint(), ingesterFlags) + querier := e2emimir.NewQuerier("querier", consul.NetworkHTTPEndpoint(), flags) + require.NoError(t, s.StartAndWaitReady(distributor, ingester, querier)) + + // Wait until distributor has updated the ring. + require.NoError(t, distributor.WaitSumMetricsWithOptions(e2e.Equals(1), []string{"cortex_ring_members"}, e2e.WithLabelMatchers( + labels.MustNewMatcher(labels.MatchEqual, "name", "ingester"), + labels.MustNewMatcher(labels.MatchEqual, "state", "ACTIVE")))) + + // Wait until querier has updated the ring. + require.NoError(t, querier.WaitSumMetricsWithOptions(e2e.Equals(1), []string{"cortex_ring_members"}, e2e.WithLabelMatchers( + labels.MustNewMatcher(labels.MatchEqual, "name", "ingester"), + labels.MustNewMatcher(labels.MatchEqual, "state", "ACTIVE")))) + + client, err := e2emimir.NewClient(distributor.HTTPEndpoint(), querier.HTTPEndpoint(), "", "", userID) + require.NoError(t, err) + + queryEnd := time.Now().Round(time.Second) + queryStart := queryEnd.Add(-1 * time.Hour) + queryStep := 5 * time.Minute + + // Two samples in the same request, each carrying its own StartTimestamp, 15 minutes apart: + // this is only expressible via Remote Write 2.0's per-sample start_timestamp, not the older + // per-series created_timestamp, which could only ever describe a single counter generation + // per request. Each distinct start timestamp should independently trigger its own zero + // sample, not just the first one in the batch. + rw2req := &promRW2.Request{ + Timeseries: []promRW2.TimeSeries{ + { + LabelsRefs: []uint32{1, 2}, + Samples: []promRW2.Sample{ + {Timestamp: queryStart.Add(1 * time.Second).UnixMilli(), Value: 100, StartTimestamp: queryStart.UnixMilli()}, + {Timestamp: queryStart.Add(15*time.Minute + 1*time.Second).UnixMilli(), Value: 200, StartTimestamp: queryStart.Add(15 * time.Minute).UnixMilli()}, + }, + Metadata: promRW2.Metadata{ + Type: promRW2.Metadata_METRIC_TYPE_COUNTER, + HelpRef: 3, + UnitRef: 4, + }, + }, + }, + Symbols: []string{"", "__name__", "foobarST_total", "some helpST", "someunitST"}, + } + + writeRes, err := client.PushRW2(rw2req) + require.NoError(t, err) + require.Equal(t, http.StatusOK, writeRes.StatusCode) + + want := model.Matrix{{ + Metric: model.Metric{"__name__": "foobarST_total"}, + Values: []model.SamplePair{ + {Timestamp: model.Time(queryStart.UnixMilli()), Value: model.SampleValue(0)}, + {Timestamp: model.Time(queryStart.Add(5 * time.Minute).UnixMilli()), Value: model.SampleValue(100)}, + {Timestamp: model.Time(queryStart.Add(15 * time.Minute).UnixMilli()), Value: model.SampleValue(0)}, + {Timestamp: model.Time(queryStart.Add(20 * time.Minute).UnixMilli()), Value: model.SampleValue(200)}, + }, + }} + got, _, _, err := client.QueryRange("foobarST_total", queryStart, queryEnd, queryStep) + require.NoError(t, err) + require.Equal(t, want.String(), got.String()) +} + func testDistributorCases(t *testing.T, cachingUnmarshalDataEnabled bool, rwVersion string, queryStart, queryEnd time.Time, queryStep time.Duration, testCases map[string]distributorTestCase) { s, err := e2e.NewScenario(networkName) require.NoError(t, err) diff --git a/pkg/blockbuilder/tsdb.go b/pkg/blockbuilder/tsdb.go index aa50265743e..68104fda6d0 100644 --- a/pkg/blockbuilder/tsdb.go +++ b/pkg/blockbuilder/tsdb.go @@ -141,18 +141,18 @@ func (b *TSDBBuilder) PushToStorageAndReleaseRequest(ctx context.Context, req *m // and NOT the stable hashing because that's what TSDB expects. We don't need stable hashing in block builder. ref, copiedLabels := app.GetRef(nonCopiedLabels, hash) - ingestCreatedTimestamp := ts.CreatedTimestamp > 0 + var prevSampleStartTimestamp int64 for _, s := range ts.Samples { - if ingestCreatedTimestamp && ts.CreatedTimestamp < s.TimestampMs && + if s.StartTimestamp > 0 && s.StartTimestamp != prevSampleStartTimestamp && s.StartTimestamp < s.TimestampMs && (!nativeHistogramsIngestionEnabled || len(ts.Histograms) == 0 || ts.Histograms[0].Timestamp >= s.TimestampMs) { if ref != 0 { // If the cached reference exists, we try to use it. - _, err = app.AppendSTZeroSample(ref, copiedLabels, s.TimestampMs, ts.CreatedTimestamp) + _, err = app.AppendSTZeroSample(ref, copiedLabels, s.TimestampMs, s.StartTimestamp) } else { // Copy the label set because TSDB may retain it. copiedLabels = mimirpb.CopyLabels(nonCopiedLabels) - ref, err = app.AppendSTZeroSample(0, copiedLabels, s.TimestampMs, ts.CreatedTimestamp) + ref, err = app.AppendSTZeroSample(0, copiedLabels, s.TimestampMs, s.StartTimestamp) } if err != nil && !errors.Is(err, storage.ErrDuplicateSampleForTimestamp) && !errors.Is(err, storage.ErrOutOfOrderST) && !errors.Is(err, storage.ErrOutOfOrderSample) { // According to OTEL spec: https://opentelemetry.io/docs/specs/otel/metrics/data-model/#cumulative-streams-handling-unknown-start-time @@ -164,7 +164,7 @@ func (b *TSDBBuilder) PushToStorageAndReleaseRequest(ctx context.Context, req *m level.Warn(b.logger).Log("msg", "failed to store zero float sample for created timestamp", "tenant", tenantID, "err", err) discardedSamples++ } - ingestCreatedTimestamp = false // Only try to append created timestamp once per series. + prevSampleStartTimestamp = s.StartTimestamp // Only try to append a given start timestamp once per series. } if ref != 0 { @@ -194,8 +194,10 @@ func (b *TSDBBuilder) PushToStorageAndReleaseRequest(ctx context.Context, req *m continue } + var prevHistogramStartTimestamp int64 + for _, h := range ts.Histograms { - if ingestCreatedTimestamp && ts.CreatedTimestamp < h.Timestamp { + if h.StartTimestamp > 0 && h.StartTimestamp != prevHistogramStartTimestamp && h.StartTimestamp < h.Timestamp { var ( ih *histogram.Histogram fh *histogram.FloatHistogram @@ -208,11 +210,11 @@ func (b *TSDBBuilder) PushToStorageAndReleaseRequest(ctx context.Context, req *m ih = zeroHistogram } if ref != 0 { - _, err = app.AppendHistogramSTZeroSample(ref, copiedLabels, h.Timestamp, ts.CreatedTimestamp, ih, fh) + _, err = app.AppendHistogramSTZeroSample(ref, copiedLabels, h.Timestamp, h.StartTimestamp, ih, fh) } else { // Copy the label set because both TSDB and the active series tracker may retain it. copiedLabels = mimirpb.CopyLabels(nonCopiedLabels) - ref, err = app.AppendHistogramSTZeroSample(0, copiedLabels, h.Timestamp, ts.CreatedTimestamp, ih, fh) + ref, err = app.AppendHistogramSTZeroSample(0, copiedLabels, h.Timestamp, h.StartTimestamp, ih, fh) } if err != nil && !errors.Is(err, storage.ErrDuplicateSampleForTimestamp) && !errors.Is(err, storage.ErrOutOfOrderST) && !errors.Is(err, storage.ErrOutOfOrderSample) { // According to OTEL spec: https://opentelemetry.io/docs/specs/otel/metrics/data-model/#cumulative-streams-handling-unknown-start-time @@ -224,7 +226,7 @@ func (b *TSDBBuilder) PushToStorageAndReleaseRequest(ctx context.Context, req *m level.Warn(b.logger).Log("msg", "failed to store zero histogram sample for created timestamp", "tenant", tenantID, "err", err) discardedSamples++ } - ingestCreatedTimestamp = false // Only try to append created timestamp once per series. + prevHistogramStartTimestamp = h.StartTimestamp // Only try to append a given start timestamp once per series. } var ( ih *histogram.Histogram diff --git a/pkg/blockbuilder/tsdb_test.go b/pkg/blockbuilder/tsdb_test.go index e1ee1a68c98..2f6e4d4a41e 100644 --- a/pkg/blockbuilder/tsdb_test.go +++ b/pkg/blockbuilder/tsdb_test.go @@ -754,8 +754,9 @@ func defaultLimitsTestConfig() validation.Limits { return limits } -// TestBuilderCreatedTimestamp tests the Remote Write protocol Created timestamp field is correctly handled. -// The Created timestamp injects extra zero samples at the specified timestamp - if it's before the next sample. +// TestBuilderCreatedTimestamp tests the Remote Write 2.0 protocol per-sample/per-histogram +// start timestamp field is correctly handled. +// The start timestamp injects extra zero samples at the specified timestamp - if it's before the sample/histogram it is attached to. func TestBuilderCreatedTimestamp(t *testing.T) { var ( user1 = "user_ooo_disabled" @@ -817,64 +818,60 @@ func TestBuilderCreatedTimestamp(t *testing.T) { }, } } + // histogramWithStartTimestamp sets the start timestamp (ST) on a mimirpb.Histogram + // returned by one of the constructors above, since the field can't be set inline + // on a function call result inside a slice literal. + histogramWithStartTimestamp := func(h mimirpb.Histogram, st int64) mimirpb.Histogram { + h.StartTimestamp = st + return h + } testCases := map[string]struct { - input []mimirpb.TimeSeries // We'll generate the Labels, just set the samples and created timestamp. + input []mimirpb.TimeSeries // We'll generate the Labels, just set the samples/histograms and their start timestamps. expectSamples []test.Sample expectDiscarded int }{ "float samples": { input: []mimirpb.TimeSeries{ { - // Samples and Created timestamp (CT) outside the current block. + // Samples and start timestamp (ST) outside the current block. Samples: []mimirpb.Sample{ {TimestampMs: lastEnd - 50000 + 100, Value: 1}, - {TimestampMs: lastEnd - 50000 + 300, Value: 2}, + {TimestampMs: lastEnd - 50000 + 300, Value: 2, StartTimestamp: lastEnd - 50000 + 200}, }, - CreatedTimestamp: lastEnd - 50000 + 200, }, { - // Sample inside the current block, but CT outside. + // Sample inside the current block, but ST outside. Samples: []mimirpb.Sample{ - {TimestampMs: lastEnd + 100, Value: 3}, + {TimestampMs: lastEnd + 100, Value: 3, StartTimestamp: lastEnd - 50000 + 200}, }, - CreatedTimestamp: lastEnd - 50000 + 200, }, { - // Samples and CT inside the current block. + // Samples and ST inside the current block. Samples: []mimirpb.Sample{ - {TimestampMs: lastEnd + 300, Value: 4}, + {TimestampMs: lastEnd + 300, Value: 4, StartTimestamp: lastEnd + 200}, {TimestampMs: lastEnd + 400, Value: 5}, }, - CreatedTimestamp: lastEnd + 200, }, { - // Repeated CT. + // Repeated ST. Samples: []mimirpb.Sample{ - {TimestampMs: lastEnd + 500, Value: 6}, + {TimestampMs: lastEnd + 500, Value: 6, StartTimestamp: lastEnd + 200}, }, - CreatedTimestamp: lastEnd + 200, }, { - // Samples and CT mixed in in the current block. + // Samples and ST mixed in in the current block. Samples: []mimirpb.Sample{ {TimestampMs: lastEnd + 600, Value: 7}, - {TimestampMs: lastEnd + 800, Value: 8}, + {TimestampMs: lastEnd + 800, Value: 8, StartTimestamp: lastEnd + 700}, }, - CreatedTimestamp: lastEnd + 700, - }, - { - // CT alone is ignored. - Samples: []mimirpb.Sample{}, - CreatedTimestamp: lastEnd + 1000, }, { - // CT inside current block but some samples in the next block. + // ST inside current block but some samples in the next block. Samples: []mimirpb.Sample{ {TimestampMs: currEnd - 200, Value: 9}, - {TimestampMs: currEnd + 200, Value: 10}, + {TimestampMs: currEnd + 200, Value: 10, StartTimestamp: currEnd - 100}, }, - CreatedTimestamp: currEnd - 100, }, }, expectSamples: []test.Sample{ @@ -897,62 +894,50 @@ func TestBuilderCreatedTimestamp(t *testing.T) { "histogram samples": { input: []mimirpb.TimeSeries{ { - // Samples and Created timestamp (CT) outside the current block. + // Histograms and start timestamp (ST) outside the current block. Histograms: []mimirpb.Histogram{ simpleTestHistogram(lastEnd-50000+100, 1), - simpleTestHistogram(lastEnd-50000+300, 2), + histogramWithStartTimestamp(simpleTestHistogram(lastEnd-50000+300, 2), lastEnd-50000+200), }, - CreatedTimestamp: lastEnd - 50000 + 200, }, { - // Sample inside the current block, but CT outside. + // Histogram inside the current block, but ST outside. Histograms: []mimirpb.Histogram{ - simpleTestHistogram(lastEnd+100, 3), + histogramWithStartTimestamp(simpleTestHistogram(lastEnd+100, 3), lastEnd-50000+200), }, - CreatedTimestamp: lastEnd - 50000 + 200, }, { - // Samples and CT inside the current block. + // Histograms and ST inside the current block. Histograms: []mimirpb.Histogram{ - simpleTestHistogram(lastEnd+300, 4), + histogramWithStartTimestamp(simpleTestHistogram(lastEnd+300, 4), lastEnd+200), simpleTestHistogram(lastEnd+400, 5), }, - CreatedTimestamp: lastEnd + 200, }, { - // Repeated CT. + // Repeated ST. Histograms: []mimirpb.Histogram{ - simpleTestHistogram(lastEnd+500, 6), + histogramWithStartTimestamp(simpleTestHistogram(lastEnd+500, 6), lastEnd+200), }, - CreatedTimestamp: lastEnd + 200, }, { - // Samples and CT mixed in in the current block. + // Histograms and ST mixed in in the current block. Histograms: []mimirpb.Histogram{ simpleTestHistogram(lastEnd+600, 7), - simpleTestHistogram(lastEnd+800, 8), + histogramWithStartTimestamp(simpleTestHistogram(lastEnd+800, 8), lastEnd+700), }, - CreatedTimestamp: lastEnd + 700, - }, - { - // CT alone is ignored. - Histograms: []mimirpb.Histogram{}, - CreatedTimestamp: lastEnd + 900, }, { // Test float histogram produces the correct zero sample. Histograms: []mimirpb.Histogram{ - simpleTestFloatHistogram(lastEnd+1100, 8.5), + histogramWithStartTimestamp(simpleTestFloatHistogram(lastEnd+1100, 8.5), lastEnd+1000), }, - CreatedTimestamp: lastEnd + 1000, }, { - // CT inside current block but some samples in the next block. + // ST inside current block but some samples in the next block. Histograms: []mimirpb.Histogram{ simpleTestHistogram(currEnd-200, 9), - simpleTestHistogram(currEnd+200, 10), + histogramWithStartTimestamp(simpleTestHistogram(currEnd+200, 10), currEnd-100), }, - CreatedTimestamp: currEnd - 100, }, }, expectSamples: []test.Sample{ @@ -983,9 +968,8 @@ func TestBuilderCreatedTimestamp(t *testing.T) { }, { Samples: []mimirpb.Sample{ - {TimestampMs: lastEnd + 200, Value: 8}, + {TimestampMs: lastEnd + 200, Value: 8, StartTimestamp: lastEnd + 100}, // Duplicate the previous sample. }, - CreatedTimestamp: lastEnd + 100, // Duplicate the previous sample. }, }, expectSamples: []test.Sample{ @@ -1002,9 +986,9 @@ func TestBuilderCreatedTimestamp(t *testing.T) { }, { Histograms: []mimirpb.Histogram{ - simpleTestHistogram(lastEnd+200, 8), + // Duplicate the previous sample. + histogramWithStartTimestamp(simpleTestHistogram(lastEnd+200, 8), lastEnd+100), }, - CreatedTimestamp: lastEnd + 100, // Duplicate the previous sample. }, }, expectSamples: []test.Sample{ @@ -1012,6 +996,27 @@ func TestBuilderCreatedTimestamp(t *testing.T) { expectedHistogram(lastEnd+200, 8), }, }, + "float samples with multiple distinct start timestamps in one batch": { + input: []mimirpb.TimeSeries{ + { + // The second sample repeats the first sample's start timestamp (ST), so it + // must not trigger a second zero-sample. The third sample has a different + // ST, so it must trigger a second zero-sample. + Samples: []mimirpb.Sample{ + {TimestampMs: lastEnd + 200, Value: 1, StartTimestamp: lastEnd + 100}, + {TimestampMs: lastEnd + 300, Value: 2, StartTimestamp: lastEnd + 100}, + {TimestampMs: lastEnd + 500, Value: 3, StartTimestamp: lastEnd + 400}, + }, + }, + }, + expectSamples: []test.Sample{ + {TS: lastEnd + 100, Val: 0}, + {TS: lastEnd + 200, Val: 1}, + {TS: lastEnd + 300, Val: 2}, + {TS: lastEnd + 400, Val: 0}, + {TS: lastEnd + 500, Val: 3}, + }, + }, } logger := log.NewNopLogger() @@ -1033,10 +1038,9 @@ func TestBuilderCreatedTimestamp(t *testing.T) { // Process the write requests. for _, input := range tc.input { ts := &mimirpb.TimeSeries{ - Labels: mimirpb.FromLabelsToLabelAdapters(labels.FromStrings(model.MetricNameLabel, metricName)), - Samples: make([]mimirpb.Sample, len(input.Samples)), - Histograms: make([]mimirpb.Histogram, len(input.Histograms)), - CreatedTimestamp: input.CreatedTimestamp, + Labels: mimirpb.FromLabelsToLabelAdapters(labels.FromStrings(model.MetricNameLabel, metricName)), + Samples: make([]mimirpb.Sample, len(input.Samples)), + Histograms: make([]mimirpb.Histogram, len(input.Histograms)), } copy(ts.Samples, input.Samples) copy(ts.Histograms, input.Histograms) diff --git a/pkg/continuoustest/ingest_storage_record.go b/pkg/continuoustest/ingest_storage_record.go index 225e4a037fa..14e48b42912 100644 --- a/pkg/continuoustest/ingest_storage_record.go +++ b/pkg/continuoustest/ingest_storage_record.go @@ -523,9 +523,6 @@ func TimeseriesEqual(this *mimirpb.TimeSeries, that interface{}) bool { } else if this == nil { return false } - if this.CreatedTimestamp != that1.CreatedTimestamp { - return false - } if len(this.Labels) != len(that1.Labels) { return false } @@ -587,7 +584,7 @@ func SampleEqual(this *mimirpb.Sample, that interface{}) bool { if !floatEqualsEquateNaN(this.Value, that1.Value) { return false } - return true + return this.StartTimestamp == that1.StartTimestamp } // ExemplarEqual is a copy of mimirpb.Exemplar.Equal but equates NaN values. @@ -736,7 +733,7 @@ func HistogramEqual(this *mimirpb.Histogram, that interface{}) bool { return false } } - return true + return this.StartTimestamp == that1.StartTimestamp } func floatEqualsEquateNaN(a, b float64) bool { diff --git a/pkg/distributor/otel_test.go b/pkg/distributor/otel_test.go index 6b831b83d57..65d457dc08c 100644 --- a/pkg/distributor/otel_test.go +++ b/pkg/distributor/otel_test.go @@ -961,9 +961,8 @@ func TestOTelCTZeroIngestion(t *testing.T) { return md }(), expected: mimirpb.TimeSeries{ - Labels: []mimirpb.LabelAdapter{{Name: "__name__", Value: "test_metric"}, {Name: "metric_attr", Value: "metric value"}}, - Samples: []mimirpb.Sample{{TimestampMs: ts.UnixMilli(), Value: 5}}, - CreatedTimestamp: ts.Add(-time.Minute).UnixMilli(), + Labels: []mimirpb.LabelAdapter{{Name: "__name__", Value: "test_metric"}, {Name: "metric_attr", Value: "metric value"}}, + Samples: []mimirpb.Sample{{TimestampMs: ts.UnixMilli(), Value: 5, StartTimestamp: ts.Add(-time.Minute).UnixMilli()}}, }, }, { @@ -985,9 +984,8 @@ func TestOTelCTZeroIngestion(t *testing.T) { return md }(), expected: mimirpb.TimeSeries{ - Labels: []mimirpb.LabelAdapter{{Name: "__name__", Value: "test_metric"}, {Name: "metric_attr", Value: "metric value"}}, - Samples: []mimirpb.Sample{{TimestampMs: ts.UnixMilli(), Value: 5}}, - CreatedTimestamp: 0, + Labels: []mimirpb.LabelAdapter{{Name: "__name__", Value: "test_metric"}, {Name: "metric_attr", Value: "metric value"}}, + Samples: []mimirpb.Sample{{TimestampMs: ts.UnixMilli(), Value: 5}}, }, }, } diff --git a/pkg/distributor/otlpappender/mimir_appender.go b/pkg/distributor/otlpappender/mimir_appender.go index e09f1270706..5b3a1409240 100644 --- a/pkg/distributor/otlpappender/mimir_appender.go +++ b/pkg/distributor/otlpappender/mimir_appender.go @@ -28,8 +28,12 @@ type MimirAppender struct { EnableCreatedTimestampZeroIngestion bool ValidIntervalCreatedTimestampZeroIngestion int64 - series []mimirpb.PreallocTimeseries - metadata []*mimirpb.MetricMetadata + series []mimirpb.PreallocTimeseries + // seriesStartTimestamp tracks the start timestamp of the last data point appended to the + // corresponding entry in series, parallel to it by index. Used by ctRequiresNewSeries to + // detect a new counter generation (a different start timestamp) within the same push. + seriesStartTimestamp []int64 + metadata []*mimirpb.MetricMetadata // To avoid creating extra time series when the same label set is used // multiple times, we keep track of the appended time series. refs map[uint64]labelsIdx @@ -67,11 +71,15 @@ func (c *MimirAppender) Append(_ storage.SeriesRef, ls labels.Labels, ct, t int6 switch { case fh != nil: - c.series[idx.idx].Histograms = append(c.series[idx.idx].Histograms, mimirpb.FromFloatHistogramToHistogramProto(t, fh)) + hp := mimirpb.FromFloatHistogramToHistogramProto(t, fh) + hp.StartTimestamp = ct + c.series[idx.idx].Histograms = append(c.series[idx.idx].Histograms, hp) case h != nil: - c.series[idx.idx].Histograms = append(c.series[idx.idx].Histograms, mimirpb.FromHistogramToHistogramProto(t, h)) + hp := mimirpb.FromHistogramToHistogramProto(t, h) + hp.StartTimestamp = ct + c.series[idx.idx].Histograms = append(c.series[idx.idx].Histograms, hp) default: - c.series[idx.idx].Samples = append(c.series[idx.idx].Samples, mimirpb.Sample{TimestampMs: t, Value: v}) + c.series[idx.idx].Samples = append(c.series[idx.idx].Samples, mimirpb.Sample{TimestampMs: t, Value: v, StartTimestamp: ct}) } c.appendExemplars(idx.idx, opts.Exemplars) c.appendMetadata(opts.MetricFamilyName, opts.Metadata) @@ -91,9 +99,9 @@ func (c *MimirAppender) recalcCreatedTimestamp(t, ct int64) int64 { } // ctRequiresNewSeries checks if the created timestamp is meaningful and different -// from the one already stored in the series at the given index. +// from the one already stored for the series at the given index. func (c *MimirAppender) ctRequiresNewSeries(seriesIdx int, ct int64) bool { - return ct > 0 && c.series[seriesIdx].CreatedTimestamp != ct + return ct > 0 && c.seriesStartTimestamp[seriesIdx] != ct } // processLabelsAndMetadata figures out if we have already seen this @@ -142,8 +150,8 @@ func (c *MimirAppender) processLabelsAndMetadata(ls labels.Labels) (hash uint64, func (c *MimirAppender) createNewSeries(idx *labelsIdx, collisionIdx int, hash uint64, ls labels.Labels, ct int64) { ts := mimirpb.TimeseriesFromPool() ts.Labels = mimirpb.FromLabelsToLabelAdapters(ls) - ts.CreatedTimestamp = ct c.series = append(c.series, mimirpb.PreallocTimeseries{TimeSeries: ts}) + c.seriesStartTimestamp = append(c.seriesStartTimestamp, ct) idx.idx = len(c.series) - 1 if collisionIdx == -1 { diff --git a/pkg/distributor/otlpappender/mimir_appender_test.go b/pkg/distributor/otlpappender/mimir_appender_test.go index 794f9b8ecf9..f2d43c58777 100644 --- a/pkg/distributor/otlpappender/mimir_appender_test.go +++ b/pkg/distributor/otlpappender/mimir_appender_test.go @@ -39,6 +39,12 @@ func appendHistogram(t *testing.T, ca *MimirAppender, ls labels.Labels, meta met require.NoError(t, err) } +// histogramProtoWithStartTimestamp returns a copy of h with StartTimestamp set, to inline into test fixtures. +func histogramProtoWithStartTimestamp(h mimirpb.Histogram, st int64) mimirpb.Histogram { + h.StartTimestamp = st + return h +} + func TestMimirAppender(t *testing.T) { collidingLabels1, collidingLabels2 := labelsWithHashCollision() @@ -73,9 +79,8 @@ func TestMimirAppender(t *testing.T) { TimeSeries: &mimirpb.TimeSeries{ Labels: []mimirpb.LabelAdapter{{Name: model.MetricNameLabel, Value: "spam"}, {Name: "a", Value: "ham"}}, Samples: []mimirpb.Sample{ - {TimestampMs: 2000, Value: 42.0}, + {TimestampMs: 2000, Value: 42.0, StartTimestamp: 1000}, }, - CreatedTimestamp: 1000, Exemplars: []mimirpb.Exemplar{ { Labels: []mimirpb.LabelAdapter{{Name: "traceId", Value: "myid"}}, @@ -116,10 +121,9 @@ func TestMimirAppender(t *testing.T) { TimeSeries: &mimirpb.TimeSeries{ Labels: []mimirpb.LabelAdapter{{Name: model.MetricNameLabel, Value: "spam"}, {Name: "a", Value: "ham"}}, Samples: []mimirpb.Sample{ - {TimestampMs: 2000, Value: 42.0}, - {TimestampMs: 3000, Value: 52.0}, + {TimestampMs: 2000, Value: 42.0, StartTimestamp: 1000}, + {TimestampMs: 3000, Value: 52.0, StartTimestamp: 1000}, }, - CreatedTimestamp: 1000, Exemplars: []mimirpb.Exemplar{ { Labels: []mimirpb.LabelAdapter{{Name: "traceId", Value: "myid"}}, @@ -165,9 +169,8 @@ func TestMimirAppender(t *testing.T) { TimeSeries: &mimirpb.TimeSeries{ Labels: []mimirpb.LabelAdapter{{Name: model.MetricNameLabel, Value: "spam"}, {Name: "a", Value: "ham"}}, Samples: []mimirpb.Sample{ - {TimestampMs: 2000, Value: 42.0}, + {TimestampMs: 2000, Value: 42.0, StartTimestamp: 1000}, }, - CreatedTimestamp: 1000, Exemplars: []mimirpb.Exemplar{ { Labels: []mimirpb.LabelAdapter{{Name: "traceId", Value: "myid"}}, @@ -181,9 +184,8 @@ func TestMimirAppender(t *testing.T) { TimeSeries: &mimirpb.TimeSeries{ Labels: []mimirpb.LabelAdapter{{Name: model.MetricNameLabel, Value: "spam"}, {Name: "a", Value: "cheese"}}, Samples: []mimirpb.Sample{ - {TimestampMs: 3000, Value: 52.0}, + {TimestampMs: 3000, Value: 52.0, StartTimestamp: 1000}, }, - CreatedTimestamp: 1000, Exemplars: []mimirpb.Exemplar{ { Labels: []mimirpb.LabelAdapter{{Name: "traceId", Value: "myid2"}}, @@ -224,9 +226,8 @@ func TestMimirAppender(t *testing.T) { TimeSeries: &mimirpb.TimeSeries{ Labels: []mimirpb.LabelAdapter{{Name: model.MetricNameLabel, Value: "spam"}, {Name: "a", Value: "ham"}}, Samples: []mimirpb.Sample{ - {TimestampMs: 2000, Value: 42.0}, + {TimestampMs: 2000, Value: 42.0, StartTimestamp: 1000}, }, - CreatedTimestamp: 1000, Exemplars: []mimirpb.Exemplar{ { Labels: []mimirpb.LabelAdapter{{Name: "traceId", Value: "myid"}}, @@ -240,9 +241,8 @@ func TestMimirAppender(t *testing.T) { TimeSeries: &mimirpb.TimeSeries{ Labels: []mimirpb.LabelAdapter{{Name: model.MetricNameLabel, Value: "spam"}, {Name: "a", Value: "ham"}}, Samples: []mimirpb.Sample{ - {TimestampMs: 3000, Value: 52.0}, + {TimestampMs: 3000, Value: 52.0, StartTimestamp: 2400}, }, - CreatedTimestamp: 2400, Exemplars: []mimirpb.Exemplar{ { Labels: []mimirpb.LabelAdapter{{Name: "traceId", Value: "myid2"}}, @@ -261,7 +261,6 @@ func TestMimirAppender(t *testing.T) { {TimestampMs: 2000, Value: 42.0}, {TimestampMs: 3000, Value: 52.0}, }, - CreatedTimestamp: 0, Exemplars: []mimirpb.Exemplar{ { Labels: []mimirpb.LabelAdapter{{Name: "traceId", Value: "myid"}}, @@ -310,7 +309,6 @@ func TestMimirAppender(t *testing.T) { {TimestampMs: defaultIntervalForStartTimestamps + 2000, Value: 42.0}, {TimestampMs: defaultIntervalForStartTimestamps + 3000, Value: 52.0}, }, - CreatedTimestamp: 0, Exemplars: []mimirpb.Exemplar{ { Labels: []mimirpb.LabelAdapter{{Name: "traceId", Value: "myid"}}, @@ -356,10 +354,9 @@ func TestMimirAppender(t *testing.T) { TimeSeries: &mimirpb.TimeSeries{ Labels: []mimirpb.LabelAdapter{{Name: model.MetricNameLabel, Value: "spam"}, {Name: "a", Value: "ham"}}, Samples: []mimirpb.Sample{ - {TimestampMs: defaultIntervalForStartTimestamps - 2000, Value: 42.0}, + {TimestampMs: defaultIntervalForStartTimestamps - 2000, Value: 42.0, StartTimestamp: 1000}, {TimestampMs: defaultIntervalForStartTimestamps + 3000, Value: 52.0}, }, - CreatedTimestamp: 1000, Exemplars: []mimirpb.Exemplar{ { Labels: []mimirpb.LabelAdapter{{Name: "traceId", Value: "myid"}}, @@ -399,9 +396,8 @@ func TestMimirAppender(t *testing.T) { TimeSeries: &mimirpb.TimeSeries{ Labels: []mimirpb.LabelAdapter{{Name: model.MetricNameLabel, Value: "spam"}, {Name: "a", Value: "ham"}}, Histograms: []mimirpb.Histogram{ - mimirpb.FromHistogramToHistogramProto(2000, test.GenerateTestHistogram(1)), + histogramProtoWithStartTimestamp(mimirpb.FromHistogramToHistogramProto(2000, test.GenerateTestHistogram(1)), 1000), }, - CreatedTimestamp: 1000, Exemplars: []mimirpb.Exemplar{ { Labels: []mimirpb.LabelAdapter{{Name: "traceId", Value: "myid"}}, @@ -442,12 +438,11 @@ func TestMimirAppender(t *testing.T) { TimeSeries: &mimirpb.TimeSeries{ Labels: []mimirpb.LabelAdapter{{Name: model.MetricNameLabel, Value: "spam_count"}, {Name: "a", Value: "ham"}}, Samples: []mimirpb.Sample{ - {TimestampMs: 2000, Value: 42.0}, + {TimestampMs: 2000, Value: 42.0, StartTimestamp: 1000}, }, Histograms: []mimirpb.Histogram{ - mimirpb.FromHistogramToHistogramProto(3000, test.GenerateTestHistogram(2)), + histogramProtoWithStartTimestamp(mimirpb.FromHistogramToHistogramProto(3000, test.GenerateTestHistogram(2)), 1000), }, - CreatedTimestamp: 1000, Exemplars: []mimirpb.Exemplar{ { Labels: []mimirpb.LabelAdapter{{Name: "traceId", Value: "myid"}}, @@ -512,7 +507,6 @@ func TestMimirAppender(t *testing.T) { {TimestampMs: 1000, Value: 42.0}, {TimestampMs: 3000, Value: 46.0}, }, - CreatedTimestamp: 0, }, }, { @@ -522,7 +516,6 @@ func TestMimirAppender(t *testing.T) { {TimestampMs: 2000, Value: 44.0}, {TimestampMs: 4000, Value: 48.0}, }, - CreatedTimestamp: 0, }, }, }, @@ -555,8 +548,16 @@ func TestMimirAppender(t *testing.T) { } else if tc.expectTimeseries != nil { expectedTimeseries = make([]mimirpb.PreallocTimeseries, len(tc.expectTimeseries)) for i, ts := range tc.expectTimeseries { - innerTs := *ts.TimeSeries // Shallow copy to modify CreatedTimestamp. - innerTs.CreatedTimestamp = 0 // Set CreatedTimestamp to 0 if the feature is disabled. + innerTs := *ts.TimeSeries // Shallow copy to modify Samples/Histograms. + // Set StartTimestamp to 0 on every sample/histogram if the feature is disabled. + innerTs.Samples = append([]mimirpb.Sample(nil), innerTs.Samples...) + for j := range innerTs.Samples { + innerTs.Samples[j].StartTimestamp = 0 + } + innerTs.Histograms = append([]mimirpb.Histogram(nil), innerTs.Histograms...) + for j := range innerTs.Histograms { + innerTs.Histograms[j].StartTimestamp = 0 + } expectedTimeseries[i].TimeSeries = &innerTs } } diff --git a/pkg/ingester/ingester_push.go b/pkg/ingester/ingester_push.go index c14ec544cf4..cca51a98d79 100644 --- a/pkg/ingester/ingester_push.go +++ b/pkg/ingester/ingester_push.go @@ -608,7 +608,7 @@ func (i *Ingester) pushSamplesToAppender( // To find out if any sample was added to this series, we keep old value. oldSucceededSamplesCount := stats.succeededSamplesCount - ingestCreatedTimestamp := ts.CreatedTimestamp > 0 + var prevSampleStartTimestamp int64 for _, s := range ts.Samples { var err error @@ -622,13 +622,13 @@ func (i *Ingester) pushSamplesToAppender( continue } - if ingestCreatedTimestamp && ts.CreatedTimestamp < s.TimestampMs && (!nativeHistogramsIngestionEnabled || len(ts.Histograms) == 0 || ts.Histograms[0].Timestamp >= s.TimestampMs) { + if s.StartTimestamp > 0 && s.StartTimestamp != prevSampleStartTimestamp && s.StartTimestamp < s.TimestampMs && (!nativeHistogramsIngestionEnabled || len(ts.Histograms) == 0 || ts.Histograms[0].Timestamp >= s.TimestampMs) { if ref != 0 { - _, err = app.AppendSTZeroSample(ref, copiedLabels, s.TimestampMs, ts.CreatedTimestamp) + _, err = app.AppendSTZeroSample(ref, copiedLabels, s.TimestampMs, s.StartTimestamp) } else { // Copy the label set because both TSDB and the active series tracker may retain it. copiedLabels = mimirpb.CopyLabels(nonCopiedLabels) - ref, err = app.AppendSTZeroSample(0, copiedLabels, s.TimestampMs, ts.CreatedTimestamp) + ref, err = app.AppendSTZeroSample(0, copiedLabels, s.TimestampMs, s.StartTimestamp) } if err == nil { stats.succeededSamplesCount++ @@ -639,9 +639,9 @@ func (i *Ingester) pushSamplesToAppender( // samples. Thus we ignore if zero sample would cause duplicate. // We also ignore out of order sample as created timestamp is out of order most of the time, // except when written before the first sample. - errProcessor.ProcessErr(err, ts.CreatedTimestamp, ts.Labels) + errProcessor.ProcessErr(err, s.StartTimestamp, ts.Labels) } - ingestCreatedTimestamp = false // Only try to append created timestamp once per series. + prevSampleStartTimestamp = s.StartTimestamp // Only try to append a given start timestamp once per series. } // If the cached reference exists, we try to use it. @@ -672,6 +672,8 @@ func (i *Ingester) pushSamplesToAppender( numNativeHistogramBuckets := -1 if nativeHistogramsIngestionEnabled { + var prevHistogramStartTimestamp int64 + for _, h := range ts.Histograms { var ( err error @@ -693,13 +695,13 @@ func (i *Ingester) pushSamplesToAppender( ih = mimirpb.FromHistogramProtoToHistogram(&h) } - if ingestCreatedTimestamp && ts.CreatedTimestamp < h.Timestamp { + if h.StartTimestamp > 0 && h.StartTimestamp != prevHistogramStartTimestamp && h.StartTimestamp < h.Timestamp { if ref != 0 { - _, err = app.AppendHistogramSTZeroSample(ref, copiedLabels, h.Timestamp, ts.CreatedTimestamp, ih, fh) + _, err = app.AppendHistogramSTZeroSample(ref, copiedLabels, h.Timestamp, h.StartTimestamp, ih, fh) } else { // Copy the label set because both TSDB and the active series tracker may retain it. copiedLabels = mimirpb.CopyLabels(nonCopiedLabels) - ref, err = app.AppendHistogramSTZeroSample(0, copiedLabels, h.Timestamp, ts.CreatedTimestamp, ih, fh) + ref, err = app.AppendHistogramSTZeroSample(0, copiedLabels, h.Timestamp, h.StartTimestamp, ih, fh) } if err == nil { stats.succeededSamplesCount++ @@ -710,9 +712,9 @@ func (i *Ingester) pushSamplesToAppender( // samples. Thus we ignore if zero sample would cause duplicate. // We also ignore out of order sample as created timestamp is out of order most of the time, // except when written before the first sample. - errProcessor.ProcessErr(err, ts.CreatedTimestamp, ts.Labels) + errProcessor.ProcessErr(err, h.StartTimestamp, ts.Labels) } - ingestCreatedTimestamp = false // Only try to append created timestamp once per series. + prevHistogramStartTimestamp = h.StartTimestamp // Only try to append a given start timestamp once per series. } // If the cached reference exists, we try to use it. diff --git a/pkg/ingester/ingester_test.go b/pkg/ingester/ingester_test.go index 41fa9dd680c..a07ecf1e6f0 100644 --- a/pkg/ingester/ingester_test.go +++ b/pkg/ingester/ingester_test.go @@ -482,6 +482,14 @@ func mustNewNativeHistogramValidationError(t *testing.T, originalErr error, time return res } +// histogramWithStartTimestamp sets the start timestamp on h and returns it. It exists because +// mimirpb.FromHistogramToHistogramProto() returns a value, so its StartTimestamp field cannot be +// set inline within a slice literal. +func histogramWithStartTimestamp(h mimirpb.Histogram, startTimestamp int64) mimirpb.Histogram { + h.StartTimestamp = startTimestamp + return h +} + func TestIngester_Push(t *testing.T) { metricLabelAdapters := []mimirpb.LabelAdapter{{Name: model.MetricNameLabel, Value: "test"}} metricLabelSet := mimirpb.FromLabelAdaptersToMetric(metricLabelAdapters) @@ -3632,9 +3640,8 @@ func TestIngester_Push(t *testing.T) { TimeSeries: &mimirpb.TimeSeries{ Labels: []mimirpb.LabelAdapter{metricLabelAdapters[0]}, // Cannot reuse test slice var because it is cleared and returned to the pool Samples: []mimirpb.Sample{ - {Value: 1, TimestampMs: 600}, + {Value: 1, TimestampMs: 600, StartTimestamp: 500}, }, - CreatedTimestamp: 500, }, }, }, @@ -3645,9 +3652,8 @@ func TestIngester_Push(t *testing.T) { TimeSeries: &mimirpb.TimeSeries{ Labels: []mimirpb.LabelAdapter{metricLabelAdapters[0]}, // Cannot reuse test slice var because it is cleared and returned to the pool Histograms: []mimirpb.Histogram{ - mimirpb.FromHistogramToHistogramProto(800, util_test.GenerateTestHistogram(1)), + histogramWithStartTimestamp(mimirpb.FromHistogramToHistogramProto(800, util_test.GenerateTestHistogram(1)), 700), }, - CreatedTimestamp: 700, }, }, }, @@ -3717,9 +3723,8 @@ func TestIngester_Push(t *testing.T) { TimeSeries: &mimirpb.TimeSeries{ Labels: []mimirpb.LabelAdapter{metricLabelAdapters[0]}, // Cannot reuse test slice var because it is cleared and returned to the pool Samples: []mimirpb.Sample{ - {Value: 1, TimestampMs: 600}, + {Value: 1, TimestampMs: 600, StartTimestamp: 1500}, }, - CreatedTimestamp: 1500, }, }, }, @@ -3730,9 +3735,8 @@ func TestIngester_Push(t *testing.T) { TimeSeries: &mimirpb.TimeSeries{ Labels: []mimirpb.LabelAdapter{metricLabelAdapters[0]}, // Cannot reuse test slice var because it is cleared and returned to the pool Histograms: []mimirpb.Histogram{ - mimirpb.FromHistogramToHistogramProto(800, util_test.GenerateTestHistogram(1)), + histogramWithStartTimestamp(mimirpb.FromHistogramToHistogramProto(800, util_test.GenerateTestHistogram(1)), 1500), }, - CreatedTimestamp: 1500, }, }, }, @@ -3800,9 +3804,8 @@ func TestIngester_Push(t *testing.T) { TimeSeries: &mimirpb.TimeSeries{ Labels: []mimirpb.LabelAdapter{metricLabelAdapters[0]}, // Cannot reuse test slice var because it is cleared and returned to the pool Samples: []mimirpb.Sample{ - {Value: 1, TimestampMs: 600}, + {Value: 1, TimestampMs: 600, StartTimestamp: 600}, }, - CreatedTimestamp: 600, }, }, }, @@ -3813,9 +3816,8 @@ func TestIngester_Push(t *testing.T) { TimeSeries: &mimirpb.TimeSeries{ Labels: []mimirpb.LabelAdapter{metricLabelAdapters[0]}, // Cannot reuse test slice var because it is cleared and returned to the pool Histograms: []mimirpb.Histogram{ - mimirpb.FromHistogramToHistogramProto(800, util_test.GenerateTestHistogram(1)), + histogramWithStartTimestamp(mimirpb.FromHistogramToHistogramProto(800, util_test.GenerateTestHistogram(1)), 800), }, - CreatedTimestamp: 800, }, }, }, @@ -3884,10 +3886,9 @@ func TestIngester_Push(t *testing.T) { Labels: []mimirpb.LabelAdapter{metricLabelAdapters[0]}, // Cannot reuse test slice var because it is cleared and returned to the pool Histograms: []mimirpb.Histogram{ mimirpb.FromHistogramToHistogramProto(400, util_test.GenerateTestHistogram(1)), - mimirpb.FromHistogramToHistogramProto(600, util_test.GenerateTestHistogram(2)), + histogramWithStartTimestamp(mimirpb.FromHistogramToHistogramProto(600, util_test.GenerateTestHistogram(2)), 500), mimirpb.FromHistogramToHistogramProto(700, util_test.GenerateTestHistogram(3)), }, - CreatedTimestamp: 500, }, }, }, @@ -3898,9 +3899,8 @@ func TestIngester_Push(t *testing.T) { TimeSeries: &mimirpb.TimeSeries{ Labels: []mimirpb.LabelAdapter{metricLabelAdapters[0]}, // Cannot reuse test slice var because it is cleared and returned to the pool Histograms: []mimirpb.Histogram{ - mimirpb.FromHistogramToHistogramProto(800, util_test.GenerateTestHistogram(4)), + histogramWithStartTimestamp(mimirpb.FromHistogramToHistogramProto(800, util_test.GenerateTestHistogram(4)), 500), }, - CreatedTimestamp: 500, }, }, }, @@ -3912,10 +3912,9 @@ func TestIngester_Push(t *testing.T) { Labels: []mimirpb.LabelAdapter{metricLabelAdapters[0]}, // Cannot reuse test slice var because it is cleared and returned to the pool Samples: []mimirpb.Sample{ {Value: 1, TimestampMs: 1400}, - {Value: 2, TimestampMs: 1600}, + {Value: 2, TimestampMs: 1600, StartTimestamp: 1500}, {Value: 3, TimestampMs: 1700}, }, - CreatedTimestamp: 1500, }, }, }, @@ -3926,9 +3925,8 @@ func TestIngester_Push(t *testing.T) { TimeSeries: &mimirpb.TimeSeries{ Labels: []mimirpb.LabelAdapter{metricLabelAdapters[0]}, // Cannot reuse test slice var because it is cleared and returned to the pool Samples: []mimirpb.Sample{ - {Value: 4, TimestampMs: 1800}, + {Value: 4, TimestampMs: 1800, StartTimestamp: 1500}, }, - CreatedTimestamp: 1500, }, }, }, @@ -3999,10 +3997,9 @@ func TestIngester_Push(t *testing.T) { Labels: []mimirpb.LabelAdapter{metricLabelAdapters[0]}, // Cannot reuse test slice var because it is cleared and returned to the pool Histograms: []mimirpb.Histogram{ mimirpb.FromHistogramToHistogramProto(400, util_test.GenerateTestHistogram(1)), - mimirpb.FromHistogramToHistogramProto(600, util_test.GenerateTestHistogram(2)), + histogramWithStartTimestamp(mimirpb.FromHistogramToHistogramProto(600, util_test.GenerateTestHistogram(2)), 500), mimirpb.FromHistogramToHistogramProto(700, util_test.GenerateTestHistogram(3)), }, - CreatedTimestamp: 500, }, }, }, @@ -4013,9 +4010,8 @@ func TestIngester_Push(t *testing.T) { TimeSeries: &mimirpb.TimeSeries{ Labels: []mimirpb.LabelAdapter{metricLabelAdapters[0]}, // Cannot reuse test slice var because it is cleared and returned to the pool Histograms: []mimirpb.Histogram{ - mimirpb.FromHistogramToHistogramProto(800, util_test.GenerateTestHistogram(4)), + histogramWithStartTimestamp(mimirpb.FromHistogramToHistogramProto(800, util_test.GenerateTestHistogram(4)), 500), }, - CreatedTimestamp: 500, }, }, }, @@ -4027,10 +4023,9 @@ func TestIngester_Push(t *testing.T) { Labels: []mimirpb.LabelAdapter{metricLabelAdapters[0]}, // Cannot reuse test slice var because it is cleared and returned to the pool Samples: []mimirpb.Sample{ {Value: 1, TimestampMs: 1400}, - {Value: 2, TimestampMs: 1600}, + {Value: 2, TimestampMs: 1600, StartTimestamp: 1500}, {Value: 3, TimestampMs: 1700}, }, - CreatedTimestamp: 1500, }, }, }, @@ -4041,9 +4036,8 @@ func TestIngester_Push(t *testing.T) { TimeSeries: &mimirpb.TimeSeries{ Labels: []mimirpb.LabelAdapter{metricLabelAdapters[0]}, // Cannot reuse test slice var because it is cleared and returned to the pool Samples: []mimirpb.Sample{ - {Value: 4, TimestampMs: 1800}, + {Value: 4, TimestampMs: 1800, StartTimestamp: 1500}, }, - CreatedTimestamp: 1500, }, }, }, @@ -4103,12 +4097,11 @@ func TestIngester_Push(t *testing.T) { cortex_ingester_tsdb_out_of_order_samples_appended_total{user="test"} 0 `, }, - "should ignore created timestamp if no samples": { + "should succeed on created timestamp being duplicate sample": { allowOOO: true, nativeHistograms: true, reqs: []*mimirpb.WriteRequest{ { - // Initialize the user TSDB. Timeseries: []mimirpb.PreallocTimeseries{ { TimeSeries: &mimirpb.TimeSeries{ @@ -4124,8 +4117,10 @@ func TestIngester_Push(t *testing.T) { Timeseries: []mimirpb.PreallocTimeseries{ { TimeSeries: &mimirpb.TimeSeries{ - Labels: []mimirpb.LabelAdapter{metricLabelAdapters[0]}, // Cannot reuse test slice var because it is cleared and returned to the pool - CreatedTimestamp: 500, + Labels: []mimirpb.LabelAdapter{metricLabelAdapters[0]}, // Cannot reuse test slice var because it is cleared and returned to the pool + Samples: []mimirpb.Sample{ + {Value: 2, TimestampMs: 500, StartTimestamp: 400}, + }, }, }, }, @@ -4137,6 +4132,7 @@ func TestIngester_Push(t *testing.T) { Metric: metricLabelSet, Values: []model.SamplePair{ {Value: 1, Timestamp: model.Time(400)}, + {Value: 2, Timestamp: model.Time(500)}, }, }, }, @@ -4150,7 +4146,7 @@ func TestIngester_Push(t *testing.T) { cortex_ingester_ingested_samples_failures_total{user="test"} 0 # HELP cortex_ingester_ingested_samples_total The total number of samples ingested per user. # TYPE cortex_ingester_ingested_samples_total counter - cortex_ingester_ingested_samples_total{user="test"} 1 + cortex_ingester_ingested_samples_total{user="test"} 2 # HELP cortex_ingester_memory_series The current number of series in memory. # TYPE cortex_ingester_memory_series gauge cortex_ingester_memory_series 1 @@ -4165,7 +4161,7 @@ func TestIngester_Push(t *testing.T) { cortex_ingester_memory_users 1 # HELP cortex_ingester_tsdb_head_max_timestamp_seconds Maximum timestamp of the head block across all tenants. # TYPE cortex_ingester_tsdb_head_max_timestamp_seconds gauge - cortex_ingester_tsdb_head_max_timestamp_seconds 0.4 + cortex_ingester_tsdb_head_max_timestamp_seconds 0.5 # HELP cortex_ingester_tsdb_head_min_timestamp_seconds Minimum timestamp of the head block across all tenants. # TYPE cortex_ingester_tsdb_head_min_timestamp_seconds gauge cortex_ingester_tsdb_head_min_timestamp_seconds 0.4 @@ -4174,7 +4170,7 @@ func TestIngester_Push(t *testing.T) { cortex_ingester_tsdb_out_of_order_samples_appended_total{user="test"} 0 `, }, - "should succeed on created timestamp being duplicate sample": { + "should succeed with histograms on created timestamp causing an error": { allowOOO: true, nativeHistograms: true, reqs: []*mimirpb.WriteRequest{ @@ -4183,8 +4179,8 @@ func TestIngester_Push(t *testing.T) { { TimeSeries: &mimirpb.TimeSeries{ Labels: []mimirpb.LabelAdapter{metricLabelAdapters[0]}, // Cannot reuse test slice var because it is cleared and returned to the pool - Samples: []mimirpb.Sample{ - {Value: 1, TimestampMs: 400}, + Histograms: []mimirpb.Histogram{ + mimirpb.FromHistogramToHistogramProto(400, util_test.GenerateTestHistogram(1)), }, }, }, @@ -4195,10 +4191,9 @@ func TestIngester_Push(t *testing.T) { { TimeSeries: &mimirpb.TimeSeries{ Labels: []mimirpb.LabelAdapter{metricLabelAdapters[0]}, // Cannot reuse test slice var because it is cleared and returned to the pool - Samples: []mimirpb.Sample{ - {Value: 2, TimestampMs: 500}, + Histograms: []mimirpb.Histogram{ + histogramWithStartTimestamp(mimirpb.FromHistogramToHistogramProto(500, util_test.GenerateTestHistogram(2)), 400), }, - CreatedTimestamp: 400, }, }, }, @@ -4208,14 +4203,20 @@ func TestIngester_Push(t *testing.T) { expectedIngested: model.Matrix{ &model.SampleStream{ Metric: metricLabelSet, - Values: []model.SamplePair{ - {Value: 1, Timestamp: model.Time(400)}, - {Value: 2, Timestamp: model.Time(500)}, + Histograms: []model.SampleHistogramPair{ + {Histogram: mimirpb.FromHistogramToPromHistogram(util_test.GenerateTestHistogram(1)), Timestamp: model.Time(400)}, + {Histogram: mimirpb.FromHistogramToPromHistogram(util_test.GenerateTestHistogram(2)), Timestamp: model.Time(500)}, }, }, }, additionalMetrics: []string{"cortex_ingester_tsdb_out_of_order_samples_appended_total"}, expectedMetrics: ` + # HELP cortex_ingester_active_native_histogram_buckets Number of currently active native histogram buckets per user. + # TYPE cortex_ingester_active_native_histogram_buckets gauge + cortex_ingester_active_native_histogram_buckets{user="test"} 8 + # HELP cortex_ingester_active_native_histogram_series Number of currently active native histogram series per user. + # TYPE cortex_ingester_active_native_histogram_series gauge + cortex_ingester_active_native_histogram_series{user="test"} 1 # HELP cortex_ingester_active_series Number of currently active series per user. # TYPE cortex_ingester_active_series gauge cortex_ingester_active_series{user="test"} 1 @@ -4248,7 +4249,7 @@ func TestIngester_Push(t *testing.T) { cortex_ingester_tsdb_out_of_order_samples_appended_total{user="test"} 0 `, }, - "should succeed with histograms on created timestamp causing an error": { + "should ingest a zero sample again when the start timestamp changes within the same batch": { allowOOO: true, nativeHistograms: true, reqs: []*mimirpb.WriteRequest{ @@ -4257,22 +4258,11 @@ func TestIngester_Push(t *testing.T) { { TimeSeries: &mimirpb.TimeSeries{ Labels: []mimirpb.LabelAdapter{metricLabelAdapters[0]}, // Cannot reuse test slice var because it is cleared and returned to the pool - Histograms: []mimirpb.Histogram{ - mimirpb.FromHistogramToHistogramProto(400, util_test.GenerateTestHistogram(1)), - }, - }, - }, - }, - }, - { - Timeseries: []mimirpb.PreallocTimeseries{ - { - TimeSeries: &mimirpb.TimeSeries{ - Labels: []mimirpb.LabelAdapter{metricLabelAdapters[0]}, // Cannot reuse test slice var because it is cleared and returned to the pool - Histograms: []mimirpb.Histogram{ - mimirpb.FromHistogramToHistogramProto(500, util_test.GenerateTestHistogram(2)), + Samples: []mimirpb.Sample{ + {Value: 1, TimestampMs: 1000, StartTimestamp: 500}, + {Value: 2, TimestampMs: 2000, StartTimestamp: 500}, + {Value: 3, TimestampMs: 3000, StartTimestamp: 2500}, }, - CreatedTimestamp: 400, }, }, }, @@ -4282,20 +4272,17 @@ func TestIngester_Push(t *testing.T) { expectedIngested: model.Matrix{ &model.SampleStream{ Metric: metricLabelSet, - Histograms: []model.SampleHistogramPair{ - {Histogram: mimirpb.FromHistogramToPromHistogram(util_test.GenerateTestHistogram(1)), Timestamp: model.Time(400)}, - {Histogram: mimirpb.FromHistogramToPromHistogram(util_test.GenerateTestHistogram(2)), Timestamp: model.Time(500)}, + Values: []model.SamplePair{ + {Value: 0, Timestamp: model.Time(500)}, + {Value: 1, Timestamp: model.Time(1000)}, + {Value: 2, Timestamp: model.Time(2000)}, + {Value: 0, Timestamp: model.Time(2500)}, + {Value: 3, Timestamp: model.Time(3000)}, }, }, }, additionalMetrics: []string{"cortex_ingester_tsdb_out_of_order_samples_appended_total"}, expectedMetrics: ` - # HELP cortex_ingester_active_native_histogram_buckets Number of currently active native histogram buckets per user. - # TYPE cortex_ingester_active_native_histogram_buckets gauge - cortex_ingester_active_native_histogram_buckets{user="test"} 8 - # HELP cortex_ingester_active_native_histogram_series Number of currently active native histogram series per user. - # TYPE cortex_ingester_active_native_histogram_series gauge - cortex_ingester_active_native_histogram_series{user="test"} 1 # HELP cortex_ingester_active_series Number of currently active series per user. # TYPE cortex_ingester_active_series gauge cortex_ingester_active_series{user="test"} 1 @@ -4304,7 +4291,7 @@ func TestIngester_Push(t *testing.T) { cortex_ingester_ingested_samples_failures_total{user="test"} 0 # HELP cortex_ingester_ingested_samples_total The total number of samples ingested per user. # TYPE cortex_ingester_ingested_samples_total counter - cortex_ingester_ingested_samples_total{user="test"} 2 + cortex_ingester_ingested_samples_total{user="test"} 5 # HELP cortex_ingester_memory_series The current number of series in memory. # TYPE cortex_ingester_memory_series gauge cortex_ingester_memory_series 1 @@ -4319,10 +4306,10 @@ func TestIngester_Push(t *testing.T) { cortex_ingester_memory_users 1 # HELP cortex_ingester_tsdb_head_max_timestamp_seconds Maximum timestamp of the head block across all tenants. # TYPE cortex_ingester_tsdb_head_max_timestamp_seconds gauge - cortex_ingester_tsdb_head_max_timestamp_seconds 0.5 + cortex_ingester_tsdb_head_max_timestamp_seconds 3 # HELP cortex_ingester_tsdb_head_min_timestamp_seconds Minimum timestamp of the head block across all tenants. # TYPE cortex_ingester_tsdb_head_min_timestamp_seconds gauge - cortex_ingester_tsdb_head_min_timestamp_seconds 0.4 + cortex_ingester_tsdb_head_min_timestamp_seconds 0.5 # HELP cortex_ingester_tsdb_out_of_order_samples_appended_total Total number of out-of-order samples appended. # TYPE cortex_ingester_tsdb_out_of_order_samples_appended_total counter cortex_ingester_tsdb_out_of_order_samples_appended_total{user="test"} 0 diff --git a/pkg/mimirpb/compat_rw2.go b/pkg/mimirpb/compat_rw2.go index 4055b6a86f4..644367e666f 100644 --- a/pkg/mimirpb/compat_rw2.go +++ b/pkg/mimirpb/compat_rw2.go @@ -106,3 +106,18 @@ func (m *WriteRequest) ProtocolVersion() string { } return RemoteWriteVersion2 } + +// legacyCreatedTimestampForMarshal returns the value to marshal into the reserved +// TimeSeries/TimeSeriesRW2 field 6 (formerly created_timestamp), for backward compatibility with +// not-yet-upgraded readers (e.g. during a rolling upgrade) that don't understand the newer +// per-sample/per-histogram StartTimestamp fields. It takes the StartTimestamp of the first +// sample, or if there are no samples, of the first histogram. +func legacyCreatedTimestampForMarshal(samples []Sample, histograms []Histogram) int64 { + if len(samples) > 0 { + return samples[0].StartTimestamp + } + if len(histograms) > 0 { + return histograms[0].StartTimestamp + } + return 0 +} diff --git a/pkg/mimirpb/compat_rw2_test.go b/pkg/mimirpb/compat_rw2_test.go index 763f1a90675..d86519e7372 100644 --- a/pkg/mimirpb/compat_rw2_test.go +++ b/pkg/mimirpb/compat_rw2_test.go @@ -44,16 +44,8 @@ func TestRW2TypesCompatible(t *testing.T) { rootNode.Nodes[1].Nodes[1].Nodes[0].Value = secondValue rootNode.Nodes[1].Nodes[1].Nodes[1].Value = strings.ReplaceAll(firstValue, "TimestampMs", "Timestamp") - // We are freezing our API at RW2.0-rc3. That means we do not yet support StartTimestamp on samples nor histograms, and we retain CreatedTimestamp on TimeSeries. - rootNode, _ = expectedTree.(*treeprint.Node) - rootNode.Nodes[1].AddNode("+0 CreatedTimestamp: int64 protobuf:varint,6") - // TimeSeries is node 1 of the request, Sample is node 1 of TimeSeries, StartTimestamp is node 2 of Sample. - require.Contains(t, rootNode.Nodes[1].Nodes[1].Nodes[2].String(), " StartTimestamp:") - rootNode.Nodes[1].Nodes[1].Nodes = rootNode.Nodes[1].Nodes[1].Nodes[0:2] - require.Contains(t, rootNode.Nodes[1].Nodes[2].Nodes[14].String(), " StartTimestamp:") - // TimeSeries is node 1 of the request, Histogram is node 2 of TimeSeries, StartTimestamp is node 14 of Histogram. - rootNode.Nodes[1].Nodes[2].Nodes = rootNode.Nodes[1].Nodes[2].Nodes[0:14] - + // Our Sample and Histogram messages now carry StartTimestamp (matching upstream's RW2 spec), and + // TimeSeries/TimeSeriesRW2 no longer carry CreatedTimestamp (reserved), so the shapes should match directly. require.Equal(t, expectedTree.String(), actualTree.String(), "Proto types are not compatible") } @@ -128,6 +120,186 @@ func TestRW2Unmarshal(t *testing.T) { require.Equal(t, expected, &received) }) + t.Run("Sample and Histogram StartTimestamp round-trip via RW2", func(t *testing.T) { + syms := rw2util.NewSymbolTableBuilder(nil) + writeRequest := &WriteRequest{ + TimeseriesRW2: []TimeSeriesRW2{ + { + LabelsRefs: []uint32{syms.GetSymbol("__name__"), syms.GetSymbol("test_metric_total")}, + Samples: []Sample{ + {Value: 123.456, TimestampMs: 1000, StartTimestamp: 500}, + }, + Histograms: []Histogram{ + FromHistogramToHistogramProto(2000, test.GenerateTestHistogram(1)), + }, + }, + }, + } + writeRequest.TimeseriesRW2[0].Histograms[0].StartTimestamp = 1500 + writeRequest.SymbolsRW2 = syms.GetSymbols() + data, err := writeRequest.Marshal() + require.NoError(t, err) + + received := PreallocWriteRequest{} + received.UnmarshalFromRW2 = true + err = received.Unmarshal(data) + require.NoError(t, err) + + require.Len(t, received.Timeseries, 1) + require.Len(t, received.Timeseries[0].Samples, 1) + require.Equal(t, int64(500), received.Timeseries[0].Samples[0].StartTimestamp) + require.Len(t, received.Timeseries[0].Histograms, 1) + require.Equal(t, int64(1500), received.Timeseries[0].Histograms[0].StartTimestamp) + }) + + t.Run("Sample and Histogram StartTimestamp round-trip via RW1", func(t *testing.T) { + writeRequest := &WriteRequest{ + Timeseries: []PreallocTimeseries{ + { + TimeSeries: &TimeSeries{ + Labels: []LabelAdapter{{Name: "__name__", Value: "test_metric_total"}}, + Samples: []Sample{ + {Value: 123.456, TimestampMs: 1000, StartTimestamp: 500}, + }, + Histograms: []Histogram{ + FromHistogramToHistogramProto(2000, test.GenerateTestHistogram(1)), + }, + }, + }, + }, + } + writeRequest.Timeseries[0].Histograms[0].StartTimestamp = 1500 + data, err := writeRequest.Marshal() + require.NoError(t, err) + + received := PreallocWriteRequest{} + err = received.Unmarshal(data) + require.NoError(t, err) + + require.Len(t, received.Timeseries, 1) + require.Len(t, received.Timeseries[0].Samples, 1) + require.Equal(t, int64(500), received.Timeseries[0].Samples[0].StartTimestamp) + require.Len(t, received.Timeseries[0].Histograms, 1) + require.Equal(t, int64(1500), received.Timeseries[0].Histograms[0].StartTimestamp) + }) + + t.Run("legacy per-series created_timestamp (pre-final RW2 wire shape) fans out to Samples and Histograms", func(t *testing.T) { + // Simulate a legacy sender that still sets the reserved TimeSeriesRW2 field 6 (formerly + // created_timestamp) instead of a per-sample/per-histogram StartTimestamp. The current + // TimeSeriesRW2 Go type can no longer express that field (it's reserved), so the wire bytes + // are built by hand. + syms := rw2util.NewSymbolTableBuilder(nil) + ts := TimeSeriesRW2{ + LabelsRefs: []uint32{syms.GetSymbol("__name__"), syms.GetSymbol("test_metric_total")}, + Samples: []Sample{ + {Value: 1, TimestampMs: 1000}, + {Value: 2, TimestampMs: 2000}, + }, + Histograms: []Histogram{ + FromHistogramToHistogramProto(3000, test.GenerateTestHistogram(1)), + }, + } + tsData, err := ts.Marshal() + require.NoError(t, err) + tsData = appendVarintField(tsData, 6, 500) + + var data []byte + for _, s := range syms.GetSymbols() { + data = appendBytesField(data, 4, []byte(s)) + } + data = appendBytesField(data, 5, tsData) + + received := PreallocWriteRequest{} + received.UnmarshalFromRW2 = true + err = received.Unmarshal(data) + require.NoError(t, err) + + require.Len(t, received.Timeseries, 1) + require.Len(t, received.Timeseries[0].Samples, 2) + require.Equal(t, int64(500), received.Timeseries[0].Samples[0].StartTimestamp) + require.Equal(t, int64(500), received.Timeseries[0].Samples[1].StartTimestamp) + require.Len(t, received.Timeseries[0].Histograms, 1) + require.Equal(t, int64(500), received.Timeseries[0].Histograms[0].StartTimestamp) + }) + + t.Run("legacy per-series created_timestamp does not override an explicit per-sample StartTimestamp", func(t *testing.T) { + syms := rw2util.NewSymbolTableBuilder(nil) + ts := TimeSeriesRW2{ + LabelsRefs: []uint32{syms.GetSymbol("__name__"), syms.GetSymbol("test_metric_total")}, + Samples: []Sample{ + {Value: 1, TimestampMs: 1000, StartTimestamp: 900}, + }, + } + tsData, err := ts.Marshal() + require.NoError(t, err) + tsData = appendVarintField(tsData, 6, 500) + + var data []byte + for _, s := range syms.GetSymbols() { + data = appendBytesField(data, 4, []byte(s)) + } + data = appendBytesField(data, 5, tsData) + + received := PreallocWriteRequest{} + received.UnmarshalFromRW2 = true + err = received.Unmarshal(data) + require.NoError(t, err) + + require.Len(t, received.Timeseries, 1) + require.Len(t, received.Timeseries[0].Samples, 1) + require.Equal(t, int64(900), received.Timeseries[0].Samples[0].StartTimestamp) + }) + + t.Run("legacy per-series created_timestamp fans out via the plain TimeSeries.Unmarshal too", func(t *testing.T) { + // The plain (non-RW2) TimeSeries.Unmarshal is used not just for Remote Write 1.0, but also + // for internal distributor->ingester gRPC and ingest-storage Kafka records. A not-yet- + // upgraded Mimir component in those internal paths would still encode the reserved field 6 + // (formerly created_timestamp) instead of a per-sample/per-histogram StartTimestamp, so + // this must fan out here too, not just in UnmarshalRW2. + ts := TimeSeries{ + Labels: []LabelAdapter{{Name: "__name__", Value: "test_metric_total"}}, + Samples: []Sample{ + {Value: 1, TimestampMs: 1000}, + {Value: 2, TimestampMs: 2000}, + }, + Histograms: []Histogram{ + FromHistogramToHistogramProto(3000, test.GenerateTestHistogram(1)), + }, + } + data, err := ts.Marshal() + require.NoError(t, err) + data = appendVarintField(data, 6, 500) + + received := TimeSeries{} + err = received.Unmarshal(data) + require.NoError(t, err) + + require.Len(t, received.Samples, 2) + require.Equal(t, int64(500), received.Samples[0].StartTimestamp) + require.Equal(t, int64(500), received.Samples[1].StartTimestamp) + require.Len(t, received.Histograms, 1) + require.Equal(t, int64(500), received.Histograms[0].StartTimestamp) + }) + + t.Run("legacy per-series created_timestamp via the plain TimeSeries.Unmarshal does not override an explicit per-sample StartTimestamp", func(t *testing.T) { + ts := TimeSeries{ + Labels: []LabelAdapter{{Name: "__name__", Value: "test_metric_total"}}, + Samples: []Sample{ + {Value: 1, TimestampMs: 1000, StartTimestamp: 900}, + }, + } + data, err := ts.Marshal() + require.NoError(t, err) + data = appendVarintField(data, 6, 500) + + received := TimeSeries{} + err = received.Unmarshal(data) + require.NoError(t, err) + + require.Len(t, received.Samples, 1) + require.Equal(t, int64(900), received.Samples[0].StartTimestamp) + }) + t.Run("zero timeseries does not panic", func(t *testing.T) { syms := rw2util.NewSymbolTableBuilder(nil) syms.GetSymbol("unused_symbol") @@ -872,6 +1044,108 @@ func TestRW2Unmarshal(t *testing.T) { }) } +// TestMarshalLegacyCreatedTimestamp verifies that TimeSeries/TimeSeriesRW2 marshalling still +// writes the reserved per-series created_timestamp field (6), for the benefit of not-yet-upgraded +// readers (e.g. during a rolling upgrade) that only understand that field, not the newer +// per-sample/per-histogram StartTimestamp fields. +func TestMarshalLegacyCreatedTimestamp(t *testing.T) { + t.Run("TimeSeries.Marshal writes the first sample's StartTimestamp into the legacy field", func(t *testing.T) { + ts := TimeSeries{ + Labels: []LabelAdapter{{Name: "__name__", Value: "test_metric_total"}}, + Samples: []Sample{ + {Value: 1, TimestampMs: 1000, StartTimestamp: 500}, + {Value: 2, TimestampMs: 2000, StartTimestamp: 500}, + }, + } + data, err := ts.Marshal() + require.NoError(t, err) + + got, found := readVarintField(data, 6) + require.True(t, found) + require.Equal(t, int64(500), got) + }) + + t.Run("TimeSeries.Marshal falls back to the first histogram's StartTimestamp when there are no samples", func(t *testing.T) { + ts := TimeSeries{ + Labels: []LabelAdapter{{Name: "__name__", Value: "test_metric_total"}}, + Histograms: []Histogram{ + FromHistogramToHistogramProto(3000, test.GenerateTestHistogram(1)), + }, + } + ts.Histograms[0].StartTimestamp = 700 + data, err := ts.Marshal() + require.NoError(t, err) + + got, found := readVarintField(data, 6) + require.True(t, found) + require.Equal(t, int64(700), got) + }) + + t.Run("TimeSeries.Marshal omits the legacy field when there is no start timestamp to report", func(t *testing.T) { + ts := TimeSeries{ + Labels: []LabelAdapter{{Name: "__name__", Value: "test_metric_total"}}, + Samples: []Sample{{Value: 1, TimestampMs: 1000}}, + } + data, err := ts.Marshal() + require.NoError(t, err) + + _, found := readVarintField(data, 6) + require.False(t, found) + }) + + t.Run("TimeSeriesRW2.Marshal writes the first sample's StartTimestamp into the legacy field", func(t *testing.T) { + syms := rw2util.NewSymbolTableBuilder(nil) + ts := TimeSeriesRW2{ + LabelsRefs: []uint32{syms.GetSymbol("__name__"), syms.GetSymbol("test_metric_total")}, + Samples: []Sample{ + {Value: 1, TimestampMs: 1000, StartTimestamp: 500}, + }, + } + data, err := ts.Marshal() + require.NoError(t, err) + + got, found := readVarintField(data, 6) + require.True(t, found) + require.Equal(t, int64(500), got) + }) + + t.Run("TimeSeriesRW2.Marshal falls back to the first histogram's StartTimestamp when there are no samples", func(t *testing.T) { + syms := rw2util.NewSymbolTableBuilder(nil) + ts := TimeSeriesRW2{ + LabelsRefs: []uint32{syms.GetSymbol("__name__"), syms.GetSymbol("test_metric_total")}, + Histograms: []Histogram{ + FromHistogramToHistogramProto(3000, test.GenerateTestHistogram(1)), + }, + } + ts.Histograms[0].StartTimestamp = 700 + data, err := ts.Marshal() + require.NoError(t, err) + + got, found := readVarintField(data, 6) + require.True(t, found) + require.Equal(t, int64(700), got) + }) + + t.Run("a round trip through Marshal and Unmarshal by an upgraded reader is unaffected", func(t *testing.T) { + // The legacy field is redundant once decoded by an upgraded reader: every sample/histogram + // already carries its own StartTimestamp from the modern fields, so the fan-out in + // Unmarshal (which only fills in a zero StartTimestamp) is a no-op here. + ts := TimeSeries{ + Labels: []LabelAdapter{{Name: "__name__", Value: "test_metric_total"}}, + Samples: []Sample{ + {Value: 1, TimestampMs: 1000, StartTimestamp: 500}, + {Value: 2, TimestampMs: 2000, StartTimestamp: 900}, + }, + } + data, err := ts.Marshal() + require.NoError(t, err) + + received := TimeSeries{} + require.NoError(t, received.Unmarshal(data)) + require.Equal(t, ts, received) + }) +} + func makeTestRW2WriteRequest(syms *rw2util.SymbolTableBuilder) *WriteRequest { req := &WriteRequest{ TimeseriesRW2: []TimeSeriesRW2{ @@ -902,3 +1176,85 @@ func makeTestRW2WriteRequest(syms *rw2util.SymbolTableBuilder) *WriteRequest { return req } + +// appendVarint appends v to buf using standard protobuf varint encoding. +func appendVarint(buf []byte, v uint64) []byte { + for v >= 0x80 { + buf = append(buf, byte(v)|0x80) + v >>= 7 + } + return append(buf, byte(v)) +} + +// appendVarintField appends a varint-wiretype field (tag + value) to buf, to hand-craft wire +// messages containing fields that are no longer expressible through the generated Go types +// (e.g. reserved fields). +func appendVarintField(buf []byte, fieldNum int, value int64) []byte { + buf = appendVarint(buf, uint64(fieldNum)<<3) // wire type 0: varint + return appendVarint(buf, uint64(value)) +} + +// appendBytesField appends a length-delimited-wiretype field (tag + length + data) to buf. +func appendBytesField(buf []byte, fieldNum int, data []byte) []byte { + buf = appendVarint(buf, uint64(fieldNum)<<3|2) + buf = appendVarint(buf, uint64(len(data))) + return append(buf, data...) +} + +// readVarint reads a standard protobuf varint from the start of buf, returning the value and the +// number of bytes consumed (0 if buf doesn't start with a valid varint). +func readVarint(buf []byte) (uint64, int) { + var v uint64 + for i, b := range buf { + v |= uint64(b&0x7F) << (7 * i) + if b < 0x80 { + return v, i + 1 + } + } + return 0, 0 +} + +// readVarintField scans data for the first occurrence of fieldNum with a varint wire type and +// returns its value, without going through the (symbol-aware) generated Unmarshal path. Used to +// verify legacy-compat fields are actually present on the wire, independent of whether the +// current Go types can still decode them into a named field. +func readVarintField(data []byte, fieldNum int) (int64, bool) { + i := 0 + for i < len(data) { + tag, n := readVarint(data[i:]) + if n == 0 { + return 0, false + } + i += n + fn := int(tag >> 3) + wt := int(tag & 0x7) + if fn == fieldNum && wt == 0 { + v, n := readVarint(data[i:]) + if n == 0 { + return 0, false + } + return int64(v), true + } + switch wt { + case 0: // varint + _, n := readVarint(data[i:]) + if n == 0 { + return 0, false + } + i += n + case 1: // fixed64 + i += 8 + case 2: // length-delimited + l, n := readVarint(data[i:]) + if n == 0 { + return 0, false + } + i += n + int(l) + case 5: // fixed32 + i += 4 + default: + return 0, false + } + } + return 0, false +} diff --git a/pkg/mimirpb/mimir.pb.go b/pkg/mimirpb/mimir.pb.go index 464997d2cd9..e5cdebc4d05 100644 --- a/pkg/mimirpb/mimir.pb.go +++ b/pkg/mimirpb/mimir.pb.go @@ -498,12 +498,6 @@ type TimeSeries struct { Samples []Sample `protobuf:"bytes,2,rep,name=samples,proto3" json:"samples"` Exemplars []Exemplar `protobuf:"bytes,3,rep,name=exemplars,proto3" json:"exemplars"` Histograms []Histogram `protobuf:"bytes,4,rep,name=histograms,proto3" json:"histograms"` - // Copy from remote write 2.0. - // Note that the "optional" keyword is omitted due to - // https://cloud.google.com/apis/design/design_patterns.md#optional_primitive_fields - // Zero value means value not set. If you need to use exactly zero value for - // the timestamp, use 1 millisecond before or after. - CreatedTimestamp int64 `protobuf:"varint,6,opt,name=created_timestamp,json=createdTimestamp,proto3" json:"created_timestamp,omitempty"` // Skip unmarshaling of exemplars. SkipUnmarshalingExemplars bool @@ -562,13 +556,6 @@ func (m *TimeSeries) GetHistograms() []Histogram { return nil } -func (m *TimeSeries) GetCreatedTimestamp() int64 { - if m != nil { - return m.CreatedTimestamp - } - return 0 -} - type LabelPair struct { Name []byte `protobuf:"bytes,1,opt,name=name,proto3" json:"name,omitempty"` Value []byte `protobuf:"bytes,2,opt,name=value,proto3" json:"value,omitempty"` @@ -625,6 +612,12 @@ func (m *LabelPair) GetValue() []byte { type Sample struct { TimestampMs int64 `protobuf:"varint,2,opt,name=timestamp_ms,json=timestampMs,proto3" json:"timestamp_ms,omitempty"` Value float64 `protobuf:"fixed64,1,opt,name=value,proto3" json:"value,omitempty"` + // Copy from remote write 2.0. + // Note that the "optional" keyword is omitted due to + // https://cloud.google.com/apis/design/design_patterns.md#optional_primitive_fields + // Zero value means value not set. If you need to use exactly zero value for + // the timestamp, use 1 millisecond before or after. + StartTimestamp int64 `protobuf:"varint,3,opt,name=start_timestamp,json=startTimestamp,proto3" json:"start_timestamp,omitempty"` } func (m *Sample) Reset() { *m = Sample{} } @@ -673,6 +666,13 @@ func (m *Sample) GetValue() float64 { return 0 } +func (m *Sample) GetStartTimestamp() int64 { + if m != nil { + return m.StartTimestamp + } + return 0 +} + type MetricMetadata struct { Type MetricMetadata_MetricType `protobuf:"varint,1,opt,name=type,proto3,enum=cortexpb.MetricMetadata_MetricType" json:"type,omitempty"` MetricFamilyName string `protobuf:"bytes,2,opt,name=metric_family_name,json=metricFamilyName,proto3" json:"metric_family_name,omitempty"` @@ -872,6 +872,12 @@ type Histogram struct { // Used only for converting from OpenTelemetry to Prometheus internally and // to unmarshal Remote Write 2.0 messages. CustomValues []float64 `protobuf:"fixed64,16,rep,packed,name=custom_values,json=customValues,proto3" json:"custom_values,omitempty"` + // Copy from remote write 2.0. + // Note that the "optional" keyword is omitted due to + // https://cloud.google.com/apis/design/design_patterns.md#optional_primitive_fields + // Zero value means value not set. If you need to use exactly zero value for + // the timestamp, use 1 millisecond before or after. + StartTimestamp int64 `protobuf:"varint,17,opt,name=start_timestamp,json=startTimestamp,proto3" json:"start_timestamp,omitempty"` } func (m *Histogram) Reset() { *m = Histogram{} } @@ -1062,6 +1068,13 @@ func (m *Histogram) GetCustomValues() []float64 { return nil } +func (m *Histogram) GetStartTimestamp() int64 { + if m != nil { + return m.StartTimestamp + } + return 0 +} + // XXX_OneofWrappers is for the internal use of the proto package. func (*Histogram) XXX_OneofWrappers() []interface{} { return []interface{}{ @@ -2145,23 +2158,6 @@ type TimeSeriesRW2 struct { Exemplars []ExemplarRW2 `protobuf:"bytes,4,rep,name=exemplars,proto3" json:"exemplars"` // metadata represents the metadata associated with the given series' samples. Metadata MetadataRW2 `protobuf:"bytes,5,opt,name=metadata,proto3" json:"metadata"` - // created_timestamp represents an optional created timestamp associated with - // this series' samples in ms format, typically for counter or histogram type - // metrics. Created timestamp represents the time when the counter started - // counting (sometimes referred to as start timestamp), which can increase - // the accuracy of query results. - // - // Note that some receivers might require this and in return fail to - // ingest such samples within the Request. - // - // For Go, see github.com/prometheus/prometheus/model/timestamp/timestamp.go - // for conversion from/to time.Time to Prometheus timestamp. - // - // Note that the "optional" keyword is omitted due to - // https://cloud.google.com/apis/design/design_patterns.md#optional_primitive_fields - // Zero value means value not set. If you need to use exactly zero value for - // the timestamp, use 1 millisecond before or after. - CreatedTimestamp int64 `protobuf:"varint,6,opt,name=created_timestamp,json=createdTimestamp,proto3" json:"created_timestamp,omitempty"` } func (m *TimeSeriesRW2) Reset() { *m = TimeSeriesRW2{} } @@ -2231,13 +2227,6 @@ func (m *TimeSeriesRW2) GetMetadata() MetadataRW2 { return MetadataRW2{} } -func (m *TimeSeriesRW2) GetCreatedTimestamp() int64 { - if m != nil { - return m.CreatedTimestamp - } - return 0 -} - type ExemplarRW2 struct { // labels_refs is an optional list of label name-value pair references, encoded // as indices to the Request.symbols array. This list's len is always @@ -2414,161 +2403,162 @@ func init() { func init() { proto.RegisterFile("mimir.proto", fileDescriptor_86d4d7485f544059) } var fileDescriptor_86d4d7485f544059 = []byte{ - // 2457 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xac, 0x59, 0xcd, 0x73, 0xdb, 0xd6, - 0x11, 0x27, 0x48, 0x8a, 0x1f, 0x4b, 0x52, 0x82, 0x9f, 0x2d, 0x87, 0x76, 0x6c, 0x4a, 0x46, 0x9a, - 0x54, 0x71, 0x5b, 0xa5, 0xe3, 0x7c, 0x78, 0x9c, 0x49, 0x9a, 0x01, 0x41, 0x48, 0xa2, 0xc3, 0x0f, - 0xf9, 0x01, 0x94, 0xaa, 0x5e, 0x30, 0x10, 0xf5, 0x24, 0x21, 0x21, 0x09, 0x16, 0x00, 0x1d, 0xab, - 0xa7, 0x5e, 0x3a, 0xed, 0x74, 0x7a, 0xe8, 0xb9, 0xb7, 0x4e, 0x0f, 0xed, 0x5f, 0xd1, 0x73, 0x8e, - 0x3e, 0xa6, 0x9d, 0xd6, 0x53, 0xcb, 0x17, 0xf7, 0xe6, 0xe9, 0xb1, 0xbd, 0x74, 0xde, 0x7b, 0xf8, - 0x26, 0xdd, 0xa8, 0x89, 0x6f, 0x78, 0xbb, 0xbf, 0xdd, 0xb7, 0xbb, 0x6f, 0x77, 0xf9, 0xde, 0x12, - 0x2a, 0x63, 0x6b, 0x6c, 0x39, 0x9b, 0x53, 0xc7, 0xf6, 0x6c, 0x54, 0x1a, 0xda, 0x8e, 0x47, 0x1e, - 0x4d, 0x0f, 0xaf, 0x5f, 0x39, 0xb1, 0x4f, 0x6c, 0x46, 0x7c, 0x87, 0x7e, 0x71, 0xbe, 0xf4, 0xb7, - 0x1c, 0x54, 0xf7, 0x1d, 0xcb, 0x23, 0x98, 0xfc, 0x74, 0x46, 0x5c, 0x0f, 0xed, 0x02, 0x78, 0xd6, - 0x98, 0xb8, 0xc4, 0xb1, 0x88, 0x5b, 0x17, 0xd6, 0x73, 0x1b, 0x95, 0x3b, 0x57, 0x36, 0x03, 0x2d, - 0x9b, 0xba, 0x35, 0x26, 0x1a, 0xe3, 0x35, 0xaf, 0x7f, 0xf9, 0x64, 0x2d, 0xf3, 0xd7, 0x27, 0x6b, - 0x68, 0xd7, 0x21, 0xe6, 0x68, 0x64, 0x0f, 0xf5, 0x50, 0x0e, 0xc7, 0x74, 0xa0, 0x7b, 0x50, 0xd0, - 0xec, 0x99, 0x33, 0x24, 0xf5, 0xec, 0xba, 0xb0, 0xb1, 0x7c, 0xe7, 0x56, 0xa4, 0x2d, 0xbe, 0xf3, - 0x26, 0x07, 0xa9, 0x93, 0xd9, 0x18, 0xfb, 0x02, 0xe8, 0x43, 0x28, 0x8d, 0x89, 0x67, 0x1e, 0x99, - 0x9e, 0x59, 0xcf, 0x31, 0x53, 0xea, 0x91, 0x70, 0x97, 0x78, 0x8e, 0x35, 0xec, 0xfa, 0xfc, 0x66, - 0xfe, 0xcb, 0x27, 0x6b, 0x02, 0x0e, 0xf1, 0xa8, 0x01, 0xe0, 0x9e, 0x8d, 0x0f, 0xed, 0x91, 0x8b, - 0xf7, 0xef, 0xd4, 0xf3, 0xeb, 0xb9, 0x8d, 0x32, 0x8e, 0x51, 0x90, 0x02, 0xb5, 0xc8, 0x48, 0x0a, - 0x59, 0x62, 0x1b, 0xbc, 0xb6, 0xc8, 0x57, 0xbc, 0x7f, 0x87, 0xe9, 0xcf, 0xe0, 0xa4, 0x0c, 0x7a, - 0x17, 0x56, 0xdd, 0xcf, 0xad, 0xa9, 0x31, 0x32, 0x0f, 0xc9, 0xc8, 0x78, 0x68, 0x8e, 0xac, 0x23, - 0xd3, 0xb3, 0xec, 0x49, 0xfd, 0x79, 0x71, 0x5d, 0xd8, 0x28, 0xe1, 0xcb, 0x94, 0xdb, 0xa1, 0xcc, - 0xbd, 0x90, 0x87, 0x7e, 0x04, 0xaf, 0xc7, 0x84, 0x86, 0xf6, 0x6c, 0xe2, 0xc5, 0x45, 0xff, 0xc9, - 0x45, 0xeb, 0xa1, 0xa8, 0x42, 0x11, 0x91, 0xbc, 0xf4, 0x36, 0x40, 0x14, 0x2b, 0x54, 0x84, 0x9c, - 0xbc, 0xdb, 0x16, 0x33, 0xa8, 0x04, 0x79, 0x3c, 0xe8, 0xa8, 0xa2, 0x40, 0xbf, 0xfa, 0x7a, 0x67, - 0x57, 0xcc, 0x4a, 0x2b, 0x50, 0xf3, 0x63, 0xec, 0x4e, 0xed, 0x89, 0x4b, 0xa4, 0x47, 0x50, 0x55, - 0x1d, 0xc7, 0x76, 0x5a, 0xc4, 0x33, 0xad, 0x91, 0x8b, 0x6e, 0xc3, 0x92, 0x62, 0xce, 0x5c, 0x52, - 0x17, 0xd8, 0xd9, 0xc4, 0x4e, 0x9a, 0xc1, 0x18, 0x0f, 0x73, 0x08, 0x42, 0x90, 0xd7, 0xec, 0x63, - 0x8f, 0x1d, 0x63, 0x09, 0xb3, 0x6f, 0xb4, 0x01, 0x2b, 0x98, 0x7c, 0x46, 0x86, 0x1e, 0x39, 0xd2, - 0xcc, 0xf1, 0x74, 0x44, 0xdc, 0x7a, 0x6e, 0x5d, 0xd8, 0xc8, 0xe1, 0x34, 0x59, 0xfa, 0x63, 0x16, - 0x20, 0x8a, 0x28, 0xda, 0x86, 0x02, 0xf3, 0x3f, 0xc8, 0xb1, 0xcb, 0xd1, 0xce, 0xcc, 0xe9, 0x5d, - 0xd3, 0x72, 0xa2, 0x14, 0x1b, 0x4c, 0x5c, 0xf3, 0x98, 0x74, 0x67, 0x9e, 0x79, 0x38, 0x22, 0x8c, - 0x8f, 0x7d, 0x71, 0xf4, 0x43, 0x28, 0xba, 0xfe, 0xce, 0x59, 0xa6, 0x49, 0x8c, 0x34, 0xf1, 0xbd, - 0xfd, 0xa3, 0x0b, 0x60, 0xe8, 0x03, 0x28, 0x93, 0x47, 0x64, 0x3c, 0x1d, 0x99, 0x8e, 0xeb, 0xa7, - 0x15, 0x8a, 0xf9, 0xed, 0xb3, 0x7c, 0xa9, 0x08, 0x8a, 0xee, 0x01, 0x9c, 0x5a, 0xae, 0x67, 0x9f, - 0x38, 0xe6, 0xd8, 0x65, 0x19, 0x95, 0x30, 0x7b, 0x27, 0xe0, 0xf9, 0x92, 0x31, 0x30, 0xfa, 0x1e, - 0x5c, 0x1a, 0x3a, 0xc4, 0xf4, 0xc8, 0x91, 0xc1, 0x12, 0xc8, 0x33, 0xc7, 0xd3, 0x7a, 0x81, 0x05, - 0x4a, 0xf4, 0x19, 0x7a, 0x40, 0x97, 0xde, 0x87, 0x72, 0x18, 0x02, 0x1a, 0xf4, 0x89, 0x39, 0xe6, - 0xe7, 0x53, 0xc5, 0xec, 0x1b, 0x5d, 0x81, 0xa5, 0x87, 0xe6, 0x68, 0xc6, 0x0b, 0xaa, 0x8a, 0xf9, - 0x42, 0x92, 0xa1, 0xc0, 0xfd, 0x45, 0xb7, 0xa0, 0x1a, 0xee, 0x62, 0x8c, 0x5d, 0x06, 0xcb, 0xe1, - 0x4a, 0x48, 0xeb, 0xba, 0x91, 0x0a, 0xaa, 0x57, 0x08, 0x54, 0xfc, 0x2e, 0x0b, 0xcb, 0xc9, 0xb2, - 0x42, 0x77, 0x21, 0xef, 0x9d, 0x4d, 0x83, 0xfc, 0x78, 0xe3, 0x65, 0xe5, 0xe7, 0x2f, 0xf5, 0xb3, - 0x29, 0xc1, 0x4c, 0x00, 0x7d, 0x1f, 0xd0, 0x98, 0xd1, 0x8c, 0x63, 0x73, 0x6c, 0x8d, 0xce, 0x0c, - 0xe6, 0x06, 0x35, 0xa5, 0x8c, 0x45, 0xce, 0xd9, 0x62, 0x8c, 0x1e, 0x75, 0x09, 0x41, 0xfe, 0x94, - 0x8c, 0xa6, 0xf5, 0x3c, 0xe3, 0xb3, 0x6f, 0x4a, 0x9b, 0x4d, 0x2c, 0xaf, 0xbe, 0xc4, 0x69, 0xf4, - 0x5b, 0x3a, 0x03, 0x88, 0x76, 0x42, 0x15, 0x28, 0x0e, 0x7a, 0x9f, 0xf6, 0xfa, 0xfb, 0x3d, 0x31, - 0x43, 0x17, 0x4a, 0x7f, 0xd0, 0xd3, 0x55, 0x2c, 0x0a, 0xa8, 0x0c, 0x4b, 0xdb, 0xf2, 0x60, 0x5b, - 0x15, 0xb3, 0xa8, 0x06, 0xe5, 0x9d, 0xb6, 0xa6, 0xf7, 0xb7, 0xb1, 0xdc, 0x15, 0x73, 0x08, 0xc1, - 0x32, 0xe3, 0x44, 0xb4, 0x3c, 0x15, 0xd5, 0x06, 0xdd, 0xae, 0x8c, 0x0f, 0xc4, 0x25, 0x5a, 0x3d, - 0xed, 0xde, 0x56, 0x5f, 0x2c, 0xa0, 0x2a, 0x94, 0x34, 0x5d, 0xd6, 0x55, 0x4d, 0xd5, 0xc5, 0xa2, - 0xf4, 0x00, 0x0a, 0x7c, 0xeb, 0x57, 0x96, 0xbb, 0xd2, 0x6f, 0x04, 0x28, 0x05, 0xf9, 0xf6, 0xea, - 0x2a, 0x22, 0x91, 0x1e, 0xc1, 0xd9, 0xce, 0x25, 0x45, 0x6e, 0x2e, 0x29, 0xa4, 0x17, 0x4b, 0x50, - 0x0e, 0xb3, 0x18, 0xdd, 0x84, 0x32, 0xef, 0x4d, 0xd6, 0xc4, 0x63, 0xc7, 0x9f, 0xdf, 0xc9, 0xe0, - 0x12, 0x23, 0xb5, 0x27, 0x1e, 0xba, 0x05, 0x15, 0xce, 0x3e, 0x1e, 0xd9, 0x26, 0x6f, 0x0a, 0xc2, - 0x4e, 0x06, 0x03, 0x23, 0x6e, 0x51, 0x1a, 0x12, 0x21, 0xe7, 0xce, 0xc6, 0x6c, 0x27, 0x01, 0xd3, - 0x4f, 0x74, 0x15, 0x0a, 0xee, 0xf0, 0x94, 0x8c, 0x4d, 0x76, 0xd0, 0x97, 0xb0, 0xbf, 0x42, 0x6f, - 0xc2, 0xf2, 0xcf, 0x88, 0x63, 0x1b, 0xde, 0xa9, 0x43, 0xdc, 0x53, 0x7b, 0x74, 0xc4, 0x0e, 0x5d, - 0xc0, 0x35, 0x4a, 0xd5, 0x03, 0x22, 0x7a, 0xcb, 0x87, 0x45, 0x76, 0x15, 0x98, 0x5d, 0x02, 0xae, - 0x52, 0xba, 0x12, 0xd8, 0x76, 0x1b, 0xc4, 0x18, 0x8e, 0x1b, 0x58, 0x64, 0x06, 0x0a, 0x78, 0x39, - 0x44, 0x72, 0x23, 0x65, 0x58, 0x9e, 0x90, 0x13, 0xd3, 0xb3, 0x1e, 0x12, 0xc3, 0x9d, 0x9a, 0x13, - 0xb7, 0x5e, 0x4a, 0xff, 0xe8, 0x35, 0x67, 0xc3, 0xcf, 0x89, 0xa7, 0x4d, 0xcd, 0x49, 0xf0, 0x2b, - 0x10, 0x48, 0x50, 0x9a, 0x8b, 0xbe, 0x0b, 0x2b, 0xa1, 0x8a, 0x23, 0x32, 0xf2, 0x4c, 0xb7, 0x5e, - 0x5e, 0xcf, 0x6d, 0x20, 0x1c, 0x6a, 0x6e, 0x31, 0x6a, 0x02, 0xc8, 0x6c, 0x73, 0xeb, 0xb0, 0x9e, - 0xdb, 0x10, 0x22, 0x20, 0x33, 0xcc, 0xa5, 0x46, 0x4d, 0x6d, 0xd7, 0x8a, 0x19, 0x55, 0xf9, 0x7a, - 0xa3, 0x02, 0x89, 0xd0, 0xa8, 0x50, 0x85, 0x6f, 0x54, 0x95, 0x1b, 0x15, 0x90, 0x23, 0xa3, 0x42, - 0xa0, 0x6f, 0x54, 0x8d, 0x1b, 0x15, 0x90, 0x7d, 0xa3, 0x3e, 0x02, 0x70, 0x88, 0x4b, 0x3c, 0xe3, - 0x94, 0x46, 0x7e, 0x99, 0x35, 0x84, 0x9b, 0x0b, 0xfa, 0xdf, 0x26, 0xa6, 0xa8, 0x1d, 0x6b, 0xe2, - 0xe1, 0xb2, 0x13, 0x7c, 0xa2, 0x1b, 0x50, 0x8e, 0x5a, 0xdf, 0x0a, 0x4b, 0xbe, 0x88, 0x80, 0xde, - 0x80, 0xda, 0x70, 0xe6, 0x7a, 0xf6, 0xd8, 0x60, 0xd9, 0xea, 0xd6, 0x45, 0x66, 0x42, 0x95, 0x13, - 0xf7, 0x18, 0x4d, 0xfa, 0x10, 0xca, 0xa1, 0xea, 0x64, 0xed, 0x17, 0x21, 0x77, 0xa0, 0x6a, 0xa2, - 0x80, 0x0a, 0x90, 0xed, 0xf5, 0xc5, 0x6c, 0x54, 0xff, 0xb9, 0xeb, 0xf9, 0x5f, 0xfd, 0xa1, 0x21, - 0x34, 0x8b, 0xb0, 0xc4, 0x9c, 0x6b, 0x56, 0x01, 0xa2, 0xdc, 0x90, 0xb6, 0xa0, 0xc2, 0xd2, 0xe0, - 0xdb, 0x76, 0xce, 0x7f, 0xe5, 0x61, 0x99, 0x29, 0x8a, 0xea, 0xc7, 0x05, 0xc4, 0xf6, 0x20, 0x8e, - 0x91, 0x0a, 0x5b, 0xad, 0xa9, 0xfe, 0xfb, 0xc9, 0x9a, 0x7c, 0x62, 0x79, 0xa7, 0xb3, 0xc3, 0xcd, - 0xa1, 0x3d, 0x7e, 0x67, 0xea, 0xd8, 0x63, 0xe2, 0x9d, 0x92, 0x99, 0x1b, 0xff, 0x1c, 0xdb, 0x47, - 0x64, 0xf4, 0x4e, 0xf8, 0x33, 0xb2, 0xa9, 0x70, 0x75, 0x51, 0x78, 0xc5, 0x61, 0x8a, 0xf2, 0x6d, - 0x0b, 0xec, 0x66, 0x3c, 0x38, 0xbc, 0x64, 0x70, 0x39, 0x2c, 0x18, 0xea, 0x3b, 0xe7, 0xf8, 0x9d, - 0x85, 0x2d, 0x16, 0x94, 0xf9, 0x2b, 0x48, 0xdf, 0x57, 0x50, 0x96, 0x6f, 0x83, 0x18, 0x5a, 0x71, - 0xc8, 0xb0, 0x41, 0x66, 0x87, 0x09, 0xcf, 0x55, 0x30, 0x68, 0xb8, 0x5b, 0x00, 0xe5, 0x95, 0x19, - 0x16, 0x6c, 0x00, 0xbd, 0x48, 0xa6, 0xde, 0xcf, 0x97, 0x04, 0x31, 0x7b, 0x3f, 0x5f, 0x2a, 0x88, - 0xc5, 0xfb, 0xf9, 0x52, 0x59, 0x84, 0xfb, 0xf9, 0x52, 0x55, 0xac, 0xdd, 0xcf, 0x97, 0x56, 0x44, - 0x11, 0x47, 0x7d, 0x15, 0xa7, 0xfa, 0x19, 0x4e, 0x37, 0x12, 0x9c, 0x2e, 0xe2, 0x58, 0xd1, 0x48, - 0x1f, 0x01, 0x44, 0x31, 0xa0, 0x47, 0x6f, 0x1f, 0x1f, 0xbb, 0x84, 0x37, 0xeb, 0x4b, 0xd8, 0x5f, - 0x51, 0xfa, 0x88, 0x4c, 0x4e, 0xbc, 0x53, 0x76, 0x6a, 0x35, 0xec, 0xaf, 0xa4, 0x19, 0xa0, 0x64, - 0xc6, 0xb2, 0xfb, 0xc6, 0x05, 0x2a, 0xe0, 0x23, 0x28, 0x87, 0x39, 0xc9, 0xf6, 0x4a, 0x5c, 0xcb, - 0x93, 0x3a, 0xfd, 0x6b, 0x79, 0x24, 0x20, 0x4d, 0x60, 0x85, 0x17, 0x5b, 0x54, 0x29, 0x61, 0x5a, - 0x09, 0x0b, 0xd2, 0x2a, 0x1b, 0xa5, 0xd5, 0xbb, 0x50, 0x0c, 0x0e, 0x87, 0x5f, 0xdb, 0xae, 0x2d, - 0xba, 0x7d, 0x31, 0x04, 0x0e, 0x90, 0x92, 0x0b, 0x2b, 0x29, 0x1e, 0x7d, 0x1a, 0x1c, 0xda, 0xb3, - 0xc9, 0x91, 0xe9, 0xbf, 0x71, 0x84, 0x8d, 0x25, 0x1c, 0xa3, 0x50, 0x7b, 0x46, 0xf6, 0x17, 0xc4, - 0x09, 0xd2, 0x9c, 0x2d, 0x28, 0x75, 0x36, 0x9d, 0x12, 0xc7, 0x4f, 0x74, 0xbe, 0x88, 0x6c, 0xcf, - 0xc7, 0x6c, 0x97, 0x46, 0x70, 0x39, 0xe5, 0x24, 0x0b, 0x6e, 0xa2, 0x07, 0x66, 0xd3, 0x3d, 0xf0, - 0xee, 0x7c, 0x5c, 0xaf, 0xa5, 0xef, 0xb2, 0xa1, 0xbe, 0x78, 0x48, 0x7f, 0x99, 0x83, 0xda, 0x83, - 0x19, 0x71, 0xce, 0x82, 0x6b, 0x3e, 0xfa, 0x01, 0x14, 0x5c, 0xcf, 0xf4, 0x66, 0xae, 0x7f, 0x6f, - 0x5b, 0x8d, 0xf4, 0x30, 0xa0, 0xc6, 0x98, 0xd8, 0x07, 0xa1, 0xbb, 0x00, 0x84, 0x5e, 0xf7, 0x0d, - 0x76, 0xd5, 0xe3, 0xcf, 0xb4, 0x7a, 0x4a, 0x84, 0xbd, 0x07, 0xd8, 0xfd, 0xae, 0x4c, 0x82, 0x4f, - 0xea, 0x3d, 0x5b, 0xb0, 0x98, 0x94, 0x31, 0x5f, 0xa0, 0x4d, 0xba, 0xbb, 0x63, 0x4d, 0x4e, 0x58, - 0x50, 0x12, 0x35, 0xab, 0x31, 0x7a, 0xcb, 0xf4, 0xcc, 0x9d, 0x0c, 0xf6, 0x51, 0x14, 0xff, 0x90, - 0x0c, 0x3d, 0xdb, 0x61, 0x4d, 0x29, 0x81, 0xdf, 0x63, 0xf4, 0x00, 0xcf, 0x51, 0x4c, 0xff, 0xd0, - 0x1c, 0x99, 0x0e, 0xfb, 0xf9, 0x4f, 0xea, 0x67, 0xf4, 0x50, 0x3f, 0x5b, 0x51, 0xfc, 0xd8, 0xf4, - 0x1c, 0xeb, 0x11, 0xeb, 0x68, 0x09, 0x7c, 0x97, 0xd1, 0x03, 0x3c, 0x47, 0xa1, 0xeb, 0x50, 0xfa, - 0xc2, 0x74, 0x26, 0xd6, 0xe4, 0x84, 0x77, 0x9d, 0x32, 0x0e, 0xd7, 0xd4, 0x63, 0x6b, 0x72, 0x6c, - 0xf3, 0x5f, 0xf8, 0x32, 0xe6, 0x8b, 0x66, 0x01, 0xf2, 0xf4, 0x16, 0x2c, 0xa9, 0x00, 0x91, 0x87, - 0xc9, 0x9f, 0x8a, 0xf2, 0xcb, 0x2e, 0x62, 0xf3, 0x15, 0x26, 0xfd, 0x42, 0x00, 0x88, 0x3c, 0x47, - 0x1f, 0x44, 0x4f, 0x1c, 0x7e, 0x35, 0xbc, 0x9a, 0x0e, 0xd0, 0xe2, 0x87, 0xce, 0x27, 0x89, 0x07, - 0x4b, 0x36, 0x5d, 0x32, 0x5c, 0xf4, 0x7f, 0x3c, 0x5b, 0x24, 0x03, 0xaa, 0x71, 0xfd, 0xb4, 0x95, - 0xf0, 0x9b, 0x3b, 0xb3, 0xa3, 0x8c, 0xfd, 0xd5, 0x37, 0xbf, 0x71, 0xfe, 0x5a, 0x80, 0x95, 0x94, - 0x19, 0x2f, 0xdd, 0x24, 0xd1, 0x76, 0xb2, 0x17, 0x68, 0x3b, 0x99, 0x58, 0x8d, 0x5c, 0xc4, 0x18, - 0x7a, 0x78, 0x61, 0xfa, 0x2c, 0xfe, 0x9d, 0xbf, 0xc8, 0xe1, 0x35, 0x01, 0xa2, 0xac, 0x42, 0xef, - 0x41, 0x21, 0x31, 0x4b, 0xb9, 0x9a, 0xce, 0x3d, 0x7f, 0x9a, 0xc2, 0x0d, 0xf6, 0xb1, 0xd2, 0xef, - 0x05, 0xa8, 0xc6, 0xd9, 0x2f, 0x0d, 0xca, 0xfb, 0xe9, 0xd7, 0xef, 0x6a, 0x2a, 0x24, 0x8b, 0x33, - 0xa3, 0x99, 0xc8, 0x0c, 0xde, 0x4c, 0x6f, 0xbc, 0x2c, 0x98, 0xec, 0xe1, 0x31, 0x9f, 0x1c, 0x9f, - 0xc1, 0x4a, 0x7c, 0x7e, 0x83, 0xf7, 0xef, 0xa0, 0x3a, 0x14, 0xfd, 0x09, 0x8b, 0x3f, 0x70, 0x09, - 0x96, 0xe8, 0xe3, 0xc4, 0x58, 0xe9, 0x42, 0xa3, 0x96, 0x98, 0x80, 0xf4, 0xe7, 0x2c, 0xd4, 0x12, - 0x18, 0xb4, 0x06, 0x15, 0xfe, 0xdc, 0x31, 0x1c, 0x72, 0xcc, 0x83, 0x5b, 0xc3, 0xc0, 0x49, 0x98, - 0x1c, 0x7f, 0x93, 0xb9, 0xc0, 0xbd, 0x05, 0x41, 0xb9, 0xe0, 0xfb, 0xfe, 0x5e, 0x7c, 0xa4, 0x90, - 0x4f, 0x1f, 0x44, 0xf0, 0xc4, 0x8b, 0x7c, 0x8b, 0x4d, 0x15, 0xee, 0xc6, 0x66, 0x5c, 0xbc, 0xfd, - 0xad, 0x26, 0x1e, 0xd9, 0x8c, 0x13, 0x49, 0x46, 0x03, 0xae, 0xff, 0x6b, 0xa6, 0x70, 0x08, 0x95, - 0x98, 0x15, 0x5f, 0x1f, 0xbd, 0xc5, 0x15, 0x9d, 0xf8, 0xfd, 0xca, 0xa5, 0x7e, 0xbf, 0xa4, 0xbf, - 0x64, 0xa1, 0x12, 0x33, 0x18, 0xbd, 0x97, 0x18, 0x1d, 0xac, 0x2f, 0xf4, 0x6a, 0x7e, 0x6e, 0x70, - 0x0d, 0x4a, 0xf4, 0xf5, 0x4f, 0x0d, 0x63, 0x5b, 0xd4, 0x70, 0x91, 0xae, 0x31, 0x39, 0xa6, 0xac, - 0xd9, 0xc4, 0xf2, 0x18, 0x2b, 0xcf, 0x59, 0x74, 0x8d, 0xc9, 0xb1, 0xf4, 0x77, 0x21, 0x31, 0x18, - 0x78, 0x1d, 0x5e, 0xeb, 0xaa, 0x3a, 0x6e, 0x2b, 0x86, 0x7e, 0xb0, 0xab, 0x1a, 0x83, 0x9e, 0xb6, - 0xab, 0x2a, 0xed, 0xad, 0xb6, 0xda, 0x12, 0x33, 0xe8, 0x35, 0xb8, 0x1c, 0x67, 0x46, 0x43, 0x83, - 0x55, 0xb8, 0x14, 0x67, 0x04, 0x03, 0x84, 0x6b, 0xb0, 0x1a, 0x27, 0xc7, 0x87, 0x09, 0x0d, 0xb8, - 0x3e, 0x27, 0x11, 0x1f, 0x2c, 0xa4, 0xb6, 0x8a, 0x86, 0x0c, 0x57, 0x40, 0x8c, 0x33, 0xfc, 0x81, - 0x43, 0x1d, 0xae, 0x24, 0xe0, 0xe1, 0xf0, 0xe1, 0xf6, 0xf3, 0x1c, 0x40, 0x34, 0x91, 0xa3, 0x7a, - 0x55, 0x8c, 0xfb, 0xd8, 0x50, 0xe4, 0x81, 0x46, 0xfd, 0x0b, 0x1e, 0x42, 0x6f, 0x81, 0x14, 0x67, - 0x60, 0x75, 0xb7, 0xd3, 0x56, 0x64, 0xcd, 0x68, 0xb5, 0x5b, 0x46, 0xaf, 0xaf, 0x1b, 0x5d, 0x59, - 0x57, 0x76, 0x44, 0x01, 0xdd, 0x82, 0x9b, 0x71, 0x9c, 0xde, 0xef, 0x1b, 0x5d, 0xb9, 0x77, 0x60, - 0x28, 0x9d, 0x81, 0xa6, 0xab, 0x58, 0x13, 0xb3, 0xd4, 0x98, 0x38, 0xa4, 0x29, 0xb7, 0x8c, 0x96, - 0xac, 0xcb, 0x62, 0x2e, 0xbd, 0x49, 0xbb, 0xb7, 0xad, 0x6a, 0x7a, 0xbb, 0xdf, 0x33, 0xb0, 0xac, - 0xab, 0x46, 0xa7, 0xdd, 0x6d, 0xeb, 0x6a, 0x4b, 0xcc, 0xa3, 0xef, 0xc0, 0x7a, 0xd2, 0x98, 0x07, - 0x03, 0x55, 0xd3, 0x93, 0xa8, 0x25, 0x1a, 0xc3, 0xa4, 0x36, 0x4d, 0x97, 0x7b, 0x8a, 0x8f, 0x10, - 0x0b, 0xe8, 0x0d, 0x58, 0x8b, 0xf3, 0x35, 0x15, 0xef, 0xb5, 0x15, 0xea, 0xb3, 0xbc, 0x27, 0xb7, - 0x3b, 0x72, 0xb3, 0xa3, 0x8a, 0x45, 0xb4, 0x0e, 0x37, 0x12, 0xfe, 0x68, 0xad, 0x66, 0x02, 0x51, - 0x4a, 0xbb, 0x43, 0x3d, 0x6e, 0x0e, 0xb4, 0x03, 0xb1, 0x9c, 0x36, 0x53, 0x69, 0x63, 0x65, 0xd0, - 0xd6, 0x8d, 0x26, 0x56, 0xe5, 0x4f, 0x55, 0x6c, 0xf4, 0x77, 0xd5, 0x9e, 0x08, 0x48, 0x82, 0x46, - 0x1c, 0xd5, 0x55, 0xf5, 0x9d, 0x3e, 0x8f, 0xa9, 0xdc, 0xe9, 0xf4, 0xf7, 0xd5, 0x96, 0x58, 0x41, - 0x37, 0xa0, 0x9e, 0xd8, 0x43, 0xed, 0xc9, 0x3d, 0xdd, 0x77, 0xa4, 0x8a, 0xde, 0x84, 0x5b, 0x71, - 0xae, 0xac, 0xe8, 0xed, 0x3d, 0xe6, 0x4f, 0x5b, 0xd5, 0xc2, 0x78, 0xd4, 0x6e, 0x7f, 0x02, 0x95, - 0xd8, 0x1d, 0x0d, 0x5d, 0x05, 0xf4, 0x60, 0xa0, 0xe2, 0x03, 0x96, 0x0d, 0x03, 0xcd, 0x60, 0x2a, - 0xc4, 0x0c, 0xf5, 0x27, 0x41, 0xd7, 0x06, 0x8a, 0xa2, 0x6a, 0x9a, 0x28, 0xdc, 0xfe, 0x4f, 0x16, - 0x96, 0x93, 0x57, 0x36, 0x9a, 0xc2, 0x1c, 0xcc, 0x0d, 0x60, 0xd9, 0xd5, 0xeb, 0xf7, 0x54, 0x31, - 0x43, 0x6d, 0x9e, 0x63, 0xe9, 0xed, 0xae, 0xda, 0x1f, 0xe8, 0xa2, 0x80, 0x6e, 0xc2, 0xb5, 0x39, - 0xae, 0x42, 0x8f, 0xa7, 0xa3, 0xb6, 0xc4, 0x2c, 0x3d, 0xbb, 0x39, 0xb6, 0xfa, 0x63, 0x55, 0x19, - 0xd0, 0x74, 0x10, 0x73, 0x0b, 0xc5, 0xc3, 0x44, 0xca, 0x2f, 0x64, 0xb7, 0x69, 0x31, 0xf6, 0xe4, - 0x8e, 0xb8, 0x44, 0x0f, 0x75, 0x8e, 0x1d, 0x3f, 0xd4, 0xc2, 0xc2, 0xfd, 0xe9, 0x91, 0x6c, 0xf5, - 0x07, 0xbd, 0x96, 0x58, 0xa4, 0xb9, 0xb3, 0x90, 0x2f, 0x2b, 0x8a, 0xba, 0xab, 0xfb, 0x99, 0xf1, - 0x16, 0x48, 0xf3, 0x11, 0x08, 0x0a, 0xc2, 0x4f, 0x5a, 0x8d, 0xe7, 0xc9, 0x42, 0x5c, 0x47, 0xc6, - 0xdb, 0xaa, 0xa1, 0xf6, 0x74, 0x7c, 0x20, 0x42, 0xf3, 0xe3, 0xc7, 0x4f, 0x1b, 0x99, 0xaf, 0x9e, - 0x36, 0x32, 0x2f, 0x9e, 0x36, 0x84, 0x9f, 0x9f, 0x37, 0x84, 0x3f, 0x9d, 0x37, 0x84, 0x2f, 0xcf, - 0x1b, 0xc2, 0xe3, 0xf3, 0x86, 0xf0, 0x8f, 0xf3, 0x86, 0xf0, 0xfc, 0xbc, 0x91, 0x79, 0x71, 0xde, - 0x10, 0x7e, 0xfb, 0xac, 0x91, 0x79, 0xfc, 0xac, 0x91, 0xf9, 0xea, 0x59, 0x23, 0xf3, 0x93, 0x22, - 0xfb, 0xd3, 0x66, 0x7a, 0x78, 0x58, 0x60, 0xff, 0xcb, 0xbc, 0xfb, 0xdf, 0x00, 0x00, 0x00, 0xff, - 0xff, 0xd9, 0x11, 0x1d, 0x78, 0xc6, 0x19, 0x00, 0x00, + // 2474 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xac, 0x59, 0xcb, 0x73, 0x1b, 0xc7, + 0xd1, 0xc7, 0x02, 0x20, 0x1e, 0x0d, 0x80, 0x5c, 0x8d, 0x44, 0x19, 0x92, 0x25, 0x90, 0x5a, 0x7f, + 0xf6, 0x47, 0xab, 0x12, 0x3a, 0x25, 0xd9, 0x56, 0xc9, 0x25, 0xc7, 0xb5, 0x58, 0x2c, 0x49, 0xc8, + 0x78, 0x50, 0xb3, 0x0b, 0x32, 0xcc, 0x65, 0x6b, 0x09, 0x0e, 0xc9, 0x95, 0x01, 0x2c, 0xb2, 0xbb, + 0x90, 0xc5, 0x9c, 0x72, 0x49, 0x25, 0x95, 0xca, 0x21, 0x55, 0x39, 0x25, 0xb7, 0x54, 0x2e, 0xa9, + 0xfc, 0x25, 0x3a, 0xea, 0xe8, 0xa4, 0x12, 0x55, 0x44, 0x5d, 0x94, 0x9b, 0x2b, 0xc7, 0xe4, 0x92, + 0x9a, 0x99, 0x7d, 0x13, 0x8c, 0x59, 0xb6, 0x6e, 0x3b, 0xdd, 0xbf, 0xee, 0xe9, 0xee, 0xe9, 0x6e, + 0xcc, 0x34, 0xa0, 0x32, 0xb6, 0xc6, 0x96, 0xb3, 0x3e, 0x75, 0x6c, 0xcf, 0x46, 0xa5, 0xa1, 0xed, + 0x78, 0xe4, 0xe9, 0x74, 0xff, 0xfa, 0x95, 0x23, 0xfb, 0xc8, 0x66, 0xc4, 0x0f, 0xe8, 0x17, 0xe7, + 0x4b, 0x7f, 0xcb, 0x41, 0x75, 0xd7, 0xb1, 0x3c, 0x82, 0xc9, 0x4f, 0x66, 0xc4, 0xf5, 0xd0, 0x36, + 0x80, 0x67, 0x8d, 0x89, 0x4b, 0x1c, 0x8b, 0xb8, 0x75, 0x61, 0x35, 0xb7, 0x56, 0xb9, 0x73, 0x65, + 0x3d, 0xd0, 0xb2, 0xae, 0x5b, 0x63, 0xa2, 0x31, 0x5e, 0xf3, 0xfa, 0xb3, 0x17, 0x2b, 0x99, 0xbf, + 0xbe, 0x58, 0x41, 0xdb, 0x0e, 0x31, 0x47, 0x23, 0x7b, 0xa8, 0x87, 0x72, 0x38, 0xa6, 0x03, 0xdd, + 0x87, 0x82, 0x66, 0xcf, 0x9c, 0x21, 0xa9, 0x67, 0x57, 0x85, 0xb5, 0xc5, 0x3b, 0xb7, 0x22, 0x6d, + 0xf1, 0x9d, 0xd7, 0x39, 0x48, 0x9d, 0xcc, 0xc6, 0xd8, 0x17, 0x40, 0x9f, 0x40, 0x69, 0x4c, 0x3c, + 0xf3, 0xc0, 0xf4, 0xcc, 0x7a, 0x8e, 0x99, 0x52, 0x8f, 0x84, 0xbb, 0xc4, 0x73, 0xac, 0x61, 0xd7, + 0xe7, 0x37, 0xf3, 0xcf, 0x5e, 0xac, 0x08, 0x38, 0xc4, 0xa3, 0x06, 0x80, 0x7b, 0x32, 0xde, 0xb7, + 0x47, 0x2e, 0xde, 0xbd, 0x53, 0xcf, 0xaf, 0xe6, 0xd6, 0xca, 0x38, 0x46, 0x41, 0x0a, 0xd4, 0x22, + 0x23, 0x29, 0x64, 0x81, 0x6d, 0xf0, 0xd6, 0x3c, 0x5f, 0xf1, 0xee, 0x1d, 0xa6, 0x3f, 0x83, 0x93, + 0x32, 0xe8, 0x2e, 0x2c, 0xbb, 0x5f, 0x58, 0x53, 0x63, 0x64, 0xee, 0x93, 0x91, 0xf1, 0xc4, 0x1c, + 0x59, 0x07, 0xa6, 0x67, 0xd9, 0x93, 0xfa, 0xeb, 0xe2, 0xaa, 0xb0, 0x56, 0xc2, 0x97, 0x29, 0xb7, + 0x43, 0x99, 0x3b, 0x21, 0x0f, 0xfd, 0x10, 0xde, 0x8e, 0x09, 0x0d, 0xed, 0xd9, 0xc4, 0x8b, 0x8b, + 0xfe, 0x93, 0x8b, 0xd6, 0x43, 0x51, 0x85, 0x22, 0x22, 0x79, 0xe9, 0x7d, 0x80, 0x28, 0x56, 0xa8, + 0x08, 0x39, 0x79, 0xbb, 0x2d, 0x66, 0x50, 0x09, 0xf2, 0x78, 0xd0, 0x51, 0x45, 0x81, 0x7e, 0xf5, + 0xf5, 0xce, 0xb6, 0x98, 0x95, 0x96, 0xa0, 0xe6, 0xc7, 0xd8, 0x9d, 0xda, 0x13, 0x97, 0x48, 0x4f, + 0xa1, 0xaa, 0x3a, 0x8e, 0xed, 0xb4, 0x88, 0x67, 0x5a, 0x23, 0x17, 0xdd, 0x86, 0x05, 0xc5, 0x9c, + 0xb9, 0xa4, 0x2e, 0xb0, 0xb3, 0x89, 0x9d, 0x34, 0x83, 0x31, 0x1e, 0xe6, 0x10, 0x84, 0x20, 0xaf, + 0xd9, 0x87, 0x1e, 0x3b, 0xc6, 0x12, 0x66, 0xdf, 0x68, 0x0d, 0x96, 0x30, 0x79, 0x4c, 0x86, 0x1e, + 0x39, 0xd0, 0xcc, 0xf1, 0x74, 0x44, 0xdc, 0x7a, 0x6e, 0x55, 0x58, 0xcb, 0xe1, 0x34, 0x59, 0xfa, + 0x6d, 0x16, 0x20, 0x8a, 0x28, 0xda, 0x84, 0x02, 0xf3, 0x3f, 0xc8, 0xb1, 0xcb, 0xd1, 0xce, 0xcc, + 0xe9, 0x6d, 0xd3, 0x72, 0xa2, 0x14, 0x1b, 0x4c, 0x5c, 0xf3, 0x90, 0x74, 0x67, 0x9e, 0xb9, 0x3f, + 0x22, 0x8c, 0x8f, 0x7d, 0x71, 0xf4, 0x03, 0x28, 0xba, 0xfe, 0xce, 0x59, 0xa6, 0x49, 0x8c, 0x34, + 0xf1, 0xbd, 0xfd, 0xa3, 0x0b, 0x60, 0xe8, 0x63, 0x28, 0x93, 0xa7, 0x64, 0x3c, 0x1d, 0x99, 0x8e, + 0xeb, 0xa7, 0x15, 0x8a, 0xf9, 0xed, 0xb3, 0x7c, 0xa9, 0x08, 0x8a, 0xee, 0x03, 0x1c, 0x5b, 0xae, + 0x67, 0x1f, 0x39, 0xe6, 0xd8, 0x65, 0x19, 0x95, 0x30, 0x7b, 0x2b, 0xe0, 0xf9, 0x92, 0x31, 0xf0, + 0xc3, 0x7c, 0xa9, 0x20, 0x16, 0xf1, 0xa5, 0xa1, 0x43, 0x4c, 0x8f, 0x1c, 0x18, 0x2c, 0x89, 0x3c, + 0x73, 0x3c, 0x95, 0x3e, 0x82, 0x72, 0xe8, 0x2e, 0x0d, 0xf0, 0xc4, 0x1c, 0xf3, 0xb3, 0xa8, 0x62, + 0xf6, 0x8d, 0xae, 0xc0, 0xc2, 0x13, 0x73, 0x34, 0xe3, 0xc5, 0x53, 0xc5, 0x7c, 0x21, 0x3d, 0x86, + 0x02, 0xf7, 0x0d, 0xdd, 0x82, 0x6a, 0xa8, 0xcd, 0x18, 0xbb, 0x0c, 0x96, 0xc3, 0x95, 0x90, 0xd6, + 0x75, 0x23, 0x15, 0x54, 0xaf, 0xe0, 0xab, 0x40, 0xff, 0x0f, 0x4b, 0xae, 0x67, 0x3a, 0x5e, 0x64, + 0x8c, 0x7f, 0x72, 0x8b, 0x8c, 0xac, 0x87, 0x26, 0xfe, 0x3e, 0x0b, 0x8b, 0xc9, 0x5a, 0x43, 0xf7, + 0x20, 0xef, 0x9d, 0x4c, 0x83, 0xa4, 0x79, 0xe7, 0xbc, 0x9a, 0xf4, 0x97, 0xfa, 0xc9, 0x94, 0x60, + 0x26, 0x80, 0xbe, 0x07, 0x68, 0xcc, 0x68, 0xc6, 0xa1, 0x39, 0xb6, 0x46, 0x27, 0x06, 0xf3, 0x97, + 0xda, 0x5c, 0xc6, 0x22, 0xe7, 0x6c, 0x30, 0x46, 0x8f, 0xfa, 0x8e, 0x20, 0x7f, 0x4c, 0x46, 0xd3, + 0x7a, 0x9e, 0xf1, 0xd9, 0x37, 0xa5, 0xcd, 0x26, 0x96, 0x57, 0x5f, 0xe0, 0x34, 0xfa, 0x2d, 0x9d, + 0x00, 0x44, 0x3b, 0xa1, 0x0a, 0x14, 0x07, 0xbd, 0xcf, 0x7b, 0xfd, 0xdd, 0x9e, 0x98, 0xa1, 0x0b, + 0xa5, 0x3f, 0xe8, 0xe9, 0x2a, 0x16, 0x05, 0x54, 0x86, 0x85, 0x4d, 0x79, 0xb0, 0xa9, 0x8a, 0x59, + 0x54, 0x83, 0xf2, 0x56, 0x5b, 0xd3, 0xfb, 0x9b, 0x58, 0xee, 0x8a, 0x39, 0x84, 0x60, 0x91, 0x71, + 0x22, 0x5a, 0x9e, 0x8a, 0x6a, 0x83, 0x6e, 0x57, 0xc6, 0x7b, 0xe2, 0x02, 0x2d, 0xa9, 0x76, 0x6f, + 0xa3, 0x2f, 0x16, 0x50, 0x15, 0x4a, 0x9a, 0x2e, 0xeb, 0xaa, 0xa6, 0xea, 0x62, 0x51, 0x7a, 0x04, + 0x05, 0xbe, 0xf5, 0x1b, 0x4b, 0x68, 0xe9, 0xd7, 0x02, 0x94, 0x82, 0x24, 0x7c, 0x73, 0x65, 0x92, + 0xc8, 0xa3, 0x30, 0x09, 0xd2, 0xd9, 0x93, 0x3b, 0x93, 0x3d, 0xd2, 0xef, 0x0a, 0x50, 0x0e, 0x53, + 0x1b, 0xdd, 0x84, 0x32, 0x6f, 0x58, 0xd6, 0xc4, 0x63, 0xc7, 0x9f, 0xdf, 0xca, 0xe0, 0x12, 0x23, + 0xb5, 0x27, 0x1e, 0xba, 0x05, 0x15, 0xce, 0x3e, 0x1c, 0xd9, 0x26, 0xef, 0x14, 0xc2, 0x56, 0x06, + 0x03, 0x23, 0x6e, 0x50, 0x1a, 0x12, 0x21, 0xe7, 0xce, 0xc6, 0x6c, 0x27, 0x01, 0xd3, 0x4f, 0x74, + 0x15, 0x0a, 0xee, 0xf0, 0x98, 0x8c, 0x4d, 0x76, 0xd0, 0x97, 0xb0, 0xbf, 0x42, 0xef, 0xc2, 0xe2, + 0x4f, 0x89, 0x63, 0x1b, 0xde, 0xb1, 0x43, 0xdc, 0x63, 0x7b, 0x74, 0xc0, 0x0e, 0x5d, 0xc0, 0x35, + 0x4a, 0xd5, 0x03, 0x22, 0x7a, 0xcf, 0x87, 0x45, 0x76, 0x15, 0x98, 0x5d, 0x02, 0xae, 0x52, 0xba, + 0x12, 0xd8, 0x76, 0x1b, 0xc4, 0x18, 0x8e, 0x1b, 0x58, 0x64, 0x06, 0x0a, 0x78, 0x31, 0x44, 0x72, + 0x23, 0x65, 0x58, 0x9c, 0x90, 0x23, 0xd3, 0xb3, 0x9e, 0x10, 0xc3, 0x9d, 0x9a, 0x13, 0xb7, 0x5e, + 0x4a, 0xff, 0x12, 0x36, 0x67, 0xc3, 0x2f, 0x88, 0xa7, 0x4d, 0xcd, 0x49, 0xf0, 0xd3, 0x10, 0x48, + 0x50, 0x9a, 0x4b, 0xeb, 0x2b, 0x54, 0x71, 0x40, 0x46, 0x9e, 0xe9, 0xd6, 0xcb, 0xab, 0xb9, 0x35, + 0x84, 0x43, 0xcd, 0x2d, 0x46, 0x4d, 0x00, 0x99, 0x6d, 0x6e, 0x1d, 0x56, 0x73, 0x6b, 0x42, 0x04, + 0x64, 0x86, 0xb9, 0xd4, 0xa8, 0xa9, 0xed, 0x5a, 0x31, 0xa3, 0x2a, 0xdf, 0x6c, 0x54, 0x20, 0x11, + 0x1a, 0x15, 0xaa, 0xf0, 0x8d, 0xaa, 0x72, 0xa3, 0x02, 0x72, 0x64, 0x54, 0x08, 0xf4, 0x8d, 0xaa, + 0x71, 0xa3, 0x02, 0xb2, 0x6f, 0xd4, 0x03, 0x00, 0x87, 0xb8, 0xc4, 0x33, 0x8e, 0x69, 0xe4, 0x17, + 0x59, 0x43, 0xb8, 0x39, 0xa7, 0x29, 0xae, 0x63, 0x8a, 0xda, 0xb2, 0x26, 0x1e, 0x2e, 0x3b, 0xc1, + 0x27, 0xba, 0x01, 0xe5, 0xa8, 0xfd, 0x2c, 0xb1, 0xe4, 0x8b, 0x08, 0xe8, 0x1d, 0xa8, 0x0d, 0x67, + 0xae, 0x67, 0x8f, 0x0d, 0x96, 0xad, 0x6e, 0x5d, 0x64, 0x26, 0x54, 0x39, 0x71, 0x87, 0xd1, 0xe6, + 0xf5, 0xb1, 0x4b, 0x73, 0xfb, 0xd8, 0x27, 0x50, 0x0e, 0x6d, 0x48, 0x36, 0x89, 0x22, 0xe4, 0xf6, + 0x54, 0x4d, 0x14, 0x50, 0x01, 0xb2, 0xbd, 0xbe, 0x98, 0x8d, 0x1a, 0x45, 0xee, 0x7a, 0xfe, 0x97, + 0x7f, 0x6c, 0x08, 0xcd, 0x22, 0x2c, 0xb0, 0x28, 0x34, 0xab, 0x00, 0x51, 0x12, 0x49, 0x1b, 0x50, + 0x61, 0xf9, 0xf2, 0x1d, 0x7b, 0xb1, 0xf4, 0xaf, 0x3c, 0x2c, 0x32, 0x45, 0x51, 0xa1, 0xb9, 0x80, + 0xd8, 0x1e, 0xc4, 0x31, 0x52, 0xf1, 0xad, 0x35, 0xd5, 0x7f, 0xbf, 0x58, 0x91, 0x8f, 0x2c, 0xef, + 0x78, 0xb6, 0xbf, 0x3e, 0xb4, 0xc7, 0x1f, 0x4c, 0x1d, 0x7b, 0x4c, 0xbc, 0x63, 0x32, 0x73, 0xe3, + 0x9f, 0x63, 0xfb, 0x80, 0x8c, 0x3e, 0x08, 0x7f, 0x84, 0xd6, 0x15, 0xae, 0x2e, 0x3a, 0x07, 0x71, + 0x98, 0xa2, 0x7c, 0xd7, 0x4a, 0xbc, 0x19, 0x0f, 0x0e, 0xaf, 0x2d, 0x5c, 0x0e, 0x2b, 0x8b, 0xfa, + 0xce, 0x39, 0x7e, 0x0b, 0x62, 0x8b, 0x39, 0xfd, 0xe0, 0x0d, 0xe4, 0xf9, 0x1b, 0xa8, 0xdf, 0xf7, + 0x41, 0x0c, 0xad, 0xd8, 0x67, 0xd8, 0xa0, 0x04, 0xc2, 0xca, 0xe0, 0x2a, 0x18, 0x34, 0xdc, 0x2d, + 0x80, 0xf2, 0x12, 0x0e, 0x2b, 0x3b, 0x80, 0x5e, 0x24, 0xa5, 0x1f, 0xe6, 0x4b, 0x82, 0x98, 0xe5, + 0x77, 0x86, 0x87, 0xf9, 0x52, 0x59, 0x84, 0x87, 0xf9, 0x52, 0x55, 0xac, 0x3d, 0xcc, 0x97, 0x96, + 0x44, 0x11, 0x47, 0x0d, 0x18, 0xa7, 0x1a, 0x1f, 0x4e, 0x77, 0x1c, 0x9c, 0xae, 0xf6, 0x58, 0x75, + 0x49, 0x0f, 0x00, 0xa2, 0x18, 0xd0, 0xa3, 0xb7, 0x0f, 0x0f, 0x5d, 0xc2, 0xbb, 0xfa, 0x25, 0xec, + 0xaf, 0x28, 0x7d, 0x44, 0x26, 0x47, 0xde, 0x31, 0x3b, 0xb5, 0x1a, 0xf6, 0x57, 0xd2, 0x0c, 0x50, + 0x32, 0x63, 0xd9, 0x0d, 0xe6, 0x02, 0x15, 0xf0, 0x00, 0xca, 0x61, 0x4e, 0xb2, 0xbd, 0x12, 0x97, + 0xfa, 0xa4, 0x4e, 0xff, 0x52, 0x1f, 0x09, 0x48, 0x13, 0x58, 0xe2, 0xc5, 0x16, 0x55, 0x4a, 0x98, + 0x56, 0xc2, 0x9c, 0xb4, 0xca, 0x46, 0x69, 0x75, 0x17, 0x8a, 0xc1, 0xe1, 0xf0, 0x4b, 0xdf, 0xb5, + 0x79, 0x77, 0x37, 0x86, 0xc0, 0x01, 0x52, 0x72, 0x61, 0x29, 0xc5, 0xa3, 0x0f, 0x8b, 0x7d, 0x7b, + 0x36, 0x39, 0x30, 0xfd, 0x17, 0x92, 0xb0, 0xb6, 0x80, 0x63, 0x14, 0x6a, 0xcf, 0xc8, 0xfe, 0x92, + 0x38, 0x41, 0x9a, 0xb3, 0x05, 0xa5, 0xce, 0xa6, 0x53, 0xe2, 0xf8, 0x89, 0xce, 0x17, 0x91, 0xed, + 0xf9, 0x98, 0xed, 0xd2, 0x08, 0x2e, 0xa7, 0x9c, 0x64, 0xc1, 0x4d, 0x34, 0xcb, 0x6c, 0xba, 0x59, + 0xde, 0x3b, 0x1b, 0xd7, 0x6b, 0xe9, 0x9b, 0x70, 0xa8, 0x2f, 0x1e, 0xd2, 0x5f, 0xe4, 0xa0, 0xf6, + 0x68, 0x46, 0x9c, 0x93, 0xe0, 0x91, 0x80, 0xbe, 0x0f, 0x05, 0xd7, 0x33, 0xbd, 0x99, 0xeb, 0x5f, + 0xf0, 0x96, 0x23, 0x3d, 0x0c, 0xa8, 0x31, 0x26, 0xf6, 0x41, 0xe8, 0x1e, 0x00, 0xa1, 0x8f, 0x05, + 0x83, 0xdd, 0x09, 0xf9, 0x23, 0xaf, 0x9e, 0x12, 0x61, 0xaf, 0x09, 0x76, 0x11, 0x2c, 0x93, 0xe0, + 0x93, 0x7a, 0xcf, 0x16, 0x2c, 0x26, 0x65, 0xcc, 0x17, 0x68, 0x9d, 0xee, 0xee, 0x58, 0x93, 0x23, + 0x16, 0x94, 0x44, 0xcd, 0x6a, 0x8c, 0xde, 0x32, 0x3d, 0x73, 0x2b, 0x83, 0x7d, 0x14, 0xc5, 0x3f, + 0x21, 0x43, 0xcf, 0x76, 0x58, 0x53, 0x4a, 0xe0, 0x77, 0x18, 0x3d, 0xc0, 0x73, 0x14, 0xd3, 0x3f, + 0x34, 0x47, 0xa6, 0xc3, 0xee, 0x09, 0x49, 0xfd, 0x8c, 0x1e, 0xea, 0x67, 0x2b, 0x8a, 0x1f, 0x9b, + 0x9e, 0x63, 0x3d, 0x65, 0x1d, 0x2d, 0x81, 0xef, 0x32, 0x7a, 0x80, 0xe7, 0x28, 0x74, 0x1d, 0x4a, + 0x5f, 0x9a, 0xce, 0xc4, 0x9a, 0x1c, 0xf1, 0xae, 0x53, 0xc6, 0xe1, 0x9a, 0x7a, 0x6c, 0x4d, 0x0e, + 0x6d, 0x7e, 0x15, 0x28, 0x63, 0xbe, 0x68, 0x16, 0x20, 0x4f, 0xaf, 0xcb, 0x92, 0x0a, 0x10, 0x79, + 0x98, 0xfc, 0xa9, 0x28, 0x9f, 0x77, 0x63, 0x3b, 0x5b, 0x61, 0xd2, 0xcf, 0x05, 0x80, 0xc8, 0x73, + 0xf4, 0x71, 0xf4, 0x40, 0xe2, 0x77, 0xc8, 0xab, 0xe9, 0x00, 0xcd, 0x7f, 0x26, 0x7d, 0x96, 0x78, + 0xee, 0x64, 0xd3, 0x25, 0xc3, 0x45, 0xff, 0xc7, 0xa3, 0x47, 0x32, 0xa0, 0x1a, 0xd7, 0x4f, 0x5b, + 0x09, 0xbf, 0xe2, 0x33, 0x3b, 0xca, 0xd8, 0x5f, 0x7d, 0xfb, 0xab, 0xe9, 0xaf, 0x04, 0x58, 0x4a, + 0x99, 0x71, 0xee, 0x26, 0x89, 0xb6, 0x93, 0xbd, 0x40, 0xdb, 0xc9, 0xc4, 0x6a, 0xe4, 0x22, 0xc6, + 0xd0, 0xc3, 0x0b, 0xd3, 0xe7, 0x9c, 0x37, 0xd7, 0x05, 0x0e, 0xaf, 0x09, 0x10, 0x65, 0x15, 0xfa, + 0x10, 0x0a, 0x89, 0x49, 0xcc, 0xd5, 0x74, 0xee, 0xf9, 0xb3, 0x18, 0x6e, 0xb0, 0x8f, 0x95, 0xfe, + 0x20, 0x40, 0x35, 0xce, 0x3e, 0x37, 0x28, 0x1f, 0xa5, 0xdf, 0xce, 0xcb, 0xa9, 0x90, 0xcc, 0xcf, + 0x8c, 0x66, 0x22, 0x33, 0x78, 0x33, 0xbd, 0x71, 0x5e, 0x30, 0xd9, 0x0b, 0xe5, 0x6c, 0x72, 0x3c, + 0x86, 0xa5, 0xf8, 0xf4, 0x07, 0xef, 0xde, 0x41, 0x75, 0x28, 0xfa, 0xf3, 0x19, 0x7f, 0x5c, 0x13, + 0x2c, 0xd1, 0xa7, 0x89, 0xa1, 0xd4, 0x85, 0x06, 0x35, 0x31, 0x01, 0xe9, 0xcf, 0x59, 0xa8, 0x25, + 0x30, 0x68, 0x05, 0x2a, 0xfc, 0x5d, 0x64, 0x38, 0xe4, 0x90, 0x07, 0xb7, 0x86, 0x81, 0x93, 0x30, + 0x39, 0xfc, 0x36, 0x53, 0x85, 0xfb, 0x73, 0x82, 0x72, 0xb1, 0xe9, 0x00, 0xba, 0x1f, 0x1f, 0x48, + 0xe4, 0xd3, 0x07, 0x11, 0xbc, 0x05, 0x23, 0xdf, 0x62, 0x33, 0x89, 0x7b, 0xb1, 0x09, 0x19, 0x6f, + 0x7f, 0xcb, 0x89, 0xd7, 0x38, 0xe3, 0x44, 0x92, 0x21, 0xf8, 0xfc, 0x89, 0xc4, 0x3e, 0x54, 0x62, + 0x3b, 0x7e, 0x73, 0xa4, 0xe6, 0x57, 0x6f, 0xe2, 0xb7, 0x2a, 0x97, 0xfa, 0xad, 0x92, 0xfe, 0x92, + 0x85, 0x4a, 0xcc, 0x38, 0xf4, 0x61, 0x62, 0x9e, 0xb0, 0x3a, 0xd7, 0x83, 0xb3, 0xc3, 0x84, 0x6b, + 0x50, 0x3a, 0x26, 0xa3, 0x29, 0x35, 0x8c, 0x6d, 0x51, 0xc3, 0x45, 0xba, 0xc6, 0xe4, 0x90, 0xb2, + 0x66, 0x13, 0xcb, 0x63, 0xac, 0x3c, 0x67, 0xd1, 0x35, 0x26, 0x87, 0xd2, 0xdf, 0x85, 0xc4, 0xb4, + 0xe0, 0x6d, 0x78, 0xab, 0xab, 0xea, 0xb8, 0xad, 0x18, 0xfa, 0xde, 0xb6, 0x6a, 0x0c, 0x7a, 0xda, + 0xb6, 0xaa, 0xb4, 0x37, 0xda, 0x6a, 0x4b, 0xcc, 0xa0, 0xb7, 0xe0, 0x72, 0x9c, 0x19, 0x4d, 0x12, + 0x96, 0xe1, 0x52, 0x9c, 0x11, 0x4c, 0x15, 0xae, 0xc1, 0x72, 0x9c, 0x1c, 0x9f, 0x30, 0x34, 0xe0, + 0xfa, 0x19, 0x89, 0xf8, 0xb4, 0x21, 0xb5, 0x55, 0x34, 0x79, 0xb8, 0x02, 0x62, 0x9c, 0xe1, 0x4f, + 0x21, 0xea, 0x70, 0x25, 0x01, 0x0f, 0x27, 0x12, 0xb7, 0x5f, 0xe7, 0x00, 0xa2, 0xd9, 0x1d, 0xd5, + 0xab, 0x62, 0xdc, 0xc7, 0x86, 0x22, 0x0f, 0x34, 0xea, 0x5f, 0xf0, 0xe8, 0x79, 0x0f, 0xa4, 0x38, + 0x03, 0xab, 0xdb, 0x9d, 0xb6, 0x22, 0x6b, 0x46, 0xab, 0xdd, 0x32, 0x7a, 0x7d, 0xdd, 0xe8, 0xca, + 0xba, 0xb2, 0x25, 0x0a, 0xe8, 0x16, 0xdc, 0x8c, 0xe3, 0xf4, 0x7e, 0xdf, 0xe8, 0xca, 0xbd, 0x3d, + 0x43, 0xe9, 0x0c, 0x34, 0x5d, 0xc5, 0x9a, 0x98, 0xa5, 0xc6, 0xc4, 0x21, 0x4d, 0xb9, 0x65, 0xb4, + 0x64, 0x5d, 0x16, 0x73, 0xe9, 0x4d, 0xda, 0xbd, 0x4d, 0x55, 0xd3, 0xdb, 0xfd, 0x9e, 0x81, 0x65, + 0x5d, 0x35, 0x3a, 0xed, 0x6e, 0x5b, 0x57, 0x5b, 0x62, 0x1e, 0xfd, 0x1f, 0xac, 0x26, 0x8d, 0x79, + 0x34, 0x50, 0x35, 0x3d, 0x89, 0x5a, 0xa0, 0x31, 0x4c, 0x6a, 0xd3, 0x74, 0xb9, 0xa7, 0xf8, 0x08, + 0xb1, 0x80, 0xde, 0x81, 0x95, 0x38, 0x5f, 0x53, 0xf1, 0x4e, 0x5b, 0xa1, 0x3e, 0xcb, 0x3b, 0x72, + 0xbb, 0x23, 0x37, 0x3b, 0xaa, 0x58, 0x44, 0xab, 0x70, 0x23, 0xe1, 0x8f, 0xd6, 0x6a, 0x26, 0x10, + 0xa5, 0xb4, 0x3b, 0xd4, 0xe3, 0xe6, 0x40, 0xdb, 0x13, 0xcb, 0x69, 0x33, 0x95, 0x36, 0x56, 0x06, + 0x6d, 0xdd, 0x68, 0x62, 0x55, 0xfe, 0x5c, 0xc5, 0x46, 0x7f, 0x5b, 0xed, 0x89, 0x80, 0x24, 0x68, + 0xc4, 0x51, 0x5d, 0x55, 0xdf, 0xea, 0xf3, 0x98, 0xca, 0x9d, 0x4e, 0x7f, 0x57, 0x6d, 0x89, 0x15, + 0x74, 0x03, 0xea, 0x89, 0x3d, 0xd4, 0x9e, 0xdc, 0xd3, 0x7d, 0x47, 0xaa, 0xe8, 0x5d, 0xb8, 0x15, + 0xe7, 0xca, 0x8a, 0xde, 0xde, 0x61, 0xfe, 0xb4, 0x55, 0x2d, 0x8c, 0x47, 0xed, 0xf6, 0x67, 0x50, + 0x89, 0xdd, 0xc7, 0xd0, 0x55, 0x40, 0x8f, 0x06, 0x2a, 0xde, 0x63, 0xd9, 0x30, 0xd0, 0x0c, 0xa6, + 0x42, 0xcc, 0x50, 0x7f, 0x12, 0x74, 0x6d, 0xa0, 0x28, 0xaa, 0xa6, 0x89, 0xc2, 0xed, 0xff, 0x64, + 0x61, 0x31, 0x79, 0x3d, 0xa3, 0x29, 0xcc, 0xc1, 0xdc, 0x00, 0x96, 0x5d, 0xbd, 0x7e, 0x4f, 0x15, + 0x33, 0xd4, 0xe6, 0x33, 0x2c, 0xbd, 0xdd, 0x55, 0xfb, 0x03, 0x5d, 0x14, 0xd0, 0x4d, 0xb8, 0x76, + 0x86, 0xab, 0xd0, 0xe3, 0xe9, 0xa8, 0x2d, 0x31, 0x4b, 0xcf, 0xee, 0x0c, 0x5b, 0xfd, 0x91, 0xaa, + 0x0c, 0x68, 0x3a, 0x88, 0xb9, 0xb9, 0xe2, 0x61, 0x22, 0xe5, 0xe7, 0xb2, 0xdb, 0xb4, 0x18, 0x7b, + 0x72, 0x47, 0x5c, 0xa0, 0x87, 0x7a, 0x86, 0x1d, 0x3f, 0xd4, 0xc2, 0xdc, 0xfd, 0xe9, 0x91, 0x6c, + 0xf4, 0x07, 0xbd, 0x96, 0x58, 0xa4, 0xb9, 0x33, 0x97, 0x2f, 0x2b, 0x8a, 0xba, 0xad, 0xfb, 0x99, + 0xf1, 0x1e, 0x48, 0x67, 0x23, 0x10, 0x14, 0x84, 0x9f, 0xb4, 0x1a, 0xcf, 0x93, 0xb9, 0xb8, 0x8e, + 0x8c, 0x37, 0x55, 0x43, 0xed, 0xe9, 0x78, 0x4f, 0x84, 0xe6, 0xa7, 0xcf, 0x5f, 0x36, 0x32, 0x5f, + 0xbd, 0x6c, 0x64, 0xbe, 0x7e, 0xd9, 0x10, 0x7e, 0x76, 0xda, 0x10, 0xfe, 0x74, 0xda, 0x10, 0x9e, + 0x9d, 0x36, 0x84, 0xe7, 0xa7, 0x0d, 0xe1, 0x1f, 0xa7, 0x0d, 0xe1, 0xf5, 0x69, 0x23, 0xf3, 0xf5, + 0x69, 0x43, 0xf8, 0xcd, 0xab, 0x46, 0xe6, 0xf9, 0xab, 0x46, 0xe6, 0xab, 0x57, 0x8d, 0xcc, 0x8f, + 0x8b, 0xec, 0xef, 0x9d, 0xe9, 0xfe, 0x7e, 0x81, 0xfd, 0x83, 0x73, 0xf7, 0xbf, 0x01, 0x00, 0x00, + 0xff, 0xff, 0xc8, 0x23, 0xee, 0x98, 0xf0, 0x19, 0x00, 0x00, } func (x ErrorCause) String() string { @@ -2784,9 +2774,6 @@ func (this *TimeSeries) Equal(that interface{}) bool { return false } } - if this.CreatedTimestamp != that1.CreatedTimestamp { - return false - } return true } func (this *LabelPair) Equal(that interface{}) bool { @@ -2841,6 +2828,9 @@ func (this *Sample) Equal(that interface{}) bool { if this.Value != that1.Value { return false } + if this.StartTimestamp != that1.StartTimestamp { + return false + } return true } func (this *MetricMetadata) Equal(that interface{}) bool { @@ -3048,6 +3038,9 @@ func (this *Histogram) Equal(that interface{}) bool { return false } } + if this.StartTimestamp != that1.StartTimestamp { + return false + } return true } func (this *Histogram_CountInt) Equal(that interface{}) bool { @@ -3878,9 +3871,6 @@ func (this *TimeSeriesRW2) Equal(that interface{}) bool { if !this.Metadata.Equal(&that1.Metadata) { return false } - if this.CreatedTimestamp != that1.CreatedTimestamp { - return false - } return true } func (this *ExemplarRW2) Equal(that interface{}) bool { @@ -3997,7 +3987,7 @@ func (this *TimeSeries) GoString() string { if this == nil { return "nil" } - s := make([]string, 0, 9) + s := make([]string, 0, 8) s = append(s, "&mimirpb.TimeSeries{") s = append(s, "Labels: "+fmt.Sprintf("%#v", this.Labels)+",\n") if this.Samples != nil { @@ -4021,7 +4011,6 @@ func (this *TimeSeries) GoString() string { } s = append(s, "Histograms: "+fmt.Sprintf("%#v", vs)+",\n") } - s = append(s, "CreatedTimestamp: "+fmt.Sprintf("%#v", this.CreatedTimestamp)+",\n") s = append(s, "}") return strings.Join(s, "") } @@ -4040,10 +4029,11 @@ func (this *Sample) GoString() string { if this == nil { return "nil" } - s := make([]string, 0, 6) + s := make([]string, 0, 7) s = append(s, "&mimirpb.Sample{") s = append(s, "TimestampMs: "+fmt.Sprintf("%#v", this.TimestampMs)+",\n") s = append(s, "Value: "+fmt.Sprintf("%#v", this.Value)+",\n") + s = append(s, "StartTimestamp: "+fmt.Sprintf("%#v", this.StartTimestamp)+",\n") s = append(s, "}") return strings.Join(s, "") } @@ -4086,7 +4076,7 @@ func (this *Histogram) GoString() string { if this == nil { return "nil" } - s := make([]string, 0, 20) + s := make([]string, 0, 21) s = append(s, "&mimirpb.Histogram{") if this.Count != nil { s = append(s, "Count: "+fmt.Sprintf("%#v", this.Count)+",\n") @@ -4118,6 +4108,7 @@ func (this *Histogram) GoString() string { s = append(s, "ResetHint: "+fmt.Sprintf("%#v", this.ResetHint)+",\n") s = append(s, "Timestamp: "+fmt.Sprintf("%#v", this.Timestamp)+",\n") s = append(s, "CustomValues: "+fmt.Sprintf("%#v", this.CustomValues)+",\n") + s = append(s, "StartTimestamp: "+fmt.Sprintf("%#v", this.StartTimestamp)+",\n") s = append(s, "}") return strings.Join(s, "") } @@ -4439,7 +4430,7 @@ func (this *TimeSeriesRW2) GoString() string { if this == nil { return "nil" } - s := make([]string, 0, 10) + s := make([]string, 0, 9) s = append(s, "&mimirpb.TimeSeriesRW2{") s = append(s, "LabelsRefs: "+fmt.Sprintf("%#v", this.LabelsRefs)+",\n") if this.Samples != nil { @@ -4464,7 +4455,6 @@ func (this *TimeSeriesRW2) GoString() string { s = append(s, "Exemplars: "+fmt.Sprintf("%#v", vs)+",\n") } s = append(s, "Metadata: "+strings.Replace(this.Metadata.GoString(), `&`, ``, 1)+",\n") - s = append(s, "CreatedTimestamp: "+fmt.Sprintf("%#v", this.CreatedTimestamp)+",\n") s = append(s, "}") return strings.Join(s, "") } @@ -4689,8 +4679,8 @@ func (m *TimeSeries) MarshalToSizedBuffer(dAtA []byte) (int, error) { _ = i var l int _ = l - if m.CreatedTimestamp != 0 { - i = encodeVarintMimir(dAtA, i, uint64(m.CreatedTimestamp)) + if legacyCreatedTimestamp := legacyCreatedTimestampForMarshal(m.Samples, m.Histograms); legacyCreatedTimestamp != 0 { + i = encodeVarintMimir(dAtA, i, uint64(legacyCreatedTimestamp)) i-- dAtA[i] = 0x30 } @@ -4810,6 +4800,11 @@ func (m *Sample) MarshalToSizedBuffer(dAtA []byte) (int, error) { _ = i var l int _ = l + if m.StartTimestamp != 0 { + i = encodeVarintMimir(dAtA, i, uint64(m.StartTimestamp)) + i-- + dAtA[i] = 0x18 + } if m.TimestampMs != 0 { i = encodeVarintMimir(dAtA, i, uint64(m.TimestampMs)) i-- @@ -4978,6 +4973,13 @@ func (m *Histogram) MarshalToSizedBuffer(dAtA []byte) (int, error) { _ = i var l int _ = l + if m.StartTimestamp != 0 { + i = encodeVarintMimir(dAtA, i, uint64(m.StartTimestamp)) + i-- + dAtA[i] = 0x1 + i-- + dAtA[i] = 0x88 + } if len(m.CustomValues) > 0 { for iNdEx := len(m.CustomValues) - 1; iNdEx >= 0; iNdEx-- { f1 := math.Float64bits(float64(m.CustomValues[iNdEx])) @@ -6057,8 +6059,8 @@ func (m *TimeSeriesRW2) MarshalToSizedBuffer(dAtA []byte) (int, error) { _ = i var l int _ = l - if m.CreatedTimestamp != 0 { - i = encodeVarintMimir(dAtA, i, uint64(m.CreatedTimestamp)) + if legacyCreatedTimestamp := legacyCreatedTimestampForMarshal(m.Samples, m.Histograms); legacyCreatedTimestamp != 0 { + i = encodeVarintMimir(dAtA, i, uint64(legacyCreatedTimestamp)) i-- dAtA[i] = 0x30 } @@ -6347,8 +6349,8 @@ func (m *TimeSeries) Size() (n int) { n += 1 + l + sovMimir(uint64(l)) } } - if m.CreatedTimestamp != 0 { - n += 1 + sovMimir(uint64(m.CreatedTimestamp)) + if legacyCreatedTimestamp := legacyCreatedTimestampForMarshal(m.Samples, m.Histograms); legacyCreatedTimestamp != 0 { + n += 1 + sovMimir(uint64(legacyCreatedTimestamp)) } return n } @@ -6382,6 +6384,9 @@ func (m *Sample) Size() (n int) { if m.TimestampMs != 0 { n += 1 + sovMimir(uint64(m.TimestampMs)) } + if m.StartTimestamp != 0 { + n += 1 + sovMimir(uint64(m.StartTimestamp)) + } return n } @@ -6507,6 +6512,9 @@ func (m *Histogram) Size() (n int) { if len(m.CustomValues) > 0 { n += 2 + sovMimir(uint64(len(m.CustomValues)*8)) + len(m.CustomValues)*8 } + if m.StartTimestamp != 0 { + n += 2 + sovMimir(uint64(m.StartTimestamp)) + } return n } @@ -6969,8 +6977,8 @@ func (m *TimeSeriesRW2) Size() (n int) { } l = m.Metadata.Size() n += 1 + l + sovMimir(uint64(l)) - if m.CreatedTimestamp != 0 { - n += 1 + sovMimir(uint64(m.CreatedTimestamp)) + if legacyCreatedTimestamp := legacyCreatedTimestampForMarshal(m.Samples, m.Histograms); legacyCreatedTimestamp != 0 { + n += 1 + sovMimir(uint64(legacyCreatedTimestamp)) } return n } @@ -7092,7 +7100,6 @@ func (this *TimeSeries) String() string { `Samples:` + repeatedStringForSamples + `,`, `Exemplars:` + repeatedStringForExemplars + `,`, `Histograms:` + repeatedStringForHistograms + `,`, - `CreatedTimestamp:` + fmt.Sprintf("%v", this.CreatedTimestamp) + `,`, `}`, }, "") return s @@ -7115,6 +7122,7 @@ func (this *Sample) String() string { s := strings.Join([]string{`&Sample{`, `Value:` + fmt.Sprintf("%v", this.Value) + `,`, `TimestampMs:` + fmt.Sprintf("%v", this.TimestampMs) + `,`, + `StartTimestamp:` + fmt.Sprintf("%v", this.StartTimestamp) + `,`, `}`, }, "") return s @@ -7183,6 +7191,7 @@ func (this *Histogram) String() string { `ResetHint:` + fmt.Sprintf("%v", this.ResetHint) + `,`, `Timestamp:` + fmt.Sprintf("%v", this.Timestamp) + `,`, `CustomValues:` + fmt.Sprintf("%v", this.CustomValues) + `,`, + `StartTimestamp:` + fmt.Sprintf("%v", this.StartTimestamp) + `,`, `}`, }, "") return s @@ -7531,7 +7540,6 @@ func (this *TimeSeriesRW2) String() string { `Histograms:` + repeatedStringForHistograms + `,`, `Exemplars:` + repeatedStringForExemplars + `,`, `Metadata:` + strings.Replace(strings.Replace(this.Metadata.String(), "MetadataRW2", "MetadataRW2", 1), `&`, ``, 1) + `,`, - `CreatedTimestamp:` + fmt.Sprintf("%v", this.CreatedTimestamp) + `,`, `}`, }, "") return s @@ -8002,6 +8010,12 @@ func (m *ErrorDetails) Unmarshal(dAtA []byte) error { return nil } func (m *TimeSeries) Unmarshal(dAtA []byte) error { + // legacyCreatedTimestamp decodes the reserved field 6 (formerly created_timestamp), kept only + // for wire compatibility with senders (including internal distributor->ingester gRPC and + // ingest-storage Kafka records from a not-yet-upgraded Mimir component) still using the + // pre-final Remote Write 2.0 shape. It is fanned out to every Sample/Histogram in this series + // once the rest of the message is decoded, see below. + var legacyCreatedTimestamp int64 l := len(dAtA) iNdEx := 0 for iNdEx < l { @@ -8172,7 +8186,7 @@ func (m *TimeSeries) Unmarshal(dAtA []byte) error { if wireType != 0 { return fmt.Errorf("proto: wrong wireType = %d for field CreatedTimestamp", wireType) } - m.CreatedTimestamp = 0 + legacyCreatedTimestamp = 0 for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowMimir @@ -8182,7 +8196,7 @@ func (m *TimeSeries) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - m.CreatedTimestamp |= int64(b&0x7F) << shift + legacyCreatedTimestamp |= int64(b&0x7F) << shift if b < 0x80 { break } @@ -8206,6 +8220,21 @@ func (m *TimeSeries) Unmarshal(dAtA []byte) error { if iNdEx > l { return io.ErrUnexpectedEOF } + if legacyCreatedTimestamp != 0 { + // Fan the legacy per-series value out to every sample/histogram that doesn't already carry + // its own StartTimestamp, for wire compatibility with senders still using the pre-final + // Remote Write 2.0 shape. + for i := range m.Samples { + if m.Samples[i].StartTimestamp == 0 { + m.Samples[i].StartTimestamp = legacyCreatedTimestamp + } + } + for i := range m.Histograms { + if m.Histograms[i].StartTimestamp == 0 { + m.Histograms[i].StartTimestamp = legacyCreatedTimestamp + } + } + } return nil } func (m *LabelPair) Unmarshal(dAtA []byte) error { @@ -8385,6 +8414,25 @@ func (m *Sample) Unmarshal(dAtA []byte) error { break } } + case 3: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field StartTimestamp", wireType) + } + m.StartTimestamp = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowMimir + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + m.StartTimestamp |= int64(b&0x7F) << shift + if b < 0x80 { + break + } + } default: iNdEx = preIndex skippy, err := skipMimir(dAtA[iNdEx:]) @@ -9327,6 +9375,25 @@ func (m *Histogram) Unmarshal(dAtA []byte) error { } else { return fmt.Errorf("proto: wrong wireType = %d for field CustomValues", wireType) } + case 17: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field StartTimestamp", wireType) + } + m.StartTimestamp = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowMimir + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + m.StartTimestamp |= int64(b&0x7F) << shift + if b < 0x80 { + break + } + } default: iNdEx = preIndex skippy, err := skipMimir(dAtA[iNdEx:]) @@ -11524,6 +11591,11 @@ func (m *TimeSeriesRW2) Unmarshal(dAtA []byte) error { } func (m *TimeSeries) UnmarshalRW2(dAtA []byte, symbols *rw2PagedSymbols, metadata metadataSet, skipNormalizeMetricName bool) error { var metricName string + // legacyCreatedTimestamp decodes the reserved field 6 (formerly created_timestamp), kept only + // for wire compatibility with senders still using the pre-final Remote Write 2.0 shape. It is + // fanned out to every Sample/Histogram in this series once the rest of the message is decoded, + // see below. + var legacyCreatedTimestamp int64 l := len(dAtA) iNdEx := 0 for iNdEx < l { @@ -11779,7 +11851,7 @@ func (m *TimeSeries) UnmarshalRW2(dAtA []byte, symbols *rw2PagedSymbols, metadat if wireType != 0 { return fmt.Errorf("proto: wrong wireType = %d for field CreatedTimestamp", wireType) } - m.CreatedTimestamp = 0 + legacyCreatedTimestamp = 0 for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowMimir @@ -11789,7 +11861,7 @@ func (m *TimeSeries) UnmarshalRW2(dAtA []byte, symbols *rw2PagedSymbols, metadat } b := dAtA[iNdEx] iNdEx++ - m.CreatedTimestamp |= int64(b&0x7F) << shift + legacyCreatedTimestamp |= int64(b&0x7F) << shift if b < 0x80 { break } @@ -11813,6 +11885,21 @@ func (m *TimeSeries) UnmarshalRW2(dAtA []byte, symbols *rw2PagedSymbols, metadat if iNdEx > l { return io.ErrUnexpectedEOF } + if legacyCreatedTimestamp != 0 { + // Fan the legacy per-series value out to every sample/histogram that doesn't already carry + // its own StartTimestamp, for wire compatibility with senders still using the pre-final + // Remote Write 2.0 shape. + for i := range m.Samples { + if m.Samples[i].StartTimestamp == 0 { + m.Samples[i].StartTimestamp = legacyCreatedTimestamp + } + } + for i := range m.Histograms { + if m.Histograms[i].StartTimestamp == 0 { + m.Histograms[i].StartTimestamp = legacyCreatedTimestamp + } + } + } return nil } func (m *ExemplarRW2) Unmarshal(dAtA []byte) error { diff --git a/pkg/mimirpb/mimir.pb.go.expdiff b/pkg/mimirpb/mimir.pb.go.expdiff index 7c2239da892..d65638c527f 100644 --- a/pkg/mimirpb/mimir.pb.go.expdiff +++ b/pkg/mimirpb/mimir.pb.go.expdiff @@ -1,5 +1,5 @@ diff --git a/pkg/mimirpb/mimir.pb.go b/pkg/mimirpb/mimir.pb.go -index 464997d2cd..ff23608339 100644 +index e5cdebc4d0..88e26dc84c 100644 --- a/pkg/mimirpb/mimir.pb.go +++ b/pkg/mimirpb/mimir.pb.go @@ -14,7 +14,6 @@ import ( @@ -42,7 +42,7 @@ index 464997d2cd..ff23608339 100644 } func (m *WriteRequest) Reset() { *m = WriteRequest{} } -@@ -487,11 +468,6 @@ func (m *ErrorDetails) GetRejectedSamples() int64 { +@@ -487,20 +468,12 @@ func (m *ErrorDetails) GetRejectedSamples() int64 { return 0 } @@ -54,17 +54,40 @@ index 464997d2cd..ff23608339 100644 type TimeSeries struct { Labels []UnsafeMutableLabel `protobuf:"bytes,1,rep,name=labels,proto3,customtype=UnsafeMutableLabel" json:"labels"` // Sorted by time, oldest sample first. -@@ -504,9 +480,6 @@ type TimeSeries struct { - // Zero value means value not set. If you need to use exactly zero value for - // the timestamp, use 1 millisecond before or after. - CreatedTimestamp int64 `protobuf:"varint,6,opt,name=created_timestamp,json=createdTimestamp,proto3" json:"created_timestamp,omitempty"` + Samples []Sample `protobuf:"bytes,2,rep,name=samples,proto3" json:"samples"` + Exemplars []Exemplar `protobuf:"bytes,3,rep,name=exemplars,proto3" json:"exemplars"` + Histograms []Histogram `protobuf:"bytes,4,rep,name=histograms,proto3" json:"histograms"` - - // Skip unmarshaling of exemplars. - SkipUnmarshalingExemplars bool } func (m *TimeSeries) Reset() { *m = TimeSeries{} } -@@ -6115,25 +6088,19 @@ func (m *TimeSeriesRW2) MarshalToSizedBuffer(dAtA []byte) (int, error) { +@@ -4679,11 +4652,6 @@ func (m *TimeSeries) MarshalToSizedBuffer(dAtA []byte) (int, error) { + _ = i + var l int + _ = l +- if legacyCreatedTimestamp := legacyCreatedTimestampForMarshal(m.Samples, m.Histograms); legacyCreatedTimestamp != 0 { +- i = encodeVarintMimir(dAtA, i, uint64(legacyCreatedTimestamp)) +- i-- +- dAtA[i] = 0x30 +- } + if len(m.Histograms) > 0 { + for iNdEx := len(m.Histograms) - 1; iNdEx >= 0; iNdEx-- { + { +@@ -6059,11 +6027,6 @@ func (m *TimeSeriesRW2) MarshalToSizedBuffer(dAtA []byte) (int, error) { + _ = i + var l int + _ = l +- if legacyCreatedTimestamp := legacyCreatedTimestampForMarshal(m.Samples, m.Histograms); legacyCreatedTimestamp != 0 { +- i = encodeVarintMimir(dAtA, i, uint64(legacyCreatedTimestamp)) +- i-- +- dAtA[i] = 0x30 +- } + { + size, err := m.Metadata.MarshalToSizedBuffer(dAtA[:i]) + if err != nil { +@@ -6117,25 +6080,19 @@ func (m *TimeSeriesRW2) MarshalToSizedBuffer(dAtA []byte) (int, error) { } } if len(m.LabelsRefs) > 0 { @@ -95,7 +118,7 @@ index 464997d2cd..ff23608339 100644 i = encodeVarintMimir(dAtA, i, uint64(j21)) i-- dAtA[i] = 0xa -@@ -6173,25 +6140,19 @@ func (m *ExemplarRW2) MarshalToSizedBuffer(dAtA []byte) (int, error) { +@@ -6175,25 +6132,19 @@ func (m *ExemplarRW2) MarshalToSizedBuffer(dAtA []byte) (int, error) { dAtA[i] = 0x11 } if len(m.LabelsRefs) > 0 { @@ -126,7 +149,27 @@ index 464997d2cd..ff23608339 100644 i = encodeVarintMimir(dAtA, i, uint64(j23)) i-- dAtA[i] = 0xa -@@ -7569,9 +7530,6 @@ func valueToStringMimir(v interface{}) string { +@@ -6349,9 +6300,6 @@ func (m *TimeSeries) Size() (n int) { + n += 1 + l + sovMimir(uint64(l)) + } + } +- if legacyCreatedTimestamp := legacyCreatedTimestampForMarshal(m.Samples, m.Histograms); legacyCreatedTimestamp != 0 { +- n += 1 + sovMimir(uint64(legacyCreatedTimestamp)) +- } + return n + } + +@@ -6977,9 +6925,6 @@ func (m *TimeSeriesRW2) Size() (n int) { + } + l = m.Metadata.Size() + n += 1 + l + sovMimir(uint64(l)) +- if legacyCreatedTimestamp := legacyCreatedTimestampForMarshal(m.Samples, m.Histograms); legacyCreatedTimestamp != 0 { +- n += 1 + sovMimir(uint64(legacyCreatedTimestamp)) +- } + return n + } + +@@ -7577,9 +7522,6 @@ func valueToStringMimir(v interface{}) string { return fmt.Sprintf("*%v", pv) } func (m *WriteRequest) Unmarshal(dAtA []byte) error { @@ -136,7 +179,7 @@ index 464997d2cd..ff23608339 100644 l := len(dAtA) iNdEx := 0 for iNdEx < l { -@@ -7601,9 +7559,6 @@ func (m *WriteRequest) Unmarshal(dAtA []byte) error { +@@ -7609,9 +7551,6 @@ func (m *WriteRequest) Unmarshal(dAtA []byte) error { } switch fieldNum { case 1: @@ -146,7 +189,7 @@ index 464997d2cd..ff23608339 100644 if wireType != 2 { return fmt.Errorf("proto: wrong wireType = %d for field Timeseries", wireType) } -@@ -7633,8 +7588,7 @@ func (m *WriteRequest) Unmarshal(dAtA []byte) error { +@@ -7641,8 +7580,7 @@ func (m *WriteRequest) Unmarshal(dAtA []byte) error { return io.ErrUnexpectedEOF } m.Timeseries = append(m.Timeseries, PreallocTimeseries{}) @@ -156,7 +199,7 @@ index 464997d2cd..ff23608339 100644 return err } iNdEx = postIndex -@@ -7658,9 +7612,6 @@ func (m *WriteRequest) Unmarshal(dAtA []byte) error { +@@ -7666,9 +7604,6 @@ func (m *WriteRequest) Unmarshal(dAtA []byte) error { } } case 3: @@ -166,7 +209,7 @@ index 464997d2cd..ff23608339 100644 if wireType != 2 { return fmt.Errorf("proto: wrong wireType = %d for field Metadata", wireType) } -@@ -7695,9 +7646,6 @@ func (m *WriteRequest) Unmarshal(dAtA []byte) error { +@@ -7703,9 +7638,6 @@ func (m *WriteRequest) Unmarshal(dAtA []byte) error { } iNdEx = postIndex case 4: @@ -176,7 +219,7 @@ index 464997d2cd..ff23608339 100644 if wireType != 2 { return fmt.Errorf("proto: wrong wireType = %d for field SymbolsRW2", wireType) } -@@ -7727,16 +7675,9 @@ func (m *WriteRequest) Unmarshal(dAtA []byte) error { +@@ -7735,16 +7667,9 @@ func (m *WriteRequest) Unmarshal(dAtA []byte) error { if postIndex > l { return io.ErrUnexpectedEOF } @@ -194,7 +237,7 @@ index 464997d2cd..ff23608339 100644 if wireType != 2 { return fmt.Errorf("proto: wrong wireType = %d for field TimeseriesRW2", wireType) } -@@ -7765,12 +7706,8 @@ func (m *WriteRequest) Unmarshal(dAtA []byte) error { +@@ -7773,12 +7698,8 @@ func (m *WriteRequest) Unmarshal(dAtA []byte) error { if postIndex > l { return io.ErrUnexpectedEOF } @@ -209,7 +252,7 @@ index 464997d2cd..ff23608339 100644 return err } iNdEx = postIndex -@@ -7833,14 +7770,6 @@ func (m *WriteRequest) Unmarshal(dAtA []byte) error { +@@ -7841,14 +7762,6 @@ func (m *WriteRequest) Unmarshal(dAtA []byte) error { if iNdEx > l { return io.ErrUnexpectedEOF } @@ -224,7 +267,20 @@ index 464997d2cd..ff23608339 100644 return nil } func (m *WriteResponse) Unmarshal(dAtA []byte) error { -@@ -8127,11 +8056,9 @@ func (m *TimeSeries) Unmarshal(dAtA []byte) error { +@@ -8010,12 +7923,6 @@ func (m *ErrorDetails) Unmarshal(dAtA []byte) error { + return nil + } + func (m *TimeSeries) Unmarshal(dAtA []byte) error { +- // legacyCreatedTimestamp decodes the reserved field 6 (formerly created_timestamp), kept only +- // for wire compatibility with senders (including internal distributor->ingester gRPC and +- // ingest-storage Kafka records from a not-yet-upgraded Mimir component) still using the +- // pre-final Remote Write 2.0 shape. It is fanned out to every Sample/Histogram in this series +- // once the rest of the message is decoded, see below. +- var legacyCreatedTimestamp int64 + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { +@@ -8141,11 +8048,9 @@ func (m *TimeSeries) Unmarshal(dAtA []byte) error { if postIndex > l { return io.ErrUnexpectedEOF } @@ -239,7 +295,55 @@ index 464997d2cd..ff23608339 100644 } iNdEx = postIndex case 4: -@@ -11520,10 +11447,6 @@ func (m *WriteRequestRW2) Unmarshal(dAtA []byte) error { +@@ -8182,25 +8087,6 @@ func (m *TimeSeries) Unmarshal(dAtA []byte) error { + return err + } + iNdEx = postIndex +- case 6: +- if wireType != 0 { +- return fmt.Errorf("proto: wrong wireType = %d for field CreatedTimestamp", wireType) +- } +- legacyCreatedTimestamp = 0 +- for shift := uint(0); ; shift += 7 { +- if shift >= 64 { +- return ErrIntOverflowMimir +- } +- if iNdEx >= l { +- return io.ErrUnexpectedEOF +- } +- b := dAtA[iNdEx] +- iNdEx++ +- legacyCreatedTimestamp |= int64(b&0x7F) << shift +- if b < 0x80 { +- break +- } +- } + default: + iNdEx = preIndex + skippy, err := skipMimir(dAtA[iNdEx:]) +@@ -8220,21 +8106,6 @@ func (m *TimeSeries) Unmarshal(dAtA []byte) error { + if iNdEx > l { + return io.ErrUnexpectedEOF + } +- if legacyCreatedTimestamp != 0 { +- // Fan the legacy per-series value out to every sample/histogram that doesn't already carry +- // its own StartTimestamp, for wire compatibility with senders still using the pre-final +- // Remote Write 2.0 shape. +- for i := range m.Samples { +- if m.Samples[i].StartTimestamp == 0 { +- m.Samples[i].StartTimestamp = legacyCreatedTimestamp +- } +- } +- for i := range m.Histograms { +- if m.Histograms[i].StartTimestamp == 0 { +- m.Histograms[i].StartTimestamp = legacyCreatedTimestamp +- } +- } +- } + return nil + } + func (m *LabelPair) Unmarshal(dAtA []byte) error { +@@ -11587,15 +11458,6 @@ func (m *WriteRequestRW2) Unmarshal(dAtA []byte) error { return nil } func (m *TimeSeriesRW2) Unmarshal(dAtA []byte) error { @@ -247,10 +351,15 @@ index 464997d2cd..ff23608339 100644 -} -func (m *TimeSeries) UnmarshalRW2(dAtA []byte, symbols *rw2PagedSymbols, metadata metadataSet, skipNormalizeMetricName bool) error { - var metricName string +- // legacyCreatedTimestamp decodes the reserved field 6 (formerly created_timestamp), kept only +- // for wire compatibility with senders still using the pre-final Remote Write 2.0 shape. It is +- // fanned out to every Sample/Histogram in this series once the rest of the message is decoded, +- // see below. +- var legacyCreatedTimestamp int64 l := len(dAtA) iNdEx := 0 for iNdEx < l { -@@ -11554,7 +11477,22 @@ func (m *TimeSeries) UnmarshalRW2(dAtA []byte, symbols *rw2PagedSymbols, metadat +@@ -11626,7 +11488,22 @@ func (m *TimeSeries) UnmarshalRW2(dAtA []byte, symbols *rw2PagedSymbols, metadat switch fieldNum { case 1: if wireType == 0 { @@ -274,7 +383,7 @@ index 464997d2cd..ff23608339 100644 } else if wireType == 2 { var packedLen int for shift := uint(0); ; shift += 7 { -@@ -11589,14 +11527,9 @@ func (m *TimeSeries) UnmarshalRW2(dAtA []byte, symbols *rw2PagedSymbols, metadat +@@ -11661,14 +11538,9 @@ func (m *TimeSeries) UnmarshalRW2(dAtA []byte, symbols *rw2PagedSymbols, metadat } } elementCount = count @@ -291,7 +400,7 @@ index 464997d2cd..ff23608339 100644 for iNdEx < postIndex { var v uint32 for shift := uint(0); ; shift += 7 { -@@ -11613,27 +11546,7 @@ func (m *TimeSeries) UnmarshalRW2(dAtA []byte, symbols *rw2PagedSymbols, metadat +@@ -11685,27 +11557,7 @@ func (m *TimeSeries) UnmarshalRW2(dAtA []byte, symbols *rw2PagedSymbols, metadat break } } @@ -320,7 +429,7 @@ index 464997d2cd..ff23608339 100644 } } else { return fmt.Errorf("proto: wrong wireType = %d for field LabelsRefs", wireType) -@@ -11735,11 +11648,9 @@ func (m *TimeSeries) UnmarshalRW2(dAtA []byte, symbols *rw2PagedSymbols, metadat +@@ -11807,11 +11659,9 @@ func (m *TimeSeries) UnmarshalRW2(dAtA []byte, symbols *rw2PagedSymbols, metadat if postIndex > l { return io.ErrUnexpectedEOF } @@ -335,7 +444,7 @@ index 464997d2cd..ff23608339 100644 } iNdEx = postIndex case 5: -@@ -11771,7 +11682,7 @@ func (m *TimeSeries) UnmarshalRW2(dAtA []byte, symbols *rw2PagedSymbols, metadat +@@ -11843,29 +11693,10 @@ func (m *TimeSeries) UnmarshalRW2(dAtA []byte, symbols *rw2PagedSymbols, metadat if postIndex > l { return io.ErrUnexpectedEOF } @@ -344,7 +453,47 @@ index 464997d2cd..ff23608339 100644 return err } iNdEx = postIndex -@@ -11816,10 +11727,6 @@ func (m *TimeSeries) UnmarshalRW2(dAtA []byte, symbols *rw2PagedSymbols, metadat +- case 6: +- if wireType != 0 { +- return fmt.Errorf("proto: wrong wireType = %d for field CreatedTimestamp", wireType) +- } +- legacyCreatedTimestamp = 0 +- for shift := uint(0); ; shift += 7 { +- if shift >= 64 { +- return ErrIntOverflowMimir +- } +- if iNdEx >= l { +- return io.ErrUnexpectedEOF +- } +- b := dAtA[iNdEx] +- iNdEx++ +- legacyCreatedTimestamp |= int64(b&0x7F) << shift +- if b < 0x80 { +- break +- } +- } + default: + iNdEx = preIndex + skippy, err := skipMimir(dAtA[iNdEx:]) +@@ -11885,28 +11716,9 @@ func (m *TimeSeries) UnmarshalRW2(dAtA []byte, symbols *rw2PagedSymbols, metadat + if iNdEx > l { + return io.ErrUnexpectedEOF + } +- if legacyCreatedTimestamp != 0 { +- // Fan the legacy per-series value out to every sample/histogram that doesn't already carry +- // its own StartTimestamp, for wire compatibility with senders still using the pre-final +- // Remote Write 2.0 shape. +- for i := range m.Samples { +- if m.Samples[i].StartTimestamp == 0 { +- m.Samples[i].StartTimestamp = legacyCreatedTimestamp +- } +- } +- for i := range m.Histograms { +- if m.Histograms[i].StartTimestamp == 0 { +- m.Histograms[i].StartTimestamp = legacyCreatedTimestamp +- } +- } +- } return nil } func (m *ExemplarRW2) Unmarshal(dAtA []byte) error { @@ -355,7 +504,7 @@ index 464997d2cd..ff23608339 100644 l := len(dAtA) iNdEx := 0 for iNdEx < l { -@@ -11850,7 +11757,22 @@ func (m *Exemplar) UnmarshalRW2(dAtA []byte, symbols *rw2PagedSymbols) error { +@@ -11937,7 +11749,22 @@ func (m *Exemplar) UnmarshalRW2(dAtA []byte, symbols *rw2PagedSymbols) error { switch fieldNum { case 1: if wireType == 0 { @@ -379,23 +528,23 @@ index 464997d2cd..ff23608339 100644 } else if wireType == 2 { var packedLen int for shift := uint(0); ; shift += 7 { -@@ -11885,13 +11807,9 @@ func (m *Exemplar) UnmarshalRW2(dAtA []byte, symbols *rw2PagedSymbols) error { +@@ -11972,13 +11799,9 @@ func (m *Exemplar) UnmarshalRW2(dAtA []byte, symbols *rw2PagedSymbols) error { } } elementCount = count - if elementCount%2 != 0 { - return errorOddNumberOfExemplarLabelRefs -- } -- if elementCount != 0 && len(m.Labels) == 0 { -- m.Labels = make([]LabelAdapter, 0, elementCount/2) + if elementCount != 0 && len(m.LabelsRefs) == 0 { + m.LabelsRefs = make([]uint32, 0, elementCount) } +- if elementCount != 0 && len(m.Labels) == 0 { +- m.Labels = make([]LabelAdapter, 0, elementCount/2) +- } - idx := 0 for iNdEx < postIndex { var v uint32 for shift := uint(0); ; shift += 7 { -@@ -11908,20 +11826,7 @@ func (m *Exemplar) UnmarshalRW2(dAtA []byte, symbols *rw2PagedSymbols) error { +@@ -11995,20 +11818,7 @@ func (m *Exemplar) UnmarshalRW2(dAtA []byte, symbols *rw2PagedSymbols) error { break } } @@ -417,7 +566,7 @@ index 464997d2cd..ff23608339 100644 } } else { return fmt.Errorf("proto: wrong wireType = %d for field LabelsRefs", wireType) -@@ -11941,7 +11846,7 @@ func (m *Exemplar) UnmarshalRW2(dAtA []byte, symbols *rw2PagedSymbols) error { +@@ -12028,7 +11838,7 @@ func (m *Exemplar) UnmarshalRW2(dAtA []byte, symbols *rw2PagedSymbols) error { if wireType != 0 { return fmt.Errorf("proto: wrong wireType = %d for field Timestamp", wireType) } @@ -426,7 +575,7 @@ index 464997d2cd..ff23608339 100644 for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowMimir -@@ -11951,7 +11856,7 @@ func (m *Exemplar) UnmarshalRW2(dAtA []byte, symbols *rw2PagedSymbols) error { +@@ -12038,7 +11848,7 @@ func (m *Exemplar) UnmarshalRW2(dAtA []byte, symbols *rw2PagedSymbols) error { } b := dAtA[iNdEx] iNdEx++ @@ -435,7 +584,7 @@ index 464997d2cd..ff23608339 100644 if b < 0x80 { break } -@@ -11978,16 +11883,6 @@ func (m *Exemplar) UnmarshalRW2(dAtA []byte, symbols *rw2PagedSymbols) error { +@@ -12065,16 +11875,6 @@ func (m *Exemplar) UnmarshalRW2(dAtA []byte, symbols *rw2PagedSymbols) error { return nil } func (m *MetadataRW2) Unmarshal(dAtA []byte) error { @@ -452,7 +601,7 @@ index 464997d2cd..ff23608339 100644 l := len(dAtA) iNdEx := 0 for iNdEx < l { -@@ -12020,7 +11915,7 @@ func MetricMetadataUnmarshalRW2(dAtA []byte, symbols *rw2PagedSymbols, metadata +@@ -12107,7 +11907,7 @@ func MetricMetadataUnmarshalRW2(dAtA []byte, symbols *rw2PagedSymbols, metadata if wireType != 0 { return fmt.Errorf("proto: wrong wireType = %d for field Type", wireType) } @@ -461,7 +610,7 @@ index 464997d2cd..ff23608339 100644 for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowMimir -@@ -12030,7 +11925,7 @@ func MetricMetadataUnmarshalRW2(dAtA []byte, symbols *rw2PagedSymbols, metadata +@@ -12117,7 +11917,7 @@ func MetricMetadataUnmarshalRW2(dAtA []byte, symbols *rw2PagedSymbols, metadata } b := dAtA[iNdEx] iNdEx++ @@ -470,7 +619,7 @@ index 464997d2cd..ff23608339 100644 if b < 0x80 { break } -@@ -12039,7 +11934,7 @@ func MetricMetadataUnmarshalRW2(dAtA []byte, symbols *rw2PagedSymbols, metadata +@@ -12126,7 +11926,7 @@ func MetricMetadataUnmarshalRW2(dAtA []byte, symbols *rw2PagedSymbols, metadata if wireType != 0 { return fmt.Errorf("proto: wrong wireType = %d for field HelpRef", wireType) } @@ -479,7 +628,7 @@ index 464997d2cd..ff23608339 100644 for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowMimir -@@ -12049,20 +11944,16 @@ func MetricMetadataUnmarshalRW2(dAtA []byte, symbols *rw2PagedSymbols, metadata +@@ -12136,20 +11936,16 @@ func MetricMetadataUnmarshalRW2(dAtA []byte, symbols *rw2PagedSymbols, metadata } b := dAtA[iNdEx] iNdEx++ @@ -502,7 +651,7 @@ index 464997d2cd..ff23608339 100644 for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowMimir -@@ -12072,15 +11963,11 @@ func MetricMetadataUnmarshalRW2(dAtA []byte, symbols *rw2PagedSymbols, metadata +@@ -12159,15 +11955,11 @@ func MetricMetadataUnmarshalRW2(dAtA []byte, symbols *rw2PagedSymbols, metadata } b := dAtA[iNdEx] iNdEx++ @@ -519,7 +668,7 @@ index 464997d2cd..ff23608339 100644 default: iNdEx = preIndex skippy, err := skipMimir(dAtA[iNdEx:]) -@@ -12100,23 +11987,6 @@ func MetricMetadataUnmarshalRW2(dAtA []byte, symbols *rw2PagedSymbols, metadata +@@ -12187,23 +11979,6 @@ func MetricMetadataUnmarshalRW2(dAtA []byte, symbols *rw2PagedSymbols, metadata if iNdEx > l { return io.ErrUnexpectedEOF } diff --git a/pkg/mimirpb/mimir.proto b/pkg/mimirpb/mimir.proto index c84b526f87a..0870a6628e7 100644 --- a/pkg/mimirpb/mimir.proto +++ b/pkg/mimirpb/mimir.proto @@ -78,12 +78,10 @@ message TimeSeries { repeated Exemplar exemplars = 3 [(gogoproto.nullable) = false]; repeated Histogram histograms = 4 [(gogoproto.nullable) = false]; - // Copy from remote write 2.0. - // Note that the "optional" keyword is omitted due to - // https://cloud.google.com/apis/design/design_patterns.md#optional_primitive_fields - // Zero value means value not set. If you need to use exactly zero value for - // the timestamp, use 1 millisecond before or after. - int64 created_timestamp = 6; + // Formerly: created_timestamp. Moved to Sample.start_timestamp and Histogram.start_timestamp, + // matching the Remote Write 2.0 spec. + reserved 6; + reserved "created_timestamp"; } message LabelPair { @@ -96,6 +94,13 @@ message LabelPair { message Sample { int64 timestamp_ms = 2; double value = 1; + + // Copy from remote write 2.0. + // Note that the "optional" keyword is omitted due to + // https://cloud.google.com/apis/design/design_patterns.md#optional_primitive_fields + // Zero value means value not set. If you need to use exactly zero value for + // the timestamp, use 1 millisecond before or after. + int64 start_timestamp = 3; } message MetricMetadata { @@ -189,6 +194,13 @@ message Histogram { // Used only for converting from OpenTelemetry to Prometheus internally and // to unmarshal Remote Write 2.0 messages. repeated double custom_values = 16; + + // Copy from remote write 2.0. + // Note that the "optional" keyword is omitted due to + // https://cloud.google.com/apis/design/design_patterns.md#optional_primitive_fields + // Zero value means value not set. If you need to use exactly zero value for + // the timestamp, use 1 millisecond before or after. + int64 start_timestamp = 17; } // @@ -382,23 +394,10 @@ message TimeSeriesRW2 { // metadata represents the metadata associated with the given series' samples. MetadataRW2 metadata = 5 [(gogoproto.nullable) = false]; - // created_timestamp represents an optional created timestamp associated with - // this series' samples in ms format, typically for counter or histogram type - // metrics. Created timestamp represents the time when the counter started - // counting (sometimes referred to as start timestamp), which can increase - // the accuracy of query results. - // - // Note that some receivers might require this and in return fail to - // ingest such samples within the Request. - // - // For Go, see github.com/prometheus/prometheus/model/timestamp/timestamp.go - // for conversion from/to time.Time to Prometheus timestamp. - // - // Note that the "optional" keyword is omitted due to - // https://cloud.google.com/apis/design/design_patterns.md#optional_primitive_fields - // Zero value means value not set. If you need to use exactly zero value for - // the timestamp, use 1 millisecond before or after. - int64 created_timestamp = 6; + // Formerly: created_timestamp. Moved to Sample.start_timestamp and Histogram.start_timestamp, + // matching the Remote Write 2.0 spec. + reserved 6; + reserved "created_timestamp"; } message ExemplarRW2 { diff --git a/pkg/mimirpb/prealloc_rw2.go b/pkg/mimirpb/prealloc_rw2.go index 73d27117d31..72764d5ff56 100644 --- a/pkg/mimirpb/prealloc_rw2.go +++ b/pkg/mimirpb/prealloc_rw2.go @@ -158,12 +158,11 @@ func FromWriteRequestToRW2Request(rw1 *WriteRequest, commonSymbols *CommonSymbol } rw2Timeseries = append(rw2Timeseries, TimeSeriesRW2{ - LabelsRefs: refs, - Samples: ts.Samples, - Histograms: ts.Histograms, - Exemplars: FromExemplarsToExemplarsRW2(ts.Exemplars, symbols), - Metadata: metadata, - CreatedTimestamp: ts.CreatedTimestamp, + LabelsRefs: refs, + Samples: ts.Samples, + Histograms: ts.Histograms, + Exemplars: FromExemplarsToExemplarsRW2(ts.Exemplars, symbols), + Metadata: metadata, }) } diff --git a/pkg/mimirpb/prealloc_rw2_test.go b/pkg/mimirpb/prealloc_rw2_test.go index 2ca183e76f4..adb45e0e931 100644 --- a/pkg/mimirpb/prealloc_rw2_test.go +++ b/pkg/mimirpb/prealloc_rw2_test.go @@ -76,17 +76,17 @@ func TestWriteRequestRW2Conversion(t *testing.T) { } }) - t.Run("TimeSeries CreatedTimestamp", func(t *testing.T) { + t.Run("Sample and Histogram StartTimestamp", func(t *testing.T) { req := &WriteRequest{ Timeseries: []PreallocTimeseries{ { TimeSeries: &TimeSeries{ - CreatedTimestamp: 1234567890, + Samples: []Sample{{Value: 1, TimestampMs: 10, StartTimestamp: 5}}, }, }, { TimeSeries: &TimeSeries{ - CreatedTimestamp: 1234567900, + Histograms: []Histogram{{Timestamp: 20, StartTimestamp: 15}}, }, }, }, @@ -96,8 +96,8 @@ func TestWriteRequestRW2Conversion(t *testing.T) { require.NoError(t, err) require.Len(t, rw2.TimeseriesRW2, 2) - require.Equal(t, int64(1234567890), rw2.TimeseriesRW2[0].CreatedTimestamp) - require.Equal(t, int64(1234567900), rw2.TimeseriesRW2[1].CreatedTimestamp) + require.Equal(t, int64(5), rw2.TimeseriesRW2[0].Samples[0].StartTimestamp) + require.Equal(t, int64(15), rw2.TimeseriesRW2[1].Histograms[0].StartTimestamp) }) t.Run("samples", func(t *testing.T) { diff --git a/pkg/mimirpb/split_test.go b/pkg/mimirpb/split_test.go index 60e6250c4d6..6d7e3d9f686 100644 --- a/pkg/mimirpb/split_test.go +++ b/pkg/mimirpb/split_test.go @@ -830,11 +830,10 @@ func mergeRW2s(partials []*WriteRequest) *WriteRequest { } newTS := TimeSeriesRW2{ - LabelsRefs: newLbls, - Samples: ts.Samples, - Exemplars: newExemplars, - Histograms: ts.Histograms, - CreatedTimestamp: ts.CreatedTimestamp, + LabelsRefs: newLbls, + Samples: ts.Samples, + Exemplars: newExemplars, + Histograms: ts.Histograms, Metadata: MetadataRW2{ Type: ts.Metadata.Type, HelpRef: helpRef, diff --git a/pkg/mimirpb/timeseries.go b/pkg/mimirpb/timeseries.go index f812dd94ff5..a7295a3af00 100644 --- a/pkg/mimirpb/timeseries.go +++ b/pkg/mimirpb/timeseries.go @@ -600,7 +600,6 @@ func DeepCopyTimeseries(dst, src PreallocTimeseries, keepHistograms, keepExempla dstTs.Labels, buf = copyToYoloLabels(buf, dstTs.Labels, srcTs.Labels) // Copy scalar properties. - dstTs.CreatedTimestamp = srcTs.CreatedTimestamp dstTs.SkipUnmarshalingExemplars = srcTs.SkipUnmarshalingExemplars // Copy the samples. @@ -748,6 +747,7 @@ func copyHistogram(src Histogram) Histogram { ResetHint: src.ResetHint, Timestamp: src.Timestamp, CustomValues: slices.Clone(src.CustomValues), + StartTimestamp: src.StartTimestamp, } } diff --git a/pkg/mimirpb/timeseries_pools.go b/pkg/mimirpb/timeseries_pools.go index f4d15ebaff8..41e9087544f 100644 --- a/pkg/mimirpb/timeseries_pools.go +++ b/pkg/mimirpb/timeseries_pools.go @@ -49,7 +49,7 @@ func TimeseriesFromPool() *TimeSeries { // Panic if the pool returns a TimeSeries that wasn't properly cleaned, // which is indicative of a hard bug that we want to catch as soon as possible. - if len(ts.Labels) > 0 || len(ts.Samples) > 0 || len(ts.Histograms) > 0 || len(ts.Exemplars) > 0 || ts.CreatedTimestamp != 0 || ts.SkipUnmarshalingExemplars { + if len(ts.Labels) > 0 || len(ts.Samples) > 0 || len(ts.Histograms) > 0 || len(ts.Exemplars) > 0 || ts.SkipUnmarshalingExemplars { panic("pool returned dirty TimeSeries: this indicates a bug where ReuseTimeseries was called on a TimeSeries still in use") } @@ -86,7 +86,6 @@ func ReuseTimeseries(ts *TimeSeries) { ts.Histograms = ts.Histograms[:0] } - ts.CreatedTimestamp = 0 ts.SkipUnmarshalingExemplars = false ClearExemplars(ts) diff --git a/pkg/mimirpb/timeseries_pools_test.go b/pkg/mimirpb/timeseries_pools_test.go index 99014299b5a..08562ca915d 100644 --- a/pkg/mimirpb/timeseries_pools_test.go +++ b/pkg/mimirpb/timeseries_pools_test.go @@ -38,7 +38,6 @@ func TestTimeseriesFromPool(t *testing.T) { {"samples", &TimeSeries{Samples: []Sample{{Value: 1, TimestampMs: 2}}}}, {"histograms", &TimeSeries{Histograms: []Histogram{{Sum: 1.0}}}}, {"exemplars", &TimeSeries{Exemplars: []Exemplar{{Value: 1, TimestampMs: 2}}}}, - {"CreatedTimestamp", &TimeSeries{CreatedTimestamp: 1234567890}}, {"SkipUnmarshalingExemplars", &TimeSeries{SkipUnmarshalingExemplars: true}}, } for _, tc := range dirtyPoolTests { diff --git a/pkg/mimirpb/timeseries_test.go b/pkg/mimirpb/timeseries_test.go index d2b42a4fc3a..595eddd838c 100644 --- a/pkg/mimirpb/timeseries_test.go +++ b/pkg/mimirpb/timeseries_test.go @@ -129,7 +129,6 @@ func TestDeepCopyTimeseries(t *testing.T) { {Name: "exemplarLabel2", Value: "exemplarValue2"}, }, }}, - CreatedTimestamp: 1234567890, SkipUnmarshalingExemplars: true, }, } @@ -137,7 +136,6 @@ func TestDeepCopyTimeseries(t *testing.T) { dst = DeepCopyTimeseries(dst, src, true, true) // Check that scalar properties are copied. - assert.Equal(t, src.CreatedTimestamp, dst.CreatedTimestamp) assert.Equal(t, src.SkipUnmarshalingExemplars, dst.SkipUnmarshalingExemplars) // Check that the values in src and dst are the same. @@ -303,7 +301,7 @@ func TestDeepCopyTimeseriesCopiesAllFields(t *testing.T) { {Name: "label1", Value: "value1"}, }, Samples: []Sample{ - {Value: 1, TimestampMs: 2}, + {Value: 1, TimestampMs: 2, StartTimestamp: 3}, }, Exemplars: []Exemplar{ {Value: 1, TimestampMs: 2, Labels: []LabelAdapter{{Name: "e1", Value: "v1"}}}, @@ -324,9 +322,9 @@ func TestDeepCopyTimeseriesCopiesAllFields(t *testing.T) { ResetHint: Histogram_YES, Timestamp: 100, CustomValues: []float64{5.0, 6.0}, + StartTimestamp: 99, }, }, - CreatedTimestamp: 1234567890, SkipUnmarshalingExemplars: true, }, }