diff --git a/cmd/sippy/load.go b/cmd/sippy/load.go index c45c408734..0099141a8e 100644 --- a/cmd/sippy/load.go +++ b/cmd/sippy/load.go @@ -32,6 +32,7 @@ import ( v1 "github.com/openshift/sippy/pkg/apis/config/v1" "github.com/openshift/sippy/pkg/dataloader" "github.com/openshift/sippy/pkg/dataloader/bugloader" + "github.com/openshift/sippy/pkg/dataloader/gateststatus" "github.com/openshift/sippy/pkg/dataloader/jiraloader" "github.com/openshift/sippy/pkg/dataloader/loaderwithmetrics" "github.com/openshift/sippy/pkg/dataloader/prowloader" @@ -66,6 +67,7 @@ type LoadFlags struct { LogLevel string ProwLoadSince string SkipMatviewRefresh bool + ForceGARefresh bool } // want a single total load and refresh time @@ -107,6 +109,7 @@ func (f *LoadFlags) BindFlags(fs *pflag.FlagSet) { fs.StringVar(&f.LogLevel, "log-level", "info", "Log level") fs.StringVar(&f.ProwLoadSince, "prow-load-since", "", "Override how far back to load prow jobs (e.g. 2024-01-15T00:00:00Z or 72h for 72 hours ago)") fs.BoolVar(&f.SkipMatviewRefresh, "skip-matview-refresh", false, "Skip refreshing materialized views after loading") + fs.BoolVar(&f.ForceGARefresh, "force-ga-refresh", false, "Force re-population of GA test status data from BigQuery") } // nolint:gocyclo @@ -357,6 +360,17 @@ func NewLoadCommand() *cobra.Command { loaders = append(loaders, fgLoader) } + if l == "ga-test-status" { + refreshMatviews = true + if bigqueryErr != nil { + return errors.Wrap(bigqueryErr, "CRITICAL error getting BigQuery client which prevents ga-test-status loading") + } + if dbErr != nil { + return errors.Wrap(dbErr, "CRITICAL error getting postgres client which prevents ga-test-status loading") + } + loaders = append(loaders, gateststatus.New(ctx, dbc, bqc, f.ForceGARefresh, f.Releases)) + } + } // Run loaders with the metrics wrapper diff --git a/cmd/sippy/seed_data.go b/cmd/sippy/seed_data.go index d5a685b391..9ae185b4cb 100644 --- a/cmd/sippy/seed_data.go +++ b/cmd/sippy/seed_data.go @@ -3,6 +3,7 @@ package main import ( "context" "database/sql" + stderrors "errors" "fmt" "os" "sort" @@ -16,6 +17,8 @@ import ( "github.com/spf13/cobra" "github.com/spf13/pflag" "gopkg.in/yaml.v3" + "gorm.io/gorm" + "k8s.io/apimachinery/pkg/util/sets" componentreadiness "github.com/openshift/sippy/pkg/api/componentreadiness" pgprovider "github.com/openshift/sippy/pkg/api/componentreadiness/dataprovider/postgres" @@ -504,6 +507,12 @@ func seedSyntheticData(dbc *db.DB) error { return fmt.Errorf("failed to backfill %s: %w", table, err) } } + + if err := seedGARawTestData(dbc); err != nil { + return errors.WithMessage(err, "failed to seed GA raw test data") + } + log.Info("Seeded GA raw test data") + if err := sippyserver.RefreshData(dbc, nil, sippyserver.RefreshOptions{}); err != nil { return fmt.Errorf("failed to refresh data: %w", err) } @@ -1045,3 +1054,114 @@ func seedFeatureGates(dbc *db.DB) error { log.Infof("Created %d feature gate records", len(featureGates)) return nil } + +// seedGARawTestData populates prow_ga_raw_test_data for GA releases using +// the same synthetic test/job definitions. This gives the +// prow_ga_test_statuses_matview data to aggregate when refreshed. +func seedGARawTestData(dbc *db.DB) error { + var gaReleases []models.ReleaseDefinition + if err := dbc.DB.Where("ga_date IS NOT NULL AND ga_date < CURRENT_DATE").Find(&gaReleases).Error; err != nil { + return fmt.Errorf("querying GA releases: %w", err) + } + + if len(gaReleases) == 0 { + log.Info("No GA releases found, skipping GA raw test data seeding") + return nil + } + + testIDCache := make(map[string]uint) + jobIDCache := make(map[string]uint) + var suiteID uint + + var suite models.Suite + if err := dbc.DB.Where("name = ?", "synthetic").First(&suite).Error; err != nil { + if !stderrors.Is(err, gorm.ErrRecordNotFound) { + return fmt.Errorf("looking up suite 'synthetic': %w", err) + } + log.Warn("Suite 'synthetic' not found, GA raw test data will use suite_id=0") + } else { + suiteID = suite.ID + } + + var rows []models.ProwGARawTestDatum + for _, rel := range gaReleases { + for _, windowDays := range utils.GAWindows { + for _, spec := range syntheticTests { + testID, ok := testIDCache[spec.testName] + if !ok { + var test models.Test + if err := dbc.DB.Where("name = ?", spec.testName).First(&test).Error; err != nil { + if !stderrors.Is(err, gorm.ErrRecordNotFound) { + return fmt.Errorf("looking up test %q: %w", spec.testName, err) + } + continue + } + testID = test.ID + testIDCache[spec.testName] = testID + } + + for jobTemplate, releaseCounts := range spec.jobCounts { + counts, ok := releaseCounts[rel.Release] + if !ok { + continue + } + jobName := fmt.Sprintf(jobTemplate, rel.Release) + prowJobID, ok := jobIDCache[jobName] + if !ok { + var job models.ProwJob + if err := dbc.DB.Where("name = ?", jobName).First(&job).Error; err != nil { + if !stderrors.Is(err, gorm.ErrRecordNotFound) { + return fmt.Errorf("looking up prow job %q: %w", jobName, err) + } + continue + } + prowJobID = job.ID + jobIDCache[jobName] = prowJobID + } + + scale := int64(windowDays) + rows = append(rows, models.ProwGARawTestDatum{ + Release: rel.Release, + WindowDays: windowDays, + TestID: testID, + ProwJobID: prowJobID, + SuiteID: suiteID, + Passes: int64(counts.success) * scale, + Failures: int64(counts.total-counts.success-counts.flake) * scale, + Flakes: int64(counts.flake) * scale, + Runs: int64(counts.total) * scale, + }) + } + } + } + } + + if len(rows) > 0 { + if err := dbc.DB.CreateInBatches(rows, 500).Error; err != nil { + return fmt.Errorf("inserting GA raw test data: %w", err) + } + } + + releasesWithRows := sets.New[string]() + for _, row := range rows { + releasesWithRows.Insert(row.Release) + } + + for _, rel := range gaReleases { + if !releasesWithRows.Has(rel.Release) { + log.WithField("release", rel.Release).Warn("No GA seed data generated, skipping ga_data_loaded_date") + continue + } + gaDate := civil.DateOf(rel.GADate.UTC()) + if err := dbc.DB.Model(&models.ReleaseDefinition{}). + Where("release = ?", rel.Release). + Update("ga_data_loaded_date", gaDate).Error; err != nil { + return fmt.Errorf("updating ga_data_loaded_date for %s: %w", rel.Release, err) + } + } + + log.WithField("rows", len(rows)). + WithField("releases", len(gaReleases)). + Info("Seeded GA raw test data") + return nil +} diff --git a/pkg/api/componentreadiness/utils/utils.go b/pkg/api/componentreadiness/utils/utils.go index 856a4fa521..1744c5eb98 100644 --- a/pkg/api/componentreadiness/utils/utils.go +++ b/pkg/api/componentreadiness/utils/utils.go @@ -11,6 +11,7 @@ import ( "sync" "time" + "cloud.google.com/go/civil" "github.com/sirupsen/logrus" "k8s.io/apimachinery/pkg/util/sets" @@ -340,6 +341,22 @@ func GenerateTestDetailsURL( return u.String(), nil } +// GAWindows lists the distinct lookback periods (in days) for which GA base +// data is fetched from BigQuery and stored in prow_ga_raw_test_data. +var GAWindows = []int{1, 30, 90} + +// GAWindowStart returns the first day of a GA base window with the given +// lookback period. +func GAWindowStart(gaDate civil.Date, windowDays int) civil.Date { + return gaDate.AddDays(-windowDays) +} + +// GAWindowEnd returns the exclusive end of the GA base window (the day after +// the GA date). +func GAWindowEnd(gaDate civil.Date) civil.Date { + return gaDate.AddDays(1) +} + // EnqueueAsync is a helper function to asynchronously enqueue something (like errors) on a channel from a synchronous context. func EnqueueAsync[T any](wg *sync.WaitGroup, channel chan T, things ...T) { wg.Add(1) diff --git a/pkg/bigquery/bqlabel/labels.go b/pkg/bigquery/bqlabel/labels.go index c1b593bd7d..ff738aec88 100644 --- a/pkg/bigquery/bqlabel/labels.go +++ b/pkg/bigquery/bqlabel/labels.go @@ -100,6 +100,7 @@ const ( JobVariants QueryValue = "job-variants" PRTestResults QueryValue = "pr-test-results" CacheLookup QueryValue = "cache-lookup" + GATestStatusLoader QueryValue = "ga-test-status-loader" ) // sanitizeLabelValue sanitizes a label value to meet BigQuery requirements: diff --git a/pkg/dataloader/gateststatus/loader.go b/pkg/dataloader/gateststatus/loader.go new file mode 100644 index 0000000000..e0eb5ab45d --- /dev/null +++ b/pkg/dataloader/gateststatus/loader.go @@ -0,0 +1,320 @@ +package gateststatus + +import ( + "context" + "fmt" + "time" + + "cloud.google.com/go/bigquery" + "cloud.google.com/go/civil" + "github.com/jackc/pgx/v4" + "github.com/jackc/pgx/v4/stdlib" + log "github.com/sirupsen/logrus" + "google.golang.org/api/iterator" + + "github.com/openshift/sippy/pkg/api/componentreadiness/utils" + bqcachedclient "github.com/openshift/sippy/pkg/bigquery" + "github.com/openshift/sippy/pkg/bigquery/bqlabel" + "github.com/openshift/sippy/pkg/db" + "github.com/openshift/sippy/pkg/db/models" +) + +// GATestStatusLoader populates prow_ga_raw_test_data for releases that have reached GA. +// +// The loader has two conceptual phases: +// 1. Fetch: for each GA release, query BigQuery once for all configured windows +// and persist the raw results in prow_ga_raw_test_data. This runs only when +// the raw data is missing or the GA date changed, or when forced. +// 2. Aggregate: handled by the prow_ga_test_statuses_matview, which joins +// raw data with current dimension tables on each refresh cycle. +type GATestStatusLoader struct { + ctx context.Context + dbc *db.DB + bqClient *bqcachedclient.Client + force bool + releases []string + errs []error +} + +func New(ctx context.Context, dbc *db.DB, bqClient *bqcachedclient.Client, force bool, releases []string) *GATestStatusLoader { + return &GATestStatusLoader{ + ctx: ctx, + dbc: dbc, + bqClient: bqClient, + force: force, + releases: releases, + } +} + +func (l *GATestStatusLoader) Name() string { return "ga-test-status" } +func (l *GATestStatusLoader) Errors() []error { return l.errs } + +func (l *GATestStatusLoader) Load() { + start := time.Now() + + allReleases, err := l.gaReleases() + if err != nil { + l.errs = append(l.errs, err) + return + } + if len(allReleases) == 0 { + log.Info("ga-test-status: no GA releases found, skipping") + return + } + + loaded := 0 + for _, rel := range allReleases { + gaDate := civil.DateOf(rel.GADate.UTC()) + if !l.force && rel.LoadedGADate != nil && *rel.LoadedGADate == gaDate { + log.WithField("release", rel.Release).Debug("ga-test-status: already loaded, skipping") + continue + } + if err := l.loadRelease(rel.Release, gaDate); err != nil { + l.errs = append(l.errs, fmt.Errorf("release %s: %w", rel.Release, err)) + } else { + loaded++ + } + } + + log.WithField("elapsed", time.Since(start)). + WithField("loaded", loaded). + Info("ga-test-status: load complete") +} + +func (l *GATestStatusLoader) loadRelease(release string, gaDate civil.Date) error { + rLog := log.WithField("release", release) + gaEnd := utils.GAWindowEnd(gaDate) + + var windows []releaseWindow + for _, windowDays := range utils.GAWindows { + windows = append(windows, releaseWindow{ + Release: release, + WindowDays: int64(windowDays), + StartDate: utils.GAWindowStart(gaDate, windowDays), + EndDate: gaEnd, + }) + } + + bqStart := time.Now() + rLog.Info("ga-test-status: fetching all windows from BigQuery") + rows, err := l.fetchFromBigQuery(windows) + if err != nil { + return fmt.Errorf("fetching from BigQuery: %w", err) + } + rLog.WithField("bq_rows", len(rows)). + WithField("bq_elapsed", time.Since(bqStart)). + Info("ga-test-status: fetched from BigQuery") + + pgStart := time.Now() + if err := l.persist(release, gaDate, rows); err != nil { + return fmt.Errorf("persisting: %w", err) + } + rLog.WithField("pg_elapsed", time.Since(pgStart)). + Info("ga-test-status: persisted to Postgres") + + return nil +} + +func (l *GATestStatusLoader) persist(release string, gaDate civil.Date, rows []stagingRow) error { + sqlDB, err := l.dbc.DB.DB() + if err != nil { + return fmt.Errorf("getting sql.DB: %w", err) + } + conn, err := stdlib.AcquireConn(sqlDB) + if err != nil { + return fmt.Errorf("acquiring pgx conn: %w", err) + } + defer func() { + if releaseErr := stdlib.ReleaseConn(sqlDB, conn); releaseErr != nil { + log.WithError(releaseErr).Error("failed to release pgx conn") + } + }() + + if len(rows) > 0 { + copyStart := time.Now() + cleanup, err := db.CopyToTempTable(l.ctx, conn, "tmp_ga_raw", rows, tempCols) + if err != nil { + return fmt.Errorf("copying to temp table: %w", err) + } + defer cleanup() + log.WithField("rows", len(rows)). + WithField("elapsed", time.Since(copyStart)). + Info("ga-test-status: copied to temp table") + } + + tx, err := conn.Begin(l.ctx) + if err != nil { + return fmt.Errorf("beginning transaction: %w", err) + } + defer func() { + if rollbackErr := tx.Rollback(l.ctx); rollbackErr != nil && rollbackErr != pgx.ErrTxClosed { + log.WithError(rollbackErr).Error("failed to rollback transaction") + } + }() + + if _, err := tx.Exec(l.ctx, + "DELETE FROM prow_ga_raw_test_data WHERE release = $1", release); err != nil { + return fmt.Errorf("deleting existing raw rows: %w", err) + } + + if len(rows) > 0 { + insertStart := time.Now() + result, err := tx.Exec(l.ctx, ` + INSERT INTO prow_ga_raw_test_data + (release, window_days, test_id, prow_job_id, suite_id, passes, failures, flakes, runs) + SELECT + $1, tmp.window_days, t.id, pj.id, COALESCE(s.id, 0), + tmp.passes, tmp.failures, tmp.flakes, tmp.runs + FROM tmp_ga_raw tmp + INNER JOIN tests t ON t.name = tmp.test_name AND t.deleted_at IS NULL + INNER JOIN prow_jobs pj ON pj.name = tmp.job_name AND pj.deleted_at IS NULL + LEFT JOIN suites s ON s.name = tmp.suite_name AND s.deleted_at IS NULL + `, release) + if err != nil { + return fmt.Errorf("INSERT...SELECT from temp table: %w", err) + } + log.WithField("rows", result.RowsAffected()). + WithField("elapsed", time.Since(insertStart)). + Info("ga-test-status: inserted from temp table") + } + + if _, err := tx.Exec(l.ctx, + "UPDATE release_definitions SET ga_data_loaded_date = $1 WHERE release = $2", + gaDate, release); err != nil { + return fmt.Errorf("recording load status: %w", err) + } + + commitStart := time.Now() + if err := tx.Commit(l.ctx); err != nil { + return fmt.Errorf("committing transaction: %w", err) + } + log.WithField("elapsed", time.Since(commitStart)).Info("ga-test-status: committed") + + return nil +} + +var tempCols = []db.TempColumn[stagingRow]{ + {Name: "window_days", Type: "integer", Value: func(r *stagingRow) any { return int(r.WindowDays) }}, + {Name: "test_name", Type: "text", Value: func(r *stagingRow) any { return r.TestName }}, + {Name: "job_name", Type: "text", Value: func(r *stagingRow) any { return r.JobName }}, + {Name: "suite_name", Type: "text", Value: func(r *stagingRow) any { return r.Suite }}, + {Name: "passes", Type: "bigint", Value: func(r *stagingRow) any { return r.Passes }}, + {Name: "failures", Type: "bigint", Value: func(r *stagingRow) any { return r.Failures }}, + {Name: "flakes", Type: "bigint", Value: func(r *stagingRow) any { return r.Flakes }}, + {Name: "runs", Type: "bigint", Value: func(r *stagingRow) any { return r.Runs }}, +} + +func (l *GATestStatusLoader) gaReleases() ([]models.ReleaseDefinition, error) { + var defs []models.ReleaseDefinition + query := l.dbc.DB.WithContext(l.ctx).Where("ga_date < CURRENT_DATE") + if len(l.releases) > 0 { + query = query.Where("release IN ?", l.releases) + } + if err := query.Find(&defs).Error; err != nil { + return nil, fmt.Errorf("querying release_definitions: %w", err) + } + return defs, nil +} + +func (l *GATestStatusLoader) fetchFromBigQuery(windows []releaseWindow) ([]stagingRow, error) { + earliestStart := windows[0].StartDate + latestEnd := windows[0].EndDate + for _, w := range windows[1:] { + if w.StartDate.Before(earliestStart) { + earliestStart = w.StartDate + } + if w.EndDate.After(latestEnd) { + latestEnd = w.EndDate + } + } + + query := l.bqClient.Query(l.ctx, bqlabel.GATestStatusLoader, fmt.Sprintf(` + WITH params AS ( + SELECT * FROM UNNEST(@windows) + ), + deduped AS ( + SELECT + p.release, + p.window_days, + junit.test_name, + jobs.prowjob_job_name AS job_name, + COALESCE(junit.testsuite, '') AS suite_name, + CASE WHEN junit.flake_count > 0 THEN 0 ELSE junit.success_val END AS adjusted_success, + CASE WHEN junit.flake_count > 0 THEN 1 ELSE 0 END AS is_flake, + ROW_NUMBER() OVER( + PARTITION BY p.release, p.window_days, junit.file_path, junit.test_name, junit.testsuite + ORDER BY CASE WHEN junit.flake_count > 0 THEN 0 WHEN junit.success_val > 0 THEN 1 ELSE 2 END + ) AS row_num + FROM params p + JOIN %[1]s.junit ON junit.release = p.release + AND junit.modified_time >= DATETIME(@earliest_start) + AND junit.modified_time < DATETIME(@latest_end) + AND junit.modified_time >= DATETIME(p.start_date) + AND junit.modified_time < DATETIME(p.end_date) + AND junit.skipped = FALSE + JOIN %[1]s.jobs jobs ON junit.prowjob_build_id = jobs.prowjob_build_id + AND jobs.prowjob_start >= DATETIME(p.start_date) + AND jobs.prowjob_start < DATETIME(p.end_date) + JOIN %[1]s.job_variants jv ON jobs.prowjob_job_name = jv.job_name + AND jv.variant_name = 'Release' AND jv.variant_value = p.release + ) + SELECT + release, + window_days, + test_name, + job_name, + suite_name, + SUM(adjusted_success) AS passes, + SUM(CASE WHEN adjusted_success = 0 AND is_flake = 0 THEN 1 ELSE 0 END) AS failures, + SUM(is_flake) AS flakes, + COUNT(*) AS runs + FROM deduped + WHERE row_num = 1 + GROUP BY release, window_days, test_name, job_name, suite_name + `, l.bqClient.Dataset)) + query.Parameters = []bigquery.QueryParameter{ + {Name: "windows", Value: windows}, + {Name: "earliest_start", Value: earliestStart}, + {Name: "latest_end", Value: latestEnd}, + } + + iter, err := query.Read(l.ctx) + if err != nil { + return nil, fmt.Errorf("executing query: %w", err) + } + + var result []stagingRow + for { + var row stagingRow + err := iter.Next(&row) + if err == iterator.Done { + break + } + if err != nil { + return nil, fmt.Errorf("reading row: %w", err) + } + result = append(result, row) + } + return result, nil +} + +// releaseWindow defines a single (release, window) combination for the batched BQ query. +// Passed as an ARRAY> parameter. +type releaseWindow struct { + Release string `bigquery:"release"` + WindowDays int64 `bigquery:"window_days"` + StartDate civil.Date `bigquery:"start_date"` + EndDate civil.Date `bigquery:"end_date"` +} + +type stagingRow struct { + Release string `bigquery:"release"` + WindowDays int64 `bigquery:"window_days"` + TestName string `bigquery:"test_name"` + JobName string `bigquery:"job_name"` + Suite string `bigquery:"suite_name"` + Passes int64 `bigquery:"passes"` + Failures int64 `bigquery:"failures"` + Flakes int64 `bigquery:"flakes"` + Runs int64 `bigquery:"runs"` +} diff --git a/pkg/db/db.go b/pkg/db/db.go index d80ebec8b9..d4d96f695a 100644 --- a/pkg/db/db.go +++ b/pkg/db/db.go @@ -146,6 +146,7 @@ func (d *DB) UpdateSchema(reportEnd *time.Time) error { &models.ReleasePullRequest{}, &models.ReleaseRepository{}, &models.ReleaseJobRun{}, + &models.ProwGARawTestDatum{}, &models.VariantCombination{}, &models.ProwJob{}, &models.ProwJobRun{}, diff --git a/pkg/db/models/prow.go b/pkg/db/models/prow.go index a8de3da835..d8c1d23943 100644 --- a/pkg/db/models/prow.go +++ b/pkg/db/models/prow.go @@ -219,6 +219,23 @@ type TestCumulativeSummary struct { PrefixSumRuns int64 `gorm:"column:prefix_sum_runs;not null;default:0"` } +// ProwGARawTestDatum stores raw BigQuery test results for GA release windows. +// Fetched once per GA date and persisted so that the aggregation into +// prow_ga_test_statuses_matview can be re-run cheaply when dimension tables change. +// Each (release, window_days) pair holds results aggregated over a different lookback +// period (e.g. 1, 30, or 90 days before GA). +type ProwGARawTestDatum struct { + Release string `gorm:"not null;index:idx_prow_ga_raw_release_window"` + WindowDays int `gorm:"not null;default:30;index:idx_prow_ga_raw_release_window"` + TestID uint `gorm:"not null"` + ProwJobID uint `gorm:"not null"` + SuiteID uint `gorm:"not null;default:0"` + Passes int64 `gorm:"not null;default:0"` + Failures int64 `gorm:"not null;default:0"` + Flakes int64 `gorm:"not null;default:0"` + Runs int64 `gorm:"not null;default:0"` +} + // Bug represents a Jira bug. type Bug struct { ID uint `json:"id" gorm:"primaryKey"` diff --git a/pkg/db/models/releases.go b/pkg/db/models/releases.go index a02f533f72..01cc7c62c1 100644 --- a/pkg/db/models/releases.go +++ b/pkg/db/models/releases.go @@ -3,6 +3,7 @@ package models import ( "time" + "cloud.google.com/go/civil" "github.com/lib/pq" ) @@ -17,6 +18,7 @@ type ReleaseDefinition struct { Patch *int `json:"patch,omitempty" gorm:"column:patch"` PreviousRelease string `json:"previous_release" gorm:"column:previous_release"` GADate *time.Time `json:"ga_date,omitempty" gorm:"column:ga_date;type:date"` + LoadedGADate *civil.Date `json:"loaded_ga_date,omitempty" gorm:"column:ga_data_loaded_date;type:date"` DevelopmentStartDate *time.Time `json:"development_start_date" gorm:"column:development_start_date;type:date"` Product string `json:"product" gorm:"column:product"` Status string `json:"status" gorm:"column:status"` diff --git a/pkg/db/views.go b/pkg/db/views.go index 6618719e4c..0747adcdba 100644 --- a/pkg/db/views.go +++ b/pkg/db/views.go @@ -83,6 +83,12 @@ var PostgresMatViews = []PostgresView{ IndexColumns: []string{"release", "architecture", "stream", "prow_job_run_id", "test_id", "suite_id"}, ReplaceStrings: map[string]string{}, }, + { + Name: "prow_ga_test_statuses_matview", + Definition: gaTestStatusMatView, + IndexColumns: []string{"release", "window_days", "test_id", "suite_id", "variant_combination_id"}, + RefreshPhase: 1, + }, } // PostgresViews are regular, non-materialized views: @@ -486,3 +492,18 @@ WHERE AND pj.id = pjr.prow_job_id ORDER BY pjrt.id DESC ` + +const gaTestStatusMatView = ` +SELECT + raw.test_id, + raw.suite_id, + pj.variant_combination_id, + raw.release, + raw.window_days, + SUM(raw.runs)::int AS total_count, + SUM(raw.passes + raw.flakes)::int AS success_count, + SUM(raw.flakes)::int AS flake_count +FROM prow_ga_raw_test_data raw +JOIN prow_jobs pj ON pj.id = raw.prow_job_id AND pj.deleted_at IS NULL AND pj.variant_combination_id IS NOT NULL +GROUP BY raw.test_id, raw.suite_id, pj.variant_combination_id, raw.release, raw.window_days +`