diff --git a/pkg/bigquery/bqlabel/labels.go b/pkg/bigquery/bqlabel/labels.go index dff1590434..695fb8378d 100644 --- a/pkg/bigquery/bqlabel/labels.go +++ b/pkg/bigquery/bqlabel/labels.go @@ -81,7 +81,6 @@ const ( BugLoaderTestBugs QueryValue = "bug-loader-test-bugs" BugLoaderJobBugs QueryValue = "bug-loader-job-bugs" ProwLoaderProwJobs QueryValue = "prow-loader-prow-jobs" - ProwLoaderTestAnalysis QueryValue = "prow-loader-test-analysis" ProwLoaderJobLabels QueryValue = "prow-loader-job-labels" VariantRegistryDeleteJobBatch QueryValue = "variant-registry-delete-job-batch" VariantRegistryDeleteVariant QueryValue = "variant-registry-delete-variant" diff --git a/pkg/dataloader/prowloader/prow.go b/pkg/dataloader/prowloader/prow.go index 8631c657e0..0466582cca 100644 --- a/pkg/dataloader/prowloader/prow.go +++ b/pkg/dataloader/prowloader/prow.go @@ -17,10 +17,7 @@ import ( "time" "cloud.google.com/go/bigquery" - "cloud.google.com/go/civil" "cloud.google.com/go/storage" - "github.com/jackc/pgtype" - "github.com/jackc/pgx/v4" "github.com/jackc/pgx/v4/stdlib" "github.com/lib/pq" "github.com/openshift/sippy/pkg/bigquery/bqlabel" @@ -34,7 +31,6 @@ import ( "k8s.io/apimachinery/pkg/util/sets" bqcachedclient "github.com/openshift/sippy/pkg/bigquery" - "github.com/openshift/sippy/pkg/db/query" v1config "github.com/openshift/sippy/pkg/apis/config/v1" "github.com/openshift/sippy/pkg/apis/junit" @@ -348,13 +344,6 @@ func (pl *ProwLoader) Load() { pl.errors = append(pl.errors, err) } - // load the test analysis by job data into tables partitioned by day, letting bigquery do the - // heavy lifting for us. - err = pl.loadDailyTestAnalysisByJob(pl.ctx) - if err != nil { - pl.errors = append(pl.errors, errors.Wrap(err, "error updating daily test analysis by job")) - } - if len(pl.errors) > 0 { log.Warningf("encountered %d errors while importing job runs", len(pl.errors)) } @@ -366,261 +355,6 @@ func (pl *ProwLoader) Load() { } } -// tempBQTestAnalysisByJobForDate is a dupe type to work around date parsing issues. -type tempBQTestAnalysisByJobForDate struct { - Date civil.Date - TestID uint - Release string - TestName string `bigquery:"test_name"` - JobName string `bigquery:"job_name"` - Runs int - Passes int - Flakes int - Failures int -} - -// getTestAnalysisByJobFromToDates uses the last daily report date to calculate the -// date range we should request from bigquery for import. We don't want to import -// the prior day if jobs are still running, so if we're not at least 8 hours into -// the current day (UTC), we will keep waiting to import yesterday. Once we cross -// that threshold we will import. (assume hourly imports) -// If our most recent import is yesterday, we're done for the day. -// If the lastDailySummary is empty, this implies a new database, and we'll do an initial -// bulk load. -// -// Returns a slice of day strings YYYY-MM-DD in ascending order. We'll import a day at -// a time, with a separate transaction for each. If something goes wrong we can fail and -// pick up at that date the next time. -// -// At present in prod, each day takes about 20 minutes -func getTestAnalysisByJobFromToDates(lastDailySummary, now time.Time, loadSince *time.Time) []string { - to := now.UTC().Add(-32 * time.Hour) - - // If this is a new db, do an initial larger import: - if lastDailySummary.IsZero() { - return DaysBetween(resolveFrom(loadSince, to), to) - } - - ldsStr := lastDailySummary.UTC().Format("2006-01-02") - if ldsStr == to.Format("2006-01-02") { - return []string{} - } - from := lastDailySummary.UTC().Add(24 * time.Hour) - return DaysBetween(from, to) -} - -// DaysBetween returns a slice of strings representing each day in YYYY-MM-DD format between two dates -func DaysBetween(start, end time.Time) []string { - var days []string - - // Normalize times to midnight to count full days - start = start.Truncate(24 * time.Hour) - end = end.Truncate(24 * time.Hour) - - // Ensure start is before or equal to end - if end.Before(start) { - start, end = end, start - } - - // Iterate from start to end date - for d := start; !d.After(end); d = d.Add(24 * time.Hour) { - days = append(days, d.Format("2006-01-02")) - } - - return days -} - -// NextDay takes a date string in YYYY-MM-DD format and returns the date string for the following day. -func NextDay(dateStr string) (string, error) { - // Parse the input date string - date, err := time.Parse("2006-01-02", dateStr) - if err != nil { - return "", fmt.Errorf("invalid date format: %v", err) - } - - // Add one day to the parsed date - nextDay := date.Add(24 * time.Hour) - - // Format the next day back to YYYY-MM-DD - return nextDay.Format("2006-01-02"), nil -} - -// loadDailyTestAnalysisByJob loads test analysis data into partitioned tables in postgres, one per -// day. The data is calculated by querying bigquery to do the heavy lifting for us. Each day is committed -// transactionally so the process is safe to interrupt and resume later. The process takes about 20 minutes -// per day at the time of writing, so an initial load for all releases can be quite time consuming. -func (pl *ProwLoader) loadDailyTestAnalysisByJob(ctx context.Context) error { - - // Figure out our last imported daily summary. - var lastDailySummary time.Time - row := pl.dbc.DB.Table("test_analysis_by_job_by_dates").Select("MAX(date)").Row() - - // Ignoring error, the function below handles the zero time if needed: (new db) - _ = row.Scan(&lastDailySummary) - - importDates := getTestAnalysisByJobFromToDates(lastDailySummary, time.Now(), pl.loadSince) - if len(importDates) == 0 { - log.Info("test analysis summary already completed today") - return nil - } - log.Infof("importing test analysis by job for dates: %v", importDates) - - jobCache, err := query.LoadProwJobCache(pl.dbc) - if err != nil { - log.WithError(err).Error("error loading job cache") - return err - } - - testCache, err := query.LoadTestCache(pl.dbc, []string{}) - if err != nil { - log.WithError(err).Error("error loading test cache") - return err - } - - for _, dateToImport := range importDates { - dLog := log.WithField("date", dateToImport) - - dLog.Infof("Loading test analysis by job daily summaries") - - q := pl.bigQueryClient.Query(ctx, bqlabel.ProwLoaderTestAnalysis, fmt.Sprintf(`WITH - deduped_testcases AS ( - SELECT - junit.*, - ROW_NUMBER() OVER(PARTITION BY file_path, test_name, testsuite ORDER BY CASE WHEN flake_count > 0 THEN 0 WHEN success_val > 0 THEN 1 ELSE 2 END ) AS row_num, - jobs. prowjob_job_name AS variant_registry_job_name, - jobs.org, - jobs.repo, - jobs.pr_number, - jobs.pr_sha, - CASE - WHEN flake_count > 0 THEN 0 - ELSE success_val - END - AS adjusted_success_val, - CASE - WHEN flake_count > 0 THEN 1 - ELSE 0 - END - AS adjusted_flake_count - FROM - %s.junit - INNER JOIN - %s.jobs jobs - ON - junit.prowjob_build_id = jobs.prowjob_build_id - AND DATE(jobs.prowjob_start) <= DATE(@DateToImport) - WHERE - DATE(junit.modified_time) = DATE(@DateToImport) - AND skipped = FALSE ) -SELECT - test_name, - DATE(modified_time) AS date, - prowjob_name AS job_name, - branch AS release, - COUNT(*) AS runs, - SUM(adjusted_success_val) AS passes, - SUM(adjusted_flake_count) AS flakes, -FROM - deduped_testcases -WHERE - row_num = 1 - AND branch IN UNNEST(@Releases) -GROUP BY - test_name, - date, - release, - prowjob_name -ORDER BY - date, - test_name, - prowjob_name -`, pl.bigQueryClient.Dataset, pl.bigQueryClient.Dataset)) - q.Parameters = []bigquery.QueryParameter{ - { - Name: "DateToImport", - Value: dateToImport, - }, - { - Name: "Releases", - Value: pl.releases, - }, - } - it, err := q.Read(ctx) - if err != nil { - dLog.WithError(err).Error("error querying test analysis from bigquery") - return err - } - - insertRows := []models.TestAnalysisByJobByDate{} - for { - row := tempBQTestAnalysisByJobForDate{} - err := it.Next(&row) - if err == iterator.Done { - break - } - if err != nil { - log.WithError(err).Error("error parsing prowjob from bigquery") - return err - } - psqlDate := pgtype.Date{} - err = psqlDate.Set(row.Date.String()) - if err != nil { - return err - } - - // Skip jobs and tests we don't know about in our postgres db: - test, ok := testCache[row.TestName] - if !ok { - continue - } - - if _, ok := jobCache[row.JobName]; !ok { - continue - } - // we have to infer failures due to the bigquery query we leveraged: - failures := row.Runs - row.Passes - row.Flakes - - // convert to a db row for postgres insertion: - psqlRow := models.TestAnalysisByJobByDate{ - Date: row.Date.In(time.UTC), - TestID: test.ID, - Release: row.Release, - TestName: row.TestName, - JobName: row.JobName, - Runs: row.Runs, - Passes: row.Passes, - Flakes: row.Flakes, - Failures: failures, - } - insertRows = append(insertRows, psqlRow) - } - st := time.Now() - dLog.Infof("inserting %d rows", len(insertRows)) - n, err := pl.dbc.CopyFrom(ctx, "test_analysis_by_job_by_dates", - []string{"date", "test_id", "release", "job_name", "test_name", "runs", "passes", "flakes", "failures"}, - pgx.CopyFromSlice(len(insertRows), func(i int) ([]any, error) { - r := &insertRows[i] - return []any{ - r.Date, - r.TestID, - r.Release, - r.JobName, - r.TestName, - r.Runs, - r.Passes, - r.Flakes, - r.Failures, - }, nil - }), - ) - if err != nil { - return fmt.Errorf("COPY test_analysis_by_job_by_dates: %w", err) - } - dLog.Infof("inserted %d rows in %s", n, time.Since(st)) - } - return nil -} - // matchRelease returns the release a prow job belongs to, or "" if it // doesn't match any configured release. For /payload sub-jobs (identified // by releaseJobName annotation + PR refs), it returns the Presubmits diff --git a/pkg/dataloader/prowloader/prow_test.go b/pkg/dataloader/prowloader/prow_test.go index 0e9d1a6dcf..2ee5d50c39 100644 --- a/pkg/dataloader/prowloader/prow_test.go +++ b/pkg/dataloader/prowloader/prow_test.go @@ -114,97 +114,6 @@ func TestParseVariantDataFile(t *testing.T) { assert.Equal(t, "foo", clusterData["AddonProp1"]) } -func TestGetTestAnalysisByJobFromToDates(t *testing.T) { - tests := []struct { - name string - lastSummary time.Time - now time.Time - expectedDates []string - }{ - { - name: "empty db to yesterday", - lastSummary: time.Time{}, - now: time.Date(2024, time.October, 31, 9, 0, 0, 0, time.UTC), - expectedDates: []string{ - "2024-10-16", - "2024-10-17", - "2024-10-18", - "2024-10-19", - "2024-10-20", - "2024-10-21", - "2024-10-22", - "2024-10-23", - "2024-10-24", - "2024-10-25", - "2024-10-26", - "2024-10-27", - "2024-10-28", - "2024-10-29", - "2024-10-30", - }, - }, - { - name: "empty db to two days ago if early", - lastSummary: time.Time{}, - now: time.Date(2024, time.October, 31, 3, 0, 0, 0, time.UTC), - expectedDates: []string{ - "2024-10-15", - "2024-10-16", - "2024-10-17", - "2024-10-18", - "2024-10-19", - "2024-10-20", - "2024-10-21", - "2024-10-22", - "2024-10-23", - "2024-10-24", - "2024-10-25", - "2024-10-26", - "2024-10-27", - "2024-10-28", - "2024-10-29", - }, - }, - { - name: "yesterday", - lastSummary: time.Date(2024, time.October, 29, 0, 0, 0, 0, time.UTC), - now: time.Date(2024, time.October, 31, 9, 0, 0, 0, time.UTC), - expectedDates: []string{"2024-10-30"}, - }, - { - name: "too early", - lastSummary: time.Date(2024, time.October, 29, 0, 0, 0, 0, time.UTC), - now: time.Date(2024, time.October, 31, 2, 0, 0, 0, time.UTC), - expectedDates: []string{}, - }, - { - name: "already updated today", - lastSummary: time.Date(2024, time.October, 30, 0, 0, 0, 0, time.UTC), - now: time.Date(2024, time.October, 31, 9, 0, 0, 0, time.UTC), - expectedDates: []string{}, - }, - { - name: "last 5 days", - lastSummary: time.Date(2024, time.October, 24, 0, 0, 0, 0, time.UTC), - now: time.Date(2024, time.October, 31, 9, 0, 0, 0, time.UTC), - expectedDates: []string{ - "2024-10-25", - "2024-10-26", - "2024-10-27", - "2024-10-28", - "2024-10-29", - "2024-10-30", - }, - }, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - dates := getTestAnalysisByJobFromToDates(tt.lastSummary, tt.now, nil) - assert.Equal(t, tt.expectedDates, dates) - }) - } -} - func TestIsPayloadPresubmit(t *testing.T) { tests := []struct { name string diff --git a/pkg/db/db.go b/pkg/db/db.go index f7d8b576f0..28a4ac1a18 100644 --- a/pkg/db/db.go +++ b/pkg/db/db.go @@ -22,15 +22,14 @@ import ( type SchemaHashType string const ( - hashTypeMatView SchemaHashType = "matview" - hashTypeView SchemaHashType = "view" - hashTypeMatViewIndex SchemaHashType = "matview_index" - hashTypeFunction SchemaHashType = "function" - partitionedTableProwJobRunTests = "prow_job_run_tests" - partitionedTableProwJobRunTestsOutputs = "prow_job_run_test_outputs" - partitionedTableTestAnalysisByJobByDates = "test_analysis_by_job_by_dates" - partitionedTableTestDailyTotals = "test_daily_totals" - partitionedTableTestCumulativeSummaries = "test_cumulative_summaries" + hashTypeMatView SchemaHashType = "matview" + hashTypeView SchemaHashType = "view" + hashTypeMatViewIndex SchemaHashType = "matview_index" + hashTypeFunction SchemaHashType = "function" + partitionedTableProwJobRunTests = "prow_job_run_tests" + partitionedTableProwJobRunTestsOutputs = "prow_job_run_test_outputs" + partitionedTableTestDailyTotals = "test_daily_totals" + partitionedTableTestCumulativeSummaries = "test_cumulative_summaries" ) type DB struct { @@ -86,9 +85,10 @@ func New(dsn string, logLevel gormlogger.LogLevel, opts ...Option) (*DB, error) return nil, err } // Prevent PostgreSQL from generating generic plans for prepared statements. - // With 10k+ partitions on tables like test_analysis_by_job_by_dates, generic - // plan generation alone can take 17+ minutes as the planner enumerates all - // partitions. Custom plans use actual parameter values for partition pruning. + // After 5 executions PostgreSQL normally switches to a generic plan that + // cannot use parameter values for partition pruning. GORM generates + // parameterized queries, so without this setting the planner would scan + // all partitions instead of pruning to the relevant ones. pgxConfig.RuntimeParams["plan_cache_mode"] = "force_custom_plan" pgxConfig.RuntimeParams["work_mem"] = "128MB" pgxConfig.RuntimeParams["idle_in_transaction_session_timeout"] = "60s" @@ -219,7 +219,6 @@ func (d *DB) PartitionedTables() []string { return []string{ partitionedTableProwJobRunTests, partitionedTableProwJobRunTestsOutputs, - partitionedTableTestAnalysisByJobByDates, partitionedTableTestDailyTotals, partitionedTableTestCumulativeSummaries, } @@ -247,7 +246,7 @@ func (d *DB) EnsurePartitions(releases []string, startDate, endDate time.Time, d dateColumn = "prow_job_run_timestamp" case partitionedTableProwJobRunTestsOutputs: dateColumn = "prow_job_run_test_timestamp" - case partitionedTableTestAnalysisByJobByDates, partitionedTableTestDailyTotals, partitionedTableTestCumulativeSummaries: + case partitionedTableTestDailyTotals, partitionedTableTestCumulativeSummaries: dateColumn = "date" default: log.Warnf("unknown partitioned table: %s", tableName) diff --git a/pkg/db/migrate/migrate.go b/pkg/db/migrate/migrate.go index 01b4d34e66..21efa51cbf 100644 --- a/pkg/db/migrate/migrate.go +++ b/pkg/db/migrate/migrate.go @@ -91,9 +91,9 @@ func RunMigrations(gormDB *gorm.DB) error { } if !hasSchemaMigrations { - hasExistingTable, err := tableExists(sqlDB, "test_analysis_by_job_by_dates") + hasExistingTable, err := tableExists(sqlDB, "prow_job_run_tests") if err != nil { - return fmt.Errorf("failed to check for test_analysis_by_job_by_dates table: %w", err) + return fmt.Errorf("failed to check for prow_job_run_tests table: %w", err) } if hasExistingTable { diff --git a/pkg/db/migrations/000010_drop_test_analysis_by_job_by_dates.down.sql b/pkg/db/migrations/000010_drop_test_analysis_by_job_by_dates.down.sql new file mode 100644 index 0000000000..93f19c5f1c --- /dev/null +++ b/pkg/db/migrations/000010_drop_test_analysis_by_job_by_dates.down.sql @@ -0,0 +1,20 @@ +-- Recreate the parent partitioned table. No data is restored. +-- Partitions were previously managed by gopar and will not be +-- recreated automatically. +CREATE TABLE IF NOT EXISTS test_analysis_by_job_by_dates ( + date TIMESTAMP WITH TIME ZONE, + test_id BIGINT, + release TEXT NOT NULL, + job_name TEXT, + test_name TEXT, + runs BIGINT, + passes BIGINT, + flakes BIGINT, + failures BIGINT +) PARTITION BY LIST (release); + +CREATE UNIQUE INDEX IF NOT EXISTS idx_test_analysis_test_release_date + ON test_analysis_by_job_by_dates (date, test_id, release, job_name); + +CREATE INDEX IF NOT EXISTS idx_test_analysis_name_and_job + ON test_analysis_by_job_by_dates (test_name, job_name); diff --git a/pkg/db/migrations/000010_drop_test_analysis_by_job_by_dates.up.sql b/pkg/db/migrations/000010_drop_test_analysis_by_job_by_dates.up.sql new file mode 100644 index 0000000000..f6e7832313 --- /dev/null +++ b/pkg/db/migrations/000010_drop_test_analysis_by_job_by_dates.up.sql @@ -0,0 +1,20 @@ +DROP VIEW IF EXISTS prow_test_analysis_by_variant_14d_view; +DROP TABLE IF EXISTS test_analysis_by_job_by_dates; + +-- Drop any detached partitions that are no longer children of the parent +-- table. The partition lifecycle detaches old partitions before dropping +-- them; those detached tables survive the parent DROP TABLE above. +DO $$ +DECLARE + tbl TEXT; +BEGIN + FOR tbl IN + SELECT tablename + FROM pg_tables + WHERE schemaname = 'public' + AND tablename LIKE 'test_analysis_by_job_by_dates_%' + LOOP + EXECUTE format('DROP TABLE IF EXISTS %I', tbl); + END LOOP; +END; +$$; diff --git a/pkg/db/migrations/MANIFEST b/pkg/db/migrations/MANIFEST index 45596fb394..050732d30a 100644 --- a/pkg/db/migrations/MANIFEST +++ b/pkg/db/migrations/MANIFEST @@ -16,3 +16,4 @@ 000007_drop_test_daily_summaries 000008_add_summary_timestamps 000009_add_lifecycle_to_prow_job_run_tests +000010_drop_test_analysis_by_job_by_dates diff --git a/pkg/db/models/prow.go b/pkg/db/models/prow.go index 209f20d0d9..02a1b0d385 100644 --- a/pkg/db/models/prow.go +++ b/pkg/db/models/prow.go @@ -160,18 +160,6 @@ type Suite struct { Name string `gorm:"uniqueIndex"` } -type TestAnalysisByJobByDate struct { - Date time.Time `gorm:"index:test_release_date,unique"` - TestID uint `gorm:"index:test_release_date,unique"` - Release string `gorm:"index:test_release_date,unique"` - JobName string `gorm:"index:test_release_date,unique"` - TestName string - Runs int - Passes int - Flakes int - Failures int -} - // TestDailyTotal stores pre-aggregated daily test results. // Table is partitioned (LIST by release, RANGE by date) - // schema managed by migration 000006, not AutoMigrate. diff --git a/pkg/db/query/job_queries.go b/pkg/db/query/job_queries.go index 7df5bffb5f..e0f1ee2d43 100644 --- a/pkg/db/query/job_queries.go +++ b/pkg/db/query/job_queries.go @@ -12,22 +12,6 @@ import ( "github.com/openshift/sippy/pkg/filter" ) -func LoadProwJobCache(dbc *db.DB) (map[string]*models.ProwJob, error) { - prowJobCache := map[string]*models.ProwJob{} - var allJobs []*models.ProwJob - res := dbc.DB.Model(&models.ProwJob{}).Find(&allJobs) - if res.Error != nil { - return map[string]*models.ProwJob{}, res.Error - } - for _, j := range allJobs { - if _, ok := prowJobCache[j.Name]; !ok { - prowJobCache[j.Name] = j - } - } - log.Infof("job cache created with %d entries from database", len(prowJobCache)) - return prowJobCache, nil -} - func JobRunTestCount(dbc *db.DB, jobRunID int64, release string, timestamp time.Time) (int, error) { var prowJobRunTestCount int64 diff --git a/pkg/db/query/test_queries.go b/pkg/db/query/test_queries.go index afd11f1deb..fb27fb2a87 100644 --- a/pkg/db/query/test_queries.go +++ b/pkg/db/query/test_queries.go @@ -90,33 +90,6 @@ const ( ) t` ) -func LoadTestCache(dbc *db.DB, preloads []string) (map[string]*models.Test, error) { - // Cache all tests by name to their ID, used for the join object. - testCache := map[string]*models.Test{} - q := dbc.DB.Model(&models.Test{}) - for _, p := range preloads { - q = q.Preload(p) - } - - // Kube exceeds 60000 tests, more than postgres can load at once: - testsBatch := []*models.Test{} - res := q.FindInBatches(&testsBatch, 5000, func(tx *gorm.DB, batch int) error { - for _, idn := range testsBatch { - if _, ok := testCache[idn.Name]; !ok { - testCache[idn.Name] = idn - } - } - return nil - }) - - if res.Error != nil { - return map[string]*models.Test{}, res.Error - } - - log.Infof("test cache created with %d entries from database", len(testCache)) - return testCache, nil -} - // TestReportsByVariant returns per-variant test report rows for every test matching // the given name criteria. When includeAll is true, an additional "All" aggregate row // per test is included via UNION ALL (Variant = "All", counting runs across all diff --git a/test/e2e/db/partitions/partitions_test.go b/test/e2e/db/partitions/partitions_test.go index 818e4e3a95..afa6101b49 100644 --- a/test/e2e/db/partitions/partitions_test.go +++ b/test/e2e/db/partitions/partitions_test.go @@ -39,7 +39,7 @@ func TestPartitionLifecycle(t *testing.T) { FROM pg_inherits JOIN pg_class parent ON pg_inherits.inhparent = parent.oid JOIN pg_class child ON pg_inherits.inhrelid = child.oid - WHERE parent.relname IN ('prow_job_run_tests', 'prow_job_run_test_outputs', 'test_analysis_by_job_by_dates') + WHERE parent.relname IN ('prow_job_run_tests', 'prow_job_run_test_outputs') `).Scan(&partitionCount).Error require.NoError(t, err) assert.Greater(t, partitionCount, int64(0), "should have created partitions") @@ -96,7 +96,7 @@ func TestPartitionLifecycle(t *testing.T) { FROM pg_inherits JOIN pg_class parent ON pg_inherits.inhparent = parent.oid JOIN pg_class child ON pg_inherits.inhrelid = child.oid - WHERE parent.relname IN ('prow_job_run_tests', 'prow_job_run_test_outputs', 'test_analysis_by_job_by_dates') + WHERE parent.relname IN ('prow_job_run_tests', 'prow_job_run_test_outputs') `).Scan(&attachedPartitions).Error require.NoError(t, err) @@ -198,7 +198,7 @@ func TestPartitionLifecycle(t *testing.T) { 0 AS level FROM pg_class c JOIN pg_namespace n ON n.oid = c.relnamespace - WHERE c.relname IN ('prow_job_run_tests', 'prow_job_run_test_outputs', 'test_analysis_by_job_by_dates') + WHERE c.relname IN ('prow_job_run_tests', 'prow_job_run_test_outputs') AND n.nspname = 'public' UNION ALL @@ -261,7 +261,7 @@ func TestPartitionLifecycle(t *testing.T) { 0 AS level FROM pg_class c JOIN pg_namespace n ON n.oid = c.relnamespace - WHERE c.relname IN ('prow_job_run_tests', 'prow_job_run_test_outputs', 'test_analysis_by_job_by_dates') + WHERE c.relname IN ('prow_job_run_tests', 'prow_job_run_test_outputs') AND n.nspname = 'public' UNION ALL @@ -336,7 +336,7 @@ func TestPartitionLifecycle(t *testing.T) { SELECT c.oid, c.relname AS table_name FROM pg_class c JOIN pg_namespace n ON n.oid = c.relnamespace - WHERE c.relname IN ('prow_job_run_tests', 'prow_job_run_test_outputs', 'test_analysis_by_job_by_dates') + WHERE c.relname IN ('prow_job_run_tests', 'prow_job_run_test_outputs') AND n.nspname = 'public' UNION ALL SELECT child.oid, child.relname @@ -369,7 +369,7 @@ func TestPartitionLifecycle(t *testing.T) { SELECT COUNT(*) FROM pg_inherits JOIN pg_class parent ON pg_inherits.inhparent = parent.oid - WHERE parent.relname IN ('prow_job_run_tests', 'prow_job_run_test_outputs', 'test_analysis_by_job_by_dates') + WHERE parent.relname IN ('prow_job_run_tests', 'prow_job_run_test_outputs') `).Scan(&partitionCountBefore).Error require.NoError(t, err) @@ -384,7 +384,7 @@ func TestPartitionLifecycle(t *testing.T) { SELECT COUNT(*) FROM pg_inherits JOIN pg_class parent ON pg_inherits.inhparent = parent.oid - WHERE parent.relname IN ('prow_job_run_tests', 'prow_job_run_test_outputs', 'test_analysis_by_job_by_dates') + WHERE parent.relname IN ('prow_job_run_tests', 'prow_job_run_test_outputs') `).Scan(&partitionCountAfter).Error require.NoError(t, err) diff --git a/test/integration/jobs_test.go b/test/integration/jobs_test.go index a09ffff66e..209bbfdfd7 100644 --- a/test/integration/jobs_test.go +++ b/test/integration/jobs_test.go @@ -765,31 +765,6 @@ func createSingleRun(t *testing.T, dbc *db.DB, jobID uint, release string, spec return run } -func TestLoadProwJobCache(t *testing.T) { - dbc := intutil.NewTestDB(t, pgContainer) - - intutil.CreateProwJob(t, dbc, "periodic-ci-e2e-aws-4.16", "4.16", []string{"aws"}) - intutil.CreateProwJob(t, dbc, "periodic-ci-e2e-gcp-4.16", "4.16", []string{"gcp"}) - intutil.CreateProwJob(t, dbc, "periodic-ci-e2e-aws-4.15", "4.15", []string{"aws"}) - - cache, err := query.LoadProwJobCache(dbc) - require.NoError(t, err) - - assert.Len(t, cache, 3) - assert.Contains(t, cache, "periodic-ci-e2e-aws-4.16") - assert.Contains(t, cache, "periodic-ci-e2e-gcp-4.16") - assert.Contains(t, cache, "periodic-ci-e2e-aws-4.15") - assert.Equal(t, "4.16", cache["periodic-ci-e2e-aws-4.16"].Release) -} - -func TestLoadProwJobCacheEmpty(t *testing.T) { - dbc := intutil.NewTestDB(t, pgContainer) - - cache, err := query.LoadProwJobCache(dbc) - require.NoError(t, err) - assert.Empty(t, cache) -} - func TestJobRunTestCount(t *testing.T) { dbc := intutil.NewTestDB(t, pgContainer) diff --git a/test/integration/util/schema.go b/test/integration/util/schema.go index 755996fad2..f1707d7f22 100644 --- a/test/integration/util/schema.go +++ b/test/integration/util/schema.go @@ -57,7 +57,6 @@ func SetupIntegrationSchema(dbc *db.DB) error { // Created here as regular tables for integration testing. &models.ProwJobRunTest{}, &models.ProwJobRunTestOutput{}, - &models.TestAnalysisByJobByDate{}, &models.TestDailyTotal{}, &models.TestCumulativeSummary{}, }