From 12f9de7d69562af7757e3edfb64de4b9f694f0c6 Mon Sep 17 00:00:00 2001 From: Matthew Staebler Date: Thu, 23 Jul 2026 14:01:57 -0400 Subject: [PATCH 1/2] Add analyze-db.sh script for post-restore database warmup After cloning or restoring the production database to staging, query performance is degraded due to stale planner statistics and lazy-loaded EBS storage. This script warms up the database via a detached pod that runs ANALYZE VERBOSE and REINDEX DATABASE CONCURRENTLY. Also adds argument validation to backfill-summaries.sh to produce clear error messages when a flag value is missing. Co-Authored-By: Claude Opus 4.6 --- README.md | 19 ++++++++ scripts/analyze-db.sh | 81 +++++++++++++++++++++++++++++++++++ scripts/backfill-summaries.sh | 14 +++--- 3 files changed, 107 insertions(+), 7 deletions(-) create mode 100755 scripts/analyze-db.sh diff --git a/README.md b/README.md index c4042d47b0..ca4d14f0fb 100644 --- a/README.md +++ b/README.md @@ -24,6 +24,25 @@ See [the API documentation](pkg/api/README.md) See [the front end documentation](sippy-ng/README.md) +## Database Operations + +After cloning or restoring the production database to staging, query +performance will be degraded due to stale planner statistics and +lazy-loaded EBS storage. Run `scripts/analyze-db.sh` to warm up the +database via a one-shot pod that runs `ANALYZE VERBOSE` (updates planner +statistics) and `REINDEX DATABASE CONCURRENTLY` (forces index pages off +lazy-loaded storage): + +```bash +./scripts/analyze-db.sh +``` + +The pod runs detached, so your local machine does not need to stay +connected. Use `--wait` to block until completion instead. The script +defaults to the `sippy` namespace and `postgres-aws` secret. Use +`--namespace` and `--db-secret` to override, and `--dry-run` to preview +the command without executing. + ## Chat See [the chat documentation](chat/README.md) diff --git a/scripts/analyze-db.sh b/scripts/analyze-db.sh new file mode 100755 index 0000000000..411de7f42f --- /dev/null +++ b/scripts/analyze-db.sh @@ -0,0 +1,81 @@ +#!/bin/bash +# Warm up the sippy database after cloning or restoring from a snapshot. +# Runs ANALYZE VERBOSE (updates planner statistics) followed by +# REINDEX DATABASE CONCURRENTLY (forces index pages off lazy-loaded EBS +# storage). Both are necessary for full query performance. +# +# The pod runs detached so your local machine does not need to stay +# connected. Use --wait to poll until completion instead. +# +# Usage: +# ./scripts/analyze-db.sh [OPTIONS] +# +# Options: +# --namespace NS Kubernetes namespace (default: sippy) +# --db-secret NAME Database secret name (default: postgres-aws) +# --wait Poll until the pod completes, then print logs +# --dry-run Print the command without executing + +set -euo pipefail + +NAMESPACE=sippy +DB_SECRET="postgres-aws" +WAIT=false +DRY_RUN=false + +while [[ $# -gt 0 ]]; do + case "$1" in + --namespace) [[ $# -ge 2 ]] || { echo "Error: --namespace requires a value" >&2; exit 1; }; NAMESPACE="$2"; shift 2 ;; + --db-secret) [[ $# -ge 2 ]] || { echo "Error: --db-secret requires a value" >&2; exit 1; }; DB_SECRET="$2"; shift 2 ;; + --wait) WAIT=true; shift ;; + --dry-run) DRY_RUN=true; shift ;; + *) echo "Unknown option: $1" >&2; exit 1 ;; + esac +done + +POD_NAME="sippy-analyze-db" +IMAGE="registry.redhat.io/rhel9/postgresql-16:latest" + +if [[ "$DRY_RUN" == "true" ]]; then + echo "Would create pod $POD_NAME to run ANALYZE VERBOSE and REINDEX" + exit 0 +fi + +oc -n "$NAMESPACE" delete pod "$POD_NAME" --ignore-not-found --wait >/dev/null 2>&1 || true + +echo "Creating pod $POD_NAME to run ANALYZE VERBOSE and REINDEX..." + +oc -n "$NAMESPACE" run "$POD_NAME" --restart=Never \ + --image="$IMAGE" \ + --overrides="{ + \"spec\": { + \"containers\": [{ + \"name\": \"$POD_NAME\", + \"image\": \"$IMAGE\", + \"command\": [\"sh\", \"-c\"], + \"args\": [\"echo 'Starting ANALYZE VERBOSE...' && psql \\\"\$SIPPY_DATABASE_DSN\\\" -c 'ANALYZE VERBOSE;' && echo 'Starting REINDEX DATABASE...' && psql \\\"\$SIPPY_DATABASE_DSN\\\" -c 'REINDEX DATABASE CONCURRENTLY sippy_openshift;' && echo 'Warmup complete.'\"], + \"env\": [{ + \"name\": \"SIPPY_DATABASE_DSN\", + \"valueFrom\": {\"secretKeyRef\": {\"name\": \"$DB_SECRET\", \"key\": \"dsn\"}} + }] + }] + } + }" + +if [[ "$WAIT" == "true" ]]; then + echo "Waiting for pod to complete..." + oc -n "$NAMESPACE" wait --for=jsonpath='{.status.phase}'=Succeeded --timeout=120m "pod/$POD_NAME" 2>/dev/null || { + STATUS=$(oc -n "$NAMESPACE" get pod "$POD_NAME" -o jsonpath='{.status.phase}' 2>/dev/null || echo "Unknown") + echo "Pod finished with status: $STATUS" >&2 + oc -n "$NAMESPACE" logs "$POD_NAME" --tail=20 2>/dev/null || true + exit 1 + } + echo "Warmup complete." + oc -n "$NAMESPACE" logs "$POD_NAME" --tail=5 + oc -n "$NAMESPACE" delete pod "$POD_NAME" --ignore-not-found >/dev/null 2>&1 || true +else + echo "Pod is running detached. To follow progress:" + echo " oc -n $NAMESPACE logs -f $POD_NAME" + echo "To check status:" + echo " oc -n $NAMESPACE get pod $POD_NAME" +fi diff --git a/scripts/backfill-summaries.sh b/scripts/backfill-summaries.sh index 6a47d60410..f667f1d64a 100755 --- a/scripts/backfill-summaries.sh +++ b/scripts/backfill-summaries.sh @@ -29,13 +29,13 @@ DRY_RUN=false while [[ $# -gt 0 ]]; do case "$1" in - --table) TABLE="$2"; shift 2 ;; - --days) DAYS="$2"; shift 2 ;; - --namespace) NAMESPACE="$2"; shift 2 ;; - --image) IMAGE="$2"; shift 2 ;; - --db-secret) DB_SECRET="$2"; shift 2 ;; - --pause) PAUSE="$2"; shift 2 ;; - --batch) BATCH="$2"; shift 2 ;; + --table) [[ $# -ge 2 ]] || { echo "Error: --table requires a value" >&2; exit 1; }; TABLE="$2"; shift 2 ;; + --days) [[ $# -ge 2 ]] || { echo "Error: --days requires a value" >&2; exit 1; }; DAYS="$2"; shift 2 ;; + --namespace) [[ $# -ge 2 ]] || { echo "Error: --namespace requires a value" >&2; exit 1; }; NAMESPACE="$2"; shift 2 ;; + --image) [[ $# -ge 2 ]] || { echo "Error: --image requires a value" >&2; exit 1; }; IMAGE="$2"; shift 2 ;; + --db-secret) [[ $# -ge 2 ]] || { echo "Error: --db-secret requires a value" >&2; exit 1; }; DB_SECRET="$2"; shift 2 ;; + --pause) [[ $# -ge 2 ]] || { echo "Error: --pause requires a value" >&2; exit 1; }; PAUSE="$2"; shift 2 ;; + --batch) [[ $# -ge 2 ]] || { echo "Error: --batch requires a value" >&2; exit 1; }; BATCH="$2"; shift 2 ;; --dry-run) DRY_RUN=true; shift ;; *) echo "Unknown option: $1" >&2; exit 1 ;; esac From 3b8b69dbf0e434ce416d112b949a53842cab69d3 Mon Sep 17 00:00:00 2001 From: Matthew Staebler Date: Sat, 25 Jul 2026 13:17:44 -0400 Subject: [PATCH 2/2] Add dynamic cache warming step to analyze-db.sh Adds a third warmup step that runs SELECT count(*) on all user tables to pull heap pages off lazy-loaded EBS storage. Tables with date/timestamp columns are scoped to the last 30 days for partition pruning. Partition children are skipped (queried through the parent). The pod command is refactored from inline shell to a heredoc-based script written to a temp file, avoiding deep escaping. Co-Authored-By: Claude Opus 4.6 --- README.md | 8 +++-- scripts/analyze-db.sh | 81 +++++++++++++++++++++++++++++++++++++++---- 2 files changed, 80 insertions(+), 9 deletions(-) diff --git a/README.md b/README.md index ca4d14f0fb..b0fe708340 100644 --- a/README.md +++ b/README.md @@ -29,9 +29,11 @@ See [the front end documentation](sippy-ng/README.md) After cloning or restoring the production database to staging, query performance will be degraded due to stale planner statistics and lazy-loaded EBS storage. Run `scripts/analyze-db.sh` to warm up the -database via a one-shot pod that runs `ANALYZE VERBOSE` (updates planner -statistics) and `REINDEX DATABASE CONCURRENTLY` (forces index pages off -lazy-loaded storage): +database via a one-shot pod that runs three steps: `ANALYZE VERBOSE` +(updates planner statistics), `REINDEX DATABASE CONCURRENTLY` (forces +index pages off lazy-loaded storage), and a cache warming pass that runs +`SELECT count(*)` on every table (scoped to the last 30 days for tables +with date/timestamp columns) to pull hot heap pages off S3: ```bash ./scripts/analyze-db.sh diff --git a/scripts/analyze-db.sh b/scripts/analyze-db.sh index 411de7f42f..622580b93a 100755 --- a/scripts/analyze-db.sh +++ b/scripts/analyze-db.sh @@ -1,8 +1,11 @@ #!/bin/bash # Warm up the sippy database after cloning or restoring from a snapshot. -# Runs ANALYZE VERBOSE (updates planner statistics) followed by -# REINDEX DATABASE CONCURRENTLY (forces index pages off lazy-loaded EBS -# storage). Both are necessary for full query performance. +# Runs three steps: +# 1. ANALYZE VERBOSE — updates planner statistics +# 2. REINDEX DATABASE CONCURRENTLY — forces index pages off lazy-loaded +# EBS storage +# 3. Cache warming — SELECT count(*) on all tables, scoped to the last +# 30 days for tables with date/timestamp columns # # The pod runs detached so your local machine does not need to stay # connected. Use --wait to poll until completion instead. @@ -37,13 +40,79 @@ POD_NAME="sippy-analyze-db" IMAGE="registry.redhat.io/rhel9/postgresql-16:latest" if [[ "$DRY_RUN" == "true" ]]; then - echo "Would create pod $POD_NAME to run ANALYZE VERBOSE and REINDEX" + echo "Would create pod $POD_NAME to run ANALYZE, REINDEX, and cache warming" exit 0 fi oc -n "$NAMESPACE" delete pod "$POD_NAME" --ignore-not-found --wait >/dev/null 2>&1 || true -echo "Creating pod $POD_NAME to run ANALYZE VERBOSE and REINDEX..." +echo "Creating pod $POD_NAME to warm up the database..." + +# The pod runs a shell script that writes a SQL warmup script to a temp +# file, then executes the three warmup steps in sequence. +WARMUP_SCRIPT=$(cat << 'SCRIPT_EOF' +set -e + +cat > /tmp/warmup.sql << 'SQL_EOF' +DO $$ +DECLARE + tbl RECORD; + date_col TEXT; + row_count BIGINT; + query TEXT; +BEGIN + FOR tbl IN + SELECT t.tablename + FROM pg_tables t + WHERE t.schemaname = 'public' + AND NOT EXISTS ( + SELECT 1 FROM pg_inherits i + JOIN pg_class c ON c.oid = i.inhrelid + WHERE c.relname = t.tablename + ) + ORDER BY t.tablename + LOOP + SELECT c.column_name INTO date_col + FROM information_schema.columns c + WHERE c.table_schema = 'public' + AND c.table_name = tbl.tablename + AND c.data_type IN ('date', 'timestamp with time zone', 'timestamp without time zone') + ORDER BY c.column_name + LIMIT 1; + + IF date_col IS NOT NULL THEN + query := format( + 'SELECT count(*) FROM %I WHERE %I >= now() - interval ''30 days''', + tbl.tablename, date_col + ); + ELSE + query := format('SELECT count(*) FROM %I', tbl.tablename); + END IF; + + EXECUTE query INTO row_count; + + IF date_col IS NOT NULL THEN + RAISE NOTICE 'Warmed % (last 30 days via %): % rows', tbl.tablename, date_col, row_count; + ELSE + RAISE NOTICE 'Warmed % (full table): % rows', tbl.tablename, row_count; + END IF; + END LOOP; +END +$$; +SQL_EOF + +echo "Step 1/3: ANALYZE VERBOSE..." +psql "$SIPPY_DATABASE_DSN" -c "ANALYZE VERBOSE;" + +echo "Step 2/3: REINDEX DATABASE..." +psql "$SIPPY_DATABASE_DSN" -c "REINDEX DATABASE CONCURRENTLY sippy_openshift;" + +echo "Step 3/3: Cache warming..." +psql "$SIPPY_DATABASE_DSN" -f /tmp/warmup.sql + +echo "Warmup complete." +SCRIPT_EOF +) oc -n "$NAMESPACE" run "$POD_NAME" --restart=Never \ --image="$IMAGE" \ @@ -53,7 +122,7 @@ oc -n "$NAMESPACE" run "$POD_NAME" --restart=Never \ \"name\": \"$POD_NAME\", \"image\": \"$IMAGE\", \"command\": [\"sh\", \"-c\"], - \"args\": [\"echo 'Starting ANALYZE VERBOSE...' && psql \\\"\$SIPPY_DATABASE_DSN\\\" -c 'ANALYZE VERBOSE;' && echo 'Starting REINDEX DATABASE...' && psql \\\"\$SIPPY_DATABASE_DSN\\\" -c 'REINDEX DATABASE CONCURRENTLY sippy_openshift;' && echo 'Warmup complete.'\"], + \"args\": [$(echo "$WARMUP_SCRIPT" | python3 -c "import sys,json; print(json.dumps(sys.stdin.read()))")], \"env\": [{ \"name\": \"SIPPY_DATABASE_DSN\", \"valueFrom\": {\"secretKeyRef\": {\"name\": \"$DB_SECRET\", \"key\": \"dsn\"}}