alerts: a flapping crash reason no longer wipes the app's diagnosis

A crashlooping pod alternates between Waiting/CrashLoopBackOff and
terminated-with-nonzero-exit, so detectPodAlertAt returns 'CrashLoopBackOff'
or 'Error' for one unchanged failure. Every reason comparison in the watcher
was a raw string equality, so each flap read as a new incident: the ELSE
branch of touchAppHealthAlertSeen wrote NULLIF('','') over cause, cause_line
and cause_kind even on a tick that read no fresh evidence, and
first_detected_at was reset in both touchAppHealthAlertSeen and
claimAppHealthAlertSlot, so the escalating cooldown ladder could never
advance and 'crashing since' was wrong.

Live proof 2026-09-27 [psql + kubectl]: wow83168-gmail-com-prod/nodejs-argo
(683 restarts, log says Cannot find module '/tmp/index.js') and
lifecoachrussia-yandex-ru-prod/gulyaev-ai-core (1512 restarts, log says Full
Knowledge Vault could not be opened) both sat in CrashLoopBackOff with empty
cause_kind while the cause was readable from their logs; nodejs-argo still
carried last_sent_cause_kind='app_code', proving a verdict had been known and
then erased. Across the table 2/82 rows show that emailed-then-erased
signature, 21/82 have first_detected_at newer than last_sent_at (an email
sent before the incident supposedly began) and 32/64 successfully sent alert
emails carried no diagnosis.

Reasons are now compared by incident class: CrashLoopBackOff and Error are
one class, ImagePullBackOff and ErrImagePull are one class, OOMKilled and
anything unrecognized map to themselves. The class mapping lives once in
reasonIncidentClass and once as the SQL fragment appHealthReasonClassSQLCase,
reused by both statements so they cannot drift. maybeCauseRefresh's cheap-skip
compares classes too, so a pure flap no longer forces a live kube GetLogs call
every tick for every crashlooping app.

A genuine class change still clears a stale verdict, which is the regression
this guard exists for: an old ErrImagePull verdict must not survive the flip
to CrashLoopBackOff and keep blaming the registry for the user's own crash.

Tests RAN against a real postgres (17 RUN, 17 PASS, 0 SKIP), including the six
pre-existing app-health guards. Mutation check: collapsing reasonIncidentClass
to the raw reason fails three class tests, restoring passes.

Author: Алекс

automator/state/backlog/items/0521-upload-без-манифеста-framework-даёт-success-билд.md

@@ -0,0 +1,262 @@
+package api
+
+import (
+	"context"
+	"testing"
+	"time"
+
+	"github.com/google/uuid"
+)
+
+// TestTouchAppHealthAlertSeenSurvivesSameClassFlapWhenNotRefreshed is
+// RED-proof for the live bug (2026-09-27): wow83168-gmail-com-prod/
+// nodejs-argo and lifecoachrussia-yandex-ru-prod/gulyaev-ai-core both have a
+// crash whose exact log line (see the seeded cause below) has not changed,
+// yet their live app_health_alerts row has cause/cause_line/cause_kind all
+// empty because detectPodAlertAt's raw reason flapped between
+// reasonCrashLoopBackOff and reasonError on the same underlying crash, and
+// the old code compared raw reason strings: the moment the reason flipped,
+// touchAppHealthAlertSeen's cause CASE fell into its ELSE branch and wrote
+// NULLIF passed empty strings resulting in a NULL write unconditionally, even though refreshed=false (no
+// new evidence was read this tick). This proves a same-CLASS flap
+// (CrashLoopBackOff -> Error) with refreshed=false must keep the stored
+// triplet intact.
+func TestTouchAppHealthAlertSeenSurvivesSameClassFlapWhenNotRefreshed(t *testing.T) {
+	pool := testAdvisoryPool(t)
+	ctx := context.Background()
+	ns := "test-ns-flap-keep-" + uuid.NewString()[:8]
+	t.Cleanup(func() {
+		_, _ = pool.Exec(context.Background(), `DELETE FROM app_health_alerts WHERE namespace = $1`, ns)
+	})
+
+	touchAppHealthAlertSeen(ctx, pool, ns, "nodejs-argo", reasonCrashLoopBackOff, "pod-1/nodejs-argo",
+		"Судя по логам, это ошибка в коде приложения (Node.js).", "Error: Cannot find module '/tmp/index.js'", "app_code", true)
+
+	touchAppHealthAlertSeen(ctx, pool, ns, "nodejs-argo", reasonError, "pod-1/nodejs-argo exit=1", "", "", "", false)
+
+	var cause, causeLine, causeKind string
+	if err := pool.QueryRow(ctx,
+		`SELECT COALESCE(cause, ''), COALESCE(cause_line, ''), COALESCE(cause_kind, '') FROM app_health_alerts WHERE namespace = $1 AND app_name = 'nodejs-argo'`,
+		ns).Scan(&cause, &causeLine, &causeKind); err != nil {
+		t.Fatalf("read back row after flap: %v", err)
+	}
+	if cause == "" || causeLine == "" || causeKind == "" {
+		t.Fatalf("expected the stored diagnosis to survive a same-class flap (CrashLoopBackOff -> Error) with refreshed=false, got cause=%q cause_line=%q cause_kind=%q", cause, causeLine, causeKind)
+	}
+	if causeLine != "Error: Cannot find module '/tmp/index.js'" {
+		t.Fatalf("expected the original cause_line to be unchanged, got %q", causeLine)
+	}
+
+	touchAppHealthAlertSeen(ctx, pool, ns, "nodejs-argo", reasonCrashLoopBackOff, "pod-1/nodejs-argo", "", "", "", false)
+	if err := pool.QueryRow(ctx,
+		`SELECT COALESCE(cause, ''), COALESCE(cause_line, ''), COALESCE(cause_kind, '') FROM app_health_alerts WHERE namespace = $1 AND app_name = 'nodejs-argo'`,
+		ns).Scan(&cause, &causeLine, &causeKind); err != nil {
+		t.Fatalf("read back row after flap back: %v", err)
+	}
+	if cause == "" || causeLine == "" || causeKind == "" {
+		t.Fatalf("expected the stored diagnosis to survive flapping back to CrashLoopBackOff, got cause=%q cause_line=%q cause_kind=%q", cause, causeLine, causeKind)
+	}
+}
+
+// TestTouchAppHealthAlertSeenClearsOnGenuineClassChange is the CRITICAL
+// correctness guard this fix must preserve (see task diagnosis): an old
+// ErrImagePull verdict (cause_kind=platform_registry, the live shape from
+// the 2026-08-30 gateway/internal-prod incident) must NOT survive a genuine
+// class change to CrashLoopBackOff, even though both reasons individually
+// flap against a sibling (ErrImagePull<->ImagePullBackOff,
+// CrashLoopBackOff<->Error). ImagePullBackOff and CrashLoopBackOff are
+// different classes, so this transition with no new verdict (refreshed=true,
+// causeKind empty -- the fresh log matched no signature) must clear the
+// stale triplet, exactly as the pre-existing
+// TestTouchAppHealthAlertSeenClearsStaleCauseWhenRefreshed test already
+// pins for the raw-reason case; this test pins the same contract survives
+// the class-based comparison.
+func TestTouchAppHealthAlertSeenClearsOnGenuineClassChange(t *testing.T) {
+	pool := testAdvisoryPool(t)
+	ctx := context.Background()
+	ns := "test-ns-class-change-clear-" + uuid.NewString()[:8]
+	t.Cleanup(func() {
+		_, _ = pool.Exec(context.Background(), `DELETE FROM app_health_alerts WHERE namespace = $1`, ns)
+	})
+
+	touchAppHealthAlertSeen(ctx, pool, ns, "gateway", reasonErrImagePull, "pod-1/gateway",
+		"Похоже, проблема на стороне реестра образов.", "", "platform_registry", true)
+
+	var cause, causeLine, causeKind string
+	if err := pool.QueryRow(ctx,
+		`SELECT COALESCE(cause, ''), COALESCE(cause_line, ''), COALESCE(cause_kind, '') FROM app_health_alerts WHERE namespace = $1 AND app_name = 'gateway'`,
+		ns).Scan(&cause, &causeLine, &causeKind); err != nil {
+		t.Fatalf("read back seeded row: %v", err)
+	}
+	if causeKind != "platform_registry" {
+		t.Fatalf("expected the seeded platform_registry cause to be stored, got cause_kind=%q", causeKind)
+	}
+
+	touchAppHealthAlertSeen(ctx, pool, ns, "gateway", reasonCrashLoopBackOff, "pod-2/gateway", "", "", "", true)
+
+	if err := pool.QueryRow(ctx,
+		`SELECT COALESCE(cause, ''), COALESCE(cause_line, ''), COALESCE(cause_kind, '') FROM app_health_alerts WHERE namespace = $1 AND app_name = 'gateway'`,
+		ns).Scan(&cause, &causeLine, &causeKind); err != nil {
+		t.Fatalf("read back row after class change: %v", err)
+	}
+	if cause != "" || causeLine != "" || causeKind != "" {
+		t.Fatalf("expected the stale platform_registry verdict to be cleared on a genuine class change (ImagePull -> CrashLoop), got cause=%q cause_line=%q cause_kind=%q", cause, causeLine, causeKind)
+	}
+}
+
+// TestTouchAppHealthAlertSeenOverwritesOnPositiveNewVerdictAcrossFlap proves
+// the fourth required behavior: a positive new classification (OOMKilled,
+// refreshed=true) always overwrites, even when it arrives on top of an
+// incident that was itself mid-flap between CrashLoopBackOff and Error. The
+// class guard must never block a genuine new diagnosis from landing.
+func TestTouchAppHealthAlertSeenOverwritesOnPositiveNewVerdictAcrossFlap(t *testing.T) {
+	pool := testAdvisoryPool(t)
+	ctx := context.Background()
+	ns := "test-ns-oom-overwrite-" + uuid.NewString()[:8]
+	t.Cleanup(func() {
+		_, _ = pool.Exec(context.Background(), `DELETE FROM app_health_alerts WHERE namespace = $1`, ns)
+	})
+
+	touchAppHealthAlertSeen(ctx, pool, ns, "worker", reasonCrashLoopBackOff, "pod-1/worker",
+		"Судя по логам, это ошибка в коде приложения.", "panic: boom", "app_code", true)
+	touchAppHealthAlertSeen(ctx, pool, ns, "worker", reasonError, "pod-1/worker exit=2", "", "", "", false)
+
+	touchAppHealthAlertSeen(ctx, pool, ns, "worker", reasonOOMKilled, "pod-1/worker",
+		"Приложению не хватает памяти.", "", "resource_limit", true)
+
+	var cause, causeLine, causeKind string
+	if err := pool.QueryRow(ctx,
+		`SELECT COALESCE(cause, ''), COALESCE(cause_line, ''), COALESCE(cause_kind, '') FROM app_health_alerts WHERE namespace = $1 AND app_name = 'worker'`,
+		ns).Scan(&cause, &causeLine, &causeKind); err != nil {
+		t.Fatalf("read back row after OOM overwrite: %v", err)
+	}
+	if causeKind != "resource_limit" {
+		t.Fatalf("expected the OOMKilled verdict to overwrite the previous app_code verdict, got cause_kind=%q", causeKind)
+	}
+	if causeLine != "" {
+		t.Fatalf("resource_limit verdicts carry no cause_line by design, got %q", causeLine)
+	}
+}
+
+// TestFirstDetectedAtSurvivesSameClassFlap proves the incident-age fix
+// directly: first_detected_at must not reset to now() on a same-class flap.
+// Live proof this matters: gulyaev-ai-core has 1512 restarts but
+// first_detected_at was observed at 2026-09-27 05:56, i.e. minutes before
+// the row was read, because every flap between CrashLoopBackOff and Error
+// was resetting the incident's age and so the escalating cooldown ladder
+// (appHealthAlertEscalationSteps: 24h -> 72h after 3d -> weekly after 14d)
+// could never advance for a genuinely long-running, constantly-flapping
+// incident.
+func TestFirstDetectedAtSurvivesSameClassFlap(t *testing.T) {
+	pool := testAdvisoryPool(t)
+	ctx := context.Background()
+	ns := "test-ns-first-detected-flap-" + uuid.NewString()[:8]
+	t.Cleanup(func() {
+		_, _ = pool.Exec(context.Background(), `DELETE FROM app_health_alerts WHERE namespace = $1`, ns)
+	})
+
+	touchAppHealthAlertSeen(ctx, pool, ns, "gulyaev-ai-core", reasonCrashLoopBackOff, "pod-1/gulyaev-ai-core", "", "", "", false)
+
+	if _, err := pool.Exec(ctx,
+		`UPDATE app_health_alerts SET first_detected_at = now() - interval '5 days' WHERE namespace = $1 AND app_name = 'gulyaev-ai-core'`,
+		ns); err != nil {
+		t.Fatalf("backdate first_detected_at: %v", err)
+	}
+
+	var firstDetected time.Time
+	if err := pool.QueryRow(ctx,
+		`SELECT first_detected_at FROM app_health_alerts WHERE namespace = $1 AND app_name = 'gulyaev-ai-core'`,
+		ns).Scan(&firstDetected); err != nil {
+		t.Fatalf("read back backdated first_detected_at: %v", err)
+	}
+
+	touchAppHealthAlertSeen(ctx, pool, ns, "gulyaev-ai-core", reasonError, "pod-1/gulyaev-ai-core exit=1", "", "", "", false)
+	touchAppHealthAlertSeen(ctx, pool, ns, "gulyaev-ai-core", reasonCrashLoopBackOff, "pod-1/gulyaev-ai-core", "", "", "", false)
+
+	var firstDetectedAfterFlap time.Time
+	if err := pool.QueryRow(ctx,
+		`SELECT first_detected_at FROM app_health_alerts WHERE namespace = $1 AND app_name = 'gulyaev-ai-core'`,
+		ns).Scan(&firstDetectedAfterFlap); err != nil {
+		t.Fatalf("read back first_detected_at after flapping: %v", err)
+	}
+	if !firstDetectedAfterFlap.Equal(firstDetected) {
+		t.Fatalf("expected first_detected_at to survive a same-class flap, got %v (was %v)", firstDetectedAfterFlap, firstDetected)
+	}
+}
+
+// TestFirstDetectedAtResetsOnGenuineClassChange is the paired guard: a real
+// class change (a new failure kind starting) must still reset the incident
+// age, or a stale first_detected_at would keep an unrelated NEW failure on
+// the tail end of the OLD incident's escalated (slower) cooldown cadence.
+func TestFirstDetectedAtResetsOnGenuineClassChange(t *testing.T) {
+	pool := testAdvisoryPool(t)
+	ctx := context.Background()
+	ns := "test-ns-first-detected-reset-" + uuid.NewString()[:8]
+	t.Cleanup(func() {
+		_, _ = pool.Exec(context.Background(), `DELETE FROM app_health_alerts WHERE namespace = $1`, ns)
+	})
+
+	touchAppHealthAlertSeen(ctx, pool, ns, "web", reasonImagePullBackOff, "pod-1/web", "", "", "", false)
+	if _, err := pool.Exec(ctx,
+		`UPDATE app_health_alerts SET first_detected_at = now() - interval '10 days' WHERE namespace = $1 AND app_name = 'web'`,
+		ns); err != nil {
+		t.Fatalf("backdate first_detected_at: %v", err)
+	}
+
+	touchAppHealthAlertSeen(ctx, pool, ns, "web", reasonCrashLoopBackOff, "pod-2/web", "", "", "", false)
+
+	var firstDetected time.Time
+	if err := pool.QueryRow(ctx,
+		`SELECT first_detected_at FROM app_health_alerts WHERE namespace = $1 AND app_name = 'web'`,
+		ns).Scan(&firstDetected); err != nil {
+		t.Fatalf("read back first_detected_at after class change: %v", err)
+	}
+	if time.Since(firstDetected) > time.Minute {
+		t.Fatalf("expected first_detected_at to reset to roughly now() on a genuine class change, got %v (age %v)", firstDetected, time.Since(firstDetected))
+	}
+}
+
+// TestClaimAppHealthAlertSlotFirstDetectedAtSurvivesSameClassFlap proves the
+// same first_detected_at protection applies to claimAppHealthAlertSlot, the
+// second write path the diagnosis identifies (line ~555): the cooldown
+// claim's own first_detected_at CASE must also compare by class, not by raw
+// reason, or the escalation ladder still resets every time an email claim
+// lands mid-flap even though touchAppHealthAlertSeen's copy is fixed.
+func TestClaimAppHealthAlertSlotFirstDetectedAtSurvivesSameClassFlap(t *testing.T) {
+	pool := testAdvisoryPool(t)
+	ctx := context.Background()
+	ns := "test-ns-claim-flap-" + uuid.NewString()[:8]
+	t.Cleanup(func() {
+		_, _ = pool.Exec(context.Background(), `DELETE FROM app_health_alerts WHERE namespace = $1`, ns)
+	})
+
+	if !claimAppHealthAlertSlot(ctx, pool, ns, "web", reasonCrashLoopBackOff, "pod/web", 24*time.Hour) {
+		t.Fatalf("first claim must succeed")
+	}
+	if _, err := pool.Exec(ctx,
+		`UPDATE app_health_alerts SET first_detected_at = now() - interval '20 days', last_sent_at = now() - interval '8 days' WHERE namespace = $1 AND app_name = 'web'`,
+		ns); err != nil {
+		t.Fatalf("backdate row: %v", err)
+	}
+
+	var firstDetected time.Time
+	if err := pool.QueryRow(ctx,
+		`SELECT first_detected_at FROM app_health_alerts WHERE namespace = $1 AND app_name = 'web'`,
+		ns).Scan(&firstDetected); err != nil {
+		t.Fatalf("read back backdated first_detected_at: %v", err)
+	}
+
+	if !claimAppHealthAlertSlot(ctx, pool, ns, "web", reasonError, "pod/web exit=1", 24*time.Hour) {
+		t.Fatalf("claim on a same-class flap past the escalated cooldown must still succeed")
+	}
+
+	var firstDetectedAfter time.Time
+	if err := pool.QueryRow(ctx,
+		`SELECT first_detected_at FROM app_health_alerts WHERE namespace = $1 AND app_name = 'web'`,
+		ns).Scan(&firstDetectedAfter); err != nil {
+		t.Fatalf("read back first_detected_at after claim: %v", err)
+	}
+	if !firstDetectedAfter.Equal(firstDetected) {
+		t.Fatalf("expected claimAppHealthAlertSlot to preserve first_detected_at across a same-class flap, got %v (was %v)", firstDetectedAfter, firstDetected)
+	}
+}