fix(kubernetes): fail sandbox after main container restart

kubernetes/internal/controller/apis.go

@@ -27,6 +27,7 @@ const (
 	AnnoAllocStatusKey           = "sandbox.opensandbox.io/alloc-status"
 	AnnoAllocReleaseKey          = "sandbox.opensandbox.io/alloc-release"
 	AnnoAllocReleasedKey         = "sandbox.opensandbox.io/alloc-released"
+	AnnoRestartBaselineKey       = "sandbox.opensandbox.io/restart-baseline"
 	LabelBatchSandboxPodIndexKey = "batch-sandbox.sandbox.opensandbox.io/pod-index"
 	LabelBatchSandboxNameKey     = "batch-sandbox.sandbox.opensandbox.io/name"
 	LabelPrivilegedNodeAccess    = "sandbox.opensandbox.io/privileged-node-access"

kubernetes/internal/controller/batchsandbox_pause_resume_test.go

@@ -16,6 +16,7 @@ package controller
 
 import (
 	"context"
+	"encoding/json"
 	"fmt"
 	"sync"
 	"testing"
@@ -1807,6 +1808,331 @@ func TestBuildRuntimeView_AggregatesPodFailuresInSteadyState(t *testing.T) {
 	assert.Equal(t, "3/4 observed pods failed; primary reason=ErrImagePull; sample pod=err-image-0", podFailed.Message)
 }
 
+func restartTestSandbox(readyAt metav1.Time, endpoint string) *sandboxv1alpha1.BatchSandbox {
+	return &sandboxv1alpha1.BatchSandbox{
+		ObjectMeta: metav1.ObjectMeta{Annotations: map[string]string{AnnotationSandboxEndpoints: fmt.Sprintf(`[%q]`, endpoint)}},
+		Status: sandboxv1alpha1.BatchSandboxStatus{
+			Phase:    sandboxv1alpha1.BatchSandboxPhaseSucceed,
+			Replicas: 1,
+			Conditions: []sandboxv1alpha1.BatchSandboxCondition{{
+				Type:               sandboxv1alpha1.BatchSandboxConditionReady,
+				Status:             sandboxv1alpha1.ConditionTrue,
+				Reason:             "PodsReady",
+				Message:            "Sandbox is running",
+				LastTransitionTime: &readyAt,
+			}},
+		},
+	}
+}
+
+func restartTestPod(name, endpoint string, createdAt, restartedAt metav1.Time) *corev1.Pod {
+	return &corev1.Pod{
+		ObjectMeta: metav1.ObjectMeta{Name: name, CreationTimestamp: createdAt},
+		Spec:       corev1.PodSpec{Containers: []corev1.Container{{Name: "sandbox"}}},
+		Status: corev1.PodStatus{
+			Phase:      corev1.PodRunning,
+			PodIP:      endpoint,
+			Conditions: []corev1.PodCondition{{Type: corev1.PodReady, Status: corev1.ConditionTrue}},
+			ContainerStatuses: []corev1.ContainerStatus{{
+				Name:         "sandbox",
+				RestartCount: 1,
+				LastTerminationState: corev1.ContainerState{Terminated: &corev1.ContainerStateTerminated{
+					Reason:     "OOMKilled",
+					FinishedAt: restartedAt,
+				}},
+			}},
+		},
+	}
+}
+
+func assertRestartHistoryIgnored(t *testing.T, view runtimeView, readyAt metav1.Time) {
+	t.Helper()
+	assert.Equal(t, sandboxv1alpha1.BatchSandboxPhaseSucceed, view.status.Phase)
+	for _, condition := range view.status.Conditions {
+		assert.NotEqual(t, sandboxv1alpha1.BatchSandboxConditionPodFailed, condition.Type)
+		if condition.Type == sandboxv1alpha1.BatchSandboxConditionReady {
+			require.NotNil(t, condition.LastTransitionTime)
+			assert.Equal(t, readyAt.Time, condition.LastTransitionTime.Time,
+				"pod membership changes must not alter the Ready transition time")
+		}
+	}
+}
+
+func TestBuildRuntimeView_FailsWhenContainerRestartsAfterReady(t *testing.T) {
+	readyAt := metav1.NewTime(time.Now().Add(-time.Minute))
+	restartedAt := metav1.NewTime(readyAt.Add(30 * time.Second))
+	view := buildRuntimeView(restartTestSandbox(readyAt, "10.0.0.10"), []*corev1.Pod{
+		restartTestPod("oom-restarted", "10.0.0.10", metav1.Time{}, restartedAt),
+	})
+
+	assert.Equal(t, sandboxv1alpha1.BatchSandboxPhaseFailed, view.status.Phase)
+	for _, condition := range view.status.Conditions {
+		if condition.Type == sandboxv1alpha1.BatchSandboxConditionPodFailed {
+			assert.Equal(t, "OOMKilled", condition.Reason)
+			return
+		}
+	}
+	t.Fatal("expected PodFailed condition")
+}
+
+func TestBuildRuntimeView_DetectsRestartAcrossSpecOnlyUpdate(t *testing.T) {
+	readyAt := metav1.NewTime(time.Now().Add(-time.Minute))
+	restartedAt := metav1.NewTime(readyAt.Add(30 * time.Second))
+	bs := restartTestSandbox(readyAt, "10.0.0.10")
+	bs.Generation = 2
+	bs.Status.ObservedGeneration = 1
+
+	view := buildRuntimeView(bs, []*corev1.Pod{
+		restartTestPod("oom-restarted", "10.0.0.10", metav1.Time{}, restartedAt),
+	})
+
+	assert.Equal(t, sandboxv1alpha1.BatchSandboxPhaseFailed, view.status.Phase)
+}
+
+func TestBuildRuntimeView_FailsWhenMainContainerTerminatesAfterReady(t *testing.T) {
+	readyAt := metav1.NewTime(time.Now().Add(-time.Minute))
+	terminatedAt := metav1.NewTime(readyAt.Add(30 * time.Second))
+	pod := restartTestPod("oom-terminated", "10.0.0.10", metav1.Time{}, terminatedAt)
+	pod.Status.ContainerStatuses[0].RestartCount = 0
+	pod.Status.ContainerStatuses[0].State.Terminated = pod.Status.ContainerStatuses[0].LastTerminationState.Terminated
+	pod.Status.ContainerStatuses[0].LastTerminationState = corev1.ContainerState{}
+
+	view := buildRuntimeView(restartTestSandbox(readyAt, "10.0.0.10"), []*corev1.Pod{pod})
+
+	assert.Equal(t, sandboxv1alpha1.BatchSandboxPhaseFailed, view.status.Phase)
+}
+
+func TestGetPodFailureReasonAndMessage_IgnoresSidecarRestart(t *testing.T) {
+	readyAt := metav1.NewTime(time.Now().Add(-time.Minute))
+	pod := restartTestPod("sidecar-restarted", "10.0.0.10", metav1.Time{}, metav1.NewTime(readyAt.Add(30*time.Second)))
+	pod.Spec.Containers = append(pod.Spec.Containers, corev1.Container{Name: "egress"})
+	pod.Status.ContainerStatuses[0].Name = "egress"
+
+	reason, message, failed := getPodFailureReasonAndMessage(pod, &readyAt)
+	assert.False(t, failed)
+	assert.Empty(t, reason)
+	assert.Empty(t, message)
+}
+
+func TestBuildRuntimeView_IgnoresContainerRestartBeforeReady(t *testing.T) {
+	restartedAt := metav1.NewTime(time.Now().Add(-2 * time.Minute))
+	readyAt := metav1.NewTime(restartedAt.Add(time.Minute))
+	view := buildRuntimeView(restartTestSandbox(readyAt, "10.0.0.10"), []*corev1.Pod{
+		restartTestPod("prewarmed", "10.0.0.10", metav1.Time{}, restartedAt),
+	})
+	assertRestartHistoryIgnored(t, view, readyAt)
+}
+
+func TestBuildRuntimeView_IgnoresRestartHistoryWhenPodMembershipChanges(t *testing.T) {
+	readyAt := metav1.NewTime(time.Now().Add(-2 * time.Minute))
+	restartedAt := metav1.NewTime(readyAt.Add(time.Minute))
+	view := buildRuntimeView(restartTestSandbox(readyAt, "10.0.0.9"), []*corev1.Pod{
+		restartTestPod("new-prewarmed-pod", "10.0.0.10", metav1.Time{}, restartedAt),
+	})
+	assertRestartHistoryIgnored(t, view, readyAt)
+}
+
+func TestBuildRuntimeView_IgnoresRestartHistoryFromNewPodReusingEndpoint(t *testing.T) {
+	readyAt := metav1.NewTime(time.Now().Add(-2 * time.Minute))
+	createdAt := metav1.NewTime(readyAt.Add(30 * time.Second))
+	restartedAt := metav1.NewTime(createdAt.Add(30 * time.Second))
+	view := buildRuntimeView(restartTestSandbox(readyAt, "10.0.0.10"), []*corev1.Pod{
+		restartTestPod("replacement-pod", "10.0.0.10", createdAt, restartedAt),
+	})
+	assertRestartHistoryIgnored(t, view, readyAt)
+}
+
+func TestBuildRuntimeView_DetectsExistingPodRestartDuringScaleUp(t *testing.T) {
+	readyAt := metav1.NewTime(time.Now().Add(-2 * time.Minute))
+	restartedAt := metav1.NewTime(readyAt.Add(time.Minute))
+	createdBeforeReady := metav1.NewTime(readyAt.Add(-time.Minute))
+	bs := restartTestSandbox(readyAt, "10.0.0.10")
+
+	view := buildRuntimeView(bs, []*corev1.Pod{
+		restartTestPod("existing-restarted", "10.0.0.10", createdBeforeReady, restartedAt),
+		restartTestPod("new-prewarmed", "10.0.0.11", createdBeforeReady, restartedAt),
+	})
+
+	assert.Equal(t, sandboxv1alpha1.BatchSandboxPhaseFailed, view.status.Phase)
+	for _, condition := range view.status.Conditions {
+		if condition.Type == sandboxv1alpha1.BatchSandboxConditionPodFailed {
+			assert.Equal(t, "1/2 observed pods failed; primary reason=OOMKilled; sample pod=existing-restarted", condition.Message)
+			return
+		}
+	}
+	t.Fatal("expected PodFailed condition")
+}
+
+func TestBuildRuntimeView_DetectsDelayedExistingPodRestartAfterScaleUp(t *testing.T) {
+	readyAt := metav1.NewTime(time.Now().Add(-3 * time.Minute))
+	createdBeforeReady := metav1.NewTime(readyAt.Add(-time.Minute))
+	preReadyRestart := metav1.NewTime(readyAt.Add(-30 * time.Second))
+	bs := restartTestSandbox(readyAt, "10.0.0.10")
+
+	existing := restartTestPod("existing", "10.0.0.10", createdBeforeReady, preReadyRestart)
+	newPod := restartTestPod("new-prewarmed", "10.0.0.11", createdBeforeReady, preReadyRestart)
+	firstView := buildRuntimeView(bs, []*corev1.Pod{existing, newPod})
+	require.Equal(t, sandboxv1alpha1.BatchSandboxPhaseSucceed, firstView.status.Phase)
+
+	bs.Status = *firstView.status
+	endpointRaw, err := json.Marshal(firstView.endpointIPs)
+	require.NoError(t, err)
+	baselineRaw, err := json.Marshal(firstView.restartDetectionBaseline)
+	require.NoError(t, err)
+	bs.Annotations[AnnotationSandboxEndpoints] = string(endpointRaw)
+	bs.Annotations[AnnoRestartBaselineKey] = string(baselineRaw)
+
+	// The informer reports the old Pod's restart only after the scale-up
+	// reconcile. Its original baseline must still detect the delayed evidence.
+	delayedRestart := metav1.NewTime(readyAt.Add(time.Minute))
+	existing = restartTestPod("existing", "10.0.0.10", createdBeforeReady, delayedRestart)
+	secondView := buildRuntimeView(bs, []*corev1.Pod{existing, newPod})
+	assert.Equal(t, sandboxv1alpha1.BatchSandboxPhaseFailed, secondView.status.Phase)
+}
+
+func TestBuildRuntimeView_DetectsNewPodRestartAfterEndpointExposure(t *testing.T) {
+	readyAt := metav1.NewTime(time.Now().Add(-3 * time.Minute))
+	preExposureRestart := metav1.NewTime(readyAt.Add(time.Minute))
+	bs := restartTestSandbox(readyAt, "10.0.0.10")
+	newPod := restartTestPod("new-prewarmed", "10.0.0.11", metav1.Time{}, preExposureRestart)
+
+	firstView := buildRuntimeView(bs, []*corev1.Pod{newPod})
+	assertRestartHistoryIgnored(t, firstView, readyAt)
+	exposedAtNanos, exists := firstView.restartDetectionBaseline[newPod.Name]
+	require.True(t, exists)
+	exposedAt := time.Unix(0, exposedAtNanos)
+	assert.True(t, exposedAt.After(preExposureRestart.Time))
+
+	bs.Status = *firstView.status
+	endpointRaw, err := json.Marshal(firstView.endpointIPs)
+	require.NoError(t, err)
+	baselineRaw, err := json.Marshal(firstView.restartDetectionBaseline)
+	require.NoError(t, err)
+	bs.Annotations[AnnotationSandboxEndpoints] = string(endpointRaw)
+	bs.Annotations[AnnoRestartBaselineKey] = string(baselineRaw)
+
+	postExposureRestart := metav1.NewTime(exposedAt.Add(time.Second))
+	newPod = restartTestPod("new-prewarmed", "10.0.0.11", metav1.Time{}, postExposureRestart)
+	secondView := buildRuntimeView(bs, []*corev1.Pod{newPod})
+	assert.Equal(t, sandboxv1alpha1.BatchSandboxPhaseFailed, secondView.status.Phase)
+}
+
+func TestBuildRuntimeView_InitialBaselinesMatchReadyTransition(t *testing.T) {
+	bs := &sandboxv1alpha1.BatchSandbox{}
+	pod := restartTestPod("initial", "10.0.0.10", metav1.Time{}, metav1.Time{})
+
+	view := buildRuntimeView(bs, []*corev1.Pod{pod})
+	baseline, exists := view.restartDetectionBaseline[pod.Name]
+	require.True(t, exists)
+	for _, condition := range view.status.Conditions {
+		if condition.Type == sandboxv1alpha1.BatchSandboxConditionReady {
+			require.NotNil(t, condition.LastTransitionTime)
+			assert.Equal(t, condition.LastTransitionTime.UnixNano(), baseline)
+			return
+		}
+	}
+	t.Fatal("expected Ready condition")
+}
+
+func TestPersistRuntimeView_AtomicallyPublishesEndpointBaselines(t *testing.T) {
+	readyAt := metav1.NewTime(time.Now().Add(-3 * time.Minute))
+	bs := restartTestSandbox(readyAt, "10.0.0.10")
+	bs.Name = "test-bs"
+	bs.Namespace = "default"
+	bs.UID = "test-bs-uid"
+	bs.ResourceVersion = "1"
+	existing := restartTestPod("existing", "10.0.0.10", metav1.Time{}, metav1.NewTime(readyAt.Add(-time.Minute)))
+	existing.UID = "existing-uid"
+	newPod := restartTestPod("new", "10.0.0.11", metav1.Time{}, metav1.NewTime(readyAt.Add(time.Minute)))
+	newPod.UID = "new-uid"
+	r := newTestReconciler(bs.DeepCopy())
+
+	view := buildRuntimeView(bs.DeepCopy(), []*corev1.Pod{existing, newPod})
+	_, errs := r.persistRuntimeView(context.Background(), bs.DeepCopy(), view)
+	require.Empty(t, errs)
+
+	updated := &sandboxv1alpha1.BatchSandbox{}
+	require.NoError(t, r.Get(context.Background(), types.NamespacedName{Namespace: bs.Namespace, Name: bs.Name}, updated))
+	assert.Equal(t, `["10.0.0.10","10.0.0.11"]`, updated.Annotations[AnnotationSandboxEndpoints])
+	persisted := map[string]int64{}
+	require.NoError(t, json.Unmarshal([]byte(updated.Annotations[AnnoRestartBaselineKey]), &persisted))
+	assert.Equal(t, readyAt.UnixNano(), persisted["existing-uid"])
+	assert.Greater(t, persisted["new-uid"], readyAt.UnixNano())
+
+	for _, condition := range updated.Status.Conditions {
+		if condition.Type == sandboxv1alpha1.BatchSandboxConditionReady {
+			require.NotNil(t, condition.LastTransitionTime)
+			assert.Equal(t, readyAt.Unix(), condition.LastTransitionTime.Unix())
+		}
+	}
+}
+
+func TestPersistRuntimeView_DoesNotExposeEndpointsFromStaleRuntimeView(t *testing.T) {
+	readyAt := metav1.NewTime(time.Now().Add(-3 * time.Minute))
+	bs := restartTestSandbox(readyAt, "10.0.0.10")
+	bs.Name = "test-bs"
+	bs.Namespace = "default"
+	bs.UID = "test-bs-uid"
+	bs.ResourceVersion = "1"
+	bs.Annotations[AnnoRestartBaselineKey] = fmt.Sprintf(`{"existing-uid":%d}`, readyAt.UnixNano())
+	r := newTestReconciler(bs.DeepCopy())
+	r.StatusRVExpectation.Expect(&sandboxv1alpha1.BatchSandbox{ObjectMeta: metav1.ObjectMeta{UID: bs.UID, ResourceVersion: "2"}})
+
+	view := runtimeView{
+		status:                   bs.Status.DeepCopy(),
+		endpointIPs:              []string{"10.0.0.10", "10.0.0.11"},
+		restartDetectionBaseline: map[string]int64{"existing-uid": readyAt.UnixNano(), "new-uid": time.Now().UnixNano()},
+	}
+	requeue, errs := r.persistRuntimeView(context.Background(), bs.DeepCopy(), view)
+	require.Empty(t, errs)
+	assert.Equal(t, time.Second, requeue)
+
+	updated := &sandboxv1alpha1.BatchSandbox{}
+	require.NoError(t, r.Get(context.Background(), types.NamespacedName{Namespace: bs.Namespace, Name: bs.Name}, updated))
+	assert.Equal(t, `["10.0.0.10"]`, updated.Annotations[AnnotationSandboxEndpoints])
+	assert.NotContains(t, updated.Annotations[AnnoRestartBaselineKey], "new-uid")
+}
+
+func TestBuildRuntimeView_DetectsDelayedRestartAfterPodOrderChanges(t *testing.T) {
+	readyAt := metav1.NewTime(time.Now().Add(-2 * time.Minute))
+	createdBeforeReady := metav1.NewTime(readyAt.Add(-time.Minute))
+	preReadyRestart := metav1.NewTime(readyAt.Add(-30 * time.Second))
+	bs := restartTestSandbox(readyAt, "10.0.0.1")
+	bs.Annotations[AnnotationSandboxEndpoints] = `["10.0.0.1","10.0.0.2"]`
+	bs.Status.Replicas = 2
+
+	// A Pod list reorder must not advance the Ready baseline while the
+	// endpoint membership is unchanged and restart status is still stale.
+	firstView := buildRuntimeView(bs, []*corev1.Pod{
+		restartTestPod("pod-2", "10.0.0.2", createdBeforeReady, preReadyRestart),
+		restartTestPod("pod-1", "10.0.0.1", createdBeforeReady, preReadyRestart),
+	})
+	require.Equal(t, sandboxv1alpha1.BatchSandboxPhaseSucceed, firstView.status.Phase)
+	var preservedReadyAt *metav1.Time
+	for i := range firstView.status.Conditions {
+		condition := &firstView.status.Conditions[i]
+		if condition.Type == sandboxv1alpha1.BatchSandboxConditionReady {
+			preservedReadyAt = condition.LastTransitionTime
+			break
+		}
+	}
+	require.NotNil(t, preservedReadyAt)
+	assert.Equal(t, readyAt.Time, preservedReadyAt.Time)
+
+	// A later informer update exposes a restart that happened after the
+	// original Ready transition. It must still fail the sandbox.
+	bs.Status = *firstView.status
+	reorderedEndpoints, err := json.Marshal(firstView.endpointIPs)
+	require.NoError(t, err)
+	bs.Annotations[AnnotationSandboxEndpoints] = string(reorderedEndpoints)
+	delayedRestart := metav1.NewTime(readyAt.Add(time.Minute))
+	secondView := buildRuntimeView(bs, []*corev1.Pod{
+		restartTestPod("pod-2", "10.0.0.2", createdBeforeReady, preReadyRestart),
+		restartTestPod("pod-1", "10.0.0.1", createdBeforeReady, delayedRestart),
+	})
+	assert.Equal(t, sandboxv1alpha1.BatchSandboxPhaseFailed, secondView.status.Phase)
+}
+
 func TestBuildRuntimeView_AggregatesResumeFailures(t *testing.T) {
 	bs := &sandboxv1alpha1.BatchSandbox{
 		ObjectMeta: metav1.ObjectMeta{
@@ -1882,9 +2208,14 @@ func TestBuildRuntimeView_PreservesConditionTransitionTimeWhenUnchanged(t *testi
 			Name:       "test-bs",
 			Namespace:  "default",
 			Generation: 3,
+			Annotations: map[string]string{
+				AnnotationSandboxEndpoints: `["10.0.0.10"]`,
+			},
 		},
 		Status: sandboxv1alpha1.BatchSandboxStatus{
-			Phase: sandboxv1alpha1.BatchSandboxPhaseSucceed,
+			ObservedGeneration: 3,
+			Phase:              sandboxv1alpha1.BatchSandboxPhaseSucceed,
+			Replicas:           1,
 			Conditions: []sandboxv1alpha1.BatchSandboxCondition{
 				{
 					Type:               sandboxv1alpha1.BatchSandboxConditionReady,