Skip to content

fix: keep pods healthy when tenant and host run different K8s versions #4037

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
flomedja wants to merge 4 commits into
base: main
Choose a base branch
from
Open

fix: keep pods healthy when tenant and host run different K8s versions #4037

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
f88a2ee
fix: keep pods healthy when tenant and host run different K8s versions
flomedja Jun 26, 2026
e29cfac
fix: strip per-condition observedGeneration and apply review feedback
flomedja Jun 26, 2026
2f5bc6c
fix: fetch host cluster version once and apply review feedback
flomedja Jul 6, 2026
e8d0226
fix: avoid repeated status updates for terminating pods and add WithC…
flomedja Jul 7, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion e2e/test_core/sync/test_k8sdefaultendpoint.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -78,7 +78,7 @@ func K8sDefaultEndpointSpec() {
}
g.Expect(hostClusterIps).To(Equal(vClusterIps), "IPs should match between host and vcluster endpoints")
g.Expect(hostClusterPorts).To(Equal(vClusterPorts), "Ports should match between host and vcluster endpoints")
}).
}).WithContext(ctx).
WithPolling(constants.PollingInterval).
WithTimeout(constants.PollingTimeoutVeryLong).
Should(Succeed(), "Endpoints should match after waiting")
Expand Down
2 changes: 1 addition & 1 deletion e2e/test_core/sync/test_networkpolicy.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -77,7 +77,7 @@ func NetworkPolicySyncSpec() {
networkingv1.PolicyTypeIngress,
networkingv1.PolicyTypeEgress,
), "synced policy should preserve policy types")
}).
}).WithContext(ctx).
WithPolling(constants.PollingInterval).
WithTimeout(constants.PollingTimeout).
Should(Succeed())
Expand Down
2 changes: 1 addition & 1 deletion e2e/test_core/sync/test_node.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -50,7 +50,7 @@ func NodeSyncLabelSelectorSpec() {
g.Expect(hostSyncedNodeName).ToNot(BeEmpty(), "Should find host node with matching hostname")
g.Expect(virtualNodes.Items).To(HaveLen(1), "Expected exactly one synced node")
g.Expect(virtualNodes.Items[0].Name).To(Equal(hostSyncedNodeName), "Synced node name should match")
}).
}).WithContext(ctx).
WithPolling(constants.PollingInterval).
WithTimeout(constants.PollingTimeout).
Should(Succeed(), "Node sync should work correctly")
Expand Down
110 changes: 100 additions & 10 deletions e2e/test_core/sync/test_pods.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -79,7 +79,7 @@ func PodSyncSpec() {
g.Expect(err).NotTo(HaveOccurred(), "failed to get pod %s/%s", ns, podName)
g.Expect(pod.Status.Phase).To(Equal(corev1.PodRunning),
"pod %s/%s phase is %s, not yet Running", ns, podName, pod.Status.Phase)
}).WithPolling(constants.PollingInterval).WithTimeout(constants.PollingTimeoutLong).Should(Succeed())
}).WithContext(ctx).WithPolling(constants.PollingInterval).WithTimeout(constants.PollingTimeoutLong).Should(Succeed())
}

defaultSecurityContext := func() *corev1.SecurityContext {
Expand Down Expand Up @@ -159,7 +159,7 @@ func PodSyncSpec() {
g.Expect(err).To(Succeed())
g.Expect(p.Status.EphemeralContainerStatuses).NotTo(BeEmpty(),
"expected ephemeral container statuses to be present")
}).WithPolling(constants.PollingInterval).WithTimeout(constants.PollingTimeout).Should(Succeed())
}).WithContext(ctx).WithPolling(constants.PollingInterval).WithTimeout(constants.PollingTimeout).Should(Succeed())
}
}
})
Expand Down Expand Up @@ -297,7 +297,7 @@ func PodSyncSpec() {
g.Expect(len(pPod.Status.Conditions)).To(BeNumerically(">=", 5),
"expected >= 5 conditions on host pod (4 standard + custom gate), got %d: %v",
len(pPod.Status.Conditions), pPod.Status.Conditions)
}).WithPolling(constants.PollingInterval).WithTimeout(constants.PollingTimeoutLong).Should(Succeed())
}).WithContext(ctx).WithPolling(constants.PollingInterval).WithTimeout(constants.PollingTimeoutLong).Should(Succeed())
})
})

Expand All @@ -320,7 +320,7 @@ func PodSyncSpec() {
Eventually(func(g Gomega) {
_, err := vClusterClient.CoreV1().ServiceAccounts(ns).Get(ctx, saName, metav1.GetOptions{})
g.Expect(err).To(Succeed())
}).WithPolling(constants.PollingInterval).WithTimeout(constants.PollingTimeout).Should(Succeed())
}).WithContext(ctx).WithPolling(constants.PollingInterval).WithTimeout(constants.PollingTimeout).Should(Succeed())
})

By("Creating a pod using the non-default service account", func() {
Expand Down Expand Up @@ -544,7 +544,7 @@ func PodSyncSpec() {
pSvc, err := hostClient.CoreV1().Services(hostNSForSvc).Get(ctx, pSvcName, metav1.GetOptions{})
g.Expect(err).NotTo(HaveOccurred(), "host service not yet synced")
g.Expect(pSvc.Spec.ClusterIP).NotTo(BeEmpty(), "host service ClusterIP not yet assigned")
}).WithPolling(constants.PollingInterval).WithTimeout(constants.PollingTimeout).Should(Succeed())
}).WithContext(ctx).WithPolling(constants.PollingInterval).WithTimeout(constants.PollingTimeout).Should(Succeed())

By("Creating a pod with dependent env vars", func() {
_, err := vClusterClient.CoreV1().Pods(ns).Create(ctx, &corev1.Pod{
Expand Down Expand Up @@ -672,7 +672,7 @@ func PodSyncSpec() {
g.Expect(pKey).NotTo(BeEmpty(),
"namespace label with value %q not yet propagated to host pod, labels: %v",
additionalLabelValue, pPod.GetLabels())
}).WithPolling(constants.PollingInterval).WithTimeout(constants.PollingTimeout).Should(Succeed())
}).WithContext(ctx).WithPolling(constants.PollingInterval).WithTimeout(constants.PollingTimeout).Should(Succeed())
})
})

Expand Down Expand Up @@ -797,7 +797,7 @@ func PodSyncSpec() {
"annotation not synced from host to vCluster")
g.Expect(vPod.Labels).To(HaveKeyWithValue(additionalLabelKey, additionalLabelValue),
"label not synced from host to vCluster")
}).WithPolling(constants.PollingInterval).WithTimeout(constants.PollingTimeout).Should(Succeed())
}).WithContext(ctx).WithPolling(constants.PollingInterval).WithTimeout(constants.PollingTimeout).Should(Succeed())
})

additionalLabelValueFromVCluster := "good-syncer-from-vcluster"
Expand Down Expand Up @@ -831,7 +831,7 @@ func PodSyncSpec() {
"annotation not synced from vCluster to host")
g.Expect(pPod.Labels).To(HaveKeyWithValue(additionalLabelKey, additionalLabelValueFromVCluster),
"label not synced from vCluster to host")
}).WithPolling(constants.PollingInterval).WithTimeout(constants.PollingTimeout).Should(Succeed())
}).WithContext(ctx).WithPolling(constants.PollingInterval).WithTimeout(constants.PollingTimeout).Should(Succeed())
})
})

Expand Down Expand Up @@ -866,7 +866,7 @@ func PodSyncSpec() {
g.Expect(err).To(Succeed())
g.Expect(vpod.Status.ObservedGeneration).To(BeNumerically("==", 1),
"status.observedGeneration is %d, expected 1", vpod.Status.ObservedGeneration)
}).WithPolling(constants.PollingInterval).WithTimeout(constants.PollingTimeout).Should(Succeed())
}).WithContext(ctx).WithPolling(constants.PollingInterval).WithTimeout(constants.PollingTimeout).Should(Succeed())
})

By("Updating pod tolerations to trigger a generation bump", func() {
Expand All @@ -892,7 +892,97 @@ func PodSyncSpec() {
g.Expect(err).To(Succeed())
g.Expect(vpod.Status.ObservedGeneration).To(BeNumerically(">=", 2),
"status.observedGeneration is %d, expected >= 2", vpod.Status.ObservedGeneration)
}).WithPolling(constants.PollingInterval).WithTimeout(constants.PollingTimeoutLong).Should(Succeed())
}).WithContext(ctx).WithPolling(constants.PollingInterval).WithTimeout(constants.PollingTimeoutLong).Should(Succeed())
})
})

It("should preserve virtual pod QOS class", func(ctx context.Context) {
// Regression test for https://github.com/loft-sh/vcluster/issues/3578, where pods
// got stuck at Ready=False. The syncer used to copy the virtual QOS class onto the
// host pod, so a later status update tried to change the host's QOS class. K8s 1.32+
// treats that field as immutable and rejects the update.
suffix := random.String(6)
ns := "pod-qos-test-" + suffix
createTestNamespace(ctx, ns)

podName := "qos-" + suffix
By("Creating a pod with no resource requests (virtual QOS class: BestEffort)", func() {
_, err := vClusterClient.CoreV1().Pods(ns).Create(ctx, &corev1.Pod{
ObjectMeta: metav1.ObjectMeta{Name: podName},
Spec: corev1.PodSpec{
Containers: []corev1.Container{{
Name: testingContainerName,
Image: testingContainerImage,
ImagePullPolicy: corev1.PullIfNotPresent,
SecurityContext: defaultSecurityContext(),
}},
},
}, metav1.CreateOptions{})
Expect(err).To(Succeed())
})

By("Waiting for the pod to be Running", func() {
waitPodRunning(ctx, podName, ns)
})

By("Verifying the virtual pod QOS class is BestEffort (not overwritten from host)", func() {
vpod, err := vClusterClient.CoreV1().Pods(ns).Get(ctx, podName, metav1.GetOptions{})
Expect(err).To(Succeed())
Expect(vpod.Status.QOSClass).To(Equal(corev1.PodQOSBestEffort),
"virtual pod QOS class should reflect the virtual apiserver's computation (BestEffort for no resource requests), not the host pod's QOS class")
})
})

It("should keep pod Ready condition stable after reaching Ready=True", func(ctx context.Context) {
// Regression test for https://github.com/loft-sh/vcluster/issues/3578, where pods
// kept switching back to Ready=False because the syncer wrongly thought their
// conditions had changed on every reconcile.
//
// This only happens when the host runs K8s >= 1.34 and the virtual cluster runs
// < 1.34. On a same-version cluster the bug can't reproduce, so the real check
// lives in the unit test TestDiffPodStatusObservedGeneration.
suffix := random.String(6)
ns := "pod-cond-stable-test-" + suffix
createTestNamespace(ctx, ns)

podName := "cond-stable-" + suffix
By("Creating a pod", func() {
_, err := vClusterClient.CoreV1().Pods(ns).Create(ctx, &corev1.Pod{
ObjectMeta: metav1.ObjectMeta{Name: podName},
Spec: corev1.PodSpec{
Containers: []corev1.Container{{
Name: testingContainerName,
Image: testingContainerImage,
ImagePullPolicy: corev1.PullIfNotPresent,
SecurityContext: defaultSecurityContext(),
}},
},
}, metav1.CreateOptions{})
Expect(err).To(Succeed())
})

readyCondition := And(
HaveField("Type", corev1.PodReady),
HaveField("Status", corev1.ConditionTrue),
)

By("Waiting for the pod to reach Running phase with Ready=True", func() {
waitPodRunning(ctx, podName, ns)
Eventually(func(g Gomega) {
pod, err := vClusterClient.CoreV1().Pods(ns).Get(ctx, podName, metav1.GetOptions{})
g.Expect(err).To(Succeed())
g.Expect(pod.Status.Conditions).To(ContainElement(readyCondition),
"pod Ready condition is not yet True: %v", pod.Status.Conditions)
}).WithContext(ctx).WithPolling(constants.PollingInterval).WithTimeout(constants.PollingTimeoutLong).Should(Succeed())
})

By("Verifying Ready condition stays True and does not flap", func() {
Consistently(func(g Gomega) {
pod, err := vClusterClient.CoreV1().Pods(ns).Get(ctx, podName, metav1.GetOptions{})
g.Expect(err).To(Succeed())
g.Expect(pod.Status.Conditions).To(ContainElement(readyCondition),
"pod Ready condition flapped away from True: %v", pod.Status.Conditions)
}).WithContext(ctx).WithPolling(constants.PollingInterval).WithTimeout(constants.PollingTimeoutShort).Should(Succeed())
})
})
},
Expand Down
10 changes: 5 additions & 5 deletions e2e/test_core/sync/test_pvc.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -82,7 +82,7 @@ func PVCSyncSpec() {
Eventually(func(g Gomega) {
_, err := vClusterClient.CoreV1().ServiceAccounts(nsName).Get(ctx, "default", metav1.GetOptions{})
g.Expect(err).To(Succeed())
}).WithPolling(constants.PollingInterval).WithTimeout(constants.PollingTimeout).Should(Succeed())
}).WithContext(ctx).WithPolling(constants.PollingInterval).WithTimeout(constants.PollingTimeout).Should(Succeed())
})

_, err = vClusterClient.CoreV1().Pods(nsName).Create(ctx, &corev1.Pod{
Expand Down Expand Up @@ -117,7 +117,7 @@ func PVCSyncSpec() {
g.Expect(err).To(Succeed())
g.Expect(vpvc.Status.Phase).To(Equal(corev1.ClaimBound),
"PVC phase is %s, not yet Bound", vpvc.Status.Phase)
}).WithPolling(constants.PollingInterval).WithTimeout(constants.PollingTimeoutVeryLong).Should(Succeed())
}).WithContext(ctx).WithPolling(constants.PollingInterval).WithTimeout(constants.PollingTimeoutVeryLong).Should(Succeed())
})

By("Verifying PVC status matches between vCluster and host", func() {
Expand Down Expand Up @@ -151,15 +151,15 @@ func PVCSyncSpec() {
_, err := vClusterClient.CoreV1().PersistentVolumeClaims(nsName).Get(ctx, pvcName, metav1.GetOptions{})
g.Expect(kerrors.IsNotFound(err)).To(BeTrue(),
"PVC %s/%s not yet deleted", nsName, pvcName)
}).WithPolling(constants.PollingInterval).WithTimeout(constants.PollingTimeoutLong).Should(Succeed())
}).WithContext(ctx).WithPolling(constants.PollingInterval).WithTimeout(constants.PollingTimeoutLong).Should(Succeed())
})

By("Waiting for the PV to be deleted from vCluster", func() {
Eventually(func(g Gomega) {
_, err := vClusterClient.CoreV1().PersistentVolumes().Get(ctx, pvName, metav1.GetOptions{})
g.Expect(kerrors.IsNotFound(err)).To(BeTrue(),
"PV %s not yet deleted", pvName)
}).WithPolling(constants.PollingInterval).WithTimeout(constants.PollingTimeoutLong).Should(Succeed())
}).WithContext(ctx).WithPolling(constants.PollingInterval).WithTimeout(constants.PollingTimeoutLong).Should(Succeed())
})

By("Verifying the host PVC is also deleted", func() {
Expand All @@ -168,7 +168,7 @@ func PVCSyncSpec() {
_, err := hostClient.CoreV1().PersistentVolumeClaims(hostNS).Get(ctx, hostPvcName, metav1.GetOptions{})
g.Expect(kerrors.IsNotFound(err)).To(BeTrue(),
"host PVC %s/%s not yet deleted", hostNS, hostPvcName)
}).WithPolling(constants.PollingInterval).WithTimeout(constants.PollingTimeoutLong).Should(Succeed())
}).WithContext(ctx).WithPolling(constants.PollingInterval).WithTimeout(constants.PollingTimeoutLong).Should(Succeed())
})
})
},
Expand Down
Loading
Loading