From 21ef991dc36208841dcdfd6b9a614ac8543c37de Mon Sep 17 00:00:00 2001 From: Pablo Acevedo Montserrat Date: Thu, 9 Jul 2026 17:24:18 +0200 Subject: [PATCH 1/3] Move RBAC manifests to the services that own them The cluster-policy-controller RBAC was applied by infrastructure-services-manager, forcing cluster-policy-controller to depend on it. This created a deadlock on restart: infrastructure-services-manager creates deployments whose SCC admission blocks for up to 13s waiting for namespace UID-range annotations, but those annotations are set by the namespace-security-allocation-controller inside cluster-policy-controller, which cannot start until infrastructure-services-manager finishes. On a fresh boot this was masked because target namespaces did not exist yet and the deployment create timed out at the client side, allowing a retry after the annotations were set. On restart the namespaces already exist, so the deployment create goes straight to admission and the deadlock is exposed. Move each service's RBAC into its own Run method: - cluster-policy-controller applies its 6 RBAC manifests and drops the infrastructure-services-manager dependency, restoring the original dependency on kube-apiserver only - kube-controller-manager applies its CSR approver RBAC alongside its own namespaces - infrastructure-services-manager no longer applies RBAC for other services Bug: https://issues.redhat.com/browse/OCPBUGS-98219 --- pkg/controllers/cluster-policy-controller.go | 30 ++++++++++++++++-- pkg/controllers/infra-services-controller.go | 33 -------------------- pkg/controllers/kube-controller-manager.go | 20 ++++++++++-- 3 files changed, 45 insertions(+), 38 deletions(-) diff --git a/pkg/controllers/cluster-policy-controller.go b/pkg/controllers/cluster-policy-controller.go index f074f71b8d..f8e5305929 100644 --- a/pkg/controllers/cluster-policy-controller.go +++ b/pkg/controllers/cluster-policy-controller.go @@ -22,6 +22,7 @@ import ( clusterpolicycontroller "github.com/openshift/cluster-policy-controller/pkg/cmd/cluster-policy-controller" "github.com/openshift/library-go/pkg/controller/controllercmd" "github.com/openshift/library-go/pkg/operator/events" + "github.com/openshift/microshift/pkg/assets" "github.com/openshift/microshift/pkg/config" corev1 "k8s.io/api/core/v1" unstructuredv1 "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" @@ -32,6 +33,7 @@ import ( type ClusterPolicyController struct { run func(context.Context) error + applyRBAC func(context.Context) error kubeconfig string configErr error @@ -45,11 +47,31 @@ func NewClusterPolicyController(cfg *config.Config) *ClusterPolicyController { func (s *ClusterPolicyController) Name() string { return "cluster-policy-controller" } func (s *ClusterPolicyController) Dependencies() []string { - return []string{"kube-apiserver", "infrastructure-services-manager"} + return []string{"kube-apiserver"} } func (s *ClusterPolicyController) configure(cfg *config.Config) error { s.kubeconfig = cfg.KubeConfigPath(config.ClusterPolicyController) + s.applyRBAC = func(ctx context.Context) error { + kubeconfigPath := cfg.KubeConfigPath(config.KubeAdmin) + cr := []string{ + "controllers/cluster-policy-controller/namespace-security-allocation-controller-clusterrole.yaml", + "controllers/cluster-policy-controller/podsecurity-admission-label-syncer-controller-clusterrole.yaml", + "controllers/cluster-policy-controller/podsecurity-admission-label-privileged-namespaces-syncer-controller-clusterrole.yaml", + } + crb := []string{ + "controllers/cluster-policy-controller/namespace-security-allocation-controller-clusterrolebinding.yaml", + "controllers/cluster-policy-controller/podsecurity-admission-label-syncer-controller-clusterrolebinding.yaml", + "controllers/cluster-policy-controller/podsecurity-admission-label-privileged-namespaces-syncer-controller-clusterrolebinding.yaml", + } + if err := assets.ApplyClusterRoles(ctx, cr, kubeconfigPath); err != nil { + return fmt.Errorf("failed to apply cluster-policy-controller RBAC: %w", err) + } + if err := assets.ApplyClusterRoleBindings(ctx, crb, kubeconfigPath); err != nil { + return fmt.Errorf("failed to apply cluster-policy-controller RBAC: %w", err) + } + return nil + } scheme := runtime.NewScheme() if err := openshiftcontrolplanev1.AddToScheme(scheme); err != nil { @@ -101,6 +123,10 @@ func (s *ClusterPolicyController) Run(ctx context.Context, ready chan<- struct{} return fmt.Errorf("configuration failed: %w", s.configErr) } - close(ready) // todo + if err := s.applyRBAC(ctx); err != nil { + return err + } + + close(ready) return s.run(ctx) } diff --git a/pkg/controllers/infra-services-controller.go b/pkg/controllers/infra-services-controller.go index 628a6c7076..e8e9a1b76e 100644 --- a/pkg/controllers/infra-services-controller.go +++ b/pkg/controllers/infra-services-controller.go @@ -44,48 +44,15 @@ func (s *InfrastructureServicesManager) Run(ctx context.Context, ready chan<- st defer close(stopped) defer close(ready) - if err := applyDefaultRBACs(ctx, s.cfg); err != nil { - klog.Errorf("%s unable to apply default RBACs: %v", s.Name(), err) - return err - } - priorityClasses := []string{"core/priority-class-openshift-user-critical.yaml"} if err := assets.ApplyPriorityClasses(ctx, priorityClasses, s.cfg.KubeConfigPath(config.KubeAdmin)); err != nil { klog.Errorf("%s unable to apply PriorityClasses: %v", s.Name(), err) return err } - // TO-DO add readiness check if err := components.StartComponents(s.cfg, ctx); err != nil { return err } klog.Infof("%s launched ocp componets", s.Name()) return ctx.Err() } - -func applyDefaultRBACs(ctx context.Context, cfg *config.Config) error { - kubeconfigPath := cfg.KubeConfigPath(config.KubeAdmin) - var ( - cr = []string{ - "controllers/kube-controller-manager/csr_approver_clusterrole.yaml", - "controllers/cluster-policy-controller/namespace-security-allocation-controller-clusterrole.yaml", - "controllers/cluster-policy-controller/podsecurity-admission-label-syncer-controller-clusterrole.yaml", - "controllers/cluster-policy-controller/podsecurity-admission-label-privileged-namespaces-syncer-controller-clusterrole.yaml", - } - crb = []string{ - "controllers/kube-controller-manager/csr_approver_clusterrolebinding.yaml", - "controllers/cluster-policy-controller/namespace-security-allocation-controller-clusterrolebinding.yaml", - "controllers/cluster-policy-controller/podsecurity-admission-label-syncer-controller-clusterrolebinding.yaml", - "controllers/cluster-policy-controller/podsecurity-admission-label-privileged-namespaces-syncer-controller-clusterrolebinding.yaml", - } - ) - if err := assets.ApplyClusterRoles(ctx, cr, kubeconfigPath); err != nil { - klog.Warningf("failed to apply cluster roles %v", err) - return err - } - if err := assets.ApplyClusterRoleBindings(ctx, crb, kubeconfigPath); err != nil { - klog.Warningf("failed to apply cluster roles %v", err) - return err - } - return nil -} diff --git a/pkg/controllers/kube-controller-manager.go b/pkg/controllers/kube-controller-manager.go index 409f87e694..a5f2ea662e 100644 --- a/pkg/controllers/kube-controller-manager.go +++ b/pkg/controllers/kube-controller-manager.go @@ -111,10 +111,24 @@ func configure(ctx context.Context, cfg *config.Config) (args []string, applyFn args, err = mergeAndConvertToArgs(overrides) applyFn = func() error { - return assets.ApplyNamespaces(ctx, []string{ + kubeconfigPath := cfg.KubeConfigPath(config.KubeAdmin) + if err := assets.ApplyNamespaces(ctx, []string{ "controllers/kube-controller-manager/namespace-openshift-kube-controller-manager.yaml", "core/namespace-openshift-infra.yaml", - }, cfg.KubeConfigPath(config.KubeAdmin)) + }, kubeconfigPath); err != nil { + return fmt.Errorf("failed to apply kube-controller-manager namespaces: %w", err) + } + if err := assets.ApplyClusterRoles(ctx, []string{ + "controllers/kube-controller-manager/csr_approver_clusterrole.yaml", + }, kubeconfigPath); err != nil { + return fmt.Errorf("failed to apply kube-controller-manager RBAC: %w", err) + } + if err := assets.ApplyClusterRoleBindings(ctx, []string{ + "controllers/kube-controller-manager/csr_approver_clusterrolebinding.yaml", + }, kubeconfigPath); err != nil { + return fmt.Errorf("failed to apply kube-controller-manager RBAC: %w", err) + } + return nil } return args, applyFn, err } @@ -157,7 +171,7 @@ func (s *KubeControllerManager) Run(ctx context.Context, ready chan<- struct{}, }() if err := s.applyFn(); err != nil { - return fmt.Errorf("failed to apply openshift namespaces: %w", err) + return err } select { From 949f08ed5ba29aa357e0fee8c142d4788178d472 Mon Sep 17 00:00:00 2001 From: Pablo Acevedo Montserrat Date: Thu, 9 Jul 2026 22:16:44 +0200 Subject: [PATCH 2/3] Add readiness check to cluster-policy-controller The previous commit broke the circular dependency but left a race: infrastructure-services-manager could still submit deployment creates before cluster-policy-controller had annotated namespaces with openshift.io/sa.scc.uid-range, depending on CPU speed. Start the controller in a goroutine and poll the default namespace for the UID range annotation before signaling ready. Add cluster-policy-controller as a dependency of infrastructure-services-manager so that deployment creates are guaranteed to pass SCC admission regardless of system load. Bug: https://issues.redhat.com/browse/OCPBUGS-98219 --- pkg/controllers/cluster-policy-controller.go | 43 +++++++++++++++++++- pkg/controllers/infra-services-controller.go | 2 +- 2 files changed, 43 insertions(+), 2 deletions(-) diff --git a/pkg/controllers/cluster-policy-controller.go b/pkg/controllers/cluster-policy-controller.go index f8e5305929..36fcf2486c 100644 --- a/pkg/controllers/cluster-policy-controller.go +++ b/pkg/controllers/cluster-policy-controller.go @@ -17,17 +17,24 @@ package controllers import ( "context" "fmt" + "time" openshiftcontrolplanev1 "github.com/openshift/api/openshiftcontrolplane/v1" + securityv1 "github.com/openshift/api/security/v1" clusterpolicycontroller "github.com/openshift/cluster-policy-controller/pkg/cmd/cluster-policy-controller" "github.com/openshift/library-go/pkg/controller/controllercmd" "github.com/openshift/library-go/pkg/operator/events" "github.com/openshift/microshift/pkg/assets" "github.com/openshift/microshift/pkg/config" corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" unstructuredv1 "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/runtime/serializer" + "k8s.io/apimachinery/pkg/util/wait" + "k8s.io/client-go/kubernetes" + "k8s.io/client-go/tools/clientcmd" + klog "k8s.io/klog/v2" "k8s.io/utils/clock" ) @@ -127,6 +134,40 @@ func (s *ClusterPolicyController) Run(ctx context.Context, ready chan<- struct{} return err } + errCh := make(chan error, 1) + go func() { + errCh <- s.run(ctx) + }() + + if err := waitForNamespaceSecurityAllocation(ctx, s.kubeconfig); err != nil { + return err + } + klog.Infof("%s is ready", s.Name()) close(ready) - return s.run(ctx) + + return <-errCh +} + +// waitForNamespaceSecurityAllocation waits until the namespace-security-allocation-controller +// has annotated the default namespace with the UID range annotation, proving it is running +// and processing namespaces. This must happen before infrastructure-services-manager starts +// creating deployments, because the SCC admission plugin blocks until these annotations exist. +func waitForNamespaceSecurityAllocation(ctx context.Context, kubeconfigPath string) error { + restConfig, err := clientcmd.BuildConfigFromFlags("", kubeconfigPath) + if err != nil { + return fmt.Errorf("failed to build kubeconfig for readiness check: %w", err) + } + kubeClient, err := kubernetes.NewForConfig(restConfig) + if err != nil { + return fmt.Errorf("failed to create kube client for readiness check: %w", err) + } + + return wait.PollUntilContextCancel(ctx, time.Second, true, func(ctx context.Context) (bool, error) { + ns, err := kubeClient.CoreV1().Namespaces().Get(ctx, "default", metav1.GetOptions{}) + if err != nil { + return false, nil + } + _, ok := ns.Annotations[securityv1.UIDRangeAnnotation] + return ok, nil + }) } diff --git a/pkg/controllers/infra-services-controller.go b/pkg/controllers/infra-services-controller.go index e8e9a1b76e..3f0207ec38 100644 --- a/pkg/controllers/infra-services-controller.go +++ b/pkg/controllers/infra-services-controller.go @@ -37,7 +37,7 @@ func NewInfrastructureServices(cfg *config.Config) *InfrastructureServicesManage func (s *InfrastructureServicesManager) Name() string { return "infrastructure-services-manager" } func (s *InfrastructureServicesManager) Dependencies() []string { - return []string{"kube-apiserver", "openshift-crd-manager", "route-controller-manager"} + return []string{"kube-apiserver", "cluster-policy-controller", "openshift-crd-manager", "route-controller-manager"} } func (s *InfrastructureServicesManager) Run(ctx context.Context, ready chan<- struct{}, stopped chan<- struct{}) error { From 934f14657977ac6ec2b19fedf51f5cdb2086795f Mon Sep 17 00:00:00 2001 From: Pablo Acevedo Montserrat Date: Mon, 13 Jul 2026 10:05:44 +0200 Subject: [PATCH 3/3] Address review feedback for cluster-policy-controller readiness - Use select on errCh and waitCh so that if the controller dies before the readiness poll completes, the error is surfaced immediately instead of blocking forever - Log progress every 5 attempts during the readiness poll to make hangs diagnosable - Fix comment: the annotation check guarantees SCC admission will not block, but does not prove the controller is running on restart (the annotation persists from the previous boot) - Differentiate cluster role vs cluster role binding error messages in both CPC and KCM Bug: https://issues.redhat.com/browse/OCPBUGS-98219 --- pkg/cmd/run.go | 2 +- pkg/controllers/cluster-policy-controller.go | 85 +++++++++++++------- pkg/controllers/kube-controller-manager.go | 4 +- 3 files changed, 57 insertions(+), 34 deletions(-) diff --git a/pkg/cmd/run.go b/pkg/cmd/run.go index 7ea654c2ea..ca2b132f91 100644 --- a/pkg/cmd/run.go +++ b/pkg/cmd/run.go @@ -228,8 +228,8 @@ func RunMicroshift(cfg *config.Config) error { util.Must(m.AddService(controllers.NewRouteControllerManager(cfg))) util.Must(m.AddService(controllers.NewOpenShiftDefaultSCCManager(cfg))) util.Must(m.AddService(mdns.NewMicroShiftmDNSController(cfg))) - util.Must(m.AddService(controllers.NewInfrastructureServices(cfg))) util.Must(m.AddService(controllers.NewClusterPolicyController(cfg))) + util.Must(m.AddService(controllers.NewInfrastructureServices(cfg))) util.Must(m.AddService(controllers.NewVersionManager(cfg))) util.Must(m.AddService(controllers.NewKubeletCAManager(cfg))) util.Must(m.AddService(node.NewKubeletServer(cfg))) diff --git a/pkg/controllers/cluster-policy-controller.go b/pkg/controllers/cluster-policy-controller.go index 36fcf2486c..9462406737 100644 --- a/pkg/controllers/cluster-policy-controller.go +++ b/pkg/controllers/cluster-policy-controller.go @@ -26,6 +26,7 @@ import ( "github.com/openshift/library-go/pkg/operator/events" "github.com/openshift/microshift/pkg/assets" "github.com/openshift/microshift/pkg/config" + corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" unstructuredv1 "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" @@ -39,9 +40,9 @@ import ( ) type ClusterPolicyController struct { - run func(context.Context) error - applyRBAC func(context.Context) error - kubeconfig string + run func(context.Context) error + kubeconfig string + adminKubeconfig string configErr error } @@ -59,26 +60,7 @@ func (s *ClusterPolicyController) Dependencies() []string { func (s *ClusterPolicyController) configure(cfg *config.Config) error { s.kubeconfig = cfg.KubeConfigPath(config.ClusterPolicyController) - s.applyRBAC = func(ctx context.Context) error { - kubeconfigPath := cfg.KubeConfigPath(config.KubeAdmin) - cr := []string{ - "controllers/cluster-policy-controller/namespace-security-allocation-controller-clusterrole.yaml", - "controllers/cluster-policy-controller/podsecurity-admission-label-syncer-controller-clusterrole.yaml", - "controllers/cluster-policy-controller/podsecurity-admission-label-privileged-namespaces-syncer-controller-clusterrole.yaml", - } - crb := []string{ - "controllers/cluster-policy-controller/namespace-security-allocation-controller-clusterrolebinding.yaml", - "controllers/cluster-policy-controller/podsecurity-admission-label-syncer-controller-clusterrolebinding.yaml", - "controllers/cluster-policy-controller/podsecurity-admission-label-privileged-namespaces-syncer-controller-clusterrolebinding.yaml", - } - if err := assets.ApplyClusterRoles(ctx, cr, kubeconfigPath); err != nil { - return fmt.Errorf("failed to apply cluster-policy-controller RBAC: %w", err) - } - if err := assets.ApplyClusterRoleBindings(ctx, crb, kubeconfigPath); err != nil { - return fmt.Errorf("failed to apply cluster-policy-controller RBAC: %w", err) - } - return nil - } + s.adminKubeconfig = cfg.KubeConfigPath(config.KubeAdmin) scheme := runtime.NewScheme() if err := openshiftcontrolplanev1.AddToScheme(scheme); err != nil { @@ -130,7 +112,7 @@ func (s *ClusterPolicyController) Run(ctx context.Context, ready chan<- struct{} return fmt.Errorf("configuration failed: %w", s.configErr) } - if err := s.applyRBAC(ctx); err != nil { + if err := applyClusterPolicyControllerRBAC(ctx, s.adminKubeconfig); err != nil { return err } @@ -139,19 +121,50 @@ func (s *ClusterPolicyController) Run(ctx context.Context, ready chan<- struct{} errCh <- s.run(ctx) }() - if err := waitForNamespaceSecurityAllocation(ctx, s.kubeconfig); err != nil { + waitCh := make(chan error, 1) + go func() { + waitCh <- waitForNamespaceSecurityAllocation(ctx, s.kubeconfig) + }() + + select { + case err := <-errCh: return err + case err := <-waitCh: + if err != nil { + return err + } } + klog.Infof("%s is ready", s.Name()) close(ready) return <-errCh } -// waitForNamespaceSecurityAllocation waits until the namespace-security-allocation-controller -// has annotated the default namespace with the UID range annotation, proving it is running -// and processing namespaces. This must happen before infrastructure-services-manager starts -// creating deployments, because the SCC admission plugin blocks until these annotations exist. +func applyClusterPolicyControllerRBAC(ctx context.Context, kubeconfigPath string) error { + cr := []string{ + "controllers/cluster-policy-controller/namespace-security-allocation-controller-clusterrole.yaml", + "controllers/cluster-policy-controller/podsecurity-admission-label-syncer-controller-clusterrole.yaml", + "controllers/cluster-policy-controller/podsecurity-admission-label-privileged-namespaces-syncer-controller-clusterrole.yaml", + } + crb := []string{ + "controllers/cluster-policy-controller/namespace-security-allocation-controller-clusterrolebinding.yaml", + "controllers/cluster-policy-controller/podsecurity-admission-label-syncer-controller-clusterrolebinding.yaml", + "controllers/cluster-policy-controller/podsecurity-admission-label-privileged-namespaces-syncer-controller-clusterrolebinding.yaml", + } + if err := assets.ApplyClusterRoles(ctx, cr, kubeconfigPath); err != nil { + return fmt.Errorf("failed to apply cluster-policy-controller cluster roles: %w", err) + } + if err := assets.ApplyClusterRoleBindings(ctx, crb, kubeconfigPath); err != nil { + return fmt.Errorf("failed to apply cluster-policy-controller cluster role bindings: %w", err) + } + return nil +} + +// waitForNamespaceSecurityAllocation waits until the UID range annotation exists on the +// default namespace. On first boot this annotation is set by the namespace-security-allocation-controller; +// on restart it persists from the previous boot. Either way, its presence guarantees that +// SCC admission will not block when infrastructure-services-manager creates deployments. func waitForNamespaceSecurityAllocation(ctx context.Context, kubeconfigPath string) error { restConfig, err := clientcmd.BuildConfigFromFlags("", kubeconfigPath) if err != nil { @@ -162,12 +175,22 @@ func waitForNamespaceSecurityAllocation(ctx context.Context, kubeconfigPath stri return fmt.Errorf("failed to create kube client for readiness check: %w", err) } + attempt := 0 return wait.PollUntilContextCancel(ctx, time.Second, true, func(ctx context.Context) (bool, error) { + attempt++ ns, err := kubeClient.CoreV1().Namespaces().Get(ctx, "default", metav1.GetOptions{}) if err != nil { + if attempt%5 == 0 { + klog.Infof("cluster-policy-controller: still waiting for namespace security allocation (attempt %d): %v", attempt, err) + } + return false, nil + } + if _, ok := ns.Annotations[securityv1.UIDRangeAnnotation]; !ok { + if attempt%5 == 0 { + klog.Infof("cluster-policy-controller: still waiting for %s annotation on default namespace (attempt %d)", securityv1.UIDRangeAnnotation, attempt) + } return false, nil } - _, ok := ns.Annotations[securityv1.UIDRangeAnnotation] - return ok, nil + return true, nil }) } diff --git a/pkg/controllers/kube-controller-manager.go b/pkg/controllers/kube-controller-manager.go index a5f2ea662e..373e2855cf 100644 --- a/pkg/controllers/kube-controller-manager.go +++ b/pkg/controllers/kube-controller-manager.go @@ -121,12 +121,12 @@ func configure(ctx context.Context, cfg *config.Config) (args []string, applyFn if err := assets.ApplyClusterRoles(ctx, []string{ "controllers/kube-controller-manager/csr_approver_clusterrole.yaml", }, kubeconfigPath); err != nil { - return fmt.Errorf("failed to apply kube-controller-manager RBAC: %w", err) + return fmt.Errorf("failed to apply kube-controller-manager cluster roles: %w", err) } if err := assets.ApplyClusterRoleBindings(ctx, []string{ "controllers/kube-controller-manager/csr_approver_clusterrolebinding.yaml", }, kubeconfigPath); err != nil { - return fmt.Errorf("failed to apply kube-controller-manager RBAC: %w", err) + return fmt.Errorf("failed to apply kube-controller-manager cluster role bindings: %w", err) } return nil }