[release-4.22] OCPBUGS-98603: Move RBAC manifests to the services that own them

pkg/cmd/run.go

@@ -226,8 +226,8 @@ func RunMicroshift(cfg *config.Config) error {
 	util.Must(m.AddService(controllers.NewRouteControllerManager(cfg)))
 	util.Must(m.AddService(controllers.NewOpenShiftDefaultSCCManager(cfg)))
 	util.Must(m.AddService(mdns.NewMicroShiftmDNSController(cfg)))
-	util.Must(m.AddService(controllers.NewInfrastructureServices(cfg)))
 	util.Must(m.AddService(controllers.NewClusterPolicyController(cfg)))
+	util.Must(m.AddService(controllers.NewInfrastructureServices(cfg)))
 	util.Must(m.AddService(controllers.NewVersionManager(cfg)))
 	util.Must(m.AddService(controllers.NewKubeletCAManager(cfg)))
 	util.Must(m.AddService(node.NewKubeletServer(cfg)))

pkg/controllers/cluster-policy-controller.go

@@ -17,22 +17,32 @@ package controllers
 import (
 	"context"
 	"fmt"
+	"time"
 
 	openshiftcontrolplanev1 "github.com/openshift/api/openshiftcontrolplane/v1"
+	securityv1 "github.com/openshift/api/security/v1"
 	clusterpolicycontroller "github.com/openshift/cluster-policy-controller/pkg/cmd/cluster-policy-controller"
 	"github.com/openshift/library-go/pkg/controller/controllercmd"
 	"github.com/openshift/library-go/pkg/operator/events"
+	"github.com/openshift/microshift/pkg/assets"
 	"github.com/openshift/microshift/pkg/config"
+
 	corev1 "k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	unstructuredv1 "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
 	"k8s.io/apimachinery/pkg/runtime"
 	"k8s.io/apimachinery/pkg/runtime/serializer"
+	"k8s.io/apimachinery/pkg/util/wait"
+	"k8s.io/client-go/kubernetes"
+	"k8s.io/client-go/tools/clientcmd"
+	klog "k8s.io/klog/v2"
 	"k8s.io/utils/clock"
 )
 
 type ClusterPolicyController struct {
-	run        func(context.Context) error
-	kubeconfig string
+	run             func(context.Context) error
+	kubeconfig      string
+	adminKubeconfig string
 
 	configErr error
 }
@@ -45,11 +55,12 @@ func NewClusterPolicyController(cfg *config.Config) *ClusterPolicyController {
 
 func (s *ClusterPolicyController) Name() string { return "cluster-policy-controller" }
 func (s *ClusterPolicyController) Dependencies() []string {
-	return []string{"kube-apiserver", "infrastructure-services-manager"}
+	return []string{"kube-apiserver"}
 }
 
 func (s *ClusterPolicyController) configure(cfg *config.Config) error {
 	s.kubeconfig = cfg.KubeConfigPath(config.ClusterPolicyController)
+	s.adminKubeconfig = cfg.KubeConfigPath(config.KubeAdmin)
 
 	scheme := runtime.NewScheme()
 	if err := openshiftcontrolplanev1.AddToScheme(scheme); err != nil {
@@ -101,6 +112,85 @@ func (s *ClusterPolicyController) Run(ctx context.Context, ready chan<- struct{}
 		return fmt.Errorf("configuration failed: %w", s.configErr)
 	}
 
-	close(ready) // todo
-	return s.run(ctx)
+	if err := applyClusterPolicyControllerRBAC(ctx, s.adminKubeconfig); err != nil {
+		return err
+	}
+
+	errCh := make(chan error, 1)
+	go func() {
+		errCh <- s.run(ctx)
+	}()
+
+	waitCh := make(chan error, 1)
+	go func() {
+		waitCh <- waitForNamespaceSecurityAllocation(ctx, s.kubeconfig)
+	}()
+
+	select {
+	case err := <-errCh:
+		return err
+	case err := <-waitCh:
+		if err != nil {
+			return err
+		}
+	}
+
+	klog.Infof("%s is ready", s.Name())
+	close(ready)
+
+	return <-errCh
+}
+
+func applyClusterPolicyControllerRBAC(ctx context.Context, kubeconfigPath string) error {
+	cr := []string{
+		"controllers/cluster-policy-controller/namespace-security-allocation-controller-clusterrole.yaml",
+		"controllers/cluster-policy-controller/podsecurity-admission-label-syncer-controller-clusterrole.yaml",
+		"controllers/cluster-policy-controller/podsecurity-admission-label-privileged-namespaces-syncer-controller-clusterrole.yaml",
+	}
+	crb := []string{
+		"controllers/cluster-policy-controller/namespace-security-allocation-controller-clusterrolebinding.yaml",
+		"controllers/cluster-policy-controller/podsecurity-admission-label-syncer-controller-clusterrolebinding.yaml",
+		"controllers/cluster-policy-controller/podsecurity-admission-label-privileged-namespaces-syncer-controller-clusterrolebinding.yaml",
+	}
+	if err := assets.ApplyClusterRoles(ctx, cr, kubeconfigPath); err != nil {
+		return fmt.Errorf("failed to apply cluster-policy-controller cluster roles: %w", err)
+	}
+	if err := assets.ApplyClusterRoleBindings(ctx, crb, kubeconfigPath); err != nil {
+		return fmt.Errorf("failed to apply cluster-policy-controller cluster role bindings: %w", err)
+	}
+	return nil
+}
+
+// waitForNamespaceSecurityAllocation waits until the UID range annotation exists on the
+// default namespace. On first boot this annotation is set by the namespace-security-allocation-controller;
+// on restart it persists from the previous boot. Either way, its presence guarantees that
+// SCC admission will not block when infrastructure-services-manager creates deployments.
+func waitForNamespaceSecurityAllocation(ctx context.Context, kubeconfigPath string) error {
+	restConfig, err := clientcmd.BuildConfigFromFlags("", kubeconfigPath)
+	if err != nil {
+		return fmt.Errorf("failed to build kubeconfig for readiness check: %w", err)
+	}
+	kubeClient, err := kubernetes.NewForConfig(restConfig)
+	if err != nil {
+		return fmt.Errorf("failed to create kube client for readiness check: %w", err)
+	}
+
+	attempt := 0
+	return wait.PollUntilContextCancel(ctx, time.Second, true, func(ctx context.Context) (bool, error) {
+		attempt++
+		ns, err := kubeClient.CoreV1().Namespaces().Get(ctx, "default", metav1.GetOptions{})
+		if err != nil {
+			if attempt%5 == 0 {
+				klog.Infof("cluster-policy-controller: still waiting for namespace security allocation (attempt %d): %v", attempt, err)
+			}
+			return false, nil
+		}
+		if _, ok := ns.Annotations[securityv1.UIDRangeAnnotation]; !ok {
+			if attempt%5 == 0 {
+				klog.Infof("cluster-policy-controller: still waiting for %s annotation on default namespace (attempt %d)", securityv1.UIDRangeAnnotation, attempt)
+			}
+			return false, nil
+		}
+		return true, nil
+	})
 }

pkg/controllers/infra-services-controller.go

@@ -37,55 +37,22 @@ func NewInfrastructureServices(cfg *config.Config) *InfrastructureServicesManage
 
 func (s *InfrastructureServicesManager) Name() string { return "infrastructure-services-manager" }
 func (s *InfrastructureServicesManager) Dependencies() []string {
-	return []string{"kube-apiserver", "openshift-crd-manager", "route-controller-manager"}
+	return []string{"kube-apiserver", "cluster-policy-controller", "openshift-crd-manager", "route-controller-manager"}
 }
 
 func (s *InfrastructureServicesManager) Run(ctx context.Context, ready chan<- struct{}, stopped chan<- struct{}) error {
 	defer close(stopped)
 	defer close(ready)
 
-	if err := applyDefaultRBACs(ctx, s.cfg); err != nil {
-		klog.Errorf("%s unable to apply default RBACs: %v", s.Name(), err)
-		return err
-	}
-
 	priorityClasses := []string{"core/priority-class-openshift-user-critical.yaml"}
 	if err := assets.ApplyPriorityClasses(ctx, priorityClasses, s.cfg.KubeConfigPath(config.KubeAdmin)); err != nil {
 		klog.Errorf("%s unable to apply PriorityClasses: %v", s.Name(), err)
 		return err
 	}
 
-	// TO-DO add readiness check
 	if err := components.StartComponents(s.cfg, ctx); err != nil {
 		return err
 	}
 	klog.Infof("%s launched ocp componets", s.Name())
 	return ctx.Err()
 }
-
-func applyDefaultRBACs(ctx context.Context, cfg *config.Config) error {
-	kubeconfigPath := cfg.KubeConfigPath(config.KubeAdmin)
-	var (
-		cr = []string{
-			"controllers/kube-controller-manager/csr_approver_clusterrole.yaml",
-			"controllers/cluster-policy-controller/namespace-security-allocation-controller-clusterrole.yaml",
-			"controllers/cluster-policy-controller/podsecurity-admission-label-syncer-controller-clusterrole.yaml",
-			"controllers/cluster-policy-controller/podsecurity-admission-label-privileged-namespaces-syncer-controller-clusterrole.yaml",
-		}
-		crb = []string{
-			"controllers/kube-controller-manager/csr_approver_clusterrolebinding.yaml",
-			"controllers/cluster-policy-controller/namespace-security-allocation-controller-clusterrolebinding.yaml",
-			"controllers/cluster-policy-controller/podsecurity-admission-label-syncer-controller-clusterrolebinding.yaml",
-			"controllers/cluster-policy-controller/podsecurity-admission-label-privileged-namespaces-syncer-controller-clusterrolebinding.yaml",
-		}
-	)
-	if err := assets.ApplyClusterRoles(ctx, cr, kubeconfigPath); err != nil {
-		klog.Warningf("failed to apply cluster roles %v", err)
-		return err
-	}
-	if err := assets.ApplyClusterRoleBindings(ctx, crb, kubeconfigPath); err != nil {
-		klog.Warningf("failed to apply cluster roles %v", err)
-		return err
-	}
-	return nil
-}

pkg/controllers/kube-controller-manager.go

@@ -111,10 +111,24 @@ func configure(ctx context.Context, cfg *config.Config) (args []string, applyFn
 
 	args, err = mergeAndConvertToArgs(overrides)
 	applyFn = func() error {
-		return assets.ApplyNamespaces(ctx, []string{
+		kubeconfigPath := cfg.KubeConfigPath(config.KubeAdmin)
+		if err := assets.ApplyNamespaces(ctx, []string{
 			"controllers/kube-controller-manager/namespace-openshift-kube-controller-manager.yaml",
 			"core/namespace-openshift-infra.yaml",
-		}, cfg.KubeConfigPath(config.KubeAdmin))
+		}, kubeconfigPath); err != nil {
+			return fmt.Errorf("failed to apply kube-controller-manager namespaces: %w", err)
+		}
+		if err := assets.ApplyClusterRoles(ctx, []string{
+			"controllers/kube-controller-manager/csr_approver_clusterrole.yaml",
+		}, kubeconfigPath); err != nil {
+			return fmt.Errorf("failed to apply kube-controller-manager cluster roles: %w", err)
+		}
+		if err := assets.ApplyClusterRoleBindings(ctx, []string{
+			"controllers/kube-controller-manager/csr_approver_clusterrolebinding.yaml",
+		}, kubeconfigPath); err != nil {
+			return fmt.Errorf("failed to apply kube-controller-manager cluster role bindings: %w", err)
+		}
+		return nil
 	}
 	return args, applyFn, err
 }
@@ -157,7 +171,7 @@ func (s *KubeControllerManager) Run(ctx context.Context, ready chan<- struct{},
 	}()
 
 	if err := s.applyFn(); err != nil {
-		return fmt.Errorf("failed to apply openshift namespaces: %w", err)
+		return err
 	}
 
 	select {