diff --git a/pkg/cmd/run.go b/pkg/cmd/run.go index 7ea654c2ea..ca2b132f91 100644 --- a/pkg/cmd/run.go +++ b/pkg/cmd/run.go @@ -228,8 +228,8 @@ func RunMicroshift(cfg *config.Config) error { util.Must(m.AddService(controllers.NewRouteControllerManager(cfg))) util.Must(m.AddService(controllers.NewOpenShiftDefaultSCCManager(cfg))) util.Must(m.AddService(mdns.NewMicroShiftmDNSController(cfg))) - util.Must(m.AddService(controllers.NewInfrastructureServices(cfg))) util.Must(m.AddService(controllers.NewClusterPolicyController(cfg))) + util.Must(m.AddService(controllers.NewInfrastructureServices(cfg))) util.Must(m.AddService(controllers.NewVersionManager(cfg))) util.Must(m.AddService(controllers.NewKubeletCAManager(cfg))) util.Must(m.AddService(node.NewKubeletServer(cfg))) diff --git a/pkg/controllers/cluster-policy-controller.go b/pkg/controllers/cluster-policy-controller.go index f074f71b8d..9462406737 100644 --- a/pkg/controllers/cluster-policy-controller.go +++ b/pkg/controllers/cluster-policy-controller.go @@ -17,22 +17,32 @@ package controllers import ( "context" "fmt" + "time" openshiftcontrolplanev1 "github.com/openshift/api/openshiftcontrolplane/v1" + securityv1 "github.com/openshift/api/security/v1" clusterpolicycontroller "github.com/openshift/cluster-policy-controller/pkg/cmd/cluster-policy-controller" "github.com/openshift/library-go/pkg/controller/controllercmd" "github.com/openshift/library-go/pkg/operator/events" + "github.com/openshift/microshift/pkg/assets" "github.com/openshift/microshift/pkg/config" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" unstructuredv1 "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/runtime/serializer" + "k8s.io/apimachinery/pkg/util/wait" + "k8s.io/client-go/kubernetes" + "k8s.io/client-go/tools/clientcmd" + klog "k8s.io/klog/v2" "k8s.io/utils/clock" ) type ClusterPolicyController struct { - run func(context.Context) error - kubeconfig string + run func(context.Context) error + kubeconfig string + adminKubeconfig string configErr error } @@ -45,11 +55,12 @@ func NewClusterPolicyController(cfg *config.Config) *ClusterPolicyController { func (s *ClusterPolicyController) Name() string { return "cluster-policy-controller" } func (s *ClusterPolicyController) Dependencies() []string { - return []string{"kube-apiserver", "infrastructure-services-manager"} + return []string{"kube-apiserver"} } func (s *ClusterPolicyController) configure(cfg *config.Config) error { s.kubeconfig = cfg.KubeConfigPath(config.ClusterPolicyController) + s.adminKubeconfig = cfg.KubeConfigPath(config.KubeAdmin) scheme := runtime.NewScheme() if err := openshiftcontrolplanev1.AddToScheme(scheme); err != nil { @@ -101,6 +112,85 @@ func (s *ClusterPolicyController) Run(ctx context.Context, ready chan<- struct{} return fmt.Errorf("configuration failed: %w", s.configErr) } - close(ready) // todo - return s.run(ctx) + if err := applyClusterPolicyControllerRBAC(ctx, s.adminKubeconfig); err != nil { + return err + } + + errCh := make(chan error, 1) + go func() { + errCh <- s.run(ctx) + }() + + waitCh := make(chan error, 1) + go func() { + waitCh <- waitForNamespaceSecurityAllocation(ctx, s.kubeconfig) + }() + + select { + case err := <-errCh: + return err + case err := <-waitCh: + if err != nil { + return err + } + } + + klog.Infof("%s is ready", s.Name()) + close(ready) + + return <-errCh +} + +func applyClusterPolicyControllerRBAC(ctx context.Context, kubeconfigPath string) error { + cr := []string{ + "controllers/cluster-policy-controller/namespace-security-allocation-controller-clusterrole.yaml", + "controllers/cluster-policy-controller/podsecurity-admission-label-syncer-controller-clusterrole.yaml", + "controllers/cluster-policy-controller/podsecurity-admission-label-privileged-namespaces-syncer-controller-clusterrole.yaml", + } + crb := []string{ + "controllers/cluster-policy-controller/namespace-security-allocation-controller-clusterrolebinding.yaml", + "controllers/cluster-policy-controller/podsecurity-admission-label-syncer-controller-clusterrolebinding.yaml", + "controllers/cluster-policy-controller/podsecurity-admission-label-privileged-namespaces-syncer-controller-clusterrolebinding.yaml", + } + if err := assets.ApplyClusterRoles(ctx, cr, kubeconfigPath); err != nil { + return fmt.Errorf("failed to apply cluster-policy-controller cluster roles: %w", err) + } + if err := assets.ApplyClusterRoleBindings(ctx, crb, kubeconfigPath); err != nil { + return fmt.Errorf("failed to apply cluster-policy-controller cluster role bindings: %w", err) + } + return nil +} + +// waitForNamespaceSecurityAllocation waits until the UID range annotation exists on the +// default namespace. On first boot this annotation is set by the namespace-security-allocation-controller; +// on restart it persists from the previous boot. Either way, its presence guarantees that +// SCC admission will not block when infrastructure-services-manager creates deployments. +func waitForNamespaceSecurityAllocation(ctx context.Context, kubeconfigPath string) error { + restConfig, err := clientcmd.BuildConfigFromFlags("", kubeconfigPath) + if err != nil { + return fmt.Errorf("failed to build kubeconfig for readiness check: %w", err) + } + kubeClient, err := kubernetes.NewForConfig(restConfig) + if err != nil { + return fmt.Errorf("failed to create kube client for readiness check: %w", err) + } + + attempt := 0 + return wait.PollUntilContextCancel(ctx, time.Second, true, func(ctx context.Context) (bool, error) { + attempt++ + ns, err := kubeClient.CoreV1().Namespaces().Get(ctx, "default", metav1.GetOptions{}) + if err != nil { + if attempt%5 == 0 { + klog.Infof("cluster-policy-controller: still waiting for namespace security allocation (attempt %d): %v", attempt, err) + } + return false, nil + } + if _, ok := ns.Annotations[securityv1.UIDRangeAnnotation]; !ok { + if attempt%5 == 0 { + klog.Infof("cluster-policy-controller: still waiting for %s annotation on default namespace (attempt %d)", securityv1.UIDRangeAnnotation, attempt) + } + return false, nil + } + return true, nil + }) } diff --git a/pkg/controllers/infra-services-controller.go b/pkg/controllers/infra-services-controller.go index 628a6c7076..3f0207ec38 100644 --- a/pkg/controllers/infra-services-controller.go +++ b/pkg/controllers/infra-services-controller.go @@ -37,55 +37,22 @@ func NewInfrastructureServices(cfg *config.Config) *InfrastructureServicesManage func (s *InfrastructureServicesManager) Name() string { return "infrastructure-services-manager" } func (s *InfrastructureServicesManager) Dependencies() []string { - return []string{"kube-apiserver", "openshift-crd-manager", "route-controller-manager"} + return []string{"kube-apiserver", "cluster-policy-controller", "openshift-crd-manager", "route-controller-manager"} } func (s *InfrastructureServicesManager) Run(ctx context.Context, ready chan<- struct{}, stopped chan<- struct{}) error { defer close(stopped) defer close(ready) - if err := applyDefaultRBACs(ctx, s.cfg); err != nil { - klog.Errorf("%s unable to apply default RBACs: %v", s.Name(), err) - return err - } - priorityClasses := []string{"core/priority-class-openshift-user-critical.yaml"} if err := assets.ApplyPriorityClasses(ctx, priorityClasses, s.cfg.KubeConfigPath(config.KubeAdmin)); err != nil { klog.Errorf("%s unable to apply PriorityClasses: %v", s.Name(), err) return err } - // TO-DO add readiness check if err := components.StartComponents(s.cfg, ctx); err != nil { return err } klog.Infof("%s launched ocp componets", s.Name()) return ctx.Err() } - -func applyDefaultRBACs(ctx context.Context, cfg *config.Config) error { - kubeconfigPath := cfg.KubeConfigPath(config.KubeAdmin) - var ( - cr = []string{ - "controllers/kube-controller-manager/csr_approver_clusterrole.yaml", - "controllers/cluster-policy-controller/namespace-security-allocation-controller-clusterrole.yaml", - "controllers/cluster-policy-controller/podsecurity-admission-label-syncer-controller-clusterrole.yaml", - "controllers/cluster-policy-controller/podsecurity-admission-label-privileged-namespaces-syncer-controller-clusterrole.yaml", - } - crb = []string{ - "controllers/kube-controller-manager/csr_approver_clusterrolebinding.yaml", - "controllers/cluster-policy-controller/namespace-security-allocation-controller-clusterrolebinding.yaml", - "controllers/cluster-policy-controller/podsecurity-admission-label-syncer-controller-clusterrolebinding.yaml", - "controllers/cluster-policy-controller/podsecurity-admission-label-privileged-namespaces-syncer-controller-clusterrolebinding.yaml", - } - ) - if err := assets.ApplyClusterRoles(ctx, cr, kubeconfigPath); err != nil { - klog.Warningf("failed to apply cluster roles %v", err) - return err - } - if err := assets.ApplyClusterRoleBindings(ctx, crb, kubeconfigPath); err != nil { - klog.Warningf("failed to apply cluster roles %v", err) - return err - } - return nil -} diff --git a/pkg/controllers/kube-controller-manager.go b/pkg/controllers/kube-controller-manager.go index 409f87e694..373e2855cf 100644 --- a/pkg/controllers/kube-controller-manager.go +++ b/pkg/controllers/kube-controller-manager.go @@ -111,10 +111,24 @@ func configure(ctx context.Context, cfg *config.Config) (args []string, applyFn args, err = mergeAndConvertToArgs(overrides) applyFn = func() error { - return assets.ApplyNamespaces(ctx, []string{ + kubeconfigPath := cfg.KubeConfigPath(config.KubeAdmin) + if err := assets.ApplyNamespaces(ctx, []string{ "controllers/kube-controller-manager/namespace-openshift-kube-controller-manager.yaml", "core/namespace-openshift-infra.yaml", - }, cfg.KubeConfigPath(config.KubeAdmin)) + }, kubeconfigPath); err != nil { + return fmt.Errorf("failed to apply kube-controller-manager namespaces: %w", err) + } + if err := assets.ApplyClusterRoles(ctx, []string{ + "controllers/kube-controller-manager/csr_approver_clusterrole.yaml", + }, kubeconfigPath); err != nil { + return fmt.Errorf("failed to apply kube-controller-manager cluster roles: %w", err) + } + if err := assets.ApplyClusterRoleBindings(ctx, []string{ + "controllers/kube-controller-manager/csr_approver_clusterrolebinding.yaml", + }, kubeconfigPath); err != nil { + return fmt.Errorf("failed to apply kube-controller-manager cluster role bindings: %w", err) + } + return nil } return args, applyFn, err } @@ -157,7 +171,7 @@ func (s *KubeControllerManager) Run(ctx context.Context, ready chan<- struct{}, }() if err := s.applyFn(); err != nil { - return fmt.Errorf("failed to apply openshift namespaces: %w", err) + return err } select {