Add Cilium CI test (allow no CNI testing adopted cluster)

apps/cilium/data.yaml

@@ -173,7 +173,8 @@ doc_link: https://docs.cilium.io/en/stable/internals/cilium_operator/
 # test settings
 test_deploy_chart: false
 test_install_servicetemplates: true
-test_deploy_multiclusterservice: false
+test_wait_for_pods: "cilium-operator-"
+test_adopted_nocni: true # cilium is the CNI, so the adopted cluster starts without one
 
 validated_amd64: 'y'
 validated_aws: 'y'

apps/cilium/example/values.yaml

@@ -0,0 +1,48 @@
+cilium:
+  cilium:
+    cluster:
+      name: cilium
+    hubble:
+      tls:
+        enabled: false
+      auto:
+        method: helm
+        certManagerIssuerRef: {}
+      ui:
+        enabled: false
+        ingress:
+          enabled: false
+      relay:
+        enabled: false
+    ipv4:
+      enabled: true
+    ipv6:
+      enabled: false
+    envoy:
+      enabled: false
+    egressGateway:
+      enabled: false
+    kubeProxyReplacement: "true"
+    serviceAccounts:
+      cilium:
+        name: cilium
+      operator:
+        name: cilium-operator
+    localRedirectPolicy: true
+    operator:
+      replicas: 1 # the default 2 leaves a pod Pending on single-node clusters
+    ipam:
+      mode: cluster-pool
+      operator:
+        clusterPoolIPv4PodCIDRList:
+        - "192.168.224.0/20"
+        - "192.168.210.0/20"
+        clusterPoolIPv6PodCIDRList:
+        - "fd00::/104"
+    tunnelProtocol: geneve
+    # CAPI-backed clusters (aws, azure) expose the endpoint on the Cluster object.
+    # Adopted clusters are SveltosCluster only, so fall back to cilium's "auto"
+    # lookup of the kube-public/cluster-info ConfigMap. hasKey is required
+    # because sveltos renders templates with missingkey=error.
+    k8sServiceHost: auto
+    k8sServicePort: auto

scripts/config/k0s-nocni.yaml

@@ -0,0 +1,9 @@
+apiVersion: k0s.k0sproject.io/v1beta1
+kind: ClusterConfig
+metadata:
+  name: k0s
+spec:
+  network:
+    provider: custom
+    kubeProxy:
+      disabled: true

scripts/deploy_cld.sh

@@ -36,17 +36,24 @@ elif [[ "$TEST_MODE" == adopted ]]; then
     if docker ps --filter name='^adopted$' -q | grep -q .; then
         echo "Adopted cluster already exists"
     else
-        port_args=(-p 6444:6443)
+        container_args=(-p 6444:6443)
         if [[ "${ADOPTED_EXPOSE_PORTS:-0}" == "1" ]]; then  # Avoid ephemeral port range conflicts in CI.
-            port_args+=(-p 50080:80 -p 50443:443)
+            container_args+=(-p 50080:80 -p 50443:443)
+        fi
+        k0s_cmd=(k0s controller --enable-worker)
+        if [[ "${TEST_ADOPTED_NOCNI:-false}" == true ]]; then  # Avoid ephemeral port range conflicts in CI.
+            echo "TEST_ADOPTED_NOCNI=true: starting the adopted cluster without a CNI"
+            container_args+=(-e "K0S_CONFIG=$(cat ./scripts/config/k0s-nocni.yaml)")
+            k0s_cmd=(k0s controller --enable-worker --config=/etc/k0s/config.yaml)
         fi
         docker run -d --name adopted --hostname adopted \
             --network k0rdent-net \
             -v /var/lib/k0s -v /var/log/pods \
             --tmpfs /run \
             --privileged \
-            "${port_args[@]}" \
-            docker.io/k0sproject/k0s:v1.36.3-k0s.0
+            "${container_args[@]}" \
+            docker.io/k0sproject/k0s:v1.36.3-k0s.0 \
+            "${k0s_cmd[@]}"
 
         echo "Waiting for adopted kubeconfig..."
         until docker exec adopted k0s kubeconfig admin > kcfg_adopted 2>/dev/null; do
@@ -63,20 +70,45 @@ elif [[ "$TEST_MODE" == adopted ]]; then
     sed -i.bak 's#server:.*#server: https://127.0.0.1:6444#' kcfg_adopted
     chmod 0600 kcfg_adopted
 
-    echo "Waiting for adopted kube-system pods to become Ready..."
-    until KUBECONFIG=kcfg_adopted kubectl wait -n kube-system --for=condition=Ready pod --all --timeout=2s 2>/dev/null; do
-        KUBECONFIG=kcfg_adopted kubectl get pods -n kube-system || true
-        sleep 2
-    done
+    if [[ "${TEST_ADOPTED_NOCNI:-false}" == true ]]; then
+        echo "Waiting for the adopted node to register (stays NotReady until the CNI is deployed)..."
+        until KUBECONFIG=kcfg_adopted kubectl get node adopted >/dev/null 2>&1; do
+            KUBECONFIG=kcfg_adopted kubectl get nodes || true
+            sleep 2
+        done
+        KUBECONFIG=kcfg_adopted kubectl get nodes
+    else
+        echo "Waiting for adopted kube-system pods to become Ready..."
+        until KUBECONFIG=kcfg_adopted kubectl wait -n kube-system --for=condition=Ready pod --all --timeout=2s 2>/dev/null; do
+            KUBECONFIG=kcfg_adopted kubectl get pods -n kube-system || true
+            sleep 2
+        done
+    fi
 
     # Allow workloads on the single control-plane node.
     KUBECONFIG=kcfg_adopted kubectl taint nodes adopted node-role.kubernetes.io/control-plane:NoSchedule- 2>/dev/null || true
 
     # Drop the malformed trailing-dot apiserver SAN (breaks FIPS-only clients).
     ./scripts/fix_adopted_cert_sans.sh adopted
 
-    # Default StorageClass (openebs hostpath); k0s ships none.
-    TEST_MODE=adopted ./scripts/install_openebs.sh
+    if [[ "${TEST_ADOPTED_NOCNI:-false}" == true ]]; then
+        # kube-proxy is disabled, so a CNI needs the real API endpoint to bootstrap.
+        # k0s ships no kube-public/cluster-info, which is where charts look it up
+        # (e.g. cilium's k8sServiceHost: auto).
+        api_host=$(KUBECONFIG=kcfg_adopted kubectl get node adopted \
+            -o jsonpath='{.status.addresses[?(@.type=="InternalIP")].address}')
+        echo "Publishing kube-public/cluster-info with API endpoint $api_host:6443"
+        KUBECONFIG=kcfg_adopted kubectl create configmap cluster-info -n kube-public \
+            --from-literal=kubeconfig="$(printf 'apiVersion: v1\nkind: Config\nclusters:\n- name: k0s\n  cluster:\n    server: https://%s:6443\n' "$api_host")" \
+            --dry-run=client -o yaml | KUBECONFIG=kcfg_adopted kubectl apply -f -
+
+        # openebs is skipped: its pods cannot start before the CNI is deployed, so
+        # this mode leaves the cluster without a default StorageClass.
+        echo "Skipping openebs install (no CNI yet)"
+    else
+        # Default StorageClass (openebs hostpath); k0s ships none.
+        TEST_MODE=adopted ./scripts/install_openebs.sh
+    fi
 
     # Internal kubeconfig for k0rdent (k0s sets the server to the container IP).
     ADOPTED_KUBECONFIG=$(docker exec adopted k0s kubeconfig admin | openssl base64 -A)
@@ -96,6 +128,8 @@ fi
 # For adopted, kcfg_adopted is already written above.
 chmod 0600 "kcfg_$TEST_MODE"
 
-if kubectl get ns | grep "projectsveltos"; then
+if [[ "${TEST_ADOPTED_NOCNI:-false}" == true ]]; then
+    echo "Skipping the projectsveltos wait (no CNI in the '$TEST_MODE' cluster yet)"
+elif kubectl get ns | grep "projectsveltos"; then
     NAMESPACE=projectsveltos ./scripts/wait_for_deployment.sh
 fi

scripts/deploy_mcs.sh

@@ -23,7 +23,14 @@ check_clusters() {
         # shellcheck disable=SC2030
         export TEST_MODE=$test_mode
         ./scripts/wait_for_deployment.sh
-        KUBECONFIG="kcfg_$test_mode" kubectl top nodes
+        if [[ "${TEST_ADOPTED_NOCNI:-false}" == true ]]; then
+            echo "Waiting for kube-system in '$test_mode' now that the CNI is deployed..."
+            NAMESPACE=kube-system WAIT_FOR_PODS='' WAIT_FOR_RUNNING='' WAIT_FOR_CREATING='' \
+                ./scripts/wait_for_deployment.sh
+            KUBECONFIG="kcfg_$test_mode" MAX_RETRIES=30 ./scripts/retry.sh kubectl top nodes
+        else
+            KUBECONFIG="kcfg_$test_mode" kubectl top nodes
+        fi
     )
   done
 }

scripts/utils.py

@@ -230,6 +230,9 @@ def print_test_vars(args):
     print(f"REMOVE_MULTICLUSTERSERVICE={test_remove_multiclusterservice}")
     test_check_images = str(app_data.get('test_check_images', default)).lower()
     print(f"CHECK_IMAGES={test_check_images}")
+    # CNI apps need the adopted cluster created without a CNI (see deploy_cld.sh).
+    test_adopted_nocni = str(app_data.get('test_adopted_nocni', False)).lower()
+    print(f"TEST_ADOPTED_NOCNI={test_adopted_nocni}")
 
 
 def get_wait_for_pods(args):