k8s-native-e2e-add-node.yaml

name: "K8s Native E2E - Add Node (use topology-suite-k8s-add-node.yml for multi-test runs)"
run-name: "K8s Native E2E - Add Node | ${{ inputs.cluster_environment || 'local' }} | ${{ inputs.testname || 'K8sNativeAddNodeTest' }} | ${{ inputs.cluster_params || 'ndcs=2,npcs=2,bs=4096,chunk_bs=4096,partitions=1,jm_count=4' }}"

on:
  workflow_call:
    inputs:
      testname:
        type: string
        default: 'K8sNativeAddNodeTest'
        description: 'Test class name to run (default: K8sNativeAddNodeTest)'
      operator_repo_branch:
        type: string
        default: 'main'
      simplyblock_image:
        type: string
        default: 'main'
      simplyblock_repository:
        type: string
        default: 'public.ecr.aws/simply-block/simplyblock'
      operator_repository:
        type: string
        default: 'simplyblock/simplyblock-operator'
      operator_tag:
        type: string
        default: 'main'
      csi_repository:
        type: string
        default: 'simplyblock/spdkcsi'
      csi_tag:
        type: string
        default: 'latest'
      spdk_image:
        type: string
        default: 'simplyblock/spdk:main-latest'
      worker_nodes:
        type: string
        required: true
        default: 'worker-0.ocp.simplyblock.ai,worker-1.ocp.simplyblock.ai,worker-2.ocp.simplyblock.ai,worker-3.ocp.simplyblock.ai,worker-4.ocp.simplyblock.ai,worker-5.ocp.simplyblock.ai'
      new_worker_nodes:
        type: string
        required: true
      ifc_names:
        type: string
        default: 'br-ex:enp2s0f0'
      max_subsys:
        type: string
        default: '30'
      ssh_user:
        type: string
        default: 'root'
      key_path:
        type: string
        default: '/home/ec2-user/.ssh/simplyblock-us-east-2.pem'
      send_slack_notification:
        type: boolean
        default: true
      cluster_params:
        type: string
        default: 'ndcs=2,npcs=2,bs=4096,chunk_bs=4096,partitions=1,jm_count=4'
      cluster_environment:
        type: string
        default: 'openshift-baremetal'
      skip_nfs:
        type: string
        default: 'false'
      use_existing_cluster:
        type: string
        default: 'false'
      tls_enabled:
        type: boolean
        default: true
      client_ips:
        type: string
        default: ''
      nfs_mountpoint:
        type: string
        default: '/mnt/nfs_share/'
      cluster_security:
        type: string
        default: 'none'
        description: "Cluster setup mode. 'backup' deploys MinIO and enables backup on StorageCluster."
      preserve_resources_on_failure:
        type: boolean
        default: true
        description: 'Preserve PVCs, FIO pods, snapshots on test failure for debugging'
  workflow_dispatch:
    inputs:
      testname:
        description: 'Test class name to run'
        required: false
        default: 'K8sNativeAddNodeTest'
      operator_repo_branch:
        description: 'Branch for simplyblock-operator repo'
        required: true
        default: 'main'
      simplyblock_image:
        description: 'Simplyblock image tag'
        required: true
        default: 'main'
      simplyblock_repository:
        description: 'Simplyblock image repository'
        required: false
        default: 'public.ecr.aws/simply-block/simplyblock'
      operator_repository:
        description: 'Operator image repository'
        required: false
        default: 'simplyblock/simplyblock-operator'
      operator_tag:
        description: 'Operator image tag'
        required: false
        default: 'main'
      csi_repository:
        description: 'CSI driver image repository (leave empty for chart default)'
        required: false
        default: 'simplyblock/spdkcsi'
      csi_tag:
        description: 'CSI driver image tag (leave empty for chart default)'
        required: false
        default: 'latest'
      spdk_image:
        description: 'SPDK container image'
        required: true
        default: 'simplyblock/spdk:main-latest'
      worker_nodes:
        description: 'Comma-separated initial K8s worker node names'
        required: true
        default: 'worker-0.ocp.simplyblock.ai,worker-1.ocp.simplyblock.ai,worker-2.ocp.simplyblock.ai,worker-3.ocp.simplyblock.ai,worker-4.ocp.simplyblock.ai,worker-5.ocp.simplyblock.ai'
      new_worker_nodes:
        description: 'Comma-separated new K8s worker node names to add during test (at least 1)'
        required: true
      ifc_names:
        description: 'Network interfaces (mgmt_ifc:data_nics)'
        required: false
        default: 'br-ex:enp2s0f0'
      max_subsys:
        description: 'Max logical volume count per storage node'
        required: false
        default: '30'
      ssh_user:
        description: 'SSH user for client nodes'
        required: false
        default: 'root'
      key_path:
        description: "SSH private key path on runner (default: /home/ec2-user/.ssh/simplyblock-us-east-2.pem)"
        required: true
        default: "/home/ec2-user/.ssh/simplyblock-us-east-2.pem"
      send_slack_notification:
        description: 'Send Slack notification?'
        required: false
        default: true
        type: boolean
      cluster_params:
        description: 'Cluster config: ndcs=2,npcs=2,bs=4096,chunk_bs=4096,partitions=1,jm_count=4'
        required: false
        default: 'ndcs=2,npcs=2,bs=4096,chunk_bs=4096,partitions=1,jm_count=4'
      cluster_environment:
        description: 'Target cluster environment'
        required: true
        default: 'openshift-baremetal'
        type: choice
        options:
          - local
          - aws-openshift
          - openshift-local
          - openshift-baremetal
          - gcp
      skip_nfs:
        description: 'Skip NFS mounting (use local logs directory instead)'
        required: false
        default: 'false'
        type: choice
        options:
          - 'false'
          - 'true'
      use_existing_cluster:
        description: 'Skip cluster cleanup and setup, reuse existing cluster'
        required: false
        default: 'false'
        type: choice
        options:
          - 'false'
          - 'true'
      tls_enabled:
        description: 'Enable TLS (installs cert-manager, sets tls.enabled + tls.mutual_enabled)'
        required: false
        default: true
        type: boolean
      client_ips:
        description: 'Comma-separated client IPs for external FIO (empty = FIO runs as K8s Jobs)'
        required: false
        default: ''
      nfs_mountpoint:
        description: 'NFS mountpoint on client nodes'
        required: false
        default: '/mnt/nfs_share/'
      cluster_security:
        description: "Cluster setup mode. 'backup' deploys MinIO and enables backup on StorageCluster. Auto-enabled for backup/backup-topology test names."
        required: false
        type: choice
        default: "none"
        options:
          - none
          - backup
      preserve_resources_on_failure:
        description: 'Preserve PVCs, FIO pods, snapshots on test failure for debugging'
        required: false
        default: true
        type: boolean

jobs:
  e2e-add-node:
    runs-on: ${{ inputs.cluster_environment == 'aws-openshift' && 'vm-runner-43' || 'self-hosted' }}
    timeout-minutes: 4320
    concurrency:
      group: ${{ github.workflow }}
      cancel-in-progress: false
    env:
      TESTNAME: ${{ inputs.testname || 'K8sNativeAddNodeTest' }}
      KEY_PATH: ${{ inputs.key_path || '/home/ec2-user/.ssh/simplyblock-us-east-2.pem' }}
      SSH_USER: ${{ inputs.ssh_user || 'root' }}
      SSH_PASSWORD: ${{ secrets.SSH_PASSWORD }}
    steps:
      - name: Fix workspace permissions
        run: sudo chown -R "$(id -u):$(id -g)" "$GITHUB_WORKSPACE" 2>/dev/null || true

      - name: Checkout code
        uses: actions/checkout@v4

      - uses: actions/checkout@master
        name: Checkout simplyblock-operator
        with:
          repository: simplyblock/simplyblock-operator
          ref: ${{ inputs.operator_repo_branch || 'main' }}
          path: 'simplyblock-operator'

      - name: Resolve KEY_PATH and validate key exists
        shell: bash
        run: |
          set -euxo pipefail

          kp="${KEY_PATH}"

          # Strip wrapping quotes
          kp="${kp%\"}"; kp="${kp#\"}"
          kp="${kp%\'}"; kp="${kp#\'}"

          # Normalize ".ssh/..." -> "$HOME/.ssh/..."
          if [[ "$kp" == .ssh/* ]]; then
            kp="${HOME}/${kp}"
          fi

          # Normalize "~/" -> "$HOME/"
          if [[ "$kp" == ~/* ]]; then
            kp="${HOME}/${kp#~/}"
          fi

          echo "Resolved KEY_PATH=$kp"
          echo "KEY_PATH=$kp" >> "$GITHUB_ENV"

          test -f "$kp" || (echo "ERROR: SSH key not found at $kp" && exit 1)
          chmod 600 "$kp" || true

      - name: Export KEY_NAME from KEY_PATH
        shell: bash
        run: |
          set -euxo pipefail
          key_name="$(basename "${KEY_PATH}")"
          echo "KEY_NAME=${key_name}" >> "$GITHUB_ENV"
          echo "Exported KEY_NAME=${key_name}"

      - name: Setup kubeconfig
        run: |
          mkdir -p $HOME/.kube
          KUBECONFIG_FILE="$HOME/.kube/config-${{ github.run_id }}"
          echo "${KUBECONFIG_DATA}" > "$KUBECONFIG_FILE"
          chmod 600 "$KUBECONFIG_FILE"
          echo "KUBECONFIG=$KUBECONFIG_FILE" >> $GITHUB_ENV
        env:
          KUBECONFIG_DATA: ${{
            inputs.cluster_environment == 'aws-openshift' && secrets.KUBECONFIG_AWS_OPENSHIFT ||
            inputs.cluster_environment == 'openshift-local' && secrets.KUBECONFIG_OPENSHIFT_LOCAL ||
            inputs.cluster_environment == 'openshift-baremetal' && secrets.KUBECONFIG_OPENSHIFT_BM ||
            inputs.cluster_environment == 'gcp' && secrets.KUBECONFIG_GCP ||
            secrets.KUBECONFIG_LOCAL || secrets.KUBECONFIG_CONTENT
            }}

      - name: Install Helm v3.15.4
        run: |
          if ! helm version 2>/dev/null | grep -q 'v3.15'; then
            curl -L https://get.helm.sh/helm-v3.15.4-linux-amd64.tar.gz -o helm-v3.15.4-linux-amd64.tar.gz
            tar -zxvf helm-v3.15.4-linux-amd64.tar.gz
            sudo mv linux-amd64/helm /usr/local/bin/helm
            rm -rf linux-amd64 helm-v3.15.4-linux-amd64.tar.gz
          fi
          helm version

      - name: Install kubectl v1.31.0
        run: |
          if ! kubectl version --client 2>/dev/null | grep -q 'v1.31'; then
            curl -LO "https://dl.k8s.io/release/v1.31.0/bin/linux/amd64/kubectl"
            chmod +x kubectl
            sudo mv kubectl /usr/local/bin/
          fi
          kubectl version --client

      - name: Set parameters
        run: |
          PARAMS="${{ inputs.cluster_params || 'ndcs=2,npcs=2,bs=4096,chunk_bs=4096,partitions=1,jm_count=4' }}"
          for pair in $(echo "$PARAMS" | tr ',' ' '); do
            key=$(echo "$pair" | cut -d= -f1)
            val=$(echo "$pair" | cut -d= -f2)
            case "$key" in
              ndcs)       echo "NDCS=$val" >> $GITHUB_ENV ;;
              npcs)       echo "NPCS=$val" >> $GITHUB_ENV ;;
              bs)         echo "BS=$val" >> $GITHUB_ENV ;;
              chunk_bs)   echo "CHUNK_BS=$val" >> $GITHUB_ENV ;;
              partitions) echo "PARTITIONS=$val" >> $GITHUB_ENV ;;
              jm_count)   echo "JM_COUNT=$val" >> $GITHUB_ENV ;;
            esac
          done

      - name: Verify kubectl connectivity
        run: kubectl get nodes

      - name: Ensure OVN-Kubernetes routingViaHost is enabled
        if: ${{ inputs.cluster_environment == 'aws-openshift' || inputs.cluster_environment == 'openshift-local' }}
        run: |
          CURRENT=$(oc get network.operator.openshift.io cluster \
            -o jsonpath='{.spec.defaultNetwork.ovnKubernetesConfig.gatewayConfig.routingViaHost}' 2>/dev/null || echo "")
          if [ "$CURRENT" = "true" ]; then
            echo "routingViaHost is already enabled, skipping patch"
          else
            echo "routingViaHost is not enabled (current: '$CURRENT'), applying patch..."
            oc patch network.operator.openshift.io cluster --type=merge \
              -p '{"spec":{"defaultNetwork":{"ovnKubernetesConfig":{"gatewayConfig":{"routingViaHost":true}}}}}'
            echo "Waiting 30s for OVN-Kubernetes to reconcile..."
            sleep 30
          fi

      - name: Pre-clean clients (kill fio, NVMe disconnect, unmount NFS, remove mount dirs)
        if: ${{ inputs.client_ips != '' }}
        shell: bash
        run: |
          set +e
          NFS_MOUNTPOINT="${{ inputs.nfs_mountpoint || '/mnt/nfs_share/' }}"

          run_remote_with_retry() {
            local ip="$1"; shift
            local cmd="$*"
            local max=5
            for attempt in $(seq 1 $max); do
              echo "  [attempt $attempt/$max] ssh ${SSH_USER}@${ip} ..."
              if ssh -o StrictHostKeyChecking=no -o ConnectTimeout=10 -i "${KEY_PATH}" \
                    "${SSH_USER}@${ip}" "${cmd}"; then
                return 0
              fi
              echo "  SSH failed, retrying in 5s..."
              sleep 5
            done
            echo "All $max attempts failed for $ip, continuing..."
            return 0
          }

          normalized_targets="$(echo "${{ inputs.client_ips }}" | tr ',' ' ' | xargs -n1 | sort -u | xargs || true)"

          if [ -z "${normalized_targets}" ]; then
            echo "No client IPs provided, skipping client cleanup."
            exit 0
          fi

          for ip in ${normalized_targets}; do
            echo "---- ${ip}: kill fio/tmux + disconnect NVMe + umount ${NFS_MOUNTPOINT} + clean mount dirs ----"
            run_remote_with_retry "${ip}" "set -euxo pipefail;
              pkill -9 fio || true;
              pkill -9 tmux || true;
              # Disconnect stale NVMe-oF connections from previous runs
              for nqn in \$(nvme list-subsys -o json 2>/dev/null | jq -r '.Subsystems[]?.NQN // empty' | grep lvol || true); do
                nvme disconnect -n \"\$nqn\" || true;
              done;
              # Unmount NFS
              mp='${NFS_MOUNTPOINT}';
              timeout 10 umount -f \"\$mp\" 2>/dev/null || umount -l \"\$mp\" 2>/dev/null || true;
              # Unmount and remove test mount dirs.
              # Try umount unconditionally — if not mounted it just fails silently.
              for mp in /mnt/test_location/*; do
                [ -e \"\$mp\" ] || continue;
                timeout 10 umount -f \"\$mp\" 2>/dev/null || umount -l \"\$mp\" 2>/dev/null || true;
              done;
              rm -rf /mnt/test_location/* 2>/dev/null || true"
          done

      # ── Cleanup old deployment ──────────────────────────────────────────────

      - name: Cleanup old deployment (full 12-phase cleanup)
        if: ${{ inputs.use_existing_cluster != 'true' }}
        run: |
          set +e

          # Run the operator's own cleanup script first if available
          if [ -x "$GITHUB_WORKSPACE/simplyblock-operator/helm-charts/scripts/cleanup-simplyblock.sh" ]; then
            bash $GITHUB_WORKSPACE/simplyblock-operator/helm-charts/scripts/cleanup-simplyblock.sh -f simplyblock || true
          fi

          # Combine worker_nodes and new_worker_nodes for comprehensive cleanup
          ALL_WORKERS="${{ inputs.worker_nodes }}"
          NEW_WORKERS="${{ inputs.new_worker_nodes }}"
          if [ -n "$NEW_WORKERS" ]; then
            ALL_WORKERS="${ALL_WORKERS},${NEW_WORKERS}"
          fi

          # Run full 12-phase cleanup (handles Helm R25+R26, CRs, CRDs,
          # cert-manager, kube-system, snapshots, PVCs/PVs, node cleanup,
          # labels, CSI hostpath data, namespace finalization).
          # Auto-detects Talos and skips node debug operations.
          CLUSTER_ENV="${{ inputs.cluster_environment || 'local' }}"
          bash $GITHUB_WORKSPACE/e2e/scripts/cleanup_upgrade_test.sh \
            -n simplyblock \
            -e "$CLUSTER_ENV" \
            -w "$ALL_WORKERS" \
            -c "$GITHUB_WORKSPACE/simplyblock-operator/helm-charts/charts/simplyblock-operator/crds/"

          # Remove simplyblock labels from new worker nodes specifically
          if [ -n "$NEW_WORKERS" ]; then
            IFS=',' read -ra NEW_NODES <<< "$NEW_WORKERS"
            for NODE in "${NEW_NODES[@]}"; do
              NODE=$(echo "$NODE" | xargs)
              [[ -z "$NODE" ]] && continue
              kubectl label node "$NODE" simplyblock.io/role- 2>/dev/null || true
            done
          fi

      - name: Cleanup old KMS deployment
        if: ${{ inputs.use_existing_cluster != 'true' }}
        run: |
          set +e
          echo "=== Uninstalling KMS (openbao) ==="
          helm uninstall openbao -n vault 2>/dev/null || true
          # Remove Helm release secrets in case uninstall left a broken/pending state
          kubectl delete secret -n vault -l name=openbao,owner=helm 2>/dev/null || true
          kubectl delete namespace vault --wait=false 2>/dev/null || true
          echo "Waiting for vault namespace to be deleted..."
          kubectl wait --for=delete namespace/vault --timeout=120s 2>/dev/null || true
          echo "=== KMS cleanup complete ==="

      # ── Label nodes ─────────────────────────────────────────────────────────

      - name: Label initial worker nodes
        if: ${{ inputs.use_existing_cluster != 'true' }}
        run: |
          CLUSTER_ENV="${{ inputs.cluster_environment || 'local' }}"
          IFS=',' read -ra NODES <<< "${{ inputs.worker_nodes }}"
          for NODE in "${NODES[@]}"; do
            if [ "$CLUSTER_ENV" = "local" ] || [ "$CLUSTER_ENV" = "openshift-local" ]; then
              echo "Labeling node $NODE with zone=default (local cluster)"
              kubectl label node "$NODE" topology.kubernetes.io/zone=default --overwrite
            else
              echo "Skipping zone label for $NODE (cloud cluster: $CLUSTER_ENV)"
            fi
            kubectl label node "$NODE" simplyblock.io/role=mgmt-plane --overwrite
          done

      # NOTE: new_worker_nodes are NOT labeled here. They must remain
      # unlabeled so the DaemonSet does not schedule storage-node pods on them
      # before the test adds them to the cluster via StorageNodeSet CR update.

      # ── Prepare namespace ───────────────────────────────────────────────────

      - name: Create namespace and set pod-security labels
        if: ${{ inputs.use_existing_cluster != 'true' }}
        run: |
          kubectl create namespace simplyblock --dry-run=client -o yaml | kubectl apply -f -
          kubectl label namespace simplyblock \
            pod-security.kubernetes.io/enforce=privileged \
            pod-security.kubernetes.io/warn=privileged \
            pod-security.kubernetes.io/audit=privileged \
            --overwrite

      - name: Create Docker registry secret
        if: ${{ inputs.use_existing_cluster != 'true' }}
        run: |
          kubectl create secret docker-registry regcred \
            --docker-server=https://index.docker.io/v1/ \
            --docker-username=${DOCKER_USER} \
            --docker-password=${DOCKER_PASSWORD} \
            --docker-email=a@b.com \
            -n simplyblock || true
        env:
          DOCKER_USER: ${{ secrets.DOCKER_USER }}
          DOCKER_PASSWORD: ${{ secrets.DOCKER_PASSWORD }}

      - name: Configure OpenShift SCC policies
        if: ${{ inputs.use_existing_cluster != 'true' && (inputs.cluster_environment == 'aws-openshift' || inputs.cluster_environment == 'openshift-local') }}
        run: |
          oc adm policy add-scc-to-user privileged -z default -n simplyblock
          oc adm policy add-scc-to-user anyuid -z default -n simplyblock
          oc label namespace simplyblock \
            pod-security.kubernetes.io/enforce=privileged \
            pod-security.kubernetes.io/audit=privileged \
            pod-security.kubernetes.io/warn=privileged \
            --overwrite

      # ── Deploy MinIO for backup tests (conditional) ────────────────────────

      - name: Setup MinIO for backup tests
        if: >-
          ${{
            inputs.use_existing_cluster != 'true' &&
            (inputs.cluster_security == 'backup' ||
             inputs.testname == 'backup-topology' ||
             contains(inputs.testname, 'Backup'))
          }}
        run: |
          set -euxo pipefail
          NAMESPACE=simplyblock
          EXTERNAL_MINIO="http://192.168.10.164:9000"
          MINIO_ACCESS_KEY="minioadmin"
          MINIO_SECRET_KEY="minioadmin"

          # Check if external MinIO is reachable FROM INSIDE the cluster
          # (runner can always reach it, but cluster nodes/pods may not)
          if kubectl run minio-check --rm -i --restart=Never --image=busybox:latest \
              --timeout=30s -- wget -q -O /dev/null -T 5 "${EXTERNAL_MINIO}/minio/health/live" >/dev/null 2>&1; then
            echo "=== External MinIO at ${EXTERNAL_MINIO} is reachable from cluster ==="
            MINIO_ENDPOINT="${EXTERNAL_MINIO}"
            echo "MINIO_SOURCE=external" >> "$GITHUB_ENV"
          else
            echo "=== External MinIO unreachable, deploying in-cluster MinIO ==="
            kubectl create namespace minio --dry-run=client -o yaml | kubectl apply -f -
            kubectl label namespace minio \
              pod-security.kubernetes.io/enforce=privileged \
              pod-security.kubernetes.io/warn=privileged \
              pod-security.kubernetes.io/audit=privileged \
              --overwrite

            # On OpenShift, grant the default SA the anyuid SCC so MinIO can run
            if kubectl api-resources | grep -q securitycontextconstraints; then
              oc adm policy add-scc-to-user anyuid -z default -n minio 2>/dev/null || true
            fi

            cat <<'MINIOYAML' | kubectl apply -f -
          apiVersion: apps/v1
          kind: Deployment
          metadata:
            name: minio
            namespace: minio
          spec:
            replicas: 1
            selector:
              matchLabels:
                app: minio
            template:
              metadata:
                labels:
                  app: minio
              spec:
                containers:
                - name: minio
                  image: minio/minio:latest
                  args: ["server", "/data"]
                  ports:
                  - containerPort: 9000
                  env:
                  - name: MINIO_ROOT_USER
                    value: minioadmin
                  - name: MINIO_ROOT_PASSWORD
                    value: minioadmin
                  volumeMounts:
                  - name: data
                    mountPath: /data
                volumes:
                - name: data
                  emptyDir: {}
          ---
          apiVersion: v1
          kind: Service
          metadata:
            name: minio
            namespace: minio
          spec:
            selector:
              app: minio
            ports:
            - port: 9000
              targetPort: 9000
          MINIOYAML

            echo "Waiting for MinIO pod to appear..."
            for i in $(seq 1 30); do
              if kubectl get pod -l app=minio -n minio --no-headers 2>/dev/null | grep -q .; then
                echo "MinIO pod found"
                break
              fi
              if [ "$i" -eq 30 ]; then
                echo "ERROR: MinIO pod never appeared. Deployment status:"
                kubectl -n minio describe deployment minio
                kubectl -n minio get events --sort-by='.lastTimestamp' | tail -20
                exit 1
              fi
              echo "  Waiting for pod ($i/30)..."
              sleep 5
            done
            echo "Waiting for MinIO pod to be ready..."
            kubectl wait --for=condition=ready pod -l app=minio -n minio --timeout=120s

            # Port-forward to make MinIO accessible from the runner
            kubectl port-forward -n minio svc/minio 9000:9000 &
            echo "MC_PORT_FWD_PID=$!" >> "$GITHUB_ENV"
            sleep 3

            MINIO_ENDPOINT="http://minio.minio.svc:9000"
            echo "MINIO_SOURCE=in-cluster" >> "$GITHUB_ENV"
          fi

          echo "MINIO_ENDPOINT=${MINIO_ENDPOINT}" >> "$GITHUB_ENV"

          # Create backup-credentials K8s Secret in cluster namespace
          cat <<CREDEOF | kubectl apply -f -
          apiVersion: v1
          kind: Secret
          metadata:
            name: backup-credentials
            namespace: ${NAMESPACE}
          type: Opaque
          stringData:
            access_key_id: ${MINIO_ACCESS_KEY}
            secret_access_key: ${MINIO_SECRET_KEY}
          CREDEOF

          echo "=== MinIO setup complete (source=${MINIO_SOURCE:-external}, endpoint=${MINIO_ENDPOINT}) ==="

      - name: Start MinIO trace logging
        if: >-
          ${{
            inputs.use_existing_cluster != 'true' &&
            (inputs.cluster_security == 'backup' ||
             inputs.testname == 'backup-topology' ||
             contains(inputs.testname, 'Backup'))
          }}
        run: |
          set -euxo pipefail

          # This VM hosts multiple parallel runners, so every path/alias/
          # process used here MUST be scoped to this run — a shared
          # /tmp/mc or a pattern-matching pkill would race with, corrupt,
          # or kill another job's in-flight mc install/trace.
          MC_BIN="/tmp/mc-${GITHUB_RUN_ID}"
          MC_ALIAS="myminio-${GITHUB_RUN_ID}"
          echo "MC_BIN=${MC_BIN}" >> "$GITHUB_ENV"
          echo "MC_ALIAS=${MC_ALIAS}" >> "$GITHUB_ENV"

          # 1. Install mc (MinIO Client) to our own per-run path
          curl -fsSL https://dl.min.io/client/mc/release/linux-amd64/mc -o "${MC_BIN}"
          chmod +x "${MC_BIN}"

          # 2. Configure our own mc alias — for in-cluster MinIO use localhost (port-forward)
          if [ "${MINIO_SOURCE:-external}" = "in-cluster" ]; then
            "${MC_BIN}" alias set "${MC_ALIAS}" http://localhost:9000 minioadmin minioadmin
          else
            "${MC_BIN}" alias set "${MC_ALIAS}" http://192.168.10.164:9000 minioadmin minioadmin
          fi

          # 3. Start admin trace in background
          MINIO_TRACE_LOG="/tmp/minio-trace-${GITHUB_RUN_ID}.log"
          echo "MINIO_TRACE_LOG=${MINIO_TRACE_LOG}" >> "$GITHUB_ENV"
          nohup "${MC_BIN}" admin trace -v --all "${MC_ALIAS}" > "${MINIO_TRACE_LOG}" 2>&1 &
          echo "MC_TRACE_PID=$!" >> "$GITHUB_ENV"

          echo "=== MinIO trace logging started (PID=$!, log=${MINIO_TRACE_LOG}) ==="

      - name: Wait before helm install
        if: ${{ inputs.use_existing_cluster != 'true' }}
        run: sleep 30

      # ── Deploy CSI stack ────────────────────────────────────────────────────

      - name: Install cert-manager (TLS prerequisite)
        if: ${{ inputs.use_existing_cluster != 'true' && inputs.tls_enabled }}
        run: |
          helm repo add jetstack https://charts.jetstack.io
          helm repo update
          helm upgrade --install cert-manager jetstack/cert-manager \
            --namespace cert-manager --create-namespace \
            --version v1.13.0 --set installCRDs=true

          echo "Waiting for cert-manager pods to be ready..."
          kubectl wait --for=condition=Ready pods --all -n cert-manager --timeout=120s
          echo "cert-manager is ready."

      - name: Install Helm Chart for simplyblock-operator
        if: ${{ inputs.use_existing_cluster != 'true' }}
        run: |
          set -euxo pipefail
          cd $GITHUB_WORKSPACE/simplyblock-operator/helm-charts/charts/simplyblock-operator/

          TLS_FLAGS=""
          if [ "${{ inputs.tls_enabled }}" = "true" ]; then
            TLS_FLAGS="--set tls.enabled=true --set tls.mutual_enabled=true"
          fi

          CSI_FLAGS=""
          if [ -n "${{ inputs.csi_repository }}" ]; then
            CSI_FLAGS="$CSI_FLAGS --set image.csi.repository=${{ inputs.csi_repository }}"
          fi
          if [ -n "${{ inputs.csi_tag }}" ]; then
            CSI_FLAGS="$CSI_FLAGS --set image.csi.tag=${{ inputs.csi_tag }}"
          fi

          helm upgrade --install spdk-csi ./ \
            --namespace simplyblock \
            --create-namespace \
            --debug \
            --timeout 10m \
            --set controlplane.enabled=true \
            --set operator.enabled=true \
            --set controlplane.csiHostpathDriver.enabled=true \
            --set controlplane.storageclass.name=local-hostpath \
            --set image.simplyblock.repository="${{ inputs.simplyblock_repository || 'public.ecr.aws/simply-block/simplyblock' }}" \
            --set image.simplyblock.tag="${{ inputs.simplyblock_image }}" \
            --set image.operator.repository="${{ inputs.operator_repository || 'simplyblock/simplyblock-operator' }}" \
            --set image.operator.tag="${{ inputs.operator_tag || 'main' }}" \
            --set csiConfig.simplybk.ip="http://simplyblock-webappapi.simplyblock:5000" \
            --set prometheus.server.persistentVolume.storageClass=local-hostpath \
            --set controlplane.observability.enabled=true \
            --set opensearch.persistence.storageClass=local-hostpath \
            --set controlplane.observability.minio.storageClass=local-hostpath \
            --set image.simplyblock.pullPolicy=IfNotPresent \
            --set image.operator.pullPolicy=IfNotPresent \
            $TLS_FLAGS $CSI_FLAGS

      - name: Grant OpenShift SCC to all service accounts (post-helm)
        if: ${{ inputs.use_existing_cluster != 'true' && (inputs.cluster_environment == 'aws-openshift' || inputs.cluster_environment == 'openshift-local') }}
        run: |
          for sa in $(oc get sa -n simplyblock -o name | cut -d/ -f2); do
            oc adm policy add-scc-to-user privileged -z $sa -n simplyblock
            echo "Granted privileged SCC to $sa"
          done

      - name: Patch fluent-bit daemonset (post-helm)
        if: ${{ inputs.use_existing_cluster != 'true' }}
        run: |
          NAMESPACE=simplyblock
          echo "=== Waiting for simplyblock-fluent-bit daemonset (up to 5 min) ==="
          PATCHED=false
          for i in $(seq 1 30); do
            if kubectl get daemonset simplyblock-fluent-bit -n $NAMESPACE &>/dev/null; then
              echo "fluent-bit daemonset found, patching affinity..."
              kubectl patch daemonset simplyblock-fluent-bit -n $NAMESPACE \
                --type=merge -p '{"spec":{"template":{"spec":{"affinity":null}}}}'
              echo "fluent-bit daemonset patched successfully"
              # Also patch CSI node daemonset with client toleration
              if kubectl get daemonset simplyblock-csi-node -n $NAMESPACE &>/dev/null; then
                kubectl patch daemonset simplyblock-csi-node -n $NAMESPACE \
                  --type=merge -p '{"spec":{"template":{"spec":{"tolerations":[{"key":"node-role","operator":"Equal","value":"client","effect":"NoSchedule"}]}}}}'
                echo "csi-node daemonset patched with client toleration"
              fi
              PATCHED=true
              break
            fi
            echo "fluent-bit daemonset not found yet ($i/30), waiting 10s..."
            sleep 10
          done
          echo "FLUENTBIT_PATCHED=$PATCHED" >> $GITHUB_ENV

      - name: Patch service accounts with imagePullSecrets
        if: ${{ inputs.use_existing_cluster != 'true' }}
        run: |
          for sa in $(kubectl get serviceaccounts -n simplyblock --no-headers | awk '{print $1}'); do
            kubectl patch serviceaccount "$sa" -n simplyblock \
              --patch '{"imagePullSecrets": [{"name": "regcred"}]}'
          done
          for deploy in $(kubectl get deployments -n simplyblock --no-headers -o custom-columns=:metadata.name 2>/dev/null); do
            kubectl patch deployment "$deploy" -n simplyblock \
              --type=merge -p '{"spec":{"template":{"spec":{"imagePullSecrets":[{"name":"regcred"}]}}}}' || true
          done

      - name: Delete ImagePullBackOff pods and wait for Running (pre-CRD)
        if: ${{ inputs.use_existing_cluster != 'true' }}
        run: |
          NAMESPACE=simplyblock
          for pod in $(kubectl get pods -n $NAMESPACE --no-headers 2>/dev/null | grep ImagePullBackOff | awk '{print $1}'); do
            kubectl delete pod -n $NAMESPACE "$pod" || true
          done

          echo "=== Waiting for all pods to be Running ==="
          for i in $(seq 1 10); do
            BAD_PODS=$(kubectl get pods -n $NAMESPACE --no-headers 2>/dev/null | grep -vE 'Running|Completed' | wc -l)
            if [ "$BAD_PODS" -eq 0 ]; then
              echo "All pods are Running"
              break
            fi
            echo "Still $BAD_PODS pod(s) not ready ($i/10)..."
            kubectl get pods -n $NAMESPACE --no-headers 2>/dev/null | grep -vE 'Running|Completed' || true
            sleep 10
          done

      # ── Apply custom resources (initial nodes only) ────────────────────────

      - name: Wait for operator pod to be ready
        if: ${{ inputs.use_existing_cluster != 'true' }}
        run: |
          NAMESPACE=simplyblock
          echo "=== Waiting for operator pod ==="
          for i in $(seq 1 60); do
            POD=$(kubectl -n $NAMESPACE get pods \
              -l app=simplyblock-admin-control \
              -o jsonpath='{.items[0].metadata.name}' 2>/dev/null) || true
            if [ -n "$POD" ]; then
              PHASE=$(kubectl -n $NAMESPACE get pod "$POD" \
                -o jsonpath='{.status.phase}' 2>/dev/null) || true
              if [ "$PHASE" = "Running" ]; then
                echo "Operator pod $POD is Running"
                break
              fi
            fi
            echo "Waiting for operator pod ($i/60)..."
            sleep 10
          done

      - name: Setup KMS (vault) for encryption
        if: ${{ inputs.use_existing_cluster != 'true' && inputs.tls_enabled }}
        run: |
          # Ensure vault namespace from previous run is fully terminated before installing.
          echo "Checking if vault namespace is still terminating..."
          for i in $(seq 1 60); do
            NS_STATUS=$(kubectl get namespace vault -o jsonpath='{.status.phase}' 2>/dev/null || echo "gone")
            if [ "$NS_STATUS" = "gone" ]; then
              echo "vault namespace does not exist — ready to install."
              break
            fi
            if [ "$NS_STATUS" = "Terminating" ]; then
              echo "vault namespace still Terminating ($i/60), clearing stuck finalizers..."
              kubectl get all -n vault --no-headers -o name 2>/dev/null | while read -r res; do
                kubectl patch "$res" -n vault --type=merge -p '{"metadata":{"finalizers":null}}' 2>/dev/null || true
              done
              kubectl patch namespace vault --type=merge -p '{"spec":{"finalizers":[]}}' 2>/dev/null || true
            else
              echo "vault namespace in phase '$NS_STATUS' ($i/60), waiting..."
            fi
            sleep 5
          done
          if kubectl get namespace vault &>/dev/null; then
            echo "WARNING: vault namespace still exists after 300s, force-deleting resources..."
            kubectl delete all --all -n vault --force --grace-period=0 2>/dev/null || true
            kubectl delete namespace vault --force --grace-period=0 2>/dev/null || true
            sleep 10
          fi

          # Auto-detect default StorageClass for vault PVCs
          STORAGE_CLASS=$(kubectl get sc -o jsonpath='{.items[?(@.metadata.annotations.storageclass\.kubernetes\.io/is-default-class=="true")].metadata.name}' | awk '{print $1}')
          if [ -z "$STORAGE_CLASS" ]; then
            STORAGE_CLASS=$(kubectl get sc -o jsonpath='{.items[0].metadata.name}')
          fi
          echo "Using StorageClass for KMS: $STORAGE_CLASS"

          export STORAGE_CLASS
          UNSEAL_KEYS_FILE=./openbao-init.txt bash $GITHUB_WORKSPACE/simplyblock-operator/helm-charts/scripts/setup-kms.sh

          echo "Waiting for vault pods to be ready..."
          kubectl wait --for=condition=Ready pods -l app.kubernetes.io/name=openbao -n vault --timeout=300s || true
          echo "KMS setup complete."

      - name: Apply operator custom resources
        if: ${{ inputs.use_existing_cluster != 'true' }}
        run: |
          NAMESPACE=simplyblock
          IFC_NAMES="${{ inputs.ifc_names || 'br-ex:enp2s0f0' }}"
          MGMT_IFC="${IFC_NAMES%%:*}"
          DATA_NICS="${IFC_NAMES#*:}"
          MAX_SUBSYS="${{ inputs.max_subsys || '30' }}"
          SB_REPO="${{ inputs.simplyblock_repository || 'public.ecr.aws/simply-block/simplyblock' }}"
          SB_TAG="${{ inputs.simplyblock_image }}"
          SPDK_IMAGE="${{ inputs.spdk_image }}"
          PARTITIONS="${PARTITIONS:-0}"
          JM_COUNT="${JM_COUNT:-3}"
          OPENSHIFT_CLUSTER="${{ (inputs.cluster_environment == 'aws-openshift' || inputs.cluster_environment == 'openshift-local' || inputs.cluster_environment == 'openshift-baremetal') && 'true' || 'false' }}"
          CORE_PERCENTAGE="${{ inputs.cluster_environment == 'aws-openshift' && '50' || inputs.cluster_environment == 'openshift-local' && '50' || inputs.cluster_environment == 'openshift-baremetal' && '50' || inputs.cluster_environment == 'local' && '35' || '65' }}"
          SKIP_KUBELET_CONFIG="${{ (inputs.cluster_environment == 'aws-openshift' || inputs.cluster_environment == 'openshift-local' || inputs.cluster_environment == 'openshift-baremetal') && 'false' || 'true' }}"
          FORCE_FORMAT_4K="${{ (inputs.cluster_environment == 'aws-openshift' || inputs.cluster_environment == 'openshift-local' || inputs.cluster_environment == 'openshift-baremetal') && 'true' || 'false' }}"
          DRIVE_SIZE_RANGE="${{ inputs.cluster_environment == 'openshift-baremetal' && '1500G-2000G' || '1.7T-2T' }}"
          ENABLE_CPU_TOPOLOGY="true"
          CLUSTER_ENV="${{ inputs.cluster_environment || 'local' }}"
          if [ "$CLUSTER_ENV" = "local" ]; then
            ENABLE_CPU_TOPOLOGY="false"
          fi

          # Convert partitions to enableJournalDevice (inverted: 0→true, 1→false)
          if [ "${PARTITIONS}" = "0" ]; then
            ENABLE_JOURNAL_DEVICE="true"
          else
            ENABLE_JOURNAL_DEVICE="false"
          fi

          # Calculate vcpuCount from CORE_PERCENTAGE by querying actual CPU count on a worker node
          FIRST_WORKER="${{ inputs.worker_nodes }}"
          FIRST_WORKER="${FIRST_WORKER%%,*}"
          if [ "$OPENSHIFT_CLUSTER" = "true" ]; then
            TOTAL_CPUS=$(oc debug node/"${FIRST_WORKER}" -- chroot /host nproc 2>/dev/null || echo "0")
          else
            TOTAL_CPUS=$(kubectl get node "${FIRST_WORKER}" -o jsonpath='{.status.capacity.cpu}' 2>/dev/null || echo "0")
          fi
          TOTAL_CPUS=$(echo "${TOTAL_CPUS}" | tr -d '[:space:]')
          if [ "${TOTAL_CPUS}" -gt 0 ] 2>/dev/null; then
            VCPU_COUNT=$(( TOTAL_CPUS * CORE_PERCENTAGE / 100 ))
            [ "${VCPU_COUNT}" -lt 1 ] && VCPU_COUNT=1
          else
            echo "WARNING: Could not determine CPU count, defaulting vcpuCount to 4"
            VCPU_COUNT=4
          fi
          echo "Calculated vcpuCount=${VCPU_COUNT} (${TOTAL_CPUS} CPUs * ${CORE_PERCENTAGE}%)"

          RESERVED_CPU_YAML=""
          if [ "${{ inputs.cluster_environment }}" = "openshift-baremetal" ]; then
            RESERVED_CPU_YAML='  reservedSystemCPU: "0,1,10,11,16,17,26,27"'
          fi

          PCIE_MODEL_YAML=""
          CLUSTER_ENV="${{ inputs.cluster_environment }}"
          if [ "$CLUSTER_ENV" != "openshift-baremetal" ] && [ "$CLUSTER_ENV" != "openshift-local" ]; then
            PCIE_MODEL_YAML='  pcieModel: "SAMSUNG MZQLB1T9HAJR-00007"'
          fi

          # Build workerNodes YAML list from initial worker nodes only
          WORKER_YAML=""
          IFS=',' read -ra NODES <<< "${{ inputs.worker_nodes }}"
          for NODE in "${NODES[@]}"; do
            WORKER_YAML="${WORKER_YAML}    - ${NODE}"$'\n'
          done

          # Conditionally build backup spec block
          BACKUP_SPEC=""
          CLUSTER_SEC="${{ inputs.cluster_security || 'none' }}"
          TESTNAME_INPUT="${{ inputs.testname || '' }}"
          if [ "$CLUSTER_SEC" = "backup" ] || [ "$TESTNAME_INPUT" = "backup-topology" ] || echo "$TESTNAME_INPUT" | grep -qi "Backup"; then
            BACKUP_SPEC="  backup:
              credentialsSecretRef:
                name: backup-credentials
              localEndpoint: ${MINIO_ENDPOINT}
              localTesting: true
              secondaryTarget: 0
              snapshotBackups: true
              withCompression: false"
          fi

          # Build vault settings and encryption pool blocks (conditional on TLS)
          VAULT_SETTINGS=""
          ENCRYPTION_POOL_YAML=""
          if [ "${{ inputs.tls_enabled }}" = "true" ]; then
            VAULT_SETTINGS="  hashicorpVaultSettings:
              baseURL: \"https://openbao.vault:8200\""
            ENCRYPTION_POOL_YAML="---
          apiVersion: storage.simplyblock.io/v1alpha1
          kind: StoragePool
          metadata:
            name: encryption-pool
            namespace: ${NAMESPACE}
          spec:
            clusterName: simplyblock-cluster
            storageClassParameters:
              encryption: true"
          fi

          echo "=== Applying custom resources (initial ${#NODES[@]} workers) ==="
          cat <<EOF | kubectl apply -f -
          apiVersion: storage.simplyblock.io/v1alpha1
          kind: StorageCluster
          metadata:
            name: simplyblock-cluster
            namespace: ${NAMESPACE}
          spec:
            fabricType: tcp
            enableNodeAffinity: true
            stripe:
              dataChunks: ${NDCS}
              parityChunks: ${NPCS}
            warningThreshold:
              capacity: 95
              provisionedCapacity: 97
            criticalThreshold:
              capacity: 96
              provisionedCapacity: 98
          ${BACKUP_SPEC}
          ${VAULT_SETTINGS}
            maxSubsystemCount: ${MAX_SUBSYS}
            vcpuCount: ${VCPU_COUNT}

          ---
          apiVersion: storage.simplyblock.io/v1alpha1
          kind: StoragePool
          metadata:
            name: simplyblock-pool
            namespace: ${NAMESPACE}
          spec:
            clusterName: simplyblock-cluster
          ${ENCRYPTION_POOL_YAML}
          ---
          apiVersion: storage.simplyblock.io/v1alpha1
          kind: StorageNodeSet
          metadata:
            name: simplyblock-node
            namespace: ${NAMESPACE}
          spec:
            clusterName: simplyblock-cluster
            clusterImage: "${SB_REPO}:${SB_TAG}"
            spdkImage: "${SPDK_IMAGE}"
            spdkProxyImage: "${SB_REPO}:${SB_TAG}"
          ${PCIE_MODEL_YAML}
            driveSizeRange: "${DRIVE_SIZE_RANGE}"
            mgmtIfname: ${MGMT_IFC}
            dataIfname:
              - ${DATA_NICS}
          ${RESERVED_CPU_YAML}
            enableJournalDevice: ${ENABLE_JOURNAL_DEVICE}
            journalManager:
              count: ${JM_COUNT}
              percentPerDevice: 3
            skipKubeletConfiguration: ${SKIP_KUBELET_CONFIG}
            enableCpuTopology: ${ENABLE_CPU_TOPOLOGY}
            openShiftCluster: ${OPENSHIFT_CLUSTER}
            ubuntuHost: false
            forceFormat4K: ${FORCE_FORMAT_4K}
            workerNodes:
          ${WORKER_YAML}
          EOF

          echo "Custom resources applied with initial workers"
          sleep 10
        env:
          NDCS: ${{ env.NDCS }}
          NPCS: ${{ env.NPCS }}

      - name: Patch service accounts with imagePullSecrets (post-CRD)
        if: ${{ inputs.use_existing_cluster != 'true' }}
        run: |
          for sa in $(kubectl get serviceaccounts -n simplyblock --no-headers | awk '{print $1}'); do
            kubectl patch serviceaccount "$sa" -n simplyblock \
              --patch '{"imagePullSecrets": [{"name": "regcred"}]}'
          done
          for deploy in $(kubectl get deployments -n simplyblock --no-headers -o custom-columns=:metadata.name 2>/dev/null); do
            kubectl patch deployment "$deploy" -n simplyblock \
              --type=merge -p '{"spec":{"template":{"spec":{"imagePullSecrets":[{"name":"regcred"}]}}}}' || true
          done

      - name: Delete ImagePullBackOff pods and wait for Running (post-CRD)
        if: ${{ inputs.use_existing_cluster != 'true' }}
        run: |
          NAMESPACE=simplyblock
          for pod in $(kubectl get pods -n $NAMESPACE --no-headers 2>/dev/null | grep ImagePullBackOff | awk '{print $1}'); do
            kubectl delete pod -n $NAMESPACE "$pod" || true
          done

          echo "=== Waiting for all pods to be Running ==="
          for i in $(seq 1 10); do
            BAD_PODS=$(kubectl get pods -n $NAMESPACE --no-headers 2>/dev/null | grep -vE 'Running|Completed' | wc -l)
            if [ "$BAD_PODS" -eq 0 ]; then
              echo "All pods are Running"
              break
            fi
            echo "Still $BAD_PODS pod(s) not ready ($i/10)..."
            kubectl get pods -n $NAMESPACE --no-headers 2>/dev/null | grep -vE 'Running|Completed' || true
            sleep 10
          done

      - name: Wait for storage SA, patch secrets, grant SCC and restart daemonset
        if: ${{ inputs.use_existing_cluster != 'true' }}
        run: |
          NAMESPACE=simplyblock
          echo "Waiting for simplyblock-storage-node-sa service account..."
          for i in $(seq 1 60); do
            if kubectl get sa simplyblock-storage-node-sa -n $NAMESPACE &>/dev/null; then
              echo "Service account simplyblock-storage-node-sa found"
              break
            fi
            echo "Attempt $i/60: SA not found yet, waiting 10s..."
            sleep 10
          done

          echo "Patching service accounts and deployments with imagePullSecrets..."
          for sa in $(kubectl get serviceaccounts -n $NAMESPACE --no-headers | awk '{print $1}'); do
            kubectl patch serviceaccount "$sa" -n $NAMESPACE \
              --patch '{"imagePullSecrets": [{"name": "regcred"}]}'
          done
          for deploy in $(kubectl get deployments -n $NAMESPACE --no-headers -o custom-columns=:metadata.name 2>/dev/null); do
            kubectl patch deployment "$deploy" -n $NAMESPACE \
              --type=merge -p '{"spec":{"template":{"spec":{"imagePullSecrets":[{"name":"regcred"}]}}}}' || true
          done

          if [ "$CLUSTER_ENV" = "aws-openshift" ] || [ "$CLUSTER_ENV" = "openshift-local" ]; then
            echo "Granting privileged SCC to all service accounts..."
            for sa in $(oc get sa -n $NAMESPACE -o name | cut -d/ -f2); do
              oc adm policy add-scc-to-user privileged -z $sa -n $NAMESPACE
              echo "Granted privileged SCC to $sa"
            done
          fi

          echo "=== Waiting for storage node daemonset ==="
          for i in $(seq 1 60); do
            if kubectl get daemonset simplyblock-storage-node-ds-simplyblock-cluster -n $NAMESPACE &>/dev/null; then
              echo "Daemonset found, waiting for rollout to complete..."
              kubectl rollout status daemonset simplyblock-storage-node-ds-simplyblock-cluster -n $NAMESPACE --timeout=300s || true
              kubectl get pods -n $NAMESPACE
              echo "Restarting daemonset..."
              kubectl rollout restart -n $NAMESPACE ds simplyblock-storage-node-ds-simplyblock-cluster
              break
            fi
            echo "Daemonset not found yet ($i/60), waiting 10s..."
            sleep 10
          done
        env:
          CLUSTER_ENV: ${{ inputs.cluster_environment }}

      # ── Wait for cluster readiness ──────────────────────────────────────────

      - name: Wait for cluster to become active
        timeout-minutes: ${{ inputs.cluster_environment == 'local' && 25 || 80 }}
        id: cluster_status
        run: |
          NAMESPACE=simplyblock
          ADMIN_POD=""
          MAX_POLL=300

          echo "=== Waiting for admin pod to be ready ==="
          for i in $(seq 1 60); do
            ADMIN_POD=$(kubectl -n $NAMESPACE get pods \
              -l app=simplyblock-admin-control \
              -o jsonpath='{.items[0].metadata.name}' 2>/dev/null) || true

            if [ -n "$ADMIN_POD" ]; then
              PHASE=$(kubectl -n $NAMESPACE get pod "$ADMIN_POD" \
                -o jsonpath='{.status.phase}' 2>/dev/null) || true
              if [ "$PHASE" = "Running" ]; then
                echo "Admin pod $ADMIN_POD is Running"
                break
              fi
              echo "Admin pod $ADMIN_POD phase=$PHASE ($i/60)"
            else
              echo "Admin pod not found yet ($i/60)"
            fi
            sleep 10
          done

          if [ -z "$ADMIN_POD" ]; then
            echo "ERROR: Admin pod never appeared"
            kubectl -n $NAMESPACE get pods
            exit 1
          fi

          # Expected number of snode-spdk pods = number of initial worker nodes
          IFS=',' read -ra NODES <<< "${{ inputs.worker_nodes }}"
          EXPECTED_SNODES=${#NODES[@]}
          echo "=== Waiting for $EXPECTED_SNODES snode-spdk pods to be ready (MAX_POLL=$MAX_POLL) ==="

          for i in $(seq 1 $MAX_POLL); do
            TOTAL=$(kubectl -n $NAMESPACE get pods -l role=simplyblock-storage-node --no-headers 2>/dev/null | wc -l)
            READY=$(kubectl -n $NAMESPACE get pods -l role=simplyblock-storage-node --no-headers 2>/dev/null | grep -c "Running" || true)

            if [ "$READY" -ge "$EXPECTED_SNODES" ]; then
              echo "All $READY/$EXPECTED_SNODES snode-spdk pods are Running"
              break
            fi

            echo "snode-spdk pods: $READY/$EXPECTED_SNODES Running (total=$TOTAL) ($i/$MAX_POLL)"
            if [ "$i" -eq "$MAX_POLL" ]; then
              echo "ERROR: snode-spdk pods did not become ready within timeout"
              kubectl -n $NAMESPACE get pods
              exit 1
            fi
            sleep 10
          done

          echo "=== Re-resolving admin pod before cluster status polling ==="
          ADMIN_POD=""
          for i in $(seq 1 30); do
            ADMIN_POD=$(kubectl -n $NAMESPACE get pods \
              -l app=simplyblock-admin-control \
              -o jsonpath='{.items[0].metadata.name}' 2>/dev/null) || true
            if [ -n "$ADMIN_POD" ]; then
              PHASE=$(kubectl -n $NAMESPACE get pod "$ADMIN_POD" \
                -o jsonpath='{.status.phase}' 2>/dev/null) || true
              if [ "$PHASE" = "Running" ]; then
                echo "Admin pod resolved: $ADMIN_POD (Running)"
                break
              fi
              echo "Admin pod $ADMIN_POD phase=$PHASE, retrying ($i/30)..."
              ADMIN_POD=""
            else
              echo "Admin pod not found, retrying ($i/30)..."
            fi
            sleep 10
          done
          if [ -z "$ADMIN_POD" ]; then
            echo "ERROR: Could not find running admin pod before polling"
            kubectl -n $NAMESPACE get pods
            exit 1
          fi

          echo "=== Polling cluster status (MAX_POLL=$MAX_POLL) ==="
          for i in $(seq 1 $MAX_POLL); do
            # Re-resolve admin pod each iteration in case it gets recreated
            NEW_POD=$(kubectl -n $NAMESPACE get pods \
              -l app=simplyblock-admin-control \
              -o jsonpath='{.items[0].metadata.name}' 2>/dev/null) || true
            NEW_PHASE=$(kubectl -n $NAMESPACE get pod "$NEW_POD" \
              -o jsonpath='{.status.phase}' 2>/dev/null) || true
            if [ -n "$NEW_POD" ] && [ "$NEW_PHASE" = "Running" ] && [ "$NEW_POD" != "$ADMIN_POD" ]; then
              echo "Admin pod changed: $ADMIN_POD -> $NEW_POD"
              ADMIN_POD="$NEW_POD"
            fi

            echo "[poll $i/$MAX_POLL] Using admin pod: $ADMIN_POD"
            OUTPUT=$(kubectl -n $NAMESPACE exec "$ADMIN_POD" -- \
              sbctl cluster list 2>&1) || true
            echo "[poll $i/$MAX_POLL] sbctl cluster list output:"
            echo "$OUTPUT"

            if echo "$OUTPUT" | grep -qi "active"; then
              echo "Cluster is active!"

              # Extract cluster ID and secret
              CLUSTER_ID=$(echo "$OUTPUT" | awk 'NR==4{print $2}')
              CLUSTER_SECRET=$(echo "$OUTPUT" | awk 'NR==4{print $NF}')

              # Fallback: try JSON format if table parsing fails
              if [ -z "$CLUSTER_ID" ] || [ "$CLUSTER_ID" = "+" ]; then
                echo "Table parsing failed (CLUSTER_ID=$CLUSTER_ID), trying JSON..."
                JSON_OUT=$(kubectl -n $NAMESPACE exec "$ADMIN_POD" -- \
                  sbctl cluster list --json 2>&1) || true
                echo "JSON output: $JSON_OUT"
                CLUSTER_ID=$(echo "$JSON_OUT" | jq -r '.[0].id // .[0].uuid // empty')
                CLUSTER_SECRET=$(echo "$JSON_OUT" | jq -r '.[0].secret // empty')
              fi

              echo "Parsed CLUSTER_ID=${CLUSTER_ID}"
              echo "CLUSTER_ID=${CLUSTER_ID}" >> $GITHUB_ENV
              echo "CLUSTER_SECRET=${CLUSTER_SECRET}" >> $GITHUB_ENV
              exit 0
            fi

            echo "Cluster not active yet ($i/$MAX_POLL)..."
            sleep 10
          done

          echo "WARNING: Cluster did not become active within timeout — will attempt force-activate"
          kubectl -n $NAMESPACE get pods
          kubectl -n $NAMESPACE exec "$ADMIN_POD" -- sbctl cluster list 2>&1 || true

      - name: Verify and force-activate cluster if needed
        if: ${{ inputs.use_existing_cluster != 'true' }}
        run: |
          NAMESPACE=simplyblock
          ADMIN_POD=$(kubectl -n $NAMESPACE get pods \
            -l app=simplyblock-admin-control \
            -o jsonpath='{.items[0].metadata.name}' 2>/dev/null) || true

          if [ -z "$ADMIN_POD" ]; then
            echo "ERROR: No admin pod found"
            exit 1
          fi

          # Helper: extract cluster ID and secret from sbctl output and export to GITHUB_ENV
          extract_cluster_info() {
            local output="$1"
            local cid csecret
            cid=$(echo "$output" | awk 'NR==4{print $2}')
            csecret=$(echo "$output" | awk 'NR==4{print $NF}')
            if [ -z "$cid" ] || [ "$cid" = "+" ]; then
              echo "Table parsing failed, trying JSON..." >&2
              local json_out
              json_out=$(kubectl -n $NAMESPACE exec "$ADMIN_POD" -- \
                sbctl cluster list --json 2>&1) || true
              cid=$(echo "$json_out" | jq -r '.[0].id // .[0].uuid // empty')
              csecret=$(echo "$json_out" | jq -r '.[0].secret // empty')
            fi
            if [ -n "$cid" ] && [ "$cid" != "+" ]; then
              echo "CLUSTER_ID=${cid}" >> $GITHUB_ENV
              echo "CLUSTER_SECRET=${csecret}" >> $GITHUB_ENV
              echo "Extracted CLUSTER_ID=${cid}" >&2
            fi
            echo "$cid"
          }

          echo "=== Verifying cluster activation ==="
          OUTPUT=$(kubectl -n $NAMESPACE exec "$ADMIN_POD" -- \
            sbctl cluster list 2>&1) || true
          echo "$OUTPUT"

          if echo "$OUTPUT" | grep -qi "active"; then
            echo "Cluster is active, ensuring env vars are set"
            extract_cluster_info "$OUTPUT"
            exit 0
          fi

          echo "Cluster is NOT active, attempting forced activation..."
          CID=$(extract_cluster_info "$OUTPUT")
          if [ -n "$CID" ] && [ "$CID" != "+" ]; then
            kubectl -n $NAMESPACE exec "$ADMIN_POD" -- \
              sbctl -d cluster activate "${CID}" 2>&1 || true
          else
            echo "WARNING: Could not extract cluster ID, trying activate without ID..."
            kubectl -n $NAMESPACE exec "$ADMIN_POD" -- \
              sbctl -d cluster activate 2>&1 || true
          fi

          echo "Waiting 60s for activation to take effect..."
          sleep 60

          OUTPUT=$(kubectl -n $NAMESPACE exec "$ADMIN_POD" -- \
            sbctl cluster list 2>&1) || true
          echo "$OUTPUT"

          if echo "$OUTPUT" | grep -qi "active"; then
            echo "Cluster is now active after forced activation"
            extract_cluster_info "$OUTPUT"
            exit 0
          fi

          echo "ERROR: Cluster is still not active after forced activation"
          exit 1

      - name: Patch fluent-bit daemonset (post-cluster-active)
        if: ${{ env.FLUENTBIT_PATCHED != 'true' }}
        run: |
          NAMESPACE=simplyblock
          echo "=== fluent-bit was not patched after helm install, retrying now ==="
          for i in $(seq 1 30); do
            if kubectl get daemonset simplyblock-fluent-bit -n $NAMESPACE &>/dev/null; then
              kubectl patch daemonset simplyblock-fluent-bit -n $NAMESPACE \
                --type=merge -p '{"spec":{"template":{"spec":{"affinity":null}}}}'
              echo "fluent-bit daemonset patched successfully"
              # Also patch CSI node daemonset with client toleration
              if kubectl get daemonset simplyblock-csi-node -n $NAMESPACE &>/dev/null; then
                kubectl patch daemonset simplyblock-csi-node -n $NAMESPACE \
                  --type=merge -p '{"spec":{"template":{"spec":{"tolerations":[{"key":"node-role","operator":"Equal","value":"client","effect":"NoSchedule"}]}}}}'
                echo "csi-node daemonset patched with client toleration"
              fi
              break
            fi
            echo "fluent-bit daemonset not found yet ($i/30), waiting 10s..."
            sleep 10
          done

      # ── Run tests ───────────────────────────────────────────────────────────

      - name: Set RUN_DIR_FILE
        run: echo "RUN_DIR_FILE=/tmp/sb_k8s_run_dir_${GITHUB_RUN_ID}_${GITHUB_RUN_ATTEMPT}.txt" >> "$GITHUB_ENV"

      - name: Record Test Start Time
        run: echo "TEST_START_TIME=$(date +%s)" >> $GITHUB_ENV

      - name: Setup and Run Tests
        run: |
          set -o pipefail
          cd $GITHUB_WORKSPACE/e2e
          python3 -m pip install --upgrade pip uv
          export PATH="$(python3 -c 'import os, uv; print(os.path.dirname(uv.find_uv_bin()))'):$PATH"

          export CLUSTER_ID="${{ env.CLUSTER_ID }}"
          export CLUSTER_SECRET="${{ env.CLUSTER_SECRET }}"
          export K8S_LOCAL_KUBECTL=1
          export SBCLI_CMD=sbctl

          PRESERVE_ARG=""
          if [ "${{ inputs.preserve_resources_on_failure }}" = "true" ]; then
            PRESERVE_ARG="--preserve_resources_on_failure True"
          fi

          PYTHONUNBUFFERED=1 ./e2e.py \
            --testname "${{ inputs.testname || 'K8sNativeAddNodeTest' }}" \
            --new_worker_nodes "${{ inputs.new_worker_nodes }}" \
            ${PRESERVE_ARG} \
            --ndcs $NDCS --npcs $NPCS --bs $BS --chunk_bs $CHUNK_BS \
            --run_k8s True \
            --namespace simplyblock \
            --helm_chart_path "$GITHUB_WORKSPACE/simplyblock-operator/helm-charts/charts/simplyblock-operator" \
            2>&1 | tee output.log
        env:
          NDCS: ${{ env.NDCS }}
          NPCS: ${{ env.NPCS }}
          BS: ${{ env.BS }}
          CHUNK_BS: ${{ env.CHUNK_BS }}
          SUPABASE_ANON_KEY: ${{ secrets.SUPABASE_ANON_KEY }}
          SKIP_NFS: ${{ inputs.skip_nfs || 'false' }}
          RUN_DIR_FILE: ${{ env.RUN_DIR_FILE }}

      # ── Post-test: timing, logs, notifications ──────────────────────────────

      - name: Export RUN_BASE_DIR from RUN_DIR_FILE
        if: always()
        run: |
          test -f "${RUN_DIR_FILE}" || (echo "RUN_DIR_FILE not found — skipping"; exit 0)
          RUN_BASE_DIR="$(cat "${RUN_DIR_FILE}" | tr -d '\r\n')"
          [[ -n "${RUN_BASE_DIR}" ]] && echo "RUN_BASE_DIR=${RUN_BASE_DIR}" >> "$GITHUB_ENV" || true

      - name: Copy raw output.log to each test's NFS folder
        if: always()
        shell: bash
        run: |
          set +e
          out_log="$GITHUB_WORKSPACE/e2e/output.log"
          if [ ! -f "${out_log}" ]; then
            echo "No output.log found, skipping"
            exit 0
          fi
          echo "output.log size: $(stat --printf='%s' "${out_log}" 2>/dev/null || echo '?') bytes"

          mapfile -t log_paths < <(
            grep 'Logs Path:' "${out_log}" 2>/dev/null \
              | sed 's/\x1b\[[0-9;]*m//g' \
              | sed 's/.*Logs Path: *//' \
              | sed 's/[[:space:]]*$//' || true
          )

          if [ ${#log_paths[@]} -eq 0 ]; then
            echo "No 'Logs Path:' entries found in output.log"
            if [ -n "${RUN_BASE_DIR:-}" ] && [ -d "${RUN_BASE_DIR}" ]; then
              cp "${out_log}" "${RUN_BASE_DIR}/github_raw_output.log" || true
              echo "Saved to ${RUN_BASE_DIR}/github_raw_output.log"
            fi
            exit 0
          fi

          for path in "${log_paths[@]}"; do
            if [ -d "${path}" ]; then
              cp "${out_log}" "${path}/github_raw_output.log" || true
              echo "Saved output.log -> ${path}/github_raw_output.log"
            else
              echo "WARN: path does not exist: ${path}"
            fi
          done

      - name: Record Test End Time
        if: always()
        run: echo "TEST_END_TIME=$(date +%s)" >> $GITHUB_ENV

      - name: Calculate Total Time Taken
        if: always()
        run: |
          TEST_TIME=$(($TEST_END_TIME - $TEST_START_TIME))
          TEST_TIME_HOURS=$(($TEST_TIME / 3600))
          TEST_TIME_MINS=$((($TEST_TIME % 3600) / 60))
          TEST_TIME_SECS=$(($TEST_TIME % 60))
          echo "Test runtime: ${TEST_TIME_HOURS}h ${TEST_TIME_MINS}m ${TEST_TIME_SECS}s"
          echo "TEST_TIME_HOURS=$TEST_TIME_HOURS" >> $GITHUB_ENV
          echo "TEST_TIME_MINS=$TEST_TIME_MINS" >> $GITHUB_ENV
          echo "TEST_TIME_SECS=$TEST_TIME_SECS" >> $GITHUB_ENV

          # Log collection timeout: half the test runtime, minimum 30 minutes
          LOG_COLLECT_TIMEOUT_MINS=$(( (TEST_TIME + 119) / 120 ))
          [ "$LOG_COLLECT_TIMEOUT_MINS" -lt 30 ] && LOG_COLLECT_TIMEOUT_MINS=30
          echo "LOG_COLLECT_TIMEOUT_MINS=$LOG_COLLECT_TIMEOUT_MINS" >> $GITHUB_ENV

      - name: Write Job Summary
        if: always()
        shell: bash
        run: |
          set +e
          out_log="$GITHUB_WORKSPACE/e2e/output.log"

          dur_fmt="${TEST_TIME_HOURS:-0}h ${TEST_TIME_MINS:-0}m ${TEST_TIME_SECS:-0}s"

          # Extract test-specific log path from output.log
          TEST_LOG_PATH="$(grep 'Logs Path:' "${out_log}" 2>/dev/null \
            | tail -n 1 | sed 's/\x1b\[[0-9;]*m//g' | sed 's/.*Logs Path: *//' \
            | sed 's/[[:space:]]*$//' || true)"

          total_cases="$(grep -E 'Number of Total Cases:' "${out_log}" 2>/dev/null | tail -n 1 | grep -oE '[0-9]+$' || echo '?')"
          passed_cnt="$(grep -E 'Number of Passed Cases:' "${out_log}" 2>/dev/null | tail -n 1 | grep -oE '[0-9]+$' || echo '?')"
          failed_cnt="$(grep -E 'Number of Failed Cases:' "${out_log}" 2>/dev/null | tail -n 1 | grep -oE '[0-9]+$' || echo '?')"
          skipped_cnt="$(grep -E 'Number of Skipped Cases:' "${out_log}" 2>/dev/null | tail -n 1 | grep -oE '[0-9]+$' || echo '0')"
          test_wise="$(grep -E '(PASSED|FAILED|SKIPPED) CASE' "${out_log}" 2>/dev/null \
            | sed 's/\x1b\[[0-9;]*m//g' | sed 's/.*INFO - //' || true)"

          failure_summary="(unknown)"
          log_tail=""
          if [[ -f "${out_log}" ]]; then
            failure_summary="$(grep -oE 'MultipleExceptions: .+|SkippedTestsException: .+' "${out_log}" \
              | tail -n 1 | sed 's/\x1b\[[0-9;]*m//g' || true)"
            if [[ -z "${failure_summary}" ]]; then
              failure_summary="$(grep -E 'RuntimeError:|AssertionError:|Error:' "${out_log}" \
                | tail -n 1 | sed 's/\x1b\[[0-9;]*m//g' || true)"
            fi
            [[ -z "${failure_summary}" ]] && failure_summary="(no exception line found)"
            log_tail="$(tail -n 50 "${out_log}" | sed 's/\x1b\[[0-9;]*m//g' || true)"
          fi

          conclusion="SUCCESS"
          if [[ "${{ job.status }}" != "success" ]]; then
            conclusion="FAILED"
          fi

          {
            echo "## K8s Native Add Node Summary — ${TESTNAME}"
            echo ""
            echo "**Result:** ${conclusion}"
            echo ""
            echo "### Run Info"
            echo "- **Test class:** \`${TESTNAME}\`"
            echo "- **Cluster ID:** \`${CLUSTER_ID}\`"
            echo "- **Branch:** \`${{ github.ref_name }}\`"
            echo "- **Operator Repo Branch:** \`${{ inputs.operator_repo_branch }}\`"
            echo "- **NDCS/NPCS:** \`${NDCS}/${NPCS}\`"
            echo "- **BS/Chunk BS:** \`${BS}/${CHUNK_BS}\`"
            echo "- **Initial Workers:** \`${{ inputs.worker_nodes }}\`"
            echo "- **New Workers:** \`${{ inputs.new_worker_nodes }}\`"
            echo "- **Duration:** ${dur_fmt}"
            echo ""
            echo "### Test Results"
            echo "- **Total:** ${total_cases} | **Passed:** ${passed_cnt} | **Failed:** ${failed_cnt} | **Skipped:** ${skipped_cnt}"
            if [[ -n "${test_wise}" ]]; then
              echo '```'
              printf '%s\n' "${test_wise}"
              echo '```'
            fi
            echo ""
            echo "### Failure Reason"
            echo '```'
            printf '%s\n' "${failure_summary}"
            echo '```'
            if [[ -n "${log_tail}" ]]; then
              echo ""
              echo "<details><summary>Last 50 lines of output.log</summary>"
              echo ""
              echo '```'
              printf '%s\n' "${log_tail}"
              echo '```'
              echo "</details>"
            fi
            echo ""
            echo "### NFS Log Locations"
            echo "- **Test logs:** \`${TEST_LOG_PATH:-not detected}\`"
            echo "- **RUN_BASE_DIR:** \`${RUN_BASE_DIR:-not detected}\`"
            echo "- Graylog logs: \`${RUN_BASE_DIR:-<run_dir>}/graylog_collected/\`"
            if [[ -n "${RUN_BASE_DIR:-}" && -d "${RUN_BASE_DIR}" ]]; then
              echo ""
              echo "<details><summary>NFS directory listing</summary>"
              echo ""
              echo '```'
              ls -la "${RUN_BASE_DIR}/" 2>/dev/null || echo "(empty)"
              echo '```'
              echo "</details>"
            fi
            echo ""
            echo "> **Note:** Graylog/OpenSearch logs are currently being collected and will be available in the NFS log directory shortly."
          } >> "$GITHUB_STEP_SUMMARY"
        env:
          CLUSTER_ID: ${{ env.CLUSTER_ID }}
          NDCS: ${{ env.NDCS }}
          NPCS: ${{ env.NPCS }}
          BS: ${{ env.BS }}
          CHUNK_BS: ${{ env.CHUNK_BS }}
          RUN_BASE_DIR: ${{ env.RUN_BASE_DIR }}
          TEST_TIME_HOURS: ${{ env.TEST_TIME_HOURS }}
          TEST_TIME_MINS: ${{ env.TEST_TIME_MINS }}
          TEST_TIME_SECS: ${{ env.TEST_TIME_SECS }}

      - name: Save summary for aggregation
        if: always()
        shell: bash
        run: cp "$GITHUB_STEP_SUMMARY" "$RUNNER_TEMP/test-summary.md"

      - name: Upload summary artifact
        if: always()
        uses: actions/upload-artifact@v4
        with:
          name: test-summary-${{ inputs.testname || 'K8sNativeAddNodeTest' }}
          path: ${{ runner.temp }}/test-summary.md
          retention-days: 7

      - name: Send Slack Notification
        if: always() && inputs.send_slack_notification != false
        shell: bash
        env:
          SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }}
          SLACK_BOT_TOKEN: ${{ secrets.SLACK_BOT_TOKEN }}
          SLACK_CHANNEL_ID: ${{ secrets.SLACK_CHANNEL_ID }}
          JOB_STATUS: ${{ job.status }}
          SLACK_RUN_URL: "${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}"
          GITHUB_REF_NAME: ${{ github.ref_name }}
          SLACK_WF_NAME: "K8s Native Add Node"
          TESTNAME: ${{ inputs.testname || 'K8sNativeAddNodeTest' }}
          RUN_BASE_DIR: ${{ env.RUN_BASE_DIR }}
          NDCS: ${{ env.NDCS }}
          NPCS: ${{ env.NPCS }}
          TEST_START_TIME: ${{ env.TEST_START_TIME }}
          TEST_END_TIME: ${{ env.TEST_END_TIME }}
        run: |
          python3 - <<'PYEOF'
          import json, os, re, sys, urllib.request, urllib.error
          bot_token = os.environ.get("SLACK_BOT_TOKEN", "")
          channel   = os.environ.get("SLACK_CHANNEL_ID", "")
          webhook   = os.environ.get("SLACK_WEBHOOK_URL", "")
          if not bot_token and not webhook:
              print("No SLACK_BOT_TOKEN or SLACK_WEBHOOK_URL set, skipping.")
              sys.exit(0)
          out_log = os.path.join(os.environ.get("GITHUB_WORKSPACE", "."), "e2e", "output.log")
          total = passed = failed = skipped = 0
          if os.path.isfile(out_log):
              content = open(out_log).read()
              def px(pat):
                  m = re.search(pat, content)
                  return int(m.group(1)) if m else 0
              total   = px(r'Number of Total Cases:\s*(\d+)')
              passed  = px(r'Number of Passed Cases:\s*(\d+)')
              failed  = px(r'Number of Failed Cases:\s*(\d+)')
              skipped = px(r'Number of Skipped Cases:\s*(\d+)')
          pass_pct = (passed * 100 // total) if total > 0 else 0
          s    = int(os.environ.get("TEST_START_TIME", "0") or "0")
          e    = int(os.environ.get("TEST_END_TIME",   "0") or "0")
          secs = max(0, e - s) if e >= s > 0 else 0
          dur  = f"{secs//3600}h {(secs%3600)//60}m {secs%60}s"
          run_url  = os.environ.get("SLACK_RUN_URL",   "")
          log_dir  = os.environ.get("RUN_BASE_DIR",    "N/A")
          ndcs     = os.environ.get("NDCS",            "?")
          npcs     = os.environ.get("NPCS",            "?")
          test_cls = os.environ.get("TESTNAME",        "") or "all"
          branch   = os.environ.get("GITHUB_REF_NAME", "?")
          wf_name  = os.environ.get("SLACK_WF_NAME",   "Run")
          ok       = os.environ.get("JOB_STATUS",      "") == "success"
          icon    = ":white_check_mark:" if ok else ":x:"
          status  = "SUCCESS" if ok else "FAILURE"
          mention = "" if ok else " <!channel>"
          lines = [
              f"{icon} *SimplyBlock {wf_name}*{mention}",
              f"*Status:* {status}  |  *Duration:* {dur}",
              f"*Branch:* `{branch}`  |  *NDCS/NPCS:* `{ndcs}/{npcs}`  |  *Test class:* `{test_cls}`",
              "",
          ]
          if total > 0:
              lines += [
                  f":white_check_mark: *Passed:*  {passed}/{total}  ({pass_pct}%)",
                  f":x: *Failed:*  {failed}",
                  f":fast_forward: *Skipped:*  {skipped}",
              ]
          else:
              lines.append("_(test counts not found in log)_")
          # --- Per-test results ---
          test_log_paths = []
          if os.path.isfile(out_log):
              ansi_re = re.compile(r'\x1b\[[0-9;]*m')
              test_results = []
              for raw_line in content.splitlines():
                  cl = ansi_re.sub('', raw_line)
                  if 'Logs Path:' in cl:
                      p = cl.split('Logs Path:')[-1].strip()
                      if p:
                          test_log_paths.append(p)
                  if not re.search(r'(PASSED|FAILED|SKIPPED) CASE', cl):
                      continue
                  m = re.search(r'Test[A-Za-z0-9_]+', cl)
                  if not m:
                      continue
                  name = m.group(0)
                  if   'PASSED CASE'  in cl: test_results.append(('PASSED',  name))
                  elif 'FAILED CASE'  in cl: test_results.append(('FAILED',  name))
                  elif 'SKIPPED CASE' in cl: test_results.append(('SKIPPED', name))
              if test_results:
                  lines.append("")
                  lines.append("*Test Results:*")
                  icons = {'PASSED': ':white_check_mark:', 'FAILED': ':x:', 'SKIPPED': ':fast_forward:'}
                  for st, nm in test_results:
                      lines.append(f"{icons.get(st, ':grey_question:')} `{nm}`")

          lines += [""]
          if test_log_paths:
              for tlp in test_log_paths:
                  lines.append(f":open_file_folder: *Test Logs:* `{tlp}`")
          lines += [
              f":file_folder: *E2E Run Dir:* `{log_dir}`",
              f":scroll: *Graylog:* `{log_dir}/graylog_collected/`",
              f":link: *Run:* <{run_url}|View on GitHub>",
              f":hourglass_flowing_sand: _Logs are being collected and will be available shortly._",
          ]
          # --- Send ---
          gh_env = os.environ.get("GITHUB_ENV", "")
          text   = "\n".join(lines)

          if bot_token and channel:
              api_payload = json.dumps({"channel": channel, "text": text}).encode()
              req = urllib.request.Request(
                  "https://slack.com/api/chat.postMessage",
                  data=api_payload,
                  headers={
                      "Content-Type": "application/json; charset=utf-8",
                      "Authorization": f"Bearer {bot_token}",
                  },
              )
              try:
                  resp = urllib.request.urlopen(req, timeout=15)
                  body = json.loads(resp.read().decode())
                  if body.get("ok"):
                      ts = body.get("ts", "")
                      print(f"Slack notification sent via Bot API (ts={ts}).")
                      if gh_env and ts:
                          with open(gh_env, "a") as f:
                              f.write(f"SLACK_MSG_TS={ts}\n")
                  else:
                      print(f"WARN: Slack API error: {body.get('error')}", file=sys.stderr)
              except Exception as exc:
                  print(f"WARN: Slack Bot API failed: {exc}", file=sys.stderr)
          elif webhook:
              req = urllib.request.Request(
                  webhook,
                  data=json.dumps({"text": text}).encode(),
                  headers={"Content-Type": "application/json"},
              )
              try:
                  urllib.request.urlopen(req, timeout=15)
                  print("Slack notification sent via webhook (no thread support).")
              except Exception as exc:
                  print(f"WARN: Slack webhook failed: {exc}", file=sys.stderr)
          else:
              print("No SLACK_BOT_TOKEN or SLACK_WEBHOOK_URL set, skipping.")
          PYEOF


      - name: Collect Graylog/OpenSearch logs
        if: '!cancelled()'
        timeout-minutes: ${{ fromJSON(env.LOG_COLLECT_TIMEOUT_MINS || '480') }}
        run: |
          set +e
          echo "=== Collecting Graylog/OpenSearch logs (per-hour chunks) ==="
          NAMESPACE=simplyblock

          if [ -z "${TEST_START_TIME}" ] || [ -z "${TEST_END_TIME}" ]; then
            echo "WARN: TEST_START_TIME or TEST_END_TIME not set, skipping"
            exit 0
          fi

          ELAPSED=$((TEST_END_TIME - TEST_START_TIME))
          if [ "${ELAPSED}" -le 0 ]; then
            echo "WARN: Elapsed time is zero or negative, skipping"
            exit 0
          fi

          # Total window: 1h before test start → 1h after test end
          WINDOW_START=$((TEST_START_TIME - 3600))
          WINDOW_END=$((TEST_END_TIME + 3600))
          TOTAL_SECONDS=$((WINDOW_END - WINDOW_START))
          TOTAL_HOURS=$(( (TOTAL_SECONDS + 3599) / 3600 ))
          echo "  Total window: ${TOTAL_HOURS} hour(s) to collect"

          # Resolve admin pod
          ADMIN_POD=""
          for i in $(seq 1 12); do
            ADMIN_POD=$(kubectl -n ${NAMESPACE} get pods \
              -l app=simplyblock-admin-control \
              -o jsonpath='{.items[0].metadata.name}' 2>/dev/null) || true
            if [ -n "${ADMIN_POD}" ]; then
              PHASE=$(kubectl -n ${NAMESPACE} get pod "${ADMIN_POD}" \
                -o jsonpath='{.status.phase}' 2>/dev/null) || true
              [ "${PHASE}" = "Running" ] && break
              ADMIN_POD=""
            fi
            sleep 10
          done
          if [ -z "${ADMIN_POD}" ]; then
            echo "ERROR: No running admin pod found, skipping"
            exit 0
          fi
          echo "  Admin pod: ${ADMIN_POD}"

          # Deploy updated collect_logs.py to admin pod (use /tmp since package dir is read-only)
          BUILTIN_SCRIPT="/usr/local/lib/python3.12/site-packages/simplyblock_core/scripts/collect_logs.py"
          DEPLOYED_SCRIPT="/tmp/collect_logs.py"
          SCRIPT_SRC="${GITHUB_WORKSPACE}/scripts/collect_logs.py"
          if [ -f "${SCRIPT_SRC}" ]; then
            kubectl -n ${NAMESPACE} cp "${SCRIPT_SRC}" "${ADMIN_POD}:${DEPLOYED_SCRIPT}" 2>/dev/null || \
              echo "WARN: failed to deploy updated collect_logs.py, using built-in version"
          fi
          # Determine which script to use
          COLLECT_SCRIPT="${BUILTIN_SCRIPT}"
          if kubectl -n ${NAMESPACE} exec "${ADMIN_POD}" -- test -f "${DEPLOYED_SCRIPT}" 2>/dev/null; then
            COLLECT_SCRIPT="${DEPLOYED_SCRIPT}"
            echo "  Using deployed script: ${COLLECT_SCRIPT}"
          else
            echo "  Using built-in script: ${COLLECT_SCRIPT}"
          fi

          # Get Graylog service ClusterIP for mgmt-ip
          MGMT_IP=$(kubectl get svc -n ${NAMESPACE} | grep graylog | awk '{print $3}')
          echo "  Graylog IP: ${MGMT_IP}"
          OPENSEARCH_IP=$(kubectl get svc opensearch-cluster-master -n ${NAMESPACE} -o jsonpath='{.spec.clusterIP}' 2>/dev/null || echo "")
          echo "  OpenSearch IP: ${OPENSEARCH_IP}"

          # Determine output dir from RUN_BASE_DIR
          OUTPUT_DIR=""
          if [ -n "${RUN_BASE_DIR:-}" ] && [ -d "${RUN_BASE_DIR}" ]; then
            OUTPUT_DIR="${RUN_BASE_DIR}/graylog_collected"
          else
            NFS_BASE="${NFS_MOUNTPOINT:-/mnt/nfs_share}"
            TIMESTAMP=$(date -u "+%Y%m%d-%H%M%S")
            OUTPUT_DIR="${NFS_BASE}/graylog_collected-${TIMESTAMP}"
          fi
          mkdir -p "${OUTPUT_DIR}" 2>/dev/null || true
          echo "  Output: ${OUTPUT_DIR}"

          # Helper: convert epoch to ISO-8601
          epoch_to_iso() {
            local ep=$1
            local iso
            iso=$(date -u -d "@${ep}" "+%Y-%m-%dT%H:%M:%S" 2>/dev/null)
            if [ -z "${iso}" ]; then
              iso=$(python3 -c "from datetime import datetime,timezone; print(datetime.fromtimestamp(${ep},tz=timezone.utc).strftime('%Y-%m-%dT%H:%M:%S'))")
            fi
            echo "${iso}"
          }

          # Build chunk boundaries, then iterate in REVERSE order (newest first)
          # so the most critical logs (end of test) are captured first.
          _CHUNK_STARTS=()
          _C=${WINDOW_START}
          while [ ${_C} -lt ${WINDOW_END} ]; do
            _CHUNK_STARTS+=(${_C})
            _C=$((_C + 3600))
          done
          NUM_CHUNKS=${#_CHUNK_STARTS[@]}

          # Helper: run collect_logs.py with optional extra flags
          run_collect() {
            local iso=$1 mins=$2 outdir=$3 extra_flag=${4:-}
            local mgmt_ip_to_use="${MGMT_IP}"
            if [[ "${extra_flag}" == *"--use-opensearch"* ]] && [ -n "${OPENSEARCH_IP}" ]; then
              mgmt_ip_to_use="${OPENSEARCH_IP}"
            fi
            kubectl -n ${NAMESPACE} exec "${ADMIN_POD}" -- \
              python3 "${COLLECT_SCRIPT}" \
                "${iso}" "${mins}" \
                --mode kubernetes \
                --namespace "${NAMESPACE}" \
                --monitoring-secret "${MON_SECRET}" \
                --output-dir "${outdir}" \
                ${extra_flag} \
                ${mgmt_ip_to_use:+--mgmt-ip "${mgmt_ip_to_use}"} \
                ${CLUSTER_ID:+--cluster-id "${CLUSTER_ID}"} \
            2>&1
            local rc=$?
            [ $rc -ne 0 ] && return $rc
            local has_content
            has_content=$(kubectl -n ${NAMESPACE} exec "${ADMIN_POD}" -- \
              find "${outdir}" -name "*.tar.gz" -size +0 2>/dev/null | head -1)
            if [ -z "${has_content}" ]; then
              echo "  WARN: collect_logs.py succeeded but no .tar.gz output found"
              return 1
            fi
          }

          # Adaptive collection: try full window, on failure split into
          # 5-min sub-windows, then 1-min if those also fail.
          collect_adaptive() {
            local start_epoch=$1 end_epoch=$2 outdir=$3 extra_flag=${4:-}
            local duration=$((end_epoch - start_epoch))
            local mins=$(( (duration + 59) / 60 ))
            local iso=$(epoch_to_iso ${start_epoch})
            if run_collect "${iso}" "${mins}" "${outdir}" "${extra_flag}"; then
              return 0
            fi
            echo "  WARN: ${mins}m window failed, retrying with 5-min sub-windows..."
            local sub_start=${start_epoch}
            while [ ${sub_start} -lt ${end_epoch} ]; do
              local sub_end=$((sub_start + 300))
              [ ${sub_end} -gt ${end_epoch} ] && sub_end=${end_epoch}
              local sub_mins=$(( ((sub_end - sub_start) + 59) / 60 ))
              local sub_iso=$(epoch_to_iso ${sub_start})
              if ! run_collect "${sub_iso}" "${sub_mins}" "${outdir}" "${extra_flag}"; then
                echo "  WARN: 5-min window at ${sub_iso} failed, retrying with 1-min windows..."
                local micro_start=${sub_start}
                while [ ${micro_start} -lt ${sub_end} ]; do
                  local micro_end=$((micro_start + 60))
                  [ ${micro_end} -gt ${sub_end} ] && micro_end=${sub_end}
                  local micro_mins=$(( ((micro_end - micro_start) + 59) / 60 ))
                  local micro_iso=$(epoch_to_iso ${micro_start})
                  run_collect "${micro_iso}" "${micro_mins}" "${outdir}" "${extra_flag}" || \
                    echo "  WARN: 1-min window at ${micro_iso} also failed"
                  micro_start=${micro_end}
                done
              fi
              sub_start=${sub_end}
            done
          }

          # Auto-detect preferred log backend on the first (newest) chunk:
          # try OpenSearch; if it works use it for all chunks, else use Graylog.
          PREFER_OPENSEARCH=""
          CHUNK=0

          for (( _IDX=NUM_CHUNKS-1; _IDX>=0; _IDX-- )); do
            CHUNK=$((CHUNK + 1))
            CHUNK_START=${_CHUNK_STARTS[$_IDX]}
            CHUNK_END=$((CHUNK_START + 3600))
            if [ ${CHUNK_END} -gt ${WINDOW_END} ]; then
              CHUNK_END=${WINDOW_END}
            fi
            CHUNK_MINUTES=$(( ((CHUNK_END - CHUNK_START) + 59) / 60 ))
            CHUNK_ISO=$(epoch_to_iso ${CHUNK_START})

            echo ""
            echo "--- Chunk ${CHUNK}/${NUM_CHUNKS}: ${CHUNK_ISO} for ${CHUNK_MINUTES}m (newest-first) ---"

            POD_OUTPUT_DIR="/tmp/graylog_collect_chunk${CHUNK}"
            kubectl -n ${NAMESPACE} exec "${ADMIN_POD}" -- mkdir -p "${POD_OUTPUT_DIR}" 2>/dev/null || true

            if [ -z "${PREFER_OPENSEARCH}" ]; then
              # First chunk: probe OpenSearch availability
              echo "  Probing OpenSearch availability..."
              if run_collect "${CHUNK_ISO}" "${CHUNK_MINUTES}" "${POD_OUTPUT_DIR}" "--use-opensearch"; then
                PREFER_OPENSEARCH=true
                echo "  OpenSearch works — using it for all chunks"
              else
                PREFER_OPENSEARCH=false
                echo "  OpenSearch unavailable — using Graylog for all chunks"
                kubectl -n ${NAMESPACE} exec "${ADMIN_POD}" -- rm -rf "${POD_OUTPUT_DIR}" 2>/dev/null || true
                kubectl -n ${NAMESPACE} exec "${ADMIN_POD}" -- mkdir -p "${POD_OUTPUT_DIR}" 2>/dev/null || true
                collect_adaptive ${CHUNK_START} ${CHUNK_END} "${POD_OUTPUT_DIR}" || \
                  echo "WARN: Graylog also failed for chunk ${CHUNK}"
              fi
            elif [ "${PREFER_OPENSEARCH}" = "true" ]; then
              collect_adaptive ${CHUNK_START} ${CHUNK_END} "${POD_OUTPUT_DIR}" "--use-opensearch" || {
                echo "WARN: OpenSearch failed for chunk ${CHUNK}, falling back to Graylog..."
                collect_adaptive ${CHUNK_START} ${CHUNK_END} "${POD_OUTPUT_DIR}" || \
                  echo "WARN: Graylog fallback also failed for chunk ${CHUNK}"
              }
            else
              collect_adaptive ${CHUNK_START} ${CHUNK_END} "${POD_OUTPUT_DIR}" || {
                echo "WARN: Graylog failed for chunk ${CHUNK}, falling back to OpenSearch..."
                collect_adaptive ${CHUNK_START} ${CHUNK_END} "${POD_OUTPUT_DIR}" "--use-opensearch" || \
                  echo "WARN: OpenSearch fallback also failed for chunk ${CHUNK}"
              }
            fi

            # Copy tarballs from pod to NFS for this chunk
            TARBALLS=$(kubectl -n ${NAMESPACE} exec "${ADMIN_POD}" -- \
              find "${POD_OUTPUT_DIR}" -name "*.tar.gz" -type f 2>/dev/null) || true
            if [ -n "${TARBALLS}" ]; then
              for TB in ${TARBALLS}; do
                TB_NAME=$(basename "${TB}")
                echo "  Copying ${TB_NAME}..."
                kubectl -n ${NAMESPACE} cp "${ADMIN_POD}:${TB}" "${OUTPUT_DIR}/${TB_NAME}" 2>&1 || true
              done
              for TB_FILE in "${OUTPUT_DIR}"/*.tar.gz; do
                [ -f "${TB_FILE}" ] && tar -xzf "${TB_FILE}" -C "${OUTPUT_DIR}/" 2>/dev/null || true
              done
            else
              echo "WARN: No tarballs found for chunk ${CHUNK}"
            fi

            # Cleanup this chunk in pod
            kubectl -n ${NAMESPACE} exec "${ADMIN_POD}" -- rm -rf "${POD_OUTPUT_DIR}" 2>/dev/null || true
          done

          echo ""
          echo "=== Log collection complete (${CHUNK} chunks, newest-first): ${OUTPUT_DIR} ==="
          ls -la "${OUTPUT_DIR}/" 2>/dev/null || true
        env:
          CLUSTER_ID: ${{ env.CLUSTER_ID }}
          MON_SECRET: ${{ secrets.MON_SECRET }}

      - name: Notify Slack log collection complete
        if: always() && inputs.send_slack_notification != false
        shell: bash
        env:
          SLACK_BOT_TOKEN: ${{ secrets.SLACK_BOT_TOKEN }}
          SLACK_CHANNEL_ID: ${{ secrets.SLACK_CHANNEL_ID }}
          SLACK_MSG_TS: ${{ env.SLACK_MSG_TS }}
          SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }}
          SLACK_RUN_URL: "${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}"
          SLACK_WF_NAME: "K8s Native Add Node"
          RUN_BASE_DIR: ${{ env.RUN_BASE_DIR }}
        run: |
          MSG=":white_check_mark: *Log collection complete* for *SimplyBlock ${SLACK_WF_NAME}*\n:file_folder: \`${RUN_BASE_DIR:-N/A}/graylog_collected/\`\n:link: <${SLACK_RUN_URL}|View Run>"
          if [ -n "${SLACK_BOT_TOKEN:-}" ] && [ -n "${SLACK_CHANNEL_ID:-}" ] && [ -n "${SLACK_MSG_TS:-}" ]; then
            curl -s -X POST "https://slack.com/api/chat.postMessage" \
              -H "Authorization: Bearer ${SLACK_BOT_TOKEN}" \
              -H "Content-Type: application/json; charset=utf-8" \
              -d "{\"channel\":\"${SLACK_CHANNEL_ID}\",\"thread_ts\":\"${SLACK_MSG_TS}\",\"text\":\"${MSG}\"}" || true
          elif [ -n "${SLACK_WEBHOOK_URL:-}" ]; then
            curl -s -X POST "${SLACK_WEBHOOK_URL}" \
              -H "Content-Type: application/json" \
              -d "{\"text\":\"${MSG}\"}" || true
          fi

      - name: Stop MinIO trace logging and save
        if: always()
        run: |
          set +e
          if [ -n "${MC_TRACE_PID:-}" ]; then
            echo "Stopping MinIO trace (PID=${MC_TRACE_PID})..."
            kill "${MC_TRACE_PID}" 2>/dev/null || true
            wait "${MC_TRACE_PID}" 2>/dev/null || true
          fi
          if [ -n "${MC_PORT_FWD_PID:-}" ]; then
            kill "${MC_PORT_FWD_PID}" 2>/dev/null || true
          fi
          if [ -f "${MINIO_TRACE_LOG:-}" ]; then
            LINES=$(wc -l < "${MINIO_TRACE_LOG}")
            SIZE=$(du -h "${MINIO_TRACE_LOG}" | cut -f1)
            echo "=== MinIO trace log: ${LINES} lines, ${SIZE} ==="
            cp "${MINIO_TRACE_LOG}" e2e/minio-trace.log 2>/dev/null || true
          else
            echo "No MinIO trace log found (backup tests may not have run)"
          fi
          # Cleanup: only our own per-run trace log + mc binary — never a
          # glob/pkill that could hit another parallel job's files or
          # processes on this shared VM.
          rm -f "${MINIO_TRACE_LOG:-}" 2>/dev/null || true
          rm -f "${MC_BIN:-}" 2>/dev/null || true

      - name: Upload MinIO trace log
        if: always()
        uses: actions/upload-artifact@v4
        with:
          name: minio-trace-${{ github.run_id }}
          path: e2e/minio-trace.log
          if-no-files-found: ignore

      - name: Cleanup MinIO backup bucket on success
        if: success()
        run: |
          set +e
          export PATH="/tmp:$PATH"
          CLUSTER_ID="${CLUSTER_ID:-}"
          if [ -z "$CLUSTER_ID" ]; then
            echo "No CLUSTER_ID set, skipping MinIO bucket cleanup"
            exit 0
          fi
          BUCKET="simplyblock-backup-${CLUSTER_ID}"
          if [ "${MINIO_SOURCE:-external}" = "in-cluster" ]; then
            MC_ENDPOINT="http://localhost:9000"
          else
            MC_ENDPOINT="http://192.168.10.164:9000"
          fi
          mc alias set myminio "${MC_ENDPOINT}" minioadmin minioadmin 2>/dev/null || true
          if mc ls "myminio/${BUCKET}" >/dev/null 2>&1; then
            echo "=== Cleaning up MinIO backup bucket: ${BUCKET} ==="
            mc rb --force "myminio/${BUCKET}" 2>/dev/null || true
            echo "=== Bucket cleanup complete ==="
          else
            echo "Bucket ${BUCKET} does not exist, nothing to clean"
          fi

      - name: Cleanup MinIO namespace (if deployed)
        if: always()
        run: kubectl delete namespace minio --wait=false 2>/dev/null || true

      - name: Cleanup build folder
        if: always()
        run: |
          rm -rf ./* || true
          rm -rf ./.??* || true
          rm -f "$HOME/.kube/config-${{ github.run_id }}"