e2e-docker.yml

name: End-to-end test docker
run-name: "E2E Docker | ${{ github.ref_name }} | ${{ inputs.cluster || 'c141' }}${{ inputs.test_case_to_run && format(' | {0}', inputs.test_case_to_run) || '' }}"

on:
  push:
    branches:
      - main
      - R*-PRE
      - R*-Hotfix

  workflow_dispatch:
    inputs:
      cluster:
        default: c141
        description: "Cluster to deploy on"
        required: true
        type: string
      ndcs:
        description: 'Value for NDCS'
        required: false
        default: 1
      npcs:
        description: 'Value for NPCS'
        required: false
        default: 1
      bs:
        description: 'Value for Block size'
        required: false
        default: 4096
      chunk_bs:
        description: 'Value for Chunk block size'
        required: false
        default: 4096
      send_slack_notification:
        description: 'Send Slack notification?'
        required: false
        default: true
        type: boolean
      run_ha_test:
        description: 'Run HA tests or not'
        required: false
        default: true
        type: boolean
      test_case_to_run:
        description: 'Specific test case to run (leave empty to run all)'
        required: false
        type: string
      new_node_ips:
        description: 'Spare node IPs for add-node/migration tests (space-separated)'
        required: false
        type: string

concurrency:
  group: ${{ github.workflow }}
  cancel-in-progress: false

jobs:
  cleanup:
    runs-on: self-hosted
    steps:
      - name: Fix workspace permissions
        run: sudo chown -R "$(id -u):$(id -g)" "$GITHUB_WORKSPACE" 2>/dev/null || true

      - name: Checkout deployment tooling
        uses: actions/checkout@v4
        with:
          repository: simplyblock-io/simplyBlockDeploy
          path: deploy

      - name: cleanup cluster
        timeout-minutes: 15
        run: |
          cd deploy/bare-metal
          cluster="${{ inputs.cluster || 'c141' }}"
          cluster="${cluster:1}"
          echo "cleaning up cluster $cluster"
          KEY="~/.ssh/simplyblock-us-east-2.pem"
          eval $(python3 inventory.py inventory/c${cluster}.yml)
          ssh -i $KEY -o StrictHostKeyChecking=no root@192.168.10.1 "bash proxmox_script.sh cluster_${cluster}"

  deploy:
    needs: cleanup
    uses: ./.github/workflows/bare-metal-deploy.yml
    with:
      runs_on: self-hosted
      cluster: ${{ inputs.cluster || 'c141' }}
      docker_image: simplyblock/simplyblock:${{ github.head_ref || github.ref_name }}
      sbcli_source: ${{ github.head_ref || github.ref_name }}
      ndcs: ${{ github.event.inputs.ndcs }}
      npcs: ${{ github.event.inputs.npcs }}
      bs: ${{ github.event.inputs.bs }}
      chunk_bs: ${{ github.event.inputs.chunk_bs }}
      k8s_snode: false

  e2e:
    runs-on: self-hosted
    needs: deploy
    steps:
      - name: Fix workspace permissions
        run: sudo chown -R "$(id -u):$(id -g)" "$GITHUB_WORKSPACE" 2>/dev/null || true

      - name: Checkout code
        uses: actions/checkout@v4

      - name: Set send_slack_notification for scheduled and push events
        if: github.event_name == 'schedule' || github.event_name == 'push'
        run: echo "send_slack_notification=true" >> $GITHUB_ENV

      - name: Set send_slack_notification for manual workflow_dispatch
        if: github.event_name == 'workflow_dispatch'
        run: echo "send_slack_notification=${{ github.event.inputs.send_slack_notification }}" >> $GITHUB_ENV

      - name: Record Test Start Time
        run: echo "TEST_START_TIME=$(date +%s)" >> $GITHUB_ENV

      - name: Setup Tests & Run Tests
        timeout-minutes: 120
        run: |
          cd $GITHUB_WORKSPACE/e2e
          python3 -m pip install --upgrade pip uv
          export PATH="$(python3 -c 'import os, uv; print(os.path.dirname(uv.find_uv_bin()))'):$PATH"
          echo "Running tests in namespace default"
          export CLUSTER_ID=${{ needs.deploy.outputs.cluster_id }}
          export CLUSTER_SECRET=${{ needs.deploy.outputs.cluster_secret }}
          export CLUSTER_IP=${{ needs.deploy.outputs.cluster_ip }}
          export API_BASE_URL=${{ needs.deploy.outputs.cluster_ip }}
          export KEY_NAME="simplyblock-us-east-2.pem" 
          export AWS_ACCESS_KEY_ID=${{ secrets.AWS_ACCESS_KEY_ID }}
          export AWS_SECRET_ACCESS_KEY=${{ secrets.AWS_SECRET_ACCESS_KEY }}
          export AWS_REGION=${{ secrets.AWS_REGION }}
          export SSH_USER=root
          export SUPABASE_ANON_KEY=${{ secrets.SUPABASE_ANON_KEY }}
          TEST_CASE=${{ inputs.test_case_to_run || '' }}
          NEW_NODE_IPS="${{ github.event.inputs.new_node_ips || '' }}"
          PYTHONUNBUFFERED=1 ./e2e.py ${TEST_CASE:+--testname} ${TEST_CASE} \
                  ${NEW_NODE_IPS:+--new_nodes} ${NEW_NODE_IPS} \
                  --ndcs $NDCS --npcs $NPCS --bs $BS --chunk_bs $CHUNK_BS --run_ha ${{ github.event.inputs.run_ha_test || true }}
        env:
          NDCS: ${{ github.event.inputs.ndcs || 1 }}
          NPCS: ${{ github.event.inputs.npcs || 1 }}
          BS: ${{ github.event.inputs.bs || 4096 }}
          CHUNK_BS: ${{ github.event.inputs.chunk_bs || 4096 }}
          SBCLI_CMD: "sbctl"

      - name: Record Test End Time
        if: always()
        run: echo "TEST_END_TIME=$(date +%s)" >> $GITHUB_ENV

      - name: Calculate Total Time Taken
        if: always()
        run: |
          TEST_TIME=$(($TEST_END_TIME - $TEST_START_TIME))
          TEST_TIME_HOURS=$(($TEST_TIME / 3600))
          TEST_TIME_MINS=$((($TEST_TIME % 3600) / 60))
          TEST_TIME_SECS=$(($TEST_TIME % 60))
          echo "Test runtime: ${TEST_TIME_HOURS}h ${TEST_TIME_MINS}m ${TEST_TIME_SECS}s"
          echo "TEST_TIME_HOURS=$TEST_TIME_HOURS" >> $GITHUB_ENV
          echo "TEST_TIME_MINS=$TEST_TIME_MINS" >> $GITHUB_ENV
          echo "TEST_TIME_SECS=$TEST_TIME_SECS" >> $GITHUB_ENV

      - name: Parse test results
        if: always() && (github.event_name == 'schedule' || env.send_slack_notification == 'true')
        id: parse_results
        run: |
          cd $GITHUB_WORKSPACE/e2e/logs
          echo "Looking for the latest non-empty log file..."
          # Find the latest non-empty log file
          COUNTER=0
          MAX_ATTEMPTS=10
          while [ $COUNTER -lt $MAX_ATTEMPTS ]; do
            LATEST_LOG=$(ls -t *.log | head -n 1)
            if [ -s "$LATEST_LOG" ]; then
              echo "Found non-empty log file: $LATEST_LOG"
              break
            fi
            echo "Attempt $((COUNTER + 1)): No non-empty log file found. Retrying..."
            COUNTER=$((COUNTER + 1))
            sleep 1  # Add a small delay to avoid rapid looping
          done
          if [ ! -s "$LATEST_LOG" ]; then
            echo "No non-empty log file found after $MAX_ATTEMPTS attempts"
            exit 1
          fi
          echo "Parsing the identified log file: $LATEST_LOG"
          # Parse the identified log file
          echo "Total tests"
          TOTAL_TESTS=$(grep -i "Number of Total Cases" "$LATEST_LOG" | awk '{print $NF}')
          echo "number Passed tests"
          PASSED_TESTS=$(grep -i "Number of Passed Cases" "$LATEST_LOG" | awk '{print $NF}')
          echo "number Failed tests"
          FAILED_TESTS=$(grep -i "Number of Failed Cases" "$LATEST_LOG" | awk '{print $NF}')
          echo "number Skipped tests"
          SKIPPED_TESTS=$(grep -i "Number of Skipped Cases" "$LATEST_LOG" | awk '{print $NF}')
          echo "List Passed tests"
          PASSED_CASES=$(grep "PASSED CASE" "$LATEST_LOG" | awk -F 'INFO - | FAILED CASE' '{print $2}')
          echo "List Failed tests"
          FAILED_CASES=$(grep "FAILED CASE" "$LATEST_LOG" | awk -F 'INFO - | SKIPPED CASE' '{print $2}')
          echo "List Skipped tests"
          SKIPPED_CASES=$(grep "SKIPPED CASE" "$LATEST_LOG" | awk -F 'INFO - | SKIPPED CASE' '{print $2}')
          # Format passed and failed cases as bullet points
          echo "Adding PASSED cases with bullets: $PASSED_CASES"
          echo "Adding FAILED cases with bullets: $FAILED_CASES"
          echo "Adding SKIPPED cases with bullets: $SKIPPED_CASES"
          PASSED_CASES_BULLETS=$(echo "$PASSED_CASES" | awk '{printf "    • %s\n", $0}')
          FAILED_CASES_BULLETS=$(echo "$FAILED_CASES" | awk '{printf "    • %s\n", $0}')
          SKIPPED_CASES_BULLETS=$(echo "$SKIPPED_CASES" | awk '{printf "    • %s\n", $0}')
          echo "PASSED cases with bullets: $PASSED_CASES_BULLETS"
          echo "FAILED cases with bullets: $FAILED_CASES_BULLETS"
          echo "SKIPPED cases with bullets: $SKIPPED_CASES_BULLETS"
          echo "TOTAL_TESTS=${TOTAL_TESTS}"
          echo "PASSED_TESTS=${PASSED_TESTS}"
          echo "FAILED_TESTS=${FAILED_TESTS}"
          echo "SKIPPED_TESTS=${SKIPPED_TESTS}"
          echo "PASSED_CASES=${PASSED_CASES}"
          echo "FAILED_CASES=${FAILED_CASES}"
          echo "SKIPPED_CASES=${SKIPPED_CASES}"
          echo "PASSED_TESTS=${PASSED_TESTS}" >> $GITHUB_ENV
          echo "FAILED_TESTS=${FAILED_TESTS}" >> $GITHUB_ENV
          echo "SKIPPED_TESTS=${SKIPPED_TESTS}" >> $GITHUB_ENV
          echo "TOTAL_TESTS=${TOTAL_TESTS}" >> $GITHUB_ENV
          echo "PASSED_CASES<<EOF" >> $GITHUB_ENV
          echo "${PASSED_CASES}" >> $GITHUB_ENV
          echo "EOF" >> $GITHUB_ENV
          echo "FAILED_CASES<<EOF" >> $GITHUB_ENV
          echo "${FAILED_CASES}" >> $GITHUB_ENV
          echo "EOF" >> $GITHUB_ENV
          echo "SKIPPED_CASES<<EOF" >> $GITHUB_ENV
          echo "${SKIPPED_CASES}" >> $GITHUB_ENV
          echo "EOF" >> $GITHUB_ENV
          echo "PASSED_CASES_BULLETS<<EOF" >> $GITHUB_ENV
          echo "${PASSED_CASES_BULLETS}" >> $GITHUB_ENV
          echo "EOF" >> $GITHUB_ENV
          echo "FAILED_CASES_BULLETS<<EOF" >> $GITHUB_ENV
          echo "${FAILED_CASES_BULLETS}" >> $GITHUB_ENV
          echo "EOF" >> $GITHUB_ENV
          echo "SKIPPED_CASES_BULLETS<<EOF" >> $GITHUB_ENV
          echo "${SKIPPED_CASES_BULLETS}" >> $GITHUB_ENV
          echo "EOF" >> $GITHUB_ENV

      - name: Write Job Summary
        if: always()
        shell: bash
        run: |
          set -euxo pipefail

          dur_fmt="${TEST_TIME_HOURS:-0}h ${TEST_TIME_MINS:-0}m ${TEST_TIME_SECS:-0}s"

          # --- Parse test counts ---
          total_cases="${TOTAL_TESTS:-0}"
          passed_cases="${PASSED_TESTS:-0}"
          failed_cases="${FAILED_TESTS:-0}"
          skipped_cases="${SKIPPED_TESTS:-0}"

          pass_pct=0; fail_pct=0; skip_pct=0
          if [[ "${total_cases}" =~ ^[0-9]+$ && "${total_cases}" -gt 0 ]]; then
            pass_pct=$(( (passed_cases  * 100) / total_cases ))
            fail_pct=$(( (failed_cases  * 100) / total_cases ))
            skip_pct=$(( (skipped_cases * 100) / total_cases ))
          fi

          # --- Parse per-test status from log ---
          test_details_table=""
          log_file=""
          for f in $GITHUB_WORKSPACE/e2e/logs/*.log; do
            [ -s "$f" ] && log_file="$f"
          done
          if [[ -n "${log_file}" ]]; then
            while IFS= read -r line; do
              clean="$(printf '%s' "${line}" | sed 's/\x1b\[[0-9;]*m//g')"
              test_name="$(printf '%s' "${clean}" | grep -oE 'Test[A-Za-z0-9_]+' | head -n1 || true)"
              [[ -z "${test_name}" ]] && continue
              if   printf '%s' "${clean}" | grep -qi 'PASSED CASE';  then icon="✅"; status="PASSED"
              elif printf '%s' "${clean}" | grep -qi 'FAILED CASE';  then icon="❌"; status="FAILED"
              elif printf '%s' "${clean}" | grep -qi 'SKIPPED CASE'; then icon="⏭"; status="SKIPPED"
              else continue
              fi
              test_details_table+="| \`${test_name}\` | ${icon} ${status} |"$'\n'
            done < <(grep -iE 'PASSED CASE|FAILED CASE|SKIPPED CASE' "${log_file}" 2>/dev/null || true)
          fi

          # --- Failure reason ---
          failure_reason=""
          if [[ -n "${log_file}" ]]; then
            multi="$(grep 'MultipleExceptions:' "${log_file}" | sed 's/\x1b\[[0-9;]*m//g' || true)"
            if [[ -n "${multi}" ]]; then
              failure_reason="${multi}"
            elif grep -Eqi 'Traceback \(most recent call last\)|Exception:|AssertionError|Input/output error' "${log_file}"; then
              failure_reason="$(grep -Ei 'Traceback \(most recent call last\)|Exception:|AssertionError|Input/output error' "${log_file}" | tail -n 3 | sed 's/\x1b\[[0-9;]*m//g' || true)"
            fi
          fi

          # --- Overall result ---
          conclusion="✅ SUCCESS"
          if [[ "${{ job.status }}" != "success" ]]; then
            conclusion="❌ FAILED"
          fi

          {
            echo "## SimplyBlock E2E Docker Run Summary"
            echo ""
            echo "**Result:** ${conclusion} &nbsp;|&nbsp; **Duration:** ${dur_fmt}"
            echo ""

            echo "### Test Results"
            echo "| | Count | % |"
            echo "|---|---|---|"
            echo "| ✅ Passed  | ${passed_cases}  | ${pass_pct}% |"
            echo "| ❌ Failed  | ${failed_cases}  | ${fail_pct}% |"
            echo "| ⏭ Skipped | ${skipped_cases} | ${skip_pct}% |"
            echo "| **Total**  | **${total_cases}** | |"
            echo ""

            if [[ -n "${test_details_table}" ]]; then
              echo "### Test Case Details"
              echo "| Test | Result |"
              echo "|---|---|"
              printf '%s' "${test_details_table}"
              echo ""
            fi

            echo "### Run Info"
            echo "- **Test class:** \`${TEST_CASE:-all}\`"
            echo "- **Branch:** \`${{ github.ref_name }}\`"
            echo "- **Cluster:** \`${CLUSTER:-?}\`"
            echo "- **NDCS/NPCS:** \`${NDCS}/${NPCS}\`"
            echo "- **BS/Chunk BS:** \`${BS}/${CHUNK_BS}\`"
            echo ""

            if [[ -n "${failure_reason}" ]]; then
              echo "### Failure Reason"
              echo '```'
              printf '%s\n' "${failure_reason}"
              echo '```'
              echo ""
            fi

            echo "> **Note:** Graylog/OpenSearch logs are currently being collected and will be available shortly."
          } >> "$GITHUB_STEP_SUMMARY"
        env:
          NDCS: ${{ github.event.inputs.ndcs || 1 }}
          NPCS: ${{ github.event.inputs.npcs || 1 }}
          BS: ${{ github.event.inputs.bs || 4096 }}
          CHUNK_BS: ${{ github.event.inputs.chunk_bs || 4096 }}
          TEST_CASE: ${{ inputs.test_case_to_run || '' }}
          CLUSTER: ${{ inputs.cluster || 'c141' }}

      - name: Send Slack Notification
        if: always() && (github.event_name == 'schedule' || env.send_slack_notification == 'true')
        shell: bash
        env:
          SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }}
          SLACK_BOT_TOKEN: ${{ secrets.SLACK_BOT_TOKEN }}
          SLACK_CHANNEL_ID: ${{ secrets.SLACK_CHANNEL_ID }}
          JOB_STATUS: ${{ job.status }}
          SLACK_RUN_URL: "${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}"
          GITHUB_REF_NAME: ${{ github.ref_name }}
          SLACK_WF_NAME: "E2E Docker"
          TESTNAME: ${{ inputs.test_case_to_run || 'all' }}
          NDCS: ${{ github.event.inputs.ndcs || 1 }}
          NPCS: ${{ github.event.inputs.npcs || 1 }}
          BS: ${{ github.event.inputs.bs || 4096 }}
          CHUNK_BS: ${{ github.event.inputs.chunk_bs || 4096 }}
          TEST_START_TIME: ${{ env.TEST_START_TIME }}
          TEST_END_TIME: ${{ env.TEST_END_TIME }}
        run: |
          python3 - <<'PYEOF'
          import json, os, re, sys, urllib.request, urllib.error

          bot_token = os.environ.get("SLACK_BOT_TOKEN", "")
          channel   = os.environ.get("SLACK_CHANNEL_ID", "")
          webhook   = os.environ.get("SLACK_WEBHOOK_URL", "")
          if not bot_token and not webhook:
              print("No SLACK_BOT_TOKEN or SLACK_WEBHOOK_URL set, skipping.")
              sys.exit(0)

          # Find latest log file
          import glob
          log_dir = os.path.join(os.environ.get("GITHUB_WORKSPACE", "."), "e2e", "logs")
          log_files = sorted(glob.glob(os.path.join(log_dir, "*.log")), key=os.path.getmtime, reverse=True)
          content = ""
          for lf in log_files:
              if os.path.getsize(lf) > 0:
                  content = open(lf).read()
                  break

          # --- Counts (e2e.py format) ---
          def px(pat):
              m = re.search(pat, content)
              return int(m.group(1)) if m else 0
          total   = px(r'Number of Total Cases:\s*(\d+)')
          passed  = px(r'Number of Passed Cases:\s*(\d+)')
          failed  = px(r'Number of Failed Cases:\s*(\d+)')
          skipped = px(r'Number of Skipped Cases:\s*(\d+)')
          pass_pct = (passed * 100 // total) if total > 0 else 0

          # --- Per-test results ---
          ansi = re.compile(r'\x1b\[[0-9;]*m')
          test_results = []
          for line in content.splitlines():
              clean = ansi.sub('', line)
              if not re.search(r'(PASSED|FAILED|SKIPPED) CASE', clean):
                  continue
              m = re.search(r'Test[A-Za-z0-9_]+', clean)
              if not m:
                  continue
              name = m.group(0)
              if   'PASSED CASE'  in clean: test_results.append(('PASSED',  name))
              elif 'FAILED CASE'  in clean: test_results.append(('FAILED',  name))
              elif 'SKIPPED CASE' in clean: test_results.append(('SKIPPED', name))

          # --- Failure reason ---
          failure_reason = ""
          multi = [ansi.sub('', l) for l in content.splitlines() if 'MultipleExceptions:' in l]
          if multi:
              failure_reason = multi[0][:2000]
          elif content:
              exc_lines = [ansi.sub('', l) for l in content.splitlines()
                           if re.search(r'(Exception:|AssertionError|Input/output error)', l)]
              if exc_lines:
                  failure_reason = '\n'.join(exc_lines[-5:])[:2000]

          # --- Env ---
          s    = int(os.environ.get("TEST_START_TIME", "0") or "0")
          e    = int(os.environ.get("TEST_END_TIME",   "0") or "0")
          secs = max(0, e - s) if e >= s > 0 else 0
          dur  = f"{secs//3600}h {(secs%3600)//60}m {secs%60}s"
          run_url  = os.environ.get("SLACK_RUN_URL",   "")
          ndcs     = os.environ.get("NDCS",            "?")
          npcs     = os.environ.get("NPCS",            "?")
          bs       = os.environ.get("BS",              "?")
          chunk_bs = os.environ.get("CHUNK_BS",        "?")
          test_cls = os.environ.get("TESTNAME",        "") or "all"
          branch   = os.environ.get("GITHUB_REF_NAME", "?")
          wf_name  = os.environ.get("SLACK_WF_NAME",   "Run")
          ok       = os.environ.get("JOB_STATUS",      "") == "success"

          icon    = ":white_check_mark:" if ok else ":x:"
          status  = "SUCCESS" if ok else "FAILURE"
          mention = "" if ok else " <!channel>"

          lines = [
              f"{icon} *SimplyBlock {wf_name}*{mention}",
              f"*Status:* {status}  |  *Duration:* {dur}",
              f"*Branch:* `{branch}`  |  *Test class:* `{test_cls}`",
              f"*Config:* NDCS={ndcs} NPCS={npcs} BS={bs} ChunkBS={chunk_bs}",
              "",
          ]

          if total > 0:
              lines += [
                  f":white_check_mark: *Passed:*  {passed}/{total}  ({pass_pct}%)",
                  f":x: *Failed:*  {failed}",
                  f":fast_forward: *Skipped:*  {skipped}",
              ]
          else:
              lines.append("_(test counts not found in log)_")

          if test_results:
              lines.append("")
              lines.append("*Test Results:*")
              icons = {'PASSED': ':white_check_mark:', 'FAILED': ':x:', 'SKIPPED': ':fast_forward:'}
              for st, nm in test_results:
                  lines.append(f"{icons.get(st, ':grey_question:')} `{nm}`")

          if failure_reason:
              lines += ["", "*Failure:*", f"```{failure_reason}```"]

          lines += [
              "",
              f":link: *Run:* <{run_url}|View on GitHub>",
              f":hourglass_flowing_sand: _Logs are being collected and will be available shortly._",
          ]

          # --- Send ---
          gh_env = os.environ.get("GITHUB_ENV", "")
          text   = "\n".join(lines)

          if bot_token and channel:
              api_payload = json.dumps({"channel": channel, "text": text}).encode()
              req = urllib.request.Request(
                  "https://slack.com/api/chat.postMessage",
                  data=api_payload,
                  headers={
                      "Content-Type": "application/json; charset=utf-8",
                      "Authorization": f"Bearer {bot_token}",
                  },
              )
              try:
                  resp = urllib.request.urlopen(req, timeout=15)
                  body = json.loads(resp.read().decode())
                  if body.get("ok"):
                      ts = body.get("ts", "")
                      print(f"Slack notification sent via Bot API (ts={ts}).")
                      if gh_env and ts:
                          with open(gh_env, "a") as f:
                              f.write(f"SLACK_MSG_TS={ts}\n")
                  else:
                      print(f"WARN: Slack API error: {body.get('error')}", file=sys.stderr)
              except Exception as exc:
                  print(f"WARN: Slack Bot API failed: {exc}", file=sys.stderr)
          elif webhook:
              req = urllib.request.Request(
                  webhook,
                  data=json.dumps({"text": text}).encode(),
                  headers={"Content-Type": "application/json"},
              )
              try:
                  urllib.request.urlopen(req, timeout=15)
                  print("Slack notification sent via webhook (no thread support).")
              except Exception as exc:
                  print(f"WARN: Slack webhook failed: {exc}", file=sys.stderr)
          else:
              print("No SLACK_BOT_TOKEN or SLACK_WEBHOOK_URL set, skipping.")
          PYEOF



      - name: Analyze SPDK core dumps (gdb backtraces)
        if: always()
        continue-on-error: true
        timeout-minutes: 30
        shell: bash
        env:
          STORAGE_PRIVATE_IPS: "${{ needs.deploy.outputs.storage_private_ips }}"
        run: |
          set -uo pipefail
          # For every SPDK core left on a storage host, copy it into that host's
          # SPDK container (same host = same image = symbols resolve), run
          # `bt` and `thread apply all bt full`, and pull all three artifacts
          # back. Script and rationale: e2e/scripts/analyze_core_dumps_docker.sh
          #
          # Unlike the stress workflows, e2e writes its logs under the
          # workspace rather than to an NFS run dir that the storage hosts can
          # see, so the script stages on the host and we scp the results back.
          CORE_SCRIPT="${GITHUB_WORKSPACE}/e2e/scripts/analyze_core_dumps_docker.sh"
          if [[ ! -f "${CORE_SCRIPT}" ]]; then
            echo "WARN: ${CORE_SCRIPT} not found; skipping core-dump analysis"
            exit 0
          fi
          if [[ -z "${STORAGE_PRIVATE_IPS:-}" ]]; then
            echo "WARN: STORAGE_PRIVATE_IPS empty; skipping core-dump analysis"
            exit 0
          fi
          KEY_PATH="${HOME}/.ssh/simplyblock-us-east-2.pem"
          SSH_OPTS=(-i "${KEY_PATH}" -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o ConnectTimeout=15)
          LOCAL_OUT="${GITHUB_WORKSPACE}/e2e/logs/core_backtraces"
          STAGE="/tmp/core_bt_${GITHUB_RUN_ID}"
          mkdir -p "${LOCAL_OUT}"
          for ip in $(echo "${STORAGE_PRIVATE_IPS}" | tr ' ' '\n' | sed '/^$/d' | sort -u); do
            echo "---- core-dump analysis: ${ip} ----"
            ssh "${SSH_OPTS[@]}" "root@${ip}" 'bash -s' -- "${STAGE}" < "${CORE_SCRIPT}" || { echo "WARN: analysis failed on ${ip}"; continue; }
            if ssh "${SSH_OPTS[@]}" "root@${ip}" "test -d ${STAGE}/core_backtraces" 2>/dev/null; then
              scp "${SSH_OPTS[@]}" -r "root@${ip}:${STAGE}/core_backtraces/." "${LOCAL_OUT}/" || echo "WARN: scp back failed for ${ip}"
              ssh "${SSH_OPTS[@]}" "root@${ip}" "sudo rm -rf ${STAGE}" || true
            fi
          done
          find "${LOCAL_OUT}" -mindepth 1 -maxdepth 2 2>/dev/null | head -20 || true

      - name: Collect Graylog/OpenSearch logs
        if: '!cancelled()'
        timeout-minutes: 480
        env:
          MNODES: "${{ needs.deploy.outputs.mnodes }}"
          CLUSTER_ID: "${{ needs.deploy.outputs.cluster_id }}"
        run: |
          set +e
          [ -z "${TEST_START_TIME:-}" ] || [ -z "${TEST_END_TIME:-}" ] && exit 0
          ELAPSED=$((TEST_END_TIME - TEST_START_TIME))
          [ "${ELAPSED}" -le 0 ] && exit 0

          WINDOW_START=$((TEST_START_TIME - 3600))
          WINDOW_END=$((TEST_END_TIME + 3600))

          MGMT_IP="$(echo "${MNODES}" | awk '{print $1}')"
          KEY_PATH="${HOME}/.ssh/simplyblock-us-east-2.pem"
          SSH_OPTS=(-i "${KEY_PATH}" -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o ConnectTimeout=10)

          OUTPUT_DIR="$GITHUB_WORKSPACE/e2e/logs/graylog_collected"
          mkdir -p "${OUTPUT_DIR}" 2>/dev/null || true

          epoch_to_iso() {
            python3 -c "from datetime import datetime,timezone; print(datetime.fromtimestamp($1,tz=timezone.utc).strftime('%Y-%m-%dT%H:%M:%S'))"
          }

          # Build chunk boundaries, then iterate in REVERSE order (newest first)
          _CHUNK_STARTS=()
          _C=${WINDOW_START}
          while [ ${_C} -lt ${WINDOW_END} ]; do
            _CHUNK_STARTS+=(${_C})
            _C=$((_C + 3600))
          done
          NUM_CHUNKS=${#_CHUNK_STARTS[@]}

          run_collect() {
            local iso=$1 mins=$2 outdir=$3 extra_flag=${4:-}
            ssh "${SSH_OPTS[@]}" "root@${MGMT_IP}" \
              "python3 -m simplyblock_core.scripts.collect_logs \
                '${iso}' '${mins}' \
                --mode docker \
                --output-dir '${outdir}' \
                ${extra_flag} \
                ${CLUSTER_ID:+--cluster-id '${CLUSTER_ID}'}" \
            2>&1
            local rc=$?
            [ $rc -ne 0 ] && return $rc
            local has_content
            has_content=$(ssh "${SSH_OPTS[@]}" "root@${MGMT_IP}" \
              "find '${outdir}' -name '*.tar.gz' -size +0 2>/dev/null | head -1")
            if [ -z "${has_content}" ]; then
              echo "  WARN: collect_logs.py succeeded but no .tar.gz output found"
              return 1
            fi
          }

          collect_adaptive() {
            local start_epoch=$1 end_epoch=$2 outdir=$3 extra_flag=${4:-}
            local duration=$((end_epoch - start_epoch))
            local mins=$(( (duration + 59) / 60 ))
            local iso=$(epoch_to_iso ${start_epoch})
            if run_collect "${iso}" "${mins}" "${outdir}" "${extra_flag}"; then
              return 0
            fi
            echo "  WARN: ${mins}m window failed, retrying with 5-min sub-windows..."
            local sub_start=${start_epoch}
            while [ ${sub_start} -lt ${end_epoch} ]; do
              local sub_end=$((sub_start + 300))
              [ ${sub_end} -gt ${end_epoch} ] && sub_end=${end_epoch}
              local sub_mins=$(( ((sub_end - sub_start) + 59) / 60 ))
              local sub_iso=$(epoch_to_iso ${sub_start})
              if ! run_collect "${sub_iso}" "${sub_mins}" "${outdir}" "${extra_flag}"; then
                echo "  WARN: 5-min window at ${sub_iso} failed, retrying with 1-min windows..."
                local micro_start=${sub_start}
                while [ ${micro_start} -lt ${sub_end} ]; do
                  local micro_end=$((micro_start + 60))
                  [ ${micro_end} -gt ${sub_end} ] && micro_end=${sub_end}
                  local micro_mins=$(( ((micro_end - micro_start) + 59) / 60 ))
                  local micro_iso=$(epoch_to_iso ${micro_start})
                  run_collect "${micro_iso}" "${micro_mins}" "${outdir}" "${extra_flag}" || \
                    echo "  WARN: 1-min window at ${micro_iso} also failed"
                  micro_start=${micro_end}
                done
              fi
              sub_start=${sub_end}
            done
          }

          PREFER_OPENSEARCH=""
          CHUNK=0
          for (( _IDX=NUM_CHUNKS-1; _IDX>=0; _IDX-- )); do
            CHUNK=$((CHUNK + 1))
            CHUNK_START=${_CHUNK_STARTS[$_IDX]}
            CHUNK_END=$((CHUNK_START + 3600))
            [ ${CHUNK_END} -gt ${WINDOW_END} ] && CHUNK_END=${WINDOW_END}
            CHUNK_MINUTES=$(( ((CHUNK_END - CHUNK_START) + 59) / 60 ))
            CHUNK_ISO=$(epoch_to_iso ${CHUNK_START})
            echo "--- Chunk ${CHUNK}/${NUM_CHUNKS}: ${CHUNK_ISO} for ${CHUNK_MINUTES}m (newest-first) ---"
            REMOTE_OUTPUT_DIR="/tmp/graylog_collect_chunk${CHUNK}"
            ssh "${SSH_OPTS[@]}" "root@${MGMT_IP}" "mkdir -p '${REMOTE_OUTPUT_DIR}'" 2>/dev/null || true
            if [ -z "${PREFER_OPENSEARCH}" ]; then
              echo "  Probing OpenSearch availability..."
              if run_collect "${CHUNK_ISO}" "${CHUNK_MINUTES}" "${REMOTE_OUTPUT_DIR}" "--use-opensearch"; then
                PREFER_OPENSEARCH=true
                echo "  OpenSearch works — using it for all chunks"
              else
                PREFER_OPENSEARCH=false
                echo "  OpenSearch unavailable — using Graylog for all chunks"
                ssh "${SSH_OPTS[@]}" "root@${MGMT_IP}" "rm -rf '${REMOTE_OUTPUT_DIR}'" 2>/dev/null || true
                ssh "${SSH_OPTS[@]}" "root@${MGMT_IP}" "mkdir -p '${REMOTE_OUTPUT_DIR}'" 2>/dev/null || true
                collect_adaptive ${CHUNK_START} ${CHUNK_END} "${REMOTE_OUTPUT_DIR}" || \
                  echo "WARN: Graylog also failed for chunk ${CHUNK}"
              fi
            elif [ "${PREFER_OPENSEARCH}" = "true" ]; then
              collect_adaptive ${CHUNK_START} ${CHUNK_END} "${REMOTE_OUTPUT_DIR}" "--use-opensearch" || {
                echo "WARN: OpenSearch failed, falling back to Graylog..."
                collect_adaptive ${CHUNK_START} ${CHUNK_END} "${REMOTE_OUTPUT_DIR}" || true
              }
            else
              collect_adaptive ${CHUNK_START} ${CHUNK_END} "${REMOTE_OUTPUT_DIR}" || {
                echo "WARN: Graylog failed, falling back to OpenSearch..."
                collect_adaptive ${CHUNK_START} ${CHUNK_END} "${REMOTE_OUTPUT_DIR}" "--use-opensearch" || true
              }
            fi
            TARBALLS=$(ssh "${SSH_OPTS[@]}" "root@${MGMT_IP}" \
              "find '${REMOTE_OUTPUT_DIR}' -name '*.tar.gz' -type f 2>/dev/null") || true
            if [ -n "${TARBALLS}" ]; then
              for TB in ${TARBALLS}; do
                scp "${SSH_OPTS[@]}" "root@${MGMT_IP}:${TB}" "${OUTPUT_DIR}/$(basename ${TB})" 2>&1 || true
              done
              for TB_FILE in "${OUTPUT_DIR}"/*.tar.gz; do
                [ -f "${TB_FILE}" ] && tar -xzf "${TB_FILE}" -C "${OUTPUT_DIR}/" 2>/dev/null || true
              done
            fi
            ssh "${SSH_OPTS[@]}" "root@${MGMT_IP}" "rm -rf '${REMOTE_OUTPUT_DIR}'" 2>/dev/null || true
          done
          echo "=== Log collection complete (${CHUNK} chunks, newest-first): ${OUTPUT_DIR} ==="

      - name: Notify Slack log collection complete
        if: always() && (github.event_name == 'schedule' || env.send_slack_notification == 'true')
        shell: bash
        env:
          SLACK_BOT_TOKEN: ${{ secrets.SLACK_BOT_TOKEN }}
          SLACK_CHANNEL_ID: ${{ secrets.SLACK_CHANNEL_ID }}
          SLACK_MSG_TS: ${{ env.SLACK_MSG_TS }}
          SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }}
          SLACK_RUN_URL: "${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}"
        run: |
          MSG=":white_check_mark: *Log collection complete* for *SimplyBlock E2E Docker*\n:link: <${SLACK_RUN_URL}|View Run>"
          if [ -n "${SLACK_BOT_TOKEN:-}" ] && [ -n "${SLACK_CHANNEL_ID:-}" ] && [ -n "${SLACK_MSG_TS:-}" ]; then
            curl -s -X POST "https://slack.com/api/chat.postMessage" \
              -H "Authorization: Bearer ${SLACK_BOT_TOKEN}" \
              -H "Content-Type: application/json; charset=utf-8" \
              -d "{\"channel\":\"${SLACK_CHANNEL_ID}\",\"thread_ts\":\"${SLACK_MSG_TS}\",\"text\":\"${MSG}\"}" || true
          elif [ -n "${SLACK_WEBHOOK_URL:-}" ]; then
            curl -s -X POST "${SLACK_WEBHOOK_URL}" \
              -H "Content-Type: application/json" \
              -d "{\"text\":\"${MSG}\"}" || true
          fi

      - name: Upload automation and docker logs to miniio
        run: |
          cd $GITHUB_WORKSPACE/e2e/
          ./logs/upload_logs_to_miniio.py
        if: always()
        env:
          GITHUB_RUN_ID: ${{ github.run_id }}
          MINIO_ACCESS_KEY: ${{ secrets.MINIO_ACCESS_KEY }}
          MINIO_SECRET_KEY: ${{ secrets.MINIO_SECRET_KEY }}
          MNODES: "${{ needs.deploy.outputs.mnodes }}"
          STORAGE_PRIVATE_IPS: "${{ needs.deploy.outputs.storage_private_ips }}"
          USER: "root"
      - name: 'Cleanup build folder'
        run: |
          ls -la ./
          rm -rf ./* || true
          rm -rf ./.??* || true
          ls -la ./