From 976fceb21ff3911665209ff93e67a6e10594ab70 Mon Sep 17 00:00:00 2001 From: Kasturi Narra Date: Thu, 2 Jul 2026 14:13:55 +0530 Subject: [PATCH 01/14] Add z-stream testing support to CI doctor pipeline - Add zstream-trigger.sh for detecting and triggering z-stream e2e jobs - Integrate z-stream into doctor prepare/finalize using unified release-* naming - Render z-stream sections in report with patch version labels (e.g., 4.19.3) - Add Z-Stream checkbox to toggle between regular and z-stream views - Make report component-agnostic via COMPONENT_REPOS mapping - GCS-based state persistence for CI step-registry trigger script Co-Authored-By: Claude Opus 4.6 --- .claude-plugin/marketplace.json | 2 +- plugins/lvms-ci/.claude-plugin/plugin.json | 2 +- plugins/lvms-ci/scripts/zstream-trigger.sh | 483 +++++++++++++++++++++ plugins/lvms-ci/skills/doctor/SKILL.md | 26 +- plugins/shared/scripts/create-report.py | 160 ++++++- plugins/shared/scripts/doctor.sh | 111 ++++- 6 files changed, 751 insertions(+), 33 deletions(-) create mode 100755 plugins/lvms-ci/scripts/zstream-trigger.sh diff --git a/.claude-plugin/marketplace.json b/.claude-plugin/marketplace.json index 11deda42..315e184e 100644 --- a/.claude-plugin/marketplace.json +++ b/.claude-plugin/marketplace.json @@ -58,7 +58,7 @@ "name": "lvms-ci", "source": "./plugins/lvms-ci", "description": "LVMS CI Automation", - "version": "1.1.1" + "version": "1.2.0" }, { "name": "mcp-atlassian", diff --git a/plugins/lvms-ci/.claude-plugin/plugin.json b/plugins/lvms-ci/.claude-plugin/plugin.json index bb14f33c..ab37f590 100644 --- a/plugins/lvms-ci/.claude-plugin/plugin.json +++ b/plugins/lvms-ci/.claude-plugin/plugin.json @@ -1,7 +1,7 @@ { "name": "lvms-ci", "description": "LVMS CI Automation", - "version": "1.1.1", + "version": "1.2.0", "author": { "name": "knarra" }, diff --git a/plugins/lvms-ci/scripts/zstream-trigger.sh b/plugins/lvms-ci/scripts/zstream-trigger.sh new file mode 100755 index 00000000..45e1f71f --- /dev/null +++ b/plugins/lvms-ci/scripts/zstream-trigger.sh @@ -0,0 +1,483 @@ +#!/usr/bin/bash +set -euo pipefail + +# Trigger LVMS z-stream e2e tests via Gangway API. +# +# For each configured release, checks GitHub for open z-stream PRs on +# openshift/lvm-operator, resolves the catalog image digest from the PR +# diff via the quay.io API, and triggers existing nightly e2e jobs with +# MULTISTAGE_PARAM_OVERRIDE_LVM_INDEX_IMAGE set to the resolved digest. +# +# Skips releases with no open PR or where the digest hasn't changed since +# the last run (tracked via a state file). +# +# Usage: +# zstream-trigger.sh [--releases 4.14,4.16,...] [--dry-run] [--workdir /path] +# [--force] [--token-file /path/to/token] +# +# Environment: +# GITHUB_TOKEN Optional. GitHub PAT for higher API rate limits. +# MY_APPCI_TOKEN Required (unless --dry-run). Gangway API bearer token. + +RELEASES="4.14,4.16,4.18,4.19,4.20,4.21" +DRY_RUN=false +FORCE=false +WORKDIR="" +TOKEN_FILE="" + +GANGWAY_API="https://gangway-ci.apps.ci.l2s4.p1.openshiftapps.com" +GITHUB_API="https://api.github.com" +QUAY_API="https://quay.io/api/v1" +QUAY_REPO="redhat-user-workloads/logical-volume-manag-tenant/lvm-operator-catalog" +STATE_DIR="${HOME}/.cache/lvms-zstream" +STATE_FILE="${STATE_DIR}/last-tested.json" + +NIGHTLY_JOBS=( + "e2e-aws-sno-qe-integration-tests" + "e2e-aws-sno-arm-qe-integration-tests" + "e2e-aws-mno-qe-integration-tests" + "e2e-aws-mno-arm-qe-integration-tests" + "e2e-baremetalds-sno-dualstack-qe-integration-tests" + "e2e-baremetalds-mno-dualstack-qe-integration-tests" +) + +# --------------------------------------------------------------------------- +# Helpers +# --------------------------------------------------------------------------- + +log() { echo "[$(date +%H:%M:%S)] $*" >&2; } +info() { log "INFO $*"; } +warn() { log "WARN $*"; } +err() { log "ERROR $*"; } + +github_curl() { + local -a headers=() + if [[ -n "${GITHUB_TOKEN:-}" ]]; then + headers+=(-H "Authorization: token ${GITHUB_TOKEN}") + fi + curl -sSL --connect-timeout 10 --max-time 30 "${headers[@]}" "$@" +} + +usage() { + cat >&2 <) + --token-file F Read MY_APPCI_TOKEN from file instead of environment + -h, --help Show this help +EOF + exit 1 +} + +# --------------------------------------------------------------------------- +# Parse arguments +# --------------------------------------------------------------------------- + +while [[ ${#} -gt 0 ]]; do + case "${1}" in + --releases) [[ ${#} -ge 2 ]] || { err "--releases requires an argument"; usage; } + RELEASES="${2}"; shift 2 ;; + --dry-run) DRY_RUN=true; shift ;; + --force) FORCE=true; shift ;; + --workdir) [[ ${#} -ge 2 ]] || { err "--workdir requires an argument"; usage; } + WORKDIR="${2}"; shift 2 ;; + --token-file) [[ ${#} -ge 2 ]] || { err "--token-file requires an argument"; usage; } + TOKEN_FILE="${2}"; shift 2 ;; + -h|--help) usage ;; + *) err "Unknown option: ${1}"; usage ;; + esac +done + +WORKDIR="${WORKDIR:-/tmp/lvms-zstream.$(date +%y%m%d)}" +mkdir -p "${WORKDIR}" + +if [[ -n "${TOKEN_FILE}" && -f "${TOKEN_FILE}" ]]; then + MY_APPCI_TOKEN=$(cat "${TOKEN_FILE}") +fi + +if ! ${DRY_RUN} && [[ -z "${MY_APPCI_TOKEN:-}" ]]; then + err "MY_APPCI_TOKEN is required (set env or use --token-file)" + exit 1 +fi + +# --------------------------------------------------------------------------- +# State management — track last-tested digest per release +# --------------------------------------------------------------------------- + +load_state() { + mkdir -p "${STATE_DIR}" + if [[ ! -f "${STATE_FILE}" ]]; then + echo '{}' > "${STATE_FILE}" + fi +} + +get_last_digest() { + local release="$1" + jq -r --arg r "${release}" '.[$r].digest // ""' "${STATE_FILE}" +} + +get_last_pr() { + local release="$1" + jq -r --arg r "${release}" '.[$r].pr // ""' "${STATE_FILE}" +} + +save_digest() { + local release="$1" digest="$2" pr_number="$3" image="$4" + local tmp + tmp=$(mktemp) + jq --arg r "${release}" \ + --arg d "${digest}" \ + --arg p "${pr_number}" \ + --arg i "${image}" \ + --arg t "$(date -Iseconds)" \ + '.[$r] = {digest: $d, pr: $p, image: $i, tested_at: $t}' \ + "${STATE_FILE}" > "${tmp}" && mv "${tmp}" "${STATE_FILE}" +} + +# --------------------------------------------------------------------------- +# GitHub: find open z-stream PR for a release +# --------------------------------------------------------------------------- + +find_zstream_pr() { + local release="$1" + local major minor + major=$(echo "${release}" | cut -d. -f1) + minor=$(echo "${release}" | cut -d. -f2) + + local prs_json + prs_json=$(github_curl "${GITHUB_API}/repos/openshift/lvm-operator/pulls?state=open&per_page=100") + + echo "${prs_json}" | jq -r --arg maj "${major}" --arg min "${minor}" ' + [.[] | select(.title | test(": Release " + $maj + "\\." + $min + "\\."; "i"))] | + if length > 0 then .[0] else empty end | + [.number, .title] | @tsv + ' +} + +# --------------------------------------------------------------------------- +# GitHub: check if a previously tracked PR was merged +# --------------------------------------------------------------------------- + +check_pr_merged() { + local pr_number="$1" + local pr_json + pr_json=$(github_curl "${GITHUB_API}/repos/openshift/lvm-operator/pulls/${pr_number}") + + local state merged_at + state=$(echo "${pr_json}" | jq -r '.state // ""') + merged_at=$(echo "${pr_json}" | jq -r '.merged_at // ""') + + if [[ "${state}" == "closed" && -n "${merged_at}" && "${merged_at}" != "null" ]]; then + echo "merged" + fi +} + +# --------------------------------------------------------------------------- +# GitHub: extract snapshot name from PR diff +# --------------------------------------------------------------------------- + +extract_snapshot() { + local pr_number="$1" + + local files_json + files_json=$(github_curl "${GITHUB_API}/repos/openshift/lvm-operator/pulls/${pr_number}/files") + + echo "${files_json}" | jq -r ' + [.[] | select(.filename | contains("catalog")) | .patch // ""] | + join("\n") + ' | { grep -oP '(?<=snapshot: )\S+' || true; } | head -1 +} + +# --------------------------------------------------------------------------- +# Quay: resolve snapshot to digest +# --------------------------------------------------------------------------- + +resolve_snapshot_to_digest() { + local snapshot="$1" + + # Parse version prefix and timestamp from snapshot name + # Format: lvm-operator-catalog-4-16-20260624-033025-000 + local version_prefix date_str time_str + version_prefix=$(echo "${snapshot}" | grep -oP 'lvm-operator-catalog-\d+-\d+') + date_str=$(echo "${snapshot}" | grep -oP '\d{8}(?=-\d{6}-)') + time_str=$(echo "${snapshot}" | grep -oP '(?<=\d{8}-)\d{6}') + + if [[ -z "${version_prefix}" || -z "${date_str}" || -z "${time_str}" ]]; then + err "Cannot parse snapshot name: ${snapshot}" + return 1 + fi + + # Convert snapshot timestamp to epoch for comparison + local snap_year snap_month snap_day snap_hour snap_min snap_sec snap_epoch + snap_year=${date_str:0:4} + snap_month=${date_str:4:2} + snap_day=${date_str:6:2} + snap_hour=${time_str:0:2} + snap_min=${time_str:2:2} + snap_sec=${time_str:4:2} + snap_epoch=$(date -d "${snap_year}-${snap_month}-${snap_day}T${snap_hour}:${snap_min}:${snap_sec}Z" +%s 2>/dev/null || echo 0) + + # Query quay for on-push build-image-index tags matching this version + local tags_json + tags_json=$(curl -sSL --connect-timeout 10 --max-time 30 \ + "${QUAY_API}/repository/${QUAY_REPO}/tag/?limit=50&filter_tag_name=like:${version_prefix}-on-push") + + # Find build-image-index tags created after the snapshot timestamp + # Pick the closest one (smallest positive time delta) + local result + result=$(echo "${tags_json}" | jq -r --arg snap_epoch "${snap_epoch}" ' + [.tags[] + | select(.name | endswith("build-image-index")) + | .tag_epoch = (.last_modified | strptime("%a, %d %b %Y %H:%M:%S %z") | mktime) + | .delta = (.tag_epoch - ($snap_epoch | tonumber)) + | select(.delta >= 0 and .delta < 1800) + ] | sort_by(.delta) | .[0] // empty | + [.name, .manifest_digest] | @tsv + ') + + if [[ -z "${result}" ]]; then + err "No matching quay tag found for snapshot ${snapshot}" + return 1 + fi + + local tag_name digest + tag_name=$(echo "${result}" | cut -f1) + digest=$(echo "${result}" | cut -f2) + + info "Resolved: ${snapshot} → ${tag_name} (${digest})" + echo "${digest}" +} + +# --------------------------------------------------------------------------- +# Gangway: trigger a nightly job with image override +# --------------------------------------------------------------------------- + +trigger_job() { + local job_name="$1" image="$2" + + local body + body=$(jq -cn \ + --arg img "${image}" \ + '{job_execution_type: "1", pod_spec_options: {envs: {MULTISTAGE_PARAM_OVERRIDE_LVM_INDEX_IMAGE: $img}}}') + + if ${DRY_RUN}; then + info "[dry-run] would trigger: ${job_name}" + info "[dry-run] image: ${image}" + return 0 + fi + + local http_code + http_code=$(curl -sSL -X POST -o /dev/stderr -w '%{http_code}' \ + --connect-timeout 10 --max-time 30 \ + -H "Authorization: Bearer ${MY_APPCI_TOKEN}" \ + -H "Content-Type: application/json" \ + -d "${body}" \ + "${GANGWAY_API}/v1/executions/${job_name}" 2>/dev/null) + + if [[ "${http_code}" == "200" ]]; then + info " triggered: ${job_name}" + return 0 + else + warn " failed (HTTP ${http_code}): ${job_name}" + return 1 + fi +} + +# --------------------------------------------------------------------------- +# Prow: fetch latest run results for nightly jobs +# --------------------------------------------------------------------------- + +fetch_last_runs() { + local release="$1" + local job_prefix="periodic-ci-openshift-lvm-operator-release-${release}-nightly-" + local results="[]" + + for test_name in "${NIGHTLY_JOBS[@]}"; do + local full_job="${job_prefix}${test_name}" + local prow_json + prow_json=$(curl -sSL --connect-timeout 10 --max-time 30 \ + "https://prow.ci.openshift.org/prowjobs.js?omit=annotations,labels,decoration_config,pod_spec&job=${full_job}" 2>/dev/null || true) + + if [[ -z "${prow_json}" ]]; then + results=$(echo "${results}" | jq --arg n "${test_name}" '. + [{name: $n, state: "unknown"}]') + continue + fi + + local entry + entry=$(echo "${prow_json}" | sed 's/^var allBuilds = //' | \ + jq -r --arg n "${test_name}" ' + [.items[] | select(.status.state == "success" or .status.state == "failure" or .status.state == "error" or .status.state == "aborted")] | + if length > 0 then .[0] else null end | + if . then {name: $n, state: .status.state, url: .status.url, started: .status.startTime} else {name: $n, state: "unknown"} end + ' 2>/dev/null || echo "{\"name\": \"${test_name}\", \"state\": \"unknown\"}") + + results=$(echo "${results}" | jq --argjson e "${entry}" '. + [$e]') + done + + echo "${results}" +} + +# --------------------------------------------------------------------------- +# Process a single release +# --------------------------------------------------------------------------- + +process_release() { + local release="$1" + local major minor + major=$(echo "${release}" | cut -d. -f1) + minor=$(echo "${release}" | cut -d. -f2) + + info "--- Release ${release} ---" + + # Find open z-stream PR + local pr_info + pr_info=$(find_zstream_pr "${release}" || true) + + if [[ -z "${pr_info}" ]]; then + # Check if a previously tracked PR was merged + local last_pr + last_pr=$(get_last_pr "${release}") + if [[ -n "${last_pr}" ]]; then + local merge_status + merge_status=$(check_pr_merged "${last_pr}" || true) + if [[ "${merge_status}" == "merged" ]]; then + info "PR #${last_pr} was merged for ${release}." + jq -n --arg r "${release}" --arg pr "${last_pr}" \ + '{($r): {status: "completed", reason: "release completed", pr: ("#" + $pr)}}' \ + > "${WORKDIR}/zstream-${release}.json" + return 0 + fi + fi + + info "No z-stream release PR found for ${release}." + jq -n --arg r "${release}" \ + '{($r): {status: "no_pr", reason: "no new releases"}}' \ + > "${WORKDIR}/zstream-${release}.json" + return 0 + fi + + local pr_number pr_title + pr_number=$(echo "${pr_info}" | cut -f1) + pr_title=$(echo "${pr_info}" | cut -f2) + info "Found PR #${pr_number}: ${pr_title}" + + # Extract snapshot from PR + local snapshot + snapshot=$(extract_snapshot "${pr_number}") + + if [[ -z "${snapshot}" ]]; then + warn "Could not extract catalog snapshot from PR #${pr_number}" + jq -n --arg r "${release}" --arg pr "${pr_number}" \ + '{($r): {status: "error", reason: "no snapshot in PR", pr: ("#" + $pr)}}' \ + > "${WORKDIR}/zstream-${release}.json" + return 0 + fi + + info "Snapshot: ${snapshot}" + + # Resolve to digest + local digest + digest=$(resolve_snapshot_to_digest "${snapshot}" || true) + + if [[ -z "${digest}" ]]; then + warn "Could not resolve snapshot to digest for ${release}" + jq -n --arg r "${release}" --arg pr "${pr_number}" --arg snap "${snapshot}" \ + '{($r): {status: "error", reason: "digest resolution failed", pr: ("#" + $pr), snapshot: $snap}}' \ + > "${WORKDIR}/zstream-${release}.json" + return 0 + fi + + local image="quay.io/${QUAY_REPO}@${digest}" + + # Check if already tested + if ! ${FORCE}; then + local last_digest + last_digest=$(get_last_digest "${release}") + if [[ "${last_digest}" == "${digest}" ]]; then + info "Already tested with same digest. Fetching last run results." + local last_runs + last_runs=$(fetch_last_runs "${release}") + jq -n --arg r "${release}" --arg pr "${pr_number}" --arg title "${pr_title}" --arg img "${image}" --arg snap "${snapshot}" \ + --argjson jobs "${last_runs}" \ + '{($r): {status: "skipped", reason: "same digest", pr: ("#" + $pr), pr_title: $title, image: $img, snapshot: $snap, jobs: $jobs}}' \ + > "${WORKDIR}/zstream-${release}.json" + return 0 + fi + fi + + # Trigger nightly jobs + local job_prefix="periodic-ci-openshift-lvm-operator-release-${release}-nightly-" + local triggered=0 failed=0 + local triggered_jobs=() + + for test_name in "${NIGHTLY_JOBS[@]}"; do + local full_job="${job_prefix}${test_name}" + if trigger_job "${full_job}" "${image}"; then + triggered=$((triggered + 1)) + triggered_jobs+=("${full_job}") + else + failed=$((failed + 1)) + fi + ${DRY_RUN} || sleep 5 + done + + info "${release}: ${triggered} triggered, ${failed} failed" + + # Update state + if ! ${DRY_RUN} && [[ ${triggered} -gt 0 && ${failed} -eq 0 ]]; then + save_digest "${release}" "${digest}" "${pr_number}" "${image}" + fi + + # Write per-release summary + jq -n \ + --arg r "${release}" \ + --arg pr "${pr_number}" \ + --arg title "${pr_title}" \ + --arg img "${image}" \ + --arg snap "${snapshot}" \ + --argjson triggered "${triggered}" \ + --argjson failed "${failed}" \ + --argjson jobs "$(printf '%s\n' "${triggered_jobs[@]}" | jq -R . | jq -s .)" \ + '{($r): {status: "triggered", pr: ("#" + $pr), pr_title: $title, image: $img, snapshot: $snap, jobs_triggered: $triggered, jobs_failed: $failed, jobs: $jobs}}' \ + > "${WORKDIR}/zstream-${release}.json" +} + +# --------------------------------------------------------------------------- +# Main +# --------------------------------------------------------------------------- + +main() { + info "=== LVMS Z-Stream Trigger ===" + info "Releases: ${RELEASES}" + info "Workdir: ${WORKDIR}" + ${DRY_RUN} && info "Mode: DRY RUN" + ${FORCE} && info "Mode: FORCE (ignoring last-tested digest)" + + load_state + + IFS=',' read -ra release_list <<< "${RELEASES}" + + for release in "${release_list[@]}"; do + release=$(echo "${release}" | xargs) + process_release "${release}" + done + + # Merge per-release summaries into one file + local summary_file="${WORKDIR}/zstream-summary.json" + jq -s 'add' "${WORKDIR}"/zstream-*.json > "${summary_file}" 2>/dev/null || echo '{}' > "${summary_file}" + + # Print summary + info "" + info "=== Summary ===" + jq -r 'to_entries[] | " \(.key): \(.value.status)\(if .value.reason then " (\(.value.reason))" elif .value.jobs_triggered then " (\(.value.jobs_triggered) jobs)" else "" end)"' \ + "${summary_file}" >&2 + + info "" + info "Summary written to: ${summary_file}" +} + +main diff --git a/plugins/lvms-ci/skills/doctor/SKILL.md b/plugins/lvms-ci/skills/doctor/SKILL.md index c9e929e3..80ab000e 100644 --- a/plugins/lvms-ci/skills/doctor/SKILL.md +++ b/plugins/lvms-ci/skills/doctor/SKILL.md @@ -82,9 +82,11 @@ Compute once at the start by running `date +%y%m%d` and substituting into the pa Use the Write tool to save the file. The file must contain the complete analysis report." ``` -3. Launch **ALL** agents in a **single message** as **foreground** agents (do NOT use `run_in_background`). Foreground agents in the same message run concurrently — this is just as fast as background agents but keeps your turn active until all complete. -4. Say "Analyzing N jobs in parallel..." in your message text alongside the Agent tool calls. -5. When all agents return, immediately proceed to Step 3 in the same turn. Do NOT stop or end your turn between Step 2 and Step 3. +3. If the prepare output contains a `"zstream"` key, process those entries the same way — they use patch versions (e.g., `4.19.3`) as the `` value but follow the identical naming and analysis pattern. + +4. Launch **ALL** agents in a **single message** as **foreground** agents (do NOT use `run_in_background`). Foreground agents in the same message run concurrently — this is just as fast as background agents but keeps your turn active until all complete. +5. Say "Analyzing N jobs in parallel..." in your message text alongside the Agent tool calls. +6. When all agents return, immediately proceed to Step 3 in the same turn. Do NOT stop or end your turn between Step 2 and Step 3. ### Step 3: Finalize — Aggregate and Generate HTML Report @@ -143,6 +145,24 @@ HTML report generated: /report-lvm-operator-ci-doctor.html - Internet access to fetch job data from Prow/GCS - Bash shell, Python 3 +### Z-Stream Integration (Optional) + +If a `zstream-summary.json` file exists in the workdir and contains releases with `"status": "skipped"` (same digest already tested), the HTML report will include a **Z-Stream** checkbox on the Periodics tab. When checked, it switches the view to show z-stream release status per version (PR info, image digest, triggered jobs). + +To populate z-stream data before generating the report: + +```text +bash plugins/lvms-ci/scripts/zstream-trigger.sh --dry-run --workdir +``` + +This queries GitHub for open z-stream release PRs on `openshift/lvm-operator`, resolves catalog image digests via quay.io, and writes `zstream-summary.json` to the workdir — without triggering any jobs (dry-run mode). + +To actually trigger z-stream e2e jobs (requires `MY_APPCI_TOKEN`): + +```text +bash plugins/lvms-ci/scripts/zstream-trigger.sh --workdir +``` + ## Related Skills - **lvms-ci:prow-job**: Single job analysis (used by Step 2 agents) diff --git a/plugins/shared/scripts/create-report.py b/plugins/shared/scripts/create-report.py index 4c0a330b..ac4a5a4f 100755 --- a/plugins/shared/scripts/create-report.py +++ b/plugins/shared/scripts/create-report.py @@ -43,6 +43,10 @@ "lvm-operator": "LVMS", } +COMPONENT_REPOS = { + "lvm-operator": "openshift/lvm-operator", + "microshift": "openshift/microshift", +} _GRADE_ORDER = {"A": 0, "B": 1, "C": 2, "D": 3, "F": 4} _GRADE_CSS = {"A": "grade-a", "B": "grade-b", "C": "grade-c", "D": "grade-d", "F": "grade-f"} @@ -191,8 +195,13 @@ if (el) el.style.display = el.style.display === 'none' ? 'block' : 'none'; } function filterToday(on) { + var zstreamCheck = document.getElementById('filter-zstream'); + if (on && zstreamCheck && zstreamCheck.checked) { + zstreamCheck.checked = false; + if (typeof filterZstream === 'function') filterZstream(false); + } var today = new Date().toISOString().split('T')[0]; - document.querySelectorAll('#tab-periodics .issue-row').forEach(function(row) { + document.querySelectorAll('#tab-periodics .release-section:not(.zstream-section) .issue-row').forEach(function(row) { var dates = (row.getAttribute('data-dates') || '').split(' '); var show = !on || dates.indexOf(today) !== -1; row.style.display = show ? '' : 'none'; @@ -202,7 +211,7 @@ detail.style.display = show ? '' : 'none'; } }); - document.querySelectorAll('#tab-periodics .release-section').forEach(function(sec) { + document.querySelectorAll('#tab-periodics .release-section:not(.zstream-section)').forEach(function(sec) { var id = sec.id.replace('release-', ''); var rows = sec.querySelectorAll('.issue-row'); var total = 0, bd = {build: 0, test: 0, infra: 0}; @@ -240,6 +249,7 @@ row.style.display = (!on || row.hasAttribute('data-latest')) ? '' : 'none'; }); } + function showGraphTab(btn, paneId) { var container = btn.closest('.perf-graphs'); container.querySelectorAll('.graph-tab-btn').forEach(function(b) { b.classList.remove('active'); }); @@ -1051,7 +1061,7 @@ def _render_index_image(index_info): short = commit[:12] if len(commit) >= 12 else commit lines.append( f' Source Commit: ' - f'{_e(short)}' + f'{_e(short)}' ) if index_info.get("error"): lines.append(f'
Inspect failed: {_e(index_info["error"])}') @@ -1059,8 +1069,11 @@ def _render_index_image(index_info): return "\n".join(lines) + + # Graph workdir — set by main() before rendering _GRAPHS_DIR = None +_COMPONENT = None def _extract_build_id(url): @@ -1203,12 +1216,22 @@ def _render_bug_links(bug_match): # HTML rendering # --------------------------------------------------------------------------- -def render_release_section(version, rdata, bug_candidates, index_info=None): +def render_release_section(version, rdata, bug_candidates, index_info=None, zstream_info=None): + is_zstream = zstream_info is not None + display = version + if is_zstream: + pr_title = zstream_info.get("pr_title", "") + m = re.search(r'(\d+\.\d+\.\d+)', pr_title) if pr_title else None + if m: + display = m.group(1) + css_class = "release-section zstream-section" if is_zstream else "release-section" + style = ' style="display:none"' if is_zstream else "" + if rdata is None: return ( - f'
\n' + f'
\n' '
\n' - f'

Release {_e(version)}

\n' + f'

Release {_e(display)}

\n' ' no data\n' '
\n' "

Analysis failed to produce results.

\n" @@ -1217,9 +1240,9 @@ def render_release_section(version, rdata, bug_candidates, index_info=None): if rdata.get("collection_error"): return ( - f'
\n' + f'
\n' '
\n' - f'

Release {_e(version)}

\n' + f'

Release {_e(display)}

\n' ' collection error\n' '
\n' f' 
Data collection failed: {_e(rdata["collection_error"])}
\n' @@ -1232,14 +1255,17 @@ def render_release_section(version, rdata, bug_candidates, index_info=None): b = rdata["breakdown"] lines = [] - lines.append(f'
') + lines.append(f'
') lines.append('
') - lines.append(f'

Release {_e(version)}🔗

') + lines.append(f'

Release {_e(display)}🔗

') label = "failure" if total == 1 else "failures" lines.append(f' {total} {label}') lines.append("
") - idx_html = _render_index_image(index_info) + if is_zstream: + idx_html = _render_zstream_info(version, zstream_info) + else: + idx_html = _render_index_image(index_info) if idx_html: lines.append(idx_html) @@ -1453,7 +1479,31 @@ def render_pr_section(pr_data, bug_candidates, pr_status, pr_error=None): return "\n".join(toc_lines) + "\n\n" + "\n".join(lines) -def generate_html(component_title, releases_data, all_bug_candidates, pr_data, pr_status, timestamp, pr_error=None, bugs_tab_data=None, images_tab_data=None, index_data=None): +# --------------------------------------------------------------------------- +# Z-Stream sections (rendered inside Periodics tab, toggled via checkbox) +# --------------------------------------------------------------------------- + + +def _render_zstream_info(version, zinfo): + """Render z-stream metadata box (PR, snapshot, image).""" + lines = ['
'] + pr = zinfo.get("pr", "") + image = zinfo.get("image", "") + snapshot = zinfo.get("snapshot", "") + if pr: + pr_display = pr.lstrip("#") + repo = COMPONENT_REPOS.get(_COMPONENT, "openshift/" + (_COMPONENT or "unknown")) + pr_url = f"https://github.com/{repo}/pull/{pr_display}" + lines.append(f' Release PR: #{_e(pr_display)}
') + if snapshot: + lines.append(f' Snapshot: {_e(snapshot)}
') + if image: + lines.append(f' Catalog Image: {_e(image)}
') + lines.append("
") + return "\n".join(lines) + + +def generate_html(component_title, releases_data, all_bug_candidates, pr_data, pr_status, timestamp, pr_error=None, bugs_tab_data=None, images_tab_data=None, index_data=None, zstream_data=None): date_str = timestamp.strftime("%Y-%m-%d") time_str = timestamp.strftime("%Y-%m-%d %H:%M:%S") @@ -1516,10 +1566,58 @@ def generate_html(component_title, releases_data, all_bug_candidates, pr_data, p for version, rdata in releases_data.items(): sections.append(render_release_section(version, rdata, all_bug_candidates, _idx.get(version))) + # Append z-stream sections using the same render_release_section path + zstream_checkbox = "" + if zstream_data: + sorted_zversions = sorted( + [zv for zv, zi in zstream_data.items() if zi.get("status") == "skipped"], + key=lambda v: [int(x) for x in v.split(".")], + ) + if sorted_zversions: + zstream_checkbox = ( + ' ' + ) + for zv in sorted_zversions: + zinfo = zstream_data[zv] + empty_bd = {"build": 0, "test": 0, "infrastructure": 0} + rdata = zinfo.get("_analyzed", {"total_failed": 0, "issues": [], "breakdown": empty_bd}) + section_id = f"{zv}-zstream" + pr_title = zinfo.get("pr_title", "") + m = re.search(r'(\d+\.\d+\.\d+)', pr_title) if pr_title else None + display = m.group(1) if m else section_id + total = rdata["total_failed"] if rdata else 0 + b = rdata.get("breakdown", empty_bd) if rdata else empty_bd + fail_word = "failure" if total == 1 else "failures" + toc.append( + f' ' + ) + sections.append(render_release_section(section_id, rdata, all_bug_candidates, zstream_info=zinfo)) + pr_section = render_pr_section(pr_data, all_bug_candidates, pr_status, pr_error) bugs_section = render_bugs_section(bugs_tab_data) if bugs_tab_data else "" images_section = render_images_section(images_tab_data) + zstream_js = "" + if zstream_checkbox: + zstream_js = """ +function filterZstream(on) { + var todayCheck = document.getElementById('filter-today'); + if (on && todayCheck && todayCheck.checked) { + todayCheck.checked = false; + filterToday(false); + } + document.querySelectorAll('#tab-periodics .toc > ul > li:not(.toc-zstream)').forEach(function(el) { el.style.display = on ? 'none' : ''; }); + document.querySelectorAll('#tab-periodics .toc > ul > li.toc-zstream').forEach(function(el) { el.style.display = on ? '' : 'none'; }); + document.querySelectorAll('#tab-periodics .release-section:not(.zstream-section)').forEach(function(el) { el.style.display = on ? 'none' : ''; }); + document.querySelectorAll('#tab-periodics .zstream-section').forEach(function(el) { el.style.display = on ? '' : 'none'; }); +}""" + js_content = JS + zstream_js + return f"""\ @@ -1551,13 +1649,15 @@ def generate_html(component_title, releases_data, all_bug_candidates, pr_data, p

Table of Contents

- +
+ +{zstream_checkbox} +
    {chr(10).join(toc)}
- {chr(10).join(sections)}
@@ -1576,7 +1676,7 @@ def generate_html(component_title, releases_data, all_bug_candidates, pr_data, p

 

 

 

 

@@ -1756,15 +1856,28 @@ def main(): images_data = load_images_data(workdir, releases) images_tab_data = build_images_tab_data(images_data, releases) if images_data else None - # Set graphs directory for rendering - global _GRAPHS_DIR + # Set globals for rendering + global _GRAPHS_DIR, _COMPONENT + _COMPONENT = component graphs_dir = os.path.join(workdir, "graphs") if os.path.isdir(graphs_dir): _GRAPHS_DIR = graphs_dir + zstream_data = load_json(os.path.join(workdir, "zstream-summary.json")) + if zstream_data: + for zv, zinfo in zstream_data.items(): + if zinfo.get("status") == "skipped": + pr_title = zinfo.get("pr_title", "") + m = re.search(r'(\d+\.\d+\.\d+)', pr_title) if pr_title else None + patch = m.group(1) if m else f"{zv}-zstream" + rdata = load_json(os.path.join(workdir, "jobs", f"release-{patch}-summary.json")) + if rdata is None: + rdata = {"total_failed": 0, "issues": [], "breakdown": _EMPTY_BREAKDOWN} + zinfo["_analyzed"] = rdata + # Generate HTML timestamp = datetime.now(timezone.utc) - html_content = generate_html(component_title, releases_data, all_bug_candidates, pr_data, pr_status, timestamp, pr_error, bugs_tab_data, images_tab_data, index_data) + html_content = generate_html(component_title, releases_data, all_bug_candidates, pr_data, pr_status, timestamp, pr_error, bugs_tab_data, images_tab_data, index_data, zstream_data) output_path = os.path.join(workdir, f"report-{component}-ci-doctor.html") with open(output_path, "w") as f: @@ -1812,6 +1925,17 @@ def main(): print(f" Release {release}: {repo_names}{grade_str}") else: print(" No container image data") + if zstream_data: + print(" Z-Stream:") + for zv in sorted(zstream_data, key=lambda v: [int(x) for x in v.split(".")]): + zi = zstream_data[zv] + zs = zi.get("status", "unknown") + extra = "" + if zs == "triggered": + extra = f' ({zi.get("jobs_triggered", 0)} jobs, PR {zi.get("pr", "")})' + elif zi.get("reason"): + extra = f' ({zi["reason"]})' + print(f" Release {zv}: {zs}{extra}") print(f"\nHTML report generated: {output_path}") diff --git a/plugins/shared/scripts/doctor.sh b/plugins/shared/scripts/doctor.sh index 569c7302..3450603a 100755 --- a/plugins/shared/scripts/doctor.sh +++ b/plugins/shared/scripts/doctor.sh @@ -121,6 +121,72 @@ cmd_prepare() { echo " Done: ${jobs_file}" >&2 done + # Collect and download for z-stream "skipped" releases + local zstream_summary="${WORKDIR}/zstream-summary.json" + declare -a ZSTREAM_RELEASES=() + local skipped_releases + skipped_releases=$(jq -r 'to_entries[] | select(.value.status == "skipped") | .key' "${zstream_summary}" 2>/dev/null || true) + + while IFS= read -r zrel; do + [[ -z "${zrel}" ]] && continue + # Extract patch version from PR title (e.g. "chore: Release 4.19.3" → "4.19.3") + local patch_version + patch_version=$(jq -r --arg r "${zrel}" '.[$r].pr_title // ""' "${zstream_summary}" 2>/dev/null \ + | grep -oP '\d+\.\d+\.\d+' || echo "${zrel}-zstream") + + # Skip if already collected as a regular release + if [[ -f "${WORKDIR}/jobs/release-${patch_version}-jobs.json" ]] && \ + [[ "$(jq 'length' "${WORKDIR}/jobs/release-${patch_version}-jobs.json")" -gt 0 ]]; then + echo " Skipping z-stream ${patch_version}: already collected" >&2 + continue + fi + + echo "=== Z-Stream ${zrel} → ${patch_version} (skipped — same digest) ===" >&2 + local zjobs_file="${WORKDIR}/jobs/release-${patch_version}-jobs.json" + + echo " Collecting failed periodic jobs..." >&2 + local zraw_json zraw_err + zraw_err=$(mktemp) + if ! zraw_json=$(bash "${SCRIPT_DIR}/prow-jobs-for-release.sh" "${COMPONENT}" "${zrel}" 2>"${zraw_err}"); then + echo " ERROR: failed to collect z-stream jobs for ${zrel}:" >&2 + local zerr_msg + zerr_msg=$(cat "${zraw_err}") + echo "${zerr_msg}" >&2 + rm -f "${zraw_err}" + echo "[]" > "${zjobs_file}" + release_errors["${patch_version}"]="${zerr_msg:-data collection failed}" + echo "${release_errors["${patch_version}"]}" > "${WORKDIR}/jobs/release-${patch_version}-error.txt" + continue + fi + rm -f "${zraw_err}" + + local zfiltered_json + zfiltered_json=$(echo "${zraw_json}" | jq '[.[] | select(.type == "periodic")]') + + local zcount + zcount=$(echo "${zfiltered_json}" | jq 'length') + + if [[ "${zcount}" -eq 0 ]]; then + echo " No failed periodic jobs found" >&2 + echo "[]" > "${zjobs_file}" + ZSTREAM_RELEASES+=("${patch_version}") + continue + fi + + echo " Found ${zcount} failed periodic jobs, downloading artifacts..." >&2 + local zdl_err + zdl_err=$(mktemp) + echo "${zfiltered_json}" | \ + bash "${SCRIPT_DIR}/download-jobs.sh" --workdir "${WORKDIR}" 2>"${zdl_err}" \ + > "${zjobs_file}" + [[ -s "${zdl_err}" ]] && cat "${zdl_err}" >&2 + rm -f "${zdl_err}" + + total_jobs=$((total_jobs + zcount)) + ZSTREAM_RELEASES+=("${patch_version}") + echo " Done: ${zjobs_file}" >&2 + done <<< "${skipped_releases}" + # Collect and download for rebase PRs if ${do_rebase}; then echo "=== Rebase Pull Requests ===" >&2 @@ -266,6 +332,22 @@ cmd_prepare() { result=$(jq -n --arg w "${WORKDIR}" --argjson rel "${releases_json}" \ '{workdir: $w, releases: $rel}') + # Add z-stream releases to output (uses patch versions, same release-* naming) + if [[ ${#ZSTREAM_RELEASES[@]} -gt 0 ]]; then + local zstream_json="[]" + for zpatch in "${ZSTREAM_RELEASES[@]}"; do + local zjobs_file="${WORKDIR}/jobs/release-${zpatch}-jobs.json" + local zcount=0 + if [[ -f "${zjobs_file}" ]]; then + zcount=$(jq 'length' "${zjobs_file}") + fi + zstream_json=$(echo "${zstream_json}" | jq \ + --arg r "${zpatch}" --argjson c "${zcount}" --arg f "${zjobs_file}" \ + '. + [{release: $r, jobs: $c, jobs_file: $f}]') + done + result=$(echo "${result}" | jq --argjson z "${zstream_json}" '. + {zstream: $z}') + fi + if ${do_rebase}; then local prs_file="${WORKDIR}/jobs/prs-jobs.json" local pr_job_count=0 @@ -329,14 +411,22 @@ cmd_finalize() { IFS=',' read -ra RELEASES <<< "${releases_arg}" - # Aggregate each release - for release in "${RELEASES[@]}"; do - release=$(echo "${release}" | xargs) - echo "=== Aggregating release ${release} ===" >&2 - python3 "${SCRIPT_DIR}/aggregate.py" \ - --release "${release}" --workdir "${WORKDIR}" >/dev/null || \ - echo " WARNING: aggregation failed for ${release}" >&2 - done + # Aggregate all job files (regular releases + z-stream patch versions) + local all_job_files + all_job_files=$(find "${WORKDIR}/jobs" -name 'release-*-jobs.json' 2>/dev/null | sort || true) + while IFS= read -r jobs_file; do + [[ -z "${jobs_file}" ]] && continue + local release + release=$(basename "${jobs_file}" | sed 's/^release-//; s/-jobs\.json$//') + local reports + reports=$(find "${WORKDIR}/jobs" -name "release-${release}-job-*.txt" 2>/dev/null | head -1) + if [[ -n "${reports}" ]]; then + echo "=== Aggregating release ${release} ===" >&2 + python3 "${SCRIPT_DIR}/aggregate.py" \ + --release "${release}" --workdir "${WORKDIR}" >/dev/null || \ + echo " WARNING: aggregation failed for ${release}" >&2 + fi + done <<< "${all_job_files}" # Aggregate PRs (if job files exist) local pr_files @@ -356,7 +446,8 @@ cmd_finalize() { local images_dir="${WORKDIR}/images" mkdir -p "${images_dir}" - for repo in ${repos}; do + while IFS= read -r repo; do + [[ -z "${repo}" ]] && continue local repo_slug="${repo//\//@}" # Fetch catalog repository ID (needed for catalog URLs) @@ -383,7 +474,7 @@ cmd_finalize() { fi done echo "${tmp_merged}" > "${outfile}" - done + done <<< "${repos}" fi # Extract index image info (LVMS-specific, no-op for other components) From 28592128d0b8e3f7e6530ea31efe4c7f8a7395bd Mon Sep 17 00:00:00 2001 From: Kasturi Narra Date: Thu, 2 Jul 2026 23:02:51 +0530 Subject: [PATCH 02/14] Fix review findings in z-stream trigger script - Switch from -on-push tags (which expire) to v{version}-{commit} merge tags - Fix empty triggered_jobs producing [""] instead of [] Co-Authored-By: Claude Opus 4.6 --- plugins/lvms-ci/scripts/zstream-trigger.sh | 19 ++++++++++++------- 1 file changed, 12 insertions(+), 7 deletions(-) diff --git a/plugins/lvms-ci/scripts/zstream-trigger.sh b/plugins/lvms-ci/scripts/zstream-trigger.sh index 45e1f71f..66084100 100755 --- a/plugins/lvms-ci/scripts/zstream-trigger.sh +++ b/plugins/lvms-ci/scripts/zstream-trigger.sh @@ -221,17 +221,22 @@ resolve_snapshot_to_digest() { snap_sec=${time_str:4:2} snap_epoch=$(date -d "${snap_year}-${snap_month}-${snap_day}T${snap_hour}:${snap_min}:${snap_sec}Z" +%s 2>/dev/null || echo 0) - # Query quay for on-push build-image-index tags matching this version + # Convert version prefix (lvm-operator-catalog-4-16) to v4.16 for tag matching + local version_dot + version_dot=$(echo "${version_prefix}" | sed 's/lvm-operator-catalog-//; s/-/./') + + # Query quay for v{version} commit tags (not -on-push, which expire) local tags_json tags_json=$(curl -sSL --connect-timeout 10 --max-time 30 \ - "${QUAY_API}/repository/${QUAY_REPO}/tag/?limit=50&filter_tag_name=like:${version_prefix}-on-push") + "${QUAY_API}/repository/${QUAY_REPO}/tag/?limit=50&filter_tag_name=like:v${version_dot}-") - # Find build-image-index tags created after the snapshot timestamp - # Pick the closest one (smallest positive time delta) + # Select only raw commit tags (v4.16-{40-hex}), exclude auxiliary suffixes + # Pick the closest one after the snapshot timestamp (smallest positive delta) local result - result=$(echo "${tags_json}" | jq -r --arg snap_epoch "${snap_epoch}" ' + result=$(echo "${tags_json}" | jq -r --arg snap_epoch "${snap_epoch}" --arg vpfx "v${version_dot}-" ' [.tags[] - | select(.name | endswith("build-image-index")) + | select(.name | startswith($vpfx)) + | select(.name | test("^v[0-9]+\\.[0-9]+-[a-f0-9]{40}$")) | .tag_epoch = (.last_modified | strptime("%a, %d %b %Y %H:%M:%S %z") | mktime) | .delta = (.tag_epoch - ($snap_epoch | tonumber)) | select(.delta >= 0 and .delta < 1800) @@ -441,7 +446,7 @@ process_release() { --arg snap "${snapshot}" \ --argjson triggered "${triggered}" \ --argjson failed "${failed}" \ - --argjson jobs "$(printf '%s\n' "${triggered_jobs[@]}" | jq -R . | jq -s .)" \ + --argjson jobs "$(if [[ ${#triggered_jobs[@]} -gt 0 ]]; then printf '%s\n' "${triggered_jobs[@]}" | jq -R . | jq -s .; else echo '[]'; fi)" \ '{($r): {status: "triggered", pr: ("#" + $pr), pr_title: $title, image: $img, snapshot: $snap, jobs_triggered: $triggered, jobs_failed: $failed, jobs: $jobs}}' \ > "${WORKDIR}/zstream-${release}.json" } From f8795e0e9a1ad95733ec04a31a10c1fb9b8cde45 Mon Sep 17 00:00:00 2001 From: Kasturi Narra Date: Thu, 2 Jul 2026 23:08:23 +0530 Subject: [PATCH 03/14] Sort fetch_last_runs results by startTime to pick latest run Co-Authored-By: Claude Opus 4.6 --- plugins/lvms-ci/scripts/zstream-trigger.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/plugins/lvms-ci/scripts/zstream-trigger.sh b/plugins/lvms-ci/scripts/zstream-trigger.sh index 66084100..e9c5a4c6 100755 --- a/plugins/lvms-ci/scripts/zstream-trigger.sh +++ b/plugins/lvms-ci/scripts/zstream-trigger.sh @@ -316,6 +316,7 @@ fetch_last_runs() { entry=$(echo "${prow_json}" | sed 's/^var allBuilds = //' | \ jq -r --arg n "${test_name}" ' [.items[] | select(.status.state == "success" or .status.state == "failure" or .status.state == "error" or .status.state == "aborted")] | + sort_by(.status.startTime) | reverse | if length > 0 then .[0] else null end | if . then {name: $n, state: .status.state, url: .status.url, started: .status.startTime} else {name: $n, state: "unknown"} end ' 2>/dev/null || echo "{\"name\": \"${test_name}\", \"state\": \"unknown\"}") From 41d15a6c0a4f8a0b42ebd008a5a5e47169730bab Mon Sep 17 00:00:00 2001 From: Kasturi Narra Date: Thu, 2 Jul 2026 23:47:52 +0530 Subject: [PATCH 04/14] Use closest tag by absolute time delta for snapshot resolution The v{version}-{commit} merge tags can appear before or after the snapshot timestamp, so pick the closest match regardless of direction instead of requiring a narrow 30-minute window. Co-Authored-By: Claude Opus 4.6 --- plugins/lvms-ci/scripts/zstream-trigger.sh | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/plugins/lvms-ci/scripts/zstream-trigger.sh b/plugins/lvms-ci/scripts/zstream-trigger.sh index e9c5a4c6..f79544ea 100755 --- a/plugins/lvms-ci/scripts/zstream-trigger.sh +++ b/plugins/lvms-ci/scripts/zstream-trigger.sh @@ -231,16 +231,15 @@ resolve_snapshot_to_digest() { "${QUAY_API}/repository/${QUAY_REPO}/tag/?limit=50&filter_tag_name=like:v${version_dot}-") # Select only raw commit tags (v4.16-{40-hex}), exclude auxiliary suffixes - # Pick the closest one after the snapshot timestamp (smallest positive delta) + # Pick the tag closest to the snapshot timestamp (smallest absolute delta) local result result=$(echo "${tags_json}" | jq -r --arg snap_epoch "${snap_epoch}" --arg vpfx "v${version_dot}-" ' [.tags[] | select(.name | startswith($vpfx)) | select(.name | test("^v[0-9]+\\.[0-9]+-[a-f0-9]{40}$")) | .tag_epoch = (.last_modified | strptime("%a, %d %b %Y %H:%M:%S %z") | mktime) - | .delta = (.tag_epoch - ($snap_epoch | tonumber)) - | select(.delta >= 0 and .delta < 1800) - ] | sort_by(.delta) | .[0] // empty | + | .abs_delta = ((.tag_epoch - ($snap_epoch | tonumber)) | fabs) + ] | sort_by(.abs_delta) | .[0] // empty | [.name, .manifest_digest] | @tsv ') From 0aada91f3c37fda02862e38140ab48b5f2c4487d Mon Sep 17 00:00:00 2001 From: Kasturi Narra Date: Thu, 2 Jul 2026 23:54:32 +0530 Subject: [PATCH 05/14] Add retry with backoff for Gangway 429 rate limiting Retry up to 3 times on HTTP 429 with exponential backoff (15s, 30s). Increase base delay between triggers from 5s to 30s. Co-Authored-By: Claude Opus 4.6 --- plugins/lvms-ci/scripts/zstream-trigger.sh | 38 ++++++++++++++-------- 1 file changed, 24 insertions(+), 14 deletions(-) diff --git a/plugins/lvms-ci/scripts/zstream-trigger.sh b/plugins/lvms-ci/scripts/zstream-trigger.sh index f79544ea..20d7666d 100755 --- a/plugins/lvms-ci/scripts/zstream-trigger.sh +++ b/plugins/lvms-ci/scripts/zstream-trigger.sh @@ -274,21 +274,31 @@ trigger_job() { return 0 fi - local http_code - http_code=$(curl -sSL -X POST -o /dev/stderr -w '%{http_code}' \ - --connect-timeout 10 --max-time 30 \ - -H "Authorization: Bearer ${MY_APPCI_TOKEN}" \ - -H "Content-Type: application/json" \ - -d "${body}" \ - "${GANGWAY_API}/v1/executions/${job_name}" 2>/dev/null) - - if [[ "${http_code}" == "200" ]]; then - info " triggered: ${job_name}" - return 0 - else + local http_code attempt max_retries=3 delay=15 + + for ((attempt = 1; attempt <= max_retries; attempt++)); do + http_code=$(curl -sSL -X POST -o /dev/stderr -w '%{http_code}' \ + --connect-timeout 10 --max-time 30 \ + -H "Authorization: Bearer ${MY_APPCI_TOKEN}" \ + -H "Content-Type: application/json" \ + -d "${body}" \ + "${GANGWAY_API}/v1/executions/${job_name}" 2>/dev/null) + + if [[ "${http_code}" == "200" ]]; then + info " triggered: ${job_name}" + return 0 + fi + + if [[ "${http_code}" == "429" && ${attempt} -lt ${max_retries} ]]; then + warn " rate limited (attempt ${attempt}/${max_retries}), retrying in ${delay}s: ${job_name}" + sleep "${delay}" + delay=$((delay * 2)) + continue + fi + warn " failed (HTTP ${http_code}): ${job_name}" return 1 - fi + done } # --------------------------------------------------------------------------- @@ -427,7 +437,7 @@ process_release() { else failed=$((failed + 1)) fi - ${DRY_RUN} || sleep 5 + ${DRY_RUN} || sleep 30 done info "${release}: ${triggered} triggered, ${failed} failed" From 7dfc019dd24e3e79c317260e3cfc13727f207d9f Mon Sep 17 00:00:00 2001 From: Kasturi Narra Date: Fri, 3 Jul 2026 12:57:09 +0530 Subject: [PATCH 06/14] lvms-zstream: filter PRs by release-management branch and match by snapshot content Filter GitHub PR search to base=release-management branch (all LVMS release PRs target this branch), eliminating the need for title regex matching. Instead, match PRs by checking if their catalog files contain a snapshot referencing the target version. This handles varying title formats and reduces API calls (5 PRs vs 100). The snapshot is now extracted during PR discovery and returned alongside the PR number/title, removing the need for a separate extract_snapshot call. Co-Authored-By: Claude Opus 4.6 --- plugins/lvms-ci/scripts/zstream-trigger.sh | 66 +++++++++------------- 1 file changed, 26 insertions(+), 40 deletions(-) diff --git a/plugins/lvms-ci/scripts/zstream-trigger.sh b/plugins/lvms-ci/scripts/zstream-trigger.sh index 20d7666d..b7eca337 100755 --- a/plugins/lvms-ci/scripts/zstream-trigger.sh +++ b/plugins/lvms-ci/scripts/zstream-trigger.sh @@ -147,15 +147,32 @@ find_zstream_pr() { local major minor major=$(echo "${release}" | cut -d. -f1) minor=$(echo "${release}" | cut -d. -f2) + local catalog_prefix="lvm-operator-catalog-${major}-${minor}" local prs_json - prs_json=$(github_curl "${GITHUB_API}/repos/openshift/lvm-operator/pulls?state=open&per_page=100") - - echo "${prs_json}" | jq -r --arg maj "${major}" --arg min "${minor}" ' - [.[] | select(.title | test(": Release " + $maj + "\\." + $min + "\\."; "i"))] | - if length > 0 then .[0] else empty end | - [.number, .title] | @tsv - ' + prs_json=$(github_curl "${GITHUB_API}/repos/openshift/lvm-operator/pulls?state=open&base=release-management&per_page=100") + + local pr_count + pr_count=$(echo "${prs_json}" | jq 'length') + + local i + for ((i = 0; i < pr_count; i++)); do + local pr_number pr_title + pr_number=$(echo "${prs_json}" | jq -r ".[$i].number") + pr_title=$(echo "${prs_json}" | jq -r ".[$i].title") + + local files_json snapshot + files_json=$(github_curl "${GITHUB_API}/repos/openshift/lvm-operator/pulls/${pr_number}/files") + snapshot=$(echo "${files_json}" | jq -r ' + [.[] | select(.filename | contains("catalog")) | .patch // ""] | + join("\n") + ' | { grep -oP "(?<=snapshot: )${catalog_prefix}\\S*" || true; } | head -1) + + if [[ -n "${snapshot}" ]]; then + printf '%s\t%s\t%s\n' "${pr_number}" "${pr_title}" "${snapshot}" + return 0 + fi + done } # --------------------------------------------------------------------------- @@ -176,22 +193,6 @@ check_pr_merged() { fi } -# --------------------------------------------------------------------------- -# GitHub: extract snapshot name from PR diff -# --------------------------------------------------------------------------- - -extract_snapshot() { - local pr_number="$1" - - local files_json - files_json=$(github_curl "${GITHUB_API}/repos/openshift/lvm-operator/pulls/${pr_number}/files") - - echo "${files_json}" | jq -r ' - [.[] | select(.filename | contains("catalog")) | .patch // ""] | - join("\n") - ' | { grep -oP '(?<=snapshot: )\S+' || true; } | head -1 -} - # --------------------------------------------------------------------------- # Quay: resolve snapshot to digest # --------------------------------------------------------------------------- @@ -342,9 +343,6 @@ fetch_last_runs() { process_release() { local release="$1" - local major minor - major=$(echo "${release}" | cut -d. -f1) - minor=$(echo "${release}" | cut -d. -f2) info "--- Release ${release} ---" @@ -375,23 +373,11 @@ process_release() { return 0 fi - local pr_number pr_title + local pr_number pr_title snapshot pr_number=$(echo "${pr_info}" | cut -f1) pr_title=$(echo "${pr_info}" | cut -f2) + snapshot=$(echo "${pr_info}" | cut -f3) info "Found PR #${pr_number}: ${pr_title}" - - # Extract snapshot from PR - local snapshot - snapshot=$(extract_snapshot "${pr_number}") - - if [[ -z "${snapshot}" ]]; then - warn "Could not extract catalog snapshot from PR #${pr_number}" - jq -n --arg r "${release}" --arg pr "${pr_number}" \ - '{($r): {status: "error", reason: "no snapshot in PR", pr: ("#" + $pr)}}' \ - > "${WORKDIR}/zstream-${release}.json" - return 0 - fi - info "Snapshot: ${snapshot}" # Resolve to digest From 7bb953ff33d70ebc37fddcb3a64461a108dc3da9 Mon Sep 17 00:00:00 2001 From: Kasturi Narra Date: Fri, 3 Jul 2026 13:33:16 +0530 Subject: [PATCH 07/14] lvms-zstream: guard against null last_modified in quay tag resolution Skip tags with missing last_modified before calling strptime to prevent jq pipeline abort during digest resolution. Co-Authored-By: Claude Opus 4.6 --- plugins/lvms-ci/scripts/zstream-trigger.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/plugins/lvms-ci/scripts/zstream-trigger.sh b/plugins/lvms-ci/scripts/zstream-trigger.sh index b7eca337..4e1ecb8a 100755 --- a/plugins/lvms-ci/scripts/zstream-trigger.sh +++ b/plugins/lvms-ci/scripts/zstream-trigger.sh @@ -238,6 +238,7 @@ resolve_snapshot_to_digest() { [.tags[] | select(.name | startswith($vpfx)) | select(.name | test("^v[0-9]+\\.[0-9]+-[a-f0-9]{40}$")) + | select(.last_modified != null and .last_modified != "") | .tag_epoch = (.last_modified | strptime("%a, %d %b %Y %H:%M:%S %z") | mktime) | .abs_delta = ((.tag_epoch - ($snap_epoch | tonumber)) | fabs) ] | sort_by(.abs_delta) | .[0] // empty | From aef588d5fb10bfd3e2b6d1315ea29c9ddc95cbf4 Mon Sep 17 00:00:00 2001 From: Kasturi Narra Date: Fri, 3 Jul 2026 23:50:06 +0530 Subject: [PATCH 08/14] lvms-zstream: separate z-stream collection from doctor.sh Move z-stream artifact collection out of the shared doctor.sh into zstream-trigger.sh so doctor stays reporting-only with no LVMS-specific logic. For skipped releases (same digest), the trigger script now collects failed periodic jobs and downloads artifacts into the workdir using the same release-*-jobs.json naming, so doctor's finalize discovers them automatically. Also make --releases a required argument instead of hardcoding the release list. Co-Authored-By: Claude Opus 4.6 --- plugins/lvms-ci/scripts/zstream-trigger.sh | 74 ++++++++++++++++++- plugins/lvms-ci/skills/doctor/SKILL.md | 16 ++--- plugins/shared/scripts/doctor.sh | 84 +--------------------- 3 files changed, 80 insertions(+), 94 deletions(-) diff --git a/plugins/lvms-ci/scripts/zstream-trigger.sh b/plugins/lvms-ci/scripts/zstream-trigger.sh index 4e1ecb8a..f0196a02 100755 --- a/plugins/lvms-ci/scripts/zstream-trigger.sh +++ b/plugins/lvms-ci/scripts/zstream-trigger.sh @@ -12,19 +12,22 @@ set -euo pipefail # the last run (tracked via a state file). # # Usage: -# zstream-trigger.sh [--releases 4.14,4.16,...] [--dry-run] [--workdir /path] +# zstream-trigger.sh --releases 4.14,4.16,... [--dry-run] [--workdir /path] # [--force] [--token-file /path/to/token] # # Environment: # GITHUB_TOKEN Optional. GitHub PAT for higher API rate limits. # MY_APPCI_TOKEN Required (unless --dry-run). Gangway API bearer token. -RELEASES="4.14,4.16,4.18,4.19,4.20,4.21" +RELEASES="" DRY_RUN=false FORCE=false WORKDIR="" TOKEN_FILE="" +TRIGGER_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +COMPONENT="lvm-operator" + GANGWAY_API="https://gangway-ci.apps.ci.l2s4.p1.openshiftapps.com" GITHUB_API="https://api.github.com" QUAY_API="https://quay.io/api/v1" @@ -63,7 +66,7 @@ usage() { Usage: $(basename "$0") [OPTIONS] Options: - --releases LIST Comma-separated releases (default: ${RELEASES}) + --releases LIST Comma-separated releases (required) --dry-run Print what would be triggered without calling Gangway --force Trigger even if digest hasn't changed since last run --workdir DIR Write summary JSON here (default: /tmp/lvms-zstream.) @@ -92,6 +95,11 @@ while [[ ${#} -gt 0 ]]; do esac done +if [[ -z "${RELEASES}" ]]; then + err "--releases is required" + usage +fi + WORKDIR="${WORKDIR:-/tmp/lvms-zstream.$(date +%y%m%d)}" mkdir -p "${WORKDIR}" @@ -338,6 +346,64 @@ fetch_last_runs() { echo "${results}" } +# --------------------------------------------------------------------------- +# Collect artifacts for a skipped release (same digest, already tested) +# --------------------------------------------------------------------------- + +collect_skipped_artifacts() { + local release="$1" + local pr_title="$2" + + local patch_version + patch_version=$(echo "${pr_title}" | grep -oP '\d+\.\d+\.\d+' || echo "${release}-zstream") + + local jobs_dir="${WORKDIR}/jobs" + mkdir -p "${jobs_dir}" + + local jobs_file="${jobs_dir}/release-${patch_version}-jobs.json" + + # Skip if already collected + if [[ -f "${jobs_file}" ]] && [[ "$(jq 'length' "${jobs_file}")" -gt 0 ]]; then + info " Artifacts already collected for ${patch_version}" + return 0 + fi + + info " Collecting failed periodic jobs for ${release}..." + local raw_json raw_err + raw_err=$(mktemp) + if ! raw_json=$(bash "${TRIGGER_DIR}/prow-jobs-for-release.sh" "${COMPONENT}" "${release}" 2>"${raw_err}"); then + warn "Failed to collect jobs for ${release}:" + cat "${raw_err}" >&2 + rm -f "${raw_err}" + echo "[]" > "${jobs_file}" + return 0 + fi + rm -f "${raw_err}" + + local filtered_json + filtered_json=$(echo "${raw_json}" | jq '[.[] | select(.type == "periodic")]') + + local count + count=$(echo "${filtered_json}" | jq 'length') + + if [[ "${count}" -eq 0 ]]; then + info " No failed periodic jobs found for ${patch_version}" + echo "[]" > "${jobs_file}" + return 0 + fi + + info " Found ${count} failed periodic jobs, downloading artifacts..." + local dl_err + dl_err=$(mktemp) + echo "${filtered_json}" | \ + bash "${TRIGGER_DIR}/download-jobs.sh" --workdir "${WORKDIR}" 2>"${dl_err}" \ + > "${jobs_file}" + [[ -s "${dl_err}" ]] && cat "${dl_err}" >&2 + rm -f "${dl_err}" + + info " Done: ${jobs_file}" +} + # --------------------------------------------------------------------------- # Process a single release # --------------------------------------------------------------------------- @@ -407,6 +473,8 @@ process_release() { --argjson jobs "${last_runs}" \ '{($r): {status: "skipped", reason: "same digest", pr: ("#" + $pr), pr_title: $title, image: $img, snapshot: $snap, jobs: $jobs}}' \ > "${WORKDIR}/zstream-${release}.json" + + collect_skipped_artifacts "${release}" "${pr_title}" return 0 fi fi diff --git a/plugins/lvms-ci/skills/doctor/SKILL.md b/plugins/lvms-ci/skills/doctor/SKILL.md index 80ab000e..e68fc8b1 100644 --- a/plugins/lvms-ci/skills/doctor/SKILL.md +++ b/plugins/lvms-ci/skills/doctor/SKILL.md @@ -82,7 +82,7 @@ Compute once at the start by running `date +%y%m%d` and substituting into the pa Use the Write tool to save the file. The file must contain the complete analysis report." ``` -3. If the prepare output contains a `"zstream"` key, process those entries the same way — they use patch versions (e.g., `4.19.3`) as the `` value but follow the identical naming and analysis pattern. +3. If z-stream job files exist in `/jobs/` (e.g., `release-4.19.3-jobs.json` — written by `zstream-trigger.sh`), analyze those jobs the same way — they use patch versions (e.g., `4.19.3`) as the `` value but follow the identical naming and analysis pattern. 4. Launch **ALL** agents in a **single message** as **foreground** agents (do NOT use `run_in_background`). Foreground agents in the same message run concurrently — this is just as fast as background agents but keeps your turn active until all complete. 5. Say "Analyzing N jobs in parallel..." in your message text alongside the Agent tool calls. @@ -147,20 +147,20 @@ HTML report generated: /report-lvm-operator-ci-doctor.html ### Z-Stream Integration (Optional) -If a `zstream-summary.json` file exists in the workdir and contains releases with `"status": "skipped"` (same digest already tested), the HTML report will include a **Z-Stream** checkbox on the Periodics tab. When checked, it switches the view to show z-stream release status per version (PR info, image digest, triggered jobs). +Run `zstream-trigger.sh` before doctor to populate z-stream data. For skipped releases (same digest already tested), the trigger script collects failed periodic job artifacts into `/jobs/release--jobs.json` — these are discovered automatically by the finalize step alongside regular releases. -To populate z-stream data before generating the report: +The HTML report includes a **Z-Stream** checkbox on the Periodics tab when `zstream-summary.json` exists in the workdir. + +To populate z-stream data (dry-run — no job triggering): ```text -bash plugins/lvms-ci/scripts/zstream-trigger.sh --dry-run --workdir +bash plugins/lvms-ci/scripts/zstream-trigger.sh --releases --dry-run --workdir ``` -This queries GitHub for open z-stream release PRs on `openshift/lvm-operator`, resolves catalog image digests via quay.io, and writes `zstream-summary.json` to the workdir — without triggering any jobs (dry-run mode). - -To actually trigger z-stream e2e jobs (requires `MY_APPCI_TOKEN`): +To trigger z-stream e2e jobs and collect skipped release artifacts (requires `MY_APPCI_TOKEN`): ```text -bash plugins/lvms-ci/scripts/zstream-trigger.sh --workdir +bash plugins/lvms-ci/scripts/zstream-trigger.sh --releases --workdir ``` ## Related Skills diff --git a/plugins/shared/scripts/doctor.sh b/plugins/shared/scripts/doctor.sh index 3450603a..f87c768d 100755 --- a/plugins/shared/scripts/doctor.sh +++ b/plugins/shared/scripts/doctor.sh @@ -121,72 +121,6 @@ cmd_prepare() { echo " Done: ${jobs_file}" >&2 done - # Collect and download for z-stream "skipped" releases - local zstream_summary="${WORKDIR}/zstream-summary.json" - declare -a ZSTREAM_RELEASES=() - local skipped_releases - skipped_releases=$(jq -r 'to_entries[] | select(.value.status == "skipped") | .key' "${zstream_summary}" 2>/dev/null || true) - - while IFS= read -r zrel; do - [[ -z "${zrel}" ]] && continue - # Extract patch version from PR title (e.g. "chore: Release 4.19.3" → "4.19.3") - local patch_version - patch_version=$(jq -r --arg r "${zrel}" '.[$r].pr_title // ""' "${zstream_summary}" 2>/dev/null \ - | grep -oP '\d+\.\d+\.\d+' || echo "${zrel}-zstream") - - # Skip if already collected as a regular release - if [[ -f "${WORKDIR}/jobs/release-${patch_version}-jobs.json" ]] && \ - [[ "$(jq 'length' "${WORKDIR}/jobs/release-${patch_version}-jobs.json")" -gt 0 ]]; then - echo " Skipping z-stream ${patch_version}: already collected" >&2 - continue - fi - - echo "=== Z-Stream ${zrel} → ${patch_version} (skipped — same digest) ===" >&2 - local zjobs_file="${WORKDIR}/jobs/release-${patch_version}-jobs.json" - - echo " Collecting failed periodic jobs..." >&2 - local zraw_json zraw_err - zraw_err=$(mktemp) - if ! zraw_json=$(bash "${SCRIPT_DIR}/prow-jobs-for-release.sh" "${COMPONENT}" "${zrel}" 2>"${zraw_err}"); then - echo " ERROR: failed to collect z-stream jobs for ${zrel}:" >&2 - local zerr_msg - zerr_msg=$(cat "${zraw_err}") - echo "${zerr_msg}" >&2 - rm -f "${zraw_err}" - echo "[]" > "${zjobs_file}" - release_errors["${patch_version}"]="${zerr_msg:-data collection failed}" - echo "${release_errors["${patch_version}"]}" > "${WORKDIR}/jobs/release-${patch_version}-error.txt" - continue - fi - rm -f "${zraw_err}" - - local zfiltered_json - zfiltered_json=$(echo "${zraw_json}" | jq '[.[] | select(.type == "periodic")]') - - local zcount - zcount=$(echo "${zfiltered_json}" | jq 'length') - - if [[ "${zcount}" -eq 0 ]]; then - echo " No failed periodic jobs found" >&2 - echo "[]" > "${zjobs_file}" - ZSTREAM_RELEASES+=("${patch_version}") - continue - fi - - echo " Found ${zcount} failed periodic jobs, downloading artifacts..." >&2 - local zdl_err - zdl_err=$(mktemp) - echo "${zfiltered_json}" | \ - bash "${SCRIPT_DIR}/download-jobs.sh" --workdir "${WORKDIR}" 2>"${zdl_err}" \ - > "${zjobs_file}" - [[ -s "${zdl_err}" ]] && cat "${zdl_err}" >&2 - rm -f "${zdl_err}" - - total_jobs=$((total_jobs + zcount)) - ZSTREAM_RELEASES+=("${patch_version}") - echo " Done: ${zjobs_file}" >&2 - done <<< "${skipped_releases}" - # Collect and download for rebase PRs if ${do_rebase}; then echo "=== Rebase Pull Requests ===" >&2 @@ -332,22 +266,6 @@ cmd_prepare() { result=$(jq -n --arg w "${WORKDIR}" --argjson rel "${releases_json}" \ '{workdir: $w, releases: $rel}') - # Add z-stream releases to output (uses patch versions, same release-* naming) - if [[ ${#ZSTREAM_RELEASES[@]} -gt 0 ]]; then - local zstream_json="[]" - for zpatch in "${ZSTREAM_RELEASES[@]}"; do - local zjobs_file="${WORKDIR}/jobs/release-${zpatch}-jobs.json" - local zcount=0 - if [[ -f "${zjobs_file}" ]]; then - zcount=$(jq 'length' "${zjobs_file}") - fi - zstream_json=$(echo "${zstream_json}" | jq \ - --arg r "${zpatch}" --argjson c "${zcount}" --arg f "${zjobs_file}" \ - '. + [{release: $r, jobs: $c, jobs_file: $f}]') - done - result=$(echo "${result}" | jq --argjson z "${zstream_json}" '. + {zstream: $z}') - fi - if ${do_rebase}; then local prs_file="${WORKDIR}/jobs/prs-jobs.json" local pr_job_count=0 @@ -411,7 +329,7 @@ cmd_finalize() { IFS=',' read -ra RELEASES <<< "${releases_arg}" - # Aggregate all job files (regular releases + z-stream patch versions) + # Aggregate all job files local all_job_files all_job_files=$(find "${WORKDIR}/jobs" -name 'release-*-jobs.json' 2>/dev/null | sort || true) while IFS= read -r jobs_file; do From 186deb2c47d7af980605d46f5b45e54544d410ee Mon Sep 17 00:00:00 2001 From: Kasturi Narra Date: Wed, 8 Jul 2026 16:22:36 +0530 Subject: [PATCH 09/14] move z-stream results from Periodics tab to Pull Requests tab Z-stream entries with PRs now display alongside rebase PRs in the Pull Requests tab using release version headers (e.g. "Release 4.18.5"). Entries without PRs (no_pr, completed) are omitted. Removes the z-stream checkbox, filterZstream JS, and zstream-section CSS from the Periodics tab. Co-Authored-By: Claude Opus 4.6 --- plugins/lvms-ci/skills/doctor/SKILL.md | 2 +- plugins/shared/scripts/create-report.py | 133 +++++++++++++----------- 2 files changed, 72 insertions(+), 63 deletions(-) diff --git a/plugins/lvms-ci/skills/doctor/SKILL.md b/plugins/lvms-ci/skills/doctor/SKILL.md index e68fc8b1..0986c41a 100644 --- a/plugins/lvms-ci/skills/doctor/SKILL.md +++ b/plugins/lvms-ci/skills/doctor/SKILL.md @@ -149,7 +149,7 @@ HTML report generated: /report-lvm-operator-ci-doctor.html Run `zstream-trigger.sh` before doctor to populate z-stream data. For skipped releases (same digest already tested), the trigger script collects failed periodic job artifacts into `/jobs/release--jobs.json` — these are discovered automatically by the finalize step alongside regular releases. -The HTML report includes a **Z-Stream** checkbox on the Periodics tab when `zstream-summary.json` exists in the workdir. +The HTML report includes z-stream results in the **Pull Requests** tab when `zstream-summary.json` exists in the workdir. To populate z-stream data (dry-run — no job triggering): diff --git a/plugins/shared/scripts/create-report.py b/plugins/shared/scripts/create-report.py index ac4a5a4f..9d64ecef 100755 --- a/plugins/shared/scripts/create-report.py +++ b/plugins/shared/scripts/create-report.py @@ -195,13 +195,8 @@ if (el) el.style.display = el.style.display === 'none' ? 'block' : 'none'; } function filterToday(on) { - var zstreamCheck = document.getElementById('filter-zstream'); - if (on && zstreamCheck && zstreamCheck.checked) { - zstreamCheck.checked = false; - if (typeof filterZstream === 'function') filterZstream(false); - } var today = new Date().toISOString().split('T')[0]; - document.querySelectorAll('#tab-periodics .release-section:not(.zstream-section) .issue-row').forEach(function(row) { + document.querySelectorAll('#tab-periodics .release-section .issue-row').forEach(function(row) { var dates = (row.getAttribute('data-dates') || '').split(' '); var show = !on || dates.indexOf(today) !== -1; row.style.display = show ? '' : 'none'; @@ -211,7 +206,7 @@ detail.style.display = show ? '' : 'none'; } }); - document.querySelectorAll('#tab-periodics .release-section:not(.zstream-section)').forEach(function(sec) { + document.querySelectorAll('#tab-periodics .release-section').forEach(function(sec) { var id = sec.id.replace('release-', ''); var rows = sec.querySelectorAll('.issue-row'); var total = 0, bd = {build: 0, test: 0, infra: 0}; @@ -1224,8 +1219,8 @@ def render_release_section(version, rdata, bug_candidates, index_info=None, zstr m = re.search(r'(\d+\.\d+\.\d+)', pr_title) if pr_title else None if m: display = m.group(1) - css_class = "release-section zstream-section" if is_zstream else "release-section" - style = ' style="display:none"' if is_zstream else "" + css_class = "release-section" + style = "" if rdata is None: return ( @@ -1316,7 +1311,7 @@ def render_release_section(version, rdata, bug_candidates, index_info=None, zstr return "\n".join(lines) -def render_pr_section(pr_data, bug_candidates, pr_status, pr_error=None): +def render_pr_section(pr_data, bug_candidates, pr_status, pr_error=None, zstream_data=None): """Render the Pull Requests tab. pr_data: analyzed PR summary (from aggregate), may be None. @@ -1372,7 +1367,7 @@ def render_pr_section(pr_data, bug_candidates, pr_status, pr_error=None): "analysis": pr, }) - if not all_prs: + if not all_prs and not zstream_data: return ( '
\n' '
\n' @@ -1400,6 +1395,35 @@ def render_pr_section(pr_data, bug_candidates, pr_status, pr_error=None): f'
  • PR# {pr["number"]}' f' — {pr["failed"]} failures ({b.get("build", 0)} build, {b.get("test", 0)} test, {b.get("infrastructure", 0)} infra){suffix}
    • ' ) + if zstream_data: + sorted_zversions = sorted( + zstream_data.keys(), + key=lambda v: [int(x) for x in v.split(".")], + ) + for zv in sorted_zversions: + zinfo = zstream_data[zv] + zstatus = zinfo.get("status", "unknown") + if not zinfo.get("pr"): + continue + pr_title = zinfo.get("pr_title", "") + m = re.search(r'(\d+\.\d+\.\d+)', pr_title) if pr_title else None + display = m.group(1) if m else zv + if zstatus == "skipped": + rdata = zinfo.get("_analyzed") + total = rdata["total_failed"] if rdata else 0 + b = rdata.get("breakdown", {"build": 0, "test": 0, "infrastructure": 0}) if rdata else {"build": 0, "test": 0, "infrastructure": 0} + fail_word = "failure" if total == 1 else "failures" + toc_lines.append( + f'
  • Release {_e(display)}' + f' — {total} {fail_word} ({b["build"]} build, {b["test"]} test, {b["infrastructure"]} infra)
    • ' + ) + elif zstatus == "triggered": + njobs = zinfo.get("jobs_triggered", 0) + toc_lines.append( + f'
  • Release {_e(display)}' + f' — {njobs} jobs running
    • ' + ) + toc_lines.append(' ') toc_lines.append('
    ') @@ -1476,11 +1500,43 @@ def render_pr_section(pr_data, bug_candidates, pr_status, pr_error=None): lines.append(' ') lines.append("
    ") + + if zstream_data: + sorted_zversions = sorted( + zstream_data.keys(), + key=lambda v: [int(x) for x in v.split(".")], + ) + for zv in sorted_zversions: + zinfo = zstream_data[zv] + zstatus = zinfo.get("status", "unknown") + if not zinfo.get("pr"): + continue + if zstatus == "skipped": + rdata = zinfo.get("_analyzed") + empty_bd = {"build": 0, "test": 0, "infrastructure": 0} + if rdata is None: + rdata = {"total_failed": 0, "issues": [], "breakdown": empty_bd} + section_id = f"{zv}-zstream" + lines.append(render_release_section(section_id, rdata, bug_candidates, zstream_info=zinfo)) + elif zstatus == "triggered": + pr_title = zinfo.get("pr_title", "") + m = re.search(r'(\d+\.\d+\.\d+)', pr_title) if pr_title else None + display = m.group(1) if m else zv + njobs = zinfo.get("jobs_triggered", 0) + section_id = f"zstream-{zv}" + lines.append(f'
    ') + lines.append('
    ') + lines.append(f'

    Release {_e(display)}🔗

    ') + lines.append(f' {njobs} jobs running') + lines.append("
    ") + lines.append(_render_zstream_info(zv, zinfo)) + lines.append("
    ") + return "\n".join(toc_lines) + "\n\n" + "\n".join(lines) # --------------------------------------------------------------------------- -# Z-Stream sections (rendered inside Periodics tab, toggled via checkbox) +# Z-stream metadata rendering # --------------------------------------------------------------------------- @@ -1540,7 +1596,7 @@ def generate_html(component_title, releases_data, all_bug_candidates, pr_data, p cards.append( '
    \n' f'
    {pr_failed_count}
    \n' - f'
    Rebase PRs
    \n' + f'
    Pull Requests
    \n' "
    " ) @@ -1566,57 +1622,11 @@ def generate_html(component_title, releases_data, all_bug_candidates, pr_data, p for version, rdata in releases_data.items(): sections.append(render_release_section(version, rdata, all_bug_candidates, _idx.get(version))) - # Append z-stream sections using the same render_release_section path - zstream_checkbox = "" - if zstream_data: - sorted_zversions = sorted( - [zv for zv, zi in zstream_data.items() if zi.get("status") == "skipped"], - key=lambda v: [int(x) for x in v.split(".")], - ) - if sorted_zversions: - zstream_checkbox = ( - ' ' - ) - for zv in sorted_zversions: - zinfo = zstream_data[zv] - empty_bd = {"build": 0, "test": 0, "infrastructure": 0} - rdata = zinfo.get("_analyzed", {"total_failed": 0, "issues": [], "breakdown": empty_bd}) - section_id = f"{zv}-zstream" - pr_title = zinfo.get("pr_title", "") - m = re.search(r'(\d+\.\d+\.\d+)', pr_title) if pr_title else None - display = m.group(1) if m else section_id - total = rdata["total_failed"] if rdata else 0 - b = rdata.get("breakdown", empty_bd) if rdata else empty_bd - fail_word = "failure" if total == 1 else "failures" - toc.append( - f' ' - ) - sections.append(render_release_section(section_id, rdata, all_bug_candidates, zstream_info=zinfo)) - - pr_section = render_pr_section(pr_data, all_bug_candidates, pr_status, pr_error) + pr_section = render_pr_section(pr_data, all_bug_candidates, pr_status, pr_error, zstream_data) bugs_section = render_bugs_section(bugs_tab_data) if bugs_tab_data else "" images_section = render_images_section(images_tab_data) - zstream_js = "" - if zstream_checkbox: - zstream_js = """ -function filterZstream(on) { - var todayCheck = document.getElementById('filter-today'); - if (on && todayCheck && todayCheck.checked) { - todayCheck.checked = false; - filterToday(false); - } - document.querySelectorAll('#tab-periodics .toc > ul > li:not(.toc-zstream)').forEach(function(el) { el.style.display = on ? 'none' : ''; }); - document.querySelectorAll('#tab-periodics .toc > ul > li.toc-zstream').forEach(function(el) { el.style.display = on ? '' : 'none'; }); - document.querySelectorAll('#tab-periodics .release-section:not(.zstream-section)').forEach(function(el) { el.style.display = on ? 'none' : ''; }); - document.querySelectorAll('#tab-periodics .zstream-section').forEach(function(el) { el.style.display = on ? '' : 'none'; }); -}""" - js_content = JS + zstream_js + js_content = JS return f"""\ @@ -1651,7 +1661,6 @@ def generate_html(component_title, releases_data, all_bug_candidates, pr_data, p

    Table of Contents

    -{zstream_checkbox}
      From f7ac1b0b7c1ee395117ac67164753a1a140ad114 Mon Sep 17 00:00:00 2001 From: Kasturi Narra Date: Thu, 9 Jul 2026 13:48:52 +0530 Subject: [PATCH 10/14] Replace z-stream trigger with Prow presubmit PR collection MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Remove zstream-trigger.sh (Gangway API triggering) — z-stream tests now run as Prow presubmits on release-management branch. Add LVMS-specific prow-jobs-for-pull-requests.sh to collect open PRs and their presubmit results from GCS. Make doctor.sh component-aware for PR collection (--author for MicroShift, --base for LVMS) and add --release-prs alias. Remove all z-stream rendering from create-report.py (keeping _pick_zstream_tag used by Images tab). Co-Authored-By: Claude Opus 4.6 --- .../scripts/prow-jobs-for-pull-requests.sh | 441 ++++++++++++++ plugins/lvms-ci/scripts/zstream-trigger.sh | 553 ------------------ plugins/lvms-ci/skills/doctor/SKILL.md | 29 +- plugins/shared/scripts/create-report.py | 130 +--- plugins/shared/scripts/doctor.sh | 16 +- 5 files changed, 465 insertions(+), 704 deletions(-) create mode 100755 plugins/lvms-ci/scripts/prow-jobs-for-pull-requests.sh delete mode 100755 plugins/lvms-ci/scripts/zstream-trigger.sh diff --git a/plugins/lvms-ci/scripts/prow-jobs-for-pull-requests.sh b/plugins/lvms-ci/scripts/prow-jobs-for-pull-requests.sh new file mode 100755 index 00000000..0caf1a63 --- /dev/null +++ b/plugins/lvms-ci/scripts/prow-jobs-for-pull-requests.sh @@ -0,0 +1,441 @@ +#!/usr/bin/bash +set -euo pipefail + +# Prow Jobs for Pull Requests (LVMS) +# Data modes (summary, detail): JSON on stdout +# Action modes (approve, restart): text on stdout +# Progress/errors: stderr + +GCS_API="https://storage.googleapis.com/storage/v1/b/test-platform-results/o" +GCS_BASE="https://storage.googleapis.com/test-platform-results" +PROW_VIEW="https://prow.ci.openshift.org/view/gs/test-platform-results" +GH_REPO="openshift/lvm-operator" +GCS_PR_PREFIX="pr-logs/pull/openshift_lvm-operator" +SIGNATURE=$'\n'"*Added by $(basename "${0}")* :robot:"$'\n' +SCRIPT_TMPDIR=$(mktemp -d) +trap 'rm -rf "${SCRIPT_TMPDIR}"' EXIT + +# Get open PRs as JSON array +# Args: filter, author, base, extra_fields (comma-separated, appended to default fields) +fetch_open_prs() { + local filter="${1:-}" + local author="${2:-}" + local base="${3:-}" + local extra_fields="${4:-}" + local fields="number,title,url" + [[ -n "${extra_fields}" ]] && fields+=",${extra_fields}" + local -a gh_args=(--repo "${GH_REPO}" --state open --limit 100 --json "${fields}") + + [[ -n "${author}" ]] && gh_args+=(--author "${author}") + [[ -n "${base}" ]] && gh_args+=(--base "${base}") + + local pr_data + pr_data=$(gh pr list "${gh_args[@]}") + + if [[ -n "${filter}" ]]; then + echo "${pr_data}" | jq -c --arg f "${filter}" '[.[] | select(.title | contains($f))]' + else + echo "${pr_data}" + fi +} + +# List job names for a PR from GCS +list_pr_jobs() { + local pr="${1}" + curl -s --max-time 60 --retry 3 --retry-delay 5 "${GCS_API}?prefix=${GCS_PR_PREFIX}/${pr}/&delimiter=/" | \ + jq -r '.prefixes[]? // empty' | \ + sed "s|${GCS_PR_PREFIX}/${pr}/||; s|/$||" +} + +# Get latest build result as a JSON object +# Returns PENDING status for jobs still running (no finished.json). +get_latest_build() { + local pr="${1}" job="${2}" + local build_id finished_json + + build_id=$(curl -s --max-time 60 --retry 3 --retry-delay 5 "${GCS_BASE}/${GCS_PR_PREFIX}/${pr}/${job}/latest-build.txt" 2>/dev/null) || return 1 + [[ -z "${build_id}" || "${build_id}" == *"<"* ]] && return 1 + + local url="${PROW_VIEW}/pr-logs/pull/openshift_lvm-operator/${pr}/${job}/${build_id}" + + finished_json=$(curl -s --max-time 60 --retry 3 --retry-delay 5 "${GCS_BASE}/${GCS_PR_PREFIX}/${pr}/${job}/${build_id}/finished.json" 2>/dev/null) + + if [[ -z "${finished_json}" || "${finished_json}" == *"NoSuchKey"* || "${finished_json}" == *"<"* ]]; then + jq -nc --arg job "${job}" --arg url "${url}" --arg build_id "${build_id}" \ + '{job: $job, status: "PENDING", url: $url, build_id: $build_id, finished: null}' + return 0 + fi + + echo "${finished_json}" | jq -c \ + --arg job "${job}" --arg url "${url}" --arg build_id "${build_id}" \ + '{ + job: $job, + status: (.result // "PENDING"), + url: $url, + build_id: $build_id, + finished: (if (.timestamp // 0) > 0 then .timestamp | todate else null end) + }' +} + +# Fetch job results for a single PR into temp dir (parallelized) +# Skips individual jobs that fail to fetch (e.g. no latest-build.txt). +fetch_pr_results() { + local pr="${1}" + local tmpdir="${2}" + local jobs + + jobs=$(list_pr_jobs "${pr}") || return 1 + [[ -z "${jobs}" ]] && return 0 + + while IFS= read -r job; do + ( + result=$(get_latest_build "${pr}" "${job}" 2>/dev/null) || exit 0 + if [[ -n "${result}" ]]; then + echo "${result}" > "${tmpdir}/${job}.json" + fi + ) & + done <<< "${jobs}" + wait + return 0 +} + +# Collect per-job JSON files into a single JSON array +collect_jobs_json() { + local tmpdir="${1}" + local files=("${tmpdir}"/*.json) + if [[ ! -f "${files[0]}" ]]; then + echo "[]" + return + fi + cat "${files[@]}" | jq -s '.' +} + +# Summary mode: JSON array of PRs with pass/fail counts +mode_summary() { + local filter="${1:-}" author="${2:-}" base="${3:-}" + local pr_data output_tmp + + echo "Fetching open PRs..." >&2 + pr_data=$(fetch_open_prs "${filter}" "${author}" "${base}") + [[ "$(echo "${pr_data}" | jq 'length')" -eq 0 ]] && { echo "[]"; return; } + + echo "Fetching job results..." >&2 + local output_tmp="${SCRIPT_TMPDIR}/summary-output" + echo -n "" > "${output_tmp}" + + while IFS=$'\t' read -r pr_number pr_title pr_url; do + local tmpdir="${SCRIPT_TMPDIR}/summary-pr-${pr_number}" + mkdir -p "${tmpdir}" + + if ! fetch_pr_results "${pr_number}" "${tmpdir}"; then + echo "PR #${pr_number}: incomplete job results, skipping" >&2 + continue + fi + + local passed=0 failed=0 other=0 total=0 + for f in "${tmpdir}"/*.json; do + [[ -f "${f}" ]] || continue + local status + status=$(jq -r '.status' "${f}") + total=$((total + 1)) + case "${status}" in + SUCCESS) passed=$((passed + 1)) ;; + FAILURE) failed=$((failed + 1)) ;; + *) other=$((other + 1)) ;; + esac + done + + jq -nc --argjson n "${pr_number}" --arg t "${pr_title}" --arg u "${pr_url}" \ + --argjson p "${passed}" --argjson f "${failed}" \ + --argjson o "${other}" --argjson to "${total}" \ + '{pr_number: $n, title: $t, url: $u, passed: $p, failed: $f, other: $o, total: $to}' \ + >> "${output_tmp}" + done < <(echo "${pr_data}" | jq -r '.[] | [.number, .title, .url] | @tsv') + + jq -s '.' "${output_tmp}" +} + +# Detail mode: JSON array of PRs with full job lists +mode_detail() { + local filter="${1:-}" author="${2:-}" base="${3:-}" + local pr_data output_tmp + + echo "Fetching open PRs..." >&2 + pr_data=$(fetch_open_prs "${filter}" "${author}" "${base}") + [[ "$(echo "${pr_data}" | jq 'length')" -eq 0 ]] && { echo "[]"; return; } + + echo "Fetching job results..." >&2 + local output_tmp="${SCRIPT_TMPDIR}/detail-output" + echo -n "" > "${output_tmp}" + + while IFS=$'\t' read -r pr_number pr_title pr_url; do + local tmpdir="${SCRIPT_TMPDIR}/detail-pr-${pr_number}" + mkdir -p "${tmpdir}" + + if ! fetch_pr_results "${pr_number}" "${tmpdir}"; then + echo "PR #${pr_number}: incomplete job results, skipping" >&2 + continue + fi + + local jobs_json + jobs_json=$(collect_jobs_json "${tmpdir}") + + jq -nc --argjson n "${pr_number}" --arg t "${pr_title}" --arg u "${pr_url}" \ + --argjson jobs "${jobs_json}" \ + '{pr_number: $n, title: $t, url: $u, jobs: $jobs}' >> "${output_tmp}" + done < <(echo "${pr_data}" | jq -r '.[] | [.number, .title, .url] | @tsv') + + jq -s '.' "${output_tmp}" +} + +# Approve mode: add /verified to PRs where all jobs pass +mode_approve() { + local filter="${1:-}" author="${2:-}" base="${3:-}" execute="${4:-false}" + local pr_data had_error=0 + + ${execute} || echo "[DRY-RUN] Use --execute to actually post comments" >&2 + echo "Fetching open PRs..." >&2 + pr_data=$(fetch_open_prs "${filter}" "${author}" "${base}") + [[ "$(echo "${pr_data}" | jq 'length')" -eq 0 ]] && { echo "No open pull requests found."; return; } + + echo "Fetching job results..." >&2 + + while IFS=$'\t' read -r pr_number pr_title pr_url; do + local tmpdir="${SCRIPT_TMPDIR}/approve-pr-${pr_number}" + mkdir -p "${tmpdir}" + + if ! fetch_pr_results "${pr_number}" "${tmpdir}"; then + echo "PR #${pr_number}: incomplete job results, skipping" + continue + fi + + local total=0 success=0 + for f in "${tmpdir}"/*.json; do + [[ -f "${f}" ]] || continue + local status + status=$(jq -r '.status' "${f}") + total=$((total + 1)) + [[ "${status}" == "SUCCESS" ]] && success=$((success + 1)) + done + + if [[ "${total}" -eq 0 ]]; then + echo "PR #${pr_number}: No jobs found, skipping" + continue + fi + + if [[ "${success}" -eq "${total}" ]]; then + local comment=$'/verified by ci\n' + comment+="${SIGNATURE}" + echo "PR #${pr_number}: All ${total} jobs passed, approving..." + if ${execute}; then + if gh pr comment "${pr_number}" --repo "${GH_REPO}" --body "${comment}"; then + echo "PR #${pr_number}: Approved" + else + echo "PR #${pr_number}: Failed to post approve comment" >&2 + had_error=1 + fi + else + echo "gh pr comment ${pr_number} --repo ${GH_REPO} --body '/verified by ci ...'" + fi + else + echo "PR #${pr_number}: ${success}/${total} jobs passed, skipping" + fi + done < <(echo "${pr_data}" | jq -r '.[] | [.number, .title, .url] | @tsv') + + return "${had_error}" +} + +# Restart mode: comment /test for each failed job +mode_restart() { + local filter="${1:-}" author="${2:-}" base="${3:-}" execute="${4:-false}" + local pr_data had_error=0 + + ${execute} || echo "[DRY-RUN] Use --execute to actually post comments" >&2 + echo "Fetching open PRs..." >&2 + pr_data=$(fetch_open_prs "${filter}" "${author}" "${base}") + [[ "$(echo "${pr_data}" | jq 'length')" -eq 0 ]] && { echo "No open pull requests found."; return; } + + echo "Fetching job results..." >&2 + + while IFS=$'\t' read -r pr_number pr_title pr_url; do + local tmpdir="${SCRIPT_TMPDIR}/restart-pr-${pr_number}" + mkdir -p "${tmpdir}" + + if ! fetch_pr_results "${pr_number}" "${tmpdir}"; then + echo "PR #${pr_number}: incomplete job results, skipping" + continue + fi + + local failed_jobs=() + for f in "${tmpdir}"/*.json; do + [[ -f "${f}" ]] || continue + local job status + job=$(jq -r '.job' "${f}") + status=$(jq -r '.status' "${f}") + [[ "${status}" == "FAILURE" ]] && failed_jobs+=("${job}") + done + + if [[ ${#failed_jobs[@]} -eq 0 ]]; then + echo "PR #${pr_number}: No failed jobs, skipping" + continue + fi + + local comment="" + for job in "${failed_jobs[@]}"; do + local build_id short_name + build_id=$(curl -s --max-time 60 --retry 3 --retry-delay 5 "${GCS_BASE}/${GCS_PR_PREFIX}/${pr_number}/${job}/latest-build.txt" 2>/dev/null) || continue + short_name=$(curl -s --max-time 60 --retry 3 --retry-delay 5 "${GCS_BASE}/${GCS_PR_PREFIX}/${pr_number}/${job}/${build_id}/prowjob.json" 2>/dev/null | \ + jq -r '.spec.rerun_command // empty' 2>/dev/null | sed 's|^/test ||') || short_name="" + short_name=$(echo "${short_name}" | xargs) + [[ -z "${short_name}" ]] && continue + comment+="/test ${short_name}"$'\n' + done + + if [[ -z "${comment}" ]]; then + echo "PR #${pr_number}: Could not resolve rerun commands for failed job(s), skipping" + continue + fi + + comment+="${SIGNATURE}" + echo "PR #${pr_number}: Restarting ${#failed_jobs[@]} failed job(s): ${failed_jobs[*]}" + if ${execute}; then + if gh pr comment "${pr_number}" --repo "${GH_REPO}" --body "${comment}"; then + echo "PR #${pr_number}: Restart comment posted" + else + echo "PR #${pr_number}: Failed to post restart comment" >&2 + had_error=1 + fi + else + echo "gh pr comment ${pr_number} --repo ${GH_REPO} --body '/test ...'" + fi + done < <(echo "${pr_data}" | jq -r '.[] | [.number, .title, .url] | @tsv') + + return "${had_error}" +} + +# Close-duplicates mode: close older PRs superseded by newer ones. +# Groups PRs by target branch. Within each group, keeps the newest +# PR (highest number) and closes older ones. +mode_close_duplicates() { + local filter="${1:-}" author="${2:-}" base="${3:-}" execute="${4:-false}" had_error=0 + + if [[ -z "${filter}" || -z "${author}" ]]; then + echo "Error: --filter and --author are required for close-duplicates mode" >&2 + return 1 + fi + + ${execute} || echo "[DRY-RUN] Use --execute to actually close PRs" >&2 + + echo "Fetching open PRs (author: ${author}, filter: ${filter})..." >&2 + local pr_data + pr_data=$(fetch_open_prs "${filter}" "${author}" "${base}" "baseRefName") + + [[ "$(echo "${pr_data}" | jq 'length')" -eq 0 ]] && { echo "No matching PRs found."; return; } + + local groups + groups=$(echo "${pr_data}" | jq -c ' + group_by(.baseRefName) + | map(select(length > 1) | sort_by(.number) | reverse) + ') + + local group_count + group_count=$(echo "${groups}" | jq 'length') + + if [[ "${group_count}" -eq 0 ]]; then + echo "No duplicate PRs found." + return + fi + + for i in $(seq 0 $((group_count - 1))); do + local group newest_number newest_title + group=$(echo "${groups}" | jq -c ".[$i]") + newest_number=$(echo "${group}" | jq '.[0].number') + newest_title=$(echo "${group}" | jq -r '.[0].title') + local base_ref + base_ref=$(echo "${group}" | jq -r '.[0].baseRefName') + + echo "${base_ref}: keeping PR #${newest_number} (${newest_title})" + + local dup_count + dup_count=$(echo "${group}" | jq 'length - 1') + + for j in $(seq 1 "${dup_count}"); do + local dup_number dup_title + dup_number=$(echo "${group}" | jq ".[$j].number") + dup_title=$(echo "${group}" | jq -r ".[$j].title") + + local comment="Closing as duplicate: superseded by #${newest_number}." + comment+=$'\n'"/close" + comment+="${SIGNATURE}" + + echo "Closing PR #${dup_number} (${dup_title})..." + if ${execute}; then + if gh pr comment "${dup_number}" --repo "${GH_REPO}" --body "${comment}"; then + echo "PR #${dup_number}: Close comment posted" + else + echo "PR #${dup_number}: Failed to post close comment" >&2 + had_error=1 + fi + else + echo "gh pr comment ${dup_number} --repo ${GH_REPO} --body 'Closing as duplicate: superseded by #${newest_number}. ...'" + fi + done + done + + return "${had_error}" +} + +usage() { + echo "Usage: ${0} [--mode MODE] [--filter STRING] [--author USER] [--base BRANCH] [--execute]" >&2 + echo " --mode MODE: Operation mode (default: summary)" >&2 + echo " summary: JSON array of PRs with pass/fail counts" >&2 + echo " detail: JSON array of PRs with full job lists" >&2 + echo " approve: Approve PRs where ALL test jobs passed (dry-run by default)" >&2 + echo " restart: Restart failed test jobs by commenting /test (dry-run by default)" >&2 + echo " close-duplicates: Close older PRs with same target branch and title filter (dry-run by default)" >&2 + echo " --filter STRING: Only include PRs whose title contains STRING" >&2 + echo " --author USER: Only include PRs authored by USER" >&2 + echo " --base BRANCH: Only include PRs targeting BRANCH" >&2 + echo " --execute: Actually post comments/close PRs (action modes)" >&2 + exit 1 +} + +main() { + local mode="summary" + local filter="" + local author="" + local base="" + local execute=false + + while [[ ${#} -gt 0 ]]; do + case "${1}" in + --mode) + [[ ${#} -lt 2 ]] && { echo "Error: mode requires an argument" >&2; usage; } + mode="${2}"; shift 2 ;; + --filter) + [[ ${#} -lt 2 ]] && { echo "Error: filter requires an argument" >&2; usage; } + filter="${2}"; shift 2 ;; + --author) + [[ ${#} -lt 2 ]] && { echo "Error: author requires an argument" >&2; usage; } + author="${2}"; shift 2 ;; + --base) + [[ ${#} -lt 2 ]] && { echo "Error: base requires an argument" >&2; usage; } + base="${2}"; shift 2 ;; + --execute) + execute=true; shift ;; + -*) echo "Unknown option: ${1}" >&2; usage ;; + *) echo "Unknown argument: ${1}" >&2; usage ;; + esac + done + + case "${mode}" in + summary) mode_summary "${filter}" "${author}" "${base}" ;; + detail) mode_detail "${filter}" "${author}" "${base}" ;; + approve) mode_approve "${filter}" "${author}" "${base}" "${execute}" ;; + restart) mode_restart "${filter}" "${author}" "${base}" "${execute}" ;; + close-duplicates) mode_close_duplicates "${filter}" "${author}" "${base}" "${execute}" ;; + *) echo "Error: Unknown mode '${mode}'" >&2; usage ;; + esac +} + +main "${@}" diff --git a/plugins/lvms-ci/scripts/zstream-trigger.sh b/plugins/lvms-ci/scripts/zstream-trigger.sh deleted file mode 100755 index f0196a02..00000000 --- a/plugins/lvms-ci/scripts/zstream-trigger.sh +++ /dev/null @@ -1,553 +0,0 @@ -#!/usr/bin/bash -set -euo pipefail - -# Trigger LVMS z-stream e2e tests via Gangway API. -# -# For each configured release, checks GitHub for open z-stream PRs on -# openshift/lvm-operator, resolves the catalog image digest from the PR -# diff via the quay.io API, and triggers existing nightly e2e jobs with -# MULTISTAGE_PARAM_OVERRIDE_LVM_INDEX_IMAGE set to the resolved digest. -# -# Skips releases with no open PR or where the digest hasn't changed since -# the last run (tracked via a state file). -# -# Usage: -# zstream-trigger.sh --releases 4.14,4.16,... [--dry-run] [--workdir /path] -# [--force] [--token-file /path/to/token] -# -# Environment: -# GITHUB_TOKEN Optional. GitHub PAT for higher API rate limits. -# MY_APPCI_TOKEN Required (unless --dry-run). Gangway API bearer token. - -RELEASES="" -DRY_RUN=false -FORCE=false -WORKDIR="" -TOKEN_FILE="" - -TRIGGER_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) -COMPONENT="lvm-operator" - -GANGWAY_API="https://gangway-ci.apps.ci.l2s4.p1.openshiftapps.com" -GITHUB_API="https://api.github.com" -QUAY_API="https://quay.io/api/v1" -QUAY_REPO="redhat-user-workloads/logical-volume-manag-tenant/lvm-operator-catalog" -STATE_DIR="${HOME}/.cache/lvms-zstream" -STATE_FILE="${STATE_DIR}/last-tested.json" - -NIGHTLY_JOBS=( - "e2e-aws-sno-qe-integration-tests" - "e2e-aws-sno-arm-qe-integration-tests" - "e2e-aws-mno-qe-integration-tests" - "e2e-aws-mno-arm-qe-integration-tests" - "e2e-baremetalds-sno-dualstack-qe-integration-tests" - "e2e-baremetalds-mno-dualstack-qe-integration-tests" -) - -# --------------------------------------------------------------------------- -# Helpers -# --------------------------------------------------------------------------- - -log() { echo "[$(date +%H:%M:%S)] $*" >&2; } -info() { log "INFO $*"; } -warn() { log "WARN $*"; } -err() { log "ERROR $*"; } - -github_curl() { - local -a headers=() - if [[ -n "${GITHUB_TOKEN:-}" ]]; then - headers+=(-H "Authorization: token ${GITHUB_TOKEN}") - fi - curl -sSL --connect-timeout 10 --max-time 30 "${headers[@]}" "$@" -} - -usage() { - cat >&2 <) - --token-file F Read MY_APPCI_TOKEN from file instead of environment - -h, --help Show this help -EOF - exit 1 -} - -# --------------------------------------------------------------------------- -# Parse arguments -# --------------------------------------------------------------------------- - -while [[ ${#} -gt 0 ]]; do - case "${1}" in - --releases) [[ ${#} -ge 2 ]] || { err "--releases requires an argument"; usage; } - RELEASES="${2}"; shift 2 ;; - --dry-run) DRY_RUN=true; shift ;; - --force) FORCE=true; shift ;; - --workdir) [[ ${#} -ge 2 ]] || { err "--workdir requires an argument"; usage; } - WORKDIR="${2}"; shift 2 ;; - --token-file) [[ ${#} -ge 2 ]] || { err "--token-file requires an argument"; usage; } - TOKEN_FILE="${2}"; shift 2 ;; - -h|--help) usage ;; - *) err "Unknown option: ${1}"; usage ;; - esac -done - -if [[ -z "${RELEASES}" ]]; then - err "--releases is required" - usage -fi - -WORKDIR="${WORKDIR:-/tmp/lvms-zstream.$(date +%y%m%d)}" -mkdir -p "${WORKDIR}" - -if [[ -n "${TOKEN_FILE}" && -f "${TOKEN_FILE}" ]]; then - MY_APPCI_TOKEN=$(cat "${TOKEN_FILE}") -fi - -if ! ${DRY_RUN} && [[ -z "${MY_APPCI_TOKEN:-}" ]]; then - err "MY_APPCI_TOKEN is required (set env or use --token-file)" - exit 1 -fi - -# --------------------------------------------------------------------------- -# State management — track last-tested digest per release -# --------------------------------------------------------------------------- - -load_state() { - mkdir -p "${STATE_DIR}" - if [[ ! -f "${STATE_FILE}" ]]; then - echo '{}' > "${STATE_FILE}" - fi -} - -get_last_digest() { - local release="$1" - jq -r --arg r "${release}" '.[$r].digest // ""' "${STATE_FILE}" -} - -get_last_pr() { - local release="$1" - jq -r --arg r "${release}" '.[$r].pr // ""' "${STATE_FILE}" -} - -save_digest() { - local release="$1" digest="$2" pr_number="$3" image="$4" - local tmp - tmp=$(mktemp) - jq --arg r "${release}" \ - --arg d "${digest}" \ - --arg p "${pr_number}" \ - --arg i "${image}" \ - --arg t "$(date -Iseconds)" \ - '.[$r] = {digest: $d, pr: $p, image: $i, tested_at: $t}' \ - "${STATE_FILE}" > "${tmp}" && mv "${tmp}" "${STATE_FILE}" -} - -# --------------------------------------------------------------------------- -# GitHub: find open z-stream PR for a release -# --------------------------------------------------------------------------- - -find_zstream_pr() { - local release="$1" - local major minor - major=$(echo "${release}" | cut -d. -f1) - minor=$(echo "${release}" | cut -d. -f2) - local catalog_prefix="lvm-operator-catalog-${major}-${minor}" - - local prs_json - prs_json=$(github_curl "${GITHUB_API}/repos/openshift/lvm-operator/pulls?state=open&base=release-management&per_page=100") - - local pr_count - pr_count=$(echo "${prs_json}" | jq 'length') - - local i - for ((i = 0; i < pr_count; i++)); do - local pr_number pr_title - pr_number=$(echo "${prs_json}" | jq -r ".[$i].number") - pr_title=$(echo "${prs_json}" | jq -r ".[$i].title") - - local files_json snapshot - files_json=$(github_curl "${GITHUB_API}/repos/openshift/lvm-operator/pulls/${pr_number}/files") - snapshot=$(echo "${files_json}" | jq -r ' - [.[] | select(.filename | contains("catalog")) | .patch // ""] | - join("\n") - ' | { grep -oP "(?<=snapshot: )${catalog_prefix}\\S*" || true; } | head -1) - - if [[ -n "${snapshot}" ]]; then - printf '%s\t%s\t%s\n' "${pr_number}" "${pr_title}" "${snapshot}" - return 0 - fi - done -} - -# --------------------------------------------------------------------------- -# GitHub: check if a previously tracked PR was merged -# --------------------------------------------------------------------------- - -check_pr_merged() { - local pr_number="$1" - local pr_json - pr_json=$(github_curl "${GITHUB_API}/repos/openshift/lvm-operator/pulls/${pr_number}") - - local state merged_at - state=$(echo "${pr_json}" | jq -r '.state // ""') - merged_at=$(echo "${pr_json}" | jq -r '.merged_at // ""') - - if [[ "${state}" == "closed" && -n "${merged_at}" && "${merged_at}" != "null" ]]; then - echo "merged" - fi -} - -# --------------------------------------------------------------------------- -# Quay: resolve snapshot to digest -# --------------------------------------------------------------------------- - -resolve_snapshot_to_digest() { - local snapshot="$1" - - # Parse version prefix and timestamp from snapshot name - # Format: lvm-operator-catalog-4-16-20260624-033025-000 - local version_prefix date_str time_str - version_prefix=$(echo "${snapshot}" | grep -oP 'lvm-operator-catalog-\d+-\d+') - date_str=$(echo "${snapshot}" | grep -oP '\d{8}(?=-\d{6}-)') - time_str=$(echo "${snapshot}" | grep -oP '(?<=\d{8}-)\d{6}') - - if [[ -z "${version_prefix}" || -z "${date_str}" || -z "${time_str}" ]]; then - err "Cannot parse snapshot name: ${snapshot}" - return 1 - fi - - # Convert snapshot timestamp to epoch for comparison - local snap_year snap_month snap_day snap_hour snap_min snap_sec snap_epoch - snap_year=${date_str:0:4} - snap_month=${date_str:4:2} - snap_day=${date_str:6:2} - snap_hour=${time_str:0:2} - snap_min=${time_str:2:2} - snap_sec=${time_str:4:2} - snap_epoch=$(date -d "${snap_year}-${snap_month}-${snap_day}T${snap_hour}:${snap_min}:${snap_sec}Z" +%s 2>/dev/null || echo 0) - - # Convert version prefix (lvm-operator-catalog-4-16) to v4.16 for tag matching - local version_dot - version_dot=$(echo "${version_prefix}" | sed 's/lvm-operator-catalog-//; s/-/./') - - # Query quay for v{version} commit tags (not -on-push, which expire) - local tags_json - tags_json=$(curl -sSL --connect-timeout 10 --max-time 30 \ - "${QUAY_API}/repository/${QUAY_REPO}/tag/?limit=50&filter_tag_name=like:v${version_dot}-") - - # Select only raw commit tags (v4.16-{40-hex}), exclude auxiliary suffixes - # Pick the tag closest to the snapshot timestamp (smallest absolute delta) - local result - result=$(echo "${tags_json}" | jq -r --arg snap_epoch "${snap_epoch}" --arg vpfx "v${version_dot}-" ' - [.tags[] - | select(.name | startswith($vpfx)) - | select(.name | test("^v[0-9]+\\.[0-9]+-[a-f0-9]{40}$")) - | select(.last_modified != null and .last_modified != "") - | .tag_epoch = (.last_modified | strptime("%a, %d %b %Y %H:%M:%S %z") | mktime) - | .abs_delta = ((.tag_epoch - ($snap_epoch | tonumber)) | fabs) - ] | sort_by(.abs_delta) | .[0] // empty | - [.name, .manifest_digest] | @tsv - ') - - if [[ -z "${result}" ]]; then - err "No matching quay tag found for snapshot ${snapshot}" - return 1 - fi - - local tag_name digest - tag_name=$(echo "${result}" | cut -f1) - digest=$(echo "${result}" | cut -f2) - - info "Resolved: ${snapshot} → ${tag_name} (${digest})" - echo "${digest}" -} - -# --------------------------------------------------------------------------- -# Gangway: trigger a nightly job with image override -# --------------------------------------------------------------------------- - -trigger_job() { - local job_name="$1" image="$2" - - local body - body=$(jq -cn \ - --arg img "${image}" \ - '{job_execution_type: "1", pod_spec_options: {envs: {MULTISTAGE_PARAM_OVERRIDE_LVM_INDEX_IMAGE: $img}}}') - - if ${DRY_RUN}; then - info "[dry-run] would trigger: ${job_name}" - info "[dry-run] image: ${image}" - return 0 - fi - - local http_code attempt max_retries=3 delay=15 - - for ((attempt = 1; attempt <= max_retries; attempt++)); do - http_code=$(curl -sSL -X POST -o /dev/stderr -w '%{http_code}' \ - --connect-timeout 10 --max-time 30 \ - -H "Authorization: Bearer ${MY_APPCI_TOKEN}" \ - -H "Content-Type: application/json" \ - -d "${body}" \ - "${GANGWAY_API}/v1/executions/${job_name}" 2>/dev/null) - - if [[ "${http_code}" == "200" ]]; then - info " triggered: ${job_name}" - return 0 - fi - - if [[ "${http_code}" == "429" && ${attempt} -lt ${max_retries} ]]; then - warn " rate limited (attempt ${attempt}/${max_retries}), retrying in ${delay}s: ${job_name}" - sleep "${delay}" - delay=$((delay * 2)) - continue - fi - - warn " failed (HTTP ${http_code}): ${job_name}" - return 1 - done -} - -# --------------------------------------------------------------------------- -# Prow: fetch latest run results for nightly jobs -# --------------------------------------------------------------------------- - -fetch_last_runs() { - local release="$1" - local job_prefix="periodic-ci-openshift-lvm-operator-release-${release}-nightly-" - local results="[]" - - for test_name in "${NIGHTLY_JOBS[@]}"; do - local full_job="${job_prefix}${test_name}" - local prow_json - prow_json=$(curl -sSL --connect-timeout 10 --max-time 30 \ - "https://prow.ci.openshift.org/prowjobs.js?omit=annotations,labels,decoration_config,pod_spec&job=${full_job}" 2>/dev/null || true) - - if [[ -z "${prow_json}" ]]; then - results=$(echo "${results}" | jq --arg n "${test_name}" '. + [{name: $n, state: "unknown"}]') - continue - fi - - local entry - entry=$(echo "${prow_json}" | sed 's/^var allBuilds = //' | \ - jq -r --arg n "${test_name}" ' - [.items[] | select(.status.state == "success" or .status.state == "failure" or .status.state == "error" or .status.state == "aborted")] | - sort_by(.status.startTime) | reverse | - if length > 0 then .[0] else null end | - if . then {name: $n, state: .status.state, url: .status.url, started: .status.startTime} else {name: $n, state: "unknown"} end - ' 2>/dev/null || echo "{\"name\": \"${test_name}\", \"state\": \"unknown\"}") - - results=$(echo "${results}" | jq --argjson e "${entry}" '. + [$e]') - done - - echo "${results}" -} - -# --------------------------------------------------------------------------- -# Collect artifacts for a skipped release (same digest, already tested) -# --------------------------------------------------------------------------- - -collect_skipped_artifacts() { - local release="$1" - local pr_title="$2" - - local patch_version - patch_version=$(echo "${pr_title}" | grep -oP '\d+\.\d+\.\d+' || echo "${release}-zstream") - - local jobs_dir="${WORKDIR}/jobs" - mkdir -p "${jobs_dir}" - - local jobs_file="${jobs_dir}/release-${patch_version}-jobs.json" - - # Skip if already collected - if [[ -f "${jobs_file}" ]] && [[ "$(jq 'length' "${jobs_file}")" -gt 0 ]]; then - info " Artifacts already collected for ${patch_version}" - return 0 - fi - - info " Collecting failed periodic jobs for ${release}..." - local raw_json raw_err - raw_err=$(mktemp) - if ! raw_json=$(bash "${TRIGGER_DIR}/prow-jobs-for-release.sh" "${COMPONENT}" "${release}" 2>"${raw_err}"); then - warn "Failed to collect jobs for ${release}:" - cat "${raw_err}" >&2 - rm -f "${raw_err}" - echo "[]" > "${jobs_file}" - return 0 - fi - rm -f "${raw_err}" - - local filtered_json - filtered_json=$(echo "${raw_json}" | jq '[.[] | select(.type == "periodic")]') - - local count - count=$(echo "${filtered_json}" | jq 'length') - - if [[ "${count}" -eq 0 ]]; then - info " No failed periodic jobs found for ${patch_version}" - echo "[]" > "${jobs_file}" - return 0 - fi - - info " Found ${count} failed periodic jobs, downloading artifacts..." - local dl_err - dl_err=$(mktemp) - echo "${filtered_json}" | \ - bash "${TRIGGER_DIR}/download-jobs.sh" --workdir "${WORKDIR}" 2>"${dl_err}" \ - > "${jobs_file}" - [[ -s "${dl_err}" ]] && cat "${dl_err}" >&2 - rm -f "${dl_err}" - - info " Done: ${jobs_file}" -} - -# --------------------------------------------------------------------------- -# Process a single release -# --------------------------------------------------------------------------- - -process_release() { - local release="$1" - - info "--- Release ${release} ---" - - # Find open z-stream PR - local pr_info - pr_info=$(find_zstream_pr "${release}" || true) - - if [[ -z "${pr_info}" ]]; then - # Check if a previously tracked PR was merged - local last_pr - last_pr=$(get_last_pr "${release}") - if [[ -n "${last_pr}" ]]; then - local merge_status - merge_status=$(check_pr_merged "${last_pr}" || true) - if [[ "${merge_status}" == "merged" ]]; then - info "PR #${last_pr} was merged for ${release}." - jq -n --arg r "${release}" --arg pr "${last_pr}" \ - '{($r): {status: "completed", reason: "release completed", pr: ("#" + $pr)}}' \ - > "${WORKDIR}/zstream-${release}.json" - return 0 - fi - fi - - info "No z-stream release PR found for ${release}." - jq -n --arg r "${release}" \ - '{($r): {status: "no_pr", reason: "no new releases"}}' \ - > "${WORKDIR}/zstream-${release}.json" - return 0 - fi - - local pr_number pr_title snapshot - pr_number=$(echo "${pr_info}" | cut -f1) - pr_title=$(echo "${pr_info}" | cut -f2) - snapshot=$(echo "${pr_info}" | cut -f3) - info "Found PR #${pr_number}: ${pr_title}" - info "Snapshot: ${snapshot}" - - # Resolve to digest - local digest - digest=$(resolve_snapshot_to_digest "${snapshot}" || true) - - if [[ -z "${digest}" ]]; then - warn "Could not resolve snapshot to digest for ${release}" - jq -n --arg r "${release}" --arg pr "${pr_number}" --arg snap "${snapshot}" \ - '{($r): {status: "error", reason: "digest resolution failed", pr: ("#" + $pr), snapshot: $snap}}' \ - > "${WORKDIR}/zstream-${release}.json" - return 0 - fi - - local image="quay.io/${QUAY_REPO}@${digest}" - - # Check if already tested - if ! ${FORCE}; then - local last_digest - last_digest=$(get_last_digest "${release}") - if [[ "${last_digest}" == "${digest}" ]]; then - info "Already tested with same digest. Fetching last run results." - local last_runs - last_runs=$(fetch_last_runs "${release}") - jq -n --arg r "${release}" --arg pr "${pr_number}" --arg title "${pr_title}" --arg img "${image}" --arg snap "${snapshot}" \ - --argjson jobs "${last_runs}" \ - '{($r): {status: "skipped", reason: "same digest", pr: ("#" + $pr), pr_title: $title, image: $img, snapshot: $snap, jobs: $jobs}}' \ - > "${WORKDIR}/zstream-${release}.json" - - collect_skipped_artifacts "${release}" "${pr_title}" - return 0 - fi - fi - - # Trigger nightly jobs - local job_prefix="periodic-ci-openshift-lvm-operator-release-${release}-nightly-" - local triggered=0 failed=0 - local triggered_jobs=() - - for test_name in "${NIGHTLY_JOBS[@]}"; do - local full_job="${job_prefix}${test_name}" - if trigger_job "${full_job}" "${image}"; then - triggered=$((triggered + 1)) - triggered_jobs+=("${full_job}") - else - failed=$((failed + 1)) - fi - ${DRY_RUN} || sleep 30 - done - - info "${release}: ${triggered} triggered, ${failed} failed" - - # Update state - if ! ${DRY_RUN} && [[ ${triggered} -gt 0 && ${failed} -eq 0 ]]; then - save_digest "${release}" "${digest}" "${pr_number}" "${image}" - fi - - # Write per-release summary - jq -n \ - --arg r "${release}" \ - --arg pr "${pr_number}" \ - --arg title "${pr_title}" \ - --arg img "${image}" \ - --arg snap "${snapshot}" \ - --argjson triggered "${triggered}" \ - --argjson failed "${failed}" \ - --argjson jobs "$(if [[ ${#triggered_jobs[@]} -gt 0 ]]; then printf '%s\n' "${triggered_jobs[@]}" | jq -R . | jq -s .; else echo '[]'; fi)" \ - '{($r): {status: "triggered", pr: ("#" + $pr), pr_title: $title, image: $img, snapshot: $snap, jobs_triggered: $triggered, jobs_failed: $failed, jobs: $jobs}}' \ - > "${WORKDIR}/zstream-${release}.json" -} - -# --------------------------------------------------------------------------- -# Main -# --------------------------------------------------------------------------- - -main() { - info "=== LVMS Z-Stream Trigger ===" - info "Releases: ${RELEASES}" - info "Workdir: ${WORKDIR}" - ${DRY_RUN} && info "Mode: DRY RUN" - ${FORCE} && info "Mode: FORCE (ignoring last-tested digest)" - - load_state - - IFS=',' read -ra release_list <<< "${RELEASES}" - - for release in "${release_list[@]}"; do - release=$(echo "${release}" | xargs) - process_release "${release}" - done - - # Merge per-release summaries into one file - local summary_file="${WORKDIR}/zstream-summary.json" - jq -s 'add' "${WORKDIR}"/zstream-*.json > "${summary_file}" 2>/dev/null || echo '{}' > "${summary_file}" - - # Print summary - info "" - info "=== Summary ===" - jq -r 'to_entries[] | " \(.key): \(.value.status)\(if .value.reason then " (\(.value.reason))" elif .value.jobs_triggered then " (\(.value.jobs_triggered) jobs)" else "" end)"' \ - "${summary_file}" >&2 - - info "" - info "Summary written to: ${summary_file}" -} - -main diff --git a/plugins/lvms-ci/skills/doctor/SKILL.md b/plugins/lvms-ci/skills/doctor/SKILL.md index 0986c41a..3ca5019f 100644 --- a/plugins/lvms-ci/skills/doctor/SKILL.md +++ b/plugins/lvms-ci/skills/doctor/SKILL.md @@ -43,11 +43,12 @@ Compute once at the start by running `date +%y%m%d` and substituting into the pa 2. Run the prepare script: ```text - bash plugins/lvms-ci/scripts/doctor.sh prepare --component lvm-operator --workdir + bash plugins/lvms-ci/scripts/doctor.sh prepare --component lvm-operator --workdir --release-prs ``` 3. The script deterministically: - For each release: fetches failed periodic jobs, downloads artifacts, writes `/jobs/release--jobs.json` + - Collects open z-stream PRs targeting `release-management` and fetches their presubmit results - Outputs a JSON summary listing all releases, job counts, and file paths 4. Read the JSON output to know which releases have jobs to analyze and how many @@ -82,11 +83,9 @@ Compute once at the start by running `date +%y%m%d` and substituting into the pa Use the Write tool to save the file. The file must contain the complete analysis report." ``` -3. If z-stream job files exist in `/jobs/` (e.g., `release-4.19.3-jobs.json` — written by `zstream-trigger.sh`), analyze those jobs the same way — they use patch versions (e.g., `4.19.3`) as the `` value but follow the identical naming and analysis pattern. - -4. Launch **ALL** agents in a **single message** as **foreground** agents (do NOT use `run_in_background`). Foreground agents in the same message run concurrently — this is just as fast as background agents but keeps your turn active until all complete. -5. Say "Analyzing N jobs in parallel..." in your message text alongside the Agent tool calls. -6. When all agents return, immediately proceed to Step 3 in the same turn. Do NOT stop or end your turn between Step 2 and Step 3. +3. Launch **ALL** agents in a **single message** as **foreground** agents (do NOT use `run_in_background`). Foreground agents in the same message run concurrently — this is just as fast as background agents but keeps your turn active until all complete. +4. Say "Analyzing N jobs in parallel..." in your message text alongside the Agent tool calls. +5. When all agents return, immediately proceed to Step 3 in the same turn. Do NOT stop or end your turn between Step 2 and Step 3. ### Step 3: Finalize — Aggregate and Generate HTML Report @@ -145,23 +144,9 @@ HTML report generated: /report-lvm-operator-ci-doctor.html - Internet access to fetch job data from Prow/GCS - Bash shell, Python 3 -### Z-Stream Integration (Optional) - -Run `zstream-trigger.sh` before doctor to populate z-stream data. For skipped releases (same digest already tested), the trigger script collects failed periodic job artifacts into `/jobs/release--jobs.json` — these are discovered automatically by the finalize step alongside regular releases. - -The HTML report includes z-stream results in the **Pull Requests** tab when `zstream-summary.json` exists in the workdir. - -To populate z-stream data (dry-run — no job triggering): - -```text -bash plugins/lvms-ci/scripts/zstream-trigger.sh --releases --dry-run --workdir -``` - -To trigger z-stream e2e jobs and collect skipped release artifacts (requires `MY_APPCI_TOKEN`): +### Z-Stream PR Results -```text -bash plugins/lvms-ci/scripts/zstream-trigger.sh --releases --workdir -``` +Z-stream test results are collected automatically when `--release-prs` is passed to the prepare command. The `--release-prs` flag queries open PRs targeting the `release-management` branch on `openshift/lvm-operator` and fetches their Prow presubmit results from GCS. These results appear in the **Pull Requests** tab of the HTML report. ## Related Skills diff --git a/plugins/shared/scripts/create-report.py b/plugins/shared/scripts/create-report.py index 9d64ecef..e1d25b7c 100755 --- a/plugins/shared/scripts/create-report.py +++ b/plugins/shared/scripts/create-report.py @@ -1211,14 +1211,8 @@ def _render_bug_links(bug_match): # HTML rendering # --------------------------------------------------------------------------- -def render_release_section(version, rdata, bug_candidates, index_info=None, zstream_info=None): - is_zstream = zstream_info is not None +def render_release_section(version, rdata, bug_candidates, index_info=None): display = version - if is_zstream: - pr_title = zstream_info.get("pr_title", "") - m = re.search(r'(\d+\.\d+\.\d+)', pr_title) if pr_title else None - if m: - display = m.group(1) css_class = "release-section" style = "" @@ -1257,10 +1251,7 @@ def render_release_section(version, rdata, bug_candidates, index_info=None, zstr lines.append(f' {total} {label}') lines.append("
    ") - if is_zstream: - idx_html = _render_zstream_info(version, zstream_info) - else: - idx_html = _render_index_image(index_info) + idx_html = _render_index_image(index_info) if idx_html: lines.append(idx_html) @@ -1311,7 +1302,7 @@ def render_release_section(version, rdata, bug_candidates, index_info=None, zstr return "\n".join(lines) -def render_pr_section(pr_data, bug_candidates, pr_status, pr_error=None, zstream_data=None): +def render_pr_section(pr_data, bug_candidates, pr_status, pr_error=None): """Render the Pull Requests tab. pr_data: analyzed PR summary (from aggregate), may be None. @@ -1367,7 +1358,7 @@ def render_pr_section(pr_data, bug_candidates, pr_status, pr_error=None, zstream "analysis": pr, }) - if not all_prs and not zstream_data: + if not all_prs: return ( '
    \n' '
    \n' @@ -1395,35 +1386,6 @@ def render_pr_section(pr_data, bug_candidates, pr_status, pr_error=None, zstream f'
  • PR# {pr["number"]}' f' — {pr["failed"]} failures ({b.get("build", 0)} build, {b.get("test", 0)} test, {b.get("infrastructure", 0)} infra){suffix}
    • ' ) - if zstream_data: - sorted_zversions = sorted( - zstream_data.keys(), - key=lambda v: [int(x) for x in v.split(".")], - ) - for zv in sorted_zversions: - zinfo = zstream_data[zv] - zstatus = zinfo.get("status", "unknown") - if not zinfo.get("pr"): - continue - pr_title = zinfo.get("pr_title", "") - m = re.search(r'(\d+\.\d+\.\d+)', pr_title) if pr_title else None - display = m.group(1) if m else zv - if zstatus == "skipped": - rdata = zinfo.get("_analyzed") - total = rdata["total_failed"] if rdata else 0 - b = rdata.get("breakdown", {"build": 0, "test": 0, "infrastructure": 0}) if rdata else {"build": 0, "test": 0, "infrastructure": 0} - fail_word = "failure" if total == 1 else "failures" - toc_lines.append( - f'
  • Release {_e(display)}' - f' — {total} {fail_word} ({b["build"]} build, {b["test"]} test, {b["infrastructure"]} infra)
    • ' - ) - elif zstatus == "triggered": - njobs = zinfo.get("jobs_triggered", 0) - toc_lines.append( - f'
  • Release {_e(display)}' - f' — {njobs} jobs running
    • ' - ) - toc_lines.append(' ') toc_lines.append('
    ') @@ -1501,65 +1463,10 @@ def render_pr_section(pr_data, bug_candidates, pr_status, pr_error=None, zstream lines.append("
    ") - if zstream_data: - sorted_zversions = sorted( - zstream_data.keys(), - key=lambda v: [int(x) for x in v.split(".")], - ) - for zv in sorted_zversions: - zinfo = zstream_data[zv] - zstatus = zinfo.get("status", "unknown") - if not zinfo.get("pr"): - continue - if zstatus == "skipped": - rdata = zinfo.get("_analyzed") - empty_bd = {"build": 0, "test": 0, "infrastructure": 0} - if rdata is None: - rdata = {"total_failed": 0, "issues": [], "breakdown": empty_bd} - section_id = f"{zv}-zstream" - lines.append(render_release_section(section_id, rdata, bug_candidates, zstream_info=zinfo)) - elif zstatus == "triggered": - pr_title = zinfo.get("pr_title", "") - m = re.search(r'(\d+\.\d+\.\d+)', pr_title) if pr_title else None - display = m.group(1) if m else zv - njobs = zinfo.get("jobs_triggered", 0) - section_id = f"zstream-{zv}" - lines.append(f'
    ') - lines.append('
    ') - lines.append(f'

    Release {_e(display)}🔗

    ') - lines.append(f' {njobs} jobs running') - lines.append("
    ") - lines.append(_render_zstream_info(zv, zinfo)) - lines.append("
    ") - return "\n".join(toc_lines) + "\n\n" + "\n".join(lines) -# --------------------------------------------------------------------------- -# Z-stream metadata rendering -# --------------------------------------------------------------------------- - - -def _render_zstream_info(version, zinfo): - """Render z-stream metadata box (PR, snapshot, image).""" - lines = ['
    '] - pr = zinfo.get("pr", "") - image = zinfo.get("image", "") - snapshot = zinfo.get("snapshot", "") - if pr: - pr_display = pr.lstrip("#") - repo = COMPONENT_REPOS.get(_COMPONENT, "openshift/" + (_COMPONENT or "unknown")) - pr_url = f"https://github.com/{repo}/pull/{pr_display}" - lines.append(f' Release PR: #{_e(pr_display)}
    ') - if snapshot: - lines.append(f' Snapshot: {_e(snapshot)}
    ') - if image: - lines.append(f' Catalog Image: {_e(image)}
    ') - lines.append("
    ") - return "\n".join(lines) - - -def generate_html(component_title, releases_data, all_bug_candidates, pr_data, pr_status, timestamp, pr_error=None, bugs_tab_data=None, images_tab_data=None, index_data=None, zstream_data=None): +def generate_html(component_title, releases_data, all_bug_candidates, pr_data, pr_status, timestamp, pr_error=None, bugs_tab_data=None, images_tab_data=None, index_data=None): date_str = timestamp.strftime("%Y-%m-%d") time_str = timestamp.strftime("%Y-%m-%d %H:%M:%S") @@ -1622,7 +1529,7 @@ def generate_html(component_title, releases_data, all_bug_candidates, pr_data, p for version, rdata in releases_data.items(): sections.append(render_release_section(version, rdata, all_bug_candidates, _idx.get(version))) - pr_section = render_pr_section(pr_data, all_bug_candidates, pr_status, pr_error, zstream_data) + pr_section = render_pr_section(pr_data, all_bug_candidates, pr_status, pr_error) bugs_section = render_bugs_section(bugs_tab_data) if bugs_tab_data else "" images_section = render_images_section(images_tab_data) @@ -1872,21 +1779,9 @@ def main(): if os.path.isdir(graphs_dir): _GRAPHS_DIR = graphs_dir - zstream_data = load_json(os.path.join(workdir, "zstream-summary.json")) - if zstream_data: - for zv, zinfo in zstream_data.items(): - if zinfo.get("status") == "skipped": - pr_title = zinfo.get("pr_title", "") - m = re.search(r'(\d+\.\d+\.\d+)', pr_title) if pr_title else None - patch = m.group(1) if m else f"{zv}-zstream" - rdata = load_json(os.path.join(workdir, "jobs", f"release-{patch}-summary.json")) - if rdata is None: - rdata = {"total_failed": 0, "issues": [], "breakdown": _EMPTY_BREAKDOWN} - zinfo["_analyzed"] = rdata - # Generate HTML timestamp = datetime.now(timezone.utc) - html_content = generate_html(component_title, releases_data, all_bug_candidates, pr_data, pr_status, timestamp, pr_error, bugs_tab_data, images_tab_data, index_data, zstream_data) + html_content = generate_html(component_title, releases_data, all_bug_candidates, pr_data, pr_status, timestamp, pr_error, bugs_tab_data, images_tab_data, index_data) output_path = os.path.join(workdir, f"report-{component}-ci-doctor.html") with open(output_path, "w") as f: @@ -1934,17 +1829,6 @@ def main(): print(f" Release {release}: {repo_names}{grade_str}") else: print(" No container image data") - if zstream_data: - print(" Z-Stream:") - for zv in sorted(zstream_data, key=lambda v: [int(x) for x in v.split(".")]): - zi = zstream_data[zv] - zs = zi.get("status", "unknown") - extra = "" - if zs == "triggered": - extra = f' ({zi.get("jobs_triggered", 0)} jobs, PR {zi.get("pr", "")})' - elif zi.get("reason"): - extra = f' ({zi["reason"]})' - print(f" Release {zv}: {zs}{extra}") print(f"\nHTML report generated: {output_path}") diff --git a/plugins/shared/scripts/doctor.sh b/plugins/shared/scripts/doctor.sh index f87c768d..be15d47b 100755 --- a/plugins/shared/scripts/doctor.sh +++ b/plugins/shared/scripts/doctor.sh @@ -7,7 +7,7 @@ set -euo pipefail # # Two phases called by the doctor skill with LLM steps in between: # -# doctor.sh prepare --component --workdir DIR [--rebase] +# doctor.sh prepare --component --workdir DIR [--rebase|--release-prs] # - Collects failed jobs for each release # - Downloads all artifacts in parallel # - Writes per-release and PR jobs JSON files to ${WORKDIR}/jobs/ @@ -46,7 +46,7 @@ cmd_prepare() { case "${1}" in --workdir) WORKDIR="${2}"; shift 2 ;; --component) COMPONENT="${2}"; shift 2 ;; - --rebase) do_rebase=true; shift ;; + --rebase|--release-prs) do_rebase=true; shift ;; --repo) if [[ -z "${2:-}" || "${2:-}" == -* ]]; then echo "Error: --repo requires an org/name argument" >&2; return 1 @@ -63,7 +63,7 @@ cmd_prepare() { if [[ -z "${releases_arg}" ]]; then echo "Error: releases argument required" >&2 - echo "Usage: $(basename "$0") prepare --component [--workdir DIR] [--rebase]" >&2 + echo "Usage: $(basename "$0") prepare --component [--workdir DIR] [--rebase|--release-prs]" >&2 return 1 fi @@ -132,8 +132,12 @@ cmd_prepare() { local pr_json pr_err pr_err=$(mktemp) local prs_error="" - if ! pr_json=$(bash "${SCRIPT_DIR}/prow-jobs-for-pull-requests.sh" \ - --mode detail --author "microshift-rebase-script[bot]" 2>"${pr_err}"); then + local -a pr_args=(--mode detail) + case "${COMPONENT}" in + microshift) pr_args+=(--author "microshift-rebase-script[bot]") ;; + lvm-operator) pr_args+=(--base "release-management") ;; + esac + if ! pr_json=$(bash "${SCRIPT_DIR}/prow-jobs-for-pull-requests.sh" "${pr_args[@]}" 2>"${pr_err}"); then echo " ERROR: failed to collect rebase PRs:" >&2 prs_error=$(cat "${pr_err}") echo "${prs_error}" >&2 @@ -505,7 +509,7 @@ usage() { echo "Usage: $(basename "$0") --component [--workdir DIR] [options]" >&2 echo "" >&2 echo "Commands:" >&2 - echo " prepare --component C [--workdir DIR] [--rebase] [--repo ORG/NAME] Collect jobs, download artifacts, optional source checkout" >&2 + echo " prepare --component C [--workdir DIR] [--rebase|--release-prs] [--repo ORG/NAME] Collect jobs, download artifacts, optional source checkout" >&2 echo " graphs --component C [--workdir DIR] [--timezone TZ] Generate PCP performance graphs" >&2 echo " finalize --component C [--workdir DIR] Aggregate results and generate HTML" >&2 echo " refresh --component C [--workdir DIR] [--ignore KEY1,KEY2,...] Regenerate HTML from existing workdir data" >&2 From 3d512991298070ffc5ea5bfff432d7be61743dd4 Mon Sep 17 00:00:00 2001 From: Kasturi Narra Date: Thu, 9 Jul 2026 13:51:18 +0530 Subject: [PATCH 11/14] Simplify finalize aggregation to iterate over known releases The find-based discovery of release-*-jobs.json was needed to pick up z-stream patch version files created by zstream-trigger.sh. Now that the trigger is removed, only CLI-passed releases have job files. Co-Authored-By: Claude Opus 4.6 --- plugins/shared/scripts/doctor.sh | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/plugins/shared/scripts/doctor.sh b/plugins/shared/scripts/doctor.sh index be15d47b..4bba564d 100755 --- a/plugins/shared/scripts/doctor.sh +++ b/plugins/shared/scripts/doctor.sh @@ -333,13 +333,9 @@ cmd_finalize() { IFS=',' read -ra RELEASES <<< "${releases_arg}" - # Aggregate all job files - local all_job_files - all_job_files=$(find "${WORKDIR}/jobs" -name 'release-*-jobs.json' 2>/dev/null | sort || true) - while IFS= read -r jobs_file; do - [[ -z "${jobs_file}" ]] && continue - local release - release=$(basename "${jobs_file}" | sed 's/^release-//; s/-jobs\.json$//') + # Aggregate job files for each release + for release in "${RELEASES[@]}"; do + release=$(echo "${release}" | xargs) local reports reports=$(find "${WORKDIR}/jobs" -name "release-${release}-job-*.txt" 2>/dev/null | head -1) if [[ -n "${reports}" ]]; then @@ -348,7 +344,7 @@ cmd_finalize() { --release "${release}" --workdir "${WORKDIR}" >/dev/null || \ echo " WARNING: aggregation failed for ${release}" >&2 fi - done <<< "${all_job_files}" + done # Aggregate PRs (if job files exist) local pr_files From 0b378c72dfa00b4f7b330d67c94918eb742190bf Mon Sep 17 00:00:00 2001 From: Kasturi Narra Date: Thu, 9 Jul 2026 13:53:19 +0530 Subject: [PATCH 12/14] Remove unnecessary find guard before release aggregation aggregate.py handles the no-reports case gracefully, so the find check for report files before calling it is unnecessary. Co-Authored-By: Claude Opus 4.6 --- plugins/shared/scripts/doctor.sh | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/plugins/shared/scripts/doctor.sh b/plugins/shared/scripts/doctor.sh index 4bba564d..366bd8a5 100755 --- a/plugins/shared/scripts/doctor.sh +++ b/plugins/shared/scripts/doctor.sh @@ -336,14 +336,10 @@ cmd_finalize() { # Aggregate job files for each release for release in "${RELEASES[@]}"; do release=$(echo "${release}" | xargs) - local reports - reports=$(find "${WORKDIR}/jobs" -name "release-${release}-job-*.txt" 2>/dev/null | head -1) - if [[ -n "${reports}" ]]; then - echo "=== Aggregating release ${release} ===" >&2 - python3 "${SCRIPT_DIR}/aggregate.py" \ - --release "${release}" --workdir "${WORKDIR}" >/dev/null || \ - echo " WARNING: aggregation failed for ${release}" >&2 - fi + echo "=== Aggregating release ${release} ===" >&2 + python3 "${SCRIPT_DIR}/aggregate.py" \ + --release "${release}" --workdir "${WORKDIR}" >/dev/null || \ + echo " WARNING: aggregation failed for ${release}" >&2 done # Aggregate PRs (if job files exist) From 71b1112c78f054ad703533a7493e865448e35d87 Mon Sep 17 00:00:00 2001 From: Kasturi Narra Date: Thu, 9 Jul 2026 13:57:14 +0530 Subject: [PATCH 13/14] Revert unrelated repo loop change in doctor.sh finalize MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Restore the original `for repo in ${repos}` loop from main — the while/read change was introduced on this branch but is unrelated to z-stream work. Co-Authored-By: Claude Opus 4.6 --- plugins/shared/scripts/doctor.sh | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/plugins/shared/scripts/doctor.sh b/plugins/shared/scripts/doctor.sh index 366bd8a5..d172c6d2 100755 --- a/plugins/shared/scripts/doctor.sh +++ b/plugins/shared/scripts/doctor.sh @@ -360,8 +360,7 @@ cmd_finalize() { local images_dir="${WORKDIR}/images" mkdir -p "${images_dir}" - while IFS= read -r repo; do - [[ -z "${repo}" ]] && continue + for repo in ${repos}; do local repo_slug="${repo//\//@}" # Fetch catalog repository ID (needed for catalog URLs) @@ -388,7 +387,7 @@ cmd_finalize() { fi done echo "${tmp_merged}" > "${outfile}" - done <<< "${repos}" + done fi # Extract index image info (LVMS-specific, no-op for other components) From 3aabec36823a46b208d08d8f78b18e7cb029c2a0 Mon Sep 17 00:00:00 2001 From: Kasturi Narra Date: Thu, 9 Jul 2026 14:33:32 +0530 Subject: [PATCH 14/14] Revert unrelated changes in create-report.py Remove cosmetic and scaffolding changes introduced on this branch that are not needed for the z-stream PR collection work: JS selector tweak, whitespace changes, js_content variable rename, TOC div wrapper, and display/css_class/style variables left over from z-stream rendering removal. Co-Authored-By: Claude Opus 4.6 --- plugins/shared/scripts/create-report.py | 31 ++++++++----------------- 1 file changed, 10 insertions(+), 21 deletions(-) diff --git a/plugins/shared/scripts/create-report.py b/plugins/shared/scripts/create-report.py index e1d25b7c..c177b9f3 100755 --- a/plugins/shared/scripts/create-report.py +++ b/plugins/shared/scripts/create-report.py @@ -196,7 +196,7 @@ } function filterToday(on) { var today = new Date().toISOString().split('T')[0]; - document.querySelectorAll('#tab-periodics .release-section .issue-row').forEach(function(row) { + document.querySelectorAll('#tab-periodics .issue-row').forEach(function(row) { var dates = (row.getAttribute('data-dates') || '').split(' '); var show = !on || dates.indexOf(today) !== -1; row.style.display = show ? '' : 'none'; @@ -244,7 +244,6 @@ row.style.display = (!on || row.hasAttribute('data-latest')) ? '' : 'none'; }); } - function showGraphTab(btn, paneId) { var container = btn.closest('.perf-graphs'); container.querySelectorAll('.graph-tab-btn').forEach(function(b) { b.classList.remove('active'); }); @@ -1064,8 +1063,6 @@ def _render_index_image(index_info): return "\n".join(lines) - - # Graph workdir — set by main() before rendering _GRAPHS_DIR = None _COMPONENT = None @@ -1212,15 +1209,11 @@ def _render_bug_links(bug_match): # --------------------------------------------------------------------------- def render_release_section(version, rdata, bug_candidates, index_info=None): - display = version - css_class = "release-section" - style = "" - if rdata is None: return ( - f'
    \n' + f'
    \n' '
    \n' - f'

    Release {_e(display)}

    \n' + f'

    Release {_e(version)}

    \n' ' no data\n' '
    \n' "

    Analysis failed to produce results.

    \n" @@ -1229,9 +1222,9 @@ def render_release_section(version, rdata, bug_candidates, index_info=None): if rdata.get("collection_error"): return ( - f'
    \n' + f'
    \n' '
    \n' - f'

    Release {_e(display)}

    \n' + f'

    Release {_e(version)}

    \n' ' collection error\n' '
    \n' f' 
    Data collection failed: {_e(rdata["collection_error"])}
    \n' @@ -1244,9 +1237,9 @@ def render_release_section(version, rdata, bug_candidates, index_info=None): b = rdata["breakdown"] lines = [] - lines.append(f'
    ') + lines.append(f'
    ') lines.append('
    ') - lines.append(f'

    Release {_e(display)}🔗

    ') + lines.append(f'

    Release {_e(version)}🔗

    ') label = "failure" if total == 1 else "failures" lines.append(f' {total} {label}') lines.append("
    ") @@ -1462,7 +1455,6 @@ def render_pr_section(pr_data, bug_candidates, pr_status, pr_error=None): lines.append(' ') lines.append("
    ") - return "\n".join(toc_lines) + "\n\n" + "\n".join(lines) @@ -1533,8 +1525,6 @@ def generate_html(component_title, releases_data, all_bug_candidates, pr_data, p bugs_section = render_bugs_section(bugs_tab_data) if bugs_tab_data else "" images_section = render_images_section(images_tab_data) - js_content = JS - return f"""\ @@ -1566,14 +1556,13 @@ def generate_html(component_title, releases_data, all_bug_candidates, pr_data, p

    Table of Contents

    -
    - -
    +
      {chr(10).join(toc)}
    + {chr(10).join(sections)}
    @@ -1592,7 +1581,7 @@ def generate_html(component_title, releases_data, all_bug_candidates, pr_data, p