From b85e5b88bb4a460fc805e7ad5949222017e0af7d Mon Sep 17 00:00:00 2001 From: lewismc Date: Fri, 14 Aug 2026 21:47:08 -0700 Subject: [PATCH] Trigger single-node Hadoop smoke test from PR /smoke-test comments and release tags --- .github/workflows/jenkins-smoke-test.yml | 161 ++++++++++++++++++++++ Jenkinsfile.smoke-test-single-node-hadoop | 130 ++++++++++++++++- 2 files changed, 288 insertions(+), 3 deletions(-) create mode 100644 .github/workflows/jenkins-smoke-test.yml diff --git a/.github/workflows/jenkins-smoke-test.yml b/.github/workflows/jenkins-smoke-test.yml new file mode 100644 index 0000000000..c906f6288d --- /dev/null +++ b/.github/workflows/jenkins-smoke-test.yml @@ -0,0 +1,161 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# Remotely trigger the ASF Jenkins single-node Hadoop smoke job. +# - PR comment (created only): a line that is exactly "/smoke-test" or +# "/smoke-test hadoop=" from a user with write/maintain/admin. +# - Tag push: release-. or release-.. +# Jenkins posts/updates the PR ACK comment; this workflow only triggers. +# Operator setup (secrets, Jenkins remote trigger): Nutch wiki. + +name: Jenkins smoke test trigger + +on: + issue_comment: + types: [created] + push: + tags: + - 'release-*' + +permissions: + contents: read + pull-requests: read + +jobs: + trigger-smoke-test: + runs-on: ubuntu-latest + # GitHub cannot filter issue_comment events by body under "on:", so it records + # a (skipped) workflow run for every comment. This job-level guard ensures a + # runner only starts for tag pushes or PR comments that mention /smoke-test; + # the step below still enforces the strict whole-line command grammar. + # Plain issues and unrelated comments are ignored. + if: > + github.event_name == 'push' || + (github.event_name == 'issue_comment' && + github.event.issue.pull_request && + contains(github.event.comment.body, '/smoke-test')) + steps: + - name: Resolve trigger parameters + id: params + env: + EVENT_NAME: ${{ github.event_name }} + COMMENT_BODY: ${{ github.event.comment.body }} + ISSUE_NUMBER: ${{ github.event.issue.number }} + TAG_NAME: ${{ github.ref_name }} + ACTOR: ${{ github.actor }} + REPOSITORY: ${{ github.repository }} + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + set -euo pipefail + + should_trigger=false + git_ref="" + hadoop_version="" + + if [ "${EVENT_NAME}" = "push" ]; then + if [[ "${TAG_NAME}" =~ ^release-[0-9]+\.[0-9]+(\.[0-9]+)?$ ]]; then + should_trigger=true + git_ref="${TAG_NAME}" + echo "Tag push matched release version pattern: ${TAG_NAME}" + else + echo "Skipping tag '${TAG_NAME}' (not release-X.Y or release-X.Y.Z)." + fi + else + # Case-sensitive whole-line command match. + while IFS= read -r line || [ -n "${line}" ]; do + if [[ "${line}" =~ ^/smoke-test$ ]]; then + should_trigger=true + git_ref="pr/${ISSUE_NUMBER}" + break + elif [[ "${line}" =~ ^/smoke-test[[:space:]]+hadoop=([A-Za-z0-9._-]+)$ ]]; then + should_trigger=true + git_ref="pr/${ISSUE_NUMBER}" + hadoop_version="${BASH_REMATCH[1]}" + break + fi + done <<< "${COMMENT_BODY}" + + if [ "${should_trigger}" != true ]; then + echo "Comment is not a /smoke-test command; nothing to do." + else + echo "Checking write permission for ${ACTOR} on ${REPOSITORY}..." + perm="$(gh api "repos/${REPOSITORY}/collaborators/${ACTOR}/permission" --jq .permission)" + case "${perm}" in + admin|maintain|write) + echo "Actor permission=${perm}; allowed." + ;; + *) + echo "Actor permission=${perm}; refusing /smoke-test (need write, maintain, or admin)." + should_trigger=false + git_ref="" + hadoop_version="" + ;; + esac + fi + fi + + echo "should_trigger=${should_trigger}" >> "${GITHUB_OUTPUT}" + echo "git_ref=${git_ref}" >> "${GITHUB_OUTPUT}" + echo "hadoop_version=${hadoop_version}" >> "${GITHUB_OUTPUT}" + + - name: Trigger Jenkins smoke job + if: steps.params.outputs.should_trigger == 'true' + env: + JENKINS_SMOKE_USER: ${{ secrets.JENKINS_SMOKE_USER }} + JENKINS_SMOKE_TOKEN: ${{ secrets.JENKINS_SMOKE_TOKEN }} + GIT_REF: ${{ steps.params.outputs.git_ref }} + HADOOP_VERSION: ${{ steps.params.outputs.hadoop_version }} + run: | + set -euo pipefail + + if [ -z "${JENKINS_SMOKE_USER}" ] || [ -z "${JENKINS_SMOKE_TOKEN}" ]; then + echo "Missing secrets JENKINS_SMOKE_USER and/or JENKINS_SMOKE_TOKEN." + exit 1 + fi + + JENKINS_URL="https://ci-builds.apache.org" + JOB_URL="${JENKINS_URL}/job/Nutch/job/Nutch-Smoke-Test-Single-Node-Hadoop-Cluster" + + echo "Requesting Jenkins crumb..." + crumb_json="$(curl -sS -u "${JENKINS_SMOKE_USER}:${JENKINS_SMOKE_TOKEN}" \ + "${JENKINS_URL}/crumbIssuer/api/json")" + crumb_field="$(jq -r '.crumbRequestField' <<<"${crumb_json}")" + crumb="$(jq -r '.crumb' <<<"${crumb_json}")" + if [ -z "${crumb_field}" ] || [ "${crumb_field}" = "null" ] \ + || [ -z "${crumb}" ] || [ "${crumb}" = "null" ]; then + echo "Failed to obtain Jenkins crumb: ${crumb_json}" + exit 1 + fi + + echo "Triggering ${JOB_URL}/buildWithParameters GIT_REF=${GIT_REF}" \ + "${HADOOP_VERSION:+HADOOP_VERSION=${HADOOP_VERSION}}" + + curl_args=( + -sS -f + -u "${JENKINS_SMOKE_USER}:${JENKINS_SMOKE_TOKEN}" + -H "${crumb_field}:${crumb}" + -X POST + "${JOB_URL}/buildWithParameters" + --data-urlencode "GIT_REF=${GIT_REF}" + --data-urlencode "EMAIL_RECIPIENT=" + ) + if [ -n "${HADOOP_VERSION}" ]; then + curl_args+=(--data-urlencode "HADOOP_VERSION=${HADOOP_VERSION}") + fi + + curl "${curl_args[@]}" + echo + echo "Jenkins build queued successfully." + echo "Job: ${JOB_URL}/" diff --git a/Jenkinsfile.smoke-test-single-node-hadoop b/Jenkinsfile.smoke-test-single-node-hadoop index 8458717692..95b10a052c 100644 --- a/Jenkinsfile.smoke-test-single-node-hadoop +++ b/Jenkinsfile.smoke-test-single-node-hadoop @@ -24,10 +24,132 @@ // PRs: pr/866 (fetches pull/866/head; do not pass a bare number) // Blank/whitespace GIT_REF is treated as master. // EMAIL_RECIPIENT — optional job-result notification address -// HADOOP_VERSION — Hadoop tarball version to download (default: 3.4.2) +// HADOOP_VERSION — Hadoop tarball version to download (default: 3.5.0) // -// Paste this script into the ASF Jenkins job configure page when updating the live job. -// Landing Jenkinsfile(s) in the apache/nutch git repo is tracked separately. +// Also triggered remotely by GitHub Actions (.github/workflows/jenkins-smoke-test.yml): +// - PR comment "/smoke-test" or "/smoke-test hadoop=" (write access; created only) +// → GIT_REF=pr/, optional HADOOP_VERSION; EMAIL_RECIPIENT left empty +// - Tag push matching release-X.Y or release-X.Y.Z → GIT_REF= +// Enable remote/API triggering on the live job and configure GHA secrets +// JENKINS_SMOKE_USER / JENKINS_SMOKE_TOKEN (operator details: Nutch wiki). +// +// When GIT_REF is pr/, this pipeline creates one GitHub PR comment (ACK) and +// updates that same comment with the final result. Requires Jenkins credential +// id "nutch-github-pr-comment" (secret text: GitHub token with PR comment rights). +// +// The ASF Jenkins job loads this file from SCM (apache/nutch). Merging changes +// to this Jenkinsfile on the configured branch is enough; do not paste the +// script into the job configure page. + +import groovy.json.JsonOutput +import groovy.json.JsonSlurper + +/** Build the markdown body for the GitHub PR ACK / result comment. */ +String nutchSmokePrCommentBody(String statusEmoji, String statusText, String durationText) { + def gitRef = env.RESOLVED_GIT_REF ?: params.GIT_REF ?: 'master' + def lines = [ + "🧪 Nutch smoke test — ${statusEmoji} ${statusText}", + "🌿 GIT_REF: `${gitRef}`", + "🐘 Hadoop: `${params.HADOOP_VERSION}`", + "🏷️ Build: #${env.BUILD_NUMBER}", + ] + if (durationText) { + lines.add("⏱️ ${durationText}") + } + lines.add("🔗 ${env.BUILD_URL}") + return lines.join('\n') +} + +@NonCPS +String nutchSmokeParseGithubCommentId(String response) { + def json = new JsonSlurper().parseText(response) + return json?.id?.toString() +} + +/** + * Create or update a GitHub issue/PR comment via REST. + * credentialsId nutch-github-pr-comment must be a secret-text GitHub token. + * Returns the comment id as String, or null on failure (non-fatal). + */ +String nutchSmokeGithubPrComment(String method, String urlPath, String body) { + try { + def commentId = null + withCredentials([string(credentialsId: 'nutch-github-pr-comment', variable: 'GITHUB_TOKEN')]) { + def payload = JsonOutput.toJson([body: body]) + writeFile file: 'nutch-smoke-github-comment.json', text: payload + def response = sh( + script: """#!/bin/bash + set -euo pipefail + curl -sS -X '${method}' \\ + -H "Authorization: Bearer \${GITHUB_TOKEN}" \\ + -H "Accept: application/vnd.github+json" \\ + -H "X-GitHub-Api-Version: 2022-11-28" \\ + -H "Content-Type: application/json" \\ + --data @nutch-smoke-github-comment.json \\ + "https://api.github.com/repos/apache/nutch${urlPath}" + """, + returnStdout: true + ).trim() + commentId = nutchSmokeParseGithubCommentId(response) + if (!commentId) { + echo "GitHub PR comment ${method} did not return an id: ${response}" + } + } + return commentId + } catch (Exception e) { + echo "GitHub PR comment ${method} failed (continuing job): ${e}" + return null + } +} + +void nutchSmokePostPrAckComment() { + def gitRef = env.RESOLVED_GIT_REF ?: '' + if (!(gitRef ==~ /^pr\/[0-9]+$/)) { + return + } + env.GITHUB_PR_ID = gitRef.substring(3) + def body = nutchSmokePrCommentBody('🚀', 'running', null) + def commentId = nutchSmokeGithubPrComment( + 'POST', + "/issues/${env.GITHUB_PR_ID}/comments", + body + ) + if (commentId) { + env.GITHUB_PR_COMMENT_ID = commentId + echo "Posted GitHub PR ACK comment id=${commentId} on PR #${env.GITHUB_PR_ID}" + } +} + +void nutchSmokeUpdatePrResultComment() { + if (!env.GITHUB_PR_COMMENT_ID) { + return + } + def result = currentBuild.currentResult ?: 'SUCCESS' + def statusEmoji + def statusText + switch (result) { + case 'SUCCESS': + statusEmoji = '✅' + statusText = 'SUCCESS' + break + case 'ABORTED': + statusEmoji = '⏹️' + statusText = 'ABORTED' + break + default: + statusEmoji = '❌' + statusText = result + break + } + def durationText = currentBuild.durationString?.replace(' and counting', '')?.trim() + def body = nutchSmokePrCommentBody(statusEmoji, statusText, durationText) + nutchSmokeGithubPrComment( + 'PATCH', + "/issues/comments/${env.GITHUB_PR_COMMENT_ID}", + body + ) + echo "Updated GitHub PR comment id=${env.GITHUB_PR_COMMENT_ID} with result=${result}" +} pipeline { agent { label 'ubuntu' } @@ -66,6 +188,7 @@ pipeline { currentBuild.displayName = "#${env.BUILD_NUMBER} · ${gitRef}" currentBuild.description = "Git Ref: ${gitRef}" echo "Resolved GIT_REF=${gitRef}" + nutchSmokePostPrAckComment() } } } @@ -301,6 +424,7 @@ pipeline { post { always { script { + nutchSmokeUpdatePrResultComment() if (params.EMAIL_RECIPIENT != '') { emailext ( to: params.EMAIL_RECIPIENT,