From 77c7f14984eaf500b2f5cb47fa00e9ac7734600d Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Mon, 6 Jul 2026 06:02:10 +0000 Subject: [PATCH 1/8] Add evals job: run_evals.cjs, push_evals.cjs, evals_steps.go, evals_job.go, compiler_evals.go Co-authored-by: pelikhan <4175913+pelikhan@users.noreply.github.com> --- actions/setup/js/push_evals.cjs | 225 ++++++++++++++++++ actions/setup/js/run_evals.cjs | 268 +++++++++++++++++++++ pkg/workflow/compiler_evals.go | 104 ++++++++ pkg/workflow/compiler_jobs.go | 29 +++ pkg/workflow/evals_job.go | 102 +++++++- pkg/workflow/evals_steps.go | 409 ++++++++++++++++++++++++++++++++ pkg/workflow/jobs.go | 2 + 7 files changed, 1132 insertions(+), 7 deletions(-) create mode 100644 actions/setup/js/push_evals.cjs create mode 100644 actions/setup/js/run_evals.cjs create mode 100644 pkg/workflow/compiler_evals.go create mode 100644 pkg/workflow/evals_steps.go diff --git a/actions/setup/js/push_evals.cjs b/actions/setup/js/push_evals.cjs new file mode 100644 index 00000000000..69d067ed0fc --- /dev/null +++ b/actions/setup/js/push_evals.cjs @@ -0,0 +1,225 @@ +// @ts-check +/// + +/** + * push_evals + * + * Commits BinEval result files (evals.jsonl) to a git branch using the + * GitHub GraphQL `createCommitOnBranch` mutation so commits are + * cryptographically signed (verified) by GitHub. Falls back to a plain + * `git push` via pushSignedCommits when the GraphQL path is unavailable. + * + * Environment variables (set by the compiled workflow step): + * GH_AW_EVALS_DIR - Directory containing evals.jsonl + * e.g. /tmp/gh-aw/evals-artifact + * GH_AW_EVALS_BRANCH - Target git branch for evals results + * e.g. evals/myworkflow + * GH_TOKEN / GITHUB_TOKEN - GitHub token for API access and git operations + * GITHUB_RUN_ID - Run ID used in commit messages + * GITHUB_SERVER_URL - GitHub server URL (defaults to https://github.com) + * GITHUB_REPOSITORY - "owner/repo" of the current repository + */ + +"use strict"; + +const fs = require("fs"); +const path = require("path"); + +const { getErrorMessage } = require("./error_helpers.cjs"); +const { execGitSync, getGitAuthEnv } = require("./git_helpers.cjs"); +const { pushSignedCommits } = require("./push_signed_commits.cjs"); + +/** + * Checkout or create an orphan git branch for evals results. + * Returns the remote HEAD SHA (empty string for a new branch). + * + * @param {string} branchName - Target branch name (e.g. "evals/myworkflow") + * @param {string} repoUrl - Authenticated HTTPS URL of the target repo + * @param {string} workspaceDir - Local git workspace directory + * @returns {string} baseRef (empty string when branch is brand new) + */ +function checkoutOrCreateBranch(branchName, repoUrl, workspaceDir) { + try { + execGitSync(["fetch", repoUrl, `${branchName}:${branchName}`], { + stdio: "pipe", + cwd: workspaceDir, + suppressLogs: true, + }); + execGitSync(["checkout", branchName], { stdio: "inherit", cwd: workspaceDir }); + const baseRef = execGitSync(["rev-parse", "HEAD"], { cwd: workspaceDir }).trim(); + core.info(`Checked out existing branch ${branchName}, baseRef=${baseRef}`); + return baseRef; + } catch (fetchErr) { + const msg = getErrorMessage(fetchErr); + const isMissing = /couldn't find remote ref/i.test(msg) || /remote branch .* not found/i.test(msg); + if (!isMissing) throw fetchErr; + + // Branch does not exist yet – create an orphan branch. + core.info(`Branch ${branchName} does not exist, creating orphan branch...`); + execGitSync(["checkout", "--orphan", branchName], { stdio: "inherit", cwd: workspaceDir }); + execGitSync(["read-tree", "--empty"], { stdio: "pipe", cwd: workspaceDir }); + // Remove any pre-existing working-tree files (from sparse checkout). + for (const entry of fs.readdirSync(workspaceDir)) { + if (entry !== ".git") { + fs.rmSync(path.join(workspaceDir, entry), { recursive: true, force: true }); + } + } + return ""; + } +} + +/** + * Main entry point called by the actions/github-script step. + */ +async function main() { + const evalsDir = process.env.GH_AW_EVALS_DIR || "/tmp/gh-aw/evals-artifact"; + const branchName = process.env.GH_AW_EVALS_BRANCH || ""; + const ghToken = process.env.GH_TOKEN || process.env.GITHUB_TOKEN || ""; + const githubRunId = process.env.GITHUB_RUN_ID || "unknown"; + const githubServerUrl = (process.env.GITHUB_SERVER_URL || "https://github.com").replace(/\/$/, ""); + const serverHost = githubServerUrl.replace(/^https?:\/\//, ""); + + if (!branchName) { + core.setFailed("GH_AW_EVALS_BRANCH is not set"); + return; + } + if (!ghToken) { + core.setFailed("GH_TOKEN or GITHUB_TOKEN is not set"); + return; + } + + const targetRepo = `${context.repo.owner}/${context.repo.repo}`; + const allowedRepos = new Set( + (process.env.GH_AW_ALLOWED_TARGET_REPOS || targetRepo) + .split(",") + .map(repo => repo.trim()) + .filter(Boolean) + ); + if (!allowedRepos.has(targetRepo)) { + core.setFailed(`Target repository "${targetRepo}" is not in GH_AW_ALLOWED_TARGET_REPOS. ` + `Current allowlist: ${Array.from(allowedRepos).join(", ")}`); + return; + } + const [owner, repo] = targetRepo.split("/"); + + core.info(`Pushing evals results to branch "${branchName}" in ${targetRepo}`); + + // Collect evals result files present in the artifact directory + const candidateFiles = ["evals.jsonl"]; + const filesToPush = candidateFiles.filter(name => { + const full = path.join(evalsDir, name); + return fs.existsSync(full) && fs.statSync(full).isFile(); + }); + + if (filesToPush.length === 0) { + core.info("No evals result files found – nothing to push"); + return; + } + + core.info(`Files to push: ${filesToPush.join(", ")}`); + + const workspaceDir = process.env.GITHUB_WORKSPACE || process.cwd(); + const repoUrl = `https://x-access-token:${ghToken}@${serverHost}/${targetRepo}.git`; + + // Checkout the target branch (or create it as an orphan on first run). + let baseRef; + try { + baseRef = checkoutOrCreateBranch(branchName, repoUrl, workspaceDir); + } catch (err) { + core.setFailed(`Failed to checkout branch "${branchName}": ${getErrorMessage(err)}`); + return; + } + + // Copy evals files into the workspace root. + for (const name of filesToPush) { + const src = path.join(evalsDir, name); + const dest = path.join(workspaceDir, name); + try { + fs.copyFileSync(src, dest); + core.info(`Copied ${name}`); + } catch (err) { + core.setFailed(`Failed to copy ${name}: ${getErrorMessage(err)}`); + return; + } + } + + // Stage all changes. + try { + execGitSync(["add", "--sparse", "."], { stdio: "inherit", cwd: workspaceDir }); + } catch (err) { + core.setFailed(`Failed to stage changes: ${getErrorMessage(err)}`); + return; + } + + // Check whether there are any staged changes to commit. + const status = execGitSync(["status", "--porcelain"], { cwd: workspaceDir }).trim(); + if (!status) { + core.info("No changes to evals results – skipping push"); + return; + } + + // Commit. + try { + execGitSync(["commit", "-m", `Update evals results from workflow run ${githubRunId}`], { stdio: "inherit", cwd: workspaceDir }); + } catch (err) { + core.setFailed(`Failed to commit evals results: ${getErrorMessage(err)}`); + return; + } + + // Point origin at the target repo so pushSignedCommits can resolve the remote branch HEAD. + execGitSync(["remote", "set-url", "origin", `https://${serverHost}/${targetRepo}.git`], { stdio: "pipe", cwd: workspaceDir }); + + // Push using GraphQL createCommitOnBranch (signed commits) with a plain-git fallback. + const MAX_RETRIES = 3; + const BASE_DELAY_MS = 1000; + let currentBaseRef = baseRef; + + for (let attempt = 0; attempt <= MAX_RETRIES; attempt++) { + core.info(`Pushing to ${branchName} (attempt ${attempt + 1}/${MAX_RETRIES + 1})...`); + try { + await pushSignedCommits({ + githubClient: github, + owner, + repo, + branch: branchName, + baseRef: currentBaseRef, + cwd: workspaceDir, + gitAuthEnv: getGitAuthEnv(ghToken), + }); + core.info(`Successfully pushed evals results to ${branchName}`); + return; + } catch (err) { + const errMsg = getErrorMessage(err); + if (attempt < MAX_RETRIES) { + const delay = BASE_DELAY_MS * Math.pow(2, attempt); + core.warning(`Push failed (attempt ${attempt + 1}/${MAX_RETRIES + 1}), retrying in ${delay}ms: ${errMsg}`); + await new Promise(resolve => setTimeout(resolve, delay)); + + // Refresh baseRef and fetch the updated remote history so that + // pushSignedCommits can resolve the new baseRef in git rev-list. + try { + const { stdout: lsOut } = await exec.getExecOutput("git", ["ls-remote", "origin", `refs/heads/${branchName}`], { cwd: workspaceDir }); + const remoteHead = lsOut.trim().split(/\s+/)[0] || ""; + if (remoteHead && remoteHead !== currentBaseRef) { + currentBaseRef = remoteHead; + core.info(`Refreshed baseRef for retry: ${currentBaseRef}`); + try { + execGitSync(["fetch", "origin", `refs/heads/${branchName}`], { + stdio: "pipe", + cwd: workspaceDir, + suppressLogs: true, + }); + } catch (fetchErr) { + core.info(`Fetch of branch "${branchName}" on retry failed (non-fatal): ${getErrorMessage(fetchErr)}`); + } + } + } catch { + // ls-remote failed; keep existing baseRef + } + } else { + core.setFailed(`Failed to push evals results after ${MAX_RETRIES + 1} attempts: ${errMsg}`); + } + } + } +} + +module.exports = { main, checkoutOrCreateBranch }; diff --git a/actions/setup/js/run_evals.cjs b/actions/setup/js/run_evals.cjs new file mode 100644 index 00000000000..1710d9d0c01 --- /dev/null +++ b/actions/setup/js/run_evals.cjs @@ -0,0 +1,268 @@ +// @ts-check +/// + +/** + * run_evals — BinEval binary evaluation harness. + * + * This module operates in two phases selected by GH_AW_EVALS_PHASE: + * + * Phase "setup" (default, runs BEFORE the agentic engine): + * - Reads configured eval questions from GH_AW_EVALS_QUESTIONS (JSON array) + * - Reads the agent output from /tmp/gh-aw/evals/agent_output.json + * - Builds a multi-question binary evaluation prompt + * - Writes the prompt to /tmp/gh-aw/aw-prompts/prompt.txt for the engine + * + * Phase "parse" (runs AFTER the agentic engine): + * - Reads the engine output log from /tmp/gh-aw/evals/evals.log + * - Extracts YES/NO answer for each question by ID or by position + * - Writes structured results to /tmp/gh-aw/evals.jsonl + * + * Environment variables: + * GH_AW_EVALS_QUESTIONS JSON array of { id, question } objects + * GH_AW_EVALS_PHASE "setup" (default) or "parse" + * GH_AW_EVALS_MODEL LLM model name recorded in output metadata + * + * Design note: this file is intentionally engine-agnostic. The engine is + * installed and executed by separate Go-generated GitHub Actions steps that + * call engine.GetInstallationSteps / engine.GetExecutionSteps; this module + * only handles prompt construction and result parsing. + */ + +"use strict"; + +const fs = require("fs"); +const path = require("path"); + +const EVALS_DIR = "/tmp/gh-aw/evals"; +const EVALS_LOG_PATH = "/tmp/gh-aw/evals/evals.log"; +const EVALS_OUTPUT_PATH = "/tmp/gh-aw/evals.jsonl"; +const AGENT_OUTPUT_FILENAME = "agent_output.json"; + +// --------------------------------------------------------------------------- +// Phase 1 – setup: write multi-question evaluation prompt +// --------------------------------------------------------------------------- + +/** + * Reads eval questions and agent output, constructs a BinEval prompt, and + * writes it to the standard GH_AW_PROMPT path for the agentic engine. + * @returns {Promise} + */ +async function setupMain() { + const questionsRaw = process.env.GH_AW_EVALS_QUESTIONS; + if (!questionsRaw) { + core.setFailed("ERR_VALIDATION: GH_AW_EVALS_QUESTIONS is not set"); + return; + } + + let questions; + try { + questions = JSON.parse(questionsRaw); + } catch (e) { + core.setFailed("ERR_VALIDATION: GH_AW_EVALS_QUESTIONS is not valid JSON: " + e.message); + return; + } + + if (!Array.isArray(questions) || questions.length === 0) { + core.setFailed("ERR_VALIDATION: GH_AW_EVALS_QUESTIONS must be a non-empty JSON array"); + return; + } + + fs.mkdirSync(EVALS_DIR, { recursive: true }); + + // Load agent output for evaluation context + const agentOutputPath = path.join(EVALS_DIR, AGENT_OUTPUT_FILENAME); + let agentOutputContent = ""; + if (fs.existsSync(agentOutputPath)) { + const stats = fs.statSync(agentOutputPath); + agentOutputContent = fs.readFileSync(agentOutputPath, "utf-8"); + core.info(`Agent output loaded: ${agentOutputPath} (${stats.size} bytes)`); + } else { + core.warning(`ERR_VALIDATION: Agent output not found at ${agentOutputPath}. ` + "Ensure the agent artifact includes agent_output.json. " + "Evaluation will proceed without agent context."); + } + + const prompt = buildEvalPrompt(questions, agentOutputContent); + + fs.mkdirSync("/tmp/gh-aw/aw-prompts", { recursive: true }); + fs.writeFileSync("/tmp/gh-aw/aw-prompts/prompt.txt", prompt); + core.exportVariable("GH_AW_PROMPT", "/tmp/gh-aw/aw-prompts/prompt.txt"); + + core.info(`BinEval setup complete: wrote prompt with ${questions.length} question(s)`); + + await core.summary + .addRaw("
\nBinEval Evaluation Prompt\n\n") + .addRaw("``````markdown\n" + prompt + "\n``````\n\n
\n") + .write(); +} + +// --------------------------------------------------------------------------- +// Phase 2 – parse: extract answers and write evals.jsonl +// --------------------------------------------------------------------------- + +/** + * Reads the engine log, extracts per-question YES/NO answers, and writes + * structured JSONL records to the evals output file. + * @returns {Promise} + */ +async function parseMain() { + const questionsRaw = process.env.GH_AW_EVALS_QUESTIONS; + const model = process.env.GH_AW_EVALS_MODEL || ""; + + /** @type {Array<{id: string, question: string}>} */ + let questions = []; + if (questionsRaw) { + try { + questions = JSON.parse(questionsRaw); + } catch { + core.warning("GH_AW_EVALS_QUESTIONS is not valid JSON; result IDs will be positional"); + } + } + + if (!fs.existsSync(EVALS_LOG_PATH)) { + core.warning(`ERR_VALIDATION: Evals log not found at ${EVALS_LOG_PATH}; no results written`); + fs.writeFileSync(EVALS_OUTPUT_PATH, ""); + return; + } + + const logContent = fs.readFileSync(EVALS_LOG_PATH, "utf-8"); + core.info(`Parsing evals log: ${EVALS_LOG_PATH} (${logContent.length} bytes)`); + + // Collect all positional Q1/Q2/... answers from the log for fallback lookup + const positionalAnswers = extractAllPositionalAnswers(logContent); + + const timestamp = new Date().toISOString(); + const results = []; + + for (let i = 0; i < questions.length; i++) { + const q = questions[i]; + + // Try ID-specific match first (e.g. "builds: YES"), then positional (Q1: YES) + let answer = extractAnswerByID(logContent, q.id); + if (answer === "UNKNOWN" && i < positionalAnswers.length && positionalAnswers[i]) { + answer = positionalAnswers[i]; + } + + const record = { + id: q.id, + question: q.question, + answer, + model, + timestamp, + }; + results.push(record); + core.info(`Q[${q.id}]: ${answer}`); + } + + // Write JSONL — one JSON object per line + const jsonlLines = results.map(r => JSON.stringify(r)); + fs.writeFileSync(EVALS_OUTPUT_PATH, jsonlLines.join("\n") + (jsonlLines.length > 0 ? "\n" : "")); + core.info(`BinEval results written to ${EVALS_OUTPUT_PATH} (${results.length} record(s))`); + + const yesCount = results.filter(r => r.answer === "YES").length; + const noCount = results.filter(r => r.answer === "NO").length; + const unknownCount = results.filter(r => r.answer === "UNKNOWN").length; + + await core.summary + .addHeading("BinEval Results", 2) + .addTable([ + [ + { data: "ID", header: true }, + { data: "Question", header: true }, + { data: "Answer", header: true }, + ], + ...results.map(r => [r.id, r.question, r.answer]), + ["", `YES: ${yesCount} | NO: ${noCount} | UNKNOWN: ${unknownCount}`, ""], + ]) + .write(); +} + +// --------------------------------------------------------------------------- +// Main entry point +// --------------------------------------------------------------------------- + +/** + * Dispatches to setupMain or parseMain based on GH_AW_EVALS_PHASE. + * @returns {Promise} + */ +async function main() { + const phase = process.env.GH_AW_EVALS_PHASE || "setup"; + if (phase === "parse") { + await parseMain(); + } else { + await setupMain(); + } +} + +// --------------------------------------------------------------------------- +// Helpers +// --------------------------------------------------------------------------- + +/** + * Builds a multi-question binary evaluation prompt. + * @param {Array<{id: string, question: string}>} questions + * @param {string} agentOutput + * @returns {string} + */ +function buildEvalPrompt(questions, agentOutput) { + const questionList = questions.map((q, i) => `Q${i + 1} [id=${q.id}]: ${q.question}`).join("\n"); + + const agentSection = agentOutput ? `## Agent Output\n\n${agentOutput}` : "## Agent Output\n\n(no agent output available)"; + + return `# BinEval: Binary Evaluation + +You are evaluating the output of an AI agentic workflow using BinEval (binary evaluation). +For each question below, answer with exactly YES or NO based on the agent output provided. + +## Questions + +${questionList} + +${agentSection} + +## Instructions + +Answer each question on a separate line using EXACTLY this format: +Q1: YES +Q2: NO + +Use only YES or NO. Do not provide explanations or reasoning. +Evaluate each question solely based on the agent output shown above.`; +} + +/** + * Extracts all positional Q1/Q2/... answers from log content. + * Returns a 0-indexed array where index 0 = Q1's answer. + * @param {string} logContent + * @returns {string[]} + */ +function extractAllPositionalAnswers(logContent) { + /** @type {string[]} */ + const answers = []; + for (const line of logContent.split("\n")) { + const match = line.trim().match(/^Q(\d+):\s+(YES|NO)\b/i); + if (match) { + const idx = parseInt(match[1], 10) - 1; // Convert 1-indexed to 0-indexed + if (idx >= 0) { + answers[idx] = match[2].toUpperCase(); + } + } + } + return answers; +} + +/** + * Tries to find an answer for a question by its id using flexible pattern matching. + * Returns "YES", "NO", or "UNKNOWN". + * @param {string} logContent + * @param {string} id + * @returns {string} + */ +function extractAnswerByID(logContent, id) { + const escaped = id.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"); + const yesPattern = new RegExp(`\\b${escaped}\\b[:\\s]+(YES)\\b`, "i"); + const noPattern = new RegExp(`\\b${escaped}\\b[:\\s]+(NO)\\b`, "i"); + if (yesPattern.test(logContent)) return "YES"; + if (noPattern.test(logContent)) return "NO"; + return "UNKNOWN"; +} + +module.exports = { main, setupMain, parseMain }; diff --git a/pkg/workflow/compiler_evals.go b/pkg/workflow/compiler_evals.go new file mode 100644 index 00000000000..39496427df2 --- /dev/null +++ b/pkg/workflow/compiler_evals.go @@ -0,0 +1,104 @@ +// Package workflow - push_evals job assembler. +package workflow + +import ( + "fmt" + "strings" + + "github.com/github/gh-aw/pkg/constants" +) + +// buildPushEvalsJob creates a job that downloads the evals artifact and commits it +// to a git branch ("evals/{sanitizedWorkflowID}") for durable storage across runs. +// Returns nil when evals are not declared in the workflow frontmatter. +func (c *Compiler) buildPushEvalsJob(data *WorkflowData) (*Job, error) { + if !data.Evals.HasEvals() { + return nil, nil + } + + var steps []string + + // Setup step so the push_evals.cjs script is available. + setupActionRef := c.resolveActionReference("./actions/setup", data) + if setupActionRef != "" || c.actionMode.IsScript() { + steps = append(steps, c.generateCheckoutActionsFolder(data)...) + traceID := fmt.Sprintf("${{ needs.%s.outputs.setup-trace-id }}", constants.ActivationJobName) + parentSpanID := setupParentSpanNeedsExpr(constants.ActivationJobName) + steps = append(steps, c.generateSetupStep(data, setupActionRef, SetupActionDestination, false, traceID, parentSpanID)...) + } + + // Checkout step – configure git credentials without downloading workspace files. + var checkoutStep strings.Builder + checkoutStep.WriteString(" - name: Checkout repository\n") + fmt.Fprintf(&checkoutStep, " uses: %s\n", getActionPin("actions/checkout")) + checkoutStep.WriteString(" with:\n") + checkoutStep.WriteString(" persist-credentials: false\n") + checkoutStep.WriteString(" sparse-checkout: .\n") + steps = append(steps, checkoutStep.String()) + + // Git configuration (author, email). + steps = append(steps, c.generateGitConfigurationSteps()...) + + // Download the evals artifact uploaded by the evals job. + evalsArtifactName := artifactPrefixExprForAgentDownstreamJob(data) + constants.EvalsArtifactName + var downloadStep strings.Builder + downloadStep.WriteString(" - name: Download evals artifact\n") + fmt.Fprintf(&downloadStep, " uses: %s\n", c.getActionPin("actions/download-artifact")) + downloadStep.WriteString(" continue-on-error: true\n") + downloadStep.WriteString(" with:\n") + fmt.Fprintf(&downloadStep, " name: %s\n", evalsArtifactName) + fmt.Fprintf(&downloadStep, " path: %s\n", evalsArtifactDownloadDir) + steps = append(steps, downloadStep.String()) + + // Push evals results to the git branch via push_evals.cjs. + branchName := evalsBranchName(data.WorkflowID) + + var pushStep strings.Builder + pushStep.WriteString(" - name: Push evals results to git\n") + pushStep.WriteString(" id: push_evals\n") + pushStep.WriteString(" if: always()\n") + fmt.Fprintf(&pushStep, " uses: %s\n", getCachedActionPin("actions/github-script", data)) + pushStep.WriteString(" env:\n") + pushStep.WriteString(" GH_TOKEN: ${{ github.token }}\n") + pushStep.WriteString(" GITHUB_RUN_ID: ${{ github.run_id }}\n") + pushStep.WriteString(" GITHUB_SERVER_URL: ${{ github.server_url }}\n") + fmt.Fprintf(&pushStep, " GH_AW_EVALS_DIR: %s\n", evalsArtifactDownloadDir) + fmt.Fprintf(&pushStep, " GH_AW_EVALS_BRANCH: %s\n", branchName) + pushStep.WriteString(" with:\n") + pushStep.WriteString(" script: |\n") + pushStep.WriteString(" const { setupGlobals } = require('" + SetupActionDestination + "/setup_globals.cjs');\n") + pushStep.WriteString(" setupGlobals(core, github, context, exec, io, getOctokit);\n") + pushStep.WriteString(" const { main } = require('" + SetupActionDestination + "/push_evals.cjs');\n") + pushStep.WriteString(" await main();\n") + steps = append(steps, pushStep.String()) + + // Restore the checkout in dev mode (same reason as push_repo_memory and push_experiments_state). + if c.actionMode.IsDev() { + steps = append(steps, c.generateRestoreActionsSetupStep()) + } + + // The push_evals job runs after the evals job completes (success or failure). + // It does not block on evals succeeding — we always want to persist whatever results were generated. + evalsNotFailure := BuildNotEquals( + BuildPropertyAccess(fmt.Sprintf("needs.%s.result", constants.EvalsJobName)), + BuildStringLiteral("failure"), + ) + notCancelled := &NotNode{Child: BuildFunctionCall("cancelled")} + jobCondition := RenderCondition(BuildAnd(BuildAnd(BuildFunctionCall("always"), notCancelled), evalsNotFailure)) + + job := &Job{ + Name: pushEvalsJobName, + RunsOn: c.formatFrameworkJobRunsOn(data), + If: jobCondition, + Permissions: "permissions:\n contents: write", + Needs: []string{string(constants.EvalsJobName), string(constants.ActivationJobName)}, + Steps: steps, + } + + return job, nil +} + +// evalsBranchName returns the git branch name used to persist evals results for the given workflow ID. +func evalsBranchName(workflowID string) string { + return string(constants.EvalsBranchPrefix) + "/" + SanitizeWorkflowIDForCacheKey(workflowID) +} diff --git a/pkg/workflow/compiler_jobs.go b/pkg/workflow/compiler_jobs.go index f12c8d1c509..b13a209d82d 100644 --- a/pkg/workflow/compiler_jobs.go +++ b/pkg/workflow/compiler_jobs.go @@ -348,6 +348,11 @@ func (c *Compiler) buildMemoryManagementJobs(data *WorkflowData) error { return err } + // Build push_evals job when evals are declared + if err := c.buildPushEvalsJobAndAdd(data); err != nil { + return err + } + // Update conclusion job dependencies if err := c.updateConclusionJobDependencies(pushRepoMemoryJobName, updateCacheMemoryJobName, pushExperimentsJobName); err != nil { return err @@ -438,6 +443,30 @@ func (c *Compiler) buildPushExperimentsStateJobWrapper(data *WorkflowData) (stri return job.Name, nil } +// buildPushEvalsJobAndAdd builds the push_evals job (if evals are declared) and +// adds it to the job manager. The conclusion job picks it up via ensureConclusionIsLastJob. +func (c *Compiler) buildPushEvalsJobAndAdd(data *WorkflowData) error { + if !data.Evals.HasEvals() { + return nil + } + + compilerJobsLog.Print("Building push_evals job") + job, err := c.buildPushEvalsJob(data) + if err != nil { + return fmt.Errorf("failed to build push_evals job: %w", err) + } + if job == nil { + return nil + } + + if err := c.jobManager.AddJob(job); err != nil { + return fmt.Errorf("failed to add push_evals job: %w", err) + } + + compilerJobsLog.Printf("Successfully added push_evals job: %s", job.Name) + return nil +} + // updateConclusionJobDependencies updates the conclusion job to depend on memory management jobs if they exist. func (c *Compiler) updateConclusionJobDependencies(pushRepoMemoryJobName, updateCacheMemoryJobName, pushExperimentsJobName string) error { conclusionJob, exists := c.jobManager.GetJob("conclusion") diff --git a/pkg/workflow/evals_job.go b/pkg/workflow/evals_job.go index 3349923290d..0e21f26b92e 100644 --- a/pkg/workflow/evals_job.go +++ b/pkg/workflow/evals_job.go @@ -1,12 +1,100 @@ // Package workflow - BinEval evaluation job assembler. package workflow -// buildEvalsJob will build the evals job for BinEval-style evaluations. -// -// TODO: Implement evals job using the detection job as a base, sharing the -// job harness infrastructure with a different harness, artifact name, and output names. -// -// Returns nil (no-op) until the implementation is complete. +import ( + "fmt" + + "github.com/github/gh-aw/pkg/constants" +) + +// buildEvalsJob creates a separate evals job that runs after the safe_outputs job +// (or directly after the agent job if safe_outputs is not configured). +// The job downloads the agent artifact to access output files, runs a BinEval +// multi-question evaluation via an agentic engine, and uploads evals.jsonl as an artifact. +// Returns nil if evals are not declared in the workflow frontmatter. func (c *Compiler) buildEvalsJob(data *WorkflowData) (*Job, error) { - return nil, nil + if !data.Evals.HasEvals() { + return nil, nil + } + + var steps []string + + // Add setup action steps (installs the agentic engine helper scripts). + setupActionRef := c.resolveActionReference("./actions/setup", data) + if setupActionRef != "" || c.actionMode.IsScript() { + // For dev mode (local action path), checkout the actions folder first. + steps = append(steps, c.generateCheckoutActionsFolder(data)...) + // Reuse the activation job trace ID so all jobs share one OTLP trace. + evalsTraceID := fmt.Sprintf("${{ needs.%s.outputs.setup-trace-id }}", constants.ActivationJobName) + evalsParentSpanID := setupParentSpanNeedsExpr(constants.ActivationJobName) + steps = append(steps, c.generateSetupStep(data, setupActionRef, SetupActionDestination, false, evalsTraceID, evalsParentSpanID)...) + } + + // Download agent output artifact to access output files (prompt.txt, agent_output.json). + // Use agent-downstream prefix since this job depends on the agent job (via safe_outputs). + agentArtifactPrefix := artifactPrefixExprForAgentDownstreamJob(data) + steps = append(steps, buildAgentOutputDownloadSteps(agentArtifactPrefix, c.getActionPin)...) + + // Download experiment artifact so the evals agent can read the current variant assignments. + steps = append(steps, buildExperimentArtifactDownloadSteps(data, c.getActionPin)...) + + // Add all evals steps: engine install, engine execution, parse, redact, upload. + steps = append(steps, c.buildEvalsJobSteps(data)...) + + // Determine job dependencies. + // Evals runs after safe_outputs when it is configured; otherwise directly after agent. + var needs []string + if data.SafeOutputs != nil { + needs = []string{string(constants.SafeOutputsJobName), string(constants.ActivationJobName)} + } else { + needs = []string{string(constants.AgentJobName), string(constants.ActivationJobName)} + } + + // Evals job condition: always run but skip if the upstream job was skipped. + // This matches the detection job pattern so conclusion still sees a non-skipped evals result. + var upstreamJobName string + if data.SafeOutputs != nil { + upstreamJobName = string(constants.SafeOutputsJobName) + } else { + upstreamJobName = string(constants.AgentJobName) + } + alwaysFunc := BuildFunctionCall("always") + upstreamNotSkipped := BuildNotEquals( + BuildPropertyAccess(fmt.Sprintf("needs.%s.result", upstreamJobName)), + BuildStringLiteral("skipped"), + ) + jobConditionNode := BuildAnd(alwaysFunc, upstreamNotSkipped) + jobCondition := RenderCondition(jobConditionNode) + + // Determine runs-on: use evals override if set, otherwise ubuntu-latest. + runsOn := "runs-on: ubuntu-latest" + if data.Evals != nil && data.Evals.RunsOn != "" { + runsOn = normalizeRunsOnSnippet(data.Evals.RunsOn) + } + + // Determine permissions for the evals job (same rationale as the detection job). + copilotRequestsEnabled := hasCopilotRequestsWritePermission(data) + perms := NewPermissionsContentsRead() + if copilotRequestsEnabled { + perms.Set(PermissionCopilotRequests, PermissionWrite) + } + if data.EngineConfig != nil && data.EngineConfig.Auth != nil && data.EngineConfig.Auth.Type == "github-oidc" { + perms.Set(PermissionIdToken, PermissionWrite) + } + if hasOTLPGitHubOIDCAuth(data.ParsedFrontmatter, data.RawFrontmatter) { + perms.Set(PermissionIdToken, PermissionWrite) + } + permissions := perms.RenderToYAML() + + job := &Job{ + Name: string(constants.EvalsJobName), + Needs: needs, + If: jobCondition, + RunsOn: c.indentYAMLLines(runsOn, " "), + Environment: c.indentYAMLLines(data.Environment, " "), + Permissions: permissions, + Steps: steps, + } + + return job, nil } diff --git a/pkg/workflow/evals_steps.go b/pkg/workflow/evals_steps.go new file mode 100644 index 00000000000..46aa5a4dea1 --- /dev/null +++ b/pkg/workflow/evals_steps.go @@ -0,0 +1,409 @@ +// Package workflow - step builders for the BinEval evaluation job. +package workflow + +import ( + "fmt" + "strings" + + "github.com/github/gh-aw/pkg/constants" + "github.com/github/gh-aw/pkg/logger" + "github.com/github/gh-aw/pkg/workflow/compilerenv" +) + +var evalsStepsLog = logger.New("workflow:evals_steps") + +const ( + // evalsDir is the BinEval working directory on the runner. + evalsDir = "/tmp/gh-aw/evals" + + // evalsLogPath is the engine output log written during the evals engine execution step. + evalsLogPath = "/tmp/gh-aw/evals/evals.log" + + // evalsResultsPath is the parsed JSONL results file produced by the parse step. + evalsResultsPath = "/tmp/gh-aw/" + constants.EvalsResultFilename + + // evalsArtifactDownloadDir is the directory into which the evals artifact is + // downloaded in the push_evals job. + evalsArtifactDownloadDir = "/tmp/gh-aw/evals-artifact" +) + +// buildEvalsJobSteps builds all steps that run inside the evals job. +// The steps analyse the agent artifact using a BinEval prompt and write +// per-question YES/NO results to evals.jsonl. +func (c *Compiler) buildEvalsJobSteps(data *WorkflowData) []string { + if !data.Evals.HasEvals() { + return nil + } + + var steps []string + + steps = append(steps, " # --- BinEval Evaluations ---\n") + + // Step 1: Clean stale firewall files from the agent artifact download so the + // AWF squid container does not fail when the evals job pre-pulls images. + steps = append(steps, c.buildCleanFirewallDirsStep()...) + + // Step 2: Pre-pull AWF container images for faster engine execution. + steps = append(steps, c.buildPullAWFContainersStep(data)...) + + // Step 3: Copy agent output files into the evals working directory. + steps = append(steps, buildPrepareEvalsFilesStep()...) + + // Step 4: Setup evals – writes the multi-question BinEval prompt via JS. + steps = append(steps, c.buildSetupEvalsStep(data)...) + + // Step 5: Ensure the evals directory and log file exist before engine execution. + steps = append(steps, buildEnsureEvalsDirStep()...) + + // Steps 6 & 7: Install engine and execute via AWF (network-restricted sandbox). + steps = append(steps, c.buildEvalsEngineSteps(data)...) + + // Step 8: Parse engine output and write evals.jsonl. + steps = append(steps, c.buildParseEvalsResultsStep(data)...) + + // Step 9: Redact secrets from evals results before upload. + steps = append(steps, c.buildRedactEvalsSecretsStep(data)...) + + // Step 10: Upload evals.jsonl as the evals artifact. + steps = append(steps, c.buildUploadEvalsArtifactStep(data)...) + + return steps +} + +// buildPrepareEvalsFilesStep creates a step that copies agent output files into the +// evals working directory so the JS harness can read them for context. +func buildPrepareEvalsFilesStep() []string { + return []string{ + " - name: Prepare evals files\n", + " run: |\n", + fmt.Sprintf(" mkdir -p %s\n", evalsDir), + " cp /tmp/gh-aw/agent_output.json " + evalsDir + "/agent_output.json 2>/dev/null || true\n", + " cp /tmp/gh-aw/aw-prompts/prompt.txt " + evalsDir + "/prompt.txt 2>/dev/null || true\n", + fmt.Sprintf(" ls -la %s/ 2>/dev/null || true\n", evalsDir), + } +} + +// buildEnsureEvalsDirStep creates a step that ensures the evals directory and log +// file are present before the engine execution step writes to them. +func buildEnsureEvalsDirStep() []string { + return []string{ + " - name: Ensure evals directory and log\n", + " run: |\n", + fmt.Sprintf(" mkdir -p %s\n", evalsDir), + fmt.Sprintf(" touch %s\n", evalsLogPath), + } +} + +// buildSetupEvalsStep creates the github-script step that writes the multi-question +// BinEval evaluation prompt to /tmp/gh-aw/aw-prompts/prompt.txt. +func (c *Compiler) buildSetupEvalsStep(data *WorkflowData) []string { + if data.Evals == nil { + return nil + } + + questionsJSON := marshalEvalsQuestions(data.Evals.Questions) + model := data.Evals.Model + if model == "" { + model = "small" + } + + script := `const { setupGlobals } = require('` + SetupActionDestination + `/setup_globals.cjs'); +setupGlobals(core, github, context, exec, io, getOctokit); +const { main } = require('` + SetupActionDestination + `/run_evals.cjs'); +await main();` + + steps := []string{ + " - name: Setup BinEval evaluations\n", + " if: always()\n", + fmt.Sprintf(" uses: %s\n", getCachedActionPin("actions/github-script", data)), + " env:\n", + fmt.Sprintf(" GH_AW_EVALS_QUESTIONS: %s\n", quoteSingleLineYAML(questionsJSON)), + fmt.Sprintf(" GH_AW_EVALS_MODEL: %q\n", model), + " GH_AW_EVALS_PHASE: setup\n", + " with:\n", + " script: |\n", + } + steps = append(steps, FormatJavaScriptForYAML(script)...) + return steps +} + +// buildEvalsEngineSteps generates the engine installation and engine execution steps +// for the evals job. These mirror the inline detection engine execution path: +// 1. Install the agentic engine (same binary as the agent job) +// 2. Execute the engine through AWF (network-restricted sandbox) to answer eval questions +func (c *Compiler) buildEvalsEngineSteps(data *WorkflowData) []string { + // Determine engine ID (same resolution order as detection). + engineID := c.getEvalsEngineID(data) + + engineConfig := data.EngineConfig + // Build the evals engine config inheriting from the main engine config. + evalsEngineConfig := engineConfig + if evalsEngineConfig == nil { + evalsEngineConfig = &EngineConfig{ID: engineID} + } else { + evalsEngineConfig = &EngineConfig{ + ID: evalsEngineConfig.ID, + Model: evalsEngineConfig.Model, + Version: evalsEngineConfig.Version, + Env: evalsEngineConfig.Env, + Config: evalsEngineConfig.Config, + Args: evalsEngineConfig.Args, + APITarget: evalsEngineConfig.APITarget, + HarnessScript: evalsEngineConfig.HarnessScript, + Driver: evalsEngineConfig.Driver, + } + } + if evalsEngineConfig.ID == "" { + evalsEngineConfig.ID = engineID + } + + // Override model from evals frontmatter if specified. + if data.Evals != nil && data.Evals.Model != "" { + evalsEngineConfig.Model = data.Evals.Model + } + + // Apply engine and enterprise default detection model (cost-effective for Q&A tasks). + engine, err := c.getAgenticEngine(engineID) + if err != nil { + return []string{ + " # Evals engine not available, skipping engine installation and execution\n", + } + } + + if evalsEngineConfig.Model == "" { + if defaultModel := compilerenv.ResolveDefaultDetectionModel(""); defaultModel != "" { + evalsEngineConfig.Model = defaultModel + } else if defaultModel := engine.GetDefaultDetectionModel(); defaultModel != "" { + evalsEngineConfig.Model = defaultModel + } + } + + // Inherit APITarget from the main engine config for GHE/custom endpoints. + if evalsEngineConfig.APITarget == "" && data.EngineConfig != nil && data.EngineConfig.APITarget != "" { + evalsEngineConfig.APITarget = data.EngineConfig.APITarget + } + + // Normalize Pi engine model to bare model ID for Copilot CLI. + originalEngineID := data.AI + if data.EngineConfig != nil && data.EngineConfig.ID != "" { + originalEngineID = data.EngineConfig.ID + } + if engineID == "copilot" && originalEngineID == "pi" { + evalsEngineConfig.Model = extractPiModelID(evalsEngineConfig.Model) + } + + // Build a minimal WorkflowData for evals engine execution. + // IsDetectionRun reuses detection-style network restrictions and MaxAI credits, + // which are appropriate for binary (YES/NO) evaluation tasks. + evalsData := &WorkflowData{ + Tools: map[string]any{ + "bash": []any{"*"}, + }, + SafeOutputs: nil, + EngineConfig: evalsEngineConfig, + AI: engineID, + Features: data.Features, + Permissions: data.Permissions, + CachedPermissions: data.CachedPermissions, + IsDetectionRun: true, + NetworkPermissions: &NetworkPermissions{ + Allowed: getThreatDetectionAdditionalAllowedDomains(data), + }, + SandboxConfig: &SandboxConfig{ + Agent: &AgentSandboxConfig{ + Type: SandboxTypeAWF, + }, + }, + } + + var steps []string + + // Install the engine binary (fresh runner has no engine installed). + installSteps := engine.GetInstallationSteps(evalsData) + + // Ensure Node.js is on PATH when the engine harness requires it. + // Guard against engines whose install steps already bundle Setup Node.js. + if engineRequiresNodeHarness(engine) && !installStepsContainNodeSetup(installSteps) { + for _, line := range GenerateNodeJsSetupStep() { + steps = append(steps, line+"\n") + } + } + for _, step := range installSteps { + for _, line := range step { + steps = append(steps, line+"\n") + } + } + + // Codex requires MCP gateway config (OpenAI proxy provider in config.toml). + if engine.GetID() == "codex" { + var mcpSetup strings.Builder + if err := c.generateMCPSetup(&mcpSetup, evalsData.Tools, engine, evalsData); err == nil { + for line := range strings.SplitSeq(mcpSetup.String(), "\n") { + if line != "" { + steps = append(steps, line+"\n") + } + } + } else { + evalsStepsLog.Printf("Failed to generate MCP setup for Codex evals; OpenAI proxy configuration may be incomplete: %v", err) + } + } + + // Execute the engine through AWF; output is written to evalsLogPath. + executionSteps := engine.GetExecutionSteps(evalsData, evalsLogPath) + for _, step := range executionSteps { + for i, line := range step { + // Prefix step IDs to avoid collisions with agent job step IDs. + prefixed := strings.Replace(line, "id: agentic_execution", "id: evals_agentic_execution", 1) + steps = append(steps, prefixed+"\n") + // Inject always() condition and continue-on-error after the first line (- name:) + // so that infrastructure failures do not block the parse step that follows. + if i == 0 { + steps = append(steps, " if: always()\n") + steps = append(steps, " continue-on-error: true\n") + } + } + } + + return steps +} + +// buildParseEvalsResultsStep creates the github-script step that reads the engine +// output log and writes structured per-question YES/NO records to evals.jsonl. +func (c *Compiler) buildParseEvalsResultsStep(data *WorkflowData) []string { + if data.Evals == nil { + return nil + } + + questionsJSON := marshalEvalsQuestions(data.Evals.Questions) + model := data.Evals.Model + if model == "" { + model = "small" + } + + script := `const { setupGlobals } = require('` + SetupActionDestination + `/setup_globals.cjs'); +setupGlobals(core, github, context, exec, io, getOctokit); +const { main } = require('` + SetupActionDestination + `/run_evals.cjs'); +await main();` + + steps := []string{ + " - name: Parse BinEval results\n", + " if: always()\n", + " continue-on-error: true\n", + fmt.Sprintf(" uses: %s\n", getCachedActionPin("actions/github-script", data)), + " env:\n", + fmt.Sprintf(" GH_AW_EVALS_QUESTIONS: %s\n", quoteSingleLineYAML(questionsJSON)), + fmt.Sprintf(" GH_AW_EVALS_MODEL: %q\n", model), + " GH_AW_EVALS_PHASE: parse\n", + " with:\n", + " script: |\n", + } + steps = append(steps, FormatJavaScriptForYAML(script)...) + return steps +} + +// buildRedactEvalsSecretsStep creates a step that runs redact_secrets.cjs to +// remove any credential patterns from evals.jsonl before the artifact is uploaded. +func (c *Compiler) buildRedactEvalsSecretsStep(data *WorkflowData) []string { + script := `const { setupGlobals } = require('` + SetupActionDestination + `/setup_globals.cjs'); +setupGlobals(core, github, context, exec, io, getOctokit); +const { main } = require('` + SetupActionDestination + `/redact_secrets.cjs'); +await main();` + + steps := []string{ + " - name: Redact secrets in evals results\n", + " if: always()\n", + " continue-on-error: true\n", + fmt.Sprintf(" uses: %s\n", getCachedActionPin("actions/github-script", data)), + " with:\n", + " script: |\n", + } + steps = append(steps, FormatJavaScriptForYAML(script)...) + return steps +} + +// buildUploadEvalsArtifactStep creates the step that uploads evals.jsonl as the +// evals artifact so it can be downloaded by the push_evals job. +func (c *Compiler) buildUploadEvalsArtifactStep(data *WorkflowData) []string { + evalsArtifactName := artifactPrefixExprForAgentDownstreamJob(data) + constants.EvalsArtifactName + return []string{ + " - name: Upload evals results\n", + " if: always()\n", + fmt.Sprintf(" uses: %s\n", c.getActionPin("actions/upload-artifact")), + " with:\n", + " name: " + evalsArtifactName + "\n", + " path: " + evalsResultsPath + "\n", + " if-no-files-found: ignore\n", + } +} + +// --------------------------------------------------------------------------- +// Engine configuration helpers +// --------------------------------------------------------------------------- + +// getEvalsEngineID returns the engine ID to use for evals execution. +// Evals reuse the main workflow engine. +func (c *Compiler) getEvalsEngineID(data *WorkflowData) string { + if data.EngineConfig != nil && data.EngineConfig.ID != "" { + return data.EngineConfig.ID + } + if data.AI != "" { + return data.AI + } + return "copilot" +} + +// --------------------------------------------------------------------------- +// Utility helpers +// --------------------------------------------------------------------------- + +// marshalEvalsQuestions serialises eval question definitions to a compact JSON +// array string suitable for embedding in a GitHub Actions env var. +func marshalEvalsQuestions(questions []EvalDefinition) string { + if len(questions) == 0 { + return "[]" + } + var sb strings.Builder + sb.WriteString("[") + for i, q := range questions { + if i > 0 { + sb.WriteString(",") + } + fmt.Fprintf(&sb, `{"id":%s,"question":%s}`, + jsonStringLiteral(q.ID), + jsonStringLiteral(q.Question), + ) + } + sb.WriteString("]") + return sb.String() +} + +// jsonStringLiteral returns a JSON-encoded string literal (with double quotes). +func jsonStringLiteral(s string) string { + var sb strings.Builder + sb.WriteByte('"') + for _, r := range s { + switch r { + case '"': + sb.WriteString(`\"`) + case '\\': + sb.WriteString(`\\`) + case '\n': + sb.WriteString(`\n`) + case '\r': + sb.WriteString(`\r`) + case '\t': + sb.WriteString(`\t`) + default: + sb.WriteRune(r) + } + } + sb.WriteByte('"') + return sb.String() +} + +// quoteSingleLineYAML wraps a value in single quotes for use in a YAML scalar, +// escaping any embedded single quotes by doubling them. +func quoteSingleLineYAML(v string) string { + escaped := strings.ReplaceAll(v, "'", "''") + return "'" + escaped + "'" +} diff --git a/pkg/workflow/jobs.go b/pkg/workflow/jobs.go index 46b7001e712..de61aef7b88 100644 --- a/pkg/workflow/jobs.go +++ b/pkg/workflow/jobs.go @@ -20,6 +20,7 @@ const runtimeFeaturesEnvVarExpression = "${{ vars.GH_AW_RUNTIME_FEATURES }}" const pushExperimentsStateJobName = "push_experiments_state" const pushRepoMemoryJobName = "push_repo_memory" const updateCacheMemoryJobName = "update_cache_memory" +const pushEvalsJobName = "push_evals" var runtimeFeaturesBuiltInJobNames = map[string]struct{}{ string(constants.AgentJobName): {}, @@ -34,6 +35,7 @@ var runtimeFeaturesBuiltInJobNames = map[string]struct{}{ pushExperimentsStateJobName: {}, pushRepoMemoryJobName: {}, updateCacheMemoryJobName: {}, + pushEvalsJobName: {}, } // Job represents a GitHub Actions job with all its properties From fdf8860988735ac89dd8c6fd3db8340b3f959154 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Mon, 6 Jul 2026 06:07:26 +0000 Subject: [PATCH 2/8] Address code review: use ERR_VALIDATION constant, fix warning prefixes, clarify step ID replacement comment Co-authored-by: pelikhan <4175913+pelikhan@users.noreply.github.com> --- actions/setup/js/run_evals.cjs | 12 +++++++----- pkg/workflow/evals_steps.go | 6 +++++- 2 files changed, 12 insertions(+), 6 deletions(-) diff --git a/actions/setup/js/run_evals.cjs b/actions/setup/js/run_evals.cjs index 1710d9d0c01..28ff7a40f5b 100644 --- a/actions/setup/js/run_evals.cjs +++ b/actions/setup/js/run_evals.cjs @@ -33,6 +33,8 @@ const fs = require("fs"); const path = require("path"); +const { ERR_VALIDATION } = require("./error_codes.cjs"); + const EVALS_DIR = "/tmp/gh-aw/evals"; const EVALS_LOG_PATH = "/tmp/gh-aw/evals/evals.log"; const EVALS_OUTPUT_PATH = "/tmp/gh-aw/evals.jsonl"; @@ -50,7 +52,7 @@ const AGENT_OUTPUT_FILENAME = "agent_output.json"; async function setupMain() { const questionsRaw = process.env.GH_AW_EVALS_QUESTIONS; if (!questionsRaw) { - core.setFailed("ERR_VALIDATION: GH_AW_EVALS_QUESTIONS is not set"); + core.setFailed(`${ERR_VALIDATION}: GH_AW_EVALS_QUESTIONS is not set`); return; } @@ -58,12 +60,12 @@ async function setupMain() { try { questions = JSON.parse(questionsRaw); } catch (e) { - core.setFailed("ERR_VALIDATION: GH_AW_EVALS_QUESTIONS is not valid JSON: " + e.message); + core.setFailed(`${ERR_VALIDATION}: GH_AW_EVALS_QUESTIONS is not valid JSON: ` + e.message); return; } if (!Array.isArray(questions) || questions.length === 0) { - core.setFailed("ERR_VALIDATION: GH_AW_EVALS_QUESTIONS must be a non-empty JSON array"); + core.setFailed(`${ERR_VALIDATION}: GH_AW_EVALS_QUESTIONS must be a non-empty JSON array`); return; } @@ -77,7 +79,7 @@ async function setupMain() { agentOutputContent = fs.readFileSync(agentOutputPath, "utf-8"); core.info(`Agent output loaded: ${agentOutputPath} (${stats.size} bytes)`); } else { - core.warning(`ERR_VALIDATION: Agent output not found at ${agentOutputPath}. ` + "Ensure the agent artifact includes agent_output.json. " + "Evaluation will proceed without agent context."); + core.warning(`Agent output not found at ${agentOutputPath}. ` + "Ensure the agent artifact includes agent_output.json. " + "Evaluation will proceed without agent context."); } const prompt = buildEvalPrompt(questions, agentOutputContent); @@ -118,7 +120,7 @@ async function parseMain() { } if (!fs.existsSync(EVALS_LOG_PATH)) { - core.warning(`ERR_VALIDATION: Evals log not found at ${EVALS_LOG_PATH}; no results written`); + core.warning(`Evals log not found at ${EVALS_LOG_PATH}; no results written`); fs.writeFileSync(EVALS_OUTPUT_PATH, ""); return; } diff --git a/pkg/workflow/evals_steps.go b/pkg/workflow/evals_steps.go index 46aa5a4dea1..6ee3d612fae 100644 --- a/pkg/workflow/evals_steps.go +++ b/pkg/workflow/evals_steps.go @@ -252,7 +252,11 @@ func (c *Compiler) buildEvalsEngineSteps(data *WorkflowData) []string { executionSteps := engine.GetExecutionSteps(evalsData, evalsLogPath) for _, step := range executionSteps { for i, line := range step { - // Prefix step IDs to avoid collisions with agent job step IDs. + // Prefix the agentic_execution step ID to avoid collisions with the agent job step + // IDs — job managers validate for duplicate step IDs across the compiled YAML. + // This mirrors the same pattern used in buildDetectionEngineExecutionStep (see + // threat_detection_inline_engine.go), where the ID is also a well-known literal + // produced by every engine's GetExecutionSteps implementation. prefixed := strings.Replace(line, "id: agentic_execution", "id: evals_agentic_execution", 1) steps = append(steps, prefixed+"\n") // Inject always() condition and continue-on-error after the first line (- name:) From aec5e25e9d018a107df1f611ea9a6d120c72b88c Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Mon, 6 Jul 2026 07:13:10 +0000 Subject: [PATCH 3/8] Remove push_evals job and associated code Co-authored-by: pelikhan <4175913+pelikhan@users.noreply.github.com> --- actions/setup/js/push_evals.cjs | 225 -------------------------------- pkg/constants/job_constants.go | 4 - pkg/workflow/compiler_evals.go | 104 --------------- pkg/workflow/compiler_jobs.go | 29 ---- pkg/workflow/evals_steps.go | 4 - pkg/workflow/jobs.go | 2 - 6 files changed, 368 deletions(-) delete mode 100644 actions/setup/js/push_evals.cjs delete mode 100644 pkg/workflow/compiler_evals.go diff --git a/actions/setup/js/push_evals.cjs b/actions/setup/js/push_evals.cjs deleted file mode 100644 index 69d067ed0fc..00000000000 --- a/actions/setup/js/push_evals.cjs +++ /dev/null @@ -1,225 +0,0 @@ -// @ts-check -/// - -/** - * push_evals - * - * Commits BinEval result files (evals.jsonl) to a git branch using the - * GitHub GraphQL `createCommitOnBranch` mutation so commits are - * cryptographically signed (verified) by GitHub. Falls back to a plain - * `git push` via pushSignedCommits when the GraphQL path is unavailable. - * - * Environment variables (set by the compiled workflow step): - * GH_AW_EVALS_DIR - Directory containing evals.jsonl - * e.g. /tmp/gh-aw/evals-artifact - * GH_AW_EVALS_BRANCH - Target git branch for evals results - * e.g. evals/myworkflow - * GH_TOKEN / GITHUB_TOKEN - GitHub token for API access and git operations - * GITHUB_RUN_ID - Run ID used in commit messages - * GITHUB_SERVER_URL - GitHub server URL (defaults to https://github.com) - * GITHUB_REPOSITORY - "owner/repo" of the current repository - */ - -"use strict"; - -const fs = require("fs"); -const path = require("path"); - -const { getErrorMessage } = require("./error_helpers.cjs"); -const { execGitSync, getGitAuthEnv } = require("./git_helpers.cjs"); -const { pushSignedCommits } = require("./push_signed_commits.cjs"); - -/** - * Checkout or create an orphan git branch for evals results. - * Returns the remote HEAD SHA (empty string for a new branch). - * - * @param {string} branchName - Target branch name (e.g. "evals/myworkflow") - * @param {string} repoUrl - Authenticated HTTPS URL of the target repo - * @param {string} workspaceDir - Local git workspace directory - * @returns {string} baseRef (empty string when branch is brand new) - */ -function checkoutOrCreateBranch(branchName, repoUrl, workspaceDir) { - try { - execGitSync(["fetch", repoUrl, `${branchName}:${branchName}`], { - stdio: "pipe", - cwd: workspaceDir, - suppressLogs: true, - }); - execGitSync(["checkout", branchName], { stdio: "inherit", cwd: workspaceDir }); - const baseRef = execGitSync(["rev-parse", "HEAD"], { cwd: workspaceDir }).trim(); - core.info(`Checked out existing branch ${branchName}, baseRef=${baseRef}`); - return baseRef; - } catch (fetchErr) { - const msg = getErrorMessage(fetchErr); - const isMissing = /couldn't find remote ref/i.test(msg) || /remote branch .* not found/i.test(msg); - if (!isMissing) throw fetchErr; - - // Branch does not exist yet – create an orphan branch. - core.info(`Branch ${branchName} does not exist, creating orphan branch...`); - execGitSync(["checkout", "--orphan", branchName], { stdio: "inherit", cwd: workspaceDir }); - execGitSync(["read-tree", "--empty"], { stdio: "pipe", cwd: workspaceDir }); - // Remove any pre-existing working-tree files (from sparse checkout). - for (const entry of fs.readdirSync(workspaceDir)) { - if (entry !== ".git") { - fs.rmSync(path.join(workspaceDir, entry), { recursive: true, force: true }); - } - } - return ""; - } -} - -/** - * Main entry point called by the actions/github-script step. - */ -async function main() { - const evalsDir = process.env.GH_AW_EVALS_DIR || "/tmp/gh-aw/evals-artifact"; - const branchName = process.env.GH_AW_EVALS_BRANCH || ""; - const ghToken = process.env.GH_TOKEN || process.env.GITHUB_TOKEN || ""; - const githubRunId = process.env.GITHUB_RUN_ID || "unknown"; - const githubServerUrl = (process.env.GITHUB_SERVER_URL || "https://github.com").replace(/\/$/, ""); - const serverHost = githubServerUrl.replace(/^https?:\/\//, ""); - - if (!branchName) { - core.setFailed("GH_AW_EVALS_BRANCH is not set"); - return; - } - if (!ghToken) { - core.setFailed("GH_TOKEN or GITHUB_TOKEN is not set"); - return; - } - - const targetRepo = `${context.repo.owner}/${context.repo.repo}`; - const allowedRepos = new Set( - (process.env.GH_AW_ALLOWED_TARGET_REPOS || targetRepo) - .split(",") - .map(repo => repo.trim()) - .filter(Boolean) - ); - if (!allowedRepos.has(targetRepo)) { - core.setFailed(`Target repository "${targetRepo}" is not in GH_AW_ALLOWED_TARGET_REPOS. ` + `Current allowlist: ${Array.from(allowedRepos).join(", ")}`); - return; - } - const [owner, repo] = targetRepo.split("/"); - - core.info(`Pushing evals results to branch "${branchName}" in ${targetRepo}`); - - // Collect evals result files present in the artifact directory - const candidateFiles = ["evals.jsonl"]; - const filesToPush = candidateFiles.filter(name => { - const full = path.join(evalsDir, name); - return fs.existsSync(full) && fs.statSync(full).isFile(); - }); - - if (filesToPush.length === 0) { - core.info("No evals result files found – nothing to push"); - return; - } - - core.info(`Files to push: ${filesToPush.join(", ")}`); - - const workspaceDir = process.env.GITHUB_WORKSPACE || process.cwd(); - const repoUrl = `https://x-access-token:${ghToken}@${serverHost}/${targetRepo}.git`; - - // Checkout the target branch (or create it as an orphan on first run). - let baseRef; - try { - baseRef = checkoutOrCreateBranch(branchName, repoUrl, workspaceDir); - } catch (err) { - core.setFailed(`Failed to checkout branch "${branchName}": ${getErrorMessage(err)}`); - return; - } - - // Copy evals files into the workspace root. - for (const name of filesToPush) { - const src = path.join(evalsDir, name); - const dest = path.join(workspaceDir, name); - try { - fs.copyFileSync(src, dest); - core.info(`Copied ${name}`); - } catch (err) { - core.setFailed(`Failed to copy ${name}: ${getErrorMessage(err)}`); - return; - } - } - - // Stage all changes. - try { - execGitSync(["add", "--sparse", "."], { stdio: "inherit", cwd: workspaceDir }); - } catch (err) { - core.setFailed(`Failed to stage changes: ${getErrorMessage(err)}`); - return; - } - - // Check whether there are any staged changes to commit. - const status = execGitSync(["status", "--porcelain"], { cwd: workspaceDir }).trim(); - if (!status) { - core.info("No changes to evals results – skipping push"); - return; - } - - // Commit. - try { - execGitSync(["commit", "-m", `Update evals results from workflow run ${githubRunId}`], { stdio: "inherit", cwd: workspaceDir }); - } catch (err) { - core.setFailed(`Failed to commit evals results: ${getErrorMessage(err)}`); - return; - } - - // Point origin at the target repo so pushSignedCommits can resolve the remote branch HEAD. - execGitSync(["remote", "set-url", "origin", `https://${serverHost}/${targetRepo}.git`], { stdio: "pipe", cwd: workspaceDir }); - - // Push using GraphQL createCommitOnBranch (signed commits) with a plain-git fallback. - const MAX_RETRIES = 3; - const BASE_DELAY_MS = 1000; - let currentBaseRef = baseRef; - - for (let attempt = 0; attempt <= MAX_RETRIES; attempt++) { - core.info(`Pushing to ${branchName} (attempt ${attempt + 1}/${MAX_RETRIES + 1})...`); - try { - await pushSignedCommits({ - githubClient: github, - owner, - repo, - branch: branchName, - baseRef: currentBaseRef, - cwd: workspaceDir, - gitAuthEnv: getGitAuthEnv(ghToken), - }); - core.info(`Successfully pushed evals results to ${branchName}`); - return; - } catch (err) { - const errMsg = getErrorMessage(err); - if (attempt < MAX_RETRIES) { - const delay = BASE_DELAY_MS * Math.pow(2, attempt); - core.warning(`Push failed (attempt ${attempt + 1}/${MAX_RETRIES + 1}), retrying in ${delay}ms: ${errMsg}`); - await new Promise(resolve => setTimeout(resolve, delay)); - - // Refresh baseRef and fetch the updated remote history so that - // pushSignedCommits can resolve the new baseRef in git rev-list. - try { - const { stdout: lsOut } = await exec.getExecOutput("git", ["ls-remote", "origin", `refs/heads/${branchName}`], { cwd: workspaceDir }); - const remoteHead = lsOut.trim().split(/\s+/)[0] || ""; - if (remoteHead && remoteHead !== currentBaseRef) { - currentBaseRef = remoteHead; - core.info(`Refreshed baseRef for retry: ${currentBaseRef}`); - try { - execGitSync(["fetch", "origin", `refs/heads/${branchName}`], { - stdio: "pipe", - cwd: workspaceDir, - suppressLogs: true, - }); - } catch (fetchErr) { - core.info(`Fetch of branch "${branchName}" on retry failed (non-fatal): ${getErrorMessage(fetchErr)}`); - } - } - } catch { - // ls-remote failed; keep existing baseRef - } - } else { - core.setFailed(`Failed to push evals results after ${MAX_RETRIES + 1} attempts: ${errMsg}`); - } - } - } -} - -module.exports = { main, checkoutOrCreateBranch }; diff --git a/pkg/constants/job_constants.go b/pkg/constants/job_constants.go index d99cf08542f..9c1418ef643 100644 --- a/pkg/constants/job_constants.go +++ b/pkg/constants/job_constants.go @@ -105,10 +105,6 @@ const EvalsArtifactName = "evals" // EvalsResultFilename is the filename of the evaluation results JSONL file. const EvalsResultFilename = "evals.jsonl" -// EvalsBranchPrefix is the git branch prefix for persisting evaluation results. -// Results are stored in branches named evals//evals.jsonl. -const EvalsBranchPrefix = "evals" - // LegacyDetectionArtifactName is the old artifact name used before the rename. // Kept for backward compatibility when downloading artifacts from older workflow runs. const LegacyDetectionArtifactName = "threat-detection.log" diff --git a/pkg/workflow/compiler_evals.go b/pkg/workflow/compiler_evals.go deleted file mode 100644 index 39496427df2..00000000000 --- a/pkg/workflow/compiler_evals.go +++ /dev/null @@ -1,104 +0,0 @@ -// Package workflow - push_evals job assembler. -package workflow - -import ( - "fmt" - "strings" - - "github.com/github/gh-aw/pkg/constants" -) - -// buildPushEvalsJob creates a job that downloads the evals artifact and commits it -// to a git branch ("evals/{sanitizedWorkflowID}") for durable storage across runs. -// Returns nil when evals are not declared in the workflow frontmatter. -func (c *Compiler) buildPushEvalsJob(data *WorkflowData) (*Job, error) { - if !data.Evals.HasEvals() { - return nil, nil - } - - var steps []string - - // Setup step so the push_evals.cjs script is available. - setupActionRef := c.resolveActionReference("./actions/setup", data) - if setupActionRef != "" || c.actionMode.IsScript() { - steps = append(steps, c.generateCheckoutActionsFolder(data)...) - traceID := fmt.Sprintf("${{ needs.%s.outputs.setup-trace-id }}", constants.ActivationJobName) - parentSpanID := setupParentSpanNeedsExpr(constants.ActivationJobName) - steps = append(steps, c.generateSetupStep(data, setupActionRef, SetupActionDestination, false, traceID, parentSpanID)...) - } - - // Checkout step – configure git credentials without downloading workspace files. - var checkoutStep strings.Builder - checkoutStep.WriteString(" - name: Checkout repository\n") - fmt.Fprintf(&checkoutStep, " uses: %s\n", getActionPin("actions/checkout")) - checkoutStep.WriteString(" with:\n") - checkoutStep.WriteString(" persist-credentials: false\n") - checkoutStep.WriteString(" sparse-checkout: .\n") - steps = append(steps, checkoutStep.String()) - - // Git configuration (author, email). - steps = append(steps, c.generateGitConfigurationSteps()...) - - // Download the evals artifact uploaded by the evals job. - evalsArtifactName := artifactPrefixExprForAgentDownstreamJob(data) + constants.EvalsArtifactName - var downloadStep strings.Builder - downloadStep.WriteString(" - name: Download evals artifact\n") - fmt.Fprintf(&downloadStep, " uses: %s\n", c.getActionPin("actions/download-artifact")) - downloadStep.WriteString(" continue-on-error: true\n") - downloadStep.WriteString(" with:\n") - fmt.Fprintf(&downloadStep, " name: %s\n", evalsArtifactName) - fmt.Fprintf(&downloadStep, " path: %s\n", evalsArtifactDownloadDir) - steps = append(steps, downloadStep.String()) - - // Push evals results to the git branch via push_evals.cjs. - branchName := evalsBranchName(data.WorkflowID) - - var pushStep strings.Builder - pushStep.WriteString(" - name: Push evals results to git\n") - pushStep.WriteString(" id: push_evals\n") - pushStep.WriteString(" if: always()\n") - fmt.Fprintf(&pushStep, " uses: %s\n", getCachedActionPin("actions/github-script", data)) - pushStep.WriteString(" env:\n") - pushStep.WriteString(" GH_TOKEN: ${{ github.token }}\n") - pushStep.WriteString(" GITHUB_RUN_ID: ${{ github.run_id }}\n") - pushStep.WriteString(" GITHUB_SERVER_URL: ${{ github.server_url }}\n") - fmt.Fprintf(&pushStep, " GH_AW_EVALS_DIR: %s\n", evalsArtifactDownloadDir) - fmt.Fprintf(&pushStep, " GH_AW_EVALS_BRANCH: %s\n", branchName) - pushStep.WriteString(" with:\n") - pushStep.WriteString(" script: |\n") - pushStep.WriteString(" const { setupGlobals } = require('" + SetupActionDestination + "/setup_globals.cjs');\n") - pushStep.WriteString(" setupGlobals(core, github, context, exec, io, getOctokit);\n") - pushStep.WriteString(" const { main } = require('" + SetupActionDestination + "/push_evals.cjs');\n") - pushStep.WriteString(" await main();\n") - steps = append(steps, pushStep.String()) - - // Restore the checkout in dev mode (same reason as push_repo_memory and push_experiments_state). - if c.actionMode.IsDev() { - steps = append(steps, c.generateRestoreActionsSetupStep()) - } - - // The push_evals job runs after the evals job completes (success or failure). - // It does not block on evals succeeding — we always want to persist whatever results were generated. - evalsNotFailure := BuildNotEquals( - BuildPropertyAccess(fmt.Sprintf("needs.%s.result", constants.EvalsJobName)), - BuildStringLiteral("failure"), - ) - notCancelled := &NotNode{Child: BuildFunctionCall("cancelled")} - jobCondition := RenderCondition(BuildAnd(BuildAnd(BuildFunctionCall("always"), notCancelled), evalsNotFailure)) - - job := &Job{ - Name: pushEvalsJobName, - RunsOn: c.formatFrameworkJobRunsOn(data), - If: jobCondition, - Permissions: "permissions:\n contents: write", - Needs: []string{string(constants.EvalsJobName), string(constants.ActivationJobName)}, - Steps: steps, - } - - return job, nil -} - -// evalsBranchName returns the git branch name used to persist evals results for the given workflow ID. -func evalsBranchName(workflowID string) string { - return string(constants.EvalsBranchPrefix) + "/" + SanitizeWorkflowIDForCacheKey(workflowID) -} diff --git a/pkg/workflow/compiler_jobs.go b/pkg/workflow/compiler_jobs.go index b13a209d82d..f12c8d1c509 100644 --- a/pkg/workflow/compiler_jobs.go +++ b/pkg/workflow/compiler_jobs.go @@ -348,11 +348,6 @@ func (c *Compiler) buildMemoryManagementJobs(data *WorkflowData) error { return err } - // Build push_evals job when evals are declared - if err := c.buildPushEvalsJobAndAdd(data); err != nil { - return err - } - // Update conclusion job dependencies if err := c.updateConclusionJobDependencies(pushRepoMemoryJobName, updateCacheMemoryJobName, pushExperimentsJobName); err != nil { return err @@ -443,30 +438,6 @@ func (c *Compiler) buildPushExperimentsStateJobWrapper(data *WorkflowData) (stri return job.Name, nil } -// buildPushEvalsJobAndAdd builds the push_evals job (if evals are declared) and -// adds it to the job manager. The conclusion job picks it up via ensureConclusionIsLastJob. -func (c *Compiler) buildPushEvalsJobAndAdd(data *WorkflowData) error { - if !data.Evals.HasEvals() { - return nil - } - - compilerJobsLog.Print("Building push_evals job") - job, err := c.buildPushEvalsJob(data) - if err != nil { - return fmt.Errorf("failed to build push_evals job: %w", err) - } - if job == nil { - return nil - } - - if err := c.jobManager.AddJob(job); err != nil { - return fmt.Errorf("failed to add push_evals job: %w", err) - } - - compilerJobsLog.Printf("Successfully added push_evals job: %s", job.Name) - return nil -} - // updateConclusionJobDependencies updates the conclusion job to depend on memory management jobs if they exist. func (c *Compiler) updateConclusionJobDependencies(pushRepoMemoryJobName, updateCacheMemoryJobName, pushExperimentsJobName string) error { conclusionJob, exists := c.jobManager.GetJob("conclusion") diff --git a/pkg/workflow/evals_steps.go b/pkg/workflow/evals_steps.go index 6ee3d612fae..9e3fec830fa 100644 --- a/pkg/workflow/evals_steps.go +++ b/pkg/workflow/evals_steps.go @@ -21,10 +21,6 @@ const ( // evalsResultsPath is the parsed JSONL results file produced by the parse step. evalsResultsPath = "/tmp/gh-aw/" + constants.EvalsResultFilename - - // evalsArtifactDownloadDir is the directory into which the evals artifact is - // downloaded in the push_evals job. - evalsArtifactDownloadDir = "/tmp/gh-aw/evals-artifact" ) // buildEvalsJobSteps builds all steps that run inside the evals job. diff --git a/pkg/workflow/jobs.go b/pkg/workflow/jobs.go index de61aef7b88..46b7001e712 100644 --- a/pkg/workflow/jobs.go +++ b/pkg/workflow/jobs.go @@ -20,7 +20,6 @@ const runtimeFeaturesEnvVarExpression = "${{ vars.GH_AW_RUNTIME_FEATURES }}" const pushExperimentsStateJobName = "push_experiments_state" const pushRepoMemoryJobName = "push_repo_memory" const updateCacheMemoryJobName = "update_cache_memory" -const pushEvalsJobName = "push_evals" var runtimeFeaturesBuiltInJobNames = map[string]struct{}{ string(constants.AgentJobName): {}, @@ -35,7 +34,6 @@ var runtimeFeaturesBuiltInJobNames = map[string]struct{}{ pushExperimentsStateJobName: {}, pushRepoMemoryJobName: {}, updateCacheMemoryJobName: {}, - pushEvalsJobName: {}, } // Job represents a GitHub Actions job with all its properties From 3c99164a7db29255b4865a9f3d9f664c7041a8a0 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Mon, 6 Jul 2026 07:24:09 +0000 Subject: [PATCH 4/8] Use XML tags to structure eval prompt sections Co-authored-by: pelikhan <4175913+pelikhan@users.noreply.github.com> --- actions/setup/js/run_evals.cjs | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/actions/setup/js/run_evals.cjs b/actions/setup/js/run_evals.cjs index 28ff7a40f5b..efe6ce4b43d 100644 --- a/actions/setup/js/run_evals.cjs +++ b/actions/setup/js/run_evals.cjs @@ -205,29 +205,29 @@ async function main() { * @returns {string} */ function buildEvalPrompt(questions, agentOutput) { - const questionList = questions.map((q, i) => `Q${i + 1} [id=${q.id}]: ${q.question}`).join("\n"); + const questionList = questions.map((q, i) => `${q.question}`).join("\n"); - const agentSection = agentOutput ? `## Agent Output\n\n${agentOutput}` : "## Agent Output\n\n(no agent output available)"; + const agentSection = agentOutput ? `\n${agentOutput}\n` : "\n(no agent output available)\n"; return `# BinEval: Binary Evaluation You are evaluating the output of an AI agentic workflow using BinEval (binary evaluation). For each question below, answer with exactly YES or NO based on the agent output provided. -## Questions - + ${questionList} + ${agentSection} -## Instructions - + Answer each question on a separate line using EXACTLY this format: Q1: YES Q2: NO Use only YES or NO. Do not provide explanations or reasoning. -Evaluate each question solely based on the agent output shown above.`; +Evaluate each question solely based on the agent output shown above. +`; } /** From 73f8758d89e24b7510afce60efbb60d5b7ac0648 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Mon, 6 Jul 2026 08:13:14 +0000 Subject: [PATCH 5/8] Fix artifact prefix, JSON escaping, and runtime features for evals job Co-authored-by: gh-aw-bot <259018956+gh-aw-bot@users.noreply.github.com> --- pkg/workflow/evals_steps.go | 11 ++++++++--- pkg/workflow/jobs.go | 1 + 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/pkg/workflow/evals_steps.go b/pkg/workflow/evals_steps.go index 9e3fec830fa..010b5e7abce 100644 --- a/pkg/workflow/evals_steps.go +++ b/pkg/workflow/evals_steps.go @@ -322,9 +322,9 @@ await main();` } // buildUploadEvalsArtifactStep creates the step that uploads evals.jsonl as the -// evals artifact so it can be downloaded by the push_evals job. +// evals artifact for downstream consumption. func (c *Compiler) buildUploadEvalsArtifactStep(data *WorkflowData) []string { - evalsArtifactName := artifactPrefixExprForAgentDownstreamJob(data) + constants.EvalsArtifactName + evalsArtifactName := artifactPrefixExprForDownstreamJob(data) + constants.EvalsArtifactName return []string{ " - name: Upload evals results\n", " if: always()\n", @@ -394,7 +394,12 @@ func jsonStringLiteral(s string) string { case '\t': sb.WriteString(`\t`) default: - sb.WriteRune(r) + // Escape control characters (< 0x20) that JSON doesn't allow + if r < 0x20 { + fmt.Fprintf(&sb, `\u%04x`, r) + } else { + sb.WriteRune(r) + } } } sb.WriteByte('"') diff --git a/pkg/workflow/jobs.go b/pkg/workflow/jobs.go index 46b7001e712..c1cd522db3c 100644 --- a/pkg/workflow/jobs.go +++ b/pkg/workflow/jobs.go @@ -26,6 +26,7 @@ var runtimeFeaturesBuiltInJobNames = map[string]struct{}{ string(constants.ActivationJobName): {}, string(constants.PreActivationJobName): {}, string(constants.DetectionJobName): {}, + string(constants.EvalsJobName): {}, string(constants.SafeOutputsJobName): {}, string(constants.UploadAssetsJobName): {}, string(constants.UploadCodeScanningJobName): {}, From 72ea6a019eab37ed3a4fc53e09dd4f68cf3b39c6 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Mon, 6 Jul 2026 09:53:00 +0000 Subject: [PATCH 6/8] Fix artifact prefix in evals job to use activation-derived prefix The evals job depends on activation in both branches (safe_outputs + activation, or agent + activation), but not always on agent. When safe_outputs is enabled, needs.agent is not available. Use artifactPrefixExprForDownstreamJob (needs.activation.outputs.artifact_prefix) instead of artifactPrefixExprForAgentDownstreamJob (needs.agent.outputs.artifact_prefix) to fix artifact name resolution in workflow_call runs. Co-authored-by: gh-aw-bot <259018956+gh-aw-bot@users.noreply.github.com> --- pkg/workflow/evals_job.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/workflow/evals_job.go b/pkg/workflow/evals_job.go index 0e21f26b92e..31a85d3230b 100644 --- a/pkg/workflow/evals_job.go +++ b/pkg/workflow/evals_job.go @@ -31,8 +31,8 @@ func (c *Compiler) buildEvalsJob(data *WorkflowData) (*Job, error) { } // Download agent output artifact to access output files (prompt.txt, agent_output.json). - // Use agent-downstream prefix since this job depends on the agent job (via safe_outputs). - agentArtifactPrefix := artifactPrefixExprForAgentDownstreamJob(data) + // Use activation-derived prefix since this job always depends on activation. + agentArtifactPrefix := artifactPrefixExprForDownstreamJob(data) steps = append(steps, buildAgentOutputDownloadSteps(agentArtifactPrefix, c.getActionPin)...) // Download experiment artifact so the evals agent can read the current variant assignments. From 5f49dea936673ab9abffa2b728a57c14696bcc5d Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Tue, 7 Jul 2026 02:35:16 +0000 Subject: [PATCH 7/8] Use core.summary.addDetails helper for BinEval prompt Co-authored-by: pelikhan <4175913+pelikhan@users.noreply.github.com> --- actions/setup/js/run_evals.cjs | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/actions/setup/js/run_evals.cjs b/actions/setup/js/run_evals.cjs index efe6ce4b43d..72c6bf1f601 100644 --- a/actions/setup/js/run_evals.cjs +++ b/actions/setup/js/run_evals.cjs @@ -90,10 +90,8 @@ async function setupMain() { core.info(`BinEval setup complete: wrote prompt with ${questions.length} question(s)`); - await core.summary - .addRaw("
\nBinEval Evaluation Prompt\n\n") - .addRaw("``````markdown\n" + prompt + "\n``````\n\n
\n") - .write(); + core.summary.addDetails("BinEval Evaluation Prompt", "\n\n``````markdown\n" + prompt + "\n``````\n\n"); + await core.summary.write(); } // --------------------------------------------------------------------------- From 3f2d731b5fdcbe9953885580cb0868a7c565f9fc Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Tue, 7 Jul 2026 22:54:24 +0000 Subject: [PATCH 8/8] Fix Auth field copy and step injection robustness in evals job - Use shallow copy of EngineConfig instead of manual field assignment to preserve Auth (OIDC/Azure), LLMProvider, and other fields - Make step injection more robust by searching for "- name:" line instead of assuming it's always at index 0 - Addresses review feedback r3526809994 and r3526809999 Co-authored-by: gh-aw-bot <259018956+gh-aw-bot@users.noreply.github.com> --- pkg/workflow/evals_steps.go | 38 ++++++++++++++++++------------------- 1 file changed, 18 insertions(+), 20 deletions(-) diff --git a/pkg/workflow/evals_steps.go b/pkg/workflow/evals_steps.go index 010b5e7abce..e3d936725ff 100644 --- a/pkg/workflow/evals_steps.go +++ b/pkg/workflow/evals_steps.go @@ -131,27 +131,20 @@ func (c *Compiler) buildEvalsEngineSteps(data *WorkflowData) []string { // Determine engine ID (same resolution order as detection). engineID := c.getEvalsEngineID(data) - engineConfig := data.EngineConfig - // Build the evals engine config inheriting from the main engine config. - evalsEngineConfig := engineConfig - if evalsEngineConfig == nil { + // Build the evals engine config by shallow-copying the main engine config. + // This preserves all fields including Auth (OIDC/Azure), LLMProvider, permissions, + // token weights, and other engine settings that should apply to eval runs. + var evalsEngineConfig *EngineConfig + if data.EngineConfig == nil { evalsEngineConfig = &EngineConfig{ID: engineID} } else { - evalsEngineConfig = &EngineConfig{ - ID: evalsEngineConfig.ID, - Model: evalsEngineConfig.Model, - Version: evalsEngineConfig.Version, - Env: evalsEngineConfig.Env, - Config: evalsEngineConfig.Config, - Args: evalsEngineConfig.Args, - APITarget: evalsEngineConfig.APITarget, - HarnessScript: evalsEngineConfig.HarnessScript, - Driver: evalsEngineConfig.Driver, + // Shallow copy all fields from the main engine config + copy := *data.EngineConfig + evalsEngineConfig = © + if evalsEngineConfig.ID == "" { + evalsEngineConfig.ID = engineID } } - if evalsEngineConfig.ID == "" { - evalsEngineConfig.ID = engineID - } // Override model from evals frontmatter if specified. if data.Evals != nil && data.Evals.Model != "" { @@ -247,7 +240,9 @@ func (c *Compiler) buildEvalsEngineSteps(data *WorkflowData) []string { // Execute the engine through AWF; output is written to evalsLogPath. executionSteps := engine.GetExecutionSteps(evalsData, evalsLogPath) for _, step := range executionSteps { - for i, line := range step { + // Track whether we've injected the if/continue-on-error fields yet + injected := false + for _, line := range step { // Prefix the agentic_execution step ID to avoid collisions with the agent job step // IDs — job managers validate for duplicate step IDs across the compiled YAML. // This mirrors the same pattern used in buildDetectionEngineExecutionStep (see @@ -255,11 +250,14 @@ func (c *Compiler) buildEvalsEngineSteps(data *WorkflowData) []string { // produced by every engine's GetExecutionSteps implementation. prefixed := strings.Replace(line, "id: agentic_execution", "id: evals_agentic_execution", 1) steps = append(steps, prefixed+"\n") - // Inject always() condition and continue-on-error after the first line (- name:) + // Inject always() condition and continue-on-error after the "- name:" line // so that infrastructure failures do not block the parse step that follows. - if i == 0 { + // Search for the name field instead of assuming it's always at index 0 to handle + // engines that might emit comments or other fields before the name. + if !injected && strings.Contains(strings.TrimSpace(line), "- name:") { steps = append(steps, " if: always()\n") steps = append(steps, " continue-on-error: true\n") + injected = true } } }