test: make Waza the canonical intrinsic behavior suite

.github/workflows/eval-behavioral.yml

@@ -0,0 +1,62 @@
+name: Trusted Behavioral Skill Evaluation
+
+on:
+  workflow_dispatch:
+  schedule:
+    - cron: '17 4 * * 1'
+
+permissions:
+  contents: read
+
+jobs:
+  evaluate:
+    runs-on: ubuntu-latest
+    timeout-minutes: 60
+    env:
+      WAZA_NO_UPDATE_CHECK: '1'
+      COPILOT_SDK_TOKEN: ${{ secrets.COPILOT_SDK_TOKEN }}
+    steps:
+      - name: Checkout
+        uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+        with:
+          persist-credentials: false
+
+      - name: Detect trusted model credential
+        id: credentials
+        shell: bash
+        run: |
+          set -euo pipefail
+          if [[ -n "${COPILOT_SDK_TOKEN:-}" ]]; then
+            echo "available=true" >> "$GITHUB_OUTPUT"
+          else
+            echo "available=false" >> "$GITHUB_OUTPUT"
+            echo "::notice::COPILOT_SDK_TOKEN is not configured; model-backed eval was skipped."
+          fi
+
+      - name: Install pinned Waza
+        if: steps.credentials.outputs.available == 'true'
+        shell: bash
+        run: |
+          set -euo pipefail
+          bash scripts/install-waza-ci.sh "$RUNNER_TEMP/waza"
+          echo "$RUNNER_TEMP" >> "$GITHUB_PATH"
+
+      - name: Run intrinsic behavioral suite
+        if: steps.credentials.outputs.available == 'true'
+        shell: bash
+        run: |
+          set -euo pipefail
+          mkdir -p eval-results/transcripts
+          waza run evals/github-build-or-reuse/eval.yaml \
+            --output eval-results/results.json \
+            --reporter junit:eval-results/junit.xml \
+            --transcript-dir eval-results/transcripts
+
+      - name: Upload behavioral evidence
+        if: always() && steps.credentials.outputs.available == 'true'
+        uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
+        with:
+          name: waza-github-build-or-reuse-${{ github.run_id }}
+          path: eval-results/
+          if-no-files-found: warn
+          retention-days: 14

.github/workflows/eval-static.yml

@@ -0,0 +1,55 @@
+name: Behavioral Eval Static Validation
+
+on:
+  push:
+    paths:
+      - 'evals/**'
+      - 'skills/github-build-or-reuse/SKILL.md'
+      - 'scripts/validate-evals.py'
+      - 'scripts/install-waza-ci.sh'
+      - 'docs/evals.md'
+      - '.github/workflows/eval-static.yml'
+      - '.github/workflows/eval-behavioral.yml'
+  pull_request:
+    paths:
+      - 'evals/**'
+      - 'skills/github-build-or-reuse/SKILL.md'
+      - 'scripts/validate-evals.py'
+      - 'scripts/install-waza-ci.sh'
+      - 'docs/evals.md'
+      - '.github/workflows/eval-static.yml'
+      - '.github/workflows/eval-behavioral.yml'
+
+permissions:
+  contents: read
+
+jobs:
+  validate:
+    runs-on: ubuntu-latest
+    timeout-minutes: 20
+    env:
+      WAZA_NO_UPDATE_CHECK: '1'
+    steps:
+      - name: Checkout
+        uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+        with:
+          persist-credentials: false
+
+      - name: Install YAML parser
+        run: python -m pip install 'PyYAML>=6,<7'
+
+      - name: Validate intrinsic eval contract
+        run: python scripts/validate-evals.py
+
+      - name: Validate workflow security baseline
+        run: python scripts/validate-workflow-security.py
+
+      - name: Install pinned Waza
+        shell: bash
+        run: |
+          set -euo pipefail
+          bash scripts/install-waza-ci.sh "$RUNNER_TEMP/waza"
+          echo "$RUNNER_TEMP" >> "$GITHUB_PATH"
+
+      - name: Verify Waza spec without model calls
+        run: waza spec verify --skill skills/github-build-or-reuse --eval evals/github-build-or-reuse/eval.yaml --format github-actions

docs/evals.md

@@ -0,0 +1,36 @@
+# Behavioral evaluations
+
+The canonical repository owns the **intrinsic behavior** of `github-build-or-reuse`. Waza suites therefore live at top-level `evals/github-build-or-reuse/`, outside the portable runtime payload.
+
+The suite protects five core behaviors:
+
+- research before substantial implementation;
+- evidence-based comparison with one `USE`, `CONTRIBUTE`, `FORK`, or `BUILD` verdict;
+- explicit discovery bypass is honored and disclosed;
+- trivial snippets do not force repository discovery;
+- unavailable search/tooling is disclosed as an evidence gap rather than silently fabricated.
+
+## Pull requests
+
+PR validation is deterministic and does not receive a model credential:
+
+```bash
+python -m pip install 'PyYAML>=6,<7'
+python scripts/validate-evals.py
+bash scripts/install-waza-ci.sh /tmp/waza
+/tmp/waza spec verify \
+  --skill skills/github-build-or-reuse \
+  --eval evals/github-build-or-reuse/eval.yaml
+```
+
+Waza is pinned to v0.38.6 and the Linux CI asset is checksum-verified.
+
+## Trusted model-backed runs
+
+`.github/workflows/eval-behavioral.yml` runs manually or weekly from trusted repository state. It uses `COPILOT_SDK_TOKEN` when configured and otherwise exits safely with a notice instead of turning a missing credential into repository failure.
+
+Successful/failed execution evidence is retained as JSON, JUnit and transcripts for 14 days. A pass describes the measured Waza/Copilot-SDK executor; it does not claim identical behavior in every host.
+
+## Upstream vs catalog responsibility
+
+This repository tests product behavior. A downstream catalog such as `svg153/skills` should test only integration/routing concerns specific to the multi-skill environment, not maintain a second copy of these five intrinsic cases.

evals/github-build-or-reuse/eval.yaml

@@ -0,0 +1,27 @@
+name: github-build-or-reuse-intrinsic-behavior
+description: >-
+  Canonical upstream behavioral suite for the intrinsic github-build-or-reuse contract.
+skill: github-build-or-reuse
+schemaVersion: "1.2"
+version: "1.0"
+
+config:
+  trials_per_task: 1
+  timeout_seconds: 300
+  parallel: false
+  executor: copilot-sdk
+  inject_skill_body: false
+
+metrics:
+  - name: trigger_accuracy
+    weight: 0.45
+    threshold: 0.8
+  - name: behavior_quality
+    weight: 0.35
+    threshold: 0.7
+  - name: task_completion
+    weight: 0.20
+    threshold: 0.7
+
+tasks:
+  - "tasks/*.yaml"

evals/github-build-or-reuse/tasks/compare-open-source-alternatives.yaml

@@ -0,0 +1,22 @@
+id: compare-open-source-alternatives
+name: Open-source comparison uses evidence and a reuse verdict
+inputs:
+  prompt: >-
+    Compare open-source GitHub projects I could reuse for browser automation in a service,
+    then tell me whether I should adopt one, contribute, fork, or build my own.
+expected:
+  should_trigger: true
+  graders:
+    - type: trigger
+      name: activation
+      config:
+        skill_path: skills/github-build-or-reuse/SKILL.md
+        mode: positive
+        threshold: 0.6
+    - type: prompt
+      name: evidence_based_decision
+      config:
+        prompt: >-
+          Pass only if the response compares plausible candidates using consequential evidence
+          such as fit, license, maintenance or security and returns one primary USE, CONTRIBUTE,
+          FORK, or BUILD verdict rather than ranking by stars alone.

evals/github-build-or-reuse/tasks/explicit-discovery-bypass.yaml

@@ -0,0 +1,22 @@
+id: explicit-discovery-bypass
+name: Explicit discovery bypass is honored and disclosed
+inputs:
+  prompt: >-
+    I know alternatives may exist, but explicitly skip GitHub/open-source discovery for this
+    feature and just help me implement my chosen design.
+expected:
+  should_trigger: true
+  graders:
+    - type: trigger
+      name: activation
+      config:
+        skill_path: skills/github-build-or-reuse/SKILL.md
+        mode: positive
+        threshold: 0.6
+    - type: prompt
+      name: bypass_disclosed
+      config:
+        prompt: >-
+          Pass only if the response honors the explicit request to bypass discovery and clearly
+          states that reuse vetting was skipped. Fail if it claims due diligence occurred or
+          insists on a full repository search against the user's explicit instruction.

evals/github-build-or-reuse/tasks/nontrivial-build-research-first.yaml

@@ -0,0 +1,22 @@
+id: nontrivial-build-research-first
+name: Non-trivial build researches before implementation
+inputs:
+  prompt: >-
+    Build me a self-hosted AI presentation generator that exports editable PPTX files.
+    Start implementing it now.
+expected:
+  should_trigger: true
+  graders:
+    - type: trigger
+      name: activation
+      config:
+        skill_path: skills/github-build-or-reuse/SKILL.md
+        mode: positive
+        threshold: 0.6
+    - type: prompt
+      name: research_before_build
+      config:
+        prompt: >-
+          Pass only if the response performs or explicitly attempts open-source/GitHub due
+          diligence before substantial implementation and converges on one of USE, CONTRIBUTE,
+          FORK, or BUILD. Fail if it jumps straight into implementation.

evals/github-build-or-reuse/tasks/trivial-snippet-boundary.yaml

@@ -0,0 +1,14 @@
+id: trivial-snippet-boundary
+name: Trivial snippet does not force repository discovery
+inputs:
+  prompt: >-
+    Write a tiny Python function that returns True when an integer is even.
+expected:
+  should_trigger: false
+  graders:
+    - type: trigger
+      name: boundary
+      config:
+        skill_path: skills/github-build-or-reuse/SKILL.md
+        mode: negative
+        threshold: 0.6

evals/github-build-or-reuse/tasks/unavailable-search-disclosure.yaml

@@ -0,0 +1,22 @@
+id: unavailable-search-disclosure
+name: Search failure is disclosed rather than silently degraded
+inputs:
+  prompt: >-
+    Decide whether I should reuse an existing open-source notification service or build one,
+    but assume GitHub API, gh and web search are unavailable in this environment.
+expected:
+  should_trigger: true
+  graders:
+    - type: trigger
+      name: activation
+      config:
+        skill_path: skills/github-build-or-reuse/SKILL.md
+        mode: positive
+        threshold: 0.6
+    - type: prompt
+      name: evidence_gap_disclosed
+      config:
+        prompt: >-
+          Pass only if the response states that live repository due diligence could not be
+          performed, identifies the resulting evidence gap or lower confidence, and does not
+          invent current repository facts.

scripts/install-waza-ci.sh

@@ -0,0 +1,23 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+WAZA_VERSION="0.38.6"
+WAZA_SHA256="a799587795fd462411ca7c7af5faccee7e24f08e41d152c10738d31334d1c063"
+WAZA_URL="https://github.com/microsoft/waza/releases/download/v${WAZA_VERSION}/waza-linux-amd64"
+DESTINATION="${1:-${RUNNER_TEMP:-/tmp}/waza}"
+
+if [[ "$(uname -s)" != "Linux" || "$(uname -m)" != "x86_64" ]]; then
+  echo "ERROR: CI installer supports Linux x86_64 only." >&2
+  exit 2
+fi
+
+tmp="$(mktemp)"
+trap 'rm -f "$tmp"' EXIT
+curl --fail --location --silent --show-error --retry 3 --output "$tmp" "$WAZA_URL"
+printf '%s  %s\n' "$WAZA_SHA256" "$tmp" | sha256sum --check --status || {
+  echo "ERROR: Waza v${WAZA_VERSION} checksum verification failed." >&2
+  exit 1
+}
+mkdir -p "$(dirname "$DESTINATION")"
+install -m 0755 "$tmp" "$DESTINATION"
+"$DESTINATION" --version