feat(judge): support judge-specific skills

CHANGELOG.md

@@ -5,6 +5,44 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## [0.6.0] - 2026-07-14
+
+### Added
+- Judge-specific Skills for `agent_judge` via `judge.skills`. These Skills are
+  installed only into the judge agent and can provide reusable grading rubrics,
+  evidence rules, and domain-specific constraints without leaking into the run
+  agent or benchmark baseline.
+- Judge Skill metadata in JSON, JUnit, and HTML reports, including the source,
+  path, target, and callable Skill name used for the evaluation.
+
+### Fixed
+- Judge prompts now identify installed Skills by their callable names and
+  explicitly require invoking each Skill tool and reading its body before
+  grading. Source paths remain only as an internal fallback when no name is
+  available.
+
+## [0.5.0] - 2026-07-09
+
+### Added
+- Configurable prompt delivery for `agent_judge` through `judge.context`.
+  Supports `minimal` and `standard` context profiles, per-field delivery
+  modes, inline-size limits, generated-file indexing, and attachment file
+  references. Reports include `judge_context` metadata describing how review
+  materials and prompts were delivered.
+- Built-in CLI agents now route large prompts through runtime-readable files
+  and stdin when they exceed `SKILL_UP_PROMPT_INLINE_MAX_BYTES` (default
+  `32768` bytes), while shorter prompts remain inline.
+
+### Changed
+- `agent_judge` now defaults to materialized review context (`profile:
+  standard`) instead of inlining the full transcript and workspace diff into
+  the judge prompt. Large artifacts are delivered through runtime-readable
+  files, while short final messages remain inline. Evals that require legacy
+  inline transcript delivery can set `judge.context.transcript: include`.
+
+The `v0.5.0` release tag is available at
+[GitHub](https://github.com/alibaba/skill-up/releases/tag/v0.5.0).
+
 ## [0.4.0] - 2026-07-07
 
 ### Added
@@ -22,12 +60,6 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
   `result.json`; turn-scoped failure details in JUnit XML).
 
 ### Changed
-- `agent_judge` now defaults to materialized review context (`profile:
-  standard`) instead of inlining full transcript and workspace diff into the
-  judge prompt. The judge receives a materials table with file references for
-  large artifacts, while short final messages remain inline. Evals that require
-  legacy inline transcript can set `judge.context.transcript: include` subject
-  to `judge.context.limits.max_bytes`.
 - `skill-up run` now validates only the cases left after
   `--include-case-name` / `--exclude-case-name` filters (plus the eval-level
   config), so an invalid case that is filtered out no longer blocks a filtered
@@ -36,14 +68,6 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
   case) regardless of filters.
 
 ### Added
-- `judge.context` for `agent_judge`, with `minimal` and `standard` profiles,
-  per-field delivery modes, inline-size limits, generated-file indexing, and
-  attachment file references. Reports now include `judge_context` metadata with
-  materialization and prompt-delivery details.
-- Built-in CLI agents now route large prompts through prompt delivery, writing
-  prompt artifacts to disk and feeding runtime-readable prompt files through
-  stdin when the prompt exceeds `SKILL_UP_PROMPT_INLINE_MAX_BYTES` (default
-  32768 bytes).
 - Config validation now rejects duplicate case IDs and duplicate `cases.files`
   references. Since reports and artifacts are keyed by case ID and one case runs
   per `cases.files` entry, a collision would silently overwrite results or
@@ -377,6 +401,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
   project and delivers the end-to-end capability to declare eval environments,
   run cases and emit structured reports as described in [README.md](README.md).
 
+[0.6.0]: https://github.com/alibaba/skill-up/releases/tag/v0.6.0
+[0.5.0]: https://github.com/alibaba/skill-up/releases/tag/v0.5.0
+[0.4.0]: https://github.com/alibaba/skill-up/releases/tag/v0.4.0
 [0.3.0]: https://github.com/alibaba/skill-up/releases/tag/v0.3.0
 [0.2.4]: https://github.com/alibaba/skill-up/releases/tag/v0.2.4
 [0.2.3]: https://github.com/alibaba/skill-up/releases/tag/v0.2.3

docs/design/custom-engine.md

@@ -464,6 +464,14 @@ The `local` transport runs a command via `runtime.Exec`. The command runs inside
 current runtime, so it can access the runtime workspace, installed skills, fixtures,
 MCP config, and environment variables.
 
+Top-level `skills` are installed before the main run agent executes. If
+`judge.type: agent_judge` declares `judge.skills`, those Skills are installed
+separately before the judge agent runs. Both paths use the Agent adapter's
+`InstallSkill` implementation; skill-up does not concatenate Skill files into
+the prompt as a fallback. Custom and remote engines that need judge Skills must
+make their agent process discover the installed Skill directory according to
+their own conventions.
+
 Example:
 
 ```yaml

docs/guide/writing-evals.md

@@ -659,6 +659,9 @@ Let an LLM grade against rubric criteria — useful when semantic understanding
 judge:
   type: agent_judge
   model: anthropic/claude-sonnet-4-6        # Model used by the judge
+  skills:                                   # Optional: judge-only Skills
+    - source: local_path
+      path: evals/fixtures/judge-rubric
   criteria:                                  # Natural-language rubric
     - "Identifies a real bug with an accurate location"
     - "Does not flag correct code as a bug"
@@ -667,6 +670,14 @@ judge:
   timeout_seconds: 60                        # Optional: bound a single judge call (0 = no judge-level deadline, parent case timeout still applies)
 ```
 
+`judge.skills` is supported only for `agent_judge`. These Skills are installed
+into the judge agent, not the run agent, and top-level `skills` are not
+automatically installed into the judge. In benchmark mode, judge Skills are
+installed for both `with_skill` and `without_skill` runs because they are
+grading tooling, not the Skill under test. Installation uses each Agent
+adapter's native Skill mechanism; skill-up does not concatenate Skill files
+into the judge prompt.
+
 `agent_judge` materializes review context into files and injects a small
 materials table into the judge prompt. When `judge.context` is omitted, the
 default profile is `standard`: `final_message` is included inline, while

docs/zh/guide/writing-evals.md

@@ -634,6 +634,9 @@ judge:
 judge:
   type: agent_judge
   model: anthropic/claude-sonnet-4-6        # 评审使用的模型
+  skills:                                   # 可选：仅供 judge 使用的 Skills
+    - source: local_path
+      path: evals/fixtures/judge-rubric
   criteria:                                  # 评估标准（自然语言描述）
     - "输出中识别了真实存在的 bug，并给出了准确位置"
     - "没有将正确代码误报为 bug"
@@ -642,6 +645,12 @@ judge:
   timeout_seconds: 60                        # 可选：限制单次 judge 调用时长（0 = 不加 judge 级 deadline，仍受 case timeout 约束）
 ```
 
+`judge.skills` 仅支持 `agent_judge`。这些 Skills 会安装到 judge agent，
+不会安装到主运行 agent；顶层 `skills` 也不会自动安装到 judge。开启
+benchmark 时，`with_skill` 和 `without_skill` 都会安装 judge Skills，因为它们是
+评分工具，不是被测 Skill。安装过程使用各 Agent adapter 原生的 Skill 机制；
+skill-up 不会把 Skill 文件内容拼接进 judge prompt。
+
 `agent_judge` 会将评审上下文物化为文件，并在 judge prompt 中注入一个很小的
 材料表。未配置 `judge.context` 时，默认使用 `standard` profile：
 `final_message` 内联，`transcript` 和 `workspace_diff` 以文件引用形式提供，

e2e/agent_test.go

@@ -441,7 +441,7 @@ input:
   prompt: Reply with exactly the word hello. Do not run commands.
 
 constraints:
-  timeout_seconds: 5
+  timeout_seconds: 30
   max_turns: 1
 
 expect:

internal/agent/agent.go

@@ -190,6 +190,11 @@ func (a *BaseAgent) Name() string {
 	return a.Cfg.Name
 }
 
+// SkillPath returns the agent's default skill installation path.
+func (a *BaseAgent) SkillPath() string {
+	return a.Cfg.SkillPath
+}
+
 // CheckCredentials checks if the required credentials are set.
 func (a *BaseAgent) CheckCredentials(ctx context.Context) error {
 	switch a.Cfg.ModelProvider {

internal/agent/codex_test.go

@@ -353,6 +353,7 @@ func TestCodexRunTurn_FirstTurnDelegatesToRun(t *testing.T) {
 	}
 	if result == nil {
 		t.Fatal("expected non-nil result")
+		return
 	}
 	// First turn should use "codex exec", not "codex resume"
 	if containsCommand(rt.commands, "codex resume") {
@@ -745,6 +746,7 @@ func TestCodexRun_PreservesArtifactsOnNonZeroExit(t *testing.T) {
 	}
 	if sessionResult == nil {
 		t.Fatal("expected session result")
+		return
 	}
 	if sessionResult.ExitCode != 1 {
 		t.Fatalf("expected exit code 1, got %d", sessionResult.ExitCode)

internal/agent/custom_test.go

@@ -370,6 +370,7 @@ func TestCustomAgent_RunLocal_NonZeroProcessExitFails(t *testing.T) {
 	}
 	if res == nil {
 		t.Fatal("expected session result to be preserved")
+		return
 	}
 	// The real process exit code must surface in the result, not the JSON's 0.
 	if res.ExitCode != 7 {

internal/cli/judge_debug.go

@@ -142,7 +142,7 @@ func runJudgeDebug(cmd *cobra.Command, args []string) error {
 	}
 
 	var input judgeDebugInput
-	if err := json.Unmarshal(data, &input); err != nil {
+	if err := json.Unmarshal(data, &input); err != nil { //nolint:musttag // debug input embeds config structs with existing tags.
 		return fmt.Errorf("parse input JSON: %w", err)
 	}
 

internal/cli/judge_debug_test.go

@@ -25,7 +25,7 @@ import (
 // writeJudgeDebugInput serialises a judgeDebugInput to a JSON file and returns the path.
 func writeJudgeDebugInput(t *testing.T, dir string, input judgeDebugInput) string {
 	t.Helper()
-	data, err := json.Marshal(input)
+	data, err := json.Marshal(input) //nolint:musttag // debug input embeds config structs with existing tags.
 	if err != nil {
 		t.Fatalf("marshal judge debug input: %v", err)
 	}
@@ -238,6 +238,7 @@ func TestMockJudgeAgent_Run(t *testing.T) {
 	}
 	if sr == nil {
 		t.Fatal("Run: nil SessionResult")
+		return
 	}
 
 	var parsed struct {

internal/cli/run.go

@@ -186,10 +186,11 @@ func loadAndPrepareConfig(ctx context.Context, cmd *cobra.Command, args []string
 		return nil, nil, nil, errors.New("no cases matched the filter")
 	}
 
-	// Validate only the selected cases (after filters). Full-suite per-case
-	// validation is the job of `skill-up validate`; here an invalid case that
-	// was filtered out must not block the run.
-	if err := config.NewValidator().ValidateCases(cases); err != nil {
+	// Validate only the selected cases (after filters), while still applying
+	// eval-level judge defaults to case-level overrides. Full-suite per-case
+	// validation is the job of `skill-up validate`; here an invalid case that was
+	// filtered out must not block the run.
+	if err := config.NewValidator().ValidateCasesWithEvalDefaults(evalCfg, cases); err != nil {
 		return nil, nil, nil, fmt.Errorf("validation failed: %w", err)
 	}
 

internal/config/loader_test.go

@@ -267,6 +267,54 @@ cases:
 	}
 }
 
+func TestLoader_LoadEvalConfig_JudgeSkills(t *testing.T) {
+	t.Parallel()
+
+	tmp := t.TempDir()
+	evalPath := filepath.Join(tmp, "eval.yaml")
+	content := `schema_version: v1alpha1
+
+environment:
+  type: none
+
+engine:
+  name: claude_code
+
+cases:
+  files:
+    - evals/cases/basic.yaml
+
+judge:
+  type: agent_judge
+  model: test-model
+  skills:
+    - source: local_path
+      path: evals/fixtures/default-judge
+    - source: local_path
+      path: evals/fixtures/security-judge
+      target: ~/.claude/skills/security-judge
+  criteria:
+    - criterion one
+`
+	if err := os.WriteFile(evalPath, []byte(content), 0o600); err != nil {
+		t.Fatalf("failed to write temp eval.yaml: %v", err)
+	}
+
+	cfg, err := NewLoader(evalPath).LoadEvalConfig()
+	if err != nil {
+		t.Fatalf("LoadEvalConfig failed: %v", err)
+	}
+	if len(cfg.Judge.Skills) != 2 {
+		t.Fatalf("Judge.Skills length = %d, want 2", len(cfg.Judge.Skills))
+	}
+	if got := cfg.Judge.Skills[0].Path; got != "evals/fixtures/default-judge" {
+		t.Fatalf("Judge.Skills[0].Path = %q", got)
+	}
+	if got := cfg.Judge.Skills[1].Target; got != "~/.claude/skills/security-judge" {
+		t.Fatalf("Judge.Skills[1].Target = %q", got)
+	}
+}
+
 // nolint:funlen // table-driven matrix over the documented defaults; keeping
 // each case inline beats spreading them across helpers.
 func TestLoader_LoadEvalConfig_AppliesDefaults(t *testing.T) {

internal/config/schema.go

@@ -190,6 +190,7 @@ type JudgeConfig struct {
 	ScriptPath string              `json:"script_path,omitempty"    yaml:"script_path,omitempty"`
 	Model      string              `json:"model,omitempty"          yaml:"model,omitempty"`
 	Criteria   []string            `json:"criteria,omitempty"       yaml:"criteria,omitempty"`
+	Skills     []SkillRef          `json:"skills,omitempty"         yaml:"skills,omitempty"`
 	Context    *JudgeContextConfig `json:"context,omitempty" yaml:"context,omitempty"`
 	// PassThreshold is the minimum pass rate for agent_judge.
 	// Nil means "not configured", so the judge layer applies its default of 0.7.

internal/config/schema_test.go

@@ -11,6 +11,7 @@ func TestDefaultEvalConfig(t *testing.T) { //nolint:cyclop,gocyclo // exhaustive
 	cfg := DefaultEvalConfig()
 	if cfg == nil {
 		t.Fatal("DefaultEvalConfig() returned nil")
+		return
 	}
 
 	// Verify schema version