feat(judge): support judge-specific skills

CHANGELOG.md

@@ -20,6 +20,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
   unsupported engines fall back to batch mode.
 - `TurnResults` field in JSON, HTML, and JUnit reports (`turn_results` in
   `result.json`; turn-scoped failure details in JUnit XML).
+- `judge.skills` for `agent_judge`, allowing judge-only Skills to be installed
+  into the judge agent as reusable grading rubrics without leaking into the run
+  agent or benchmark baseline.
 
 ### Changed
 - `agent_judge` now defaults to materialized review context (`profile:

docs/design/custom-engine.md

@@ -464,6 +464,14 @@ The `local` transport runs a command via `runtime.Exec`. The command runs inside
 current runtime, so it can access the runtime workspace, installed skills, fixtures,
 MCP config, and environment variables.
 
+Top-level `skills` are installed before the main run agent executes. If
+`judge.type: agent_judge` declares `judge.skills`, those Skills are installed
+separately before the judge agent runs. Both paths use the Agent adapter's
+`InstallSkill` implementation; skill-up does not concatenate Skill files into
+the prompt as a fallback. Custom and remote engines that need judge Skills must
+make their agent process discover the installed Skill directory according to
+their own conventions.
+
 Example:
 
 ```yaml

docs/guide/writing-evals.md

@@ -659,6 +659,9 @@ Let an LLM grade against rubric criteria — useful when semantic understanding
 judge:
   type: agent_judge
   model: anthropic/claude-sonnet-4-6        # Model used by the judge
+  skills:                                   # Optional: judge-only Skills
+    - source: local_path
+      path: evals/fixtures/judge-rubric
   criteria:                                  # Natural-language rubric
     - "Identifies a real bug with an accurate location"
     - "Does not flag correct code as a bug"
@@ -667,6 +670,14 @@ judge:
   timeout_seconds: 60                        # Optional: bound a single judge call (0 = no judge-level deadline, parent case timeout still applies)
 ```
 
+`judge.skills` is supported only for `agent_judge`. These Skills are installed
+into the judge agent, not the run agent, and top-level `skills` are not
+automatically installed into the judge. In benchmark mode, judge Skills are
+installed for both `with_skill` and `without_skill` runs because they are
+grading tooling, not the Skill under test. Installation uses each Agent
+adapter's native Skill mechanism; skill-up does not concatenate Skill files
+into the judge prompt.
+
 `agent_judge` materializes review context into files and injects a small
 materials table into the judge prompt. When `judge.context` is omitted, the
 default profile is `standard`: `final_message` is included inline, while

docs/zh/guide/writing-evals.md

@@ -634,6 +634,9 @@ judge:
 judge:
   type: agent_judge
   model: anthropic/claude-sonnet-4-6        # 评审使用的模型
+  skills:                                   # 可选：仅供 judge 使用的 Skills
+    - source: local_path
+      path: evals/fixtures/judge-rubric
   criteria:                                  # 评估标准（自然语言描述）
     - "输出中识别了真实存在的 bug，并给出了准确位置"
     - "没有将正确代码误报为 bug"
@@ -642,6 +645,12 @@ judge:
   timeout_seconds: 60                        # 可选：限制单次 judge 调用时长（0 = 不加 judge 级 deadline，仍受 case timeout 约束）
 ```
 
+`judge.skills` 仅支持 `agent_judge`。这些 Skills 会安装到 judge agent，
+不会安装到主运行 agent；顶层 `skills` 也不会自动安装到 judge。开启
+benchmark 时，`with_skill` 和 `without_skill` 都会安装 judge Skills，因为它们是
+评分工具，不是被测 Skill。安装过程使用各 Agent adapter 原生的 Skill 机制；
+skill-up 不会把 Skill 文件内容拼接进 judge prompt。
+
 `agent_judge` 会将评审上下文物化为文件，并在 judge prompt 中注入一个很小的
 材料表。未配置 `judge.context` 时，默认使用 `standard` profile：
 `final_message` 内联，`transcript` 和 `workspace_diff` 以文件引用形式提供，

e2e/agent_test.go

@@ -441,7 +441,7 @@ input:
   prompt: Reply with exactly the word hello. Do not run commands.
 
 constraints:
-  timeout_seconds: 5
+  timeout_seconds: 30
   max_turns: 1
 
 expect:

internal/agent/codex_test.go

@@ -353,6 +353,7 @@ func TestCodexRunTurn_FirstTurnDelegatesToRun(t *testing.T) {
 	}
 	if result == nil {
 		t.Fatal("expected non-nil result")
+		return
 	}
 	// First turn should use "codex exec", not "codex resume"
 	if containsCommand(rt.commands, "codex resume") {
@@ -745,6 +746,7 @@ func TestCodexRun_PreservesArtifactsOnNonZeroExit(t *testing.T) {
 	}
 	if sessionResult == nil {
 		t.Fatal("expected session result")
+		return
 	}
 	if sessionResult.ExitCode != 1 {
 		t.Fatalf("expected exit code 1, got %d", sessionResult.ExitCode)

internal/agent/custom_test.go

@@ -370,6 +370,7 @@ func TestCustomAgent_RunLocal_NonZeroProcessExitFails(t *testing.T) {
 	}
 	if res == nil {
 		t.Fatal("expected session result to be preserved")
+		return
 	}
 	// The real process exit code must surface in the result, not the JSON's 0.
 	if res.ExitCode != 7 {

internal/cli/judge_debug.go

@@ -142,7 +142,7 @@ func runJudgeDebug(cmd *cobra.Command, args []string) error {
 	}
 
 	var input judgeDebugInput
-	if err := json.Unmarshal(data, &input); err != nil {
+	if err := json.Unmarshal(data, &input); err != nil { //nolint:musttag // debug input embeds config structs with existing tags.
 		return fmt.Errorf("parse input JSON: %w", err)
 	}
 

internal/cli/judge_debug_test.go

@@ -25,7 +25,7 @@ import (
 // writeJudgeDebugInput serialises a judgeDebugInput to a JSON file and returns the path.
 func writeJudgeDebugInput(t *testing.T, dir string, input judgeDebugInput) string {
 	t.Helper()
-	data, err := json.Marshal(input)
+	data, err := json.Marshal(input) //nolint:musttag // debug input embeds config structs with existing tags.
 	if err != nil {
 		t.Fatalf("marshal judge debug input: %v", err)
 	}
@@ -238,6 +238,7 @@ func TestMockJudgeAgent_Run(t *testing.T) {
 	}
 	if sr == nil {
 		t.Fatal("Run: nil SessionResult")
+		return
 	}
 
 	var parsed struct {

internal/cli/run.go

@@ -186,10 +186,11 @@ func loadAndPrepareConfig(ctx context.Context, cmd *cobra.Command, args []string
 		return nil, nil, nil, errors.New("no cases matched the filter")
 	}
 
-	// Validate only the selected cases (after filters). Full-suite per-case
-	// validation is the job of `skill-up validate`; here an invalid case that
-	// was filtered out must not block the run.
-	if err := config.NewValidator().ValidateCases(cases); err != nil {
+	// Validate only the selected cases (after filters), while still applying
+	// eval-level judge defaults to case-level overrides. Full-suite per-case
+	// validation is the job of `skill-up validate`; here an invalid case that was
+	// filtered out must not block the run.
+	if err := config.NewValidator().ValidateCasesWithEvalDefaults(evalCfg, cases); err != nil {
 		return nil, nil, nil, fmt.Errorf("validation failed: %w", err)
 	}
 

internal/config/loader_test.go

@@ -267,6 +267,54 @@ cases:
 	}
 }
 
+func TestLoader_LoadEvalConfig_JudgeSkills(t *testing.T) {
+	t.Parallel()
+
+	tmp := t.TempDir()
+	evalPath := filepath.Join(tmp, "eval.yaml")
+	content := `schema_version: v1alpha1
+
+environment:
+  type: none
+
+engine:
+  name: claude_code
+
+cases:
+  files:
+    - evals/cases/basic.yaml
+
+judge:
+  type: agent_judge
+  model: test-model
+  skills:
+    - source: local_path
+      path: evals/fixtures/default-judge
+    - source: local_path
+      path: evals/fixtures/security-judge
+      target: ~/.claude/skills/security-judge
+  criteria:
+    - criterion one
+`
+	if err := os.WriteFile(evalPath, []byte(content), 0o600); err != nil {
+		t.Fatalf("failed to write temp eval.yaml: %v", err)
+	}
+
+	cfg, err := NewLoader(evalPath).LoadEvalConfig()
+	if err != nil {
+		t.Fatalf("LoadEvalConfig failed: %v", err)
+	}
+	if len(cfg.Judge.Skills) != 2 {
+		t.Fatalf("Judge.Skills length = %d, want 2", len(cfg.Judge.Skills))
+	}
+	if got := cfg.Judge.Skills[0].Path; got != "evals/fixtures/default-judge" {
+		t.Fatalf("Judge.Skills[0].Path = %q", got)
+	}
+	if got := cfg.Judge.Skills[1].Target; got != "~/.claude/skills/security-judge" {
+		t.Fatalf("Judge.Skills[1].Target = %q", got)
+	}
+}
+
 // nolint:funlen // table-driven matrix over the documented defaults; keeping
 // each case inline beats spreading them across helpers.
 func TestLoader_LoadEvalConfig_AppliesDefaults(t *testing.T) {

internal/config/schema.go

@@ -190,6 +190,7 @@ type JudgeConfig struct {
 	ScriptPath string              `json:"script_path,omitempty"    yaml:"script_path,omitempty"`
 	Model      string              `json:"model,omitempty"          yaml:"model,omitempty"`
 	Criteria   []string            `json:"criteria,omitempty"       yaml:"criteria,omitempty"`
+	Skills     []SkillRef          `json:"skills,omitempty"         yaml:"skills,omitempty"`
 	Context    *JudgeContextConfig `json:"context,omitempty" yaml:"context,omitempty"`
 	// PassThreshold is the minimum pass rate for agent_judge.
 	// Nil means "not configured", so the judge layer applies its default of 0.7.

internal/config/schema_test.go

@@ -11,6 +11,7 @@ func TestDefaultEvalConfig(t *testing.T) { //nolint:cyclop,gocyclo // exhaustive
 	cfg := DefaultEvalConfig()
 	if cfg == nil {
 		t.Fatal("DefaultEvalConfig() returned nil")
+		return
 	}
 
 	// Verify schema version

internal/config/validator.go

@@ -96,7 +96,7 @@ func (v *Validator) ValidateEvalConfig(cfg *EvalConfig) error {
 	}
 
 	errs = append(errs, validateCollectArtifacts("cases.defaults.collect_artifacts", cfg.Cases.Defaults.CollectArtifacts)...)
-	errs = append(errs, validateJudgeContext(cfg.Judge.Context)...)
+	errs = append(errs, validateJudgeTypeAndLocalFields(cfg.Judge)...)
 
 	if len(errs) > 0 {
 		return fmt.Errorf("validation errors:\n  - %s", strings.Join(errs, "\n  - "))
@@ -143,8 +143,8 @@ func (v *Validator) ValidateCaseConfig(cfg *CaseConfig) error {
 
 	// validate per-turn judge rules reference valid turn numbers
 	turnsTotal := len(cfg.Input.Turns)
-	if cfg.Judge.Type != "" {
-		errs = append(errs, validateJudgeTypeAndFields(cfg.Judge)...)
+	if judgeNeedsLocalValidation(cfg.Judge) {
+		errs = append(errs, validateJudgeTypeAndLocalFields(cfg.Judge)...)
 	}
 	var allRules []Rule
 	allRules = append(allRules, cfg.Judge.Success...)
@@ -160,35 +160,56 @@ func (v *Validator) ValidateCaseConfig(cfg *CaseConfig) error {
 	return nil
 }
 
-// validateJudgeTypeAndFields validates judge type and its conditional fields.
-func validateJudgeTypeAndFields(judge JudgeConfig) []string {
+func judgeNeedsLocalValidation(judge JudgeConfig) bool {
+	return judge.Type != "" || len(judge.Skills) > 0 || judge.Context != nil
+}
+
+// validateJudgeTypeAndLocalFields validates judge fields that do not depend on inheritance.
+func validateJudgeTypeAndLocalFields(judge JudgeConfig) []string {
 	var errs []string
 
 	if judge.Type != "" && !isValidJudgeType(judge.Type) {
 		errs = append(errs, "judge.type must be one of: rule_based, script, agent_judge")
 	}
+	if len(judge.Skills) > 0 && judge.Type != judgeTypeAgentJudge {
+		errs = append(errs, "judge.skills is only supported when judge.type is agent_judge")
+	}
+	errs = append(errs, validateSkillRefs("judge.skills", judge.Skills)...)
+
+	errs = append(errs, validatePassThreshold(judge.PassThreshold)...)
+	errs = append(errs, validateJudgeContext(judge.Context)...)
+
+	if judge.TimeoutSeconds != nil && *judge.TimeoutSeconds < 0 {
+		errs = append(errs, "judge.timeout_seconds must be non-negative")
+	}
 
-	// script type requires script_path
+	return errs
+}
+
+func validateJudgeTypeAndRequiredFields(judge JudgeConfig) []string {
+	errs := validateJudgeTypeAndLocalFields(judge)
 	if judge.Type == judgeTypeScript && judge.ScriptPath == "" {
 		errs = append(errs, "judge.script_path is required when judge.type is script")
 	}
-
-	// agent_judge type requires model and criteria
 	if judge.Type == judgeTypeAgentJudge && judge.Model == "" {
 		errs = append(errs, "judge.model is required when judge.type is agent_judge")
 	}
-
 	if judge.Type == judgeTypeAgentJudge && len(judge.Criteria) == 0 {
 		errs = append(errs, "judge.criteria is required when judge.type is agent_judge")
 	}
+	return errs
+}
 
-	errs = append(errs, validatePassThreshold(judge.PassThreshold)...)
-	errs = append(errs, validateJudgeContext(judge.Context)...)
-
-	if judge.TimeoutSeconds != nil && *judge.TimeoutSeconds < 0 {
-		errs = append(errs, "judge.timeout_seconds must be non-negative")
+func validateSkillRefs(field string, refs []SkillRef) []string {
+	var errs []string
+	for i, ref := range refs {
+		if strings.TrimSpace(ref.Source) == "" {
+			errs = append(errs, fmt.Sprintf("%s[%d].source is required", field, i))
+		}
+		if ref.Source == "local_path" && strings.TrimSpace(ref.Path) == "" {
+			errs = append(errs, fmt.Sprintf("%s[%d].path is required when source is local_path", field, i))
+		}
 	}
-
 	return errs
 }
 
@@ -293,7 +314,7 @@ func (v *Validator) ValidateAll(result *EvalResult) error {
 	if err := v.ValidateEvalConfig(result.Eval); err != nil {
 		return err
 	}
-	return v.ValidateCases(result.Cases)
+	return v.ValidateCasesWithEvalDefaults(result.Eval, result.Cases)
 }
 
 // ValidateCases validates a set of cases: it rejects duplicate effective IDs
@@ -319,6 +340,40 @@ func (v *Validator) ValidateCases(cases []*CaseConfig) error {
 	return nil
 }
 
+// ValidateCasesWithEvalDefaults validates already-loaded cases and effective
+// judge settings after applying eval-level defaults. Unlike ValidateAll, it
+// does not require eval.cases.files and is therefore suitable for filtered
+// runs and Anthropic evals.json auto mode where cases are loaded directly.
+func (v *Validator) ValidateCasesWithEvalDefaults(eval *EvalConfig, cases []*CaseConfig) error {
+	if err := v.ValidateCases(cases); err != nil {
+		return err
+	}
+	for _, c := range cases {
+		effectiveJudge := mergeJudgeConfigForValidation(eval.Judge, c.Judge)
+		if errs := validateJudgeTypeAndRequiredFields(effectiveJudge); len(errs) > 0 {
+			return fmt.Errorf("case %s: validation errors:\n  - %s", c.ID, strings.Join(errs, "\n  - "))
+		}
+	}
+	return nil
+}
+
+func mergeJudgeConfigForValidation(global, caseLevel JudgeConfig) JudgeConfig {
+	if caseLevel.Type != "" {
+		merged := caseLevel
+		if merged.Model == "" {
+			merged.Model = global.Model
+		}
+		if merged.PassThreshold == nil {
+			merged.PassThreshold = global.PassThreshold
+		}
+		if merged.TimeoutSeconds == nil {
+			merged.TimeoutSeconds = global.TimeoutSeconds
+		}
+		return merged
+	}
+	return global
+}
+
 // duplicateCaseErrors reports duplicate source-file references and duplicate
 // effective case IDs within the given case set. skill-up writes reports and
 // artifacts keyed by case ID and runs one case per cases.files entry, so a