From af1ba891dfd0883390ee0c65bdb5c460a051b76f Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Wed, 2 Sep 2026 11:49:18 +0900 Subject: [PATCH 1/8] test(email-writing): define admission policy RED contract --- backend/tests/test_email_writing_policy.py | 407 +++++++++++++++++++++ 1 file changed, 407 insertions(+) create mode 100644 backend/tests/test_email_writing_policy.py diff --git a/backend/tests/test_email_writing_policy.py b/backend/tests/test_email_writing_policy.py new file mode 100644 index 000000000..acfee48ae --- /dev/null +++ b/backend/tests/test_email_writing_policy.py @@ -0,0 +1,407 @@ +"""Regression contract for versioned email-writing Judge admission policy.""" + +from __future__ import annotations + +from copy import deepcopy +from datetime import datetime, timezone +import hashlib +import json +from pathlib import Path + +import pytest + +from services.email_writing_policy import ( + EmailWritingPolicyError, + evaluate_policy_admission, + load_policy_artifact, + select_rollback_policy, +) + +_POLICY_DIR = Path(__file__).resolve().parents[1] / "policies" +_POLICY_NAME = "email_writing_judge_evaluation_only_v1.json" +_NOW = datetime(2026, 9, 2, 0, 0, tzinfo=timezone.utc) +_RUNTIME_CONTRACTS = { + "naruon": "0.14.4", + "inkspan": None, + "fast_mlsirm": None, + "contextual_orchestrator": "v1", +} +_REQUIRED_CRITERIA = ( + "issue_support", + "span_fidelity", + "replacement_correctness", + "intent_preservation", + "fact_preservation", + "request_strength_preservation", + "audience_pragmatics", + "technical_precision", + "actionability", + "explanation_quality", +) + + +def _canonical_bytes(payload: dict[str, object]) -> bytes: + """Serialize an artifact exactly as the integrity manifest contract expects.""" + return ( + json.dumps(payload, ensure_ascii=False, sort_keys=True, separators=(",", ":")) + + "\n" + ).encode() + + +def _manifest_for(name: str, payload: dict[str, object]) -> dict[str, object]: + """Build a minimal manifest bound to one canonical test artifact.""" + digest = hashlib.sha256(_canonical_bytes(payload)).hexdigest() + return { + "manifest_version": 1, + "artifacts": {name: {"sha256": digest}}, + } + + +def _evaluation_policy() -> dict[str, object]: + """Return the checked-in evaluation policy as mutable test data.""" + return json.loads((_POLICY_DIR / _POLICY_NAME).read_text(encoding="utf-8")) + + +def _published_policy(*, version: str = "email-writing-policy-v1") -> dict[str, object]: + """Build a fully evidenced synthetic published policy for deterministic tests.""" + payload = _evaluation_policy() + payload.update( + { + "policy_id": "email_writing_judge_policy", + "policy_version": version, + "status": "published", + "publish_decision": "publish", + "compatible_contracts": { + "naruon": "0.14.4", + "inkspan": "0.6.0", + "fast_mlsirm": "0.9.2", + "contextual_orchestrator": "v1", + }, + "calibration_summary": { + "status": "validated", + "brier_score": 0.08, + "test_retest_reliability": 0.91, + "dif_profiles_evaluated": 8, + "temporal_drift_evaluated": True, + }, + "evidence": { + "protocol_hash": "sha256:" + "1" * 64, + "calibration_dataset_hash": "sha256:" + "2" * 64, + "locked_holdout_hash": "sha256:" + "3" * 64, + "reference_adjudication_hash": "sha256:" + "4" * 64, + "holdout_labels_accessed_after_preregistration": True, + }, + "mandatory_criterion_floors": { + criterion: 2 for criterion in _REQUIRED_CRITERIA + }, + "minimum_criterion_scores": { + criterion: 0.5 for criterion in _REQUIRED_CRITERIA + }, + } + ) + return payload + + +def _load( + payload: dict[str, object], + *, + runtime_contracts: dict[str, str | None] | None = None, +): + """Load one synthetic artifact through the production integrity boundary.""" + manifest = _manifest_for(_POLICY_NAME, payload) + return load_policy_artifact( + artifact_name=_POLICY_NAME, + artifact_bytes=_canonical_bytes(payload), + manifest_bytes=_canonical_bytes(manifest), + now=_NOW, + runtime_contracts=runtime_contracts or _RUNTIME_CONTRACTS, + ) + + +def test_policy_schema_declares_strict_lifecycle_and_publish_decision() -> None: + """Keep lifecycle/publication fields explicit and closed to unknown values.""" + schema = json.loads( + (_POLICY_DIR / "email_writing_judge_policy.schema.json").read_text( + encoding="utf-8" + ) + ) + assert schema["additionalProperties"] is False + assert schema["properties"]["publish_decision"]["enum"] == ["publish", "withhold"] + assert schema["properties"]["status"]["enum"] == [ + "evaluation_only", + "published", + "superseded", + "revoked", + ] + + +@pytest.mark.parametrize("field", ["publish_decision", "status", "policy_version"]) +def test_policy_rejects_missing_required_top_level_field(field: str) -> None: + """A policy missing lifecycle identity cannot enter the runtime registry.""" + payload = _evaluation_policy() + del payload[field] + with pytest.raises(EmailWritingPolicyError, match="policy_schema_invalid"): + _load(payload) + + +def test_evaluation_only_policy_must_withhold() -> None: + """An evaluation artifact can exercise the pipeline but cannot publish guidance.""" + payload = _evaluation_policy() + payload["publish_decision"] = "publish" + with pytest.raises(EmailWritingPolicyError, match="policy_lifecycle_invalid"): + _load(payload) + + +def test_policy_rejects_unknown_and_executable_fields() -> None: + """Field smuggling cannot add lexical tables, scripts, or executable policy content.""" + for field, value in ( + ("keyword_table", {"urgent": "critical"}), + ("executable", "python -c 'print(1)'"), + ): + payload = _evaluation_policy() + payload[field] = value + with pytest.raises(EmailWritingPolicyError, match="policy_schema_invalid"): + _load(payload) + + +def test_manifest_rejects_malformed_and_modified_artifacts() -> None: + """Only an exact artifact digest listed in the manifest can be loaded.""" + payload = _evaluation_policy() + bad_manifest = { + "manifest_version": 1, + "artifacts": {_POLICY_NAME: {"sha256": "not-a-digest"}}, + } + with pytest.raises(EmailWritingPolicyError, match="policy_manifest_invalid"): + load_policy_artifact( + artifact_name=_POLICY_NAME, + artifact_bytes=_canonical_bytes(payload), + manifest_bytes=_canonical_bytes(bad_manifest), + now=_NOW, + runtime_contracts=_RUNTIME_CONTRACTS, + ) + + manifest = _manifest_for(_POLICY_NAME, payload) + payload["limitations"] = ["modified after manifest creation"] + with pytest.raises(EmailWritingPolicyError, match="policy_integrity_mismatch"): + load_policy_artifact( + artifact_name=_POLICY_NAME, + artifact_bytes=_canonical_bytes(payload), + manifest_bytes=_canonical_bytes(manifest), + now=_NOW, + runtime_contracts=_RUNTIME_CONTRACTS, + ) + + +def test_evaluation_only_artifact_never_admits_user_facing_diagnostics() -> None: + """Runtime admission remains withheld even when Judge evidence is otherwise strong.""" + policy = _load(_evaluation_policy()) + outcome = evaluate_policy_admission( + policy=policy, + language_tag="en", + review_mode="deep", + candidate_model_profile_id="candidate-reviewer-v1", + candidate_provider_id="contextual-orchestrator", + judge_model_profile_id="independent-judge-v1", + judge_provider_id="contextual-orchestrator", + rubric_version="email_writing_judge_rubric_v1", + criterion_categories={criterion: 3 for criterion in _REQUIRED_CRITERIA}, + criterion_scores={criterion: 1.0 for criterion in _REQUIRED_CRITERIA}, + ) + assert outcome == "withhold" + + +def test_published_policy_requires_complete_calibration_and_holdout_evidence() -> None: + """Publication is invalid before preregistered holdout and adjudication evidence exists.""" + payload = _published_policy() + payload["evidence"]["locked_holdout_hash"] = None + with pytest.raises(EmailWritingPolicyError, match="policy_publication_evidence_incomplete"): + _load( + payload, + runtime_contracts={ + "naruon": "0.14.4", + "inkspan": "0.6.0", + "fast_mlsirm": "0.9.2", + "contextual_orchestrator": "v1", + }, + ) + + +def test_profile_and_contract_mismatch_fail_closed() -> None: + """Unsupported runtime contracts and language profiles cannot silently broaden claims.""" + payload = _published_policy() + with pytest.raises(EmailWritingPolicyError, match="policy_contract_incompatible"): + _load( + payload, + runtime_contracts={ + "naruon": "0.14.4", + "inkspan": "0.6.0", + "fast_mlsirm": "0.9.1", + "contextual_orchestrator": "v1", + }, + ) + + policy = _load( + payload, + runtime_contracts={ + "naruon": "0.14.4", + "inkspan": "0.6.0", + "fast_mlsirm": "0.9.2", + "contextual_orchestrator": "v1", + }, + ) + assert ( + evaluate_policy_admission( + policy=policy, + language_tag="fr", + review_mode="deep", + candidate_model_profile_id="candidate-reviewer-v1", + candidate_provider_id="contextual-orchestrator", + judge_model_profile_id="independent-judge-v1", + judge_provider_id="contextual-orchestrator", + rubric_version="email_writing_judge_rubric_v1", + criterion_categories={criterion: 3 for criterion in _REQUIRED_CRITERIA}, + criterion_scores={criterion: 1.0 for criterion in _REQUIRED_CRITERIA}, + ) + == "unsupported_profile" + ) + + +def test_mandatory_preservation_floor_cannot_be_hidden_by_high_average() -> None: + """A failed fact-preservation criterion withholds even when every other score is perfect.""" + payload = _published_policy() + policy = _load( + payload, + runtime_contracts={ + "naruon": "0.14.4", + "inkspan": "0.6.0", + "fast_mlsirm": "0.9.2", + "contextual_orchestrator": "v1", + }, + ) + categories = {criterion: 3 for criterion in _REQUIRED_CRITERIA} + scores = {criterion: 1.0 for criterion in _REQUIRED_CRITERIA} + categories["fact_preservation"] = 1 + scores["fact_preservation"] = 0.99 + assert ( + evaluate_policy_admission( + policy=policy, + language_tag="en", + review_mode="deep", + candidate_model_profile_id="candidate-reviewer-v1", + candidate_provider_id="contextual-orchestrator", + judge_model_profile_id="independent-judge-v1", + judge_provider_id="contextual-orchestrator", + rubric_version="email_writing_judge_rubric_v1", + criterion_categories=categories, + criterion_scores=scores, + ) + == "withhold" + ) + + +def test_same_model_candidate_and_judge_requires_published_compatibility() -> None: + """A same-model pair is adjudicated unless the policy explicitly calibrates that pairing.""" + payload = _published_policy() + profile = payload["approved_profiles"][0] + profile["judge_model_profile_id"] = profile["candidate_model_profile_id"] + profile["same_model_allowed"] = False + policy = _load( + payload, + runtime_contracts={ + "naruon": "0.14.4", + "inkspan": "0.6.0", + "fast_mlsirm": "0.9.2", + "contextual_orchestrator": "v1", + }, + ) + assert ( + evaluate_policy_admission( + policy=policy, + language_tag="en", + review_mode="deep", + candidate_model_profile_id="candidate-reviewer-v1", + candidate_provider_id="contextual-orchestrator", + judge_model_profile_id="candidate-reviewer-v1", + judge_provider_id="contextual-orchestrator", + rubric_version="email_writing_judge_rubric_v1", + criterion_categories={criterion: 3 for criterion in _REQUIRED_CRITERIA}, + criterion_scores={criterion: 1.0 for criterion in _REQUIRED_CRITERIA}, + ) + == "adjudicate" + ) + + +def test_mixed_criterion_identity_and_impossible_floors_fail_closed() -> None: + """Criterion identity and category semantics stay fixed rather than being coerced.""" + payload = _published_policy() + payload["mandatory_criterion_floors"]["fact_preservation"] = 4 + with pytest.raises(EmailWritingPolicyError, match="policy_floor_invalid"): + _load( + payload, + runtime_contracts={ + "naruon": "0.14.4", + "inkspan": "0.6.0", + "fast_mlsirm": "0.9.2", + "contextual_orchestrator": "v1", + }, + ) + + payload = _published_policy() + policy = _load( + payload, + runtime_contracts={ + "naruon": "0.14.4", + "inkspan": "0.6.0", + "fast_mlsirm": "0.9.2", + "contextual_orchestrator": "v1", + }, + ) + categories = {criterion: 3 for criterion in _REQUIRED_CRITERIA} + categories["invented_criterion"] = 3 + with pytest.raises(EmailWritingPolicyError, match="criterion_identity_mismatch"): + evaluate_policy_admission( + policy=policy, + language_tag="en", + review_mode="deep", + candidate_model_profile_id="candidate-reviewer-v1", + candidate_provider_id="contextual-orchestrator", + judge_model_profile_id="independent-judge-v1", + judge_provider_id="contextual-orchestrator", + rubric_version="email_writing_judge_rubric_v1", + criterion_categories=categories, + criterion_scores={criterion: 1.0 for criterion in _REQUIRED_CRITERIA}, + ) + + +def test_revoked_policy_rolls_back_only_to_explicit_compatible_manifest_entry() -> None: + """Rollback cannot silently downgrade to an unlisted, expired, or incompatible artifact.""" + current_payload = _published_policy(version="email-writing-policy-v2") + current_payload["status"] = "revoked" + current_payload["publish_decision"] = "withhold" + current_payload["rollback_version"] = "email-writing-policy-v1" + previous_payload = _published_policy(version="email-writing-policy-v1") + + current = _load( + current_payload, + runtime_contracts={ + "naruon": "0.14.4", + "inkspan": "0.6.0", + "fast_mlsirm": "0.9.2", + "contextual_orchestrator": "v1", + }, + ) + previous = _load( + previous_payload, + runtime_contracts={ + "naruon": "0.14.4", + "inkspan": "0.6.0", + "fast_mlsirm": "0.9.2", + "contextual_orchestrator": "v1", + }, + ) + assert select_rollback_policy(current_policy=current, candidates=[previous]) is previous + + unlisted = deepcopy(previous) + object.__setattr__(unlisted, "policy_version", "email-writing-policy-v0") + with pytest.raises(EmailWritingPolicyError, match="policy_rollback_unavailable"): + select_rollback_policy(current_policy=current, candidates=[unlisted]) From 9a6b636e9f4676091039ae45135a332001ab57e0 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Wed, 2 Sep 2026 11:54:13 +0900 Subject: [PATCH 2/8] feat(email-writing): add calibrated admission policy boundary --- ...mail_writing_judge_evaluation_only_v1.json | 1 + .../email_writing_judge_policy.schema.json | 1 + .../email_writing_policy_manifest.json | 1 + backend/services/email_writing_policy.py | 654 ++++++++++++++++++ 4 files changed, 657 insertions(+) create mode 100644 backend/policies/email_writing_judge_evaluation_only_v1.json create mode 100644 backend/policies/email_writing_judge_policy.schema.json create mode 100644 backend/policies/email_writing_policy_manifest.json create mode 100644 backend/services/email_writing_policy.py diff --git a/backend/policies/email_writing_judge_evaluation_only_v1.json b/backend/policies/email_writing_judge_evaluation_only_v1.json new file mode 100644 index 000000000..10737becb --- /dev/null +++ b/backend/policies/email_writing_judge_evaluation_only_v1.json @@ -0,0 +1 @@ +{"adjudication_conditions":["same_model_without_compatible_calibration","criterion_disagreement","insufficient_evidence"],"approved_profiles":[{"candidate_model_profile_id":"candidate-reviewer-v1","candidate_provider_id":"contextual-orchestrator","judge_model_profile_id":"independent-judge-v1","judge_provider_id":"contextual-orchestrator","language_tags":["en","ko"],"profile_id":"email-writing-evaluation-en-ko-v1","review_modes":["incremental","deep"],"rubric_version":"email_writing_judge_rubric_v1","same_model_allowed":false}],"calibration_summary":{"brier_score":null,"dif_profiles_evaluated":0,"status":"not_evaluated","temporal_drift_evaluated":false,"test_retest_reliability":null},"category_anchors":["no_credible_evidence","partial_or_weak_support","mostly_supported_with_gaps","fully_supported_with_accurate_evidence"],"category_count":4,"compatible_contracts":{"contextual_orchestrator":"v1","fast_mlsirm":null,"inkspan":null,"naruon":"0.14.4"},"created_at":"2026-09-01T00:00:00Z","evidence":{"calibration_dataset_hash":null,"holdout_labels_accessed_after_preregistration":false,"locked_holdout_hash":null,"protocol_hash":null,"reference_adjudication_hash":null},"expires_at":"2027-01-01T00:00:00Z","limitations":["Evaluation-only artifact. It cannot admit user-facing writing diagnostics.","No language, model, provider, calibration, DIF, reliability, or drift validity claim is published by this artifact.","Threshold values are placeholders for pipeline exercise only and are not production admission thresholds."],"mandatory_criterion_floors":{"actionability":0,"audience_pragmatics":0,"explanation_quality":0,"fact_preservation":0,"intent_preservation":0,"issue_support":0,"replacement_correctness":0,"request_strength_preservation":0,"span_fidelity":0,"technical_precision":0},"minimum_criterion_scores":{"actionability":0.0,"audience_pragmatics":0.0,"explanation_quality":0.0,"fact_preservation":0.0,"intent_preservation":0.0,"issue_support":0.0,"replacement_correctness":0.0,"request_strength_preservation":0.0,"span_fidelity":0.0,"technical_precision":0.0},"policy_id":"email_writing_judge_policy","policy_version":"email-writing-evaluation-only-v1","publish_decision":"withhold","required_criteria_by_candidate_kind":{"no_replacement_diagnostic":["issue_support","span_fidelity","intent_preservation","fact_preservation","request_strength_preservation","audience_pragmatics","technical_precision","actionability","explanation_quality"],"replacement_diagnostic":["issue_support","span_fidelity","replacement_correctness","intent_preservation","fact_preservation","request_strength_preservation","audience_pragmatics","technical_precision","actionability","explanation_quality"]},"rollback_version":null,"status":"evaluation_only"} diff --git a/backend/policies/email_writing_judge_policy.schema.json b/backend/policies/email_writing_judge_policy.schema.json new file mode 100644 index 000000000..8837f97f8 --- /dev/null +++ b/backend/policies/email_writing_judge_policy.schema.json @@ -0,0 +1 @@ +{"$id":"https://contextualwisdomlab.github.io/naruon/schemas/email_writing_judge_policy.schema.json","$schema":"https://json-schema.org/draft/2020-12/schema","additionalProperties":false,"allOf":[{"if":{"properties":{"status":{"const":"published"}},"required":["status"]},"then":{"properties":{"publish_decision":{"const":"publish"}}}},{"if":{"properties":{"status":{"enum":["evaluation_only","superseded","revoked"]}},"required":["status"]},"then":{"properties":{"publish_decision":{"const":"withhold"}}}}],"properties":{"adjudication_conditions":{"items":{"enum":["same_model_without_compatible_calibration","criterion_disagreement","insufficient_evidence"]},"type":"array","uniqueItems":true},"approved_profiles":{"items":{"additionalProperties":false,"properties":{"candidate_model_profile_id":{"type":"string"},"candidate_provider_id":{"type":"string"},"judge_model_profile_id":{"type":"string"},"judge_provider_id":{"type":"string"},"language_tags":{"items":{"maxLength":63,"minLength":1,"type":"string"},"maxItems":32,"minItems":1,"type":"array","uniqueItems":true},"profile_id":{"type":"string"},"review_modes":{"items":{"enum":["incremental","deep"]},"maxItems":2,"minItems":1,"type":"array","uniqueItems":true},"rubric_version":{"type":"string"},"same_model_allowed":{"type":"boolean"}},"required":["profile_id","language_tags","review_modes","candidate_model_profile_id","candidate_provider_id","judge_model_profile_id","judge_provider_id","rubric_version","same_model_allowed"],"type":"object"},"maxItems":64,"minItems":1,"type":"array"},"calibration_summary":{"additionalProperties":false,"properties":{"brier_score":{"maximum":1.0,"minimum":0.0,"type":["number","null"]},"dif_profiles_evaluated":{"minimum":0,"type":"integer"},"status":{"enum":["not_evaluated","validated"]},"temporal_drift_evaluated":{"type":"boolean"},"test_retest_reliability":{"maximum":1.0,"minimum":0.0,"type":["number","null"]}},"required":["status","brier_score","test_retest_reliability","dif_profiles_evaluated","temporal_drift_evaluated"],"type":"object"},"category_anchors":{"items":{"type":"string"},"maxItems":9,"minItems":2,"type":"array","uniqueItems":true},"category_count":{"maximum":9,"minimum":2,"type":"integer"},"compatible_contracts":{"additionalProperties":false,"properties":{"contextual_orchestrator":{"type":"string"},"fast_mlsirm":{"type":["string","null"]},"inkspan":{"type":["string","null"]},"naruon":{"type":"string"}},"required":["naruon","inkspan","fast_mlsirm","contextual_orchestrator"],"type":"object"},"created_at":{"format":"date-time","type":"string"},"evidence":{"additionalProperties":false,"properties":{"calibration_dataset_hash":{"pattern":"^sha256:[0-9a-f]{64}$","type":["string","null"]},"holdout_labels_accessed_after_preregistration":{"type":"boolean"},"locked_holdout_hash":{"pattern":"^sha256:[0-9a-f]{64}$","type":["string","null"]},"protocol_hash":{"pattern":"^sha256:[0-9a-f]{64}$","type":["string","null"]},"reference_adjudication_hash":{"pattern":"^sha256:[0-9a-f]{64}$","type":["string","null"]}},"required":["protocol_hash","calibration_dataset_hash","locked_holdout_hash","reference_adjudication_hash","holdout_labels_accessed_after_preregistration"],"type":"object"},"expires_at":{"format":"date-time","type":"string"},"limitations":{"items":{"maxLength":1000,"minLength":1,"type":"string"},"maxItems":64,"minItems":1,"type":"array"},"mandatory_criterion_floors":{"additionalProperties":false,"properties":{"actionability":{"maximum":8,"minimum":0,"type":"integer"},"audience_pragmatics":{"maximum":8,"minimum":0,"type":"integer"},"explanation_quality":{"maximum":8,"minimum":0,"type":"integer"},"fact_preservation":{"maximum":8,"minimum":0,"type":"integer"},"intent_preservation":{"maximum":8,"minimum":0,"type":"integer"},"issue_support":{"maximum":8,"minimum":0,"type":"integer"},"replacement_correctness":{"maximum":8,"minimum":0,"type":"integer"},"request_strength_preservation":{"maximum":8,"minimum":0,"type":"integer"},"span_fidelity":{"maximum":8,"minimum":0,"type":"integer"},"technical_precision":{"maximum":8,"minimum":0,"type":"integer"}},"required":["issue_support","span_fidelity","replacement_correctness","intent_preservation","fact_preservation","request_strength_preservation","audience_pragmatics","technical_precision","actionability","explanation_quality"],"type":"object"},"minimum_criterion_scores":{"additionalProperties":false,"properties":{"actionability":{"maximum":1.0,"minimum":0.0,"type":"number"},"audience_pragmatics":{"maximum":1.0,"minimum":0.0,"type":"number"},"explanation_quality":{"maximum":1.0,"minimum":0.0,"type":"number"},"fact_preservation":{"maximum":1.0,"minimum":0.0,"type":"number"},"intent_preservation":{"maximum":1.0,"minimum":0.0,"type":"number"},"issue_support":{"maximum":1.0,"minimum":0.0,"type":"number"},"replacement_correctness":{"maximum":1.0,"minimum":0.0,"type":"number"},"request_strength_preservation":{"maximum":1.0,"minimum":0.0,"type":"number"},"span_fidelity":{"maximum":1.0,"minimum":0.0,"type":"number"},"technical_precision":{"maximum":1.0,"minimum":0.0,"type":"number"}},"required":["issue_support","span_fidelity","replacement_correctness","intent_preservation","fact_preservation","request_strength_preservation","audience_pragmatics","technical_precision","actionability","explanation_quality"],"type":"object"},"policy_id":{"pattern":"^[A-Za-z0-9][A-Za-z0-9._:-]{0,127}$","type":"string"},"policy_version":{"pattern":"^[A-Za-z0-9][A-Za-z0-9._:-]{0,127}$","type":"string"},"publish_decision":{"enum":["publish","withhold"]},"required_criteria_by_candidate_kind":{"additionalProperties":false,"properties":{"no_replacement_diagnostic":{"items":{"enum":["issue_support","span_fidelity","replacement_correctness","intent_preservation","fact_preservation","request_strength_preservation","audience_pragmatics","technical_precision","actionability","explanation_quality"]},"minItems":1,"type":"array","uniqueItems":true},"replacement_diagnostic":{"items":{"enum":["issue_support","span_fidelity","replacement_correctness","intent_preservation","fact_preservation","request_strength_preservation","audience_pragmatics","technical_precision","actionability","explanation_quality"]},"minItems":1,"type":"array","uniqueItems":true}},"required":["replacement_diagnostic","no_replacement_diagnostic"],"type":"object"},"rollback_version":{"type":["string","null"]},"status":{"enum":["evaluation_only","published","superseded","revoked"]}},"required":["policy_id","policy_version","status","publish_decision","created_at","expires_at","compatible_contracts","approved_profiles","category_count","category_anchors","required_criteria_by_candidate_kind","mandatory_criterion_floors","minimum_criterion_scores","adjudication_conditions","calibration_summary","evidence","limitations","rollback_version"],"title":"Naruon Email Writing Judge Policy","type":"object"} diff --git a/backend/policies/email_writing_policy_manifest.json b/backend/policies/email_writing_policy_manifest.json new file mode 100644 index 000000000..9dfbaccd3 --- /dev/null +++ b/backend/policies/email_writing_policy_manifest.json @@ -0,0 +1 @@ +{"artifacts":{"email_writing_judge_evaluation_only_v1.json":{"sha256":"d3b6283c998a336b559ff909053a19678554f489b2d950362b1a365a52c38629"}},"manifest_version":1} diff --git a/backend/services/email_writing_policy.py b/backend/services/email_writing_policy.py new file mode 100644 index 000000000..5433dc343 --- /dev/null +++ b/backend/services/email_writing_policy.py @@ -0,0 +1,654 @@ +"""Integrity-bound admission policy for contextual email-writing diagnostics. + +This module validates versioned policy artifacts and decides whether already +structured Judge evidence may be admitted, withheld, or escalated. It does not +infer writing quality from text, fit calibration models, call model providers, +mutate mail, or decide whether an email may be sent. +""" + +from __future__ import annotations + +from collections.abc import Mapping, Sequence +from datetime import datetime, timezone +import hashlib +import re +from typing import Final, Literal, TypeAlias + +from pydantic import ( + BaseModel, + ConfigDict, + Field, + PrivateAttr, + StrictBool, + StrictFloat, + StrictInt, + StrictStr, + ValidationError, + field_validator, + model_validator, +) + +from services.email_writing_contracts import ( + StrictEmailWritingJsonError, + parse_strict_email_writing_json, +) +from services.email_writing_judge import EMAIL_WRITING_JUDGE_CRITERION_IDS + +PolicyStatus: TypeAlias = Literal[ + "evaluation_only", + "published", + "superseded", + "revoked", +] +PublishDecision: TypeAlias = Literal["publish", "withhold"] +AdmissionOutcome: TypeAlias = Literal[ + "admit", + "withhold", + "adjudicate", + "unsupported_profile", + "policy_unavailable", +] +CandidateKind: TypeAlias = Literal[ + "replacement_diagnostic", + "no_replacement_diagnostic", +] +ReviewMode: TypeAlias = Literal["incremental", "deep"] + +_HASH_RE: Final = re.compile(r"^sha256:[0-9a-f]{64}$") +_HEX_DIGEST_RE: Final = re.compile(r"^[0-9a-f]{64}$") +_IDENTIFIER_RE: Final = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._:-]{0,127}$") +_ALLOWED_ADJUDICATION_CONDITIONS: Final = frozenset( + { + "same_model_without_compatible_calibration", + "criterion_disagreement", + "insufficient_evidence", + } +) +_CANDIDATE_KINDS: Final = ( + "replacement_diagnostic", + "no_replacement_diagnostic", +) + + +class EmailWritingPolicyError(ValueError): + """Stable payload-redacted failure raised by policy integrity validation.""" + + def __init__(self, code: str) -> None: + """Create an error that exposes only one stable application code.""" + super().__init__(code) + self.code = code + + def __repr__(self) -> str: + """Return a representation that cannot contain policy payload text.""" + return f"EmailWritingPolicyError({self.code!r})" + + +class _PolicyModel(BaseModel): + """Strict base class for immutable policy and manifest JSON models.""" + + model_config = ConfigDict(extra="forbid", strict=True, frozen=True) + + +class _ManifestArtifact(_PolicyModel): + """Digest identity for one policy artifact listed by the manifest.""" + + sha256: StrictStr + + @field_validator("sha256") + @classmethod + def validate_sha256(cls, value: str) -> str: + """Require one lowercase SHA-256 digest without an algorithm prefix.""" + if _HEX_DIGEST_RE.fullmatch(value) is None: + raise ValueError("manifest_digest_invalid") + return value + + +class _PolicyManifest(_PolicyModel): + """Exact artifact allowlist used before any policy JSON is trusted.""" + + manifest_version: Literal[1] + artifacts: dict[str, _ManifestArtifact] + + @field_validator("artifacts") + @classmethod + def validate_artifacts( + cls, + value: dict[str, _ManifestArtifact], + ) -> dict[str, _ManifestArtifact]: + """Require at least one safely named immutable JSON artifact.""" + if not value: + raise ValueError("manifest_artifacts_empty") + for name in value: + if ( + not name.endswith(".json") + or "/" in name + or "\\" in name + or name.startswith(".") + or ".." in name + ): + raise ValueError("manifest_artifact_name_invalid") + return value + + +class EmailWritingCompatibleContracts(_PolicyModel): + """Exact runtime contract versions against which a policy was evaluated.""" + + naruon: StrictStr + inkspan: StrictStr | None + fast_mlsirm: StrictStr | None + contextual_orchestrator: StrictStr + + @field_validator("naruon", "inkspan", "fast_mlsirm", "contextual_orchestrator") + @classmethod + def validate_contract_id(cls, value: str | None) -> str | None: + """Keep non-null dependency identities bounded and non-executable.""" + if value is None: + return None + if _IDENTIFIER_RE.fullmatch(value) is None: + raise ValueError("contract_identity_invalid") + return value + + +class EmailWritingApprovedProfile(_PolicyModel): + """One explicitly evaluated language/model/provider/rubric combination.""" + + profile_id: StrictStr + language_tags: list[StrictStr] + review_modes: list[ReviewMode] + candidate_model_profile_id: StrictStr + candidate_provider_id: StrictStr + judge_model_profile_id: StrictStr + judge_provider_id: StrictStr + rubric_version: StrictStr + same_model_allowed: StrictBool + + @field_validator( + "profile_id", + "candidate_model_profile_id", + "candidate_provider_id", + "judge_model_profile_id", + "judge_provider_id", + "rubric_version", + ) + @classmethod + def validate_identifier(cls, value: str) -> str: + """Reject profile identifiers that could smuggle commands or paths.""" + if _IDENTIFIER_RE.fullmatch(value) is None: + raise ValueError("profile_identity_invalid") + return value + + @field_validator("language_tags") + @classmethod + def validate_language_tags(cls, value: list[str]) -> list[str]: + """Require an explicit non-empty language claim without wildcard profiles.""" + if not value or len(set(value)) != len(value): + raise ValueError("language_profiles_invalid") + for language_tag in value: + if ( + not language_tag + or language_tag == "*" + or len(language_tag) > 63 + or any(character.isspace() for character in language_tag) + ): + raise ValueError("language_profile_invalid") + return value + + @field_validator("review_modes") + @classmethod + def validate_review_modes(cls, value: list[ReviewMode]) -> list[ReviewMode]: + """Require at least one unique supported review mode.""" + if not value or len(set(value)) != len(value): + raise ValueError("review_modes_invalid") + return value + + +class EmailWritingCalibrationSummary(_PolicyModel): + """Privacy-minimized measurement summary required before publication.""" + + status: Literal["not_evaluated", "validated"] + brier_score: StrictFloat | None + test_retest_reliability: StrictFloat | None + dif_profiles_evaluated: StrictInt + temporal_drift_evaluated: StrictBool + + @field_validator("brier_score", "test_retest_reliability") + @classmethod + def validate_unit_metric(cls, value: float | None) -> float | None: + """Require finite unit-interval metrics when measurement evidence exists.""" + if value is None: + return None + if value != value or value in {float("inf"), float("-inf")}: + raise ValueError("calibration_metric_non_finite") + if not 0.0 <= value <= 1.0: + raise ValueError("calibration_metric_out_of_range") + return value + + @field_validator("dif_profiles_evaluated") + @classmethod + def validate_dif_count(cls, value: int) -> int: + """Require a non-negative count of explicitly evaluated DIF profiles.""" + if value < 0: + raise ValueError("dif_profile_count_invalid") + return value + + +class EmailWritingPolicyEvidence(_PolicyModel): + """Content hashes proving preregistration and evaluated reference material.""" + + protocol_hash: StrictStr | None + calibration_dataset_hash: StrictStr | None + locked_holdout_hash: StrictStr | None + reference_adjudication_hash: StrictStr | None + holdout_labels_accessed_after_preregistration: StrictBool + + @field_validator( + "protocol_hash", + "calibration_dataset_hash", + "locked_holdout_hash", + "reference_adjudication_hash", + ) + @classmethod + def validate_hash(cls, value: str | None) -> str | None: + """Require algorithm-qualified SHA-256 identities when evidence exists.""" + if value is not None and _HASH_RE.fullmatch(value) is None: + raise ValueError("evidence_hash_invalid") + return value + + +class EmailWritingJudgePolicy(_PolicyModel): + """Validated policy artifact that contains no authored or model plaintext.""" + + policy_id: StrictStr + policy_version: StrictStr + status: PolicyStatus + publish_decision: PublishDecision + created_at: StrictStr + expires_at: StrictStr + compatible_contracts: EmailWritingCompatibleContracts + approved_profiles: list[EmailWritingApprovedProfile] + category_count: StrictInt + category_anchors: list[StrictStr] + required_criteria_by_candidate_kind: dict[str, list[StrictStr]] + mandatory_criterion_floors: dict[str, StrictInt] + minimum_criterion_scores: dict[str, StrictFloat] + adjudication_conditions: list[StrictStr] + calibration_summary: EmailWritingCalibrationSummary + evidence: EmailWritingPolicyEvidence + limitations: list[StrictStr] + rollback_version: StrictStr | None + + _artifact_name: str = PrivateAttr(default="") + _artifact_sha256: str = PrivateAttr(default="") + + @field_validator("policy_id", "policy_version", "rollback_version") + @classmethod + def validate_policy_identifier(cls, value: str | None) -> str | None: + """Require bounded opaque policy identities rather than executable text.""" + if value is None: + return None + if _IDENTIFIER_RE.fullmatch(value) is None: + raise ValueError("policy_identity_invalid") + return value + + @field_validator("created_at", "expires_at") + @classmethod + def validate_timestamp(cls, value: str) -> str: + """Require an offset-aware RFC 3339-compatible timestamp.""" + _parse_timestamp(value) + return value + + @field_validator("approved_profiles") + @classmethod + def validate_profiles( + cls, + value: list[EmailWritingApprovedProfile], + ) -> list[EmailWritingApprovedProfile]: + """Require unique profile identities and at least one bounded profile.""" + if not value: + raise ValueError("approved_profiles_empty") + profile_ids = [profile.profile_id for profile in value] + if len(profile_ids) != len(set(profile_ids)): + raise ValueError("approved_profile_duplicate") + return value + + @field_validator("category_count") + @classmethod + def validate_category_count(cls, value: int) -> int: + """Bound ordered polytomous categories without selecting a production value.""" + if value < 2 or value > 9: + raise ValueError("category_count_invalid") + return value + + @field_validator("category_anchors") + @classmethod + def validate_anchors(cls, value: list[str]) -> list[str]: + """Require unique stable anchor identities without interpreting prose.""" + if not value or len(value) != len(set(value)): + raise ValueError("category_anchors_invalid") + for anchor in value: + if _IDENTIFIER_RE.fullmatch(anchor) is None: + raise ValueError("category_anchor_invalid") + return value + + @field_validator("adjudication_conditions") + @classmethod + def validate_adjudication_conditions(cls, value: list[str]) -> list[str]: + """Restrict escalation triggers to structured workflow states.""" + if len(value) != len(set(value)): + raise ValueError("adjudication_condition_duplicate") + if not set(value).issubset(_ALLOWED_ADJUDICATION_CONDITIONS): + raise ValueError("adjudication_condition_invalid") + return value + + @field_validator("limitations") + @classmethod + def validate_limitations(cls, value: list[str]) -> list[str]: + """Keep limitations explicit, bounded, and non-empty for auditability.""" + if not value: + raise ValueError("limitations_empty") + for limitation in value: + if not limitation or len(limitation) > 1_000: + raise ValueError("limitation_invalid") + return value + + @model_validator(mode="after") + def validate_policy_contract(self) -> "EmailWritingJudgePolicy": + """Enforce lifecycle, criterion, threshold, and evidence invariants.""" + if len(self.category_anchors) != self.category_count: + raise ValueError("category_anchor_count_mismatch") + + required_keys = set(_CANDIDATE_KINDS) + if set(self.required_criteria_by_candidate_kind) != required_keys: + raise ValueError("candidate_kind_contract_invalid") + canonical = tuple(EMAIL_WRITING_JUDGE_CRITERION_IDS) + canonical_set = set(canonical) + for candidate_kind, criterion_ids in self.required_criteria_by_candidate_kind.items(): + if not criterion_ids or len(criterion_ids) != len(set(criterion_ids)): + raise ValueError("required_criteria_invalid") + if not set(criterion_ids).issubset(canonical_set): + raise ValueError("required_criteria_invalid") + if tuple(criterion for criterion in canonical if criterion in criterion_ids) != tuple( + criterion_ids + ): + raise ValueError("required_criteria_order_invalid") + if candidate_kind == "replacement_diagnostic": + if "replacement_correctness" not in criterion_ids: + raise ValueError("replacement_criterion_required") + elif "replacement_correctness" in criterion_ids: + raise ValueError("replacement_criterion_forbidden") + + expected_floor_ids = set().union( + *(set(values) for values in self.required_criteria_by_candidate_kind.values()) + ) + if set(self.mandatory_criterion_floors) != expected_floor_ids: + raise ValueError("criterion_floor_identity_invalid") + if set(self.minimum_criterion_scores) != expected_floor_ids: + raise ValueError("criterion_score_floor_identity_invalid") + for floor in self.mandatory_criterion_floors.values(): + if floor < 0 or floor >= self.category_count: + raise ValueError("policy_floor_invalid") + for score in self.minimum_criterion_scores.values(): + if score != score or score in {float("inf"), float("-inf")}: + raise ValueError("policy_score_floor_non_finite") + if not 0.0 <= score <= 1.0: + raise ValueError("policy_score_floor_invalid") + + if self.status == "published": + if self.publish_decision != "publish": + raise ValueError("published_policy_must_publish") + if self.calibration_summary.status != "validated": + raise ValueError("published_calibration_missing") + if any( + value is None + for value in ( + self.compatible_contracts.inkspan, + self.compatible_contracts.fast_mlsirm, + self.evidence.protocol_hash, + self.evidence.calibration_dataset_hash, + self.evidence.locked_holdout_hash, + self.evidence.reference_adjudication_hash, + ) + ): + raise ValueError("published_evidence_missing") + if not self.evidence.holdout_labels_accessed_after_preregistration: + raise ValueError("published_holdout_order_invalid") + if self.calibration_summary.brier_score is None: + raise ValueError("published_brier_missing") + if self.calibration_summary.test_retest_reliability is None: + raise ValueError("published_reliability_missing") + if self.calibration_summary.dif_profiles_evaluated <= 0: + raise ValueError("published_dif_missing") + if not self.calibration_summary.temporal_drift_evaluated: + raise ValueError("published_drift_missing") + else: + if self.publish_decision != "withhold": + raise ValueError("nonpublished_policy_must_withhold") + if self.status == "evaluation_only": + if self.evidence.holdout_labels_accessed_after_preregistration: + raise ValueError("evaluation_holdout_access_forbidden") + + return self + + +def _parse_timestamp(value: str) -> datetime: + """Parse one offset-aware policy timestamp and normalize it to UTC.""" + normalized = value[:-1] + "+00:00" if value.endswith("Z") else value + try: + parsed = datetime.fromisoformat(normalized) + except ValueError as error: + raise ValueError("policy_timestamp_invalid") from error + if parsed.tzinfo is None: + raise ValueError("policy_timestamp_timezone_required") + return parsed.astimezone(timezone.utc) + + +def _wrap_policy_validation_error( + error: Exception, + *, + default_code: str, +) -> EmailWritingPolicyError: + """Map internal parser details to one stable redacted policy error code.""" + text = str(error) + if "policy_floor_invalid" in text: + return EmailWritingPolicyError("policy_floor_invalid") + if any( + marker in text + for marker in ( + "nonpublished_policy_must_withhold", + "published_policy_must_publish", + "evaluation_holdout_access_forbidden", + ) + ): + return EmailWritingPolicyError("policy_lifecycle_invalid") + if any( + marker in text + for marker in ( + "published_evidence_missing", + "published_calibration_missing", + "published_holdout_order_invalid", + "published_brier_missing", + "published_reliability_missing", + "published_dif_missing", + "published_drift_missing", + ) + ): + return EmailWritingPolicyError("policy_publication_evidence_incomplete") + return EmailWritingPolicyError(default_code) + + +def _validate_runtime_contracts( + policy: EmailWritingJudgePolicy, + runtime_contracts: Mapping[str, str | None], +) -> None: + """Require every non-null policy contract to match the live runtime exactly.""" + expected = { + "naruon": policy.compatible_contracts.naruon, + "inkspan": policy.compatible_contracts.inkspan, + "fast_mlsirm": policy.compatible_contracts.fast_mlsirm, + "contextual_orchestrator": policy.compatible_contracts.contextual_orchestrator, + } + if set(runtime_contracts) != set(expected): + raise EmailWritingPolicyError("policy_contract_incompatible") + for name, expected_version in expected.items(): + if expected_version is None: + continue + if runtime_contracts[name] != expected_version: + raise EmailWritingPolicyError("policy_contract_incompatible") + + +def load_policy_artifact( + *, + artifact_name: str, + artifact_bytes: bytes, + manifest_bytes: bytes, + now: datetime, + runtime_contracts: Mapping[str, str | None], +) -> EmailWritingJudgePolicy: + """Load one integrity-bound policy after strict manifest and lifecycle validation. + + The function validates bytes before parsing policy content, rejects duplicate + or unexpected JSON members, checks temporal validity and exact runtime + contracts, and returns no user-facing admission merely because an artifact + loaded successfully. + """ + try: + manifest = parse_strict_email_writing_json(manifest_bytes, _PolicyManifest) + except (StrictEmailWritingJsonError, ValidationError, ValueError) as error: + raise EmailWritingPolicyError("policy_manifest_invalid") from error + + entry = manifest.artifacts.get(artifact_name) + if entry is None: + raise EmailWritingPolicyError("policy_manifest_unknown_artifact") + actual_digest = hashlib.sha256(artifact_bytes).hexdigest() + if actual_digest != entry.sha256: + raise EmailWritingPolicyError("policy_integrity_mismatch") + + try: + policy = parse_strict_email_writing_json(artifact_bytes, EmailWritingJudgePolicy) + except (StrictEmailWritingJsonError, ValidationError, ValueError) as error: + raise _wrap_policy_validation_error( + error, + default_code="policy_schema_invalid", + ) from error + + if now.tzinfo is None: + raise EmailWritingPolicyError("policy_clock_invalid") + now_utc = now.astimezone(timezone.utc) + created_at = _parse_timestamp(policy.created_at) + expires_at = _parse_timestamp(policy.expires_at) + if expires_at <= created_at: + raise EmailWritingPolicyError("policy_time_window_invalid") + if now_utc < created_at: + raise EmailWritingPolicyError("policy_future_dated") + if now_utc >= expires_at: + raise EmailWritingPolicyError("policy_expired") + + _validate_runtime_contracts(policy, runtime_contracts) + object.__setattr__(policy, "_artifact_name", artifact_name) + object.__setattr__(policy, "_artifact_sha256", actual_digest) + return policy + + +def evaluate_policy_admission( + *, + policy: EmailWritingJudgePolicy, + language_tag: str, + review_mode: ReviewMode, + candidate_model_profile_id: str, + candidate_provider_id: str, + judge_model_profile_id: str, + judge_provider_id: str, + rubric_version: str, + criterion_categories: Mapping[str, int], + criterion_scores: Mapping[str, float], + candidate_kind: CandidateKind = "replacement_diagnostic", +) -> AdmissionOutcome: + """Apply structured published thresholds without inspecting authored text. + + Evaluation-only, revoked, superseded, or withheld policies can never admit a + diagnostic. Published policies require an exact evaluated profile and exact + criterion identity for the candidate kind. Mandatory preservation floors are + conjunctive: a high average cannot compensate for a failed criterion. + """ + if policy.status != "published" or policy.publish_decision != "publish": + return "withhold" + + matching_profile = next( + ( + profile + for profile in policy.approved_profiles + if language_tag in profile.language_tags + and review_mode in profile.review_modes + and candidate_model_profile_id == profile.candidate_model_profile_id + and candidate_provider_id == profile.candidate_provider_id + and judge_model_profile_id == profile.judge_model_profile_id + and judge_provider_id == profile.judge_provider_id + and rubric_version == profile.rubric_version + ), + None, + ) + if matching_profile is None: + return "unsupported_profile" + + if ( + candidate_model_profile_id == judge_model_profile_id + and not matching_profile.same_model_allowed + ): + return "adjudicate" + + required_criterion_ids = tuple( + policy.required_criteria_by_candidate_kind[candidate_kind] + ) + required_set = set(required_criterion_ids) + if ( + set(criterion_categories) != required_set + or set(criterion_scores) != required_set + ): + raise EmailWritingPolicyError("criterion_identity_mismatch") + + for criterion_id in required_criterion_ids: + category = criterion_categories[criterion_id] + score = criterion_scores[criterion_id] + if type(category) is not int: + raise EmailWritingPolicyError("criterion_category_invalid") + if category < 0 or category >= policy.category_count: + raise EmailWritingPolicyError("criterion_category_invalid") + if type(score) not in {int, float} or type(score) is bool: + raise EmailWritingPolicyError("criterion_score_invalid") + numeric_score = float(score) + if ( + numeric_score != numeric_score + or numeric_score in {float("inf"), float("-inf")} + or not 0.0 <= numeric_score <= 1.0 + ): + raise EmailWritingPolicyError("criterion_score_invalid") + if category < policy.mandatory_criterion_floors[criterion_id]: + return "withhold" + if numeric_score < policy.minimum_criterion_scores[criterion_id]: + return "withhold" + + return "admit" + + +def select_rollback_policy( + *, + current_policy: EmailWritingJudgePolicy, + candidates: Sequence[EmailWritingJudgePolicy], +) -> EmailWritingJudgePolicy: + """Select only the explicitly named compatible published rollback policy.""" + rollback_version = current_policy.rollback_version + if rollback_version is None: + raise EmailWritingPolicyError("policy_rollback_unavailable") + matches = [ + candidate + for candidate in candidates + if candidate.policy_version == rollback_version + and candidate.status == "published" + and candidate.publish_decision == "publish" + and candidate.compatible_contracts == current_policy.compatible_contracts + and bool(candidate._artifact_name) + and bool(candidate._artifact_sha256) + ] + if len(matches) != 1: + raise EmailWritingPolicyError("policy_rollback_unavailable") + return matches[0] From 16f640b26e77ee422bd4ad8746661c0f5e1e6df0 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Wed, 2 Sep 2026 11:56:40 +0900 Subject: [PATCH 3/8] ci(email-writing): verify Task 8 policy boundary --- .../workflows/email-writing-policy-tdd.yml | 121 ++++++++++++++++++ 1 file changed, 121 insertions(+) create mode 100644 .github/workflows/email-writing-policy-tdd.yml diff --git a/.github/workflows/email-writing-policy-tdd.yml b/.github/workflows/email-writing-policy-tdd.yml new file mode 100644 index 000000000..6d45344b0 --- /dev/null +++ b/.github/workflows/email-writing-policy-tdd.yml @@ -0,0 +1,121 @@ +name: Email Writing Policy TDD + +on: + push: + branches: + - feat/email-writing-policy-task8-current-stack + pull_request: + paths: + - backend/policies/email_writing_judge_policy.schema.json + - backend/policies/email_writing_judge_evaluation_only_v1.json + - backend/policies/email_writing_policy_manifest.json + - backend/services/email_writing_policy.py + - backend/tests/test_email_writing_policy.py + - scripts/ci/reject_terminal_output.sh + - .github/workflows/email-writing-policy-tdd.yml + workflow_dispatch: + +permissions: + contents: read + +concurrency: + group: email-writing-policy-tdd-${{ github.event.pull_request.head.sha || github.sha }} + cancel-in-progress: true + +env: + FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: true + PYTHONWARNINGS: error + +jobs: + task8-admission-policy: + runs-on: ubuntu-24.04 + timeout-minutes: 20 + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + ref: ${{ github.event.pull_request.head.sha || github.sha }} + persist-credentials: false + - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 + with: + python-version: "3.14" + cache: pip + cache-dependency-path: backend/requirements-hashes.txt + - name: Install hash-locked application dependencies + run: >- + bash scripts/ci/reject_terminal_output.sh + python -m pip install --disable-pip-version-check --require-hashes + -r backend/requirements-hashes.txt + - name: Install hash-verified coverage tool + run: | + set -euo pipefail + mkdir -p /tmp/coverage-wheel + cat >/tmp/coverage-lock.txt <<'EOF' + coverage==7.15.2 --hash=sha256:eb6bcae8d1a9d305351ecb108232441d11c5cfe9de840a04388ba5d2db8d735c + EOF + bash scripts/ci/reject_terminal_output.sh \ + python -m pip download \ + --disable-pip-version-check \ + --require-hashes \ + --no-deps \ + --only-binary=:all: \ + --platform any \ + --python-version 3.14 \ + --implementation py \ + --abi none \ + --dest /tmp/coverage-wheel \ + -r /tmp/coverage-lock.txt + bash scripts/ci/reject_terminal_output.sh \ + python -m pip install --disable-pip-version-check --no-deps \ + /tmp/coverage-wheel/coverage-7.15.2-py3-none-any.whl + - name: Run Task 8 policy tests + run: | + cd backend + bash ../scripts/ci/reject_terminal_output.sh \ + python -m pytest -q --noconftest tests/test_email_writing_policy.py + - name: Verify Task 8 statement and branch coverage + run: | + cd backend + bash ../scripts/ci/reject_terminal_output.sh python -m coverage erase + bash ../scripts/ci/reject_terminal_output.sh \ + python -m coverage run --branch \ + --include='services/email_writing_policy.py' \ + -m pytest -q --noconftest tests/test_email_writing_policy.py + bash ../scripts/ci/reject_terminal_output.sh \ + python -m coverage report --show-missing --fail-under=100 \ + services/email_writing_policy.py + - name: Verify shipped Python docstrings + run: | + set -euo pipefail + cat >/tmp/check_task8_docstrings.py <<'PY' + import ast + from pathlib import Path + + path = Path("services/email_writing_policy.py") + tree = ast.parse(path.read_text(encoding="utf-8"), filename=str(path)) + missing: list[str] = [] + if ast.get_docstring(tree) is None: + missing.append(f"{path}:") + for node in ast.walk(tree): + if isinstance(node, (ast.ClassDef, ast.FunctionDef, ast.AsyncFunctionDef)): + if ast.get_docstring(node) is None: + missing.append(f"{path}:{node.lineno}:{node.name}") + if missing: + raise SystemExit("Missing shipped docstrings:\n" + "\n".join(missing)) + print("Docstring gate passed for Task 8 policy module") + PY + cd backend + bash ../scripts/ci/reject_terminal_output.sh \ + python /tmp/check_task8_docstrings.py + - name: Lint Task 8 source and tests + run: | + cd backend + bash ../scripts/ci/reject_terminal_output.sh \ + python -m ruff check \ + services/email_writing_policy.py \ + tests/test_email_writing_policy.py + - name: Compile Task 8 source and tests + run: | + bash scripts/ci/reject_terminal_output.sh \ + python -m compileall -q \ + backend/services/email_writing_policy.py \ + backend/tests/test_email_writing_policy.py From e155375700879938ad010b46d83ac7edfbd8c504 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Wed, 2 Sep 2026 13:51:45 +0900 Subject: [PATCH 4/8] test(email-writing): reject unknown Task 8 candidate kind --- .../workflows/email-writing-policy-tdd.yml | 15 +++++-- ...est_email_writing_policy_candidate_kind.py | 40 +++++++++++++++++++ 2 files changed, 51 insertions(+), 4 deletions(-) create mode 100644 backend/tests/test_email_writing_policy_candidate_kind.py diff --git a/.github/workflows/email-writing-policy-tdd.yml b/.github/workflows/email-writing-policy-tdd.yml index 6d45344b0..0b7ebbbdf 100644 --- a/.github/workflows/email-writing-policy-tdd.yml +++ b/.github/workflows/email-writing-policy-tdd.yml @@ -11,6 +11,7 @@ on: - backend/policies/email_writing_policy_manifest.json - backend/services/email_writing_policy.py - backend/tests/test_email_writing_policy.py + - backend/tests/test_email_writing_policy_candidate_kind.py - scripts/ci/reject_terminal_output.sh - .github/workflows/email-writing-policy-tdd.yml workflow_dispatch: @@ -71,7 +72,9 @@ jobs: run: | cd backend bash ../scripts/ci/reject_terminal_output.sh \ - python -m pytest -q --noconftest tests/test_email_writing_policy.py + python -m pytest -q --noconftest \ + tests/test_email_writing_policy.py \ + tests/test_email_writing_policy_candidate_kind.py - name: Verify Task 8 statement and branch coverage run: | cd backend @@ -79,7 +82,9 @@ jobs: bash ../scripts/ci/reject_terminal_output.sh \ python -m coverage run --branch \ --include='services/email_writing_policy.py' \ - -m pytest -q --noconftest tests/test_email_writing_policy.py + -m pytest -q --noconftest \ + tests/test_email_writing_policy.py \ + tests/test_email_writing_policy_candidate_kind.py bash ../scripts/ci/reject_terminal_output.sh \ python -m coverage report --show-missing --fail-under=100 \ services/email_writing_policy.py @@ -112,10 +117,12 @@ jobs: bash ../scripts/ci/reject_terminal_output.sh \ python -m ruff check \ services/email_writing_policy.py \ - tests/test_email_writing_policy.py + tests/test_email_writing_policy.py \ + tests/test_email_writing_policy_candidate_kind.py - name: Compile Task 8 source and tests run: | bash scripts/ci/reject_terminal_output.sh \ python -m compileall -q \ backend/services/email_writing_policy.py \ - backend/tests/test_email_writing_policy.py + backend/tests/test_email_writing_policy.py \ + backend/tests/test_email_writing_policy_candidate_kind.py diff --git a/backend/tests/test_email_writing_policy_candidate_kind.py b/backend/tests/test_email_writing_policy_candidate_kind.py new file mode 100644 index 000000000..e55bb5e8d --- /dev/null +++ b/backend/tests/test_email_writing_policy_candidate_kind.py @@ -0,0 +1,40 @@ +"""Regression for fail-closed Task 8 candidate-kind admission.""" + +import pytest + +from services.email_writing_policy import ( + EmailWritingPolicyError, + evaluate_policy_admission, +) +from tests.test_email_writing_policy import ( + _REQUIRED_CRITERIA, + _load, + _published_policy, +) + +_RUNTIME_CONTRACTS = { + "naruon": "0.14.4", + "inkspan": "0.6.0", + "fast_mlsirm": "0.9.2", + "contextual_orchestrator": "v1", +} + + +def test_unknown_candidate_kind_fails_with_stable_policy_error() -> None: + """Untrusted candidate-kind input must not escape as a raw mapping KeyError.""" + policy = _load(_published_policy(), runtime_contracts=_RUNTIME_CONTRACTS) + + with pytest.raises(EmailWritingPolicyError, match="candidate_kind_invalid"): + evaluate_policy_admission( + policy=policy, + language_tag="en", + review_mode="deep", + candidate_model_profile_id="candidate-reviewer-v1", + candidate_provider_id="contextual-orchestrator", + judge_model_profile_id="independent-judge-v1", + judge_provider_id="contextual-orchestrator", + rubric_version="email_writing_judge_rubric_v1", + criterion_categories={criterion: 3 for criterion in _REQUIRED_CRITERIA}, + criterion_scores={criterion: 1.0 for criterion in _REQUIRED_CRITERIA}, + candidate_kind="unknown_candidate_kind", + ) From 21fa293d967bac0dd2b7b9a917b9fbce415134a2 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Wed, 2 Sep 2026 13:53:25 +0900 Subject: [PATCH 5/8] fix(email-writing): fail closed on unknown candidate kind --- backend/services/email_writing_policy.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/backend/services/email_writing_policy.py b/backend/services/email_writing_policy.py index 5433dc343..919286329 100644 --- a/backend/services/email_writing_policy.py +++ b/backend/services/email_writing_policy.py @@ -596,6 +596,9 @@ def evaluate_policy_admission( ): return "adjudicate" + if candidate_kind not in _CANDIDATE_KINDS: + raise EmailWritingPolicyError("candidate_kind_invalid") + required_criterion_ids = tuple( policy.required_criteria_by_candidate_kind[candidate_kind] ) From fbf32aaf67e53fcb00e640e48df62c22b2eb4f41 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Fri, 4 Sep 2026 22:25:46 +0900 Subject: [PATCH 6/8] test(email-writing): cover policy admission boundary Exercise every policy validator, lifecycle, timing, runtime-contract, admission, and rollback branch while preserving the 100 percent workflow threshold. Assisted-by: OpenAI Codex Signed-off-by: Seongho Bae --- backend/tests/test_email_writing_policy.py | 333 ++++++++++++++++++++- 1 file changed, 331 insertions(+), 2 deletions(-) diff --git a/backend/tests/test_email_writing_policy.py b/backend/tests/test_email_writing_policy.py index acfee48ae..523835ebd 100644 --- a/backend/tests/test_email_writing_policy.py +++ b/backend/tests/test_email_writing_policy.py @@ -11,6 +11,8 @@ import pytest from services.email_writing_policy import ( + EmailWritingCalibrationSummary, + EmailWritingJudgePolicy, EmailWritingPolicyError, evaluate_policy_admission, load_policy_artifact, @@ -214,7 +216,9 @@ def test_published_policy_requires_complete_calibration_and_holdout_evidence() - """Publication is invalid before preregistered holdout and adjudication evidence exists.""" payload = _published_policy() payload["evidence"]["locked_holdout_hash"] = None - with pytest.raises(EmailWritingPolicyError, match="policy_publication_evidence_incomplete"): + with pytest.raises( + EmailWritingPolicyError, match="policy_publication_evidence_incomplete" + ): _load( payload, runtime_contracts={ @@ -399,9 +403,334 @@ def test_revoked_policy_rolls_back_only_to_explicit_compatible_manifest_entry() "contextual_orchestrator": "v1", }, ) - assert select_rollback_policy(current_policy=current, candidates=[previous]) is previous + assert ( + select_rollback_policy(current_policy=current, candidates=[previous]) + is previous + ) unlisted = deepcopy(previous) object.__setattr__(unlisted, "policy_version", "email-writing-policy-v0") with pytest.raises(EmailWritingPolicyError, match="policy_rollback_unavailable"): select_rollback_policy(current_policy=current, candidates=[unlisted]) + + +def _assert_policy_rejected(payload: dict[str, object], code: str) -> None: + """Require one mutated policy to fail with a stable public error code.""" + with pytest.raises(EmailWritingPolicyError, match=code): + _load( + payload, + runtime_contracts={ + "naruon": "0.14.4", + "inkspan": "0.6.0", + "fast_mlsirm": "0.9.2", + "contextual_orchestrator": "v1", + }, + ) + + +def test_policy_error_repr_and_manifest_boundaries() -> None: + """Cover redacted error display and manifest allowlist edge cases.""" + assert repr(EmailWritingPolicyError("policy_invalid")) == ( + "EmailWritingPolicyError('policy_invalid')" + ) + payload = _evaluation_policy() + for artifacts in ({}, {"../policy.json": {"sha256": "0" * 64}}): + manifest = {"manifest_version": 1, "artifacts": artifacts} + with pytest.raises(EmailWritingPolicyError, match="policy_manifest_invalid"): + load_policy_artifact( + artifact_name=_POLICY_NAME, + artifact_bytes=_canonical_bytes(payload), + manifest_bytes=_canonical_bytes(manifest), + now=_NOW, + runtime_contracts=_RUNTIME_CONTRACTS, + ) + + with pytest.raises( + EmailWritingPolicyError, match="policy_manifest_unknown_artifact" + ): + load_policy_artifact( + artifact_name="unknown.json", + artifact_bytes=_canonical_bytes(payload), + manifest_bytes=_canonical_bytes(_manifest_for(_POLICY_NAME, payload)), + now=_NOW, + runtime_contracts=_RUNTIME_CONTRACTS, + ) + + +def test_policy_field_validators_reject_invalid_values() -> None: + """Exercise every bounded identity, profile, metric, and list validator.""" + mutations = [] + + payload = _evaluation_policy() + payload["compatible_contracts"]["naruon"] = "bad path/value" + mutations.append(payload) + + for field in ( + "profile_id", + "candidate_model_profile_id", + "candidate_provider_id", + "judge_model_profile_id", + "judge_provider_id", + "rubric_version", + ): + payload = _evaluation_policy() + payload["approved_profiles"][0][field] = "bad path/value" + mutations.append(payload) + + for language_tags in ([], ["en", "en"], ["*"]): + payload = _evaluation_policy() + payload["approved_profiles"][0]["language_tags"] = language_tags + mutations.append(payload) + + for review_modes in ([], ["deep", "deep"]): + payload = _evaluation_policy() + payload["approved_profiles"][0]["review_modes"] = review_modes + mutations.append(payload) + + for field, value in ( + ("brier_score", float("nan")), + ("brier_score", 1.1), + ("test_retest_reliability", float("inf")), + ("dif_profiles_evaluated", -1), + ): + payload = _evaluation_policy() + payload["calibration_summary"][field] = value + mutations.append(payload) + + payload = _evaluation_policy() + payload["evidence"]["protocol_hash"] = "not-a-qualified-hash" + mutations.append(payload) + + payload = _evaluation_policy() + payload["policy_id"] = "bad path/value" + mutations.append(payload) + + for payload in mutations: + _assert_policy_rejected(payload, "policy_schema_invalid") + + +def test_policy_collection_validators_reject_invalid_values() -> None: + """Reject empty, duplicate, unknown, unordered, and inconsistent policy maps.""" + mutations = [] + for profiles in ([], [_evaluation_policy()["approved_profiles"][0]] * 2): + payload = _evaluation_policy() + payload["approved_profiles"] = deepcopy(profiles) + mutations.append(payload) + + for category_count in (1, 10): + payload = _evaluation_policy() + payload["category_count"] = category_count + mutations.append(payload) + + for anchors in ([], ["same", "same"], ["bad anchor", "b", "c", "d"]): + payload = _evaluation_policy() + payload["category_anchors"] = anchors + mutations.append(payload) + + for conditions in ( + ["criterion_disagreement", "criterion_disagreement"], + ["unknown_condition"], + ): + payload = _evaluation_policy() + payload["adjudication_conditions"] = conditions + mutations.append(payload) + + for limitations in ([], [""]): + payload = _evaluation_policy() + payload["limitations"] = limitations + mutations.append(payload) + + payload = _evaluation_policy() + payload["category_anchors"] = payload["category_anchors"][:-1] + mutations.append(payload) + + payload = _evaluation_policy() + payload["required_criteria_by_candidate_kind"] = { + "replacement_diagnostic": list(_REQUIRED_CRITERIA) + } + mutations.append(payload) + + for criterion_ids in ( + [], + ["invented_criterion"], + list(reversed(_REQUIRED_CRITERIA)), + [ + criterion + for criterion in _REQUIRED_CRITERIA + if criterion != "replacement_correctness" + ], + ): + payload = _evaluation_policy() + payload["required_criteria_by_candidate_kind"]["replacement_diagnostic"] = ( + criterion_ids + ) + mutations.append(payload) + + payload = _evaluation_policy() + payload["required_criteria_by_candidate_kind"]["no_replacement_diagnostic"].insert( + 2, "replacement_correctness" + ) + mutations.append(payload) + + payload = _evaluation_policy() + payload["mandatory_criterion_floors"].pop("fact_preservation") + mutations.append(payload) + + payload = _evaluation_policy() + payload["minimum_criterion_scores"].pop("fact_preservation") + mutations.append(payload) + + for score in (float("nan"), 1.1): + payload = _evaluation_policy() + payload["minimum_criterion_scores"]["fact_preservation"] = score + mutations.append(payload) + + for payload in mutations: + _assert_policy_rejected(payload, "policy_schema_invalid") + + with pytest.raises(ValueError, match="calibration_metric_non_finite"): + EmailWritingCalibrationSummary( + status="validated", + brier_score=float("nan"), + test_retest_reliability=0.9, + dif_profiles_evaluated=1, + temporal_drift_evaluated=True, + ) + + payload = _evaluation_policy() + payload["minimum_criterion_scores"]["fact_preservation"] = float("nan") + with pytest.raises(ValueError, match="policy_score_floor_non_finite"): + EmailWritingJudgePolicy.model_validate(payload) + + +def test_policy_lifecycle_evidence_requirements() -> None: + """Reject every incomplete publication and evaluation-only holdout state.""" + lifecycle_cases = [] + + payload = _published_policy() + payload["publish_decision"] = "withhold" + lifecycle_cases.append((payload, "policy_lifecycle_invalid")) + + payload = _evaluation_policy() + payload["evidence"]["holdout_labels_accessed_after_preregistration"] = True + lifecycle_cases.append((payload, "policy_lifecycle_invalid")) + + payload = _published_policy() + payload["calibration_summary"]["status"] = "not_evaluated" + lifecycle_cases.append((payload, "policy_publication_evidence_incomplete")) + + for section, field, value in ( + ("evidence", "holdout_labels_accessed_after_preregistration", False), + ("calibration_summary", "brier_score", None), + ("calibration_summary", "test_retest_reliability", None), + ("calibration_summary", "dif_profiles_evaluated", 0), + ("calibration_summary", "temporal_drift_evaluated", False), + ): + payload = _published_policy() + payload[section][field] = value + lifecycle_cases.append((payload, "policy_publication_evidence_incomplete")) + + for payload, code in lifecycle_cases: + _assert_policy_rejected(payload, code) + + +def test_policy_time_and_runtime_contract_boundaries() -> None: + """Reject malformed clocks, invalid windows, and incomplete runtime maps.""" + payload = _evaluation_policy() + payload["created_at"] = "not-a-time" + _assert_policy_rejected(payload, "policy_schema_invalid") + + payload = _evaluation_policy() + payload["created_at"] = "2026-09-01T00:00:00" + _assert_policy_rejected(payload, "policy_schema_invalid") + + payload = _evaluation_policy() + manifest = _manifest_for(_POLICY_NAME, payload) + common = { + "artifact_name": _POLICY_NAME, + "artifact_bytes": _canonical_bytes(payload), + "manifest_bytes": _canonical_bytes(manifest), + "runtime_contracts": _RUNTIME_CONTRACTS, + } + for now, code in ( + (datetime(2026, 9, 2), "policy_clock_invalid"), + (datetime(2025, 9, 2, tzinfo=timezone.utc), "policy_future_dated"), + (datetime(2028, 9, 2, tzinfo=timezone.utc), "policy_expired"), + ): + with pytest.raises(EmailWritingPolicyError, match=code): + load_policy_artifact(now=now, **common) + + payload["expires_at"] = payload["created_at"] + with pytest.raises(EmailWritingPolicyError, match="policy_time_window_invalid"): + load_policy_artifact( + artifact_name=_POLICY_NAME, + artifact_bytes=_canonical_bytes(payload), + manifest_bytes=_canonical_bytes(_manifest_for(_POLICY_NAME, payload)), + now=_NOW, + runtime_contracts=_RUNTIME_CONTRACTS, + ) + + with pytest.raises(EmailWritingPolicyError, match="policy_contract_incompatible"): + _load(_evaluation_policy(), runtime_contracts={"naruon": "0.14.4"}) + + +def test_published_policy_admission_value_boundaries() -> None: + """Cover category, score, threshold, and successful admission outcomes.""" + policy = _load( + _published_policy(), + runtime_contracts={ + "naruon": "0.14.4", + "inkspan": "0.6.0", + "fast_mlsirm": "0.9.2", + "contextual_orchestrator": "v1", + }, + ) + categories = {criterion: 3 for criterion in _REQUIRED_CRITERIA} + scores = {criterion: 1.0 for criterion in _REQUIRED_CRITERIA} + + def evaluate(current_categories: dict[str, int], current_scores: dict[str, float]): + return evaluate_policy_admission( + policy=policy, + language_tag="en", + review_mode="deep", + candidate_model_profile_id="candidate-reviewer-v1", + candidate_provider_id="contextual-orchestrator", + judge_model_profile_id="independent-judge-v1", + judge_provider_id="contextual-orchestrator", + rubric_version="email_writing_judge_rubric_v1", + criterion_categories=current_categories, + criterion_scores=current_scores, + ) + + assert evaluate(categories, scores) == "admit" + + for value in (True, -1, policy.category_count): + invalid_categories = dict(categories) + invalid_categories["fact_preservation"] = value + with pytest.raises(EmailWritingPolicyError, match="criterion_category_invalid"): + evaluate(invalid_categories, scores) + + for value in (True, float("nan"), float("inf"), -0.1, 1.1): + invalid_scores = dict(scores) + invalid_scores["fact_preservation"] = value + with pytest.raises(EmailWritingPolicyError, match="criterion_score_invalid"): + evaluate(categories, invalid_scores) + + below_floor = dict(scores) + below_floor["fact_preservation"] = 0.1 + assert evaluate(categories, below_floor) == "withhold" + + +def test_rollback_requires_an_explicit_version() -> None: + """A policy without a named rollback target cannot select a candidate.""" + current = _load( + _published_policy(), + runtime_contracts={ + "naruon": "0.14.4", + "inkspan": "0.6.0", + "fast_mlsirm": "0.9.2", + "contextual_orchestrator": "v1", + }, + ) + with pytest.raises(EmailWritingPolicyError, match="policy_rollback_unavailable"): + select_rollback_policy(current_policy=current, candidates=[]) From 91813e60948e070ca3734ec4634adf28cb5daadd Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Fri, 4 Sep 2026 23:20:42 +0900 Subject: [PATCH 7/8] fix(email-writing): verify published evidence artifacts Resolve every published evidence envelope by digest, bind evaluation artifacts to the preregistered protocol, and reject missing, modified, or pre-protocol holdout evidence before admission. Assisted-by: OpenAI Codex Signed-off-by: Seongho Bae --- .../email_writing_judge_policy.schema.json | 2 +- backend/services/email_writing_policy.py | 82 ++++++++++++- backend/tests/test_email_writing_policy.py | 109 +++++++++++++++++- 3 files changed, 190 insertions(+), 3 deletions(-) diff --git a/backend/policies/email_writing_judge_policy.schema.json b/backend/policies/email_writing_judge_policy.schema.json index 8837f97f8..8fc0403b5 100644 --- a/backend/policies/email_writing_judge_policy.schema.json +++ b/backend/policies/email_writing_judge_policy.schema.json @@ -1 +1 @@ -{"$id":"https://contextualwisdomlab.github.io/naruon/schemas/email_writing_judge_policy.schema.json","$schema":"https://json-schema.org/draft/2020-12/schema","additionalProperties":false,"allOf":[{"if":{"properties":{"status":{"const":"published"}},"required":["status"]},"then":{"properties":{"publish_decision":{"const":"publish"}}}},{"if":{"properties":{"status":{"enum":["evaluation_only","superseded","revoked"]}},"required":["status"]},"then":{"properties":{"publish_decision":{"const":"withhold"}}}}],"properties":{"adjudication_conditions":{"items":{"enum":["same_model_without_compatible_calibration","criterion_disagreement","insufficient_evidence"]},"type":"array","uniqueItems":true},"approved_profiles":{"items":{"additionalProperties":false,"properties":{"candidate_model_profile_id":{"type":"string"},"candidate_provider_id":{"type":"string"},"judge_model_profile_id":{"type":"string"},"judge_provider_id":{"type":"string"},"language_tags":{"items":{"maxLength":63,"minLength":1,"type":"string"},"maxItems":32,"minItems":1,"type":"array","uniqueItems":true},"profile_id":{"type":"string"},"review_modes":{"items":{"enum":["incremental","deep"]},"maxItems":2,"minItems":1,"type":"array","uniqueItems":true},"rubric_version":{"type":"string"},"same_model_allowed":{"type":"boolean"}},"required":["profile_id","language_tags","review_modes","candidate_model_profile_id","candidate_provider_id","judge_model_profile_id","judge_provider_id","rubric_version","same_model_allowed"],"type":"object"},"maxItems":64,"minItems":1,"type":"array"},"calibration_summary":{"additionalProperties":false,"properties":{"brier_score":{"maximum":1.0,"minimum":0.0,"type":["number","null"]},"dif_profiles_evaluated":{"minimum":0,"type":"integer"},"status":{"enum":["not_evaluated","validated"]},"temporal_drift_evaluated":{"type":"boolean"},"test_retest_reliability":{"maximum":1.0,"minimum":0.0,"type":["number","null"]}},"required":["status","brier_score","test_retest_reliability","dif_profiles_evaluated","temporal_drift_evaluated"],"type":"object"},"category_anchors":{"items":{"type":"string"},"maxItems":9,"minItems":2,"type":"array","uniqueItems":true},"category_count":{"maximum":9,"minimum":2,"type":"integer"},"compatible_contracts":{"additionalProperties":false,"properties":{"contextual_orchestrator":{"type":"string"},"fast_mlsirm":{"type":["string","null"]},"inkspan":{"type":["string","null"]},"naruon":{"type":"string"}},"required":["naruon","inkspan","fast_mlsirm","contextual_orchestrator"],"type":"object"},"created_at":{"format":"date-time","type":"string"},"evidence":{"additionalProperties":false,"properties":{"calibration_dataset_hash":{"pattern":"^sha256:[0-9a-f]{64}$","type":["string","null"]},"holdout_labels_accessed_after_preregistration":{"type":"boolean"},"locked_holdout_hash":{"pattern":"^sha256:[0-9a-f]{64}$","type":["string","null"]},"protocol_hash":{"pattern":"^sha256:[0-9a-f]{64}$","type":["string","null"]},"reference_adjudication_hash":{"pattern":"^sha256:[0-9a-f]{64}$","type":["string","null"]}},"required":["protocol_hash","calibration_dataset_hash","locked_holdout_hash","reference_adjudication_hash","holdout_labels_accessed_after_preregistration"],"type":"object"},"expires_at":{"format":"date-time","type":"string"},"limitations":{"items":{"maxLength":1000,"minLength":1,"type":"string"},"maxItems":64,"minItems":1,"type":"array"},"mandatory_criterion_floors":{"additionalProperties":false,"properties":{"actionability":{"maximum":8,"minimum":0,"type":"integer"},"audience_pragmatics":{"maximum":8,"minimum":0,"type":"integer"},"explanation_quality":{"maximum":8,"minimum":0,"type":"integer"},"fact_preservation":{"maximum":8,"minimum":0,"type":"integer"},"intent_preservation":{"maximum":8,"minimum":0,"type":"integer"},"issue_support":{"maximum":8,"minimum":0,"type":"integer"},"replacement_correctness":{"maximum":8,"minimum":0,"type":"integer"},"request_strength_preservation":{"maximum":8,"minimum":0,"type":"integer"},"span_fidelity":{"maximum":8,"minimum":0,"type":"integer"},"technical_precision":{"maximum":8,"minimum":0,"type":"integer"}},"required":["issue_support","span_fidelity","replacement_correctness","intent_preservation","fact_preservation","request_strength_preservation","audience_pragmatics","technical_precision","actionability","explanation_quality"],"type":"object"},"minimum_criterion_scores":{"additionalProperties":false,"properties":{"actionability":{"maximum":1.0,"minimum":0.0,"type":"number"},"audience_pragmatics":{"maximum":1.0,"minimum":0.0,"type":"number"},"explanation_quality":{"maximum":1.0,"minimum":0.0,"type":"number"},"fact_preservation":{"maximum":1.0,"minimum":0.0,"type":"number"},"intent_preservation":{"maximum":1.0,"minimum":0.0,"type":"number"},"issue_support":{"maximum":1.0,"minimum":0.0,"type":"number"},"replacement_correctness":{"maximum":1.0,"minimum":0.0,"type":"number"},"request_strength_preservation":{"maximum":1.0,"minimum":0.0,"type":"number"},"span_fidelity":{"maximum":1.0,"minimum":0.0,"type":"number"},"technical_precision":{"maximum":1.0,"minimum":0.0,"type":"number"}},"required":["issue_support","span_fidelity","replacement_correctness","intent_preservation","fact_preservation","request_strength_preservation","audience_pragmatics","technical_precision","actionability","explanation_quality"],"type":"object"},"policy_id":{"pattern":"^[A-Za-z0-9][A-Za-z0-9._:-]{0,127}$","type":"string"},"policy_version":{"pattern":"^[A-Za-z0-9][A-Za-z0-9._:-]{0,127}$","type":"string"},"publish_decision":{"enum":["publish","withhold"]},"required_criteria_by_candidate_kind":{"additionalProperties":false,"properties":{"no_replacement_diagnostic":{"items":{"enum":["issue_support","span_fidelity","replacement_correctness","intent_preservation","fact_preservation","request_strength_preservation","audience_pragmatics","technical_precision","actionability","explanation_quality"]},"minItems":1,"type":"array","uniqueItems":true},"replacement_diagnostic":{"items":{"enum":["issue_support","span_fidelity","replacement_correctness","intent_preservation","fact_preservation","request_strength_preservation","audience_pragmatics","technical_precision","actionability","explanation_quality"]},"minItems":1,"type":"array","uniqueItems":true}},"required":["replacement_diagnostic","no_replacement_diagnostic"],"type":"object"},"rollback_version":{"type":["string","null"]},"status":{"enum":["evaluation_only","published","superseded","revoked"]}},"required":["policy_id","policy_version","status","publish_decision","created_at","expires_at","compatible_contracts","approved_profiles","category_count","category_anchors","required_criteria_by_candidate_kind","mandatory_criterion_floors","minimum_criterion_scores","adjudication_conditions","calibration_summary","evidence","limitations","rollback_version"],"title":"Naruon Email Writing Judge Policy","type":"object"} +{"$id":"https://contextualwisdomlab.github.io/naruon/schemas/email_writing_judge_policy.schema.json","$schema":"https://json-schema.org/draft/2020-12/schema","additionalProperties":false,"allOf":[{"if":{"properties":{"status":{"const":"published"}},"required":["status"]},"then":{"properties":{"calibration_summary":{"properties":{"status":{"const":"validated"}},"required":["status"]},"evidence":{"properties":{"calibration_dataset_hash":{"type":"string"},"holdout_labels_accessed_after_preregistration":{"const":true},"locked_holdout_hash":{"type":"string"},"protocol_hash":{"type":"string"},"reference_adjudication_hash":{"type":"string"}},"required":["protocol_hash","calibration_dataset_hash","locked_holdout_hash","reference_adjudication_hash","holdout_labels_accessed_after_preregistration"]},"publish_decision":{"const":"publish"}}}},{"if":{"properties":{"status":{"enum":["evaluation_only","superseded","revoked"]}},"required":["status"]},"then":{"properties":{"publish_decision":{"const":"withhold"}}}}],"properties":{"adjudication_conditions":{"items":{"enum":["same_model_without_compatible_calibration","criterion_disagreement","insufficient_evidence"]},"type":"array","uniqueItems":true},"approved_profiles":{"items":{"additionalProperties":false,"properties":{"candidate_model_profile_id":{"type":"string"},"candidate_provider_id":{"type":"string"},"judge_model_profile_id":{"type":"string"},"judge_provider_id":{"type":"string"},"language_tags":{"items":{"maxLength":63,"minLength":1,"type":"string"},"maxItems":32,"minItems":1,"type":"array","uniqueItems":true},"profile_id":{"type":"string"},"review_modes":{"items":{"enum":["incremental","deep"]},"maxItems":2,"minItems":1,"type":"array","uniqueItems":true},"rubric_version":{"type":"string"},"same_model_allowed":{"type":"boolean"}},"required":["profile_id","language_tags","review_modes","candidate_model_profile_id","candidate_provider_id","judge_model_profile_id","judge_provider_id","rubric_version","same_model_allowed"],"type":"object"},"maxItems":64,"minItems":1,"type":"array"},"calibration_summary":{"additionalProperties":false,"properties":{"brier_score":{"maximum":1.0,"minimum":0.0,"type":["number","null"]},"dif_profiles_evaluated":{"minimum":0,"type":"integer"},"status":{"enum":["not_evaluated","validated"]},"temporal_drift_evaluated":{"type":"boolean"},"test_retest_reliability":{"maximum":1.0,"minimum":0.0,"type":["number","null"]}},"required":["status","brier_score","test_retest_reliability","dif_profiles_evaluated","temporal_drift_evaluated"],"type":"object"},"category_anchors":{"items":{"type":"string"},"maxItems":9,"minItems":2,"type":"array","uniqueItems":true},"category_count":{"maximum":9,"minimum":2,"type":"integer"},"compatible_contracts":{"additionalProperties":false,"properties":{"contextual_orchestrator":{"type":"string"},"fast_mlsirm":{"type":["string","null"]},"inkspan":{"type":["string","null"]},"naruon":{"type":"string"}},"required":["naruon","inkspan","fast_mlsirm","contextual_orchestrator"],"type":"object"},"created_at":{"format":"date-time","type":"string"},"evidence":{"additionalProperties":false,"properties":{"calibration_dataset_hash":{"pattern":"^sha256:[0-9a-f]{64}$","type":["string","null"]},"holdout_labels_accessed_after_preregistration":{"type":"boolean"},"locked_holdout_hash":{"pattern":"^sha256:[0-9a-f]{64}$","type":["string","null"]},"protocol_hash":{"pattern":"^sha256:[0-9a-f]{64}$","type":["string","null"]},"reference_adjudication_hash":{"pattern":"^sha256:[0-9a-f]{64}$","type":["string","null"]}},"required":["protocol_hash","calibration_dataset_hash","locked_holdout_hash","reference_adjudication_hash","holdout_labels_accessed_after_preregistration"],"type":"object"},"expires_at":{"format":"date-time","type":"string"},"limitations":{"items":{"maxLength":1000,"minLength":1,"type":"string"},"maxItems":64,"minItems":1,"type":"array"},"mandatory_criterion_floors":{"additionalProperties":false,"properties":{"actionability":{"maximum":8,"minimum":0,"type":"integer"},"audience_pragmatics":{"maximum":8,"minimum":0,"type":"integer"},"explanation_quality":{"maximum":8,"minimum":0,"type":"integer"},"fact_preservation":{"maximum":8,"minimum":0,"type":"integer"},"intent_preservation":{"maximum":8,"minimum":0,"type":"integer"},"issue_support":{"maximum":8,"minimum":0,"type":"integer"},"replacement_correctness":{"maximum":8,"minimum":0,"type":"integer"},"request_strength_preservation":{"maximum":8,"minimum":0,"type":"integer"},"span_fidelity":{"maximum":8,"minimum":0,"type":"integer"},"technical_precision":{"maximum":8,"minimum":0,"type":"integer"}},"required":["issue_support","span_fidelity","replacement_correctness","intent_preservation","fact_preservation","request_strength_preservation","audience_pragmatics","technical_precision","actionability","explanation_quality"],"type":"object"},"minimum_criterion_scores":{"additionalProperties":false,"properties":{"actionability":{"maximum":1.0,"minimum":0.0,"type":"number"},"audience_pragmatics":{"maximum":1.0,"minimum":0.0,"type":"number"},"explanation_quality":{"maximum":1.0,"minimum":0.0,"type":"number"},"fact_preservation":{"maximum":1.0,"minimum":0.0,"type":"number"},"intent_preservation":{"maximum":1.0,"minimum":0.0,"type":"number"},"issue_support":{"maximum":1.0,"minimum":0.0,"type":"number"},"replacement_correctness":{"maximum":1.0,"minimum":0.0,"type":"number"},"request_strength_preservation":{"maximum":1.0,"minimum":0.0,"type":"number"},"span_fidelity":{"maximum":1.0,"minimum":0.0,"type":"number"},"technical_precision":{"maximum":1.0,"minimum":0.0,"type":"number"}},"required":["issue_support","span_fidelity","replacement_correctness","intent_preservation","fact_preservation","request_strength_preservation","audience_pragmatics","technical_precision","actionability","explanation_quality"],"type":"object"},"policy_id":{"pattern":"^[A-Za-z0-9][A-Za-z0-9._:-]{0,127}$","type":"string"},"policy_version":{"pattern":"^[A-Za-z0-9][A-Za-z0-9._:-]{0,127}$","type":"string"},"publish_decision":{"enum":["publish","withhold"]},"required_criteria_by_candidate_kind":{"additionalProperties":false,"properties":{"no_replacement_diagnostic":{"items":{"enum":["issue_support","span_fidelity","replacement_correctness","intent_preservation","fact_preservation","request_strength_preservation","audience_pragmatics","technical_precision","actionability","explanation_quality"]},"minItems":1,"type":"array","uniqueItems":true},"replacement_diagnostic":{"items":{"enum":["issue_support","span_fidelity","replacement_correctness","intent_preservation","fact_preservation","request_strength_preservation","audience_pragmatics","technical_precision","actionability","explanation_quality"]},"minItems":1,"type":"array","uniqueItems":true}},"required":["replacement_diagnostic","no_replacement_diagnostic"],"type":"object"},"rollback_version":{"type":["string","null"]},"status":{"enum":["evaluation_only","published","superseded","revoked"]}},"required":["policy_id","policy_version","status","publish_decision","created_at","expires_at","compatible_contracts","approved_profiles","category_count","category_anchors","required_criteria_by_candidate_kind","mandatory_criterion_floors","minimum_criterion_scores","adjudication_conditions","calibration_summary","evidence","limitations","rollback_version"],"title":"Naruon Email Writing Judge Policy","type":"object"} diff --git a/backend/services/email_writing_policy.py b/backend/services/email_writing_policy.py index 919286329..f2d1d55ee 100644 --- a/backend/services/email_writing_policy.py +++ b/backend/services/email_writing_policy.py @@ -17,7 +17,6 @@ from pydantic import ( BaseModel, ConfigDict, - Field, PrivateAttr, StrictBool, StrictFloat, @@ -255,6 +254,35 @@ def validate_hash(cls, value: str | None) -> str | None: return value +class _EvidenceArtifact(_PolicyModel): + """Minimal immutable evidence envelope resolved before publication.""" + + evidence_version: Literal[1] + evidence_kind: Literal[ + "protocol", + "calibration_dataset", + "locked_holdout", + "reference_adjudication", + ] + recorded_at: StrictStr + protocol_hash: StrictStr | None + + @field_validator("recorded_at") + @classmethod + def validate_recorded_at(cls, value: str) -> str: + """Require an offset-aware evidence timestamp.""" + _parse_timestamp(value) + return value + + @field_validator("protocol_hash") + @classmethod + def validate_protocol_hash(cls, value: str | None) -> str | None: + """Require an algorithm-qualified protocol identity when present.""" + if value is not None and _HASH_RE.fullmatch(value) is None: + raise ValueError("evidence_protocol_hash_invalid") + return value + + class EmailWritingJudgePolicy(_PolicyModel): """Validated policy artifact that contains no authored or model plaintext.""" @@ -496,6 +524,55 @@ def _validate_runtime_contracts( raise EmailWritingPolicyError("policy_contract_incompatible") +def _verify_published_evidence( + policy: EmailWritingJudgePolicy, + evidence_artifacts: Mapping[str, bytes] | None, +) -> None: + """Resolve and hash every published evidence envelope before admission.""" + expected = { + "protocol": policy.evidence.protocol_hash, + "calibration_dataset": policy.evidence.calibration_dataset_hash, + "locked_holdout": policy.evidence.locked_holdout_hash, + "reference_adjudication": policy.evidence.reference_adjudication_hash, + } + if evidence_artifacts is None or any(value is None for value in expected.values()): + raise EmailWritingPolicyError("policy_publication_evidence_unverified") + + resolved: dict[str, _EvidenceArtifact] = {} + for evidence_kind, qualified_digest in expected.items(): + assert qualified_digest is not None + artifact_bytes = evidence_artifacts.get(qualified_digest) + if artifact_bytes is None: + raise EmailWritingPolicyError("policy_publication_evidence_unverified") + actual_digest = "sha256:" + hashlib.sha256(artifact_bytes).hexdigest() + if actual_digest != qualified_digest: + raise EmailWritingPolicyError("policy_publication_evidence_mismatch") + try: + artifact = parse_strict_email_writing_json( + artifact_bytes, + _EvidenceArtifact, + ) + except (StrictEmailWritingJsonError, ValidationError, ValueError) as error: + raise EmailWritingPolicyError( + "policy_publication_evidence_invalid" + ) from error + if artifact.evidence_kind != evidence_kind: + raise EmailWritingPolicyError("policy_publication_evidence_invalid") + resolved[evidence_kind] = artifact + + protocol_hash = expected["protocol"] + if resolved["protocol"].protocol_hash is not None or any( + artifact.protocol_hash != protocol_hash + for evidence_kind, artifact in resolved.items() + if evidence_kind != "protocol" + ): + raise EmailWritingPolicyError("policy_publication_evidence_invalid") + if _parse_timestamp(resolved["locked_holdout"].recorded_at) <= _parse_timestamp( + resolved["protocol"].recorded_at + ): + raise EmailWritingPolicyError("policy_publication_evidence_order_invalid") + + def load_policy_artifact( *, artifact_name: str, @@ -503,6 +580,7 @@ def load_policy_artifact( manifest_bytes: bytes, now: datetime, runtime_contracts: Mapping[str, str | None], + evidence_artifacts: Mapping[str, bytes] | None = None, ) -> EmailWritingJudgePolicy: """Load one integrity-bound policy after strict manifest and lifecycle validation. @@ -544,6 +622,8 @@ def load_policy_artifact( raise EmailWritingPolicyError("policy_expired") _validate_runtime_contracts(policy, runtime_contracts) + if policy.status == "published": + _verify_published_evidence(policy, evidence_artifacts) object.__setattr__(policy, "_artifact_name", artifact_name) object.__setattr__(policy, "_artifact_sha256", actual_digest) return policy diff --git a/backend/tests/test_email_writing_policy.py b/backend/tests/test_email_writing_policy.py index 523835ebd..7a078a9d4 100644 --- a/backend/tests/test_email_writing_policy.py +++ b/backend/tests/test_email_writing_policy.py @@ -104,12 +104,55 @@ def _published_policy(*, version: str = "email-writing-policy-v1") -> dict[str, return payload +def _published_evidence_artifacts(payload: dict[str, object]) -> dict[str, bytes]: + """Build immutable evidence envelopes and bind their digests to the policy.""" + evidence = payload["evidence"] + assert isinstance(evidence, dict) + artifacts: dict[str, bytes] = {} + protocol_bytes = _canonical_bytes( + { + "evidence_version": 1, + "evidence_kind": "protocol", + "recorded_at": "2026-08-01T00:00:00Z", + "protocol_hash": None, + } + ) + protocol_hash = "sha256:" + hashlib.sha256(protocol_bytes).hexdigest() + if evidence.get("protocol_hash") == "sha256:" + "1" * 64: + evidence["protocol_hash"] = protocol_hash + artifacts[protocol_hash] = protocol_bytes + for index, evidence_kind in enumerate( + ("calibration_dataset", "locked_holdout", "reference_adjudication"), + start=2, + ): + artifact_bytes = _canonical_bytes( + { + "evidence_version": 1, + "evidence_kind": evidence_kind, + "recorded_at": f"2026-08-0{index}T00:00:00Z", + "protocol_hash": protocol_hash, + } + ) + qualified_digest = "sha256:" + hashlib.sha256(artifact_bytes).hexdigest() + field_name = f"{evidence_kind}_hash" + placeholder_digit = str(index) + if evidence.get(field_name) == "sha256:" + placeholder_digit * 64: + evidence[field_name] = qualified_digest + artifacts[qualified_digest] = artifact_bytes + return artifacts + + def _load( payload: dict[str, object], *, runtime_contracts: dict[str, str | None] | None = None, ): """Load one synthetic artifact through the production integrity boundary.""" + evidence_artifacts = ( + _published_evidence_artifacts(payload) + if payload.get("status") == "published" + else None + ) manifest = _manifest_for(_POLICY_NAME, payload) return load_policy_artifact( artifact_name=_POLICY_NAME, @@ -117,6 +160,7 @@ def _load( manifest_bytes=_canonical_bytes(manifest), now=_NOW, runtime_contracts=runtime_contracts or _RUNTIME_CONTRACTS, + evidence_artifacts=evidence_artifacts, ) @@ -135,6 +179,16 @@ def test_policy_schema_declares_strict_lifecycle_and_publish_decision() -> None: "superseded", "revoked", ] + published_contract = schema["allOf"][0]["then"]["properties"] + assert published_contract["calibration_summary"]["properties"]["status"] == { + "const": "validated" + } + assert published_contract["evidence"]["properties"]["protocol_hash"] == { + "type": "string" + } + assert published_contract["evidence"]["properties"][ + "holdout_labels_accessed_after_preregistration" + ] == {"const": True} @pytest.mark.parametrize("field", ["publish_decision", "status", "policy_version"]) @@ -230,6 +284,59 @@ def test_published_policy_requires_complete_calibration_and_holdout_evidence() - ) +def test_published_policy_resolves_evidence_bytes_and_preregistration_order() -> None: + """Reject missing, modified, or pre-protocol published evidence artifacts.""" + payload = _published_policy() + evidence_artifacts = _published_evidence_artifacts(payload) + manifest = _manifest_for(_POLICY_NAME, payload) + common = { + "artifact_name": _POLICY_NAME, + "artifact_bytes": _canonical_bytes(payload), + "manifest_bytes": _canonical_bytes(manifest), + "now": _NOW, + "runtime_contracts": { + "naruon": "0.14.4", + "inkspan": "0.6.0", + "fast_mlsirm": "0.9.2", + "contextual_orchestrator": "v1", + }, + } + with pytest.raises( + EmailWritingPolicyError, match="policy_publication_evidence_unverified" + ): + load_policy_artifact(**common) + + modified = dict(evidence_artifacts) + modified[next(iter(modified))] += b"modified" + with pytest.raises( + EmailWritingPolicyError, match="policy_publication_evidence_mismatch" + ): + load_policy_artifact(**common, evidence_artifacts=modified) + + evidence = payload["evidence"] + assert isinstance(evidence, dict) + old_holdout_hash = evidence["locked_holdout_hash"] + assert isinstance(old_holdout_hash, str) + early_holdout = _canonical_bytes( + { + "evidence_version": 1, + "evidence_kind": "locked_holdout", + "recorded_at": "2026-07-31T00:00:00Z", + "protocol_hash": evidence["protocol_hash"], + } + ) + new_holdout_hash = "sha256:" + hashlib.sha256(early_holdout).hexdigest() + evidence["locked_holdout_hash"] = new_holdout_hash + evidence_artifacts.pop(old_holdout_hash) + evidence_artifacts[new_holdout_hash] = early_holdout + common["artifact_bytes"] = _canonical_bytes(payload) + common["manifest_bytes"] = _canonical_bytes(_manifest_for(_POLICY_NAME, payload)) + with pytest.raises( + EmailWritingPolicyError, match="policy_publication_evidence_order_invalid" + ): + load_policy_artifact(**common, evidence_artifacts=evidence_artifacts) + + def test_profile_and_contract_mismatch_fail_closed() -> None: """Unsupported runtime contracts and language profiles cannot silently broaden claims.""" payload = _published_policy() @@ -378,7 +485,7 @@ def test_mixed_criterion_identity_and_impossible_floors_fail_closed() -> None: def test_revoked_policy_rolls_back_only_to_explicit_compatible_manifest_entry() -> None: - """Rollback cannot silently downgrade to an unlisted, expired, or incompatible artifact.""" + """Rollback cannot silently downgrade to an unlisted artifact.""" current_payload = _published_policy(version="email-writing-policy-v2") current_payload["status"] = "revoked" current_payload["publish_decision"] = "withhold" From e7bc7b5dc97b312ca907d2e1def15b399a51cd48 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sat, 5 Sep 2026 01:01:18 +0900 Subject: [PATCH 8/8] test(email-writing): cover policy evidence envelopes --- backend/tests/test_email_writing_policy.py | 78 ++++++++++++++++++++++ 1 file changed, 78 insertions(+) diff --git a/backend/tests/test_email_writing_policy.py b/backend/tests/test_email_writing_policy.py index 7a078a9d4..aae79a982 100644 --- a/backend/tests/test_email_writing_policy.py +++ b/backend/tests/test_email_writing_policy.py @@ -337,6 +337,84 @@ def test_published_policy_resolves_evidence_bytes_and_preregistration_order() -> load_policy_artifact(**common, evidence_artifacts=evidence_artifacts) +def test_published_evidence_envelopes_fail_closed_on_identity_mismatch() -> None: + """Reject missing, malformed, mislabeled, and self-referential evidence.""" + + def load_with( + payload: dict[str, object], artifacts: dict[str, bytes] + ) -> EmailWritingJudgePolicy: + return load_policy_artifact( + artifact_name=_POLICY_NAME, + artifact_bytes=_canonical_bytes(payload), + manifest_bytes=_canonical_bytes(_manifest_for(_POLICY_NAME, payload)), + now=_NOW, + runtime_contracts={ + "naruon": "0.14.4", + "inkspan": "0.6.0", + "fast_mlsirm": "0.9.2", + "contextual_orchestrator": "v1", + }, + evidence_artifacts=artifacts, + ) + + payload = _published_policy() + artifacts = _published_evidence_artifacts(payload) + missing = dict(artifacts) + missing.pop(next(iter(missing))) + with pytest.raises( + EmailWritingPolicyError, match="policy_publication_evidence_unverified" + ): + load_with(payload, missing) + + for evidence_kind, replacement, expected_code in ( + ( + "calibration_dataset", + { + "evidence_version": 1, + "evidence_kind": "calibration_dataset", + "recorded_at": "2026-08-02T00:00:00Z", + "protocol_hash": "invalid", + }, + "policy_publication_evidence_invalid", + ), + ( + "locked_holdout", + { + "evidence_version": 1, + "evidence_kind": "reference_adjudication", + "recorded_at": "2026-08-03T00:00:00Z", + "protocol_hash": payload["evidence"]["protocol_hash"], + }, + "policy_publication_evidence_invalid", + ), + ( + "protocol", + { + "evidence_version": 1, + "evidence_kind": "protocol", + "recorded_at": "2026-08-01T00:00:00Z", + "protocol_hash": "sha256:" + "0" * 64, + }, + "policy_publication_evidence_invalid", + ), + ): + current_payload = deepcopy(payload) + current_artifacts = dict(artifacts) + evidence = current_payload["evidence"] + assert isinstance(evidence, dict) + field_name = f"{evidence_kind}_hash" + old_hash = evidence[field_name] + assert isinstance(old_hash, str) + replacement_bytes = _canonical_bytes(replacement) + replacement_hash = "sha256:" + hashlib.sha256(replacement_bytes).hexdigest() + evidence[field_name] = replacement_hash + current_artifacts.pop(old_hash) + current_artifacts[replacement_hash] = replacement_bytes + + with pytest.raises(EmailWritingPolicyError, match=expected_code): + load_with(current_payload, current_artifacts) + + def test_profile_and_contract_mismatch_fail_closed() -> None: """Unsupported runtime contracts and language profiles cannot silently broaden claims.""" payload = _published_policy()