diff --git a/evals/source-identity-v1/README.md b/evals/source-identity-v1/README.md new file mode 100644 index 0000000..1389def --- /dev/null +++ b/evals/source-identity-v1/README.md @@ -0,0 +1,258 @@ +# Prospective source identity v1 + +A standalone, offline evaluator component for **future, explicitly opted-in +cohorts**. It reads selected source bytes, resolves a declaration spelling, and +checks exact citations. It does not use graph output, import evaluated Python +modules, expand Rust macros, invoke providers, or run evaluated source. It uses +Python 3.10+ and an isolated, pinned Rust syntax frontend (the executable +snapshot and generic-Python fixture use Python 3.12). The frontend parses source +with `syn`; it never passes evaluated bytes to a compiler. + +Nothing here changes historical v1/v2 evaluators, imports, corpus, protocols, +locks, answers, reports or scores. These tests demonstrate contract behavior on +new synthetic sources, **not measured agent benefit**. Unsupported syntax is an +explicit limitation, not evidence that an answer is a hallucination. + +## Run + +From the repository root, on Python 3.12, first build the evaluator frontend. +Rust 1.89+ and the exact crates in `syntax/Cargo.lock` must already be cached; +these commands do not install host software or access the network. Check disk +usage before creating the build directory and stop at 80% or above: + +```sh +df --output=pcent .orbit/tmp | tail -1 +CARGO_TARGET_DIR="$PWD/.orbit/tmp/source-identity-syntax-target" \ + cargo build --offline --locked --manifest-path evals/source-identity-v1/syntax/Cargo.toml +``` + +Then run the prospective gates (including the isolated frontend's format/lint +checks, which are separate from the product workspace): + +```sh +cargo fmt --manifest-path evals/source-identity-v1/syntax/Cargo.toml --check +CARGO_TARGET_DIR="$PWD/.orbit/tmp/source-identity-syntax-target" \ + cargo clippy --offline --locked --manifest-path evals/source-identity-v1/syntax/Cargo.toml -- -D warnings +cargo deny --offline --locked --manifest-path evals/source-identity-v1/syntax/Cargo.toml \ + check --config deny.toml --disable-fetch +python3 -B -m unittest discover -s evals/source-identity-v1 -p 'test_*.py' +python3 -B evals/source-identity-v1/example.py +python3 -B evals/source-identity-v1/source_identity.py --help +``` + +`example.py` prints a deterministic, test-only result: identity recall **2/3**, +citation recall **1/3**, one missing required identity, one visible unsupported +alias, and **pending** semantic review. One answer uses an equivalent +trait-qualified Rust spelling; another names the correct Python assignment but +quotes the wrong value. `example-result.json` pins the exact output, contract +bytes, implementation bytes, source bytes, manifest and Python AST version. +The frontend runtime and its complete dependency lock are also pinned. +The snapshot is a fixture, not a study result. Semantic review has no inferred +pass, combined score, or agent-effectiveness interpretation. + +To exercise the reusable scorer with the same JSON request: + +```sh +mkdir -p .orbit/tmp +python3 -B evals/source-identity-v1/example.py --request > .orbit/tmp/source-identity-request.json +python3 -B evals/source-identity-v1/source_identity.py \ + --root evals/source-identity-v1/fixtures \ + --input .orbit/tmp/source-identity-request.json +``` + +The CLI writes one JSON result to stdout and exits 0, even when submitted answers +receive no credit. Invalid evaluator inputs/truth exit 2 with a JSON error on +stderr and empty stdout. Argument-parser usage errors also exit 2. `--help` and +representative output are captured from the real executables in `cli-help.txt` +and `example-result.json`. To deliberately regenerate those fixtures after a +reviewed implementation/contract change, run: + +```sh +UPDATE_GOLDENS=1 python3 -B -m unittest discover -s evals/source-identity-v1 -p 'test_*.py' +``` + +Normal tests never rewrite them. A changed runtime version changes the pinned +output; evaluate and adopt that runtime prospectively as well. The repository's +complete validation sequence remains in `CONTRIBUTING.md`; these tests are a +separate local gate because this task does not change the historical harness. + +## Input and identity + +`source_identity.Project(root, rust_root=..., python_files=[...])` constructs a +read-once source snapshot. `Project.check(selector)` validates an item; +`score(project, required, submitted)` scores arrays; `score_request(root, +request)` is the same entry point as the CLI. The request has exactly: + +- `schema_version: 1`; +- `manifest: {"rust_root": "rust/lib.rs", "python_files": ["python/settings.py"]}` + (`rust_root` may be null); +- a nonempty `required` array and a `submitted` array. + +Every selector has exactly `language`, `name`, `file`, `line`, `kind`, and +`citation`. `language` is `rust` or `python`; file paths are canonical, +root-relative POSIX paths; lines are positive, one-based integers (not booleans). +Kinds are `fn`, `struct`, `enum`, `trait` for supported Rust declarations and +`function`, `class`, `assignment` for supported Python declarations. Python +assignments are syntactic bindings, not claims of runtime immutability. + +A citation is `{"start_line": 2, "end_line": 2, "quote": "RETRY_LIMIT = 7"}`. +It must begin on the declaration anchor line, end inside that declaration, and +match complete source lines including indentation, joined by LF without a final +newline. Substrings, different values, nearby comments, empty quotes, and +out-of-span ranges do not receive citation credit. Source encoding is UTF-8/LF; +CRLF, NUL, symlink paths, escaping paths and files over 1 MiB are rejected. + +Canonical identity consists of language, file, line, column, declaration kind, +module, verified owner, verified trait and leaf name. Rust's anchor is its name +token; Python's is the AST declaration/assignment start. Function bodies and +class spans provide citation bounds, not additional name-resolution scopes. +Hashes bind those locations to the exact source snapshot. Rust module identity +comes from the selected crate root and module declarations, **not filenames**. +Python module identity is the explicit root-relative `.py` path, with +`__init__.py` naming its package. It does not inspect `sys.path`. + +Resolve spellings across the **entire selected language index before** comparing +file, line and kind. A line number cannot break a duplicate-name tie. An +unqualified free function name considers free declarations, not methods sharing +its leaf. Use `crate::parse` or `nested::parse` to distinguish free functions in +different modules. `Compass::parse` and `Ledger::parse` identify different owners. +For owner-qualified methods, a short owner must itself be unique across indexed +types/import bindings, even if only one owner defines the method. + +For the synthetic trait implementation, these identify the same declaration: + +- `Compass::from_str` (only when no other trait/inherent declaration competes); +- `::from_str` (the import verifies `FromStr`); +- `::from_str`. + +Validated local type/import aliases and full module-qualified names are +supported. Trait-qualified names must match the verified trait binding. Rust +**dot spellings are rejected** with a reason: use `Owner::method`, not +`Owner.method`. There is no heuristic conversion or observed-name allowlist. + +## Bounded source grammar + +`syntax/` is a separate, unpublished Cargo workspace with no product dependency +edges. It uses `syn` **2.0.119** (full syntax and AST visitor) and `proc-macro2` +**1.0.106** (lexing and source spans), with JSON transport and typed errors. +`syntax/Cargo.lock` pins every transitive version, registry origin and checksum. +These crates came from the existing Cargo registry cache; their package manifests +and license files are the upstream provenance, not recovered Orbit code. The +parser and lexer are MIT/Apache-2.0 dual licensed. No vendored or downloaded +parser binary is committed. Build the frontend from these sources and lockfile +in the ignored path above. The scorer launches that fixed executable with an +empty environment and a ten-second timeout; it never builds on demand. + +The frontend parses the **entire file**, including function parameters, return +types, bodies, literals, fields and unrelated declarations. Syntax errors and +unrecognized `Verbatim` nodes refuse the Rust index. Validated source passes to +the declaration resolver; delimiter balance alone cannot grant credit. The +binary embeds its own source, manifest and lockfile, which Python compares with +the current files before accepting a successful parse. Missing, stale, failing +or timed-out frontends refuse Rust credit with a diagnostic. Returned pins hash +these files alongside the Python implementation, contract and source snapshot. +This detects accidental stale builds; it is not executable attestation or a +security boundary against an evaluator who controls the helper binary. + +Supported declarations include free/generic functions, structs/enums with +parameters and fields, local traits with supertraits/associated items, and +inherent/trait impl methods of verified local named types, including generic +owners and where clauses. Owner/trait arguments are erased **only for spelling +resolution of a written declaration**. Two specialized impls defining the same +method stay ambiguous; this never infers instantiation, trait applicability or +runtime dispatch. Blanket impls on generic parameters, qualified-self types and +negative impls remain unsupported. + +Ordinary line/block docs, explicit `doc`, lint attributes, `repr`, `inline`, +`cold`, `must_use`, `deprecated`, `non_exhaustive`, `no_mangle`, `export_name`, +`link_section` and `track_caller` preserve written binding identity. Standard +`derive` entries (`Clone`, `Copy`, `Debug`, `Default`, `Eq`, `PartialEq`, `Ord`, +`PartialOrd`, `Hash`) are admitted; generated methods are never indexed. Custom +attribute/derive macros and `cfg_attr` are refused because they can emit unknown +bindings. This is an explicit attribute-effects policy, not an ignore-all rule +or an allowlist of declaration names. Attribute argument semantics are not +compiler-checked. + +A `cfg` declaration is recorded as uncertain, with all descendants/methods of +conditional modules, owners, traits and impls also uncredited. It still competes +in ambiguity checks; a line number cannot rescue it. Named const/static/type +alias/associated bindings are unsupported candidates and blockers, so they do +not erase independently verified unrelated declarations. Macro definitions (whose matcher grammar is not validated by `syn`), +item macro invocations, foreign scopes, +glob/group/conditional imports and other contexts with an unknown binding set +still refuse the whole Rust index. Diagnostics explain the distinction. + +Inline modules and external modules with literal `#[path = "file.rs"]` retain +the explicit closed source universe. Paths work at the crate root and within +its inline modules; external files cannot declare further external modules. +Reused/cyclic files, duplicate modules, ordinary external modules without a +literal path, import collisions/cycles and shadowed `std`/`core` roots are +refused. Simple `use path;` / `use path as Alias;` can reference verified local +types or written `std::`/`core::` paths. Unknown owners/traits and type aliases +used as impl owners are refused. No extern-crate/prelude guessing occurs. + +Syntax validity means acceptance by the pinned parser's Rust grammar, **not +compilation, typechecking or behavioral correctness**. Macro token trees remain +unexpanded (their tokens need not themselves be Rust expressions), and methods +generated by macros/derives are not written declarations. Edition/feature +availability, name/type resolution inside bodies, trait completeness, attribute +semantics and runtime behavior require separate review. The synthetic sources +include syntactically valid but semantically incomplete bodies. The helper is +neither a compiler nor a security sandbox. + +Python uses `ast.parse`, never `import`, `eval`, `exec`, or `ast.literal_eval` on +source. It supports free/async functions (including generic syntax supported by +the pinned AST runtime), simple classes/methods, and single module-level named +assignments/annotated assignments whose values are literal scalars or recursively +literal tuples. Signed numeric literals are allowed. Names, calls, arithmetic, +mutable containers, annotation-only bindings, complex annotations and assignment +aliases stay unsupported. Explicit imports are recorded as unsupported bindings +without importing them. They still block ambiguous names. Duplicate/rebound +bindings cannot be rescued by quoting one occurrence. Decorated functions receive +no credit; decorated/inherited/rebound classes invalidate the Python index so +hidden method ownership cannot be inferred. Destructuring, chained/attribute +assignment, wildcard imports, conditionals, mutation and other module/class +statements invalidate the Python index. Function bodies are opaque AST subtrees; +local/nested function bindings are not scored. + +These are **syntactic declaration identities**, not evaluation of Python module +initialization or arbitrary import/decorator/call effects. A dynamic RHS does not +become a constant merely because an answer cites it. Establish runtime claims +through the separate semantic review. When one source file makes a language +index incomplete, all identities in that language are refused; another complete +language index can still be evaluated, with the refusal visible in output. + +## Scoring and prospective adoption + +The evaluator author must freeze the source root, manifest, implementation, +contract, runtime and required identities **before** collecting answers. The +manifest is trusted evaluator input, never selected independently by each answer: +a smaller file universe can hide ambiguity. Keep source files stable while the +snapshot is constructed. Files are cached once read; later disk changes do not +alter that snapshot's citations or hashes. No repository-root discovery or graph +lookup supplies missing context. + +Required identities and their citations must all validate, with no duplicate +canonical identities. Invalid/unsupported truth fails the run; it cannot shrink +the denominator. Submitted duplicates cannot increase recall. Identity recall +counts distinct required identities; citation recall counts those same identities +with exact evidence. Every submitted item survives in `items`, including +`verified_extra` declarations and uncredited/unsupported items with reasons. +`missing` retains every unsatisfied required identity. There is no automatic +penalty that labels an unsupported item hallucinated. + +`semantic_review` always remains pending. Consumers must retain this independent +review and must not treat identity/citation equivalence as semantic correctness. +Future cohorts must deliberately record and verify the returned contract, +implementation, source and runtime pins in their own prospective lock/protocol. +Never apply the new interpretation to frozen historical scores or import it into +historical evaluators. Contract/grammar changes require a new prospective pin +(and a new contract version for changed identity semantics). + +The fixtures use independently invented `Compass`, `Ledger`, `Dial`, `Socket` +and Python bindings. Tests include trait/inherent ambiguity, owner ambiguity, +wrong module/file/kind/line/quote, literal path mapping, imports, conditional +alternatives, dynamic assignments, comments/string lookalikes, malformed source, +symlink refusal, source snapshot hashing, unchanged denominators, and executable +success/error/help behavior. They are deterministic and run without providers, +network access, or host installation. Build the pinned frontend offline first. diff --git a/evals/source-identity-v1/cli-help.txt b/evals/source-identity-v1/cli-help.txt new file mode 100644 index 0000000..75bf717 --- /dev/null +++ b/evals/source-identity-v1/cli-help.txt @@ -0,0 +1,8 @@ +usage: source_identity.py [-h] --root ROOT --input INPUT + +Score prospective source identities and exact citations; JSON output only. + +options: + -h, --help show this help message and exit + --root ROOT frozen source root + --input INPUT JSON request; see README.md diff --git a/evals/source-identity-v1/contract.json b/evals/source-identity-v1/contract.json new file mode 100644 index 0000000..b9d97c7 --- /dev/null +++ b/evals/source-identity-v1/contract.json @@ -0,0 +1,31 @@ +{ + "contract": "source-identity-v1", + "schema_version": 1, + "adoption": "prospective_explicit_only", + "identity": [ + "language", + "file", + "line", + "column", + "kind", + "module", + "owner", + "trait", + "leaf" + ], + "rust_dot_spelling": "rejected_use_double_colon", + "ambiguity": "resolve_spelling_over_complete_selected_index_before_location_filter", + "citation": "exact_complete_lines_in_declaration_span_including_name_line_no_final_newline", + "python_assignment": "single_module_name_with_literal_scalar_or_recursive_tuple_value", + "rust_support": "syn_2.0.119_syntax_validated_bounded_declaration_index_no_expansion_or_typecheck", + "semantic_review": "external_pending_never_inferred_from_identity_or_citation", + "source_encoding": "utf8_lf_only", + "limits": { + "source_bytes_per_file": 1048576, + "rust_files": 128, + "python_files": 128 + }, + "dependencies": "python_3.10_or_newer_and_offline_built_syntax_frontend_locked_in_syntax/Cargo.lock", + "rust_uncertainty": "named_unsupported_or_cfg_bindings_block_ties; unknown_binding_effects_or_syntax_errors_refuse_rust_index", + "rust_generic_identity": "written_declaration_with_erased_owner_and_trait_arguments; competing_specializations_remain_ambiguous" +} diff --git a/evals/source-identity-v1/example-result.json b/evals/source-identity-v1/example-result.json new file mode 100644 index 0000000..da5b925 --- /dev/null +++ b/evals/source-identity-v1/example-result.json @@ -0,0 +1,156 @@ +{ + "citation_recall": { + "matched": 1, + "required": 3 + }, + "diagnostics": [ + { + "file": "rust/lib.rs", + "line": 5, + "name": "Err", + "reason": "named binding is outside the scored declaration kinds" + }, + { + "file": "python/settings.py", + "line": 5, + "name": "DYNAMIC", + "reason": "only module-level literal assignments supported; dynamic/alias values are uncertain" + }, + { + "file": "python/settings.py", + "line": 6, + "name": "ALIAS", + "reason": "only module-level literal assignments supported; dynamic/alias values are uncertain" + } + ], + "identity_recall": { + "matched": 2, + "required": 3 + }, + "items": [ + { + "citation_ok": true, + "citation_reason": null, + "classification": "required", + "identity": { + "column": 8, + "file": "rust/lib.rs", + "kind": "fn", + "language": "rust", + "leaf": "from_str", + "line": 6, + "module": "crate", + "owner": "crate::Compass", + "trait": "std::str::FromStr" + }, + "identity_ok": true, + "reason": null, + "selector": { + "citation": { + "end_line": 6, + "quote": " fn from_str(text: &str) -> Result { todo!() }", + "start_line": 6 + }, + "file": "rust/lib.rs", + "kind": "fn", + "language": "rust", + "line": 6, + "name": "::from_str" + } + }, + { + "citation_ok": false, + "citation_reason": "quote differs from exact complete source lines", + "classification": "required", + "identity": { + "column": 1, + "file": "python/settings.py", + "kind": "assignment", + "language": "python", + "leaf": "RETRY_LIMIT", + "line": 2, + "module": "python.settings", + "owner": null, + "trait": null + }, + "identity_ok": true, + "reason": null, + "selector": { + "citation": { + "end_line": 2, + "quote": "RETRY_LIMIT = 8", + "start_line": 2 + }, + "file": "python/settings.py", + "kind": "assignment", + "language": "python", + "line": 2, + "name": "RETRY_LIMIT" + } + }, + { + "citation_ok": false, + "citation_reason": "identity unresolved", + "classification": "uncredited", + "identity": null, + "identity_ok": false, + "reason": "only module-level literal assignments supported; dynamic/alias values are uncertain", + "selector": { + "citation": { + "end_line": 6, + "quote": "ALIAS = RETRY_LIMIT", + "start_line": 6 + }, + "file": "python/settings.py", + "kind": "assignment", + "language": "python", + "line": 6, + "name": "ALIAS" + } + } + ], + "missing": [ + { + "column": 12, + "file": "rust/lib.rs", + "kind": "fn", + "language": "rust", + "leaf": "parse", + "line": 12, + "module": "crate", + "owner": "crate::Ledger", + "trait": null + } + ], + "pins": { + "contract": "source-identity-v1", + "contract_sha256": "21fee7101c9bf93b5cad7b30982d48f992b864fb9a8c1179f91db63031e02779", + "implementation_sha256": { + "rust_source.py": "d42926bce54478739e3a52841b2f003b32885f5cc7c602bd11ad904d7840e09a", + "source_identity.py": "8fadf0f18f1642f3d0b7e9185a937a76bfeda1daddba0adadf584dff9a9c52fa", + "syntax/Cargo.lock": "dd766e08c25fbe96b4f1da71430997de006aee962fbfc5cfb76167431b478d2e", + "syntax/Cargo.toml": "8c3ab6d15e99a3142d6dcafef97dc3f4e3a4b1170a076bac53ebeee07ed2f4b7", + "syntax/src/main.rs": "b346d262b6911858e82ecb78d50c7f94c214a137188f2222ae6abc4208fe88e9" + }, + "manifest": { + "python_files": [ + "python/settings.py" + ], + "rust_root": "rust/lib.rs" + }, + "python_ast_version": "3.12", + "rust_syntax_runtime": "syn-2.0.119/proc-macro2-1.0.106; dependencies pinned by syntax/Cargo.lock", + "source_sha256": { + "python/settings.py": "9cfc03e06fa615f0fbef187c94426ea635d415be756fe9a063fa18e2358c322c", + "rust/lib.rs": "e9d11e6094576e0011160f5696ec2afcc8e784c07bc6749f6e5ebbc7d540e398", + "rust/relocated.rs": "b5f4b84461eedd5e017af1b53b47fdcf2e83cb3a8cd0e448352fb5f7baefd089" + } + }, + "schema_version": 1, + "semantic_review": { + "reason": "independent human/source review required", + "result": null, + "status": "pending" + }, + "unsupported_sources": {} +} diff --git a/evals/source-identity-v1/example.py b/evals/source-identity-v1/example.py new file mode 100644 index 0000000..fbbea39 --- /dev/null +++ b/evals/source-identity-v1/example.py @@ -0,0 +1,37 @@ +"""Runnable test-only scoring example. No providers and no historical imports.""" +import json +from pathlib import Path +import sys + +from source_identity import score_request + +ROOT = Path(__file__).resolve().parent / "fixtures" + + +def selector(language, name, file, line, kind, quote): + return {"language": language, "name": name, "file": file, "line": line, + "kind": kind, "citation": {"start_line": line, "end_line": line, "quote": quote}} + + +def request(): + conversion = selector("rust", "Compass::from_str", "rust/lib.rs", 6, "fn", + " fn from_str(text: &str) -> Result { todo!() }") + constant = selector("python", "RETRY_LIMIT", "python/settings.py", 2, + "assignment", "RETRY_LIMIT = 7") + omitted = selector("rust", "Ledger::parse", "rust/lib.rs", 12, "fn", + " pub fn parse(value: u8) -> u8 { value }") + uncertain = selector("python", "ALIAS", "python/settings.py", 6, + "assignment", "ALIAS = RETRY_LIMIT") + return {"schema_version": 1, + "manifest": {"rust_root": "rust/lib.rs", "python_files": ["python/settings.py"]}, + "required": [conversion, constant, omitted], + "submitted": [conversion | {"name": "::from_str"}, + constant | {"citation": constant["citation"] | {"quote": "RETRY_LIMIT = 8"}}, + uncertain]} + + +if __name__ == "__main__": + if sys.argv[1:] not in ([], ["--request"]): + raise SystemExit("usage: python3 -B example.py [--request]") + output = request() if sys.argv[1:] else score_request(ROOT, request()) + print(json.dumps(output, sort_keys=True, indent=2)) diff --git a/evals/source-identity-v1/fixtures/python/settings.py b/evals/source-identity-v1/fixtures/python/settings.py new file mode 100644 index 0000000..d37fe77 --- /dev/null +++ b/evals/source-identity-v1/fixtures/python/settings.py @@ -0,0 +1,16 @@ +"""Synthetic source; this module must never be imported by the scorer.""" +RETRY_LIMIT = 7 +WINDOW: int = 12 +LABELS = ("north", "south") +DYNAMIC = compute_limit() +ALIAS = RETRY_LIMIT + +def decode(value): + return value + +class Packet: + def decode(self, value): + return value + +# def counterfeit(): pass +TEXT = "def counterfeit(): pass" diff --git a/evals/source-identity-v1/fixtures/rust/lib.rs b/evals/source-identity-v1/fixtures/rust/lib.rs new file mode 100644 index 0000000..402af09 --- /dev/null +++ b/evals/source-identity-v1/fixtures/rust/lib.rs @@ -0,0 +1,28 @@ +use std::str::FromStr; +pub struct Compass; +pub struct Ledger; +impl FromStr for Compass { + type Err = (); + fn from_str(text: &str) -> Result { todo!() } +} +impl Compass { + pub fn parse(value: T) -> T { value } +} +impl Ledger { + pub fn parse(value: u8) -> u8 { value } +} +pub fn parse(value: T) -> T { value } +pub mod nested { + pub struct Dial; + impl Dial { + pub fn turn() {} + } + pub fn parse() {} +} +#[path = "relocated.rs"] +pub mod wire; +// fn counterfeit() {} +pub fn strings() { + let text = r###"impl Ghost { fn counterfeit() {} }"###; + /* outer /* fn counterfeit() {} */ comment */ +} diff --git a/evals/source-identity-v1/fixtures/rust/relocated.rs b/evals/source-identity-v1/fixtures/rust/relocated.rs new file mode 100644 index 0000000..d414690 --- /dev/null +++ b/evals/source-identity-v1/fixtures/rust/relocated.rs @@ -0,0 +1,4 @@ +pub struct Socket; +impl Socket { + pub fn connect() {} +} diff --git a/evals/source-identity-v1/rust_source.py b/evals/source-identity-v1/rust_source.py new file mode 100644 index 0000000..f907fd5 --- /dev/null +++ b/evals/source-identity-v1/rust_source.py @@ -0,0 +1,204 @@ +"""Syntax-validated Rust declaration index; no compilation or macro expansion.""" +import json +from pathlib import Path +import subprocess + +HERE = Path(__file__).resolve().parent +PARSER_SOURCES = ("syntax/Cargo.toml", "syntax/Cargo.lock", "syntax/src/main.rs") +PARSER = HERE.parents[1] / ".orbit/tmp/source-identity-syntax-target/debug/source-identity-syntax" + + +class UnsupportedRust(ValueError): + pass + + +def parse(source): + try: + process = subprocess.run([str(PARSER)], input=source, text=True, encoding="utf-8", + capture_output=True, timeout=10, env={}) + if process.returncode or process.stderr: + raise UnsupportedRust("Rust syntax frontend failed: " + process.stderr[:500]) + output = json.loads(process.stdout) + if "error" in output: + raise UnsupportedRust(output["error"]) + expected = {name: (HERE / name).read_text(encoding="utf-8") for name in PARSER_SOURCES} + if output.get("sources") != expected: + raise UnsupportedRust("stale syntax frontend; rebuild offline using README instructions") + return output["items"] + except (OSError, subprocess.TimeoutExpired, ValueError, KeyError) as error: + raise UnsupportedRust("Rust syntax frontend unavailable/invalid: " + str(error)) from error + + +class RustIndex: + def __init__(self, read, root): + self.root = root + self.read = read + self.declarations = [] + self.imports = {} + self.modules = set() + self.files = set() + self.pending = [] + self.diagnostics = [] + self.parse_file(root, (), root.rsplit("/", 1)[0] if "/" in root else "") + self.finish() + self.diagnostics = [{"file": d["file"], "line": d["line"], "name": d["leaf"], + "reason": d["unsupported"]} for d in self.declarations if d.get("unsupported")] + + def parse_file(self, file, module, module_dir, uncertain=None): + if file in self.files or len(self.files) >= 128: + raise UnsupportedRust("reused/cyclic module file or module limit exceeded") + self.files.add(file) + self.scope(parse(self.read(file)), file, module, module_dir, uncertain=uncertain) + + def scope(self, nodes, file, module, module_dir, context=None, uncertain=None): + for node in nodes: + kind = node["kind"] + reason = uncertain or node.get("unsupported") + if kind in ("fn", "struct", "enum", "trait", "unsupported"): + item = {"language": "rust", "file": file, "line": node["line"], + "column": node["column"], "end_line": node["end_line"], "kind": kind, + "module": "::".join(("crate",) + module), "owner": None, + "trait": None, "leaf": node["name"], "unsupported": reason} + if kind == "unsupported": + item["unsupported"] = reason or "named binding is outside the scored declaration kinds" + self.declarations.append(item) + if context: + self.pending.append((item, module, context)) + if kind == "trait": + self.scope(node["children"], file, module, module_dir, + ("trait", node["name"], None), reason) + elif kind == "impl": + self.scope(node["children"], file, module, module_dir, + ("impl", node["owner"], node["trait"]), reason) + elif kind == "use": + key = (module, node["name"]) + if key in self.imports or uncertain: + raise UnsupportedRust("ambiguous/conditional import binding") + self.imports[key] = node["target"] + elif kind == "mod": + name = node["name"] + child = module + (name,) + if child in self.modules: + raise UnsupportedRust("duplicate/conditional module binding") + self.modules.add(child) + if node["children"] is not None: + if node["path"]: + raise UnsupportedRust("path on inline module unsupported") + self.scope(node["children"], file, child, self.join(module_dir, name), uncertain=reason) + else: + if not node["path"]: + raise UnsupportedRust("external modules require literal #[path]") + if file != self.root: + raise UnsupportedRust("external modules declared inside external files are not supported") + child_file = self.join(module_dir, node["path"]) + child_dir = (child_file.rsplit("/", 1)[0] if child_file.endswith("/mod.rs") + else child_file.removesuffix(".rs")) + if not child_file.endswith(".rs"): + raise UnsupportedRust("module path must end in .rs") + self.parse_file(child_file, child, child_dir, reason) + else: + raise UnsupportedRust("unknown syntax frontend node") + + @staticmethod + def join(parent, child): + return parent + "/" + child if parent else child + + def qualify(self, path, module, seen=()): + parts = path.split("::") + if parts[0] == "crate": + return path + if parts[0] in ("std", "core"): + return path + if parts[0] == "self": + return "::".join(("crate",) + module + tuple(parts[1:])) + if parts[0] == "super": + base = list(module) + while parts and parts[0] == "super": + if not base: + raise UnsupportedRust("super escapes crate") + base.pop() + parts.pop(0) + return "::".join(["crate", *base, *parts]) + key = (module, parts[0]) + if key in self.imports: + if key in seen: + raise UnsupportedRust("cyclic import aliases") + imported = self.qualify(self.imports[key], module, (*seen, key)) + return "::".join([imported, *parts[1:]]) + return "::".join(("crate",) + module + tuple(parts)) + + def finish(self): + reserved = {"std", "core"} + if (any(module[-1] in reserved for module in self.modules) + or any(alias in reserved for _, alias in self.imports) + or any(item["leaf"] in reserved and item["kind"] != "fn" + for item in self.declarations)): + raise UnsupportedRust("shadowed std/core roots make external trait identity uncertain") + types = {} + associated = {id(item) for item, _, _ in self.pending} + for item in self.declarations: + if (item["kind"] in ("struct", "enum", "trait", "unsupported") + and id(item) not in associated): + path = item["module"] + "::" + item["leaf"] + if path in types or (tuple(item["module"].split("::")[1:]), item["leaf"]) in self.imports: + raise UnsupportedRust("duplicate type/import binding") + types[path] = item["kind"] + uncertain_types = {d["module"] + "::" + d["leaf"]: d["unsupported"] + for d in self.declarations if d.get("unsupported") and id(d) not in associated} + owner_bindings = {} + for path in types: + owner_bindings.setdefault(path.split("::")[-1], set()).add(path) + for (scope, alias), _ in self.imports.items(): + target = self.qualify(alias, scope) + if target not in types and not target.startswith(("std::", "core::")): + raise UnsupportedRust("only verified local type or std/core imports are supported") + # Imported names remain blockers even when no local method uses them. + owner_bindings.setdefault(alias, set()).add(target) + prefix = "::".join(("crate",) + scope) + if any(d["module"] == prefix and d["leaf"] == alias and d["owner"] is None + for d in self.declarations): + raise UnsupportedRust("import collides with a declaration binding") + for item, module, (mode, owner, trait) in self.pending: + owner = self.qualify(owner, module) + if mode == "trait": + item["owner"], item["trait"] = owner, owner + continue + if types.get(owner) not in ("struct", "enum"): + raise UnsupportedRust(f"impl owner {owner} is not a verified local type") + item["owner"] = owner + if owner in uncertain_types: + item["unsupported"] = uncertain_types[owner] + if trait: + trait = self.qualify(trait, module) + if types.get(trait) != "trait" and not trait.startswith(("std::", "core::")): + raise UnsupportedRust(f"unverified trait binding {trait}") + item["trait"] = trait + if trait in uncertain_types: + item["unsupported"] = uncertain_types[trait] + for item in self.declarations: + module = tuple(item["module"].split("::")[1:]) + owner = item["owner"] + full = (owner or item["module"]) + "::" + item["leaf"] + names = {full, full.removeprefix("crate::")} + if owner: + owners = {owner, owner.removeprefix("crate::")} + owner_leaf = owner.split("::")[-1] + if owner_bindings.get(owner_leaf) == {owner}: + owners.add(owner_leaf) + for (scope, alias) in self.imports: + if scope == module and owner_bindings.get(alias) == {owner}: + owners.add(alias) + names.update(name + "::" + item["leaf"] for name in owners) + if item["trait"]: + trait = item["trait"] + traits = {trait, trait.removeprefix("crate::")} + leaf = trait.split("::")[-1] + if self.qualify(leaf, module) == trait: + traits.add(leaf) + for (scope, alias) in self.imports: + if scope == module and self.qualify(alias, module) == trait: + traits.add(alias) + names.update(f"<{o} as {t}>::{item['leaf']}" for o in owners for t in traits) + else: + names.add(item["leaf"]) + item["spellings"] = names diff --git a/evals/source-identity-v1/source_identity.py b/evals/source-identity-v1/source_identity.py new file mode 100644 index 0000000..420836d --- /dev/null +++ b/evals/source-identity-v1/source_identity.py @@ -0,0 +1,355 @@ +"""Prospective source identity and citation contract. No source imports/execution.""" +import argparse +import ast +from collections import Counter +import hashlib +import json +from pathlib import Path, PurePosixPath +import re +import sys + +from rust_source import RustIndex, UnsupportedRust, PARSER_SOURCES + +HERE = Path(__file__).resolve().parent +CONTRACT = json.loads((HERE / "contract.json").read_text(encoding="utf-8")) +IDENTITY_FIELDS = CONTRACT["identity"] + + +class ContractError(ValueError): + """Invalid evaluator inputs/truth, as distinct from an uncredited answer.""" + + +def sha256(data): + return hashlib.sha256(data).hexdigest() + + +def canonical_path(value): + if not isinstance(value, str) or not value or "\\" in value or "\x00" in value: + raise ContractError("expected a canonical relative POSIX file path") + path = PurePosixPath(value) + if path.is_absolute() or any(p in (".", "..") for p in value.split("/")) or str(path) != value: + raise ContractError("file path must be relative, normalized and contained") + return path + + +def identity(item): + return {field: item[field] for field in IDENTITY_FIELDS} + + +def identity_key(item): + return json.dumps(item, sort_keys=True, separators=(",", ":")) + + +def literal(node): + # No ast.literal_eval: recognize syntax without constructing large values. + if isinstance(node, ast.Constant): + return type(node.value) in (str, bytes, int, float, complex, bool, type(None)) + if isinstance(node, ast.Tuple): + return all(literal(child) for child in node.elts) + if isinstance(node, ast.UnaryOp) and isinstance(node.op, (ast.UAdd, ast.USub)): + return isinstance(node.operand, ast.Constant) and type(node.operand.value) in (int, float, complex) + return False + + +def python_index(text, file): + """Index a closed module syntax; uncertain top-level effects reject the module.""" + tree = ast.parse(text, filename=file) + module = file.removesuffix(".py").replace("/", ".") + if module.endswith(".__init__"): + module = module.removesuffix(".__init__") + if not module or any(not re.fullmatch(r"[A-Za-z_]\w*", x, re.ASCII) for x in module.split(".")): + raise ContractError("Python module path must consist of identifier components") + declarations, diagnostics = [], [] + + def scope(body, owner=None): + bindings = [] + candidates = [] + for node in body: + name, kind, reason = None, None, None + if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef, ast.ClassDef)): + name = node.name + kind = "class" if isinstance(node, ast.ClassDef) else "function" + if node.decorator_list: + reason = "decorated declarations have uncertain runtime identity" + elif isinstance(node, ast.ClassDef) and (node.bases or node.keywords): + reason = "class inheritance/metaclass semantics unsupported" + elif isinstance(node, (ast.Import, ast.ImportFrom)) and owner is None: + for imported in node.names: + if imported.name == "*": + raise ContractError("wildcard import leaves the module binding set unknown") + bound = imported.asname or (imported.name.split(".")[0] + if isinstance(node, ast.Import) else imported.name) + bindings.append(bound) + candidates.append((node, bound, "import", "import aliases are not source declarations in this contract")) + continue + elif isinstance(node, ast.Assign): + if len(node.targets) == 1 and isinstance(node.targets[0], ast.Name): + name, kind = node.targets[0].id, "assignment" + if owner or not literal(node.value): + reason = "only module-level literal assignments supported; dynamic/alias values are uncertain" + else: + raise ContractError("destructuring/chained/attribute assignments are unsupported module effects") + elif isinstance(node, ast.AnnAssign) and isinstance(node.target, ast.Name): + name, kind = node.target.id, "assignment" + if owner or node.value is None or not literal(node.value): + reason = "annotated assignment needs a module-level literal value" + if not isinstance(node.annotation, (ast.Name, ast.Constant)): + reason = "executable/complex annotations unsupported" + elif isinstance(node, ast.Expr) and isinstance(node.value, ast.Constant) and isinstance(node.value.value, str): + continue + elif isinstance(node, ast.Pass): + continue + else: + raise ContractError("unsupported Python module/class statement; conditionals and mutation may change bindings") + bindings.append(name) + candidates.append((node, name, kind, reason)) + counts = Counter(bindings) + for node, name, kind, reason in candidates: + if counts[name] != 1: + reason = "duplicate/rebound Python declaration" + if reason: + diagnostics.append({"file": file, "line": node.lineno, "name": name, "reason": reason}) + if kind == "class": + raise ContractError("unsupported class binding may hide method ownership: " + reason) + start = node.lineno + item = {"language": "python", "file": file, "line": start, + "column": node.col_offset + 1, "end_line": node.end_lineno, + "kind": kind, "module": module, "owner": owner, "trait": None, + "leaf": name} + qualified = (owner or module) + "." + name + item["spellings"] = {qualified, (owner.removeprefix(module + ".") + "." + name) if owner else name} + item["unsupported"] = reason + declarations.append(item) + if kind == "class": + scope(node.body, qualified) + scope(tree.body) + return declarations, diagnostics + + +class Project: + """One explicit source universe, read once and hashed from the parsed bytes. + + The caller freezes the manifest before collecting answers. Files outside that + manifest cannot be cited, and answers never choose a smaller ambiguity universe. + """ + def __init__(self, root, *, rust_root=None, python_files=()): + self.root = Path(root).resolve(strict=True) + if not self.root.is_dir(): + raise ContractError("root is not a directory") + self.sources = {} + self.declarations = [] + self.diagnostics = [] + self.failures = {} + if rust_root is not None: + canonical_path(rust_root) + if not rust_root.endswith(".rs"): + raise ContractError("rust_root must name an .rs file") + try: + index = RustIndex(self.read, rust_root) + self.declarations.extend(index.declarations) + self.diagnostics.extend(index.diagnostics) + except (UnsupportedRust, ContractError, RecursionError) as error: + self.failures["rust"] = str(error) + if not isinstance(python_files, (list, tuple)) or len(python_files) > 128: + raise ContractError("python_files must be an array of at most 128 paths") + if any(not isinstance(file, str) for file in python_files): + raise ContractError("python_files entries must be strings") + if len(set(python_files)) != len(python_files): + raise ContractError("duplicate Python manifest path") + python_declarations = [] + for file in python_files: + canonical_path(file) + if not file.endswith(".py"): + raise ContractError("python_files must name .py files") + try: + declarations, diagnostics = python_index(self.read(file), file) + python_declarations.extend(declarations) + self.diagnostics.extend(diagnostics) + except (SyntaxError, ContractError, RecursionError) as error: + self.failures["python"] = f"{file}: {error}" + # Any incomplete language index cannot safely prove name uniqueness. + if "python" not in self.failures: + bindings = Counter(name for item in python_declarations for name in item["spellings"]) + for item in python_declarations: + if item["owner"]: + short = item["owner"].removeprefix(item["module"] + ".") + parts = short.split(".") + if any(bindings[".".join(parts[:n])] != 1 for n in range(1, len(parts) + 1)): + item["spellings"].discard(short + "." + item["leaf"]) + self.declarations.extend(python_declarations) + self.manifest = {"rust_root": rust_root, "python_files": list(python_files)} + + def read(self, file): + canonical_path(file) + if file in self.sources: + return self.sources[file]["text"] + path = self.root + for part in PurePosixPath(file).parts: + path = path / part + if path.is_symlink(): + raise ContractError(f"symlink source paths unsupported: {file}") + try: + path.resolve(strict=True).relative_to(self.root) + with path.open("rb") as handle: + raw = handle.read(CONTRACT["limits"]["source_bytes_per_file"] + 1) + except (OSError, ValueError) as error: + raise ContractError(f"cannot read contained source {file}: {error}") from error + if len(raw) > CONTRACT["limits"]["source_bytes_per_file"]: + raise ContractError(f"source size limit exceeded: {file}") + try: + text = raw.decode("utf-8") + except UnicodeDecodeError as error: + raise ContractError(f"source must be UTF-8: {file}") from error + if "\r" in text or "\x00" in text: + raise ContractError(f"source must use LF without NUL: {file}") + self.sources[file] = {"text": text, "sha256": sha256(raw)} + return text + + def check(self, selector): + result = {"identity_ok": False, "citation_ok": False, "identity": None, + "reason": None, "citation_reason": "identity unresolved"} + try: + if not isinstance(selector, dict) or set(selector) != {"language", "name", "file", "line", "kind", "citation"}: + raise ContractError("selector requires exactly language/name/file/line/kind/citation") + language = selector["language"] + if language not in ("rust", "python"): + raise ContractError("unsupported language") + canonical_path(selector["file"]) + if type(selector["line"]) is not int or selector["line"] < 1: + raise ContractError("line must be a positive integer") + name = selector["name"] + if not isinstance(name, str) or len(name) > 1024: + raise ContractError("name must be a string of at most 1024 characters") + if language == "rust": + path = r"[A-Za-z_]\w*(?:::[A-Za-z_]\w*)*" + if not re.fullmatch(rf"(?:{path}|<{path} as {path}>::[A-Za-z_]\w*)", name, re.ASCII): + raise ContractError("malformed Rust spelling; use ::, dot spellings are not admitted") + elif not re.fullmatch(r"[A-Za-z_]\w*(?:\.[A-Za-z_]\w*)*", name, re.ASCII): + raise ContractError("malformed Python spelling") + if not isinstance(selector["kind"], str): + raise ContractError("kind must be a string") + if language in self.failures: + raise ContractError("unsupported source context: " + self.failures[language]) + candidates = [d for d in self.declarations if d["language"] == language and name in d["spellings"]] + if len(candidates) != 1: + reason = "ambiguous spelling" if candidates else "unresolved or unsupported declaration" + raise ContractError(reason) + declaration = candidates[0] + if declaration.get("unsupported"): + raise ContractError(declaration["unsupported"]) + if any(selector[key] != declaration[key] for key in ("file", "line", "kind")): + raise ContractError("spelling resolves to a different file, declaration line or kind") + result.update(identity_ok=True, identity=identity(declaration)) + result["citation_ok"], result["citation_reason"] = self.citation(declaration, selector["citation"]) + except (ContractError, TypeError) as error: + result["reason"] = str(error) + return result + + def citation(self, declaration, citation): + if not isinstance(citation, dict) or set(citation) != {"start_line", "end_line", "quote"}: + return False, "citation requires exactly start_line/end_line/quote" + start, end = citation["start_line"], citation["end_line"] + if type(start) is not int or type(end) is not int or not isinstance(citation["quote"], str): + return False, "citation line bounds must be integers and quote a string" + if not (start == declaration["line"] <= end <= declaration["end_line"]): + return False, "citation must start at declaration line and remain in its span" + lines = self.sources[declaration["file"]]["text"].split("\n") + if citation["quote"] != "\n".join(lines[start - 1:end]): + return False, "quote differs from exact complete source lines" + return True, None + + def pins(self): + return {"contract": CONTRACT["contract"], + "contract_sha256": sha256((HERE / "contract.json").read_bytes()), + "implementation_sha256": {name: sha256((HERE / name).read_bytes()) + for name in ("source_identity.py", "rust_source.py", *PARSER_SOURCES)}, + "source_sha256": {name: value["sha256"] for name, value in sorted(self.sources.items())}, + "manifest": self.manifest, + "rust_syntax_runtime": "syn-2.0.119/proc-macro2-1.0.106; dependencies pinned by syntax/Cargo.lock", + "python_ast_version": f"{sys.version_info.major}.{sys.version_info.minor}"} + + +def score(project, required, submitted): + if not isinstance(required, list) or not required or not isinstance(submitted, list): + raise ContractError("required must be a nonempty array and submitted an array") + truth = {} + for selector in required: + checked = project.check(selector) + if not checked["identity_ok"] or not checked["citation_ok"]: + raise ContractError("invalid required identity/evidence: " + json.dumps(checked, sort_keys=True)) + key = identity_key(checked["identity"]) + if key in truth: + raise ContractError("duplicate required canonical identity") + truth[key] = checked["identity"] + found, cited, items = set(), set(), [] + for selector in submitted: + checked = project.check(selector) + key = identity_key(checked["identity"]) + classification = "uncredited" + if checked["identity_ok"]: + classification = "required" if key in truth else "verified_extra" + if key in truth: + found.add(key) + if checked["citation_ok"]: + cited.add(key) + items.append({"selector": selector, "classification": classification, **checked}) + return {"schema_version": 1, "pins": project.pins(), + "identity_recall": {"matched": len(found), "required": len(truth)}, + "citation_recall": {"matched": len(cited), "required": len(truth)}, + "missing": [value for key, value in truth.items() if key not in found], + "items": items, "diagnostics": project.diagnostics, + "unsupported_sources": project.failures, + "semantic_review": {"status": "pending", "result": None, + "reason": "independent human/source review required"}} + + +def score_request(root, request): + if not isinstance(request, dict) or set(request) != {"schema_version", "manifest", "required", "submitted"}: + raise ContractError("request requires exactly schema_version/manifest/required/submitted") + if type(request["schema_version"]) is not int or request["schema_version"] != 1: + raise ContractError("unsupported request schema_version") + manifest = request["manifest"] + if not isinstance(manifest, dict) or set(manifest) != {"rust_root", "python_files"}: + raise ContractError("manifest requires exactly rust_root/python_files") + return score(Project(root, **manifest), request["required"], request["submitted"]) + + +def parse_request(text): + def pairs(entries): + result = {} + for key, value in entries: + if key in result: + raise ContractError("duplicate JSON field: " + key) + result[key] = value + return result + + def constant(value): + raise ContractError("nonfinite JSON number: " + value) + + return json.loads(text, object_pairs_hook=pairs, parse_constant=constant) + + +def main(): + parser = argparse.ArgumentParser(prog="source_identity.py", description="Score prospective source identities and exact citations; JSON output only.") + parser.add_argument("--root", required=True, type=Path, help="frozen source root") + parser.add_argument("--input", required=True, type=Path, help="JSON request; see README.md") + args = parser.parse_args() + try: + request = parse_request(args.input.read_text(encoding="utf-8")) + result = score_request(args.root, request) + except (OSError, ValueError, TypeError, RecursionError) as error: + print(json.dumps({"schema_version": 1, "code": "invalid_input", "error": str(error)}), file=sys.stderr) + return 2 + try: + print(json.dumps(result, indent=2, sort_keys=True, ensure_ascii=True)) + sys.stdout.flush() + except BrokenPipeError: + try: + sys.stdout.close() + except BrokenPipeError: + pass + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/evals/source-identity-v1/syntax/Cargo.lock b/evals/source-identity-v1/syntax/Cargo.lock new file mode 100644 index 0000000..87c4168 --- /dev/null +++ b/evals/source-identity-v1/syntax/Cargo.lock @@ -0,0 +1,139 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 4 + +[[package]] +name = "itoa" +version = "1.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682" + +[[package]] +name = "memchr" +version = "2.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf8baf1c55e62ffcace7a9f06f4bd9cd3f0c4beb022d3b367256b91b87513d98" + +[[package]] +name = "proc-macro2" +version = "1.0.106" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8fd00f0bb2e90d81d1044c2b32617f68fcb9fa3bb7640c23e9c748e53fb30934" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "quote" +version = "1.0.47" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fbf4db142a473a8d80c26bbf18454ed458bf8d26c8219c331daecfdbd079001" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "serde" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4148590afebada386688f18773da617792bf2ef03ffc1e4cbd2b1d45b023e0ba" +dependencies = [ + "serde_core", +] + +[[package]] +name = "serde_core" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "67dca2c9c51e58a4791a4b1ed58308b39c64224d349a935ab5039aa360942a48" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_derive" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e7a5d71263a5a7d47b41f6b3f06ba276f10cc18b0931f1799f710578e2309348" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.6", +] + +[[package]] +name = "serde_json" +version = "1.0.150" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e8014e44b4736ed0538adeecded0fce2a272f22dc9578a7eb6b2d9993c74cfb9" +dependencies = [ + "itoa", + "memchr", + "serde", + "serde_core", + "zmij", +] + +[[package]] +name = "source-identity-syntax" +version = "0.1.0" +dependencies = [ + "proc-macro2", + "serde_json", + "syn 2.0.119", + "thiserror", +] + +[[package]] +name = "syn" +version = "2.0.119" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "872831b642d1a07999a962a351ed35b955ea2cfc8f3862091e2a240a84f17297" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "syn" +version = "3.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8593e8e72159ed2257d083c7a454a85cbf854f37a0966d8d483aff8c8a3ebcee" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "thiserror" +version = "2.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4288b5bcbc7920c07a1149a35cf9590a2aa808e0bc1eafaade0b80947865fbc4" +dependencies = [ + "thiserror-impl", +] + +[[package]] +name = "thiserror-impl" +version = "2.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ebc4ee7f67670e9b64d05fa4253e753e016c6c95ff35b89b7941d6b856dec1d5" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "unicode-ident" +version = "1.0.26" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d245f478577f809a851594d02313b640fb437e0bb33866753cff937863096954" + +[[package]] +name = "zmij" +version = "1.0.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29666d0abbfad1e3dc4dcf6144730dd3a3ab225bbbdac83319345b1b44ccfc1b" diff --git a/evals/source-identity-v1/syntax/Cargo.toml b/evals/source-identity-v1/syntax/Cargo.toml new file mode 100644 index 0000000..506108c --- /dev/null +++ b/evals/source-identity-v1/syntax/Cargo.toml @@ -0,0 +1,16 @@ +[package] +name = "source-identity-syntax" +version = "0.1.0" +edition = "2021" +rust-version = "1.89" +license = "MIT OR Apache-2.0" +publish = false + +# Isolated prospective evaluator, never a product workspace dependency. +[workspace] + +[dependencies] +syn = { version = "=2.0.119", features = ["full", "visit"] } +proc-macro2 = { version = "=1.0.106", features = ["span-locations"] } +serde_json = "=1.0.150" +thiserror = "=2.0.18" diff --git a/evals/source-identity-v1/syntax/src/main.rs b/evals/source-identity-v1/syntax/src/main.rs new file mode 100644 index 0000000..5d85fc3 --- /dev/null +++ b/evals/source-identity-v1/syntax/src/main.rs @@ -0,0 +1,289 @@ +//! Offline syntax only. Input bytes are parsed, never compiled or expanded. +use serde_json::{json, Value}; +use std::io::{self, Read, Write}; +use syn::{spanned::Spanned, visit::Visit, Attribute, Ident, Item, Path, Type}; + +#[derive(Debug, thiserror::Error)] +enum Error { + #[error("Rust syntax error: {0}")] + Syntax(#[from] syn::Error), + #[error("unsupported Rust context: {0}")] + Unsupported(String), + #[error("source read failed: {0}")] + Read(#[from] io::Error), +} +impl From<&str> for Error { + fn from(message: &str) -> Self { + Self::Unsupported(message.into()) + } +} +impl From for Error { + fn from(message: String) -> Self { + Self::Unsupported(message) + } +} +type Result = std::result::Result; + +fn path(path: &Path) -> Result { + if path.leading_colon.is_some() { + return Err("absolute extern paths are unsupported".into()); + } + Ok(path + .segments + .iter() + .map(|p| p.ident.to_string()) + .collect::>() + .join("::")) +} + +// Only attributes whose binding effects we explicitly understand are admitted. +// cfg preserves a named blocker; arbitrary attribute/derive macros can emit new +// bindings and therefore cannot be reduced to a blocker for the written name. +fn attributes(attrs: &[Attribute]) -> Result<(Option, Option)> { + let mut uncertain = None; + let mut module_path = None; + for attr in attrs { + let name = path(attr.path())?; + match name.as_str() { + "cfg" => uncertain = Some("conditional declaration (cfg is not evaluated)".into()), + "path" => { + if module_path.is_some() { + return Err("duplicate path attribute".into()); + } + if let syn::Meta::NameValue(meta) = &attr.meta { + if let syn::Expr::Lit(lit) = &meta.value { + if let syn::Lit::Str(value) = &lit.lit { + module_path = Some(value.value()); + continue; + } + } + } + return Err("module path must be a literal string".into()); + } + "derive" => { + let derives = attr + .parse_args_with( + syn::punctuated::Punctuated::::parse_terminated, + ) + .map_err(Error::Syntax)?; + for derive in derives { + // Standard derives generate trait impls, not scored written methods. + if !matches!( + path(&derive)?.as_str(), + "Clone" + | "Copy" + | "Debug" + | "Default" + | "Eq" + | "PartialEq" + | "Ord" + | "PartialOrd" + | "Hash" + ) { + return Err("custom derive can generate unknown bindings".into()); + } + } + } + "doc" | "allow" | "warn" | "deny" | "forbid" | "expect" | "repr" | "inline" + | "cold" | "must_use" | "deprecated" | "non_exhaustive" | "no_mangle" + | "export_name" | "link_section" | "track_caller" => {} + _ => { + return Err(format!("unsupported attribute {name}; binding effects unknown").into()) + } + } + } + Ok((uncertain, module_path)) +} + +fn named(kind: &str, name: &Ident, span: proc_macro2::Span, attrs: &[Attribute]) -> Result { + let (uncertain, module_path) = attributes(attrs)?; + if kind != "mod" && module_path.is_some() { + return Err("path attribute requires an external module".into()); + } + let start = name.span().start(); + Ok( + json!({"kind": kind, "name": name.to_string(), "line": start.line, + "column": start.column + 1, "end_line": span.end().line, + "unsupported": uncertain, "path": module_path}), + ) +} + +fn items(input: &[Item]) -> Result> { + input.iter().map(item).collect() +} + +fn item(input: &Item) -> Result { + match input { + Item::Fn(x) => named("fn", &x.sig.ident, x.span(), &x.attrs), + Item::Struct(x) => named("struct", &x.ident, x.span(), &x.attrs), + Item::Enum(x) => named("enum", &x.ident, x.span(), &x.attrs), + Item::Trait(x) => { + let mut value = named("trait", &x.ident, x.span(), &x.attrs)?; + let children = x + .items + .iter() + .map(|child| match child { + syn::TraitItem::Fn(f) => named("fn", &f.sig.ident, f.span(), &f.attrs), + syn::TraitItem::Const(c) => named("unsupported", &c.ident, c.span(), &c.attrs), + syn::TraitItem::Type(t) => named("unsupported", &t.ident, t.span(), &t.attrs), + _ => Err("trait macro/verbatim item leaves bindings unknown".into()), + }) + .collect::>>()?; + value["children"] = json!(children); + Ok(value) + } + Item::Impl(x) => { + let (uncertain, module_path) = attributes(&x.attrs)?; + if module_path.is_some() || x.defaultness.is_some() { + return Err("unsupported impl attributes/specialization".into()); + } + let owner = match x.self_ty.as_ref() { + Type::Path(p) if p.qself.is_none() => path(&p.path)?, + _ => return Err("impl owner is not a named local type".into()), + }; + if x.generics + .type_params() + .any(|p| owner.split("::").next() == Some(p.ident.to_string().as_str())) + { + return Err("blanket impl owner is a generic parameter".into()); + } + let trait_path = x + .trait_ + .as_ref() + .map(|(negative, p, _)| { + if negative.is_some() { + Err("negative impl is unsupported".into()) + } else { + path(p) + } + }) + .transpose()?; + let children = x + .items + .iter() + .map(|child| match child { + syn::ImplItem::Fn(f) => named("fn", &f.sig.ident, f.span(), &f.attrs), + syn::ImplItem::Const(c) => named("unsupported", &c.ident, c.span(), &c.attrs), + syn::ImplItem::Type(t) => named("unsupported", &t.ident, t.span(), &t.attrs), + _ => Err("impl macro/verbatim item leaves bindings unknown".into()), + }) + .collect::>>()?; + Ok(json!({"kind": "impl", "owner": owner, "trait": trait_path, + "unsupported": uncertain, "children": children})) + } + Item::Mod(x) => { + let mut value = named("mod", &x.ident, x.span(), &x.attrs)?; + value["children"] = match &x.content { + Some((_, content)) => json!(items(content)?), + None => Value::Null, + }; + Ok(value) + } + Item::Use(x) => { + let (uncertain, module_path) = attributes(&x.attrs)?; + if uncertain.is_some() || module_path.is_some() || x.leading_colon.is_some() { + return Err("conditional/absolute import unsupported".into()); + } + let mut tree = &x.tree; + let mut parts = Vec::new(); + while let syn::UseTree::Path(p) = tree { + parts.push(p.ident.to_string()); + tree = &p.tree; + } + let alias = match tree { + syn::UseTree::Name(n) => { + parts.push(n.ident.to_string()); + n.ident.to_string() + } + syn::UseTree::Rename(n) => { + parts.push(n.ident.to_string()); + n.rename.to_string() + } + _ => return Err("group/glob imports leave bindings unknown".into()), + }; + Ok(json!({"kind": "use", "name": alias, "target": parts.join("::")})) + } + Item::Const(x) => named("unsupported", &x.ident, x.span(), &x.attrs), + Item::Static(x) => named("unsupported", &x.ident, x.span(), &x.attrs), + Item::Type(x) => named("unsupported", &x.ident, x.span(), &x.attrs), + _ => Err("item macro/foreign/unsupported scope leaves bindings unknown".into()), + } +} + +// syn intentionally preserves some unrecognized syntax as Verbatim. Never +// accept those escape hatches as proof of syntax validity, even in a body. +#[derive(Default)] +struct NoVerbatim(bool); +impl<'ast> Visit<'ast> for NoVerbatim { + fn visit_expr(&mut self, node: &'ast syn::Expr) { + self.0 |= matches!(node, syn::Expr::Verbatim(_)); + syn::visit::visit_expr(self, node); + } + fn visit_type(&mut self, node: &'ast Type) { + self.0 |= matches!(node, Type::Verbatim(_)); + syn::visit::visit_type(self, node); + } + fn visit_pat(&mut self, node: &'ast syn::Pat) { + self.0 |= matches!(node, syn::Pat::Verbatim(_)); + syn::visit::visit_pat(self, node); + } + fn visit_impl_item(&mut self, node: &'ast syn::ImplItem) { + self.0 |= matches!(node, syn::ImplItem::Verbatim(_)); + syn::visit::visit_impl_item(self, node); + } + fn visit_trait_item(&mut self, node: &'ast syn::TraitItem) { + self.0 |= matches!(node, syn::TraitItem::Verbatim(_)); + syn::visit::visit_trait_item(self, node); + } + fn visit_foreign_item(&mut self, node: &'ast syn::ForeignItem) { + self.0 |= matches!(node, syn::ForeignItem::Verbatim(_)); + syn::visit::visit_foreign_item(self, node); + } + fn visit_item(&mut self, node: &'ast Item) { + self.0 |= matches!(node, Item::Verbatim(_)); + syn::visit::visit_item(self, node); + } +} + +fn parse(source: &str) -> Result { + let file = syn::parse_file(source)?; + let mut visitor = NoVerbatim::default(); + visitor.visit_file(&file); + if visitor.0 { + return Err("unrecognized verbatim Rust syntax".into()); + } + let (uncertain, module_path) = attributes(&file.attrs)?; + if uncertain.is_some() || module_path.is_some() { + return Err("conditional/path crate attribute unsupported".into()); + } + Ok(json!({"items": items(&file.items)?, "sources": { + "syntax/Cargo.toml": include_str!("../Cargo.toml"), + "syntax/Cargo.lock": include_str!("../Cargo.lock"), + "syntax/src/main.rs": include_str!("main.rs") + }})) +} + +fn main() { + let mut source = String::new(); + let result = io::stdin() + .take(1_048_577) + .read_to_string(&mut source) + .map_err(Error::Read) + .and_then(|_| { + if source.len() > 1_048_576 { + Err("source size limit exceeded".into()) + } else { + parse(&source) + } + }); + let output = match result { + Ok(value) => value, + Err(error) => json!({"error": error.to_string()}), + }; + // This private JSON protocol is consumed by Python; no source is executed. + if let Err(error) = writeln!(io::stdout().lock(), "{output}") { + if error.kind() != io::ErrorKind::BrokenPipe { + std::process::exit(1); + } + } +} diff --git a/evals/source-identity-v1/test_source_identity.py b/evals/source-identity-v1/test_source_identity.py new file mode 100644 index 0000000..92218b6 --- /dev/null +++ b/evals/source-identity-v1/test_source_identity.py @@ -0,0 +1,511 @@ +"""Independent source fixtures; exercise the public API and executable surface.""" +import json +import os +from pathlib import Path +import subprocess +import sys +import tempfile +import unittest + +from source_identity import Project, ContractError, score, score_request + +HERE = Path(__file__).resolve().parent + + +def rust(name, line, kind="fn", file="rust/lib.rs", quote=None): + return {"language": "rust", "name": name, "file": file, "line": line, + "kind": kind, "citation": {"start_line": line, "end_line": line, + "quote": quote if quote is not None else + (HERE / "fixtures" / file).read_text().splitlines()[line - 1]}} + + +def python(name, line, kind="assignment"): + return {"language": "python", "name": name, "file": "python/settings.py", + "line": line, "kind": kind, "citation": {"start_line": line, + "end_line": line, "quote": (HERE / "fixtures/python/settings.py") + .read_text().splitlines()[line - 1]}} + + +class SourceIdentityTests(unittest.TestCase): + def setUp(self): + self.project = Project(HERE / "fixtures", rust_root="rust/lib.rs", + python_files=["python/settings.py"]) + + def request_for(self, required, submitted): + return {"schema_version": 1, "manifest": {"rust_root": "lib.rs", "python_files": ["truth.py"]}, + "required": required, "submitted": submitted} + + def run_request_cli(self, project, request): + request_path = project.root / "request.json" + request_path.write_text(json.dumps(request)) + return subprocess.run([sys.executable, "-B", str(HERE / "source_identity.py"), + "--root", str(project.root), "--input", str(request_path)], + capture_output=True, text=True, timeout=20, env={}) + + def test_malformed_rust_public_boundaries(self): + # Independently authored balanced syntax faults, not copies of the parser. + sources = [ + "pub fn choose(value u8) {}", # parameter separator + "pub fn choose(!!!) -> Definitely nonsense {}", # header + "pub fn choose() -> & {}", # incomplete return type + "pub fn choose() -> Vec<,> {}", # malformed generic type + "pub fn choose() { let = ; }", # pattern and initializer + "pub fn choose() { let x = (1 +); }", # expression + "pub fn choose() { if true else {} }", # control flow + 'pub fn choose() { let x = "bad\\q"; }', # invalid escape + 'pub fn choose() { let x = r##"unclosed"#; }', # raw string terminator + "pub fn choose() {} /* unterminated", # lexical fault + "struct Other { field u8 }\npub fn choose() {}", # unrelated invalid source + ] + truth = {"language": "python", "name": "VALUE", "file": "truth.py", "line": 1, + "kind": "assignment", "citation": {"start_line": 1, "end_line": 1, "quote": "VALUE = 1"}} + for source in sources: + with self.subTest(source=source): + project = self.synthetic({"lib.rs": source, "truth.py": "VALUE = 1"}, + rust_root="lib.rs", python_files=["truth.py"]) + line = 2 if source.startswith("struct") else 1 + selector = rust("choose", line, file="lib.rs", quote=source.splitlines()[line - 1]) + checked = project.check(selector) + self.assertFalse(checked["identity_ok"], checked) + self.assertFalse(checked["citation_ok"], checked) + request = self.request_for([truth], [selector]) + result = score_request(project.root, request) + self.assertEqual(result["identity_recall"], {"matched": 0, "required": 1}) + self.assertEqual(result["citation_recall"], {"matched": 0, "required": 1}) + self.assertEqual(result["semantic_review"]["status"], "pending") + process = self.run_request_cli(project, request) + self.assertEqual(process.returncode, 0, process.stderr) + self.assertEqual(json.loads(process.stdout)["identity_recall"]["matched"], 0) + invalid_truth = self.request_for([selector], [selector]) + with self.assertRaises(ContractError): + score_request(project.root, invalid_truth) + process = self.run_request_cli(project, invalid_truth) + self.assertEqual(process.returncode, 2) + self.assertEqual(process.stdout, "") + self.assertEqual(json.loads(process.stderr)["code"], "invalid_input") + + def test_realistic_rust_public_boundaries(self): + source = '''//! A representative crate. +#![allow(dead_code)] +#[derive(Debug, Clone)] +struct Parcel { value: T } +trait Read: Sized { + type Output; + fn read(&self, value: T) -> Self::Output; +} +impl Parcel where T: std::fmt::Debug { + /// A documented method. + #[inline] + fn take(&self) -> &T { &self.value } +} +impl Read for Parcel { + type Output = T; + fn read(&self, value: T) -> T { value } +} +const LIMIT: usize = 3; +type Other = Option; +#[doc = "A free function"] +#[must_use] +pub fn choose(value: T) -> T { value } +''' + project = self.synthetic({"lib.rs": source, "truth.py": "VALUE = 1"}, + rust_root="lib.rs", python_files=["truth.py"]) + selectors = [rust(name, line, file="lib.rs", quote=source.splitlines()[line - 1]) + for name, line in [("Parcel::take", 12), ("::read", 16), + ("Read::read", 7), ("choose", 22)]] + for selector in selectors: + with self.subTest(selector=selector): + checked = project.check(selector) + self.assertTrue(checked["identity_ok"] and checked["citation_ok"], checked) + request = self.request_for(selectors, selectors) + result = score_request(project.root, request) + self.assertEqual(result["identity_recall"], {"matched": 4, "required": 4}) + self.assertEqual(result["citation_recall"], {"matched": 4, "required": 4}) + process = self.run_request_cli(project, request) + self.assertEqual(process.returncode, 0, process.stderr) + self.assertEqual(json.loads(process.stdout)["identity_recall"], {"matched": 4, "required": 4}) + for name in ["Other::take", "::read", "wrong::Parcel::take", "read"]: + self.assertFalse(project.check(selectors[0] | {"name": name})["identity_ok"]) + + def test_named_uncertainty_preserves_unrelated_declarations_and_blocks_ties(self): + source = '''#[cfg(unix)] fn maybe() {} +fn choose() {} +mod other { #[cfg(windows)] fn choose() {} } +type Alias = u8; +const COUNT: usize = 1; +''' + project = self.synthetic({"lib.rs": source}, rust_root="lib.rs") + choose = rust("crate::choose", 2, file="lib.rs", quote=source.splitlines()[1]) + self.assertTrue(project.check(choose)["identity_ok"], project.failures) + self.assertIn("ambiguous", project.check(choose | {"name": "choose"})["reason"]) + self.assertFalse(project.check(choose | {"name": "maybe", "line": 1})["identity_ok"]) + self.assertTrue(project.diagnostics) + for source in ["#[cfg(unix)] struct Box; impl Box { fn choose() {} }", + "struct Box; #[cfg(unix)] impl Box { fn choose() {} }", + "#[cfg(unix)] mod a { pub fn choose() {} }"]: + with self.subTest(source=source): + project = self.synthetic({"lib.rs": source}, rust_root="lib.rs") + for name in ["Box::choose", "a::choose", "choose"]: + self.assertFalse(project.check(choose | {"name": name, "line": 1})["identity_ok"]) + + def test_ordinary_context_and_free_method_control(self): + sources = [ + '#[derive(Debug)] struct Other;\npub fn choose() {}', + '#[doc = "documented"]\npub fn choose() {}', + 'struct Other(T);\npub fn choose() {}', + 'struct Other; impl Other { fn choose() {} }\npub fn choose() {}', + ] + for source in sources: + with self.subTest(source=source): + project = self.synthetic({"lib.rs": source, "truth.py": "VALUE = 1"}, rust_root="lib.rs") + selector = rust("choose", 2, file="lib.rs", quote=source.splitlines()[1]) + self.assertTrue(project.check(selector)["citation_ok"], project.failures) + request = self.request_for([selector], [selector]) + self.assertEqual(score_request(project.root, request)["identity_recall"], + {"matched": 1, "required": 1}) + process = self.run_request_cli(project, request) + self.assertEqual(process.returncode, 0, process.stderr) + self.assertEqual(json.loads(process.stdout)["citation_recall"], + {"matched": 1, "required": 1}) + + def test_missing_and_stale_frontend_refuse_public_credit(self): + from unittest.mock import patch + import rust_source + source = "fn choose() {}" + project = self.synthetic({"lib.rs": source}, rust_root="lib.rs") + selector = rust("choose", 1, file="lib.rs", quote=source) + self.assertTrue(project.check(selector)["identity_ok"]) + with patch.object(rust_source, "PARSER", project.root / "absent-frontend"): + missing = Project(project.root, rust_root="lib.rs") + self.assertFalse(missing.check(selector)["identity_ok"]) + self.assertIn("unavailable", missing.failures["rust"]) + # The real binary still runs, but an altered source manifest cannot match + # its embedded build inputs. No substitute parser or permissive fallback. + with patch.object(rust_source, "PARSER_SOURCES", ()): + stale = Project(project.root, rust_root="lib.rs") + self.assertFalse(stale.check(selector)["citation_ok"]) + self.assertIn("stale", stale.failures["rust"]) + + def test_generic_owner_ambiguity_and_macro_uncertainty(self): + sources = [ + "struct Box(T); impl Box { fn choose() {} } impl Box { fn choose() {} }", + "struct Box; impl Box { fn choose() {} }", + "#[custom] fn choose() {}", + "#[derive(Custom)] struct Box; fn choose() {}", + "macro_rules! make { () => {} } make!(); fn choose() {}", + "#[cfg_attr(unix, custom)] fn choose() {}", + "macro_rules! unused { malformed matcher } fn choose() {}", + ] + for source in sources: + with self.subTest(source=source): + project = self.synthetic({"lib.rs": source}, rust_root="lib.rs") + for name in ["Box::choose", "choose"]: + selector = rust(name, 1, file="lib.rs", quote=source) + self.assertFalse(project.check(selector)["identity_ok"]) + + def test_rust_strings_comments_and_no_execution(self): + source = '''/* nested /* fn fake() {} */ comments */ +fn choose() { + let text = r###"fn fake(!!!) { let = ; }"###; + let character = '}'; + std::fs::write("EXECUTED", text); +} +''' + project = self.synthetic({"lib.rs": source}, rust_root="lib.rs") + selector = rust("choose", 2, file="lib.rs", quote="fn choose() {") + self.assertTrue(project.check(selector)["citation_ok"], project.failures) + self.assertFalse(project.check(selector | {"name": "fake"})["identity_ok"]) + self.assertFalse((project.root / "EXECUTED").exists()) + + def test_trait_and_owner_variants(self): + spellings = ["Compass::from_str", "::from_str", + "::from_str"] + results = [self.project.check(rust(name, 6)) for name in spellings] + self.assertTrue(all(x["identity_ok"] and x["citation_ok"] for x in results)) + self.assertEqual(len({json.dumps(x["identity"], sort_keys=True) for x in results}), 1) + self.assertEqual(results[0]["identity"]["trait"], "std::str::FromStr") + + def test_owner_free_nested_and_explicit_module(self): + for item in [rust("Compass::parse", 9), rust("Ledger::parse", 12), + rust("crate::parse", 14), rust("nested::parse", 20), + rust("nested::Dial::turn", 18), + rust("wire::Socket::connect", 3, file="rust/relocated.rs")]: + with self.subTest(item=item): + result = self.project.check(item) + self.assertTrue(result["identity_ok"], result) + self.assertTrue(result["citation_ok"], result) + self.assertFalse(self.project.check(rust("parse", 14))["identity_ok"]) + + def test_python_declarations_and_static_assignments(self): + for item in [python("RETRY_LIMIT", 2), python("python.settings.WINDOW", 3), + python("LABELS", 4), python("decode", 8, "function"), + python("Packet.decode", 12, "function")]: + with self.subTest(item=item): + self.assertTrue(self.project.check(item)["identity_ok"]) + for name, line in [("DYNAMIC", 5), ("ALIAS", 6), ("counterfeit", 15)]: + self.assertFalse(self.project.check(python(name, line))["identity_ok"]) + + def test_adversarial_selectors(self): + original = rust("Compass::parse", 9) + changes = [{"name": "Ledger::parse"}, {"name": "wrong::Compass::parse"}, + {"name": "::parse"}, {"name": "Compass.parse"}, + {"file": "rust/relocated.rs"}, {"file": "../lib.rs"}, + {"kind": "struct"}, {"line": 10}, {"line": True}, + {"language": "Rust"}, {"name": "counterfeit"}, + {"name": "Compass::::parse"}, {"ignored": "field"}] + for change in changes: + with self.subTest(change=change): + self.assertFalse(self.project.check(original | change)["identity_ok"]) + + def test_citations_are_separate_and_exact(self): + original = rust("Compass::parse", 9) + for citation in [{"start_line": 9, "end_line": 9, "quote": "pub fn parse"}, + {"start_line": 8, "end_line": 8, "quote": "impl Compass {"}, + {"start_line": 9, "end_line": 90, "quote": ""}, + {"start_line": True, "end_line": 9, "quote": ""}, + None]: + result = self.project.check(original | {"citation": citation}) + self.assertTrue(result["identity_ok"], result) + self.assertFalse(result["citation_ok"], result) + + def test_score_keeps_denominator_and_extras(self): + required = [rust("Compass::from_str", 6), python("RETRY_LIMIT", 2)] + submitted = [rust("::from_str", 6), + rust("::from_str", 6), + python("ALIAS", 6), rust("Ledger::parse", 12)] + result = score(self.project, required, submitted) + self.assertEqual(result["identity_recall"], {"matched": 1, "required": 2}) + self.assertEqual(result["citation_recall"], {"matched": 1, "required": 2}) + self.assertEqual(len(result["items"]), 4) + self.assertEqual(result["semantic_review"]["status"], "pending") + self.assertEqual(len(result["missing"]), 1) + self.assertEqual(result["items"][-1]["classification"], "verified_extra") + self.assertIn("contract_sha256", result["pins"]) + self.assertEqual(len(result["pins"]["source_sha256"]), 3) + with self.assertRaises(ContractError): + score(self.project, [python("ALIAS", 6)], []) + + def synthetic(self, files, rust_root=None, python_files=()): + scratch = HERE.parents[1] / ".orbit/tmp" + scratch.mkdir(parents=True, exist_ok=True) + temporary = tempfile.TemporaryDirectory(prefix="source-identity-test-", dir=scratch) + self.addCleanup(temporary.cleanup) + root = Path(temporary.name) + for name, source in files.items(): + path = root / name + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(source) + return Project(root, rust_root=rust_root, python_files=python_files) + + def test_rust_ambiguous_and_unsupported_contexts(self): + cases = [ + "fn choose() {}\nfn choose() {}\n", + '#[cfg(unix)] fn choose() {}\n#[cfg(windows)] fn choose() {}', + 'macro_rules! make { () => { fn choose() {} } } make!();', + 'use crate::hidden::*; fn choose() {}', + 'fn choose() { "unterminated }', + 'fn choose( {}', + 'struct Box; impl Missing { fn choose() {} }', + 'struct Box; impl Unknown for Box { fn choose() {} }', + 'struct Box; impl Box { fn choose() {} }', + 'struct Box; impl Bo x { fn choose() {} }', + 'mod other; fn choose() {}', + ] + for source in cases: + with self.subTest(source=source): + project = self.synthetic({"lib.rs": source}, rust_root="lib.rs") + item = rust("choose", 1, file="rust/lib.rs") | {"file": "lib.rs"} + self.assertFalse(project.check(item)["identity_ok"]) + self.assertIsNotNone(project.check(item)["reason"]) + + def test_duplicate_trait_impls_cannot_use_line_to_disambiguate(self): + source = "struct Box;\ntrait Paint { fn coat(); }\nimpl Paint for Box { fn coat() {} }\nimpl Paint for Box { fn coat() {} }\n" + project = self.synthetic({"lib.rs": source}, rust_root="lib.rs") + item = rust("::coat", 3) | {"file": "lib.rs"} + self.assertIn("ambiguous", project.check(item)["reason"]) + + def test_trait_qualification_disambiguates_inherent_and_trait(self): + source = "struct Box;\ntrait Paint { fn coat(); }\nimpl Paint for Box { fn coat() {} }\nimpl Box { fn coat() {} }\n" + project = self.synthetic({"lib.rs": source}, rust_root="lib.rs") + item = rust("::coat", 3) | {"file": "lib.rs"} + self.assertTrue(project.check(item)["identity_ok"]) + self.assertFalse(project.check(item | {"name": "Box::coat"})["identity_ok"]) + self.assertFalse(project.check(item | {"name": "::coat"})["identity_ok"]) + + def test_owner_leaf_must_itself_be_unique(self): + source = "mod a { pub struct Box; impl Box { fn coat() {} } }\nmod b { pub struct Box; }" + project = self.synthetic({"lib.rs": source}, rust_root="lib.rs") + item = rust("a::Box::coat", 1) | {"file": "lib.rs"} + self.assertTrue(project.check(item)["identity_ok"]) + self.assertFalse(project.check(item | {"name": "Box::coat"})["identity_ok"]) + + def test_verified_import_alias_and_nested_trait(self): + source = "mod defs { pub struct Box; pub trait Paint { fn coat(); } }\nuse crate::defs::Box as CrateBox;\nuse crate::defs::Paint as Finish;\nimpl Finish for CrateBox { fn coat() {} }" + project = self.synthetic({"lib.rs": source}, rust_root="lib.rs") + item = rust("::coat", 4) | {"file": "lib.rs"} + checked = project.check(item) + self.assertTrue(checked["identity_ok"], checked) + self.assertEqual(checked["identity"]["owner"], "crate::defs::Box") + self.assertEqual(checked["identity"]["trait"], "crate::defs::Paint") + + def test_module_paths_are_not_inferred_from_filename(self): + bad = rust("relocated::Socket::connect", 3, file="rust/relocated.rs") + self.assertFalse(self.project.check(bad)["identity_ok"]) + source = '#[path = "part.rs"] mod first;\n#[path = "part.rs"] mod second;' + project = self.synthetic({"lib.rs": source, "part.rs": "fn f() {}"}, rust_root="lib.rs") + self.assertIn("reused", project.failures["rust"]) + project = self.synthetic({"lib.rs": '#[path = "../outside.rs"] mod other;'}, rust_root="lib.rs") + self.assertIn("contained", project.failures["rust"]) + + def test_python_rebinding_control_flow_and_decorators(self): + sources = ["VALUE = 1\nVALUE = 2", "VALUE = 1\nVALUE = other", "if flag:\n VALUE = 1", + "VALUE = 1\ndel VALUE", "VALUE = 1\nVALUE += 1", "from other import *\nVALUE = 1", + "@wrapper\ndef VALUE(): pass", "class VALUE(Base): pass", "VALUE: int", + "VALUE = alias", "VALUE = func()", "VALUE = [1, 2]", "VALUE = 1 + 2", + "VALUE = (1, unknown)", "VALUE =", '"VALUE = 1"', "# VALUE = 1"] + for source in sources: + with self.subTest(source=source): + project = self.synthetic({"settings.py": source}, python_files=["settings.py"]) + item = python("VALUE", 1) | {"file": "settings.py"} + self.assertFalse(project.check(item)["identity_ok"]) + + def test_unsupported_python_binding_still_blocks_ambiguous_leaf(self): + project = self.synthetic({"one.py": "VALUE = 1", "two.py": "VALUE = other"}, + python_files=["one.py", "two.py"]) + item = python("VALUE", 1) | {"file": "one.py"} + self.assertIn("ambiguous", project.check(item)["reason"]) + self.assertTrue(project.check(item | {"name": "one.VALUE"})["identity_ok"]) + + def test_python_source_is_never_executed(self): + project = self.synthetic({"settings.py": 'DYNAMIC = __import__("pathlib").Path("EXECUTED").touch()'}, + python_files=["settings.py"]) + self.assertFalse((project.root / "EXECUTED").exists()) + self.assertTrue(project.diagnostics) + + def test_symlink_source_is_refused(self): + project = self.synthetic({"real.py": "VALUE = 1"}) + (project.root / "link.py").symlink_to("real.py") + linked = Project(project.root, python_files=["link.py"]) + self.assertIn("symlink", linked.failures["python"]) + + def test_frozen_source_snapshot_and_hash(self): + import hashlib + source = "VALUE = 1" + project = self.synthetic({"settings.py": source}, python_files=["settings.py"]) + (project.root / "settings.py").write_text("VALUE = 2") + item = python("VALUE", 1) | {"file": "settings.py", + "citation": {"start_line": 1, "end_line": 1, "quote": source}} + self.assertTrue(project.check(item)["citation_ok"]) + self.assertEqual(project.pins()["source_sha256"]["settings.py"], hashlib.sha256(source.encode()).hexdigest()) + + def test_scorer_real_executable_success_and_bad_request(self): + from example import request + project = self.synthetic({}) + input_path = project.root / "request.json" + input_path.write_text(json.dumps(request())) + command = [sys.executable, "-B", str(HERE / "source_identity.py"), + "--root", str(HERE / "fixtures"), "--input", str(input_path)] + process = subprocess.run(command, capture_output=True, text=True, timeout=20, env={}) + self.assertEqual(process.returncode, 0, process.stderr) + self.assertEqual(process.stderr, "") + output = json.loads(process.stdout) + self.assertEqual(output["identity_recall"], {"matched": 2, "required": 3}) + self.assertEqual(output["citation_recall"], {"matched": 1, "required": 3}) + for bad_request in ['{"schema_version": 99}', '{', + '{"schema_version": 1, "schema_version": 1}', + '{"schema_version": NaN}']: + with self.subTest(request=bad_request): + input_path.write_text(bad_request) + process = subprocess.run(command, capture_output=True, text=True, timeout=20, env={}) + self.assertEqual(process.returncode, 2) + self.assertEqual(process.stdout, "") + self.assertEqual(json.loads(process.stderr)["code"], "invalid_input") + + def test_shadowed_standard_library_trait_path_is_refused(self): + source = "mod std { pub mod str { pub trait FromStr { fn from_str(); } } }\nstruct Box;\nimpl std::str::FromStr for Box { fn from_str() {} }" + project = self.synthetic({"lib.rs": source}, rust_root="lib.rs") + item = rust("::from_str", 3) | {"file": "lib.rs"} + self.assertFalse(project.check(item)["identity_ok"]) + + def test_python_short_owner_cannot_hide_another_class(self): + project = self.synthetic({"one.py": "class Packet:\n def decode(self): pass", + "two.py": "class Packet: pass"}, python_files=["one.py", "two.py"]) + item = python("Packet.decode", 2, "function") | {"file": "one.py"} + self.assertFalse(project.check(item)["identity_ok"]) + self.assertTrue(project.check(item | {"name": "one.Packet.decode"})["identity_ok"]) + + def test_multiline_citation_and_generic_lifetimes(self): + source = "struct Box;\nimpl Box {\n fn borrow<'a, T>(value: &'a T) -> &'a T {\n value\n }\n}" + project = self.synthetic({"lib.rs": source}, rust_root="lib.rs") + item = rust("Box::borrow", 3) | {"file": "lib.rs", "citation": { + "start_line": 3, "end_line": 5, "quote": "\n".join(source.splitlines()[2:5])}} + self.assertTrue(project.check(item)["citation_ok"]) + bad = item | {"citation": item["citation"] | {"end_line": 6}} + self.assertFalse(project.check(bad)["citation_ok"]) + + def test_closed_stdout_is_silent_success(self): + from example import request + project = self.synthetic({}) + request_path = project.root / "request.json" + request_path.write_text(json.dumps(request())) + process = subprocess.Popen([sys.executable, "-B", str(HERE / "source_identity.py"), + "--root", str(HERE / "fixtures"), "--input", str(request_path)], + stdout=subprocess.PIPE, stderr=subprocess.PIPE, env={}) + try: + process.stdout.close() + process.wait(timeout=20) + stderr = process.stderr.read() + self.assertEqual(process.returncode, 0, stderr) + self.assertEqual(stderr, b"") + finally: + if process.poll() is None: + process.kill() + process.wait(timeout=5) + process.stderr.close() + + def test_cli_help_golden(self): + process = subprocess.run([sys.executable, "-B", str(HERE / "source_identity.py"), "--help"], + capture_output=True, text=True, timeout=20, env={}) + self.assertEqual(process.returncode, 0, process.stderr) + self.assertEqual(process.stderr, "") + golden = HERE / "cli-help.txt" + if os.environ.get("UPDATE_GOLDENS") == "1": + golden.write_text(process.stdout) + self.assertEqual(process.stdout, golden.read_text()) + + def test_python_import_bindings_are_visible_without_importing(self): + source = "from nonexistent import VALUE as OTHER\nimport also_nonexistent\nVALUE: int = 5" + project = self.synthetic({"settings.py": source}, python_files=["settings.py"]) + item = python("VALUE", 3) | {"file": "settings.py"} + self.assertTrue(project.check(item)["identity_ok"]) + self.assertFalse(project.check(item | {"name": "OTHER", "line": 1})["identity_ok"]) + self.assertEqual(len(project.diagnostics), 2) + + @unittest.skipIf(sys.version_info < (3, 12), "Python generic def syntax needs Python 3.12+") + def test_python_generic_function_syntax(self): + project = self.synthetic({"settings.py": "def identity[T](value: T) -> T:\n return value"}, + python_files=["settings.py"]) + item = python("identity", 1, "function") | {"file": "settings.py"} + self.assertTrue(project.check(item)["identity_ok"]) + + def test_rejected_truth_cannot_shrink_required_denominator(self): + valid = rust("Compass::parse", 9) + with self.assertRaises(ContractError): + score(self.project, [valid, valid], [valid]) + with self.assertRaises(ContractError): + score(self.project, [], [valid]) + with self.assertRaises(ContractError): + score(self.project, [valid | {"citation": None}], [valid]) + + def test_executable_example_is_deterministic(self): + process = subprocess.run([sys.executable, "-B", str(HERE / "example.py")], + capture_output=True, text=True, timeout=20, env={}) + self.assertEqual(process.returncode, 0, process.stderr) + golden = HERE / "example-result.json" + if os.environ.get("UPDATE_GOLDENS") == "1": + golden.write_text(process.stdout) + self.assertEqual(process.stdout, golden.read_text()) + + +if __name__ == "__main__": + unittest.main()