diff --git a/CHANGELOG.md b/CHANGELOG.md index dfaf1304..b80c3971 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -49,6 +49,14 @@ versioning is loosely [SemVer](https://semver.org/) at the bundle level. remain Claude-Code-only. New guide: `docs/multi-harness.md`. ### Fixed +- **Scope: out-of-scope rules no longer fail open** (`engine/scope.py`). A deny rule written with + a port (`admin.example.com:8443`), a scheme in `scope.json` / `--out-of-scope` + (`https://admin.example.com`), a path, a trailing dot, a trailing comma, or as an IPv6 CIDR + matched no host, so the host fell through to a broader in-scope rule (`example.com`) and was + tested. Every rule from every source is now normalized to its host once, in `Scope`; IPv6 + addresses and CIDRs are supported; an uncompilable out-of-scope regex raises instead of being + skipped; rules widened to their host or unable to match are reported as `scope warning:` lines + by `scope.py` and the engine log. - `hunt-clickjacking`, `hunt-html-injection`: quoted the `description` — an unquoted `: ` (`Targets:`, `surfaces:`) broke strict YAML / Codex. Also genericized lab/harness-specific language in the new skills for cross-harness portability. diff --git a/commands/scope.md b/commands/scope.md index 163d53c0..69e2a195 100644 --- a/commands/scope.md +++ b/commands/scope.md @@ -17,8 +17,14 @@ judgment — deny wins, default deny. - apex `acme.com` → the apex **and** any subdomain - `*.acme.com` → any subdomain (NOT the bare apex) - `api.acme.com` → that exact host - - `10.0.0.0/8` → any IP in the CIDR + - `10.0.0.0/8`, `2001:db8::/32` → any IP in the CIDR (IPv4 or IPv6) - `re:^lab[0-9]+\.acme\.io$` → explicit regex (prefix `re:`) + + Rules can be pasted as the program page writes them: `https://admin.acme.com/panel`, + `admin.acme.com:8443` and `ADMIN.acme.com.` all reduce to the host `admin.acme.com`. + Scope is enforced per host, so a rule's path or port is dropped (a + `scope warning:` line says so); an out-of-scope `re:` rule that does not compile + is a hard error, never a silently skipped deny. 3. Prints `IN-SCOPE` / `OUT-OF-SCOPE ` and exits non-zero if any asset is out of scope — so it can gate automation. diff --git a/engine/engine.py b/engine/engine.py index 84ceec05..ca7f176a 100644 --- a/engine/engine.py +++ b/engine/engine.py @@ -64,6 +64,8 @@ def __init__(self, scope_path, base, model, max_hunts, max_turns, timeout, mock= self.eng.log(f"engine start | scope={self.scope.name} in={self.scope.in_scope} " f"out={self.scope.out_of_scope} seeds={self.scope.seeds} mock={mock} " f"intrusive={'ALLOWED' if allow_intrusive else 'READ-ONLY (default)'}") + for warning in self.scope.warnings: + self.eng.log(f"scope warning: {warning}") # ---------------- rules of engagement (injected into every agent) ---------------- def _roe(self): diff --git a/engine/scope.py b/engine/scope.py index db6e3fb7..11a9b03c 100644 --- a/engine/scope.py +++ b/engine/scope.py @@ -11,23 +11,85 @@ example.com -> the apex AND any subdomain (example.com, api.example.com) *.example.com -> any subdomain (NOT the bare apex) api.example.com -> that exact host - 10.0.0.0/8 -> any IP in the CIDR (IPv4) + 10.0.0.0/8 -> any IP in the CIDR (IPv4 or IPv6, e.g. 2001:db8::/32) + 10.0.0.5, ::1 -> that exact IP re:^staging[0-9]+\\.example\\.com$ -> explicit regex (prefix re:) + +Every non-regex rule is normalized to the host it names before matching, however +it was written: `https://admin.example.com/panel`, `admin.example.com:8443`, +`ADMIN.example.com.` and `[2001:db8::1]` all become a host rule. This matters most +for out-of-scope rules, where a rule that silently matched nothing failed OPEN. +A rule that loses information (a path or port cannot be enforced per host) or +still cannot match anything is reported in `Scope.warnings`; an out-of-scope +regex that does not compile raises ValueError instead of being skipped. """ import ipaddress import re from urllib.parse import urlparse +_WRAPPERS = "`\"'" + + +def _canonical_ip(text): + """Canonical text of an IP address or CIDR ('[2001:0db8::1]' -> '2001:db8::1'), + or None if `text` is neither.""" + t = text.strip() + if t.startswith("[") and "]" in t: + t = t[1:].replace("]", "", 1) + try: + if "/" in t: + return str(ipaddress.ip_network(t, strict=False)) + return str(ipaddress.ip_address(t)) + except ValueError: + return None + + def _host_of(target): t = target.strip() if "://" not in t: t = "//" + t try: parsed = urlparse(t) - return (parsed.hostname or "").lower().rstrip(".") + host = (parsed.hostname or "").lower().rstrip(".") except ValueError: return "" + return _canonical_ip(host) or host + + +def _normalize_rule(rule): + """Reduce one scope rule to the form `_match` understands. + + Returns (rule, warning). `warning` is None when nothing was lost; otherwise a + human-readable note that the rule was widened to its whole host, or that it + still cannot match any host. Regex rules (`re:`) are returned untouched. + """ + raw = rule + t = rule.strip().strip(_WRAPPERS).rstrip(",;").strip().strip(_WRAPPERS) + if not t or t[:3].lower() == "re:": + return t, None + ip = _canonical_ip(t) + if ip: + return ip, None + try: + parsed = urlparse(t if "://" in t or t.startswith("//") else "//" + t) + host = (parsed.hostname or "").rstrip(".") + port = parsed.port + except ValueError: + return t.lower(), f"rule {raw!r} is not a valid host and will never match" + if not host or any(c.isspace() for c in host) or "*" in host[2:] or \ + (host.startswith("*") and not host.startswith("*.")): + return (host or t).lower(), f"rule {raw!r} is not a valid host and will never match" + lost = [] + if port is not None: + lost.append(f"port {port}") + if parsed.path not in ("", "/") or parsed.query or parsed.fragment: + lost.append("path") + host = _canonical_ip(host) or host + if lost: + return host, (f"rule {raw!r} names a {' and '.join(lost)}, but scope is enforced " + f"per host, so it applies to every URL on {host}") + return host, None def _match(pattern, host): @@ -40,11 +102,16 @@ def _match(pattern, host): except re.error: return False p = p.lower() - if "/" in p and p.replace(".", "").replace("/", "").isdigit(): # CIDR + ip = _canonical_ip(p) + if ip: try: - return ipaddress.ip_address(host) in ipaddress.ip_network(p, strict=False) + addr = ipaddress.ip_address(host) except ValueError: - return False + return False # an IP rule never matches a hostname + if "/" in ip: + net = ipaddress.ip_network(ip) + return addr.version == net.version and addr in net + return addr == ipaddress.ip_address(ip) if p.startswith("*."): base = p[2:] return host.endswith("." + base) @@ -54,15 +121,37 @@ def _match(pattern, host): class Scope: def __init__(self, in_scope, out_of_scope=None, seeds=None, name="engagement"): - self.in_scope = [p for p in (in_scope or []) if p.strip()] - self.out_of_scope = [p for p in (out_of_scope or []) if p.strip()] + self.warnings = [] + self.in_scope = self._normalize_all(in_scope, "in-scope") + self.out_of_scope = self._normalize_all(out_of_scope, "out-of-scope") self.seeds = seeds or [] self.name = name + def _normalize_all(self, rules, kind): + out = [] + for rule in rules or []: + norm, warning = _normalize_rule(rule) + if not norm: + continue + if norm[:3].lower() == "re:": + try: + re.compile(norm[3:]) + except re.error as exc: + if kind == "out-of-scope": + # Deny wins: a deny rule that cannot be evaluated must not be skipped. + raise ValueError(f"invalid out-of-scope regex {rule!r}: {exc}; refusing to " + f"build a scope whose deny rule cannot be enforced") from None + warning = f"regex {rule!r} does not compile ({exc}) and will never match" + if warning: + self.warnings.append(f"{kind} {warning}") + out.append(norm) + return out + @classmethod def load(cls, path): import json - d = json.load(open(path, encoding="utf-8")) + with open(path, encoding="utf-8") as fh: + d = json.load(fh) return cls(d.get("in_scope", []), d.get("out_of_scope", []), d.get("seeds", []), d.get("name", "engagement")) @@ -109,24 +198,16 @@ def _selftest(): s2 = Scope(in_scope=["*.test.example.com"]) assert s2.in_scope_host("a.test.example.com") assert not s2.in_scope_host("test.example.com") + # deny rules pasted with a port / scheme / IPv6 CIDR must still deny (never fail open) + s3 = Scope(in_scope=["example.com", "re:.*"], + out_of_scope=["admin.example.com:8443", "https://ops.example.com/", "2001:db8::/32"]) + assert not s3.in_scope_host("https://admin.example.com:8443/") + assert not s3.in_scope_host("https://ops.example.com/x") + assert not s3.in_scope_host("http://[2001:db8::1]/") + assert s3.in_scope_host("https://api.example.com/") print("scope.py self-test: PASS") -def _rule_from_token(token): - """Normalize one scope.md bullet token into a host rule. - - `https://app.example.com/login?x=1` -> `app.example.com`. - Explicit-rule forms (`re:`, CIDR, `*.host`) and already-bare hosts are kept as-is. - """ - t = token.strip() - if t[:3].lower() == "re:" or ("://" not in t and not t.startswith("//")): - return t - try: - host = urlparse(t if "://" in t else "//" + t).hostname or "" - except ValueError: - return t - return host or t - def _scope_from_md(path): """Build a Scope from a scaffolded scope.md (the `hunt` template). @@ -134,9 +215,8 @@ def _scope_from_md(path): Takes the first whitespace token of each bullet (so '- api.example.com — note' yields 'api.example.com'). Skips placeholder bullets like '(paste ... here)'. A token written as a URL (the common paste-from-program-page form, - '- https://app.example.com/login — main app') is reduced to its host, because - every rule is matched against the host anyway; leaving the scheme in made the - rule match nothing and the whole engagement default-deny. + '- https://app.example.com/login — main app') is reduced to its host by + `Scope`, which normalizes rules from every source the same way. """ in_scope, out_scope, section = [], [], None with open(path, encoding="utf-8") as fh: @@ -162,12 +242,13 @@ def _scope_from_md(path): pat = pat[2:-2] # bullet written in bold, e.g. '**api.example.com**' if not pat or pat.startswith("(") or pat.startswith("["): continue # placeholder text, not a real pattern - (in_scope if section == "in" else out_scope).append(_rule_from_token(pat)) + (in_scope if section == "in" else out_scope).append(pat) return Scope(in_scope=in_scope, out_of_scope=out_scope, name=path) def _main(argv=None): import argparse + import sys ap = argparse.ArgumentParser( prog="scope.py", description="Deterministic in-scope check (deny wins, default deny).") @@ -185,14 +266,20 @@ def _main(argv=None): _selftest() return 0 - if args.md: - scope = _scope_from_md(args.md) - elif args.scope_file: - scope = Scope.load(args.scope_file) - elif args.in_scope or args.out_of_scope: - scope = Scope(in_scope=args.in_scope, out_of_scope=args.out_of_scope) - else: + if not (args.md or args.scope_file or args.in_scope or args.out_of_scope): ap.error("provide --md, --scope-file, or --in-scope/--out-of-scope patterns") + try: + if args.md: + scope = _scope_from_md(args.md) + elif args.scope_file: + scope = Scope.load(args.scope_file) + else: + scope = Scope(in_scope=args.in_scope, out_of_scope=args.out_of_scope) + except ValueError as exc: + print(f"scope error: {exc}", file=sys.stderr) + return 2 + for warning in scope.warnings: + print(f"scope warning: {warning}", file=sys.stderr) if not args.targets: ap.error("no targets given") diff --git a/engine/test_scope_rule_normalization.py b/engine/test_scope_rule_normalization.py new file mode 100644 index 00000000..338f2bcd --- /dev/null +++ b/engine/test_scope_rule_normalization.py @@ -0,0 +1,132 @@ +import io +import json +import os +import tempfile +import unittest +from contextlib import redirect_stderr, redirect_stdout + +from scope import Scope, _main, _scope_from_md + + +class OutOfScopeRulesDoNotFailOpenTests(unittest.TestCase): + """Every way a program page writes an out-of-scope asset must still deny it. + + Before normalization each of these deny rules matched nothing, so the host + fell through to the broader in-scope rule and was treated as testable. + """ + + DENY_FORMS = { + "port": "admin.example.com:8443", + "scheme": "https://admin.example.com", + "scheme and path": "https://admin.example.com/panel", + "bare path": "admin.example.com/panel", + "trailing dot": "admin.example.com.", + "upper case": "ADMIN.Example.COM", + "userinfo": "https://ops@admin.example.com", + "trailing comma": "admin.example.com,", + "backticks": "`admin.example.com`", + } + + def test_every_written_form_of_a_deny_rule_denies_the_host(self): + for form, rule in self.DENY_FORMS.items(): + with self.subTest(form=form): + scope = Scope(["example.com"], [rule]) + self.assertFalse(scope.in_scope_host("https://admin.example.com/")) + self.assertFalse(scope.in_scope_host("https://admin.example.com:8443/panel")) + self.assertTrue(scope.in_scope_host("https://api.example.com/")) + + def test_wildcard_deny_rule_written_as_url(self): + scope = Scope(["example.com"], ["https://*.corp.example.com/"]) + self.assertFalse(scope.in_scope_host("https://vpn.corp.example.com/")) + self.assertTrue(scope.in_scope_host("https://corp.example.com/")) + + def test_ipv6_cidr_deny_rule(self): + scope = Scope(["re:.*"], ["2001:db8::/32"]) + self.assertFalse(scope.in_scope_host("http://[2001:db8::1]/")) + self.assertTrue(scope.in_scope_host("http://[2001:db9::1]/")) + + def test_rules_from_scope_json_are_normalized(self): + fd, path = tempfile.mkstemp(suffix=".json") + with os.fdopen(fd, "w", encoding="utf-8") as fh: + json.dump({"name": "acme", "in_scope": ["https://example.com/"], + "out_of_scope": ["https://admin.example.com:443/"]}, fh) + try: + scope = Scope.load(path) + finally: + os.remove(path) + self.assertEqual(scope.in_scope, ["example.com"]) + self.assertEqual(scope.out_of_scope, ["admin.example.com"]) + self.assertFalse(scope.in_scope_host("https://admin.example.com/")) + self.assertTrue(scope.in_scope_host("https://www.example.com/")) + + def test_port_bullet_in_scope_md_denies(self): + fd, path = tempfile.mkstemp(suffix=".md") + with os.fdopen(fd, "w", encoding="utf-8") as fh: + fh.write("## In scope\n\n- example.com\n\n## Out of scope\n\n- admin.example.com:8443 — legacy panel\n") + try: + scope = _scope_from_md(path) + finally: + os.remove(path) + self.assertFalse(scope.in_scope_host("https://admin.example.com:8443/")) + + +class InvalidDenyRegexTests(unittest.TestCase): + def test_uncompilable_out_of_scope_regex_is_refused(self): + with self.assertRaises(ValueError): + Scope(["example.com"], ["re:^admin[.example\\.com$"]) + + def test_uncompilable_in_scope_regex_warns_and_matches_nothing(self): + scope = Scope(["re:^api[.example\\.com$"]) + self.assertFalse(scope.in_scope_host("https://api.example.com/")) + self.assertEqual(len(scope.warnings), 1) + + def test_cli_exits_2_on_unenforceable_deny_rule(self): + err = io.StringIO() + with redirect_stderr(err): + code = _main(["admin.example.com", "--in-scope", "example.com", + "--out-of-scope", "re:(unclosed"]) + self.assertEqual(code, 2) + self.assertIn("scope error", err.getvalue()) + + +class IpRuleTests(unittest.TestCase): + def test_ipv6_cidr_and_exact_address_in_scope(self): + scope = Scope(["2001:db8::/32", "[2001:db8:ffff::5]"]) + self.assertTrue(scope.in_scope_host("http://[2001:db8::1]:8080/")) + self.assertTrue(scope.in_scope_host("http://[2001:0db8:0000::0001]/")) + self.assertFalse(scope.in_scope_host("http://[2001:dead::1]/")) + + def test_ipv4_rule_does_not_match_an_ipv6_host(self): + self.assertFalse(Scope(["0.0.0.0/0"]).in_scope_host("http://[::1]/")) + + def test_ip_rules_never_match_hostnames(self): + scope = Scope(["10.0.0.0/8", "127.0.0.1"]) + self.assertFalse(scope.in_scope_host("https://10.0.0.1.example.com/")) + self.assertTrue(scope.in_scope_host("http://127.0.0.1:3000/")) + + +class WarningTests(unittest.TestCase): + def test_widened_and_unmatchable_rules_are_reported(self): + scope = Scope(["https://app.example.com/api", "api.*.example.com"], + ["admin.example.com:8443"]) + joined = "\n".join(scope.warnings) + self.assertIn("in-scope rule 'https://app.example.com/api' names a path", joined) + self.assertIn("in-scope rule 'api.*.example.com' is not a valid host", joined) + self.assertIn("out-of-scope rule 'admin.example.com:8443' names a port 8443", joined) + + def test_plain_rules_produce_no_warnings(self): + scope = Scope(["example.com", "*.example.org", "10.0.0.0/8", "re:^lab\\d+\\.x\\.io$"], + ["admin.example.com"]) + self.assertEqual(scope.warnings, []) + + def test_cli_prints_warnings_to_stderr(self): + err = io.StringIO() + with redirect_stderr(err), redirect_stdout(io.StringIO()): + code = _main(["api.example.com", "--in-scope", "example.com", + "--out-of-scope", "admin.example.com:8443"]) + self.assertEqual(code, 0) + self.assertIn("scope warning", err.getvalue()) + + +if __name__ == "__main__": + unittest.main()