fix(pingora): admit OOXML manuscripts by fail-closed structural DOCX evidence

scripts/ci/pingora_edge_policy.py

@@ -22,7 +22,7 @@
 
 Suffix decision: most research-data formats (``.xlsx``, ``.sav``, ``.rds``,
 ``.npz``, ...) have no entry in ``BINARY_DOCUMENT_MAGIC``, which only knows
-``.hwpx``/``.pdf``/``.png``. Rather than grow that registry for every such
+``.docx``/``.hwpx``/``.pdf``/``.png``. Rather than grow that registry for every such
 format, a file under a declared prefix whose suffix has no magic entry is
 admitted on the stricter complement of the UTF-8 decode this module already
 performs for every ordinarily-scanned file: no diff patch available, *and* the
@@ -32,8 +32,17 @@
 exists to do -- while still admitting genuinely opaque research binaries
 without maintaining an open-ended magic-byte catalog. A suffix that *does*
 have a magic entry keeps that entry's existing structural evidence check
-(``_is_complete_png``, ``_is_complete_hwpx``, or the raw magic-prefix check for
-``.pdf``) even under a declared prefix.
+(``_is_complete_png``, ``_is_complete_hwpx``, ``_is_complete_docx``, or the raw
+magic-prefix check for ``.pdf``) even under a declared prefix -- so a ``.docx``
+under a declared prefix is now held to the stricter structural proof rather
+than the UTF-8 complement, which fails closed in the same direction.
+
+late-life-anxiety-reanalysis#257 -- structural DOCX admission: a tracked
+research manuscript under ``docs/`` was rejected with "is not valid UTF-8"
+because ``.docx`` had no magic entry and so reached the ordinary content
+scan's UTF-8 decode. ``.docx`` is admitted on ``_is_complete_docx`` container
+evidence instead. The artifact is neither relocated nor exempted: an
+unreadable, truncated, disguised, or non-WordprocessingML package still fails.
 """
 
 from __future__ import annotations
@@ -76,11 +85,25 @@
 # (this org's own "attach the relevant paper PDF" convention) for a reason
 # that has nothing to do with the Nginx runtime policy this module enforces.
 BINARY_DOCUMENT_MAGIC = {
+    ".docx": (b"PK\x03\x04",),
     ".hwpx": (b"PK\x03\x04",),
     ".pdf": (b"%PDF-",),
     ".png": (b"\x89PNG\r\n\x1a\n",),
 }
 PNG_SIGNATURE = BINARY_DOCUMENT_MAGIC[".png"][0]
+# OOXML WordprocessingML structural admission (late-life-anxiety-reanalysis#257).
+# A ``.docx`` is admitted by proving its container structure, never by decoding
+# it as text: the exact OPC parts every conforming writer emits, plus the main
+# document part's content type, which is what distinguishes a WordprocessingML
+# package from an arbitrary ZIP (or an HWPX) renamed to ``.docx``.
+DOCX_REQUIRED_PARTS = ("[Content_Types].xml", "_rels/.rels", "word/document.xml")
+DOCX_MAIN_DOCUMENT_CONTENT_TYPE = (
+    b"application/vnd.openxmlformats-officedocument.wordprocessingml.document.main+xml"
+)
+# The content-type declaration is the only part this module reads, and it is
+# read bounded: a package whose declaration exceeds this ceiling is rejected
+# rather than streamed, so admission cannot be turned into an unbounded read.
+MAX_DOCX_CONTENT_TYPES_BYTES = 65_536
 SOURCE_TEST_SUFFIXES = frozenset({".py", ".pyi", ".js", ".mjs", ".cjs", ".ts", ".tsx", ".rs"})
 LICENSE_NAMES = frozenset({"license", "license.md", "copying", "copyrights", "notice"})
 DOCUMENTATION_DIRECTORIES = frozenset({"doc", "docs", "documentation"})
@@ -369,8 +392,8 @@ def _is_binary_documentation_asset(changed: ChangedFile, declared_prefixes: Sequ
     *declared_prefixes* (issue #2193) is the base-ref-only research/data
     artifact declaration: it replaces ONLY this function's path-shape test,
     never the content evidence a caller still confirms below. A file whose
-    suffix is a recognized ``BINARY_DOCUMENT_MAGIC`` format (``.hwpx``/
-    ``.pdf``/``.png``) is admitted under a declared prefix on the exact same
+    suffix is a recognized ``BINARY_DOCUMENT_MAGIC`` format (``.docx``/
+    ``.hwpx``/``.pdf``/``.png``) is admitted under a declared prefix on the exact same
     format evidence documentation paths already require. A file whose
     suffix has no magic entry at all (research formats such as ``.xlsx``,
     ``.sav``, ``.rds``, ``.npz`` have none) can ONLY be admitted through a
@@ -606,8 +629,9 @@ def _binary_documentation_evidence_confirms(
     also omits a patch for a textual diff that exceeds its own rendering
     limit, well under this module's ``MAX_FILE_BYTES`` content-fetch
     ceiling. Whenever the file's raw bytes can be fetched at all, this
-    verifies the declared format's magic prefix instead of trusting
-    patch-presence alone. Only a file whose content evidently exceeds the
+    verifies the declared format's structural evidence (``_is_complete_docx``
+    for the OOXML manuscript case, ``_is_complete_hwpx``, ``_is_complete_png``)
+    or magic prefix instead of trusting patch-presence alone. Only a file whose content evidently exceeds the
     Contents API's size ceiling -- the exact case ``_is_binary_documentation_asset``
     exists for, a cited, large research paper -- falls back to trusting the
     path+suffix convention for oversized PDFs only; every other
@@ -636,6 +660,8 @@ def _binary_documentation_evidence_confirms(
         return _is_complete_png(raw)
     if suffix == ".hwpx":
         return _is_complete_hwpx(raw)
+    if suffix == ".docx":
+        return _is_complete_docx(raw)
     if suffix not in BINARY_DOCUMENT_MAGIC:
         try:
             raw.decode("utf-8")
@@ -645,6 +671,49 @@ def _binary_documentation_evidence_confirms(
     return raw.startswith(BINARY_DOCUMENT_MAGIC[suffix])
 
 
+def _is_complete_docx(raw: bytes) -> bool:
+    """Confirm a bounded OOXML WordprocessingML container without reading its text.
+
+    Fail-closed structural admission for the research manuscript in
+    late-life-anxiety-reanalysis#257: require an unprefixed ZIP, its exact end
+    record, unique members, every part in ``DOCX_REQUIRED_PARTS`` present,
+    non-empty and unencrypted, and the main document part's content type
+    declared in a bounded ``[Content_Types].xml``. Unlike HWPX, a conforming
+    ``.docx`` has no stored ``mimetype`` member and DEFLATEs every part, so
+    neither is required here. Anything unreadable, truncated, prefixed,
+    appended to, encrypted, or declaring a different document type returns
+    ``False`` -- the caller then scans the bytes as it always did, which for
+    genuinely binary content fails the policy rather than skipping it.
+    """
+    if not raw.startswith(BINARY_DOCUMENT_MAGIC[".docx"][0]):
+        return False
+    try:
+        with zipfile.ZipFile(io.BytesIO(raw)) as archive:
+            archive_entries = archive.infolist()
+            member_names = [member_info.filename for member_info in archive_entries]
+            end_offset = len(raw) - 22 - len(archive.comment)
+            if end_offset < 0 or raw[end_offset:end_offset + 4] != b"PK\x05\x06":
+                return False
+            if int.from_bytes(raw[end_offset + 20:end_offset + 22], "little") != len(archive.comment):
+                return False
+            if not archive_entries or archive_entries[0].header_offset != 0:
+                return False
+            if len(member_names) != len(set(member_names)):
+                return False
+            for part_name in DOCX_REQUIRED_PARTS:
+                part_info = archive.getinfo(part_name)
+                if part_info.flag_bits & 1 or part_info.file_size == 0:
+                    return False
+            content_types_info = archive.getinfo(DOCX_REQUIRED_PARTS[0])
+            if content_types_info.file_size > MAX_DOCX_CONTENT_TYPES_BYTES:
+                return False
+            with archive.open(content_types_info) as content_types_stream:
+                declaration = content_types_stream.read(MAX_DOCX_CONTENT_TYPES_BYTES)
+            return DOCX_MAIN_DOCUMENT_CONTENT_TYPE in declaration
+    except (KeyError, UnicodeError, OSError, ValueError, NotImplementedError, zipfile.BadZipFile):
+        return False
+
+
 def _is_complete_hwpx(raw: bytes) -> bool:
     """Confirm a bounded HWPX container without extracting document content.
 

tests/test_pingora_docx_evidence.py

@@ -0,0 +1,137 @@
+"""Exercise DOCX structural admission through the production policy boundary offline."""
+import io
+import zipfile
+
+import pytest
+from tests.test_pingora_edge_policy import policy
+from tests.test_pingora_hwpx_evidence import RUNTIME_BYTES, RUNTIME_TEXT, evaluate_bytes
+
+# The real research artifact from late-life-anxiety-reanalysis#257 (exact head
+# 706a81e5a6f88ad74544ab9cf89d4da2b9e6a44d) sits at this path. It is a
+# DEFLATE-compressed OOXML package whose first member is [Content_Types].xml,
+# with no ZIP comment and no prefixed or appended bytes. The fixture below
+# reproduces exactly that structure without copying another repository's
+# artifact into this one.
+REAL_ARTIFACT_PATH = (
+    "docs/delivery_interim_20260920/"
+    "air_render_00cdc51_g7integration_20260921_205341/manuscript_interim_20260921.docx"
+)
+MAIN_DOCUMENT_CONTENT_TYPE = (
+    b"application/vnd.openxmlformats-officedocument.wordprocessingml.document.main+xml"
+)
+CONTENT_TYPES_XML = (
+    b'<?xml version="1.0" encoding="UTF-8"?>'
+    b'<Types xmlns="http://schemas.openxmlformats.org/package/2006/content-types">'
+    b'<Default Extension="rels" ContentType="application/vnd.openxmlformats-package.relationships+xml"/>'
+    b'<Override PartName="/word/document.xml" ContentType="' + MAIN_DOCUMENT_CONTENT_TYPE + b'"/>'
+    b"</Types>"
+)
+SPREADSHEET_CONTENT_TYPES_XML = CONTENT_TYPES_XML.replace(
+    MAIN_DOCUMENT_CONTENT_TYPE,
+    b"application/vnd.openxmlformats-officedocument.spreadsheetml.sheet.main+xml",
+)
+
+
+def docx_archive(
+    *,
+    content_types=CONTENT_TYPES_XML,
+    package_rels=b'<Relationships xmlns="http://schemas.openxmlformats.org/package/2006/relationships"/>',
+    document=b'<w:document xmlns:w="http://schemas.openxmlformats.org/wordprocessingml/2006/main"/>',
+    compression_type=zipfile.ZIP_DEFLATED,
+    duplicate_document=False,
+):
+    """Create a deterministic, non-sensitive OOXML boundary fixture."""
+    archive_buffer = io.BytesIO()
+    members = [
+        ("[Content_Types].xml", content_types),
+        ("_rels/.rels", package_rels),
+        ("word/document.xml", document),
+    ]
+    if duplicate_document:
+        members.append(("word/document.xml", document))
+    with zipfile.ZipFile(archive_buffer, "w") as archive_file:
+        for member_name, member_value in members:
+            if member_value is None:
+                continue
+            member_info = zipfile.ZipInfo(member_name)
+            member_info.compress_type = compression_type
+            archive_file.writestr(member_info, member_value)
+    return archive_buffer.getvalue()
+
+
+@pytest.mark.parametrize("file_path", [REAL_ARTIFACT_PATH, "docs/manuscript.docx", "Docs/MANUSCRIPT.DOCX"])
+def test_valid_docx_is_admitted_at_the_real_research_artifact_path(file_path):
+    """Admit the #257 manuscript by structure, without moving or excluding it."""
+    assert evaluate_bytes(file_path, docx_archive()) == ()
+
+
+@pytest.mark.parametrize("compression_type", [zipfile.ZIP_STORED, zipfile.ZIP_DEFLATED])
+def test_valid_docx_is_admitted_for_either_stored_or_deflated_parts(compression_type):
+    """Real writers DEFLATE every part; stored parts are equally structural."""
+    assert evaluate_bytes("docs/manuscript.docx", docx_archive(compression_type=compression_type)) == ()
+
+
+@pytest.mark.parametrize("file_bytes", [
+    b"PK\x03\x04\xff",
+    docx_archive()[:-10],
+    b"#!/bin/sh\n" + RUNTIME_BYTES + docx_archive(),
+    b"PK\x03\x04" + docx_archive(),
+    docx_archive() + b"\n" + RUNTIME_BYTES,
+    docx_archive()[:-2] + b"\x01\x00",
+    docx_archive(content_types=None),
+    docx_archive(package_rels=None),
+    docx_archive(document=None),
+    docx_archive(document=b""),
+    docx_archive(content_types=SPREADSHEET_CONTENT_TYPES_XML),
+    docx_archive(duplicate_document=True),
+])
+def test_corrupt_or_disguised_docx_container_is_not_exempt(file_bytes):
+    """Unreadable, incomplete, or non-WordprocessingML input still fails closed."""
+    with pytest.raises(policy.PolicyError):
+        evaluate_bytes("docs/manuscript.docx", file_bytes)
+
+
+def test_docx_rejects_an_encrypted_or_entryless_package():
+    """An encrypted part or an empty central directory is unverifiable, so it fails."""
+    encrypted_bytes = bytearray(docx_archive())
+    encrypted_bytes[encrypted_bytes.find(b"PK\x01\x02") + 8] |= 1
+    entryless_bytes = bytearray(docx_archive())
+    end_offset = len(entryless_bytes) - 22
+    entryless_bytes[end_offset + 8:end_offset + 12] = b"\x00\x00\x00\x00"
+    entryless_bytes[end_offset + 12:end_offset + 16] = b"\x00\x00\x00\x00"
+    entryless_bytes[end_offset + 16:end_offset + 20] = end_offset.to_bytes(4, "little")
+    for file_bytes in (bytes(encrypted_bytes), bytes(entryless_bytes)):
+        with pytest.raises(policy.PolicyError):
+            evaluate_bytes("docs/manuscript.docx", file_bytes)
+
+
+def test_oversized_content_types_declaration_is_not_read():
+    """A declaration beyond the bounded read ceiling is rejected, never streamed."""
+    padding = b"<!--" + b"p" * (policy.MAX_DOCX_CONTENT_TYPES_BYTES + 1) + b"-->"
+    with pytest.raises(policy.PolicyError):
+        evaluate_bytes("docs/manuscript.docx", docx_archive(content_types=CONTENT_TYPES_XML + padding))
+
+
+@pytest.mark.parametrize("file_path", ["docs/manuscript.docx", "local/model.zip"])
+def test_non_utf8_bytes_without_a_structural_document_still_fail(file_path):
+    """Opaque bytes that are not a recognized structural document are never admitted."""
+    with pytest.raises(policy.PolicyError):
+        evaluate_bytes(file_path, b"\x00\x01\x02\xff\xfe" + RUNTIME_BYTES)
+
+
+@pytest.mark.parametrize("patch_value", [None, "+" + RUNTIME_TEXT.rstrip("\n")])
+def test_valid_utf8_named_docx_keeps_todays_runtime_scan(patch_value):
+    """A file that decodes as UTF-8 is never treated as a binary artifact."""
+    violations = evaluate_bytes("docs/manuscript.docx", RUNTIME_BYTES, patch_value=patch_value)
+    assert [violation.rule for violation in violations] == ["nginx_runtime_path"]
+
+
+def test_docx_in_runtime_path_remains_unavailable():
+    """A structural format exception cannot exempt an active runtime location."""
+    with pytest.raises(policy.PolicyError):
+        evaluate_bytes("docs/nginx/manuscript.docx", docx_archive())
+
+
+def test_removed_docx_does_not_load_deleted_content():
+    """Deletion does not require unavailable final-head bytes."""
+    assert evaluate_bytes("docs/manuscript.docx", b"", file_status="removed") == ()