Clojure->Python CUTOVER: Step 4 — remove the Clojure tree; move the Python math engine to math/

.github/workflows/python-ci.yml

@@ -10,7 +10,11 @@ on:
     paths: &paths
       - 'delphi/**/*.py'
       - 'delphi/requirements*.txt'
+      - 'delphi/requirements.lock'
+      - 'delphi/pyproject.toml'
       - 'delphi/Dockerfile'
+      - 'math/**/*.py'
+      - 'math/pyproject.toml'
       - '.github/workflows/python-ci.yml'
   pull_request:
     branches:
@@ -97,8 +101,9 @@ jobs:
         docker compose -f docker-compose.test.yml cp delphi/real_data delphi:/app/real_data
         echo "Copying coverage script into container..."
         docker compose -f docker-compose.test.yml cp delphi/generate_coverage_md.py delphi:/app/generate_coverage_md.py
-        echo "Copying script to be tested into container..."
-        docker compose -f docker-compose.test.yml cp delphi/polismath/run_math_pipeline.py delphi:/app/run_math_pipeline.py
+        # (no run_math_pipeline.py cp: since the math package move it is a
+        # polismath submodule — installed in the image, covered by
+        # --cov=polismath; nothing imports a top-level copy.)
         docker compose -f docker-compose.test.yml cp delphi/umap_narrative delphi:/app/umap_narrative
 
         echo "Running tests and generating coverage report..."
@@ -120,7 +125,7 @@ jobs:
             echo '--- Running Pytest ---'; \
             export PYTHONPATH=\$PYTHONPATH:/app; \
             export POLIS_TEST_POSTGRES_URL=\"postgresql://\$DATABASE_USER:\$DATABASE_PASSWORD@\$DATABASE_HOST/\$DATABASE_NAME\"; \
-            pytest --cov=polismath --cov=run_math_pipeline --cov=./umap_narrative --cov-report=xml:/app/coverage.xml /app/tests --ignore=/app/tests/test_pakistan_conversation.py
+            pytest --cov=polismath --cov=./umap_narrative --cov-report=xml:/app/coverage.xml /app/tests --ignore=/app/tests/test_pakistan_conversation.py
             echo '--- Generating Coverage Comment Text ---'; \
             python /app/generate_coverage_md.py > /app/coverage-comment.md \
           "

delphi/Dockerfile

@@ -54,15 +54,24 @@ RUN --mount=type=cache,target=/root/.cache/uv \
 
   # ===== OPTIMIZATION: Copy source code LAST (busts cache on code changes) =====
   # Copy source code - this layer rebuilds when code changes but reuses dependency layers above
-  COPY polismath/ ./polismath/
   COPY umap_narrative/ ./umap_narrative/
   COPY scripts/ ./scripts/
   COPY *.py ./
 
-  # Install the project package (without dependencies - they're already installed)
-  # This registers entry points and installs the package in development mode
+  # The math engine lives OUTSIDE this build context (<repo>/math, cutover
+  # Step #4) and arrives via a named additional build context — compose
+  # declares `additional_contexts: mathsrc: ./math` on every service that
+  # builds this Dockerfile. Its deps are a subset of the requirements.lock
+  # installed above, so --no-deps is safe.
+  COPY --from=mathsrc . /math-src
+
+  # Install both project packages (without dependencies - they're already
+  # installed). This registers entry points; polismath installs from the
+  # sibling source tree, NEVER from an index (supply-chain: the PyPI name
+  # "polismath" is not ours).
   RUN --mount=type=cache,target=/root/.cache/uv \
-      uv pip install --no-deps .
+      uv pip install --no-deps . && \
+      uv pip install --no-deps /math-src
 
   RUN echo "--- PyTorch Check (after pyproject.toml installation) ---" && \
       pip show torch torchvision torchaudio && \
@@ -90,8 +99,8 @@ WORKDIR /app
 COPY --from=builder /usr/local/lib/python3.12/site-packages /usr/local/lib/python3.12/site-packages
 COPY --from=builder /usr/local/bin /usr/local/bin
 
-# Copy source code from builder stage (already built into the package)
-COPY --from=builder /app/polismath/ ./polismath/
+# Copy source code from builder stage (already built into the package;
+# polismath ships via site-packages above — no /app/polismath dir anymore)
 COPY --from=builder /app/scripts/ ./scripts/
 COPY --from=builder /app/umap_narrative/ ./umap_narrative/
 COPY --from=builder /app/*.py ./
@@ -144,9 +153,14 @@ COPY --from=builder /opt/uv/uv /usr/local/bin/uv
 ENV UV_SYSTEM_PYTHON=1
 COPY pyproject.toml .
 
-# Install dev dependencies (pytest, etc.) using caching
+# Install dev dependencies (pytest, etc.) using caching. The bare
+# `polismath` requirement is satisfied by the copy already installed in
+# the `final` stage this extends — assert that survived so a future
+# --upgrade/--reinstall can never silently pull the foreign PyPI
+# "polismath" package instead.
 RUN --mount=type=cache,target=/root/.cache/uv \
-    uv pip install ".[dev]"
+    uv pip install ".[dev]" && \
+    python -c "import polismath.poller"
 
 # Default command for test container (can be overridden)
 CMD ["tail", "-f", "/dev/null"]

delphi/pyproject.toml

@@ -9,6 +9,9 @@ description = "Mathematical analytics pipeline for Polis conversations"
 requires-python = ">=3.12,<3.13" # Pin to Python 3.12.x
 
 dependencies = [
+    # The math engine — its own package at <repo>/math since cutover Step #4,
+    # installed editable from the sibling tree (see [tool.uv.sources]).
+    "polismath",
     "numpy>=1.26.4,<2.0",
     "pandas>=2.1.4",
     "sqlalchemy>=2.0.36",
@@ -101,8 +104,8 @@ notebook = [
 [project.scripts]
 delphi = "scripts.delphi_cli:main"
 run-delphi = "run_delphi:main"
-# Pipeline entry points
-run-math-pipeline = "polismath.run_math_pipeline:main"
+# Pipeline entry points (run-math-pipeline moved to math/pyproject.toml
+# with the polismath package, cutover Step #4)
 run-umap-pipeline = "umap_narrative.run_pipeline:main"
 calculate-extremity = "umap_narrative.501_calculate_comment_extremity:main"
 calculate-priorities = "umap_narrative.502_calculate_priorities:main"
@@ -111,13 +114,17 @@ reset-conversation = "umap_narrative.reset_conversation:main"
 create-datamapplot = "umap_narrative.700_datamapplot_for_layer:main"
 setup-minio = "setup_minio:setup_minio_bucket"
 
+# uv: resolve the polismath dependency from the sibling math/ tree,
+# editable — one shared environment (delphi/.venv) for both packages.
+[tool.uv.sources]
+polismath = { path = "../math", editable = true }
+
 # Hatchling configuration
 [tool.hatch.build.targets.wheel]
-packages = ["polismath", "umap_narrative"]
+packages = ["umap_narrative"]
 
 [tool.hatch.build.targets.sdist]
 include = [
-    "/polismath",
     "/umap_narrative",
     "/scripts",
     "/tests",
@@ -161,7 +168,10 @@ relative_files = true
 # maintain this). The mirror `polis/.venv` (also real or symlink) is what lets
 # editors opening the workspace at the repo root auto-discover the interpreter.
 [tool.pyright]
-include = ["polismath", "umap_narrative", "tests", "scripts"]
+# polismath moved to <repo>/math (cutover Step #4) and has its own
+# [tool.pyright] there; here we cover the delphi tree, which imports it
+# from the shared .venv (editable install).
+include = ["umap_narrative", "tests", "scripts"]
 exclude = [
     "**/__pycache__",
     "**/.pytest_cache",

delphi/run_delphi.py

@@ -97,8 +97,10 @@ def main():
 
     # Run the math pipeline
     print(f"{GREEN}Running math pipeline...{NC}")
+    # polismath is an installed package (its source moved to <repo>/math at
+    # cutover Step #4) — invoke the pipeline as a module, not by file path.
     math_command = [
-        "python", f"{app_path}/polismath/run_math_pipeline.py",
+        "python", "-m", "polismath.run_math_pipeline",
         f"--zid={zid}",
     ]
     if max_votes_arg:

delphi/scripts/prodclone_extract.py

@@ -3,7 +3,7 @@
 a "prodclone" Postgres database (a clone of the production polis DB) into the
 replay-dataset export format, for Clojure↔Python math parity certification.
 
-See ``delphi/polismath/replay/prodclone.py`` for the pure building blocks
+See ``math/polismath/replay/prodclone.py`` for the pure building blocks
 (SQL builders, feature classifiers, CSV formatters, slug minting, the
 path-safety guard). This script is a thin click CLI wiring those together
 with a live psycopg2 connection — mirrors the style of

delphi/tests/replay_harness/test_certify.py

@@ -207,7 +207,7 @@ def test_sha256_tree_exclude_file_and_dir_prefix(tmp_path):
 def test_engine_tree_exclude_entries_exist_and_keep_replay_shapers():
     """Every exclusion names a real path under polismath/ (a rename must not
     turn it into a silent no-op), and the replay-shaping files stay hashed."""
-    pm = cert._DELPHI_ROOT / "polismath"
+    pm = cert.paths.PACKAGE_ROOT
     for e in cert._ENGINE_TREE_EXCLUDE:
         p = pm / e.rstrip("/")
         if e.endswith("/"):
@@ -219,6 +219,71 @@ def test_engine_tree_exclude_entries_exist_and_keep_replay_shapers():
     assert not kept & set(cert._ENGINE_TREE_EXCLUDE)
 
 
+def test_clj_source_hashes_missing_oracle_raises_certify_error(tmp_path, monkeypatch):
+    """Post-Step-4 the Clojure oracle tree is absent from the working tree:
+    the clj cache-key hash must fail with pointed restore instructions
+    (CertifyError, stage 'clj-oracle') — not a bare FileNotFoundError."""
+    monkeypatch.setattr(cert, "_MATH_ROOT", tmp_path / "math")
+    cert._clj_source_hashes.cache_clear()
+    try:
+        with pytest.raises(cert.CertifyError, match="git history"):
+            cert._clj_source_hashes()
+    finally:
+        cert._clj_source_hashes.cache_clear()
+
+
+class TestEnsureCljRecordingWithoutOracle:
+    """Cached pairs MUST survive the oracle tree's removal (Step 4): a
+    recording whose INPUT keys (votes/schedule) match its manifest is
+    trusted as-is; only an actual re-replay demands the restored tree."""
+
+    def _spec_and_entry(self):
+        entry = _make_entry()
+        spec = sched.preset_single_cut("vw", 100, schedule_id=entry.schedule_id)
+        return entry, spec
+
+    def _write_cached_recording(self, root, entry, spec, votes_sha):
+        clj_dir = cert.st.recording_dir(entry.dataset, entry.schedule_id, root=root) / "clj"
+        clj_dir.mkdir(parents=True)
+        cert._write_manifest(clj_dir / "cache_manifest.json", {
+            "votes_sha256": votes_sha,
+            "schedule_hash": cert.canonical_schedule_hash(spec),
+            "replay_clj_sha256": "recorded-against-this-tree",
+            "math_src_sha256": "recorded-against-this-tree",
+        })
+        return clj_dir
+
+    def test_cached_recording_is_used_without_oracle(self, tmp_path, monkeypatch):
+        monkeypatch.setattr(cert, "_MATH_ROOT", tmp_path / "no-oracle")
+        entry, spec = self._spec_and_entry()
+        root = tmp_path / "root"
+        expected_dir = self._write_cached_recording(root, entry, spec, "sha")
+        clj_dir, cached = cert.ensure_clj_recording(
+            entry, spec, "sha", tmp_path / "votes.csv", root=root
+        )
+        assert cached is True and clj_dir == expected_dir
+
+    def test_cache_miss_without_oracle_raises_clj_oracle(self, tmp_path, monkeypatch):
+        monkeypatch.setattr(cert, "_MATH_ROOT", tmp_path / "no-oracle")
+        entry, spec = self._spec_and_entry()
+        root = tmp_path / "root"
+        self._write_cached_recording(root, entry, spec, "sha")
+        with pytest.raises(cert.CertifyError, match="git history"):
+            cert.ensure_clj_recording(
+                entry, spec, "DIFFERENT-sha", tmp_path / "votes.csv", root=root
+            )
+
+    def test_refresh_without_oracle_raises_clj_oracle(self, tmp_path, monkeypatch):
+        monkeypatch.setattr(cert, "_MATH_ROOT", tmp_path / "no-oracle")
+        entry, spec = self._spec_and_entry()
+        root = tmp_path / "root"
+        self._write_cached_recording(root, entry, spec, "sha")
+        with pytest.raises(cert.CertifyError, match="git history"):
+            cert.ensure_clj_recording(
+                entry, spec, "sha", tmp_path / "votes.csv", root=root, refresh=True
+            )
+
+
 def test_engine_tree_hash_ignores_harness_edits_sees_engine_edits(tmp_path):
     pm = tmp_path / "polismath"
     for rel in ("replay/certify.py", "replay/prodclone.py", "replay/driver.py",
@@ -256,7 +321,7 @@ def fake_run(cmd, *, cwd, env):
     (pm / "replay").mkdir(parents=True)
     (pm / "replay" / "certify.py").write_text("h = 1\n")
     (pm / "replay" / "driver.py").write_text("d = 1\n")
-    monkeypatch.setattr(cert, "_DELPHI_ROOT", fake_delphi)
+    monkeypatch.setattr(cert, "_ENGINE_TREE_ROOT", pm)
     cert._engine_tree_hash_cached.cache_clear()
     try:
         entry = _make_entry()
@@ -815,6 +880,9 @@ def _seed_cached_pair(root: Path, ds: str, sid: str, *, divergent: bool) -> None
 
 def test_run_battery_parallel_matches_serial_report_and_ledger(tmp_path, monkeypatch):
     monkeypatch.setattr(cert, "_manifest_matches", lambda mp, exp: True)
+    # Oracle-absent working trees (post-Step-4) consult the input-keys
+    # seam before the full-manifest one — fake both cache hits.
+    monkeypatch.setattr(cert, "_manifest_matches_inputs", lambda mp, keys: True)
     monkeypatch.setattr(cert, "_clj_source_hashes", lambda: ("x", "y"))
 
     entries = [
@@ -843,6 +911,9 @@ def test_run_battery_parallel_matches_serial_report_and_ledger(tmp_path, monkeyp
 
 def test_run_battery_workers_one_is_default_and_identical(tmp_path, monkeypatch):
     monkeypatch.setattr(cert, "_manifest_matches", lambda mp, exp: True)
+    # Oracle-absent working trees (post-Step-4) consult the input-keys
+    # seam before the full-manifest one — fake both cache hits.
+    monkeypatch.setattr(cert, "_manifest_matches_inputs", lambda mp, keys: True)
     monkeypatch.setattr(cert, "_clj_source_hashes", lambda: ("x", "y"))
 
     entries = [_make_entry(schedule_id="w1-only")]

delphi/tests/replay_harness/test_poller_equiv_seed.py

@@ -558,7 +558,7 @@ def test_py_poller_runner_cmd_cwd_env(self):
             database_url="postgresql://x/polis_equiv", math_env="py-shadow",
             base_env={},
         )
-        assert runner.cmd == ["uv", "run", "python", "scripts/math_poller.py"]
+        assert runner.cmd == ["uv", "run", "python", "-m", "polismath.poller"]
         # Equality with _DELPHI_ROOT is the contract; asserting the directory
         # NAME was layout-fragile — CI mounts the delphi tree at /app, where
         # .name == "app" (python-ci run 30071088647, 2026-07-24).

delphi/tests/test_dynamodb_consensus_roundtrip.py

@@ -1,5 +1,5 @@
 """
-Tests for D11 cascade fix in `delphi/polismath/database/dynamodb.py`.
+Tests for D11 cascade fix in `math/polismath/database/dynamodb.py`.
 
 Investigation B (2026-06-11) found three sites in the DynamoDB writer/reader
 that either dropped the new D11 `consensus_comments` dict shape

delphi/tests/test_prodclone_extract.py

@@ -1,5 +1,5 @@
 """Tests for the prodclone extractor (delphi/scripts/prodclone_extract.py +
-delphi/polismath/replay/prodclone.py).
+math/polismath/replay/prodclone.py).
 
 Unit tests exercise the PURE building blocks (SQL builders, feature
 classifiers, CSV row formatters, slug minting, path-safety guard, map

docker-compose.test.yml

@@ -62,12 +62,14 @@ services:
     build:
       context: ./delphi
       target: final
+      additional_contexts:
+        mathsrc: ./math
       # Share every layer with the delphi service's test build (CPU torch);
       # without this the default CUDA wheels build a second multi-GB image
       # in CI for no benefit — the poller needs no torch at runtime.
       args:
         USE_CPU_TORCH: "true"
-    command: ["python", "scripts/math_poller.py"]
+    command: ["python", "-m", "polismath.poller"]
     labels:
       polis_tag: test
     environment:
@@ -196,6 +198,8 @@ services:
     build:
       context: ./delphi
       target: test
+      additional_contexts:
+        mathsrc: ./math
       args:
         USE_CPU_TORCH: "true"
     labels:

docker-compose.yml

@@ -81,6 +81,10 @@ services:
     build:
       context: ./delphi
       target: final
+      # The math engine source (installed into the image) lives outside the
+      # build context since cutover Step #4 — see delphi/Dockerfile.
+      additional_contexts:
+        mathsrc: ./math
     labels:
       polis_tag: ${TAG:-dev}
     environment:
@@ -135,9 +139,11 @@ services:
     build:
       context: ./delphi
       target: final
+      additional_contexts:
+        mathsrc: ./math
     labels:
       polis_tag: ${TAG:-dev}
-    command: ["python", "scripts/math_poller.py"]
+    command: ["python", "-m", "polismath.poller"]
     environment:
       - DATABASE_URL=${DATABASE_URL}
       - DATABASE_SSL_MODE=${DATABASE_SSL_MODE:-disable}