Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
13 changes: 7 additions & 6 deletions docker/omnigraph-server.Dockerfile
Original file line number Diff line number Diff line change
Expand Up @@ -31,13 +31,14 @@
# covered it when it did not, and a partial bump is silent until deploy.

ARG OMNIGRAPH_VERSION=0.10.0
# Upstream tag to fetch from. `edge` is the rolling build of upstream main,
# republished on every push there; a real release is `v${OMNIGRAPH_VERSION}`.
# Upstream tag to fetch from. A real release is `v${OMNIGRAPH_VERSION}`; `edge`
# is the rolling build of upstream main, republished on every push there.
# Kept separate because on a moving tag the two differ — see
# witan_core/omnigraph_install.py :: _OMNIGRAPH_RELEASE_TAG.
ARG OMNIGRAPH_RELEASE_TAG=edge
ARG OMNIGRAPH_SHA256_X86_64=6a0fba8842a2071c558abf2c1a399ce5e11d359dff78b6ae6ff3676617f95680
ARG OMNIGRAPH_SHA256_ARM64=dd40fa4169a89af41cddbdeb8fe441b714438633297e153876b4889ec0af3a86
# witan_core/omnigraph_install.py :: _OMNIGRAPH_RELEASE_TAG, which also records
# why this is a release tag again and what Lance 11 asks of an existing graph.
ARG OMNIGRAPH_RELEASE_TAG=v0.10.0
ARG OMNIGRAPH_SHA256_X86_64=05d3ce4ec0ab51a876befd89b643c3e7f2d5489be0398a38cef6fb3a0d257fc1
ARG OMNIGRAPH_SHA256_ARM64=dd3ac09123a68882454db7e689da4c306c41677826237098df4e76b0f73d8d5e

# ── Fetch + checksum-verify the release, extract both binaries ────────────────
FROM debian:trixie-slim@sha256:3a39a0592364683e6bab97937b72cad5a8fa6dcbbee90edb3bb48c7f8e94f258 AS fetch
Expand Down
13 changes: 7 additions & 6 deletions docker/witan.Dockerfile
Original file line number Diff line number Diff line change
Expand Up @@ -23,13 +23,14 @@ ARG PYTHON_VERSION=3.14
# version is an outage. Renovate covers all three; `just check-omnigraph-pins`
# is the CI backstop.
ARG OMNIGRAPH_VERSION=0.10.0
# Upstream tag to fetch from. `edge` is the rolling build of upstream main,
# republished on every push there; a real release is `v${OMNIGRAPH_VERSION}`.
# Upstream tag to fetch from. A real release is `v${OMNIGRAPH_VERSION}`; `edge`
# is the rolling build of upstream main, republished on every push there.
# Kept separate because on a moving tag the two differ — see
# witan_core/omnigraph_install.py :: _OMNIGRAPH_RELEASE_TAG.
ARG OMNIGRAPH_RELEASE_TAG=edge
ARG OMNIGRAPH_SHA256_X86_64=6a0fba8842a2071c558abf2c1a399ce5e11d359dff78b6ae6ff3676617f95680
ARG OMNIGRAPH_SHA256_ARM64=dd40fa4169a89af41cddbdeb8fe441b714438633297e153876b4889ec0af3a86
# witan_core/omnigraph_install.py :: _OMNIGRAPH_RELEASE_TAG, which also records
# why this is a release tag again and what Lance 11 asks of an existing graph.
ARG OMNIGRAPH_RELEASE_TAG=v0.10.0
ARG OMNIGRAPH_SHA256_X86_64=05d3ce4ec0ab51a876befd89b643c3e7f2d5489be0398a38cef6fb3a0d257fc1
ARG OMNIGRAPH_SHA256_ARM64=dd3ac09123a68882454db7e689da4c306c41677826237098df4e76b0f73d8d5e
# Keep in lockstep with witan-council's version (mcp/servers/witan/pyproject.toml
# [project].version / [tool.bumpversion]); it labels the built image.
ARG WITAN_VERSION=0.8.0
Expand Down
33 changes: 33 additions & 0 deletions packages/witan-core/tests/test_omnigraph.py
Original file line number Diff line number Diff line change
Expand Up @@ -1845,3 +1845,36 @@ def test_the_wait_is_capped_at_the_last_viable_moment(monkeypatch):
elapsed = time.monotonic() - started
assert elapsed < 2.0, "should give up near the viable boundary, not at 30s"
assert elapsed > 0.2, "should have waited while admission was still viable"


# ── omnigraph 0.10.0's full-text rebuild guard (upstream #581) ──────────────
#
# Literal, for the reason `_WRITE_AUTHORITY_STDERR` is: taken from the `#[error]`
# attribute on `OmniError::FullTextIndexRebuildRequired` in
# crates/omnigraph/src/error.rs at v0.10.0.
_FTS_REBUILD_STDERR = (
"omnigraph query failed:\n"
"full-text index 'memory_content' requires rebuild: analyzer generation "
"cannot be proven compatible; run omnigraph rebuild-full-text-indexes "
"<URI> --branch <branch> on the live branch (historical snapshots are "
"unchanged)"
)


def test_full_text_rebuild_required_is_terminal_on_the_cli_path():
"""Lance 11 changed the analyzer, so an index built by Lance 10 cannot serve
a 0.10.0 `search()`/`bm25()` query. Retrying never clears it — upstream says
so outright — so classifying it RETRYABLE would burn the whole attempt
budget and then report a timeout-shaped failure for a permanent condition
whose remedy is printed right there in the message.
"""
assert (
og._classify_cli_error(_FTS_REBUILD_STDERR) == _http.FULL_TEXT_REBUILD_REQUIRED
)


def test_full_text_rebuild_is_not_confused_with_a_repair():
"""NEEDS_REPAIR would run `omnigraph repair --force` on a graph that is not
damaged. Upstream is explicit that ordinary reads stay available and only
the full-text index is refused."""
assert og._classify_cli_error(_FTS_REBUILD_STDERR) != _http.NEEDS_REPAIR
46 changes: 46 additions & 0 deletions packages/witan-core/tests/test_omnigraph_http.py
Original file line number Diff line number Diff line change
Expand Up @@ -227,6 +227,52 @@ def test_repair_still_wins_over_a_conflict_status():
assert ogh.classify_status(409, message) == ogh.NEEDS_REPAIR


def test_full_text_rebuild_required_is_terminal_not_a_retryable_conflict():
"""omnigraph 0.10.0 (upstream #581) answers a stale full-text index with a
409 carrying `full_text_index_rebuild_required`. The bare-409 rule below it
would retry that to exhaustion and bury the remedy, so this must be caught
on the structured key AND on the server's prose."""
body = json.dumps(
{
"error": (
"full-text index 'memory_content' requires rebuild: analyzer "
"generation cannot be proven compatible; run omnigraph "
"rebuild-full-text-indexes <URI> --branch <branch> on the live "
"branch (historical snapshots are unchanged)"
),
"code": "conflict",
"details": {
"full_text_index_rebuild_required": {
"index": "memory_content",
"reason": "analyzer generation cannot be proven compatible",
}
},
}
)
assert ogh.classify_status(409, body) == ogh.FULL_TEXT_REBUILD_REQUIRED
assert ogh.classify_status(409, body) != ogh.RETRYABLE

# The detail key alone is enough — a relay that drops the prose still
# classifies correctly.
assert (
ogh.classify_status(
409, '{"details": {"full_text_index_rebuild_required": {}}}'
)
== ogh.FULL_TEXT_REBUILD_REQUIRED
)


def test_a_plain_conflict_is_still_retryable():
"""The guard above must not swallow the ordinary lost-race 409 that
`task_claim` depends on."""
message = (
"write authority 'graph_head:main' changed during preparation "
"(expected 01M08E24Y, current 01M08E27K) - reprepare from the "
"current branch state"
)
assert ogh.classify_status(409, message) == ogh.RETRYABLE


def test_error_message_uses_the_servers_own_wording():
body = json.dumps({"error": "policy denied action 'change'", "code": "forbidden"})
assert ogh.error_message(403, body) == (
Expand Down
30 changes: 30 additions & 0 deletions packages/witan-core/witan_core/omnigraph.py
Original file line number Diff line number Diff line change
Expand Up @@ -146,6 +146,21 @@
# node:ClaimRow); reopen the graph read-write before retrying". Matching the
# stem they share rather than either sentence.
_RECOVERY_REQUIRED = ("recovery required", "recovery_required")

# Checked BEFORE _RETRYABLE for the same reason the HTTP side checks it before
# its 409 catch-all: this is terminal, and retrying only delays the one message
# that says what to do. omnigraph 0.10.0 prints, verbatim:
#
# full-text index '<name>' requires rebuild: <reason>; run omnigraph
# rebuild-full-text-indexes <URI> --branch <branch> on the live branch
# (historical snapshots are unchanged)
#
# The command name is the marker rather than "requires rebuild" alone, which is
# generic enough to collide with a future unrelated message.
_FULL_TEXT_REBUILD_REQUIRED = (
"rebuild-full-text-indexes",
"full_text_index_rebuild_required",
)
_MAX_ATTEMPTS = 8

# omnigraph uses strict single-version storage: a release that bumps the
Expand Down Expand Up @@ -524,6 +539,8 @@ def _classify_cli_error(stderr: str) -> str:
return _http.PRECONDITION_FAILED
if any(m in lowered for m in _RECOVERY_REQUIRED):
return _http.RECOVERY_REQUIRED
if any(m in lowered for m in _FULL_TEXT_REBUILD_REQUIRED):
return _http.FULL_TEXT_REBUILD_REQUIRED
if any(m in lowered for m in _NEEDS_REPAIR):
return _http.NEEDS_REPAIR
if any(m in lowered for m in _RETRYABLE):
Expand Down Expand Up @@ -2086,6 +2103,19 @@ def _retry_loop(
f"you read it. NOTHING WAS WRITTEN, and this write must "
f"not be retried as-is; re-read and decide:\n{err.strip()}"
)
if kind == _http.FULL_TEXT_REBUILD_REQUIRED:
# TERMINAL, on the first attempt. The index cannot serve
# this engine's analyzer and no amount of retrying changes
# that; upstream says so outright. Ordinary reads are
# unaffected — only full-text queries land here — so this is
# not a broken graph, and the remedy is in the server's own
# message, which is why it is passed through whole.
raise RuntimeError(
f"omnigraph {label} was refused because a full-text "
f"index needs an explicit rebuild for this engine "
f"version. Retrying will not clear it, and ordinary "
f"(non-search) reads are unaffected:\n{err.strip()}"
)
if surface_conflict and kind == _http.RETRYABLE:
# A compare-and-swap caller wants to lose the race, not
# re-apply its write over the winner. Surface immediately.
Expand Down
36 changes: 36 additions & 0 deletions packages/witan-core/witan_core/omnigraph_http.py
Original file line number Diff line number Diff line change
Expand Up @@ -79,6 +79,30 @@
#: classification is what makes that true on our side too.
PRECONDITION_FAILED = "precondition_failed"

#: A full-text index whose analyzer generation cannot be proven compatible with
#: the running engine — omnigraph 0.10.0's ``FullTextIndexRebuildRequired``,
#: answered as HTTP 409 with a ``full_text_index_rebuild_required`` detail
#: object (upstream #581).
#:
#: ★ TERMINAL, AND IT HAS TO BE NAMED BECAUSE 409 ALREADY MEANS "RETRY ME".
#: :func:`classify_status` treats a bare 409 as :data:`RETRYABLE` on the status
#: alone — deliberately, see the comment there — so without this check first,
#: every BM25/``search()`` request against a pre-upgrade index would be retried
#: the full budget and then surface as a generic retry-exhausted error, hiding
#: the one sentence that says what to do. Upstream is explicit that it "is not
#: cleared by retrying".
#:
#: ★ IT IS NOT A BROKEN GRAPH, and specifically not :data:`NEEDS_REPAIR`.
#: Upstream's own doc comment on the variant: "Ordinary reads remain available;
#: do not return a partial indexed result." Only full-text queries are refused,
#: and the fix is one explicit command per branch:
#:
#: omnigraph rebuild-full-text-indexes <URI> --branch <branch>
#:
#: Running ``omnigraph repair`` here would be a much heavier hammer aimed at
#: the wrong thing.
FULL_TEXT_REBUILD_REQUIRED = "full_text_rebuild_required"

#: The branch-wide write barrier: "recovery required for operation …: pending
#: Load recovery operation blocks writes on branch 'main'".
#:
Expand Down Expand Up @@ -236,6 +260,18 @@ def classify_status(status: int, message: str) -> str:
for marker in ("stale view", "manifest table version", "refresh and retry")
):
return RETRYABLE
# BEFORE the 409 catch-all below, which would otherwise retry this to
# exhaustion — see FULL_TEXT_REBUILD_REQUIRED. The structured detail key is
# the primary signal because it is a wire contract; the prose is checked too
# so the CLI path and any non-JSON relay reach the same answer.
if any(
marker in lowered
for marker in (
"full_text_index_rebuild_required",
"rebuild-full-text-indexes",
)
):
return FULL_TEXT_REBUILD_REQUIRED
if status == 409:
# ★ A CONFLICT IS THE ONE STATUS THAT MEANS "RETRY ME" MOST LITERALLY,
# and it was falling through to FATAL because this function only ever
Expand Down
Loading
Loading