fix(semantic): preserve abstracts after list markers

openviking/storage/queuefs/semantic_processor.py

@@ -101,6 +101,30 @@ class SemanticProcessor(DequeueHandlerBase):
     _request_stats_by_telemetry_id: Dict[str, RequestQueueStats] = {}
     _request_stats_order: List[str] = []
     _max_cached_stats = 256
+    _non_terminal_abbreviations = frozenset(
+        {
+            "approx",
+            "co",
+            "dept",
+            "dr",
+            "e.g",
+            "etc",
+            "fig",
+            "i.e",
+            "inc",
+            "jr",
+            "ltd",
+            "mr",
+            "mrs",
+            "ms",
+            "mx",
+            "no",
+            "prof",
+            "sr",
+            "st",
+            "vs",
+        }
+    )
 
     def __init__(self, max_concurrent_llm: int = 64):
         """
@@ -1188,6 +1212,75 @@ def replace_index(match):
 
         return re.sub(r"\[(\d+)\]", replace_index, generated_content)
 
+    @staticmethod
+    def _is_list_marker_period(text: str, period_index: int) -> bool:
+        line_start = text.rfind("\n", 0, period_index) + 1
+        marker = text[line_start:period_index]
+        return re.fullmatch(r"\s*(?:\d+(?:\.\d+)*|[A-Za-z])", marker) is not None
+
+    @classmethod
+    def _is_abbreviation_period(cls, text: str, period_index: int) -> bool:
+        if not text[period_index + 1 :].strip():
+            return False
+
+        token_match = re.search(r"[A-Za-z]+(?:\.[A-Za-z]+)*$", text[:period_index])
+        if token_match is None:
+            return False
+
+        token = token_match.group(0).lower()
+        return (
+            token in cls._non_terminal_abbreviations
+            or re.fullmatch(r"(?:[a-z]\.)+[a-z]", token) is not None
+        )
+
+    @staticmethod
+    def _is_structural_paragraph(text: str, paragraph_end: int) -> bool:
+        preceding_text = text[:paragraph_end]
+        previous_breaks = list(re.finditer(r"\n[ \t]*\n+", preceding_text))
+        paragraph_start = previous_breaks[-1].end() if previous_breaks else 0
+        paragraph = preceding_text[paragraph_start:].strip()
+        if re.fullmatch(r"#{1,6}\s+\S[^\n]*", paragraph):
+            return True
+        marker = r"(?:\((?:\d+(?:\.\d+)*|[A-Za-z])\)|(?:\d+(?:\.\d+)*|[A-Za-z])[.)])"
+        return re.fullmatch(rf"{marker}\s+\S[^\n]*", paragraph) is not None
+
+    def _find_sentence_boundaries(self, text: str) -> List[int]:
+        boundaries = set()
+
+        for paragraph_break in re.finditer(r"\n[ \t]*\n+", text):
+            paragraph_end = paragraph_break.start()
+            if not self._is_structural_paragraph(text, paragraph_end):
+                boundaries.add(paragraph_end)
+
+        closing_characters = "\"'”’)]}）】》〉」』"
+        for punctuation_match in re.finditer(r"[.!?。？！]", text):
+            punctuation_index = punctuation_match.start()
+            punctuation = punctuation_match.group(0)
+            boundary = punctuation_match.end()
+
+            while boundary < len(text) and text[boundary] in closing_characters:
+                boundary += 1
+
+            if punctuation in ".!?":
+                if boundary < len(text) and not text[boundary].isspace():
+                    continue
+                if punctuation == ".":
+                    if (
+                        punctuation_index > 0
+                        and punctuation_index + 1 < len(text)
+                        and text[punctuation_index - 1].isdigit()
+                        and text[punctuation_index + 1].isdigit()
+                    ):
+                        continue
+                    if self._is_list_marker_period(text, punctuation_index):
+                        continue
+                    if self._is_abbreviation_period(text, punctuation_index):
+                        continue
+
+            boundaries.add(boundary)
+
+        return sorted(boundaries)
+
     def _truncate_generated_text(self, text: str, max_chars: int) -> str:
         if max_chars <= 0 or len(text) <= max_chars:
             return text
@@ -1197,8 +1290,7 @@ def _truncate_generated_text(self, text: str, max_chars: int) -> str:
 
         first_sentence_end = None
         last_sentence_end_within_limit = None
-        for sentence_end_match in re.finditer(r"\.(?!\d)(?=\s|$)|[!?](?=\s|$)|[。？！]", text):
-            sentence_end = sentence_end_match.end()
+        for sentence_end in self._find_sentence_boundaries(text):
             if first_sentence_end is None:
                 first_sentence_end = sentence_end
             if sentence_end <= max_chars:

tests/storage/test_semantic_processor_l0_l1.py

@@ -3,6 +3,8 @@
 
 from types import SimpleNamespace
 
+import pytest
+
 from openviking.storage.queuefs import semantic_processor as semantic_processor_module
 from openviking.storage.queuefs.semantic_processor import SemanticProcessor
 
@@ -151,3 +153,121 @@ def test_abstract_truncation_accepts_sentence_period_after_number(monkeypatch):
     _, abstract = processor._enforce_size_limits("# README\n\nBody", abstract)
 
     assert abstract == "This import check was generated at 16:55."
+
+
+@pytest.mark.parametrize(
+    ("text", "max_chars", "expected"),
+    [
+        pytest.param(
+            "1. **Some Directory**\n\n"
+            "This directory contains a detailed collection of onboarding documents",
+            80,
+            "1. **Some Directory**\n\nThis directory contains a detailed collection of...",
+            id="numbered-list-marker",
+        ),
+        pytest.param(
+            "a. **Setup**\n\n"
+            "Follow the onboarding guide to configure semantic retrieval for every workspace",
+            60,
+            "a. **Setup**\n\nFollow the onboarding guide to configure...",
+            id="lettered-list-marker",
+        ),
+        pytest.param(
+            "1.2. **Setup**\n\n"
+            "Follow the onboarding guide to configure semantic retrieval for every workspace",
+            62,
+            "1.2. **Setup**\n\nFollow the onboarding guide to configure...",
+            id="hierarchical-numbered-list-marker",
+        ),
+        pytest.param(
+            "Version 3.14 is stable. Additional compatibility details follow.",
+            35,
+            "Version 3.14 is stable.",
+            id="decimal",
+        ),
+        pytest.param(
+            "Dr. Smith explains semantic retrieval across every indexed workspace without setup",
+            50,
+            "Dr. Smith explains semantic retrieval across...",
+            id="title-abbreviation",
+        ),
+        pytest.param(
+            "Use e.g. semantic tags to improve retrieval across every indexed workspace",
+            45,
+            "Use e.g. semantic tags to improve...",
+            id="latin-abbreviation",
+        ),
+        pytest.param(
+            "第一句说明检索能力。第二句包含更多细节，需要截断。",
+            12,
+            "第一句说明检索能力。",
+            id="cjk-punctuation",
+        ),
+        pytest.param(
+            "First paragraph has no terminal punctuation\n\n"
+            "Second paragraph continues with details",
+            55,
+            "First paragraph has no terminal punctuation",
+            id="paragraph-boundary",
+        ),
+        pytest.param(
+            "# Overview\n\n"
+            "This opening sentence contains enough detail to exceed the configured limit "
+            "before it finally ends. Another sentence follows.",
+            45,
+            "# Overview\n\n"
+            "This opening sentence contains enough detail to exceed the configured limit "
+            "before it finally ends.",
+            id="long-first-sentence-after-heading",
+        ),
+        pytest.param(
+            "OK. Additional explanation follows.",
+            10,
+            "OK.",
+            id="short-complete-sentence",
+        ),
+        pytest.param("abcdefghij", 4, "a...", id="short-fragment-fallback"),
+    ],
+)
+def test_truncation_uses_meaningful_boundaries(text, max_chars, expected):
+    processor = SemanticProcessor()
+
+    assert processor._truncate_generated_text(text, max_chars) == expected
+
+
+@pytest.mark.parametrize(
+    ("max_chars", "expected"),
+    [
+        pytest.param(-1, "abcdef", id="negative-disabled"),
+        pytest.param(0, "abcdef", id="disabled"),
+        pytest.param(1, "a", id="one-character"),
+        pytest.param(2, "ab", id="two-characters"),
+        pytest.param(3, "abc", id="three-characters"),
+    ],
+)
+def test_truncation_handles_tiny_limits(max_chars, expected):
+    processor = SemanticProcessor()
+
+    assert processor._truncate_generated_text("abcdef", max_chars) == expected
+
+
+def test_normalize_overview_does_not_collapse_abstract_to_numbered_marker(monkeypatch):
+    _patch_semantic_limits(monkeypatch)
+    processor = SemanticProcessor()
+    generated = (
+        "# Some Directory\n\n"
+        "1. **Some Directory**\n\n"
+        + "This directory contains a detailed collection of onboarding documents "
+        * 8
+        + "\n\n"
+        "## Quick Navigation\n\n"
+        "- Read the onboarding guide"
+    )
+
+    overview, abstract = processor._normalize_overview_generation(generated)
+
+    assert overview == generated
+    assert abstract.startswith("1. **Some Directory**\nThis directory contains")
+    assert abstract.endswith("...")
+    assert abstract != "1."
+    assert len(abstract) <= 256