feat(document-outline.js): carry a PDF region's caption and interpretation

packages/document-outline.js/src/outline/pdf-regions.test.ts

@@ -173,12 +173,64 @@ describe("segmentPdfRegions", () => {
     expect(captionRegion?.confidence).toBeGreaterThan(0);
     expect(captionRegion?.confidence).toBeLessThanOrEqual(1);
 
+    // Recorded in both directions: the caption stays its own region, and the figure it labels now
+    // carries that text. The pass already had to work out which figure the caption belonged to in
+    // order to classify it, and used to drop the answer -- so a consumer wanting a figure's own label
+    // had to re-derive the adjacency this function had just computed.
+    expect(figureRegion?.caption).toBe(
+      "Figure 1: a chart of quarterly results.",
+    );
+    // Associated, not moved: projecting every region's text must read the caption exactly once. Asserted
+    // by counting it across every region rather than by inspecting the caption region's own items, which
+    // the fixture already guarantees and which would pass even if the figure had swallowed the item too.
+    const occurrences = regions.filter((region) =>
+      region.items.some(
+        (item) =>
+          item.kind === "text" &&
+          item.text === "Figure 1: a chart of quarterly results.",
+      ),
+    );
+    expect(occurrences).toHaveLength(1);
+
     const proseRegion = regions.find(
       (region) => region.classification === "column",
     );
     expect(proseRegion?.items).toHaveLength(4);
   });
 
+  it("gives a figure the nearer of two candidate captions", () => {
+    // A figure sandwiched between two short runs has two candidates and only one is its label. The same
+    // gap that decides a caption's own confidence decides which figure wins it, so the nearer text is
+    // the one recorded on the figure.
+    //
+    // Both gaps sit in a narrow window the segmentation forces: wider than the LOCAL cut threshold
+    // (1.5x the caption's own ~10pt font size, so ~15pt -- below that the run is not split off as its
+    // own region at all) and within CAPTION_GAP_PT (24pt, beyond which it is not a caption). Above is
+    // 18pt away, below is 22pt.
+    const figure: LayoutItem = {
+      kind: "image",
+      imageId: "img-2",
+      xPt: 100,
+      yPt: 400,
+      widthPt: 300,
+      heightPt: 200,
+    };
+    // The nearer caption is the one ABOVE, which is the arrangement that makes the tie-break
+    // load-bearing: regions arrive sorted top-to-bottom, so `above` is processed first, and a naive
+    // last-writer-wins would record `below` instead. With the fixture the other way round both rules
+    // agree and the test proves nothing -- verified by mutating the comparison to `if (true)`, under
+    // which the earlier version of this case still passed.
+    const above = line(150, 618, "Figure 2: the nearer caption.");
+    const below = line(150, 368, "Further away, below the figure.");
+
+    const regions = segmentPdfRegions(page([above, figure, below]));
+    const figureRegion = regions.find(
+      (region) => region.classification === "figure",
+    );
+
+    expect(figureRegion?.caption).toBe("Figure 2: the nearer caption.");
+  });
+
   it("does not attach a caption to a figure it is not vertically adjacent to", () => {
     const figure: LayoutItem = {
       kind: "image",
@@ -1188,7 +1240,8 @@ describe("attachCaptions", () => {
       "Figure 1.",
     );
     const [region] = attachCaptions([figureRegion, caption]);
-    expect(region).toBe(figureRegion); // unaffected, non-column/unknown region passes through
+    // The figure itself is no longer untouched: it now carries the matched caption's text on its own `caption` field, in addition to the caption staying its own 'caption'-classified region below.
+    expect(region).toEqual({ ...figureRegion, caption: "Figure 1." });
     const result = attachCaptions([figureRegion, caption])[1]!;
     expect(result.classification).toBe("caption");
   });

packages/document-outline.js/src/outline/pdf-regions.ts

@@ -1,3 +1,4 @@
+import type { ContentInterpretation } from "document-schema.js";
 import type { LayoutItem, LayoutPage } from "pdf-codec";
 import type { RegionClassification } from "./regions";
 
@@ -18,6 +19,21 @@ export interface PdfRegion {
   readonly items: readonly LayoutItem[];
   readonly classification: RegionClassification;
   readonly confidence: number;
+  // The caption text this region is labelled by, set on a 'figure' region when attachCaptions below
+  // found one. Associated, not moved: the caption remains its own 'caption'-classified region in the
+  // returned array, so a consumer projecting every region's text still reads it exactly once -- the same
+  // rule document-schema.js's ContentImageBlock.caption follows for a docx figure, and for the same
+  // reason. Absent when no adjacent short text run claimed this figure, which is the common case.
+  readonly caption?: string;
+  // What a consumer read out of this region, when it has read it -- the schema's own annotation channel,
+  // reused here rather than re-minted (ContentInterpretationSchema is exported for exactly this).
+  //
+  // Never set by segmentPdfRegions: this package infers geometry, and nothing in it calls a model or
+  // reads pixels. The field exists because a PDF region is the one place an interpretation of a *vector*
+  // figure can attach at all -- a chart drawn as paths is dozens of sibling rect/line/path items with no
+  // single content node whose extent is the chart, so the region is the only container whose content IS
+  // the thing being described. A consumer that rasterises a region and reads it puts the result here.
+  readonly interpretation?: ContentInterpretation;
 }
 
 export interface Bounds {
@@ -462,40 +478,76 @@ const CAPTION_MAX_CHARS = 160;
 const CAPTION_GAP_PT = 24;
 
 // Second pass: a short text leaf classified 'column' or 'unknown' that sits immediately above or below a 'figure' leaf, and horizontally overlaps it, is a caption -- captions are a RELATIONSHIP to a figure, not a standalone geometric signature, so this can only run after every leaf already has its own first-pass classification.
+//
+// The relationship is recorded in BOTH directions, which it was not before: the text leaf becomes a 'caption' region as it always did, and the figure it labels now carries that text on its own `caption`. The pass already had to find which figure a caption belonged to in order to classify it at all, and then dropped the answer -- so a consumer wanting a figure's own label had to re-derive the adjacency this function had just computed. That matters for the case the field exists to serve: handing a figure's region to something that reads it (a vision model, say) is far more useful when the author's own caption for that figure comes with it.
 export function attachCaptions(regions: readonly PdfRegion[]): PdfRegion[] {
   const figures = regions.filter(
     (region) => region.classification === "figure",
   );
-  return regions.map((region) => {
+
+  // Figure -> its nearest claiming caption. Nearest, because a figure sandwiched between two short runs has two candidates and only one of them is its label; the same gap that decides the caption's own confidence decides which figure wins it.
+  //
+  // On an exact tie the run ABOVE the figure wins, because `regions` arrives sorted top-to-bottom (the descending-yPt sort where the leaves are built) and the comparison below is strict. That is the less conventional answer for a figure label, so it is stated here rather than left to be inferred from a sort two hundred lines away, and a test pins it. The same strictness means a run equidistant from two figures is claimed by the upper one.
+  //
+  // The two views deliberately do not agree on counts, and a consumer should not assume they do: BOTH runs in a sandwich are reclassified 'caption' (behaviour this pass already had), while only one figure carries text. The loser is a 'caption' region that labels nothing.
+  const captionFor = new Map<PdfRegion, { text: string; gap: number }>();
+  // Caption candidate -> the gap to the figure it claims. Only the gap is kept: which figure won is
+  // recorded on the figure's own side, in `captionFor`.
+  const claimed = new Map<PdfRegion, number>();
+
+  for (const region of regions) {
     if (
       region.classification !== "column" &&
       region.classification !== "unknown"
     ) {
-      return region;
+      continue;
     }
-    const text = region.items
-      .map((item) => (item.kind === "text" ? item.text : ""))
-      .join(" ")
-      .trim();
-    if (text.length === 0 || text.length > CAPTION_MAX_CHARS) return region;
-
-    // Only the NUMBER of the nearest qualifying gap ever feeds into this region's own output (confidence below) -- which figure it came from is never observable, so there is no need to track a `nearest` figure at all, only the minimum qualifying gap itself.
-    const qualifyingGaps = figures
-      .filter((figure) => horizontallyOverlaps(region.bounds, figure.bounds))
-      .map((figure) => verticalGap(region.bounds, figure.bounds))
-      .filter(
-        (gap): gap is number => gap !== undefined && gap <= CAPTION_GAP_PT,
-      );
-    if (qualifyingGaps.length === 0) return region;
-    const nearestGap = Math.min(...qualifyingGaps);
-    return {
-      ...region,
-      classification: "caption",
-      confidence: clamp01(1 - nearestGap / CAPTION_GAP_PT),
-    };
+    const text = regionText(region);
+    if (text.length === 0 || text.length > CAPTION_MAX_CHARS) continue;
+
+    let nearest: PdfRegion | undefined;
+    let nearestGap = Number.POSITIVE_INFINITY;
+    for (const figure of figures) {
+      if (!horizontallyOverlaps(region.bounds, figure.bounds)) continue;
+      const gap = verticalGap(region.bounds, figure.bounds);
+      if (gap !== undefined && gap <= CAPTION_GAP_PT && gap < nearestGap) {
+        nearest = figure;
+        nearestGap = gap;
+      }
+    }
+    if (nearest === undefined) continue;
+
+    claimed.set(region, nearestGap);
+    const existing = captionFor.get(nearest);
+    if (existing === undefined || nearestGap < existing.gap) {
+      captionFor.set(nearest, { text, gap: nearestGap });
+    }
+  }
+
+  return regions.map((region) => {
+    const gap = claimed.get(region);
+    if (gap !== undefined) {
+      return {
+        ...region,
+        classification: "caption" as const,
+        confidence: clamp01(1 - gap / CAPTION_GAP_PT),
+      };
+    }
+    const caption = captionFor.get(region);
+    return caption === undefined
+      ? region
+      : { ...region, caption: caption.text };
   });
 }
 
+// A region's own text content, joined and trimmed. Non-text items contribute nothing.
+function regionText(region: PdfRegion): string {
+  return region.items
+    .map((item) => (item.kind === "text" ? item.text : ""))
+    .join(" ")
+    .trim();
+}
+
 export function horizontallyOverlaps(
   a: PdfRegionBounds,
   b: PdfRegionBounds,