From fd38669ff82b4022e193835c88c47fb28d77c32a Mon Sep 17 00:00:00 2001
From: arimu1 <19286898+arimu1@users.noreply.github.com>
Date: Tue, 21 Jul 2026 09:22:45 +0700
Subject: [PATCH 1/2] Resolve relative inline links to fully-qualified URLs
during scrape
Relative hrefs from source HTML (e.g. /self-hosted/latest/migration/entire-database/)
were being carried verbatim into the markdown, producing links that
LLMs can't resolve on their own. Rewrite links against each
page's URL before converting to markdown, across the Tiger, PostGIS,
and Postgres ingest pipelines.
Closes #12
---
ingest/postgis_docs.py | 2 ++
ingest/postgres_docs.py | 7 +++++--
ingest/tiger_docs.py | 18 ++++++++++++++++++
ingest/utils/beautiful_soup.py | 7 +++++++
4 files changed, 32 insertions(+), 2 deletions(-)
diff --git a/ingest/postgis_docs.py b/ingest/postgis_docs.py
index 6cb8d72..c2c78ae 100644
--- a/ingest/postgis_docs.py
+++ b/ingest/postgis_docs.py
@@ -26,6 +26,7 @@
fetch_page_as_soup,
get_postgis_page_urls,
postgis_html_to_markdown,
+ resolve_relative_links,
)
from ingest.utils.db import build_database_uri
@@ -57,6 +58,7 @@ def get_pages(self) -> Iterable[PageSource]:
title = extract_title(soup, fallback="PostGIS Documentation")
soup = clean_postgis_html(soup)
+ soup = resolve_relative_links(soup, full_url)
markdown = postgis_html_to_markdown(soup)
page = Page(
diff --git a/ingest/postgres_docs.py b/ingest/postgres_docs.py
index 8c020f8..1ab6171 100644
--- a/ingest/postgres_docs.py
+++ b/ingest/postgres_docs.py
@@ -15,6 +15,7 @@
from ingest.utils.beautiful_soup import (
extract_postgres_page_metadata,
postgres_html_to_markdown,
+ resolve_relative_links,
)
POSTGRES_DIR = THIS_DIR / "postgres"
@@ -126,7 +127,7 @@ def build_html() -> None:
)
-def build_markdown() -> None:
+def build_markdown(version: int) -> None:
print("converting to markdown...")
if MD_DIR.exists():
shutil.rmtree(MD_DIR)
@@ -162,6 +163,8 @@ def build_markdown() -> None:
except SystemError:
raise SystemError(f"No div with id found in {html_file}")
+ page_url = f"{POSTGRES_BASE_URL}/{version}/{slug}"
+ soup = resolve_relative_links(soup, page_url)
md_content = postgres_html_to_markdown(soup, is_refentry)
md_content = f"""---
title: {title_text}
@@ -229,7 +232,7 @@ def main():
print(f"Building Postgres {version} ({tag}) documentation...")
checkout_tag(tag)
build_html()
- build_markdown()
+ build_markdown(version)
PostgresDocsImporter(version).run(conn)
diff --git a/ingest/tiger_docs.py b/ingest/tiger_docs.py
index 5f15a0d..c59a710 100644
--- a/ingest/tiger_docs.py
+++ b/ingest/tiger_docs.py
@@ -576,6 +576,21 @@ def strip_data_images(self, soup):
return soup
+ def resolve_relative_links(self, soup, base_url):
+ """Rewrite relative links to fully-qualified URLs"""
+ links_resolved = 0
+
+ for link in soup.find_all("a", href=True):
+ resolved = urljoin(base_url, link["href"])
+ if resolved != link["href"]:
+ link["href"] = resolved
+ links_resolved += 1
+
+ if links_resolved > 0:
+ self.logger.debug(f"Resolved {links_resolved} relative links")
+
+ return soup
+
def convert_callouts_to_admonitions(self, soup):
"""Convert div.callout elements with h6 to admonition-style markdown callouts"""
callouts_converted = 0
@@ -940,6 +955,9 @@ def parse(self, response):
if self.should_strip_data_images:
soup = self.strip_data_images(soup)
+ # Resolve relative links to fully-qualified URLs
+ soup = self.resolve_relative_links(soup, url)
+
# Convert callout divs to admonitions
soup = self.convert_callouts_to_admonitions(soup)
diff --git a/ingest/utils/beautiful_soup.py b/ingest/utils/beautiful_soup.py
index 7cae9d2..78b97e2 100644
--- a/ingest/utils/beautiful_soup.py
+++ b/ingest/utils/beautiful_soup.py
@@ -30,6 +30,13 @@
]
+def resolve_relative_links(soup: BeautifulSoup, base_url: str) -> BeautifulSoup:
+ """Rewrite relative links to fully-qualified URLs against base_url."""
+ for link in soup.find_all("a", href=True):
+ link["href"] = urljoin(base_url, link["href"])
+ return soup
+
+
def clean_postgis_html(soup: BeautifulSoup) -> BeautifulSoup:
"""Remove navigation, scripts, styles, and data-URI images from PostGIS HTML."""
for selector in POSTGIS_REMOVE_SELECTORS:
From 1280f17f00daebfa37eb6d884255ac30c286b9fc Mon Sep 17 00:00:00 2001
From: arimu1 <19286898+arimu1@users.noreply.github.com>
Date: Tue, 21 Jul 2026 10:00:33 +0700
Subject: [PATCH 2/2] Delegate spider's resolve_relative_links to shared util
The spider method had its own divergent implementation of link
resolution (conditional rewrite + debug counter) alongside the shared
ingest.utils.beautiful_soup.resolve_relative_links (unconditional
rewrite) used by the other two pipelines. Same name, same job, two
bodies that could silently drift. The spider method is now a thin
wrapper that delegates to the shared util and keeps its
debug-level resolved-link count.
---
ingest/tiger_docs.py | 25 +++++++++++++++++--------
1 file changed, 17 insertions(+), 8 deletions(-)
diff --git a/ingest/tiger_docs.py b/ingest/tiger_docs.py
index c59a710..d6c8a93 100644
--- a/ingest/tiger_docs.py
+++ b/ingest/tiger_docs.py
@@ -18,6 +18,9 @@
OPENAI_API_KEY,
OPENAI_BASE_URL,
)
+from ingest.utils.beautiful_soup import (
+ resolve_relative_links as resolve_relative_links_util,
+)
from ingest.utils.db import build_database_uri
from langchain_text_splitters import (
MarkdownHeaderTextSplitter,
@@ -577,14 +580,20 @@ def strip_data_images(self, soup):
return soup
def resolve_relative_links(self, soup, base_url):
- """Rewrite relative links to fully-qualified URLs"""
- links_resolved = 0
-
- for link in soup.find_all("a", href=True):
- resolved = urljoin(base_url, link["href"])
- if resolved != link["href"]:
- link["href"] = resolved
- links_resolved += 1
+ """Rewrite relative links to fully-qualified URLs.
+
+ Delegates to the shared ingest.utils.beautiful_soup implementation so
+ the two spiders don't drift; keeps the debug-level resolved-link count.
+ """
+ hrefs_before = [link["href"] for link in soup.find_all("a", href=True)]
+ soup = resolve_relative_links_util(soup, base_url)
+ links_resolved = sum(
+ 1
+ for before, link in zip(
+ hrefs_before, soup.find_all("a", href=True), strict=True
+ )
+ if link["href"] != before
+ )
if links_resolved > 0:
self.logger.debug(f"Resolved {links_resolved} relative links")