Resolve relative inline links to fully-qualified URLs during scrape

ingest/postgis_docs.py

@@ -26,6 +26,7 @@
     fetch_page_as_soup,
     get_postgis_page_urls,
     postgis_html_to_markdown,
+    resolve_relative_links,
 )
 from ingest.utils.db import build_database_uri
 
@@ -57,6 +58,7 @@ def get_pages(self) -> Iterable[PageSource]:
 
             title = extract_title(soup, fallback="PostGIS Documentation")
             soup = clean_postgis_html(soup)
+            soup = resolve_relative_links(soup, full_url)
             markdown = postgis_html_to_markdown(soup)
 
             page = Page(

ingest/postgres_docs.py

@@ -15,6 +15,7 @@
 from ingest.utils.beautiful_soup import (
     extract_postgres_page_metadata,
     postgres_html_to_markdown,
+    resolve_relative_links,
 )
 
 POSTGRES_DIR = THIS_DIR / "postgres"
@@ -126,7 +127,7 @@ def build_html() -> None:
     )
 
 
-def build_markdown() -> None:
+def build_markdown(version: int) -> None:
     print("converting to markdown...")
     if MD_DIR.exists():
         shutil.rmtree(MD_DIR)
@@ -162,6 +163,8 @@ def build_markdown() -> None:
         except SystemError:
             raise SystemError(f"No div with id found in {html_file}")
 
+        page_url = f"{POSTGRES_BASE_URL}/{version}/{slug}"
+        soup = resolve_relative_links(soup, page_url)
         md_content = postgres_html_to_markdown(soup, is_refentry)
         md_content = f"""---
 title: {title_text}
@@ -229,7 +232,7 @@ def main():
         print(f"Building Postgres {version} ({tag}) documentation...")
         checkout_tag(tag)
         build_html()
-        build_markdown()
+        build_markdown(version)
         PostgresDocsImporter(version).run(conn)
 
 

ingest/tiger_docs.py

@@ -18,6 +18,9 @@
     OPENAI_API_KEY,
     OPENAI_BASE_URL,
 )
+from ingest.utils.beautiful_soup import (
+    resolve_relative_links as resolve_relative_links_util,
+)
 from ingest.utils.db import build_database_uri
 from langchain_text_splitters import (
     MarkdownHeaderTextSplitter,
@@ -576,6 +579,27 @@ def strip_data_images(self, soup):
 
         return soup
 
+    def resolve_relative_links(self, soup, base_url):
+        """Rewrite relative <a href> links to fully-qualified URLs.
+
+        Delegates to the shared ingest.utils.beautiful_soup implementation so
+        the two spiders don't drift; keeps the debug-level resolved-link count.
+        """
+        hrefs_before = [link["href"] for link in soup.find_all("a", href=True)]
+        soup = resolve_relative_links_util(soup, base_url)
+        links_resolved = sum(
+            1
+            for before, link in zip(
+                hrefs_before, soup.find_all("a", href=True), strict=True
+            )
+            if link["href"] != before
+        )
+
+        if links_resolved > 0:
+            self.logger.debug(f"Resolved {links_resolved} relative links")
+
+        return soup
+
     def convert_callouts_to_admonitions(self, soup):
         """Convert div.callout elements with h6 to admonition-style markdown callouts"""
         callouts_converted = 0
@@ -940,6 +964,9 @@ def parse(self, response):
             if self.should_strip_data_images:
                 soup = self.strip_data_images(soup)
 
+            # Resolve relative links to fully-qualified URLs
+            soup = self.resolve_relative_links(soup, url)
+
             # Convert callout divs to admonitions
             soup = self.convert_callouts_to_admonitions(soup)
 

ingest/utils/beautiful_soup.py

@@ -30,6 +30,13 @@
 ]
 
 
+def resolve_relative_links(soup: BeautifulSoup, base_url: str) -> BeautifulSoup:
+    """Rewrite relative <a href> links to fully-qualified URLs against base_url."""
+    for link in soup.find_all("a", href=True):
+        link["href"] = urljoin(base_url, link["href"])
+    return soup
+
+
 def clean_postgis_html(soup: BeautifulSoup) -> BeautifulSoup:
     """Remove navigation, scripts, styles, and data-URI images from PostGIS HTML."""
     for selector in POSTGIS_REMOVE_SELECTORS: