1+ """Offline-only fetcher stub — no network calls, no APIs."""
12from __future__ import annotations
23
3- import asyncio
44import logging
5- import re
65import time
76from typing import Optional
8- from urllib .parse import urlparse
97
108from moso_core .realtime .models import FetchResult
11- from moso_core .realtime .sources import SourceDefinition , get_sources_for_url
9+ from moso_core .realtime .sources import SourceDefinition
1210
1311logger = logging .getLogger (__name__ )
1412
15- FETCH_TIMEOUT = 15
16- MAX_REDIRECTS = 5
17- MAX_TEXT_SIZE = 500_000
18-
19- try :
20- import aiohttp
21-
22- AIOHTTP_AVAILABLE = True
23- except ImportError :
24- AIOHTTP_AVAILABLE = False
25-
26- try :
27- from bs4 import BeautifulSoup
28-
29- BEAUTIFULSOUP_AVAILABLE = True
30- except ImportError :
31- BEAUTIFULSOUP_AVAILABLE = False
32-
33-
34- def strip_html (html : str ) -> str :
35- if BEAUTIFULSOUP_AVAILABLE :
36- soup = BeautifulSoup (html , "html.parser" )
37- for tag in soup (["script" , "style" , "nav" , "footer" , "header" , "aside" ]):
38- tag .decompose ()
39- text = soup .get_text (separator = "\n " )
40- else :
41- text = re .sub (r"<[^>]+>" , " " , html )
42- text = re .sub (r"\s+" , " " , text )
43- lines = (line .strip () for line in text .splitlines ())
44- return "\n " .join (line for line in lines if line )
45-
46-
47- def extract_title (html : str ) -> str :
48- match = re .search (r"<title[^>]*>([^<]+)</title>" , html , re .IGNORECASE )
49- return match .group (1 ).strip () if match else ""
50-
51-
52- def truncate_text (text : str , max_chars : int = MAX_TEXT_SIZE ) -> str :
53- if len (text ) > max_chars :
54- return text [:max_chars ] + f"\n \n [...truncated at { max_chars } chars]"
55- return text
56-
5713
5814class Fetcher :
59- def __init__ (self ):
60- self ._session : Optional [aiohttp .ClientSession ] = None
61- self ._user_agent = (
62- "Mozilla/5.0 (Windows NT 10.0; Win64; x64) "
63- "AppleWebKit/537.36 (KHTML, like Gecko) "
64- "Chrome/120.0.0.0 Safari/537.36"
65- )
66-
67- async def _get_session (self ) -> aiohttp .ClientSession :
68- if self ._session is None or self ._session .closed :
69- timeout = aiohttp .ClientTimeout (total = FETCH_TIMEOUT )
70- connector = aiohttp .TCPConnector (
71- limit = 10 ,
72- ttl_dns_cache = 300 ,
73- ssl = True ,
74- )
75- self ._session = aiohttp .ClientSession (
76- timeout = timeout ,
77- connector = connector ,
78- headers = {"User-Agent" : self ._user_agent },
79- )
80- return self ._session
15+ """Stub fetcher that always returns offline results."""
8116
8217 async def fetch_url (
8318 self ,
8419 url : str ,
8520 source_name : str = "" ,
8621 tls_verify : bool = True ,
8722 ) -> FetchResult :
88- if not AIOHTTP_AVAILABLE :
89- return FetchResult (
90- url = url ,
91- source_name = source_name or url ,
92- raw_text = "" ,
93- parsed_text = "aiohttp is not installed. Install with: pip install aiohttp" ,
94- status_code = 0 ,
95- error = "aiohttp not available" ,
96- )
97-
98- session = await self ._get_session ()
99- redirect_chain : list [str ] = []
100- current_url = url
101- fetch_start = time .time ()
102-
103- for hop in range (MAX_REDIRECTS + 1 ):
104- try :
105- async with session .get (
106- current_url ,
107- allow_redirects = False ,
108- ssl = True ,
109- ) as resp :
110- status = resp .status
111- content_type = resp .content_type or "text/plain"
112-
113- if status in (301 , 302 , 303 , 307 , 308 ):
114- location = resp .headers .get ("Location" , "" )
115- if not location :
116- break
117- redirect_chain .append (current_url )
118- current_url = location
119- logger .debug ("Redirect %d -> %s" , hop + 1 , location )
120- continue
121-
122- raw = await resp .read ()
123- raw_text = raw .decode ("utf-8" , errors = "replace" )
124- raw_text = truncate_text (raw_text )
125-
126- encoding = resp .charset or "utf-8"
127- try :
128- decoded = raw .decode (encoding , errors = "replace" )
129- except (LookupError , UnicodeDecodeError ):
130- decoded = raw .decode ("utf-8" , errors = "replace" )
131-
132- parsed = extract_title (decoded ) + "\n \n " + strip_html (decoded )
133- parsed = truncate_text (parsed )
134-
135- elapsed = time .time () - fetch_start
136- logger .info (
137- "Fetched %s (%d, %.1fs, %d chars)" ,
138- url , status , elapsed , len (parsed ),
139- )
140-
141- return FetchResult (
142- url = url ,
143- source_name = source_name or url ,
144- raw_text = raw_text ,
145- parsed_text = parsed ,
146- status_code = status ,
147- timestamp = time .strftime ("%Y-%m-%dT%H:%M:%SZ" , time .gmtime ()),
148- content_type = content_type ,
149- redirect_chain = redirect_chain ,
150- tls_verified = True ,
151- )
152-
153- except asyncio .TimeoutError :
154- logger .warning ("Timeout fetching %s" , url )
155- return FetchResult (
156- url = url ,
157- source_name = source_name or url ,
158- raw_text = "" ,
159- parsed_text = "" ,
160- status_code = 0 ,
161- error = f"Timeout after { FETCH_TIMEOUT } s" ,
162- )
163- except aiohttp .ClientError as e :
164- logger .warning ("HTTP error fetching %s: %s" , url , e )
165- return FetchResult (
166- url = url ,
167- source_name = source_name or url ,
168- raw_text = "" ,
169- parsed_text = "" ,
170- status_code = 0 ,
171- error = str (e ),
172- )
173-
17423 return FetchResult (
17524 url = url ,
17625 source_name = source_name or url ,
17726 raw_text = "" ,
178- parsed_text = "" ,
27+ parsed_text = "[offline] Network disabled — no external API calls. " ,
17928 status_code = 0 ,
180- error = f"Too many redirects ({ len (redirect_chain )} )" ,
181- redirect_chain = redirect_chain ,
29+ error = "offline" ,
18230 )
18331
18432 async def fetch_multiple (
18533 self ,
18634 urls : list [str ],
18735 source_names : Optional [list [str ]] = None ,
18836 ) -> list [FetchResult ]:
189- tasks = []
190- for i , url in enumerate (urls ):
191- name = source_names [i ] if source_names and i < len (source_names ) else ""
192- tasks .append (self .fetch_url (url , source_name = name ))
193- return await asyncio .gather (* tasks )
37+ return [
38+ await self .fetch_url (url , source_name = source_names [i ] if source_names and i < len (source_names ) else "" )
39+ for i , url in enumerate (urls )
40+ ]
19441
19542 async def fetch_by_source (
19643 self ,
@@ -200,34 +47,27 @@ async def fetch_by_source(
20047 ttl_override : Optional [int ] = None ,
20148 ) -> list [FetchResult ]:
20249 results = []
203- to_fetch : list [tuple [str , str ]] = []
20450 for src in sources :
20551 if use_cache and cache :
20652 cached = cache .get (src .url )
20753 if cached is not None :
20854 results .append (cached )
20955 continue
210- to_fetch .append ((src .url , src .name ))
211-
212- if to_fetch :
213- urls = [t [0 ] for t in to_fetch ]
214- names = [t [1 ] for t in to_fetch ]
215- fetched = await self .fetch_multiple (urls , source_names = names )
216- for result in fetched :
217- if use_cache and cache and result .success :
218- cache .set (result .url , result , ttl = ttl_override )
219- results .append (result )
220-
221- results .sort (key = lambda r : r .success , reverse = True )
56+ results .append (FetchResult (
57+ url = src .url ,
58+ source_name = src .name ,
59+ raw_text = "" ,
60+ parsed_text = "[offline] Network disabled." ,
61+ status_code = 0 ,
62+ error = "offline" ,
63+ ))
22264 return results
22365
22466 async def close (self ):
225- if self ._session and not self ._session .closed :
226- await self ._session .close ()
67+ pass
22768
22869 def __enter__ (self ):
22970 return self
23071
23172 def __exit__ (self , exc_type , exc_val , exc_tb ):
232- if self ._session and not self ._session .closed :
233- asyncio .create_task (self ._session .close ())
73+ pass
0 commit comments