smarter: factual-model routing (beh-10), fetch_url main-content extraction + 8k cap (url-08), get_weather via met.no (spec-016)

fjerkroa_bot/url_reader.py

@@ -21,7 +21,7 @@ from .ai_responder import sanitize_external_text
 from .httpread import read_capped
 
 DEFAULT_MAX_BYTES = 2 * 1024 * 1024
-DEFAULT_MAX_CHARS = 6000
+DEFAULT_MAX_CHARS = 8000  # URL-08: budget goes to content now, not chrome
 DEFAULT_MAX_IMAGES = 2
 FETCH_TIMEOUT_S = 15
 MAX_REDIRECTS = 5
@@ -41,18 +41,37 @@ FETCH_URL_TOOL = {
 _META_REFRESH_URL = re.compile(r"url\s*=\s*['\"]?([^'\";\s]+)", re.I)
 
 
+_SKIP_TAGS = ("script", "style", "noscript", "svg", "nav", "header", "footer", "aside", "form", "select", "button")
+_BLOCK_TAGS = ("p", "li", "div", "section", "article", "td", "ul", "ol", "table", "h1", "h2", "h3", "h4", "h5", "h6")
+_LINK_DENSITY_MAX = 0.6  # boilerplate: block mostly link text ... (URL-08)
+_LINK_BLOCK_MAX_CHARS = 200  # ... AND short (menus, related lists); long linky paragraphs survive
+
+
 class _Extractor(HTMLParser):
     def __init__(self) -> None:
         super().__init__()
         self._skip = 0
-        self.parts: List[str] = []
+        self._links = 0
+        self._buf: List[str] = []
+        self._buf_link_chars = 0
+        self.blocks: List[Tuple[str, int]] = []  # (text, chars inside <a>)
         self.images: List[str] = []
         self.og_image: Optional[str] = None
         self.refresh_url: Optional[str] = None
 
+    def _flush(self) -> None:
+        text = " ".join(self._buf).strip()
+        if text:
+            self.blocks.append((text, self._buf_link_chars))
+        self._buf, self._buf_link_chars = [], 0
+
     def handle_starttag(self, tag: str, attrs) -> None:
-        if tag in ("script", "style", "noscript", "svg"):
+        if tag in _SKIP_TAGS:
             self._skip += 1
+        if tag == "a":
+            self._links += 1
+        if tag in _BLOCK_TAGS:
+            self._flush()
         attr = dict(attrs)
         src = attr.get("src")
         if tag == "img" and src:
@@ -67,12 +86,28 @@ class _Extractor(HTMLParser):
                 self.refresh_url = match.group(1)
 
     def handle_endtag(self, tag: str) -> None:
-        if tag in ("script", "style", "noscript", "svg") and self._skip > 0:
+        if tag in _SKIP_TAGS and self._skip > 0:
             self._skip -= 1
+        if tag == "a" and self._links > 0:
+            self._links -= 1
+        if tag in _BLOCK_TAGS:
+            self._flush()
 
     def handle_data(self, data: str) -> None:
         if self._skip == 0 and data.strip():
-            self.parts.append(data.strip())
+            self._buf.append(data.strip())
+            if self._links > 0:
+                self._buf_link_chars += len(data.strip())
+
+    def content_parts(self) -> List[str]:
+        """Blocks minus boilerplate: short blocks dominated by link text are chrome (URL-08)."""
+        self._flush()
+        out = []
+        for text, link_chars in self.blocks:
+            if link_chars / max(1, len(text)) > _LINK_DENSITY_MAX and len(text) < _LINK_BLOCK_MAX_CHARS:
+                continue
+            out.append(text)
+        return out
 
 
 def _ip_is_public(ip_str: str) -> bool:
@@ -162,7 +197,7 @@ class URLReader:
         return extractor
 
     def _to_text(self, html: str) -> str:
-        return re.sub(r"\s+\n", "\n", " ".join(self._extract(html).parts))
+        return re.sub(r"\s+\n", "\n", " ".join(self._extract(html).content_parts()))
 
     async def _ingest_images(self, html: str, base_url: str, channel: str, user: str) -> int:
         if self.image_cache is None: