url-09: fetched images become vision input — fetch_url og:image/body images ride as input_image, direct image urls ingested, data urls never in json tool text

fjerkroa_bot/url_reader.py

@@ -149,7 +149,7 @@ class URLReader:
     def enabled(self) -> bool:
         return bool(self._config().get("enable-url-reading", False))
 
-    async def _get(self, session, url: str, max_bytes: int) -> Tuple[str, bytes]:
+    async def _get(self, session, url: str, max_bytes: int) -> Tuple[str, bytes, str]:
         """Manual redirect handling so every hop is re-guarded (URL-04)."""
         current = url
         for _ in range(MAX_REDIRECTS):
@@ -161,7 +161,8 @@ class URLReader:
                     current = urljoin(current, response.headers["Location"])
                     continue
                 response.raise_for_status()
-                return str(response.url), await read_capped(response, max_bytes)
+                content_type = str(response.headers.get("Content-Type", "")).split(";")[0].strip().lower()
+                return str(response.url), await read_capped(response, max_bytes), content_type
         raise ValueError("too many redirects")
 
     async def fetch(self, url: str, channel: str, user: str) -> Dict[str, Any]:
@@ -170,9 +171,11 @@ class URLReader:
         timeout = aiohttp.ClientTimeout(total=FETCH_TIMEOUT_S)
         try:
             async with aiohttp.ClientSession(timeout=timeout, headers={"User-Agent": "FjerkroaBot/1.0"}) as session:
-                final_url, body = await self._get(session, url, max_bytes)
+                final_url, body, content_type = await self._get(session, url, max_bytes)
                 # follow a meta-refresh redirect (link shorteners / getnews stubs), re-guarded — URL-04
                 for _ in range(2):
+                    if content_type.startswith("image/"):
+                        break
                     extractor = self._extract(body.decode("utf-8", "ignore"))
                     if not extractor.refresh_url:
                         break
@@ -180,13 +183,21 @@ class URLReader:
                     if guard_url(target) is not None or target == final_url:
                         break
                     logging.info(f"url reader: following meta-refresh -> {target}")
-                    final_url, body = await self._get(session, target, max_bytes)
+                    final_url, body, content_type = await self._get(session, target, max_bytes)
         except Exception as err:
             return {"error": str(err)}
+        # a URL that IS an image: cache it and hand it over as sight (URL-09)
+        if content_type.startswith("image/"):
+            vision = []
+            if self.image_cache is not None and len(body) < max_bytes:  # >= cap means possibly truncated
+                sha = self.image_cache.ingest_bytes(body, channel, user, None)
+                data_url = self._cached_data_url(sha, channel) if sha else None
+                vision = [data_url] if data_url else []
+            return {"url": final_url, "text": "(image)", "images_cached": len(vision), "vision": vision}
         html = body.decode("utf-8", "ignore")
         clean = sanitize_external_text(self._to_text(html), int(config.get("url-max-chars", DEFAULT_MAX_CHARS)))
-        images = await self._ingest_images(html, final_url, channel, user)
-        return {"url": final_url, "text": clean, "images_cached": images}
+        vision = await self._ingest_images(html, final_url, channel, user)
+        return {"url": final_url, "text": clean, "images_cached": len(vision), "vision": vision}
 
     def _extract(self, html: str) -> "_Extractor":
         extractor = _Extractor()
@@ -199,20 +210,27 @@ class URLReader:
     def _to_text(self, html: str) -> str:
         return re.sub(r"\s+\n", "\n", " ".join(self._extract(html).content_parts()))
 
-    async def _ingest_images(self, html: str, base_url: str, channel: str, user: str) -> int:
+    async def _ingest_images(self, html: str, base_url: str, channel: str, user: str) -> List[str]:
+        """Cache page images and return their data URLs for vision input (URL-09)."""
         if self.image_cache is None:
-            return 0
+            return []
         extractor = self._extract(html)
         candidates = ([extractor.og_image] if extractor.og_image else []) + extractor.images
         limit = int(self._config().get("url-max-images", DEFAULT_MAX_IMAGES))
-        cached = 0
+        data_urls: List[str] = []
         for src in candidates:
-            if cached >= limit:
+            if len(data_urls) >= limit:
                 break
             absolute = urljoin(base_url, src)
             if guard_url(absolute) is not None:
                 continue
             sha = await self.image_cache.ingest_url(absolute, channel, user, None)
-            if sha is not None:
-                cached += 1
-        return cached
+            data_url = self._cached_data_url(sha, channel) if sha else None
+            if data_url:
+                data_urls.append(data_url)
+        return data_urls
+
+    def _cached_data_url(self, sha: str, channel: str) -> Optional[str]:
+        recent = self.image_cache.recent(channel, 8)
+        ext = next((row["ext"] for row in recent if row["sha256"] == sha), None)
+        return self.image_cache.data_url(sha, ext) if ext else None