igdb search + capped http reads: native fulltext, full-page fetch

search_games used `where name ~ "<q>"*`: prefix-only, diacritic- and
word-order-sensitive -- "MARVEL Tōkon" found nothing though IGDB has
it. Switch to IGDB's native `search` clause (relevance-ranked,
diacritic-insensitive).

fetch_url and image downloads read bodies with content.read(n), which
returns only the first buffered chunk (~7 KB): pages collapsed to
their <title>. httpread.read_capped collects chunks up to the byte
cap; image downloads read limit+1 so over-limit files are still
rejected instead of cached truncated (IMG-10).

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>

fjerkroa_bot/httpread.py

@@ -0,0 +1,18 @@
+"""Bounded HTTP body read (leaf module, no intra-package imports).
+
+`response.content.read(n)` returns whatever is buffered, not n bytes,
+so it silently truncates large or chunked bodies (and web feeds/pages
+parse to garbage). This accumulates decompressed chunks up to a hard
+cap instead.
+"""
+
+CHUNK = 65536
+
+
+async def read_capped(response, max_bytes: int) -> bytes:
+    buf = bytearray()
+    async for chunk in response.content.iter_chunked(CHUNK):
+        buf.extend(chunk)
+        if len(buf) > max_bytes:
+            break
+    return bytes(buf[:max_bytes])

fjerkroa_bot/igdblib.py

@@ -56,8 +56,12 @@ class IGDBQuery(object):
         return requests.post(igdb_url, headers=headers, data=query_body)
 
     @staticmethod
-    def build_query(fields, filters=None, limit=10, offset=None):
-        query = f"fields {','.join(fields) if fields is not None and len(fields) > 0 else '*'}; limit {limit};"
+    def build_query(fields, filters=None, limit=10, offset=None, search_term=None):
+        query = ""
+        if search_term:
+            escaped = search_term.replace("\\", "\\\\").replace('"', '\\"')
+            query += f'search "{escaped}"; '
+        query += f"fields {','.join(fields) if fields is not None and len(fields) > 0 else '*'}; limit {limit};"
         if offset is not None:
             query += f" offset {offset};"
         if filters:
@@ -65,12 +69,12 @@ class IGDBQuery(object):
             query += " where " + " & ".join(filter_statements) + ";"
         return query
 
-    def generalized_igdb_query(self, params, endpoint, fields, additional_filters=None, limit=10, offset=None):
+    def generalized_igdb_query(self, params, endpoint, fields, additional_filters=None, limit=10, offset=None, search_term=None):
         all_filters = {key: f'~ "{value}"*' for key, value in params.items() if value}
         if additional_filters:
             all_filters.update(additional_filters)
 
-        query = self.build_query(fields, all_filters, limit, offset)
+        query = self.build_query(fields, all_filters, limit, offset, search_term)
         data = self.send_igdb_request(endpoint, query)
         print(f"{endpoint}: {query} -> {data}")
         return data
@@ -134,9 +138,10 @@ class IGDBQuery(object):
             return None
 
         try:
-            # Search for games with fuzzy matching
+            # IGDB native full-text search: diacritic- and word-order-insensitive,
+            # unlike a `name ~ "..."*` prefix filter
             games = self.generalized_igdb_query(
-                {"name": query.strip()},
+                {},
                 "games",
                 [
                     "id",
@@ -155,6 +160,7 @@ class IGDBQuery(object):
                 ],
                 additional_filters={"game_type": "= 0"},  # Main games only (IGDB renamed category -> game_type)
                 limit=limit,
+                search_term=query.strip(),
             )
 
             if not games:

fjerkroa_bot/images.py

@@ -14,6 +14,7 @@ from typing import Any, Callable, Dict, List, Optional
 
 import aiohttp
 
+from .httpread import read_capped
 from .persistence import PersistentStore
 
 DEFAULT_CACHE_MB = 500
@@ -79,7 +80,9 @@ class ImageCache:
         async with aiohttp.ClientSession(timeout=timeout) as session:
             async with session.get(url) as response:
                 response.raise_for_status()
-                return await response.content.read(limit + 1)
+                # limit + 1: an over-limit body must stay over-limit so
+                # ingest_bytes rejects it instead of caching it truncated
+                return await read_capped(response, limit + 1)
 
     def data_url(self, sha256: str, ext: str) -> Optional[str]:
         path = self._path(sha256, ext)

fjerkroa_bot/url_reader.py

@@ -18,6 +18,7 @@ from urllib.parse import urljoin, urlparse
 import aiohttp
 
 from .ai_responder import sanitize_external_text
+from .httpread import read_capped
 
 DEFAULT_MAX_BYTES = 2 * 1024 * 1024
 DEFAULT_MAX_CHARS = 6000
@@ -115,7 +116,7 @@ class URLReader:
                     current = urljoin(current, response.headers["Location"])
                     continue
                 response.raise_for_status()
-                return str(response.url), await response.content.read(max_bytes + 1)
+                return str(response.url), await read_capped(response, max_bytes)
         raise ValueError("too many redirects")
 
     async def fetch(self, url: str, channel: str, user: str) -> Dict[str, Any]: