igdb search + capped http reads: native fulltext, full-page fetch

search_games used `where name ~ "<q>"*`: prefix-only, diacritic- and
word-order-sensitive -- "MARVEL Tōkon" found nothing though IGDB has
it. Switch to IGDB's native `search` clause (relevance-ranked,
diacritic-insensitive).

fetch_url and image downloads read bodies with content.read(n), which
returns only the first buffered chunk (~7 KB): pages collapsed to
their <title>. httpread.read_capped collects chunks up to the byte
cap; image downloads read limit+1 so over-limit files are still
rejected instead of cached truncated (IMG-10).

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>

fjerkroa_bot/httpread.py

@@ -0,0 +1,18 @@
+"""Bounded HTTP body read (leaf module, no intra-package imports).
+
+`response.content.read(n)` returns whatever is buffered, not n bytes,
+so it silently truncates large or chunked bodies (and web feeds/pages
+parse to garbage). This accumulates decompressed chunks up to a hard
+cap instead.
+"""
+
+CHUNK = 65536
+
+
+async def read_capped(response, max_bytes: int) -> bytes:
+    buf = bytearray()
+    async for chunk in response.content.iter_chunked(CHUNK):
+        buf.extend(chunk)
+        if len(buf) > max_bytes:
+            break
+    return bytes(buf[:max_bytes])

fjerkroa_bot/igdblib.py

@@ -56,8 +56,12 @@ class IGDBQuery(object):
         return requests.post(igdb_url, headers=headers, data=query_body)
 
     @staticmethod
-    def build_query(fields, filters=None, limit=10, offset=None):
-        query = f"fields {','.join(fields) if fields is not None and len(fields) > 0 else '*'}; limit {limit};"
+    def build_query(fields, filters=None, limit=10, offset=None, search_term=None):
+        query = ""
+        if search_term:
+            escaped = search_term.replace("\\", "\\\\").replace('"', '\\"')
+            query += f'search "{escaped}"; '
+        query += f"fields {','.join(fields) if fields is not None and len(fields) > 0 else '*'}; limit {limit};"
         if offset is not None:
             query += f" offset {offset};"
         if filters:
@@ -65,12 +69,12 @@ class IGDBQuery(object):
             query += " where " + " & ".join(filter_statements) + ";"
         return query
 
-    def generalized_igdb_query(self, params, endpoint, fields, additional_filters=None, limit=10, offset=None):
+    def generalized_igdb_query(self, params, endpoint, fields, additional_filters=None, limit=10, offset=None, search_term=None):
         all_filters = {key: f'~ "{value}"*' for key, value in params.items() if value}
         if additional_filters:
             all_filters.update(additional_filters)
 
-        query = self.build_query(fields, all_filters, limit, offset)
+        query = self.build_query(fields, all_filters, limit, offset, search_term)
         data = self.send_igdb_request(endpoint, query)
         print(f"{endpoint}: {query} -> {data}")
         return data
@@ -134,9 +138,10 @@ class IGDBQuery(object):
             return None
 
         try:
-            # Search for games with fuzzy matching
+            # IGDB native full-text search: diacritic- and word-order-insensitive,
+            # unlike a `name ~ "..."*` prefix filter
             games = self.generalized_igdb_query(
-                {"name": query.strip()},
+                {},
                 "games",
                 [
                     "id",
@@ -155,6 +160,7 @@ class IGDBQuery(object):
                 ],
                 additional_filters={"game_type": "= 0"},  # Main games only (IGDB renamed category -> game_type)
                 limit=limit,
+                search_term=query.strip(),
             )
 
             if not games:

fjerkroa_bot/images.py

@@ -14,6 +14,7 @@ from typing import Any, Callable, Dict, List, Optional
 
 import aiohttp
 
+from .httpread import read_capped
 from .persistence import PersistentStore
 
 DEFAULT_CACHE_MB = 500
@@ -79,7 +80,9 @@ class ImageCache:
         async with aiohttp.ClientSession(timeout=timeout) as session:
             async with session.get(url) as response:
                 response.raise_for_status()
-                return await response.content.read(limit + 1)
+                # limit + 1: an over-limit body must stay over-limit so
+                # ingest_bytes rejects it instead of caching it truncated
+                return await read_capped(response, limit + 1)
 
     def data_url(self, sha256: str, ext: str) -> Optional[str]:
         path = self._path(sha256, ext)

fjerkroa_bot/url_reader.py

@@ -18,6 +18,7 @@ from urllib.parse import urljoin, urlparse
 import aiohttp
 
 from .ai_responder import sanitize_external_text
+from .httpread import read_capped
 
 DEFAULT_MAX_BYTES = 2 * 1024 * 1024
 DEFAULT_MAX_CHARS = 6000
@@ -115,7 +116,7 @@ class URLReader:
                     current = urljoin(current, response.headers["Location"])
                     continue
                 response.raise_for_status()
-                return str(response.url), await response.content.read(max_bytes + 1)
+                return str(response.url), await read_capped(response, max_bytes)
         raise ValueError("too many redirects")
 
     async def fetch(self, url: str, channel: str, user: str) -> Dict[str, Any]:

tests/test_igdblib.py

@@ -174,6 +174,25 @@ class TestIGDBQuery(unittest.TestCase):
         self.assertEqual(result, [{"id": 1, "name": "Super Mario Bros"}])
 
 
+class TestIGDBNativeSearch(unittest.TestCase):
+    def test_build_query_with_search_term(self):
+        """search_games uses IGDB full-text search, not a name prefix filter."""
+        query = IGDBQuery.build_query(["name"], {"game_type": "= 0"}, limit=5, search_term="Marvel Tōkon")
+        self.assertEqual(query, 'search "Marvel Tōkon"; fields name; limit 5; where game_type = 0;')
+
+    def test_search_term_escapes_quotes_and_backslashes(self):
+        query = IGDBQuery.build_query(["name"], search_term='say "hi" \\ bye')
+        self.assertIn('search "say \\"hi\\" \\\\ bye";', query)
+
+    @patch.object(IGDBQuery, "generalized_igdb_query")
+    def test_search_games_passes_search_term(self, mock_query):
+        mock_query.return_value = []
+        IGDBQuery("cid", "token").search_games("Elden Ring", limit=3)
+        _, kwargs = mock_query.call_args
+        self.assertEqual(kwargs["search_term"], "Elden Ring")
+        self.assertEqual(mock_query.call_args.args[0], {})
+
+
 class TestIGDBTokenRefresh(unittest.TestCase):
     @staticmethod
     def _oauth_response(token="fresh_token", expires_in=5_000_000):

tests/test_spec_imgcache.py

@@ -45,6 +45,45 @@ class TestIngest(unittest.TestCase):
             self.assertIsNone(cache.ingest_bytes(PNG, "chat", "alice", "1"))
 
 
+class TestOversizedDownloadRejected(unittest.IsolatedAsyncioTestCase):
+    async def test_download_stays_over_limit_and_is_rejected(self):
+        """IMG-10: an over-limit download must be rejected, not cached truncated."""
+        with tempfile.TemporaryDirectory() as tmp:
+            _, cache = make_cache(tmp, {"image-max-bytes": 32})
+
+            class FakeContent:
+                @staticmethod
+                async def iter_chunked(size):
+                    yield PNG  # 72 bytes > 32
+
+            class FakeResp:
+                content = FakeContent()
+
+                async def __aenter__(self):
+                    return self
+
+                async def __aexit__(self, *a):
+                    return False
+
+                def raise_for_status(self):
+                    pass
+
+            class FakeSession:
+                def get(self, url):
+                    return FakeResp()
+
+                async def __aenter__(self):
+                    return self
+
+                async def __aexit__(self, *a):
+                    return False
+
+            with patch("fjerkroa_bot.images.aiohttp.ClientSession", return_value=FakeSession()):
+                data = await cache._download("http://x.com/big.png")
+                self.assertEqual(len(data), 33)  # limit + 1, not silently capped to limit
+                self.assertIsNone(await cache.ingest_url("http://x.com/big.png", "chat", "alice", "1"))
+
+
 class TestVisionDataUrls(OpsBase):
     async def test_attachment_becomes_data_url(self):
         """IMG-11: the model sees a data: URL, never the CDN link."""

tests/test_spec_url.py

@@ -91,6 +91,46 @@ class TestTextExtraction(unittest.TestCase):
         self.assertNotIn("x{}", text)
 
 
+class TestBodyReadCollectsAllChunks(unittest.IsolatedAsyncioTestCase):
+    async def test_get_reads_past_first_chunk(self):
+        """URL-05 regression: body arrives in many chunks; all are collected up to the cap."""
+        reader = URLReader(lambda: {}, None)
+        chunks = [b"<title>t</title>", b"<p>middle</p>", b"<p>end</p>"]
+
+        class FakeContent:
+            @staticmethod
+            async def iter_chunked(size):
+                for chunk in chunks:
+                    yield chunk
+
+        class FakeResp:
+            status = 200
+            headers = {}
+            url = "http://safe.example.com"
+            content = FakeContent()
+
+            async def __aenter__(self):
+                return self
+
+            async def __aexit__(self, *a):
+                return False
+
+            def raise_for_status(self):
+                pass
+
+        class FakeSession:
+            def get(self, url, allow_redirects=False):
+                return FakeResp()
+
+        with patch("fjerkroa_bot.url_reader.guard_url", return_value=None):
+            _, body = await reader._get(FakeSession(), "http://safe.example.com", 1000)
+        self.assertEqual(body, b"".join(chunks))
+
+        with patch("fjerkroa_bot.url_reader.guard_url", return_value=None):
+            _, body = await reader._get(FakeSession(), "http://safe.example.com", 20)
+        self.assertEqual(body, b"".join(chunks)[:20])
+
+
 class TestFetchSanitizes(unittest.IsolatedAsyncioTestCase):
     async def test_fetch_result_is_sanitized_and_capped(self):
         """URL-05: fetch output is length-capped and @everyone-neutralized."""