From 2caa18a17fcb8b6e6d340f4d5b84676f95d2183f Mon Sep 17 00:00:00 2001 From: Oleksandr Kozachuk Date: Mon, 13 Jul 2026 19:29:16 +0200 Subject: [PATCH] igdb search + capped http reads: native fulltext, full-page fetch MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit search_games used `where name ~ ""*`: prefix-only, diacritic- and word-order-sensitive -- "MARVEL Tōkon" found nothing though IGDB has it. Switch to IGDB's native `search` clause (relevance-ranked, diacritic-insensitive). fetch_url and image downloads read bodies with content.read(n), which returns only the first buffered chunk (~7 KB): pages collapsed to their . httpread.read_capped collects chunks up to the byte cap; image downloads read limit+1 so over-limit files are still rejected instead of cached truncated (IMG-10). Co-Authored-By: Claude Fable 5 <noreply@anthropic.com> --- fjerkroa_bot/httpread.py | 18 +++++++++++++++++ fjerkroa_bot/igdblib.py | 18 +++++++++++------ fjerkroa_bot/images.py | 5 ++++- fjerkroa_bot/url_reader.py | 3 ++- tests/test_igdblib.py | 19 ++++++++++++++++++ tests/test_spec_imgcache.py | 39 ++++++++++++++++++++++++++++++++++++ tests/test_spec_url.py | 40 +++++++++++++++++++++++++++++++++++++ 7 files changed, 134 insertions(+), 8 deletions(-) create mode 100644 fjerkroa_bot/httpread.py diff --git a/fjerkroa_bot/httpread.py b/fjerkroa_bot/httpread.py new file mode 100644 index 0000000..db4d1d6 --- /dev/null +++ b/fjerkroa_bot/httpread.py @@ -0,0 +1,18 @@ +"""Bounded HTTP body read (leaf module, no intra-package imports). + +`response.content.read(n)` returns whatever is buffered, not n bytes, +so it silently truncates large or chunked bodies (and web feeds/pages +parse to garbage). This accumulates decompressed chunks up to a hard +cap instead. +""" + +CHUNK = 65536 + + +async def read_capped(response, max_bytes: int) -> bytes: + buf = bytearray() + async for chunk in response.content.iter_chunked(CHUNK): + buf.extend(chunk) + if len(buf) > max_bytes: + break + return bytes(buf[:max_bytes]) diff --git a/fjerkroa_bot/igdblib.py b/fjerkroa_bot/igdblib.py index 1ee97e4..2b08841 100644 --- a/fjerkroa_bot/igdblib.py +++ b/fjerkroa_bot/igdblib.py @@ -56,8 +56,12 @@ class IGDBQuery(object): return requests.post(igdb_url, headers=headers, data=query_body) @staticmethod - def build_query(fields, filters=None, limit=10, offset=None): - query = f"fields {','.join(fields) if fields is not None and len(fields) > 0 else '*'}; limit {limit};" + def build_query(fields, filters=None, limit=10, offset=None, search_term=None): + query = "" + if search_term: + escaped = search_term.replace("\\", "\\\\").replace('"', '\\"') + query += f'search "{escaped}"; ' + query += f"fields {','.join(fields) if fields is not None and len(fields) > 0 else '*'}; limit {limit};" if offset is not None: query += f" offset {offset};" if filters: @@ -65,12 +69,12 @@ class IGDBQuery(object): query += " where " + " & ".join(filter_statements) + ";" return query - def generalized_igdb_query(self, params, endpoint, fields, additional_filters=None, limit=10, offset=None): + def generalized_igdb_query(self, params, endpoint, fields, additional_filters=None, limit=10, offset=None, search_term=None): all_filters = {key: f'~ "{value}"*' for key, value in params.items() if value} if additional_filters: all_filters.update(additional_filters) - query = self.build_query(fields, all_filters, limit, offset) + query = self.build_query(fields, all_filters, limit, offset, search_term) data = self.send_igdb_request(endpoint, query) print(f"{endpoint}: {query} -> {data}") return data @@ -134,9 +138,10 @@ class IGDBQuery(object): return None try: - # Search for games with fuzzy matching + # IGDB native full-text search: diacritic- and word-order-insensitive, + # unlike a `name ~ "..."*` prefix filter games = self.generalized_igdb_query( - {"name": query.strip()}, + {}, "games", [ "id", @@ -155,6 +160,7 @@ class IGDBQuery(object): ], additional_filters={"game_type": "= 0"}, # Main games only (IGDB renamed category -> game_type) limit=limit, + search_term=query.strip(), ) if not games: diff --git a/fjerkroa_bot/images.py b/fjerkroa_bot/images.py index 205e279..674a372 100644 --- a/fjerkroa_bot/images.py +++ b/fjerkroa_bot/images.py @@ -14,6 +14,7 @@ from typing import Any, Callable, Dict, List, Optional import aiohttp +from .httpread import read_capped from .persistence import PersistentStore DEFAULT_CACHE_MB = 500 @@ -79,7 +80,9 @@ class ImageCache: async with aiohttp.ClientSession(timeout=timeout) as session: async with session.get(url) as response: response.raise_for_status() - return await response.content.read(limit + 1) + # limit + 1: an over-limit body must stay over-limit so + # ingest_bytes rejects it instead of caching it truncated + return await read_capped(response, limit + 1) def data_url(self, sha256: str, ext: str) -> Optional[str]: path = self._path(sha256, ext) diff --git a/fjerkroa_bot/url_reader.py b/fjerkroa_bot/url_reader.py index 2343e2f..f119972 100644 --- a/fjerkroa_bot/url_reader.py +++ b/fjerkroa_bot/url_reader.py @@ -18,6 +18,7 @@ from urllib.parse import urljoin, urlparse import aiohttp from .ai_responder import sanitize_external_text +from .httpread import read_capped DEFAULT_MAX_BYTES = 2 * 1024 * 1024 DEFAULT_MAX_CHARS = 6000 @@ -115,7 +116,7 @@ class URLReader: current = urljoin(current, response.headers["Location"]) continue response.raise_for_status() - return str(response.url), await response.content.read(max_bytes + 1) + return str(response.url), await read_capped(response, max_bytes) raise ValueError("too many redirects") async def fetch(self, url: str, channel: str, user: str) -> Dict[str, Any]: diff --git a/tests/test_igdblib.py b/tests/test_igdblib.py index c99085e..e367903 100644 --- a/tests/test_igdblib.py +++ b/tests/test_igdblib.py @@ -174,6 +174,25 @@ class TestIGDBQuery(unittest.TestCase): self.assertEqual(result, [{"id": 1, "name": "Super Mario Bros"}]) +class TestIGDBNativeSearch(unittest.TestCase): + def test_build_query_with_search_term(self): + """search_games uses IGDB full-text search, not a name prefix filter.""" + query = IGDBQuery.build_query(["name"], {"game_type": "= 0"}, limit=5, search_term="Marvel Tōkon") + self.assertEqual(query, 'search "Marvel Tōkon"; fields name; limit 5; where game_type = 0;') + + def test_search_term_escapes_quotes_and_backslashes(self): + query = IGDBQuery.build_query(["name"], search_term='say "hi" \\ bye') + self.assertIn('search "say \\"hi\\" \\\\ bye";', query) + + @patch.object(IGDBQuery, "generalized_igdb_query") + def test_search_games_passes_search_term(self, mock_query): + mock_query.return_value = [] + IGDBQuery("cid", "token").search_games("Elden Ring", limit=3) + _, kwargs = mock_query.call_args + self.assertEqual(kwargs["search_term"], "Elden Ring") + self.assertEqual(mock_query.call_args.args[0], {}) + + class TestIGDBTokenRefresh(unittest.TestCase): @staticmethod def _oauth_response(token="fresh_token", expires_in=5_000_000): diff --git a/tests/test_spec_imgcache.py b/tests/test_spec_imgcache.py index a0fed1a..252ecd4 100644 --- a/tests/test_spec_imgcache.py +++ b/tests/test_spec_imgcache.py @@ -45,6 +45,45 @@ class TestIngest(unittest.TestCase): self.assertIsNone(cache.ingest_bytes(PNG, "chat", "alice", "1")) +class TestOversizedDownloadRejected(unittest.IsolatedAsyncioTestCase): + async def test_download_stays_over_limit_and_is_rejected(self): + """IMG-10: an over-limit download must be rejected, not cached truncated.""" + with tempfile.TemporaryDirectory() as tmp: + _, cache = make_cache(tmp, {"image-max-bytes": 32}) + + class FakeContent: + @staticmethod + async def iter_chunked(size): + yield PNG # 72 bytes > 32 + + class FakeResp: + content = FakeContent() + + async def __aenter__(self): + return self + + async def __aexit__(self, *a): + return False + + def raise_for_status(self): + pass + + class FakeSession: + def get(self, url): + return FakeResp() + + async def __aenter__(self): + return self + + async def __aexit__(self, *a): + return False + + with patch("fjerkroa_bot.images.aiohttp.ClientSession", return_value=FakeSession()): + data = await cache._download("http://x.com/big.png") + self.assertEqual(len(data), 33) # limit + 1, not silently capped to limit + self.assertIsNone(await cache.ingest_url("http://x.com/big.png", "chat", "alice", "1")) + + class TestVisionDataUrls(OpsBase): async def test_attachment_becomes_data_url(self): """IMG-11: the model sees a data: URL, never the CDN link.""" diff --git a/tests/test_spec_url.py b/tests/test_spec_url.py index 9928bea..be317b4 100644 --- a/tests/test_spec_url.py +++ b/tests/test_spec_url.py @@ -91,6 +91,46 @@ class TestTextExtraction(unittest.TestCase): self.assertNotIn("x{}", text) +class TestBodyReadCollectsAllChunks(unittest.IsolatedAsyncioTestCase): + async def test_get_reads_past_first_chunk(self): + """URL-05 regression: body arrives in many chunks; all are collected up to the cap.""" + reader = URLReader(lambda: {}, None) + chunks = [b"<title>t", b"

middle

", b"

end

"] + + class FakeContent: + @staticmethod + async def iter_chunked(size): + for chunk in chunks: + yield chunk + + class FakeResp: + status = 200 + headers = {} + url = "http://safe.example.com" + content = FakeContent() + + async def __aenter__(self): + return self + + async def __aexit__(self, *a): + return False + + def raise_for_status(self): + pass + + class FakeSession: + def get(self, url, allow_redirects=False): + return FakeResp() + + with patch("fjerkroa_bot.url_reader.guard_url", return_value=None): + _, body = await reader._get(FakeSession(), "http://safe.example.com", 1000) + self.assertEqual(body, b"".join(chunks)) + + with patch("fjerkroa_bot.url_reader.guard_url", return_value=None): + _, body = await reader._get(FakeSession(), "http://safe.example.com", 20) + self.assertEqual(body, b"".join(chunks)[:20]) + + class TestFetchSanitizes(unittest.IsolatedAsyncioTestCase): async def test_fetch_result_is_sanitized_and_capped(self): """URL-05: fetch output is length-capped and @everyone-neutralized."""