igdb search + capped http reads: native fulltext, full-page fetch

search_games used `where name ~ "<q>"*`: prefix-only, diacritic- and
word-order-sensitive -- "MARVEL Tōkon" found nothing though IGDB has
it. Switch to IGDB's native `search` clause (relevance-ranked,
diacritic-insensitive).

fetch_url and image downloads read bodies with content.read(n), which
returns only the first buffered chunk (~7 KB): pages collapsed to
their <title>. httpread.read_capped collects chunks up to the byte
cap; image downloads read limit+1 so over-limit files are still
rejected instead of cached truncated (IMG-10).

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
Oleksandr Kozachuk
2026-07-13 19:29:16 +02:00
parent d4eec4088d
commit 2caa18a17f
7 changed files with 134 additions and 8 deletions
+40
View File
@@ -91,6 +91,46 @@ class TestTextExtraction(unittest.TestCase):
self.assertNotIn("x{}", text)
class TestBodyReadCollectsAllChunks(unittest.IsolatedAsyncioTestCase):
async def test_get_reads_past_first_chunk(self):
"""URL-05 regression: body arrives in many chunks; all are collected up to the cap."""
reader = URLReader(lambda: {}, None)
chunks = [b"<title>t</title>", b"<p>middle</p>", b"<p>end</p>"]
class FakeContent:
@staticmethod
async def iter_chunked(size):
for chunk in chunks:
yield chunk
class FakeResp:
status = 200
headers = {}
url = "http://safe.example.com"
content = FakeContent()
async def __aenter__(self):
return self
async def __aexit__(self, *a):
return False
def raise_for_status(self):
pass
class FakeSession:
def get(self, url, allow_redirects=False):
return FakeResp()
with patch("fjerkroa_bot.url_reader.guard_url", return_value=None):
_, body = await reader._get(FakeSession(), "http://safe.example.com", 1000)
self.assertEqual(body, b"".join(chunks))
with patch("fjerkroa_bot.url_reader.guard_url", return_value=None):
_, body = await reader._get(FakeSession(), "http://safe.example.com", 20)
self.assertEqual(body, b"".join(chunks)[:20])
class TestFetchSanitizes(unittest.IsolatedAsyncioTestCase):
async def test_fetch_result_is_sanitized_and_capped(self):
"""URL-05: fetch output is length-capped and @everyone-neutralized."""