url reader: follow html meta-refresh redirects (getnews shortlinks) with ssrf re-guard

2026-07-13 20:09:46 +02:00
parent 7628faf551
commit a514ff652c
3 changed files with 93 additions and 12 deletions
@@ -79,6 +79,65 @@ class TestRedirectRevalidation(unittest.IsolatedAsyncioTestCase):
                await reader._get(FakeSession(), "http://safe.example.com", 1000)


+class TestMetaRefresh(unittest.IsolatedAsyncioTestCase):
+    async def test_follows_meta_refresh_to_real_article(self):
+        """URL-04: a getnews-style meta-refresh stub is followed to the real article."""
+        reader = URLReader(lambda: {}, None)
+        stub = (
+            b'<html><head><meta http-equiv="refresh" content="0;url=https://pushsquare.com/real"></head><body>Redirecting...</body></html>'
+        )
+        article = b"<html><body><h1>MARVEL Tokon</h1><p>Full article text here</p></body></html>"
+        calls = []
+
+        async def fake_get(session, url, max_bytes):
+            calls.append(url)
+            return (url, stub if "stub" in url else article)
+
+        reader._get = fake_get  # type: ignore
+        with patch("fjerkroa_bot.url_reader.guard_url", return_value=None):
+            import fjerkroa_bot.url_reader as ur
+
+            # patch the session context so fetch() runs against fake_get
+            class FakeCM:
+                async def __aenter__(self):
+                    return object()
+
+                async def __aexit__(self, *a):
+                    return False
+
+            with patch.object(ur.aiohttp, "ClientSession", return_value=FakeCM()):
+                result = await reader.fetch("https://gggemein.de/url/stub.html", "chat", "alice")
+        self.assertIn("Full article text", result["text"])
+        self.assertEqual(result["url"], "https://pushsquare.com/real")
+        self.assertIn("https://pushsquare.com/real", calls)
+
+    async def test_meta_refresh_to_internal_is_not_followed(self):
+        """URL-04: a meta-refresh pointing at an internal IP is refused (SSRF)."""
+        reader = URLReader(lambda: {}, None)
+        stub = b'<meta http-equiv="refresh" content="0; url=http://127.0.0.1/secret">Redirecting'
+
+        async def fake_get(session, url, max_bytes):
+            return (url, stub)
+
+        reader._get = fake_get  # type: ignore
+        import fjerkroa_bot.url_reader as ur
+
+        class FakeCM:
+            async def __aenter__(self):
+                return object()
+
+            async def __aexit__(self, *a):
+                return False
+
+        def guard(u):
+            return "refused" if "127.0.0.1" in u else None
+
+        with patch("fjerkroa_bot.url_reader.guard_url", side_effect=guard):
+            with patch.object(ur.aiohttp, "ClientSession", return_value=FakeCM()):
+                result = await reader.fetch("https://safe.com/x", "chat", "alice")
+        self.assertEqual(result["url"], "https://safe.com/x")  # did not follow to 127.0.0.1
+
+
 class TestTextExtraction(unittest.TestCase):
    def test_html_reduced_to_text(self):
        """URL-05: scripts/styles dropped, tags stripped."""