url reader: follow html meta-refresh redirects (getnews shortlinks) with ssrf re-guard

fjerkroa_bot/news.py

@@ -90,6 +90,102 @@ class NewsFetcher:
         return out
 
 
+DEFAULT_SEEN_CAP = 5000
+DEFAULT_POST_PER_FEED = 5
+DEFAULT_POST_MAX_PER_RUN = 8
+
+
+def item_key(item: Dict[str, str]) -> str:
+    return item.get("link") or item.get("title") or ""
+
+
+class NewsPoster:
+    """Post NEW feed items to Discord channel webhooks (ggg model, SPEC-013 NEWS-04..06)."""
+
+    def __init__(self, guard, fetch_bytes, post_webhook) -> None:
+        self._guard = guard
+        self._fetch_bytes = fetch_bytes
+        self._post_webhook = post_webhook
+
+    async def run_post(
+        self,
+        feeds: List[Tuple[str, str, str]],
+        webhooks: Dict[str, str],
+        seen: set,
+        per_feed: int,
+        max_per_run: int,
+        seed_only: bool,
+    ) -> Tuple[int, set]:
+        """Returns (posted_count, updated_seen). seed_only marks new items seen without posting."""
+        posted = 0
+        for url, label, channel in feeds:
+            reason = self._guard(url)
+            if reason:
+                logging.warning(f"news-post: skipping feed {label} — {reason}")
+                continue
+            try:
+                data = await self._fetch_bytes(url)
+            except Exception as err:
+                logging.warning(f"news-post: fetch failed for {label}: {repr(err)}")
+                continue
+            for item in parse_feed(data, label)[:per_feed]:
+                key = item_key(item)
+                if not key or key in seen:
+                    continue
+                seen.add(key)
+                may_post = not seed_only and posted < max_per_run
+                if may_post and await self._deliver(item, label, channel, webhooks):
+                    posted += 1
+        return posted, seen
+
+    async def _deliver(self, item: Dict[str, str], label: str, channel: str, webhooks: Dict[str, str]) -> bool:
+        hook = webhooks.get(channel)
+        if not hook:
+            logging.warning(f"news-post: no webhook for channel {channel!r} ({label})")
+            return False
+        title = sanitize_external_text(item["title"], 300)
+        link = item.get("link", "")
+        content = f"**[{label}]** {title}" + (f"\n{link}" if link else "")
+        try:
+            await self._post_webhook(hook, content)
+            return True
+        except Exception as err:
+            logging.warning(f"news-post: webhook post failed ({label}): {repr(err)}")
+            return False
+
+
+def load_seen(path: str) -> Tuple[set, bool]:
+    """(seen-set, existed). Missing/broken state -> empty set, existed=False (seed run)."""
+    import json
+    import os
+
+    if not os.path.exists(path):
+        return set(), False
+    try:
+        with open(path, encoding="utf-8") as fd:
+            return set(json.load(fd)), True
+    except Exception as err:
+        logging.warning(f"news-post: unreadable state {path}: {err!r} — reseeding")
+        return set(), False
+
+
+def save_seen(path: str, seen: set, cap: int = DEFAULT_SEEN_CAP) -> None:
+    import json
+
+    # keep the newest `cap` keys (insertion order preserved by Python sets? no — use a bounded slice)
+    keys = list(seen)[-cap:]
+    with open(path, "w", encoding="utf-8") as fd:
+        json.dump(keys, fd)
+
+
+def _post_feeds_from_config(config: Dict[str, Any]) -> List[Tuple[str, str, str]]:
+    feeds = []
+    for entry in config.get("news-post-feeds", []):
+        if isinstance(entry, (list, tuple)) and len(entry) >= 3:
+            feeds.append((str(entry[0]), str(entry[1]), str(entry[2])))
+    return feeds
+
+
 def _feeds_from_config(config: Dict[str, Any]) -> List[Tuple[str, str]]:
     """news-feeds = [["url", "label"], ...] or ["url", ...]."""
     feeds = []
@@ -113,6 +209,42 @@ async def _aiohttp_fetch(url: str) -> bytes:
             return await read_capped(response, 4 * 1024 * 1024)
 
 
+async def _aiohttp_post(hook: str, content: str) -> None:
+    import aiohttp
+
+    timeout = aiohttp.ClientTimeout(total=FETCH_TIMEOUT_S)
+    async with aiohttp.ClientSession(timeout=timeout) as session:
+        # allowed_mentions none: a headline can never ping the channel (SAF-02 spirit)
+        payload = {"content": content[:2000], "allowed_mentions": {"parse": []}}
+        async with session.post(hook, json=payload) as response:
+            response.raise_for_status()
+
+
+async def run_post(config: Dict[str, Any]) -> int:
+    """Webhook-posting mode (ggg): post new items to channels. Returns posted count."""
+    from .url_reader import guard_url
+
+    webhooks = dict(config.get("news-post-webhooks", {}))
+    feeds = _post_feeds_from_config(config)
+    state_path = config.get("news-post-state", "news_state.json")
+    if not webhooks or not feeds:
+        logging.error("news-post: need news-post-webhooks and news-post-feeds")
+        return 0
+    seen, existed = load_seen(state_path)
+    poster = NewsPoster(guard_url, _aiohttp_fetch, _aiohttp_post)
+    posted, seen = await poster.run_post(
+        feeds,
+        webhooks,
+        seen,
+        int(config.get("news-post-per-feed", DEFAULT_POST_PER_FEED)),
+        int(config.get("news-post-max-per-run", DEFAULT_POST_MAX_PER_RUN)),
+        seed_only=not existed,  # first run seeds without flooding the channels
+    )
+    save_seen(state_path, seen, int(config.get("news-post-seen-cap", DEFAULT_SEEN_CAP)))
+    logging.info(f"news-post: posted {posted} item(s)" + (" (seed run — nothing posted)" if not existed else ""))
+    return posted
+
+
 async def run(config: Dict[str, Any]) -> Optional[str]:
     from .url_reader import guard_url
 
@@ -140,11 +272,17 @@ def main() -> int:
     import tomlkit
 
     logging.basicConfig(level=logging.INFO, format="%(asctime)s [%(levelname)s] %(message)s")
-    parser = argparse.ArgumentParser(description="Fetch RSS/Atom feeds into the {news} digest file")
+    parser = argparse.ArgumentParser(
+        description="Fetch RSS/Atom feeds: --post to channel webhooks (ggg) or default {news} digest file (kroa)"
+    )
     parser.add_argument("--config", required=True)
+    parser.add_argument("--post", action="store_true", help="webhook-posting mode (post new items to Discord channels)")
     args = parser.parse_args()
     with open(args.config, encoding="utf-8") as fd:
         config = tomlkit.load(fd)
+    if args.post:
+        asyncio.run(run_post(config))
+        return 0
     result = asyncio.run(run(config))
     return 0 if result else 1
 

fjerkroa_bot/url_reader.py

@@ -38,6 +38,9 @@ FETCH_URL_TOOL = {
 }
 
 
+_META_REFRESH_URL = re.compile(r"url\s*=\s*['\"]?([^'\";\s]+)", re.I)
+
+
 class _Extractor(HTMLParser):
     def __init__(self) -> None:
         super().__init__()
@@ -45,6 +48,7 @@ class _Extractor(HTMLParser):
         self.parts: List[str] = []
         self.images: List[str] = []
         self.og_image: Optional[str] = None
+        self.refresh_url: Optional[str] = None
 
     def handle_starttag(self, tag: str, attrs) -> None:
         if tag in ("script", "style", "noscript", "svg"):
@@ -55,6 +59,12 @@ class _Extractor(HTMLParser):
             self.images.append(src)
         if tag == "meta" and attr.get("property") == "og:image" and attr.get("content"):
             self.og_image = attr["content"]
+        # meta-refresh redirect (link shorteners, getnews stubs) — URL-04
+        content = attr.get("content")
+        if tag == "meta" and (attr.get("http-equiv") or "").lower() == "refresh" and content:
+            match = _META_REFRESH_URL.search(content)
+            if match and self.refresh_url is None:
+                self.refresh_url = match.group(1)
 
     def handle_endtag(self, tag: str) -> None:
         if tag in ("script", "style", "noscript", "svg") and self._skip > 0:
@@ -126,29 +136,38 @@ class URLReader:
         try:
             async with aiohttp.ClientSession(timeout=timeout, headers={"User-Agent": "FjerkroaBot/1.0"}) as session:
                 final_url, body = await self._get(session, url, max_bytes)
+                # follow a meta-refresh redirect (link shorteners / getnews stubs), re-guarded — URL-04
+                for _ in range(2):
+                    extractor = self._extract(body.decode("utf-8", "ignore"))
+                    if not extractor.refresh_url:
+                        break
+                    target = urljoin(final_url, extractor.refresh_url)
+                    if guard_url(target) is not None or target == final_url:
+                        break
+                    logging.info(f"url reader: following meta-refresh -> {target}")
+                    final_url, body = await self._get(session, target, max_bytes)
         except Exception as err:
             return {"error": str(err)}
-        text = self._to_text(body.decode("utf-8", "ignore"))
+        html = body.decode("utf-8", "ignore")
-        clean = sanitize_external_text(text, int(config.get("url-max-chars", DEFAULT_MAX_CHARS)))
+        clean = sanitize_external_text(self._to_text(html), int(config.get("url-max-chars", DEFAULT_MAX_CHARS)))
-        images = await self._ingest_images(body.decode("utf-8", "ignore"), final_url, channel, user)
+        images = await self._ingest_images(html, final_url, channel, user)
         return {"url": final_url, "text": clean, "images_cached": images}
 
-    def _to_text(self, html: str) -> str:
+    def _extract(self, html: str) -> "_Extractor":
         extractor = _Extractor()
         try:
             extractor.feed(html)
         except Exception as err:
-            logging.debug(f"html parse (text) failed: {err!r}")
+            logging.debug(f"html parse failed: {err!r}")
-        return re.sub(r"\s+\n", "\n", " ".join(extractor.parts))
+        return extractor
+
+    def _to_text(self, html: str) -> str:
+        return re.sub(r"\s+\n", "\n", " ".join(self._extract(html).parts))
 
     async def _ingest_images(self, html: str, base_url: str, channel: str, user: str) -> int:
         if self.image_cache is None:
             return 0
-        extractor = _Extractor()
+        extractor = self._extract(html)
-        try:
-            extractor.feed(html)
-        except Exception as err:
-            logging.debug(f"html parse (images) failed: {err!r}")
         candidates = ([extractor.og_image] if extractor.og_image else []) + extractor.images
         limit = int(self._config().get("url-max-images", DEFAULT_MAX_IMAGES))
         cached = 0

specs/SPEC-011-url-reading.md

@@ -31,7 +31,10 @@ refused without DNS.
 
 Redirects are followed manually; each hop's target passes URL-02 and
 URL-03 again. A public URL that 302-redirects to `localhost` or an
-internal IP is refused at the redirect, not fetched.
+internal IP is refused at the redirect, not fetched. **HTML
+meta-refresh** redirects (link shorteners, the old getnews stubs) are
+also followed — the target is SSRF-re-guarded and fetched, so the
+reader returns the real article, not the "Redirecting…" stub.
 
 ### URL-05 — Fetched text is bounded and sanitized (coverage: test)
 

specs/SPEC-013-news.md

@@ -23,3 +23,32 @@ or control characters into the prompt via a headline.
 `NewsFetcher.collect` skips any feed URL the SSRF guard rejects,
 skips feeds that fail to fetch (one bad feed never sinks the run),
 and drops duplicate headlines across feeds.
+
+## Webhook posting (ggg model)
+
+`--post` mode fetches feeds mapped to channels and posts NEW items to
+the channel's Discord webhook — replacing the py3.8 `getnews.py`
+(dead play3 feed, 35 MB substring-scan state file, HTML-redirect
+cruft). Config: `news-post-feeds = [[url, label, channel], …]`,
+`news-post-webhooks = {channel = url}`, `news-post-state`.
+
+### NEWS-04 — Only unseen items post, then are marked seen (coverage: test)
+
+`NewsPoster.run_post` posts each item whose key (link, else title) is
+not in the seen-set, adds it to the set, and posts to the mapped
+channel's webhook. Re-runs over the same feed post nothing new.
+
+### NEWS-05 — First run seeds without flooding (coverage: test)
+
+With no prior state file (`seed_only`), every current item is marked
+seen but nothing is posted — migrating off getnews.py never dumps a
+backlog into the channels. `news-post-max-per-run` caps steady-state
+posts per run.
+
+### NEWS-06 — Post failures and bad channels are survived (coverage: test)
+
+A feed the SSRF guard rejects, a feed that fails to fetch, an item
+whose channel has no configured webhook, and a webhook POST that
+raises are each logged and skipped — one failure never sinks the
+run, and the seen-set still advances for successfully-processed
+items.

tests/test_spec_news.py

@@ -3,7 +3,7 @@
 import unittest
 from unittest.mock import AsyncMock
 
-from fjerkroa_bot.news import NewsFetcher, parse_feed, render_digest
+from fjerkroa_bot.news import NewsFetcher, NewsPoster, load_seen, parse_feed, render_digest, save_seen
 
 RSS = b"""<?xml version="1.0"?><rss><channel>
 <item><title>Game X released</title><link>https://ex.com/x</link></item>
@@ -73,3 +73,103 @@ class TestCollect(unittest.IsolatedAsyncioTestCase):
         fetcher = NewsFetcher(lambda u: None, AsyncMock(return_value=RSS))
         items = await fetcher.collect([("https://a.com", "A")], per_feed=1)
         self.assertEqual(len(items), 1)
+
+
+class TestPoster(unittest.IsolatedAsyncioTestCase):
+    def poster(self, posts):
+        async def fetch(url):
+            return RSS
+
+        async def post(hook, content):
+            posts.append((hook, content))
+
+        return NewsPoster(lambda u: None, fetch, post)
+
+    async def test_posts_unseen_then_dedups(self):
+        """NEWS-04: unseen items post to the mapped webhook; re-run posts nothing."""
+        posts = []
+        poster = self.poster(posts)
+        feeds = [("https://a.com/feed", "PS", "news")]
+        hooks = {"news": "https://discord.com/api/webhooks/x"}
+        posted, seen = await poster.run_post(feeds, hooks, set(), per_feed=5, max_per_run=8, seed_only=False)
+        self.assertEqual(posted, 2)
+        self.assertIn("PS", posts[0][1])
+        self.assertIn("https://discord.com/api/webhooks/x", posts[0][0])
+        # re-run with the accumulated seen -> nothing new
+        posts.clear()
+        posted2, _ = await poster.run_post(feeds, hooks, seen, per_feed=5, max_per_run=8, seed_only=False)
+        self.assertEqual(posted2, 0)
+        self.assertEqual(posts, [])
+
+    async def test_seed_run_posts_nothing(self):
+        """NEWS-05: seed_only marks items seen without posting."""
+        posts = []
+        poster = self.poster(posts)
+        feeds = [("https://a.com/feed", "PS", "news")]
+        posted, seen = await poster.run_post(feeds, {"news": "h"}, set(), 5, 8, seed_only=True)
+        self.assertEqual(posted, 0)
+        self.assertEqual(posts, [])
+        self.assertEqual(len(seen), 2)  # both marked seen
+
+    async def test_max_per_run_caps(self):
+        """NEWS-05: max-per-run caps posts; extras stay seen (not re-posted next run)."""
+        posts = []
+        poster = self.poster(posts)
+        feeds = [("https://a.com/feed", "PS", "news")]
+        posted, seen = await poster.run_post(feeds, {"news": "h"}, set(), per_feed=5, max_per_run=1, seed_only=False)
+        self.assertEqual(posted, 1)
+        self.assertEqual(len(seen), 2)  # both seen, only one posted
+
+    async def test_failures_survived(self):
+        """NEWS-06: SSRF-skip, fetch fail, missing webhook, post error each survive."""
+        posts = []
+
+        async def fetch(url):
+            if "boom" in url:
+                raise ValueError("boom")
+            return RSS
+
+        async def post(hook, content):
+            if hook == "bad":
+                raise RuntimeError("post failed")
+            posts.append((hook, content))
+
+        def guard(url):
+            return "refused" if "internal" in url else None
+
+        poster = NewsPoster(guard, fetch, post)
+        feeds = [
+            ("https://internal/feed", "I", "news"),  # SSRF-skipped
+            ("https://boom.com/feed", "B", "news"),  # fetch fails
+            ("https://ok.com/feed", "OK", "nowhere"),  # no webhook for channel
+            ("https://ok2.com/feed", "OK2", "news"),  # webhook raises
+        ]
+        posted, seen = await poster.run_post(feeds, {"news": "bad"}, set(), 5, 8, seed_only=False)
+        self.assertEqual(posted, 0)  # everything failed/skipped, no crash
+
+
+class TestSeenState(unittest.TestCase):
+    def test_roundtrip_and_seed_detection(self):
+        """NEWS-05: missing state -> (empty, existed=False); saved state reloads."""
+        import tempfile
+        from pathlib import Path
+
+        with tempfile.TemporaryDirectory() as tmp:
+            path = str(Path(tmp) / "state.json")
+            seen, existed = load_seen(path)
+            self.assertEqual((seen, existed), (set(), False))
+            save_seen(path, {"a", "b", "c"}, cap=5000)
+            reloaded, existed2 = load_seen(path)
+            self.assertEqual(reloaded, {"a", "b", "c"})
+            self.assertTrue(existed2)
+
+    def test_cap_bounds_state(self):
+        """NEWS-05: save keeps at most `cap` keys."""
+        import json
+        import tempfile
+        from pathlib import Path
+
+        with tempfile.TemporaryDirectory() as tmp:
+            path = str(Path(tmp) / "state.json")
+            save_seen(path, {f"k{i}" for i in range(100)}, cap=10)
+            self.assertEqual(len(json.load(open(path))), 10)

tests/test_spec_url.py

@@ -79,6 +79,65 @@ class TestRedirectRevalidation(unittest.IsolatedAsyncioTestCase):
                 await reader._get(FakeSession(), "http://safe.example.com", 1000)
 
 
+class TestMetaRefresh(unittest.IsolatedAsyncioTestCase):
+    async def test_follows_meta_refresh_to_real_article(self):
+        """URL-04: a getnews-style meta-refresh stub is followed to the real article."""
+        reader = URLReader(lambda: {}, None)
+        stub = (
+            b'<html><head><meta http-equiv="refresh" content="0;url=https://pushsquare.com/real"></head><body>Redirecting...</body></html>'
+        )
+        article = b"<html><body><h1>MARVEL Tokon</h1><p>Full article text here</p></body></html>"
+        calls = []
+
+        async def fake_get(session, url, max_bytes):
+            calls.append(url)
+            return (url, stub if "stub" in url else article)
+
+        reader._get = fake_get  # type: ignore
+        with patch("fjerkroa_bot.url_reader.guard_url", return_value=None):
+            import fjerkroa_bot.url_reader as ur
+
+            # patch the session context so fetch() runs against fake_get
+            class FakeCM:
+                async def __aenter__(self):
+                    return object()
+
+                async def __aexit__(self, *a):
+                    return False
+
+            with patch.object(ur.aiohttp, "ClientSession", return_value=FakeCM()):
+                result = await reader.fetch("https://gggemein.de/url/stub.html", "chat", "alice")
+        self.assertIn("Full article text", result["text"])
+        self.assertEqual(result["url"], "https://pushsquare.com/real")
+        self.assertIn("https://pushsquare.com/real", calls)
+
+    async def test_meta_refresh_to_internal_is_not_followed(self):
+        """URL-04: a meta-refresh pointing at an internal IP is refused (SSRF)."""
+        reader = URLReader(lambda: {}, None)
+        stub = b'<meta http-equiv="refresh" content="0; url=http://127.0.0.1/secret">Redirecting'
+
+        async def fake_get(session, url, max_bytes):
+            return (url, stub)
+
+        reader._get = fake_get  # type: ignore
+        import fjerkroa_bot.url_reader as ur
+
+        class FakeCM:
+            async def __aenter__(self):
+                return object()
+
+            async def __aexit__(self, *a):
+                return False
+
+        def guard(u):
+            return "refused" if "127.0.0.1" in u else None
+
+        with patch("fjerkroa_bot.url_reader.guard_url", side_effect=guard):
+            with patch.object(ur.aiohttp, "ClientSession", return_value=FakeCM()):
+                result = await reader.fetch("https://safe.com/x", "chat", "alice")
+        self.assertEqual(result["url"], "https://safe.com/x")  # did not follow to 127.0.0.1
+
+
 class TestTextExtraction(unittest.TestCase):
     def test_html_reduced_to_text(self):
         """URL-05: scripts/styles dropped, tags stripped."""