news webhook posting mode: replaces py3.8 getnews (bounded state, seed-on-first-run, ssrf-guarded)

fjerkroa_bot/news.py

@@ -90,6 +90,102 @@ class NewsFetcher:
         return out
 
 
+DEFAULT_SEEN_CAP = 5000
+DEFAULT_POST_PER_FEED = 5
+DEFAULT_POST_MAX_PER_RUN = 8
+
+
+def item_key(item: Dict[str, str]) -> str:
+    return item.get("link") or item.get("title") or ""
+
+
+class NewsPoster:
+    """Post NEW feed items to Discord channel webhooks (ggg model, SPEC-013 NEWS-04..06)."""
+
+    def __init__(self, guard, fetch_bytes, post_webhook) -> None:
+        self._guard = guard
+        self._fetch_bytes = fetch_bytes
+        self._post_webhook = post_webhook
+
+    async def run_post(
+        self,
+        feeds: List[Tuple[str, str, str]],
+        webhooks: Dict[str, str],
+        seen: set,
+        per_feed: int,
+        max_per_run: int,
+        seed_only: bool,
+    ) -> Tuple[int, set]:
+        """Returns (posted_count, updated_seen). seed_only marks new items seen without posting."""
+        posted = 0
+        for url, label, channel in feeds:
+            reason = self._guard(url)
+            if reason:
+                logging.warning(f"news-post: skipping feed {label} — {reason}")
+                continue
+            try:
+                data = await self._fetch_bytes(url)
+            except Exception as err:
+                logging.warning(f"news-post: fetch failed for {label}: {repr(err)}")
+                continue
+            for item in parse_feed(data, label)[:per_feed]:
+                key = item_key(item)
+                if not key or key in seen:
+                    continue
+                seen.add(key)
+                may_post = not seed_only and posted < max_per_run
+                if may_post and await self._deliver(item, label, channel, webhooks):
+                    posted += 1
+        return posted, seen
+
+    async def _deliver(self, item: Dict[str, str], label: str, channel: str, webhooks: Dict[str, str]) -> bool:
+        hook = webhooks.get(channel)
+        if not hook:
+            logging.warning(f"news-post: no webhook for channel {channel!r} ({label})")
+            return False
+        title = sanitize_external_text(item["title"], 300)
+        link = item.get("link", "")
+        content = f"**[{label}]** {title}" + (f"\n{link}" if link else "")
+        try:
+            await self._post_webhook(hook, content)
+            return True
+        except Exception as err:
+            logging.warning(f"news-post: webhook post failed ({label}): {repr(err)}")
+            return False
+
+
+def load_seen(path: str) -> Tuple[set, bool]:
+    """(seen-set, existed). Missing/broken state -> empty set, existed=False (seed run)."""
+    import json
+    import os
+
+    if not os.path.exists(path):
+        return set(), False
+    try:
+        with open(path, encoding="utf-8") as fd:
+            return set(json.load(fd)), True
+    except Exception as err:
+        logging.warning(f"news-post: unreadable state {path}: {err!r} — reseeding")
+        return set(), False
+
+
+def save_seen(path: str, seen: set, cap: int = DEFAULT_SEEN_CAP) -> None:
+    import json
+
+    # keep the newest `cap` keys (insertion order preserved by Python sets? no — use a bounded slice)
+    keys = list(seen)[-cap:]
+    with open(path, "w", encoding="utf-8") as fd:
+        json.dump(keys, fd)
+
+
+def _post_feeds_from_config(config: Dict[str, Any]) -> List[Tuple[str, str, str]]:
+    feeds = []
+    for entry in config.get("news-post-feeds", []):
+        if isinstance(entry, (list, tuple)) and len(entry) >= 3:
+            feeds.append((str(entry[0]), str(entry[1]), str(entry[2])))
+    return feeds
+
+
 def _feeds_from_config(config: Dict[str, Any]) -> List[Tuple[str, str]]:
     """news-feeds = [["url", "label"], ...] or ["url", ...]."""
     feeds = []
@@ -113,6 +209,42 @@ async def _aiohttp_fetch(url: str) -> bytes:
             return await read_capped(response, 4 * 1024 * 1024)
 
 
+async def _aiohttp_post(hook: str, content: str) -> None:
+    import aiohttp
+
+    timeout = aiohttp.ClientTimeout(total=FETCH_TIMEOUT_S)
+    async with aiohttp.ClientSession(timeout=timeout) as session:
+        # allowed_mentions none: a headline can never ping the channel (SAF-02 spirit)
+        payload = {"content": content[:2000], "allowed_mentions": {"parse": []}}
+        async with session.post(hook, json=payload) as response:
+            response.raise_for_status()
+
+
+async def run_post(config: Dict[str, Any]) -> int:
+    """Webhook-posting mode (ggg): post new items to channels. Returns posted count."""
+    from .url_reader import guard_url
+
+    webhooks = dict(config.get("news-post-webhooks", {}))
+    feeds = _post_feeds_from_config(config)
+    state_path = config.get("news-post-state", "news_state.json")
+    if not webhooks or not feeds:
+        logging.error("news-post: need news-post-webhooks and news-post-feeds")
+        return 0
+    seen, existed = load_seen(state_path)
+    poster = NewsPoster(guard_url, _aiohttp_fetch, _aiohttp_post)
+    posted, seen = await poster.run_post(
+        feeds,
+        webhooks,
+        seen,
+        int(config.get("news-post-per-feed", DEFAULT_POST_PER_FEED)),
+        int(config.get("news-post-max-per-run", DEFAULT_POST_MAX_PER_RUN)),
+        seed_only=not existed,  # first run seeds without flooding the channels
+    )
+    save_seen(state_path, seen, int(config.get("news-post-seen-cap", DEFAULT_SEEN_CAP)))
+    logging.info(f"news-post: posted {posted} item(s)" + (" (seed run — nothing posted)" if not existed else ""))
+    return posted
+
+
 async def run(config: Dict[str, Any]) -> Optional[str]:
     from .url_reader import guard_url
 
@@ -140,11 +272,17 @@ def main() -> int:
     import tomlkit
 
     logging.basicConfig(level=logging.INFO, format="%(asctime)s [%(levelname)s] %(message)s")
-    parser = argparse.ArgumentParser(description="Fetch RSS/Atom feeds into the {news} digest file")
+    parser = argparse.ArgumentParser(
+        description="Fetch RSS/Atom feeds: --post to channel webhooks (ggg) or default {news} digest file (kroa)"
+    )
     parser.add_argument("--config", required=True)
+    parser.add_argument("--post", action="store_true", help="webhook-posting mode (post new items to Discord channels)")
     args = parser.parse_args()
     with open(args.config, encoding="utf-8") as fd:
         config = tomlkit.load(fd)
+    if args.post:
+        asyncio.run(run_post(config))
+        return 0
     result = asyncio.run(run(config))
     return 0 if result else 1
 

specs/SPEC-013-news.md

@@ -23,3 +23,32 @@ or control characters into the prompt via a headline.
 `NewsFetcher.collect` skips any feed URL the SSRF guard rejects,
 skips feeds that fail to fetch (one bad feed never sinks the run),
 and drops duplicate headlines across feeds.
+
+## Webhook posting (ggg model)
+
+`--post` mode fetches feeds mapped to channels and posts NEW items to
+the channel's Discord webhook — replacing the py3.8 `getnews.py`
+(dead play3 feed, 35 MB substring-scan state file, HTML-redirect
+cruft). Config: `news-post-feeds = [[url, label, channel], …]`,
+`news-post-webhooks = {channel = url}`, `news-post-state`.
+
+### NEWS-04 — Only unseen items post, then are marked seen (coverage: test)
+
+`NewsPoster.run_post` posts each item whose key (link, else title) is
+not in the seen-set, adds it to the set, and posts to the mapped
+channel's webhook. Re-runs over the same feed post nothing new.
+
+### NEWS-05 — First run seeds without flooding (coverage: test)
+
+With no prior state file (`seed_only`), every current item is marked
+seen but nothing is posted — migrating off getnews.py never dumps a
+backlog into the channels. `news-post-max-per-run` caps steady-state
+posts per run.
+
+### NEWS-06 — Post failures and bad channels are survived (coverage: test)
+
+A feed the SSRF guard rejects, a feed that fails to fetch, an item
+whose channel has no configured webhook, and a webhook POST that
+raises are each logged and skipped — one failure never sinks the
+run, and the seen-set still advances for successfully-processed
+items.

tests/test_spec_news.py

@@ -3,7 +3,7 @@
 import unittest
 from unittest.mock import AsyncMock
 
-from fjerkroa_bot.news import NewsFetcher, parse_feed, render_digest
+from fjerkroa_bot.news import NewsFetcher, NewsPoster, load_seen, parse_feed, render_digest, save_seen
 
 RSS = b"""<?xml version="1.0"?><rss><channel>
 <item><title>Game X released</title><link>https://ex.com/x</link></item>
@@ -73,3 +73,103 @@ class TestCollect(unittest.IsolatedAsyncioTestCase):
         fetcher = NewsFetcher(lambda u: None, AsyncMock(return_value=RSS))
         items = await fetcher.collect([("https://a.com", "A")], per_feed=1)
         self.assertEqual(len(items), 1)
+
+
+class TestPoster(unittest.IsolatedAsyncioTestCase):
+    def poster(self, posts):
+        async def fetch(url):
+            return RSS
+
+        async def post(hook, content):
+            posts.append((hook, content))
+
+        return NewsPoster(lambda u: None, fetch, post)
+
+    async def test_posts_unseen_then_dedups(self):
+        """NEWS-04: unseen items post to the mapped webhook; re-run posts nothing."""
+        posts = []
+        poster = self.poster(posts)
+        feeds = [("https://a.com/feed", "PS", "news")]
+        hooks = {"news": "https://discord.com/api/webhooks/x"}
+        posted, seen = await poster.run_post(feeds, hooks, set(), per_feed=5, max_per_run=8, seed_only=False)
+        self.assertEqual(posted, 2)
+        self.assertIn("PS", posts[0][1])
+        self.assertIn("https://discord.com/api/webhooks/x", posts[0][0])
+        # re-run with the accumulated seen -> nothing new
+        posts.clear()
+        posted2, _ = await poster.run_post(feeds, hooks, seen, per_feed=5, max_per_run=8, seed_only=False)
+        self.assertEqual(posted2, 0)
+        self.assertEqual(posts, [])
+
+    async def test_seed_run_posts_nothing(self):
+        """NEWS-05: seed_only marks items seen without posting."""
+        posts = []
+        poster = self.poster(posts)
+        feeds = [("https://a.com/feed", "PS", "news")]
+        posted, seen = await poster.run_post(feeds, {"news": "h"}, set(), 5, 8, seed_only=True)
+        self.assertEqual(posted, 0)
+        self.assertEqual(posts, [])
+        self.assertEqual(len(seen), 2)  # both marked seen
+
+    async def test_max_per_run_caps(self):
+        """NEWS-05: max-per-run caps posts; extras stay seen (not re-posted next run)."""
+        posts = []
+        poster = self.poster(posts)
+        feeds = [("https://a.com/feed", "PS", "news")]
+        posted, seen = await poster.run_post(feeds, {"news": "h"}, set(), per_feed=5, max_per_run=1, seed_only=False)
+        self.assertEqual(posted, 1)
+        self.assertEqual(len(seen), 2)  # both seen, only one posted
+
+    async def test_failures_survived(self):
+        """NEWS-06: SSRF-skip, fetch fail, missing webhook, post error each survive."""
+        posts = []
+
+        async def fetch(url):
+            if "boom" in url:
+                raise ValueError("boom")
+            return RSS
+
+        async def post(hook, content):
+            if hook == "bad":
+                raise RuntimeError("post failed")
+            posts.append((hook, content))
+
+        def guard(url):
+            return "refused" if "internal" in url else None
+
+        poster = NewsPoster(guard, fetch, post)
+        feeds = [
+            ("https://internal/feed", "I", "news"),  # SSRF-skipped
+            ("https://boom.com/feed", "B", "news"),  # fetch fails
+            ("https://ok.com/feed", "OK", "nowhere"),  # no webhook for channel
+            ("https://ok2.com/feed", "OK2", "news"),  # webhook raises
+        ]
+        posted, seen = await poster.run_post(feeds, {"news": "bad"}, set(), 5, 8, seed_only=False)
+        self.assertEqual(posted, 0)  # everything failed/skipped, no crash
+
+
+class TestSeenState(unittest.TestCase):
+    def test_roundtrip_and_seed_detection(self):
+        """NEWS-05: missing state -> (empty, existed=False); saved state reloads."""
+        import tempfile
+        from pathlib import Path
+
+        with tempfile.TemporaryDirectory() as tmp:
+            path = str(Path(tmp) / "state.json")
+            seen, existed = load_seen(path)
+            self.assertEqual((seen, existed), (set(), False))
+            save_seen(path, {"a", "b", "c"}, cap=5000)
+            reloaded, existed2 = load_seen(path)
+            self.assertEqual(reloaded, {"a", "b", "c"})
+            self.assertTrue(existed2)
+
+    def test_cap_bounds_state(self):
+        """NEWS-05: save keeps at most `cap` keys."""
+        import json
+        import tempfile
+        from pathlib import Path
+
+        with tempfile.TemporaryDirectory() as tmp:
+            path = str(Path(tmp) / "state.json")
+            save_seen(path, {f"k{i}" for i in range(100)}, cap=10)
+            self.assertEqual(len(json.load(open(path))), 10)