diff --git a/fjerkroa_bot/news.py b/fjerkroa_bot/news.py index 7706cdf..74a5ae2 100644 --- a/fjerkroa_bot/news.py +++ b/fjerkroa_bot/news.py @@ -90,6 +90,102 @@ class NewsFetcher: return out +DEFAULT_SEEN_CAP = 5000 +DEFAULT_POST_PER_FEED = 5 +DEFAULT_POST_MAX_PER_RUN = 8 + + +def item_key(item: Dict[str, str]) -> str: + return item.get("link") or item.get("title") or "" + + +class NewsPoster: + """Post NEW feed items to Discord channel webhooks (ggg model, SPEC-013 NEWS-04..06).""" + + def __init__(self, guard, fetch_bytes, post_webhook) -> None: + self._guard = guard + self._fetch_bytes = fetch_bytes + self._post_webhook = post_webhook + + async def run_post( + self, + feeds: List[Tuple[str, str, str]], + webhooks: Dict[str, str], + seen: set, + per_feed: int, + max_per_run: int, + seed_only: bool, + ) -> Tuple[int, set]: + """Returns (posted_count, updated_seen). seed_only marks new items seen without posting.""" + posted = 0 + for url, label, channel in feeds: + reason = self._guard(url) + if reason: + logging.warning(f"news-post: skipping feed {label} — {reason}") + continue + try: + data = await self._fetch_bytes(url) + except Exception as err: + logging.warning(f"news-post: fetch failed for {label}: {repr(err)}") + continue + for item in parse_feed(data, label)[:per_feed]: + key = item_key(item) + if not key or key in seen: + continue + seen.add(key) + may_post = not seed_only and posted < max_per_run + if may_post and await self._deliver(item, label, channel, webhooks): + posted += 1 + return posted, seen + + async def _deliver(self, item: Dict[str, str], label: str, channel: str, webhooks: Dict[str, str]) -> bool: + hook = webhooks.get(channel) + if not hook: + logging.warning(f"news-post: no webhook for channel {channel!r} ({label})") + return False + title = sanitize_external_text(item["title"], 300) + link = item.get("link", "") + content = f"**[{label}]** {title}" + (f"\n{link}" if link else "") + try: + await self._post_webhook(hook, content) + return True + except Exception as err: + logging.warning(f"news-post: webhook post failed ({label}): {repr(err)}") + return False + + +def load_seen(path: str) -> Tuple[set, bool]: + """(seen-set, existed). Missing/broken state -> empty set, existed=False (seed run).""" + import json + import os + + if not os.path.exists(path): + return set(), False + try: + with open(path, encoding="utf-8") as fd: + return set(json.load(fd)), True + except Exception as err: + logging.warning(f"news-post: unreadable state {path}: {err!r} — reseeding") + return set(), False + + +def save_seen(path: str, seen: set, cap: int = DEFAULT_SEEN_CAP) -> None: + import json + + # keep the newest `cap` keys (insertion order preserved by Python sets? no — use a bounded slice) + keys = list(seen)[-cap:] + with open(path, "w", encoding="utf-8") as fd: + json.dump(keys, fd) + + +def _post_feeds_from_config(config: Dict[str, Any]) -> List[Tuple[str, str, str]]: + feeds = [] + for entry in config.get("news-post-feeds", []): + if isinstance(entry, (list, tuple)) and len(entry) >= 3: + feeds.append((str(entry[0]), str(entry[1]), str(entry[2]))) + return feeds + + def _feeds_from_config(config: Dict[str, Any]) -> List[Tuple[str, str]]: """news-feeds = [["url", "label"], ...] or ["url", ...].""" feeds = [] @@ -113,6 +209,42 @@ async def _aiohttp_fetch(url: str) -> bytes: return await read_capped(response, 4 * 1024 * 1024) +async def _aiohttp_post(hook: str, content: str) -> None: + import aiohttp + + timeout = aiohttp.ClientTimeout(total=FETCH_TIMEOUT_S) + async with aiohttp.ClientSession(timeout=timeout) as session: + # allowed_mentions none: a headline can never ping the channel (SAF-02 spirit) + payload = {"content": content[:2000], "allowed_mentions": {"parse": []}} + async with session.post(hook, json=payload) as response: + response.raise_for_status() + + +async def run_post(config: Dict[str, Any]) -> int: + """Webhook-posting mode (ggg): post new items to channels. Returns posted count.""" + from .url_reader import guard_url + + webhooks = dict(config.get("news-post-webhooks", {})) + feeds = _post_feeds_from_config(config) + state_path = config.get("news-post-state", "news_state.json") + if not webhooks or not feeds: + logging.error("news-post: need news-post-webhooks and news-post-feeds") + return 0 + seen, existed = load_seen(state_path) + poster = NewsPoster(guard_url, _aiohttp_fetch, _aiohttp_post) + posted, seen = await poster.run_post( + feeds, + webhooks, + seen, + int(config.get("news-post-per-feed", DEFAULT_POST_PER_FEED)), + int(config.get("news-post-max-per-run", DEFAULT_POST_MAX_PER_RUN)), + seed_only=not existed, # first run seeds without flooding the channels + ) + save_seen(state_path, seen, int(config.get("news-post-seen-cap", DEFAULT_SEEN_CAP))) + logging.info(f"news-post: posted {posted} item(s)" + (" (seed run — nothing posted)" if not existed else "")) + return posted + + async def run(config: Dict[str, Any]) -> Optional[str]: from .url_reader import guard_url @@ -140,11 +272,17 @@ def main() -> int: import tomlkit logging.basicConfig(level=logging.INFO, format="%(asctime)s [%(levelname)s] %(message)s") - parser = argparse.ArgumentParser(description="Fetch RSS/Atom feeds into the {news} digest file") + parser = argparse.ArgumentParser( + description="Fetch RSS/Atom feeds: --post to channel webhooks (ggg) or default {news} digest file (kroa)" + ) parser.add_argument("--config", required=True) + parser.add_argument("--post", action="store_true", help="webhook-posting mode (post new items to Discord channels)") args = parser.parse_args() with open(args.config, encoding="utf-8") as fd: config = tomlkit.load(fd) + if args.post: + asyncio.run(run_post(config)) + return 0 result = asyncio.run(run(config)) return 0 if result else 1 diff --git a/specs/SPEC-013-news.md b/specs/SPEC-013-news.md index 5c10473..c692325 100644 --- a/specs/SPEC-013-news.md +++ b/specs/SPEC-013-news.md @@ -23,3 +23,32 @@ or control characters into the prompt via a headline. `NewsFetcher.collect` skips any feed URL the SSRF guard rejects, skips feeds that fail to fetch (one bad feed never sinks the run), and drops duplicate headlines across feeds. + +## Webhook posting (ggg model) + +`--post` mode fetches feeds mapped to channels and posts NEW items to +the channel's Discord webhook — replacing the py3.8 `getnews.py` +(dead play3 feed, 35 MB substring-scan state file, HTML-redirect +cruft). Config: `news-post-feeds = [[url, label, channel], …]`, +`news-post-webhooks = {channel = url}`, `news-post-state`. + +### NEWS-04 — Only unseen items post, then are marked seen (coverage: test) + +`NewsPoster.run_post` posts each item whose key (link, else title) is +not in the seen-set, adds it to the set, and posts to the mapped +channel's webhook. Re-runs over the same feed post nothing new. + +### NEWS-05 — First run seeds without flooding (coverage: test) + +With no prior state file (`seed_only`), every current item is marked +seen but nothing is posted — migrating off getnews.py never dumps a +backlog into the channels. `news-post-max-per-run` caps steady-state +posts per run. + +### NEWS-06 — Post failures and bad channels are survived (coverage: test) + +A feed the SSRF guard rejects, a feed that fails to fetch, an item +whose channel has no configured webhook, and a webhook POST that +raises are each logged and skipped — one failure never sinks the +run, and the seen-set still advances for successfully-processed +items. diff --git a/tests/test_spec_news.py b/tests/test_spec_news.py index 057304b..d802705 100644 --- a/tests/test_spec_news.py +++ b/tests/test_spec_news.py @@ -3,7 +3,7 @@ import unittest from unittest.mock import AsyncMock -from fjerkroa_bot.news import NewsFetcher, parse_feed, render_digest +from fjerkroa_bot.news import NewsFetcher, NewsPoster, load_seen, parse_feed, render_digest, save_seen RSS = b""" Game X releasedhttps://ex.com/x @@ -73,3 +73,103 @@ class TestCollect(unittest.IsolatedAsyncioTestCase): fetcher = NewsFetcher(lambda u: None, AsyncMock(return_value=RSS)) items = await fetcher.collect([("https://a.com", "A")], per_feed=1) self.assertEqual(len(items), 1) + + +class TestPoster(unittest.IsolatedAsyncioTestCase): + def poster(self, posts): + async def fetch(url): + return RSS + + async def post(hook, content): + posts.append((hook, content)) + + return NewsPoster(lambda u: None, fetch, post) + + async def test_posts_unseen_then_dedups(self): + """NEWS-04: unseen items post to the mapped webhook; re-run posts nothing.""" + posts = [] + poster = self.poster(posts) + feeds = [("https://a.com/feed", "PS", "news")] + hooks = {"news": "https://discord.com/api/webhooks/x"} + posted, seen = await poster.run_post(feeds, hooks, set(), per_feed=5, max_per_run=8, seed_only=False) + self.assertEqual(posted, 2) + self.assertIn("PS", posts[0][1]) + self.assertIn("https://discord.com/api/webhooks/x", posts[0][0]) + # re-run with the accumulated seen -> nothing new + posts.clear() + posted2, _ = await poster.run_post(feeds, hooks, seen, per_feed=5, max_per_run=8, seed_only=False) + self.assertEqual(posted2, 0) + self.assertEqual(posts, []) + + async def test_seed_run_posts_nothing(self): + """NEWS-05: seed_only marks items seen without posting.""" + posts = [] + poster = self.poster(posts) + feeds = [("https://a.com/feed", "PS", "news")] + posted, seen = await poster.run_post(feeds, {"news": "h"}, set(), 5, 8, seed_only=True) + self.assertEqual(posted, 0) + self.assertEqual(posts, []) + self.assertEqual(len(seen), 2) # both marked seen + + async def test_max_per_run_caps(self): + """NEWS-05: max-per-run caps posts; extras stay seen (not re-posted next run).""" + posts = [] + poster = self.poster(posts) + feeds = [("https://a.com/feed", "PS", "news")] + posted, seen = await poster.run_post(feeds, {"news": "h"}, set(), per_feed=5, max_per_run=1, seed_only=False) + self.assertEqual(posted, 1) + self.assertEqual(len(seen), 2) # both seen, only one posted + + async def test_failures_survived(self): + """NEWS-06: SSRF-skip, fetch fail, missing webhook, post error each survive.""" + posts = [] + + async def fetch(url): + if "boom" in url: + raise ValueError("boom") + return RSS + + async def post(hook, content): + if hook == "bad": + raise RuntimeError("post failed") + posts.append((hook, content)) + + def guard(url): + return "refused" if "internal" in url else None + + poster = NewsPoster(guard, fetch, post) + feeds = [ + ("https://internal/feed", "I", "news"), # SSRF-skipped + ("https://boom.com/feed", "B", "news"), # fetch fails + ("https://ok.com/feed", "OK", "nowhere"), # no webhook for channel + ("https://ok2.com/feed", "OK2", "news"), # webhook raises + ] + posted, seen = await poster.run_post(feeds, {"news": "bad"}, set(), 5, 8, seed_only=False) + self.assertEqual(posted, 0) # everything failed/skipped, no crash + + +class TestSeenState(unittest.TestCase): + def test_roundtrip_and_seed_detection(self): + """NEWS-05: missing state -> (empty, existed=False); saved state reloads.""" + import tempfile + from pathlib import Path + + with tempfile.TemporaryDirectory() as tmp: + path = str(Path(tmp) / "state.json") + seen, existed = load_seen(path) + self.assertEqual((seen, existed), (set(), False)) + save_seen(path, {"a", "b", "c"}, cap=5000) + reloaded, existed2 = load_seen(path) + self.assertEqual(reloaded, {"a", "b", "c"}) + self.assertTrue(existed2) + + def test_cap_bounds_state(self): + """NEWS-05: save keeps at most `cap` keys.""" + import json + import tempfile + from pathlib import Path + + with tempfile.TemporaryDirectory() as tmp: + path = str(Path(tmp) / "state.json") + save_seen(path, {f"k{i}" for i in range(100)}, cap=10) + self.assertEqual(len(json.load(open(path))), 10)