"""News digest fetcher (SPEC-013, FDB-012 news rewrite). Replaces the broken pre-1.0-openai `news_feed.py`. Fetches configured RSS/Atom feeds (stdlib, no feedparser dep), builds a compact sanitized headline digest, and writes it to the `{news}` file the responder injects (AIResponder.message). Feeds are external input: titles are sanitized (SAF-03) and each feed URL is SSRF-guarded before fetching. CLI: python -m fjerkroa_bot.news --config kroa.toml """ import argparse import logging import sys import time from typing import Any, Dict, List, Optional, Tuple import defusedxml.ElementTree as ElementTree # hardened XML: feeds are untrusted (XXE/billion-laughs) from .ai_responder import sanitize_external_text DEFAULT_PER_FEED = 3 DEFAULT_MAX_ITEMS = 15 FETCH_TIMEOUT_S = 15 _ATOM = "{http://www.w3.org/2005/Atom}" def parse_feed(data: bytes, source: str = "") -> List[Dict[str, str]]: """Parse RSS or Atom bytes into [{title, link, source}] (tolerant).""" try: root = ElementTree.fromstring(data) except Exception as err: # malformed XML or a blocked entity/DTD attack — tolerate, never raise (NEWS-01) logging.warning(f"news: unparseable/unsafe feed {source!r}: {err!r}") return [] items: List[Dict[str, str]] = [] # RSS: <link/> for item in root.iter("item"): title = (item.findtext("title") or "").strip() link = (item.findtext("link") or "").strip() if title: items.append({"title": title, "link": link, "source": source}) # Atom: <feed><entry><title/><link href=/> for entry in root.iter(f"{_ATOM}entry"): title = (entry.findtext(f"{_ATOM}title") or "").strip() link_el = entry.find(f"{_ATOM}link") link = link_el.get("href", "") if link_el is not None else "" if title: items.append({"title": title, "link": link, "source": source}) return items def render_digest(items: List[Dict[str, str]], max_items: int = DEFAULT_MAX_ITEMS) -> str: """Compact sanitized digest for the {news} prompt slot.""" lines = [] for item in items[:max_items]: title = sanitize_external_text(item["title"], 200) source = item.get("source", "") link = item.get("link", "") prefix = f"[{source}] " if source else "" lines.append(f"- {prefix}{title}" + (f" ({link})" if link else "")) return "\n".join(lines) class NewsFetcher: def __init__(self, guard, fetch_bytes) -> None: # injected so tests need no network; production wires aiohttp + guard_url self._guard = guard self._fetch_bytes = fetch_bytes async def collect(self, feeds: List[Tuple[str, str]], per_feed: int) -> List[Dict[str, str]]: """feeds = [(url, label)]; returns deduped items, order preserved.""" seen = set() out: List[Dict[str, str]] = [] for url, label in feeds: reason = self._guard(url) if reason: logging.warning(f"news: skipping feed {label} — {reason}") continue try: data = await self._fetch_bytes(url) except Exception as err: logging.warning(f"news: fetch failed for {label}: {repr(err)}") continue for item in parse_feed(data, label)[:per_feed]: key = item["title"] if key not in seen: seen.add(key) out.append(item) return out def _feeds_from_config(config: Dict[str, Any]) -> List[Tuple[str, str]]: """news-feeds = [["url", "label"], ...] or ["url", ...].""" feeds = [] for entry in config.get("news-feeds", []): if isinstance(entry, (list, tuple)): feeds.append((str(entry[0]), str(entry[1]) if len(entry) > 1 else "")) else: feeds.append((str(entry), "")) return feeds async def _aiohttp_fetch(url: str) -> bytes: import aiohttp from .httpread import read_capped timeout = aiohttp.ClientTimeout(total=FETCH_TIMEOUT_S) async with aiohttp.ClientSession(timeout=timeout, headers={"User-Agent": "Mozilla/5.0 (compatible; FjerkroaBot-news/1.0)"}) as session: async with session.get(url) as response: response.raise_for_status() return await read_capped(response, 4 * 1024 * 1024) async def run(config: Dict[str, Any]) -> Optional[str]: from .url_reader import guard_url out_path = config.get("news") if not out_path: logging.error("news: no `news` output path in config") return None feeds = _feeds_from_config(config) if not feeds: logging.error("news: no `news-feeds` configured") return None fetcher = NewsFetcher(guard_url, _aiohttp_fetch) items = await fetcher.collect(feeds, int(config.get("news-per-feed", DEFAULT_PER_FEED))) digest = render_digest(items, int(config.get("news-max-items", DEFAULT_MAX_ITEMS))) header = f"News as of {time.strftime('%Y-%m-%d %H:%M UTC', time.gmtime())}:\n" with open(out_path, "w", encoding="utf-8") as fd: fd.write(header + digest + "\n") logging.info(f"news: wrote {len(items)} items to {out_path}") return out_path def main() -> int: import asyncio import tomlkit logging.basicConfig(level=logging.INFO, format="%(asctime)s [%(levelname)s] %(message)s") parser = argparse.ArgumentParser(description="Fetch RSS/Atom feeds into the {news} digest file") parser.add_argument("--config", required=True) args = parser.parse_args() with open(args.config, encoding="utf-8") as fd: config = tomlkit.load(fd) result = asyncio.run(run(config)) return 0 if result else 1 if __name__ == "__main__": sys.exit(main())