discord_bot/fjerkroa_bot/news.py

"""News digest fetcher (SPEC-013, FDB-012 news rewrite).

Replaces the broken pre-1.0-openai `news_feed.py`. Fetches configured
RSS/Atom feeds (stdlib, no feedparser dep), builds a compact sanitized
headline digest, and writes it to the `{news}` file the responder
injects (AIResponder.message). Feeds are external input: titles are
sanitized (SAF-03) and each feed URL is SSRF-guarded before fetching.

CLI: python -m fjerkroa_bot.news --config kroa.toml
"""

import argparse
import logging
import sys
import time
from typing import Any, Dict, List, Optional, Tuple

import defusedxml.ElementTree as ElementTree  # hardened XML: feeds are untrusted (XXE/billion-laughs)

from .ai_responder import sanitize_external_text

DEFAULT_PER_FEED = 3
DEFAULT_MAX_ITEMS = 15
FETCH_TIMEOUT_S = 15
_ATOM = "{http://www.w3.org/2005/Atom}"


def parse_feed(data: bytes, source: str = "") -> List[Dict[str, str]]:
    """Parse RSS or Atom bytes into [{title, link, source}] (tolerant)."""
    try:
        root = ElementTree.fromstring(data)
    except Exception as err:
        # malformed XML or a blocked entity/DTD attack — tolerate, never raise (NEWS-01)
        logging.warning(f"news: unparseable/unsafe feed {source!r}: {err!r}")
        return []
    items: List[Dict[str, str]] = []
    # RSS: <rss><channel><item><title/><link/>
    for item in root.iter("item"):
        title = (item.findtext("title") or "").strip()
        link = (item.findtext("link") or "").strip()
        if title:
            items.append({"title": title, "link": link, "source": source})
    # Atom: <feed><entry><title/><link href=/>
    for entry in root.iter(f"{_ATOM}entry"):
        title = (entry.findtext(f"{_ATOM}title") or "").strip()
        link_el = entry.find(f"{_ATOM}link")
        link = link_el.get("href", "") if link_el is not None else ""
        if title:
            items.append({"title": title, "link": link, "source": source})
    return items


def render_digest(items: List[Dict[str, str]], max_items: int = DEFAULT_MAX_ITEMS) -> str:
    """Compact sanitized digest for the {news} prompt slot."""
    lines = []
    for item in items[:max_items]:
        title = sanitize_external_text(item["title"], 200)
        source = item.get("source", "")
        link = item.get("link", "")
        prefix = f"[{source}] " if source else ""
        lines.append(f"- {prefix}{title}" + (f" ({link})" if link else ""))
    return "\n".join(lines)


class NewsFetcher:
    def __init__(self, guard, fetch_bytes) -> None:
        # injected so tests need no network; production wires aiohttp + guard_url
        self._guard = guard
        self._fetch_bytes = fetch_bytes

    async def collect(self, feeds: List[Tuple[str, str]], per_feed: int) -> List[Dict[str, str]]:
        """feeds = [(url, label)]; returns deduped items, order preserved."""
        seen = set()
        out: List[Dict[str, str]] = []
        for url, label in feeds:
            reason = self._guard(url)
            if reason:
                logging.warning(f"news: skipping feed {label} — {reason}")
                continue
            try:
                data = await self._fetch_bytes(url)
            except Exception as err:
                logging.warning(f"news: fetch failed for {label}: {repr(err)}")
                continue
            for item in parse_feed(data, label)[:per_feed]:
                key = item["title"]
                if key not in seen:
                    seen.add(key)
                    out.append(item)
        return out


def _feeds_from_config(config: Dict[str, Any]) -> List[Tuple[str, str]]:
    """news-feeds = [["url", "label"], ...] or ["url", ...]."""
    feeds = []
    for entry in config.get("news-feeds", []):
        if isinstance(entry, (list, tuple)):
            feeds.append((str(entry[0]), str(entry[1]) if len(entry) > 1 else ""))
        else:
            feeds.append((str(entry), ""))
    return feeds


async def _aiohttp_fetch(url: str) -> bytes:
    import aiohttp

    from .httpread import read_capped

    timeout = aiohttp.ClientTimeout(total=FETCH_TIMEOUT_S)
    async with aiohttp.ClientSession(timeout=timeout, headers={"User-Agent": "Mozilla/5.0 (compatible; FjerkroaBot-news/1.0)"}) as session:
        async with session.get(url) as response:
            response.raise_for_status()
            return await read_capped(response, 4 * 1024 * 1024)


async def run(config: Dict[str, Any]) -> Optional[str]:
    from .url_reader import guard_url

    out_path = config.get("news")
    if not out_path:
        logging.error("news: no `news` output path in config")
        return None
    feeds = _feeds_from_config(config)
    if not feeds:
        logging.error("news: no `news-feeds` configured")
        return None
    fetcher = NewsFetcher(guard_url, _aiohttp_fetch)
    items = await fetcher.collect(feeds, int(config.get("news-per-feed", DEFAULT_PER_FEED)))
    digest = render_digest(items, int(config.get("news-max-items", DEFAULT_MAX_ITEMS)))
    header = f"News as of {time.strftime('%Y-%m-%d %H:%M UTC', time.gmtime())}:\n"
    with open(out_path, "w", encoding="utf-8") as fd:
        fd.write(header + digest + "\n")
    logging.info(f"news: wrote {len(items)} items to {out_path}")
    return out_path


def main() -> int:
    import asyncio

    import tomlkit

    logging.basicConfig(level=logging.INFO, format="%(asctime)s [%(levelname)s] %(message)s")
    parser = argparse.ArgumentParser(description="Fetch RSS/Atom feeds into the {news} digest file")
    parser.add_argument("--config", required=True)
    args = parser.parse_args()
    with open(args.config, encoding="utf-8") as fd:
        config = tomlkit.load(fd)
    result = asyncio.run(run(config))
    return 0 if result else 1


if __name__ == "__main__":
    sys.exit(main())