diff --git a/fjerkroa_bot/news.py b/fjerkroa_bot/news.py index 636aef2..1349843 100644 --- a/fjerkroa_bot/news.py +++ b/fjerkroa_bot/news.py @@ -27,6 +27,7 @@ DEFAULT_SUMMARY_CHARS = 200 DEFAULT_NEWS_KEEP = 400 FETCH_TIMEOUT_S = 15 _ATOM = "{http://www.w3.org/2005/Atom}" +_RSS1 = "{http://purl.org/rss/1.0/}" # RSS 1.0 / RDF (e.g. 4gamer.net) namespaces //<link> _TAG_RE = re.compile(r"<[^>]+>") @@ -36,22 +37,28 @@ def _clean_summary(raw: str, max_len: int = 300) -> str: return re.sub(r"\s+", " ", text).strip()[:max_len] +def _rss_items(root: Any, ns: str, source: str) -> List[Dict[str, str]]: + """RSS 2.0 (ns='') and RSS 1.0/RDF (ns=_RSS1) both use <item><title><link><description>.""" + out: List[Dict[str, str]] = [] + for item in root.iter(f"{ns}item"): + title = (item.findtext(f"{ns}title") or "").strip() + link = (item.findtext(f"{ns}link") or "").strip() + summary = _clean_summary(item.findtext(f"{ns}description") or "") + if title: + out.append({"title": title, "link": link, "source": source, "summary": summary}) + return out + + def parse_feed(data: bytes, source: str = "") -> List[Dict[str, str]]: - """Parse RSS or Atom bytes into [{title, link, source}] (tolerant).""" + """Parse RSS 2.0, RSS 1.0/RDF, or Atom bytes into [{title, link, source, summary}] (tolerant).""" try: root = ElementTree.fromstring(data) except Exception as err: # malformed XML or a blocked entity/DTD attack — tolerate, never raise (NEWS-01) logging.warning(f"news: unparseable/unsafe feed {source!r}: {err!r}") return [] - items: List[Dict[str, str]] = [] - # RSS: <rss><channel><item><title/><link/><description/> - for item in root.iter("item"): - title = (item.findtext("title") or "").strip() - link = (item.findtext("link") or "").strip() - summary = _clean_summary(item.findtext("description") or "") - if title: - items.append({"title": title, "link": link, "source": source, "summary": summary}) + # RSS 2.0 (unqualified) + RSS 1.0/RDF (namespaced, e.g. 4gamer) share <item><title><link><description> + items: List[Dict[str, str]] = _rss_items(root, "", source) + _rss_items(root, _RSS1, source) # Atom: <feed><entry><title/><link href=/><summary|content/> for entry in root.iter(f"{_ATOM}entry"): title = (entry.findtext(f"{_ATOM}title") or "").strip() diff --git a/specs/SPEC-013-news.md b/specs/SPEC-013-news.md index cdf5694..5c60cc7 100644 --- a/specs/SPEC-013-news.md +++ b/specs/SPEC-013-news.md @@ -6,7 +6,7 @@ Replaces the broken pre-1.0-openai `news_feed.py`. A CLI `AIResponder.message` injects into the `{news}` slot. Feeds are external input and operator-configured. -### NEWS-01 — RSS and Atom parse to items (coverage: test) +### NEWS-01 — RSS (2.0 and 1.0/RDF) and Atom parse to items (coverage: test) `parse_feed(bytes, label)` extracts `{title, link, source}` from both RSS (`<item>`) and Atom (`<entry>`) documents, tolerates malformed diff --git a/tests/test_spec_news.py b/tests/test_spec_news.py index d51faac..d472bf1 100644 --- a/tests/test_spec_news.py +++ b/tests/test_spec_news.py @@ -29,6 +29,16 @@ ATOM = b"""<?xml version="1.0"?><feed xmlns="http://www.w3.org/2005/Atom"> <entry><title>Atom headline """ +RSS1 = ( + '' + '' + 'Feed' + 'ゲームニュースhttps://ex.jp/1' + "本文ここ" + 'Secondhttps://ex.jp/2' + "" +).encode("utf-8") + class TestParse(unittest.TestCase): def test_rss(self): @@ -44,6 +54,13 @@ class TestParse(unittest.TestCase): self.assertEqual(items[0]["title"], "Atom headline") self.assertEqual(items[0]["link"], "https://ex.com/a") + def test_rss1_rdf(self): + """NEWS-01: RSS 1.0/RDF (namespaced , e.g. 4gamer.net) parses like RSS 2.0.""" + items = parse_feed(RSS1, "JP") + self.assertEqual([i["title"] for i in items], ["ゲームニュース", "Second"]) + self.assertEqual(items[0]["link"], "https://ex.jp/1") + self.assertEqual(items[0]["summary"], "本文ここ") + def test_malformed_never_raises(self): """NEWS-01: garbage XML returns [] without raising.""" self.assertEqual(parse_feed(b"