Fjerkroa/discord_bot
1
1
Fork 0
Code Issues Pull Requests Releases Wiki Activity

news memory + get_news tool (news-07..12): feed summaries, deduped rolling store (schema v6), on-demand filtered retrieval

Browse Source
This commit is contained in:
Oleksandr Kozachuk
2026-07-14 11:34:23 +02:00
parent 891fbdc101
commit 5d01400638
6 changed files with 390 additions and 15 deletions
Download Patch File Download Diff File Expand all files Collapse all files
fjerkroa_bot/news.py
+110 -10
View File
@@ -11,8 +11,10 @@ CLI: python -m fjerkroa_bot.news --config kroa.toml
import argparse
import logging
import re
import sys
import time
from html import unescape
from typing import Any, Dict, List, Optional, Tuple
import defusedxml.ElementTree as ElementTree # hardened XML: feeds are untrusted (XXE/billion-laughs)
@@ -21,8 +23,17 @@ from .ai_responder import sanitize_external_text
DEFAULT_PER_FEED = 3
DEFAULT_MAX_ITEMS = 15
DEFAULT_SUMMARY_CHARS = 200
DEFAULT_NEWS_KEEP = 400
FETCH_TIMEOUT_S = 15
_ATOM = "{http://www.w3.org/2005/Atom}"
_TAG_RE = re.compile(r"<[^>]+>")
def _clean_summary(raw: str, max_len: int = 300) -> str:
"""Strip HTML, unescape entities, collapse whitespace (feed descriptions are often HTML)."""
text = unescape(_TAG_RE.sub(" ", raw or ""))
return re.sub(r"\s+", " ", text).strip()[:max_len]
def parse_feed(data: bytes, source: str = "") -> List[Dict[str, str]]:
@@ -34,31 +45,39 @@ def parse_feed(data: bytes, source: str = "") -> List[Dict[str, str]]:
logging.warning(f"news: unparseable/unsafe feed {source!r}: {err!r}")
return []
items: List[Dict[str, str]] = []
# RSS: <rss><channel><item><title/><link/>
# RSS: <rss><channel><item><title/><link/><description/>
for item in root.iter("item"):
title = (item.findtext("title") or "").strip()
link = (item.findtext("link") or "").strip()
summary = _clean_summary(item.findtext("description") or "")
if title:
items.append({"title": title, "link": link, "source": source})
# Atom: <feed><entry><title/><link href=/>
items.append({"title": title, "link": link, "source": source, "summary": summary})
# Atom: <feed><entry><title/><link href=/><summary|content/>
for entry in root.iter(f"{_ATOM}entry"):
title = (entry.findtext(f"{_ATOM}title") or "").strip()
link_el = entry.find(f"{_ATOM}link")
link = link_el.get("href", "") if link_el is not None else ""
summary = _clean_summary(entry.findtext(f"{_ATOM}summary") or entry.findtext(f"{_ATOM}content") or "")
if title:
items.append({"title": title, "link": link, "source": source})
items.append({"title": title, "link": link, "source": source, "summary": summary})
return items
def render_digest(items: List[Dict[str, str]], max_items: int = DEFAULT_MAX_ITEMS) -> str:
"""Compact sanitized digest for the {news} prompt slot."""
def render_digest(items: List[Dict[str, str]], max_items: int = DEFAULT_MAX_ITEMS, summary_chars: int = DEFAULT_SUMMARY_CHARS) -> str:
"""Compact sanitized digest for the {news} prompt slot (title + short summary + link)."""
lines = []
for item in items[:max_items]:
title = sanitize_external_text(item["title"], 200)
source = item.get("source", "")
link = item.get("link", "")
summary = sanitize_external_text(item.get("summary", ""), summary_chars) if summary_chars else ""
prefix = f"[{source}] " if source else ""
lines.append(f"- {prefix}{title}" + (f" ({link})" if link else ""))
line = f"- {prefix}{title}"
if summary:
line += f" — {summary}"
if link:
line += f" ({link})"
lines.append(line)
return "\n".join(lines)
@@ -102,10 +121,11 @@ def item_key(item: Dict[str, str]) -> str:
class NewsPoster:
"""Post NEW feed items to Discord channel webhooks (ggg model, SPEC-013 NEWS-04..06)."""
def __init__(self, guard, fetch_bytes, post_webhook) -> None:
def __init__(self, guard, fetch_bytes, post_webhook, store: Any = None) -> None:
self._guard = guard
self._fetch_bytes = fetch_bytes
self._post_webhook = post_webhook
self._store = store
async def run_post(
self,
@@ -115,9 +135,11 @@ class NewsPoster:
per_feed: int,
max_per_run: int,
seed_only: bool,
keep: int = DEFAULT_NEWS_KEEP,
) -> Tuple[int, set]:
"""Returns (posted_count, updated_seen). seed_only marks new items seen without posting."""
posted = 0
harvested: List[Dict[str, str]] = []
for url, label, channel in feeds:
reason = self._guard(url)
if reason:
@@ -129,6 +151,7 @@ class NewsPoster:
logging.warning(f"news-post: fetch failed for {label}: {repr(err)}")
continue
for item in parse_feed(data, label)[:per_feed]:
harvested.append(item) # NEWS-09: everything parsed feeds the searchable store
key = item_key(item)
if not key or key in seen:
continue
@@ -136,6 +159,9 @@ class NewsPoster:
may_post = not seed_only and posted < max_per_run
if may_post and await self._deliver(item, label, channel, webhooks):
posted += 1
if self._store is not None and harvested:
self._store.add_news_items(harvested)
self._store.prune_news(keep)
return posted, seen
async def _deliver(self, item: Dict[str, str], label: str, channel: str, webhooks: Dict[str, str]) -> bool:
@@ -154,6 +180,76 @@ class NewsPoster:
return False
# --- news memory + on-demand retrieval tool (SPEC-013 NEWS-09..12) ---
GET_NEWS_TOOL = {
"name": "get_news",
"description": "Fetch recent real-world news the bot has collected from its RSS feeds (local, national, world, sport, "
"culture). Use when someone asks what is new or what is happening, optionally about a topic or from a particular "
"source. Returns headlines with a short summary and a link to read more.",
"parameters": {
"type": "object",
"properties": {
"topic": {"type": "string", "description": "Optional keywords to filter by, e.g. 'Nordland', 'football', 'weather'."},
"source": {"type": "string", "description": "Optional source label, e.g. 'NRK', 'Aftenposten', 'Verden', 'Sport'."},
"limit": {"type": "integer", "description": "How many items to return (default 10, max 30)."},
},
"required": [],
},
}
def _news_terms(topic: Optional[str]) -> List[str]:
return [t for t in re.split(r"\W+", (topic or "").lower()) if len(t) > 1][:5]
def query_news(
store: Any, topic: Optional[str] = None, source: Optional[str] = None, limit: int = 10, summary_chars: int = DEFAULT_SUMMARY_CHARS
) -> Dict[str, Any]:
"""Retrieve stored news for the get_news tool: term/source-filtered, sanitized (NEWS-11)."""
if store is None:
return {"error": "news store unavailable"}
limit = max(1, min(int(limit or 10), 30))
src = (str(source).strip() or None) if source else None
terms = _news_terms(topic)
try:
rows = store.search_news(terms, limit, src) if terms else store.recent_news(limit, src)
except Exception as err:
logging.warning(f"news: query failed: {err!r}")
return {"error": "news lookup failed"}
results = [
{
"source": row.get("source", ""),
"title": sanitize_external_text(row.get("title", ""), 200),
"summary": sanitize_external_text(row.get("summary", ""), summary_chars),
"link": row.get("link", ""),
}
for row in rows
]
return {"topic": topic or "", "source": src or "", "results": results}
def _open_store(config: Dict[str, Any]) -> Any:
directory = config.get("history-directory")
if not directory:
return None
from pathlib import Path
from .persistence import PersistentStore
return PersistentStore(Path(str(directory)).expanduser() / "bot.db")
def persist_news(config: Dict[str, Any], items: List[Dict[str, str]]) -> int:
"""Upsert fetched items into the news store, prune to the rolling window (NEWS-09)."""
store = _open_store(config)
if store is None or not items:
return 0
added = store.add_news_items(items)
store.prune_news(int(config.get("news-keep", DEFAULT_NEWS_KEEP)))
return added
def load_seen(path: str) -> Tuple[set, bool]:
"""(seen-set, existed). Missing/broken state -> empty set, existed=False (seed run)."""
import json
@@ -231,7 +327,7 @@ async def run_post(config: Dict[str, Any]) -> int:
logging.error("news-post: need news-post-webhooks and news-post-feeds")
return 0
seen, existed = load_seen(state_path)
poster = NewsPoster(guard_url, _aiohttp_fetch, _aiohttp_post)
poster = NewsPoster(guard_url, _aiohttp_fetch, _aiohttp_post, store=_open_store(config))
posted, seen = await poster.run_post(
feeds,
webhooks,
@@ -239,6 +335,7 @@ async def run_post(config: Dict[str, Any]) -> int:
int(config.get("news-post-per-feed", DEFAULT_POST_PER_FEED)),
int(config.get("news-post-max-per-run", DEFAULT_POST_MAX_PER_RUN)),
seed_only=not existed, # first run seeds without flooding the channels
keep=int(config.get("news-keep", DEFAULT_NEWS_KEEP)),
)
save_seen(state_path, seen, int(config.get("news-post-seen-cap", DEFAULT_SEEN_CAP)))
logging.info(f"news-post: posted {posted} item(s)" + (" (seed run — nothing posted)" if not existed else ""))
@@ -258,7 +355,10 @@ async def run(config: Dict[str, Any]) -> Optional[str]:
return None
fetcher = NewsFetcher(guard_url, _aiohttp_fetch)
items = await fetcher.collect(feeds, int(config.get("news-per-feed", DEFAULT_PER_FEED)))
digest = render_digest(items, int(config.get("news-max-items", DEFAULT_MAX_ITEMS)))
persist_news(config, items) # NEWS-09: feed the searchable rolling store for get_news
digest = render_digest(
items, int(config.get("news-max-items", DEFAULT_MAX_ITEMS)), int(config.get("news-summary-chars", DEFAULT_SUMMARY_CHARS))
)
header = f"News as of {time.strftime('%Y-%m-%d %H:%M UTC', time.gmtime())}:\n"
with open(out_path, "w", encoding="utf-8") as fd:
fd.write(header + digest + "\n")