Compare commits

...

4 Commits

17 changed files with 550 additions and 20 deletions
+1 -1
View File
@@ -1,6 +1,6 @@
# Fjerkroa Bot Development Makefile (uv-managed) # Fjerkroa Bot Development Makefile (uv-managed)
.PHONY: help install install-dev clean test test-cov test-fast lint format format-check type-check security-check audit trace check all-checks pre-commit run run-dev build ci .PHONY: deploy backup help install install-dev clean test test-cov test-fast lint format format-check type-check security-check audit trace check all-checks pre-commit run run-dev build ci
# Default target # Default target
help: ## Show this help message help: ## Show this help message
+4
View File
@@ -105,6 +105,7 @@ class AIMessage(AIMessageBase):
self.channel = channel self.channel = channel
self.direct = direct self.direct = direct
self.historise_question = historise_question self.historise_question = historise_question
self.factual = False # classifier verdict; may route to factual-model (BEH-10)
self.vars = ["user", "message", "channel", "direct", "historise_question"] self.vars = ["user", "message", "channel", "direct", "historise_question"]
@@ -340,6 +341,9 @@ class AIResponder(AIResponderBase):
# Get the history limit from the configuration # Get the history limit from the configuration
limit = self.config["history-limit"] limit = self.config["history-limit"]
# Factual verdict routes this call to factual-model if configured (BEH-10)
self._factual = bool(getattr(message, "factual", False))
# Check if a short path applies, return an empty AIResponse if it does # Check if a short path applies, return an empty AIResponse if it does
if self.short_path(message, limit): if self.short_path(message, limit):
await self._persist_history() await self._persist_history()
+15 -2
View File
@@ -1,5 +1,6 @@
import argparse import argparse
import asyncio import asyncio
import fnmatch
import logging import logging
import random import random
import re import re
@@ -481,7 +482,7 @@ class FjerkroaBot(commands.Bot):
return fallback_channel return fallback_channel
if channel_name.startswith("#"): if channel_name.startswith("#"):
channel_name = channel_name[1:] channel_name = channel_name[1:]
if not no_ignore and channel_name in self.config.get("ignore-channels", []): if not no_ignore and self.channel_ignored(channel_name):
return fallback_channel return fallback_channel
for guild in self.guilds: for guild in self.guilds:
channel = discord.utils.get(guild.channels, name=channel_name) channel = discord.utils.get(guild.channels, name=channel_name)
@@ -494,8 +495,12 @@ class FjerkroaBot(commands.Bot):
return str(channel.recipient.name) return str(channel.recipient.name)
return str(channel.id) if isinstance(channel, DMChannel) else str(channel.name) return str(channel.id) if isinstance(channel, DMChannel) else str(channel.name)
def channel_ignored(self, channel_name) -> bool:
"""fnmatch patterns; plain names match exactly as before (BEH-09)."""
return any(fnmatch.fnmatchcase(str(channel_name), pattern) for pattern in self.config.get("ignore-channels", []))
def ignore_message(self, channel_name, message): def ignore_message(self, channel_name, message):
return channel_name in self.config.get("ignore-channels", []) and not message.direct return self.channel_ignored(channel_name) and not message.direct
def log_message_action(self, action, message, channel_name): def log_message_action(self, action, message, channel_name):
logging.info(f"{action} message {repr(message)} for channel {channel_name}") logging.info(f"{action} message {repr(message)} for channel {channel_name}")
@@ -521,6 +526,11 @@ class FjerkroaBot(commands.Bot):
async def handle_message_through_responder(self, message): async def handle_message_through_responder(self, message):
"""Handle a message through the AI responder""" """Handle a message through the AI responder"""
# Ignored channels are fully silent — before the classifier gate,
# so no emoji reaction leaks either (BEH-09). DMs are never ignored.
if not isinstance(message.channel, DMChannel) and self.channel_ignored(self.get_channel_name(message.channel)):
self.log_message_action("ignore", message, self.get_channel_name(message.channel))
return
message_content = str(message.content).strip() message_content = str(message.content).strip()
if message.reference and message.reference.resolved and isinstance(message.reference.resolved.content, str): if message.reference and message.reference.resolved and isinstance(message.reference.resolved.content, str):
reference_content = str(message.reference.resolved.content).replace("\n", "> \n") reference_content = str(message.reference.resolved.content).replace("\n", "> \n")
@@ -691,6 +701,9 @@ class FjerkroaBot(commands.Bot):
# Get the AI responder based on the channel name # Get the AI responder based on the channel name
airesponder = self.get_ai_responder(channel_name) airesponder = self.get_ai_responder(channel_name)
# Classifier verdict rides along: factual questions may use factual-model (BEH-10)
message.factual = factual
# Send the user message to the AI responder, with typing indicators. # Send the user message to the AI responder, with typing indicators.
# A raised call = a broken API path (cf. the gpt-5.6 tools incident): # A raised call = a broken API path (cf. the gpt-5.6 tools incident):
# count it, alert staff at threshold, never crash the handler (OPS-16). # count it, alert staff at threshold, never crash the handler (OPS-16).
+17 -2
View File
@@ -198,7 +198,12 @@ GET_NEWS_TOOL = {
"parameters": { "parameters": {
"type": "object", "type": "object",
"properties": { "properties": {
"topic": {"type": "string", "description": "Optional keywords to filter by, e.g. 'Nordland', 'football', 'weather'."}, "topic": {
"type": "string",
"description": "Optional filter: one or two keywords, in the language the feeds are written in "
"(e.g. Norwegian for Norwegian news: 'Nordland', 'fotball', 'trafikkulykke'). If nothing matches "
"exactly, related or recent items come back with a `note` saying so.",
},
"source": {"type": "string", "description": "Optional source label, e.g. 'NRK', 'Aftenposten', 'Verden', 'Sport'."}, "source": {"type": "string", "description": "Optional source label, e.g. 'NRK', 'Aftenposten', 'Verden', 'Sport'."},
"limit": {"type": "integer", "description": "How many items to return (default 10, max 30)."}, "limit": {"type": "integer", "description": "How many items to return (default 10, max 30)."},
}, },
@@ -220,8 +225,15 @@ def query_news(
limit = max(1, min(int(limit or 10), 30)) limit = max(1, min(int(limit or 10), 30))
src = (str(source).strip() or None) if source else None src = (str(source).strip() or None) if source else None
terms = _news_terms(topic) terms = _news_terms(topic)
note = None
try: try:
rows = store.search_news(terms, limit, src) if terms else store.recent_news(limit, src) rows = store.search_news(terms, limit, src) if terms else store.recent_news(limit, src)
if terms and not rows: # NEWS-13: soft degradation, never empty-handed
rows = store.search_news(terms, limit, src, match_any=True)
note = "no item matches all keywords; showing items matching some of them"
if terms and not rows:
rows = store.recent_news(limit, src)
note = "nothing matches the topic; showing the newest stored items instead"
except Exception as err: except Exception as err:
logging.warning(f"news: query failed: {err!r}") logging.warning(f"news: query failed: {err!r}")
return {"error": "news lookup failed"} return {"error": "news lookup failed"}
@@ -234,7 +246,10 @@ def query_news(
} }
for row in rows for row in rows
] ]
return {"topic": topic or "", "source": src or "", "results": results} payload = {"topic": topic or "", "source": src or "", "results": results}
if note:
payload["note"] = note
return payload
def _open_store(config: Dict[str, Any]) -> Any: def _open_store(config: Dict[str, Any]) -> Any:
+12
View File
@@ -17,6 +17,7 @@ from .leonardo_draw import LeonardoAIDrawMixIn
from .news import GET_NEWS_TOOL, query_news from .news import GET_NEWS_TOOL, query_news
from .quota import QuotaLedger from .quota import QuotaLedger
from .url_reader import FETCH_URL_TOOL, URLReader from .url_reader import FETCH_URL_TOOL, URLReader
from .weather import GET_WEATHER_TOOL, Weather
from .websearch import DEFAULT_RESULTS as WEB_DEFAULT_RESULTS from .websearch import DEFAULT_RESULTS as WEB_DEFAULT_RESULTS
from .websearch import WEB_SEARCH_TOOL, WebSearch from .websearch import WEB_SEARCH_TOOL, WebSearch
@@ -170,6 +171,7 @@ class OpenAIResponder(AIResponder, LeonardoAIDrawMixIn):
self.codex = CodexSearch(lambda: self.config) self.codex = CodexSearch(lambda: self.config)
# Web search (SPEC-015) via Exa; general "look it up" beyond fetch_url/news/codex # Web search (SPEC-015) via Exa; general "look it up" beyond fetch_url/news/codex
self.web_search = WebSearch(lambda: self.config) self.web_search = WebSearch(lambda: self.config)
self.weather = Weather(lambda: self.config)
def _available_tools(self) -> List[Dict[str, Any]]: def _available_tools(self) -> List[Dict[str, Any]]:
"""Assemble the function-tool list from every enabled provider (URL-01).""" """Assemble the function-tool list from every enabled provider (URL-01)."""
@@ -189,6 +191,8 @@ class OpenAIResponder(AIResponder, LeonardoAIDrawMixIn):
functions.append(GET_NEWS_TOOL) functions.append(GET_NEWS_TOOL)
if self.web_search.enabled(): # WEB-01 if self.web_search.enabled(): # WEB-01
functions.append(WEB_SEARCH_TOOL) functions.append(WEB_SEARCH_TOOL)
if self.weather.enabled(): # WEA-01
functions.append(GET_WEATHER_TOOL)
return functions return functions
async def _dispatch_tool(self, name: str, args: Dict[str, Any], author: str) -> Any: async def _dispatch_tool(self, name: str, args: Dict[str, Any], author: str) -> Any:
@@ -219,6 +223,12 @@ class OpenAIResponder(AIResponder, LeonardoAIDrawMixIn):
return {"error": "daily web search limit reached"} return {"error": "daily web search limit reached"}
self.ledger._add(f"web:{author}", 1) self.ledger._add(f"web:{author}", 1)
return await self.web_search.search(str(args.get("query", "")), int(args.get("num_results", WEB_DEFAULT_RESULTS))) return await self.web_search.search(str(args.get("query", "")), int(args.get("num_results", WEB_DEFAULT_RESULTS)))
if name == "get_weather":
per_user_cap = int(self.config.get("weather-daily-per-user", 30))
if self.ledger._get(f"weather:{author}") >= per_user_cap: # WEA-04
return {"error": "daily weather lookup limit reached"}
self.ledger._add(f"weather:{author}", 1)
return await self.weather.forecast(args.get("location"))
return await self._execute_igdb_function(name, args) return await self._execute_igdb_function(name, args)
async def draw_openai(self, description: str, count: int = 1) -> List[BytesIO]: async def draw_openai(self, description: str, count: int = 1) -> List[BytesIO]:
@@ -292,6 +302,8 @@ class OpenAIResponder(AIResponder, LeonardoAIDrawMixIn):
model = self.config["model-vision"] model = self.config["model-vision"]
else: else:
messages[-1]["content"] = messages[-1]["content"][0]["text"] messages[-1]["content"] = messages[-1]["content"][0]["text"]
if getattr(self, "_factual", False) and "factual-model" in self.config:
model = self.config["factual-model"] # BEH-10: facts get the stronger tier
if self._use_retry_model and "retry-model" in self.config: if self._use_retry_model and "retry-model" in self.config:
model = self.config["retry-model"] model = self.config["retry-model"]
except (KeyError, IndexError, TypeError) as e: except (KeyError, IndexError, TypeError) as e:
+15 -2
View File
@@ -152,14 +152,27 @@ class PersistentStore:
rows = conn.execute(sql, params).fetchall() rows = conn.execute(sql, params).fetchall()
return [{"source": r[0], "title": r[1], "link": r[2], "summary": r[3]} for r in rows] return [{"source": r[0], "title": r[1], "link": r[2], "summary": r[3]} for r in rows]
def search_news(self, terms: List[str], limit: int = 20, source: Optional[str] = None) -> List[Dict[str, Any]]: def search_news(self, terms: List[str], limit: int = 20, source: Optional[str] = None, match_any: bool = False) -> List[Dict[str, Any]]:
"""Rows where every term appears in title or summary; optional source filter (NEWS-11).""" """Rows where every term appears in title/summary/source; match_any ranks by how many terms hit (NEWS-11/13)."""
params: List[Any] = [] params: List[Any] = []
clauses = [] clauses = []
for term in terms: for term in terms:
clauses.append("(title LIKE ? OR summary LIKE ? OR source LIKE ?)") clauses.append("(title LIKE ? OR summary LIKE ? OR source LIKE ?)")
like = f"%{term}%" like = f"%{term}%"
params += [like, like, like] params += [like, like, like]
if match_any and clauses:
hits = " + ".join(clauses)
where = "hits > 0"
if source:
where += " AND source = ?"
params.append(source)
params.append(int(limit))
sql = (
f"SELECT source, title, link, summary FROM " # nosec B608 - fixed templates; values parameterised
f"(SELECT id, source, title, link, summary, {hits} AS hits FROM news) "
f"WHERE {where} ORDER BY hits DESC, id DESC LIMIT ?"
)
else:
where = " AND ".join(clauses) if clauses else "1=1" where = " AND ".join(clauses) if clauses else "1=1"
if source: if source:
where = f"({where}) AND source = ?" where = f"({where}) AND source = ?"
+3 -1
View File
@@ -20,7 +20,9 @@ DEFAULT_TASKGEN_INTERVAL_HOURS = 6.0
DEFAULT_BORENESS_PROMPT = ( DEFAULT_BORENESS_PROMPT = (
"A thought just occurred to you. Anchor it to something real you know — recent news (use get_news), a game " "A thought just occurred to you. Anchor it to something real you know — recent news (use get_news), a game "
"releasing soon, the weather, or a regular you remember — not a generic musing. Share it briefly, in your own " "releasing soon, the weather, or a regular you remember — not a generic musing. Share it briefly, in your own "
"voice, as an observation, a gentle question, or a joke; never an advertisement. Read the room and stay in character." "voice, as an observation, a gentle question, or a joke; never an advertisement. Read the room and stay in character. "
"Check your own recent posts in the history first: pick a subject you have not touched lately and a different form "
"than last time, and never open with a fixed label or heading — just start mid-thought."
) )
ExecuteCallback = Callable[[str, str], Awaitable[None]] ExecuteCallback = Callable[[str, str], Awaitable[None]]
+41 -6
View File
@@ -21,7 +21,7 @@ from .ai_responder import sanitize_external_text
from .httpread import read_capped from .httpread import read_capped
DEFAULT_MAX_BYTES = 2 * 1024 * 1024 DEFAULT_MAX_BYTES = 2 * 1024 * 1024
DEFAULT_MAX_CHARS = 6000 DEFAULT_MAX_CHARS = 8000 # URL-08: budget goes to content now, not chrome
DEFAULT_MAX_IMAGES = 2 DEFAULT_MAX_IMAGES = 2
FETCH_TIMEOUT_S = 15 FETCH_TIMEOUT_S = 15
MAX_REDIRECTS = 5 MAX_REDIRECTS = 5
@@ -41,18 +41,37 @@ FETCH_URL_TOOL = {
_META_REFRESH_URL = re.compile(r"url\s*=\s*['\"]?([^'\";\s]+)", re.I) _META_REFRESH_URL = re.compile(r"url\s*=\s*['\"]?([^'\";\s]+)", re.I)
_SKIP_TAGS = ("script", "style", "noscript", "svg", "nav", "header", "footer", "aside", "form", "select", "button")
_BLOCK_TAGS = ("p", "li", "div", "section", "article", "td", "ul", "ol", "table", "h1", "h2", "h3", "h4", "h5", "h6")
_LINK_DENSITY_MAX = 0.6 # boilerplate: block mostly link text ... (URL-08)
_LINK_BLOCK_MAX_CHARS = 200 # ... AND short (menus, related lists); long linky paragraphs survive
class _Extractor(HTMLParser): class _Extractor(HTMLParser):
def __init__(self) -> None: def __init__(self) -> None:
super().__init__() super().__init__()
self._skip = 0 self._skip = 0
self.parts: List[str] = [] self._links = 0
self._buf: List[str] = []
self._buf_link_chars = 0
self.blocks: List[Tuple[str, int]] = [] # (text, chars inside <a>)
self.images: List[str] = [] self.images: List[str] = []
self.og_image: Optional[str] = None self.og_image: Optional[str] = None
self.refresh_url: Optional[str] = None self.refresh_url: Optional[str] = None
def _flush(self) -> None:
text = " ".join(self._buf).strip()
if text:
self.blocks.append((text, self._buf_link_chars))
self._buf, self._buf_link_chars = [], 0
def handle_starttag(self, tag: str, attrs) -> None: def handle_starttag(self, tag: str, attrs) -> None:
if tag in ("script", "style", "noscript", "svg"): if tag in _SKIP_TAGS:
self._skip += 1 self._skip += 1
if tag == "a":
self._links += 1
if tag in _BLOCK_TAGS:
self._flush()
attr = dict(attrs) attr = dict(attrs)
src = attr.get("src") src = attr.get("src")
if tag == "img" and src: if tag == "img" and src:
@@ -67,12 +86,28 @@ class _Extractor(HTMLParser):
self.refresh_url = match.group(1) self.refresh_url = match.group(1)
def handle_endtag(self, tag: str) -> None: def handle_endtag(self, tag: str) -> None:
if tag in ("script", "style", "noscript", "svg") and self._skip > 0: if tag in _SKIP_TAGS and self._skip > 0:
self._skip -= 1 self._skip -= 1
if tag == "a" and self._links > 0:
self._links -= 1
if tag in _BLOCK_TAGS:
self._flush()
def handle_data(self, data: str) -> None: def handle_data(self, data: str) -> None:
if self._skip == 0 and data.strip(): if self._skip == 0 and data.strip():
self.parts.append(data.strip()) self._buf.append(data.strip())
if self._links > 0:
self._buf_link_chars += len(data.strip())
def content_parts(self) -> List[str]:
"""Blocks minus boilerplate: short blocks dominated by link text are chrome (URL-08)."""
self._flush()
out = []
for text, link_chars in self.blocks:
if link_chars / max(1, len(text)) > _LINK_DENSITY_MAX and len(text) < _LINK_BLOCK_MAX_CHARS:
continue
out.append(text)
return out
def _ip_is_public(ip_str: str) -> bool: def _ip_is_public(ip_str: str) -> bool:
@@ -162,7 +197,7 @@ class URLReader:
return extractor return extractor
def _to_text(self, html: str) -> str: def _to_text(self, html: str) -> str:
return re.sub(r"\s+\n", "\n", " ".join(self._extract(html).parts)) return re.sub(r"\s+\n", "\n", " ".join(self._extract(html).content_parts()))
async def _ingest_images(self, html: str, base_url: str, channel: str, user: str) -> int: async def _ingest_images(self, html: str, base_url: str, channel: str, user: str) -> int:
if self.image_cache is None: if self.image_cache is None:
+111
View File
@@ -0,0 +1,111 @@
"""Weather tool via MET Norway Locationforecast (SPEC-016).
A `get_weather` function tool: both personas talk about weather (the
sea over the skerries, rain on patch day) but had to guess it. The
free api.met.no compact forecast grounds it. Locations are
host-configured `[name, lat, lon]` entries the model picks by name
and never supplies coordinates or URLs, so there is no SSRF surface.
"""
import logging
from typing import Any, Callable, Dict, List, Optional, Tuple
import aiohttp
from .ai_responder import sanitize_external_text
MET_COMPACT_URL = "https://api.met.no/weatherapi/locationforecast/2.0/compact"
USER_AGENT = "fjerkroa-discord-bot/3 (https://fjerkroa.no)"
FETCH_TIMEOUT_S = 15
FORECAST_POINT_INDICES = (6, 12, 24) # hourly series: ~6h/12h/24h ahead
GET_WEATHER_TOOL = {
"name": "get_weather",
"description": "Current weather and a short forecast for the configured local places. Use this whenever weather comes "
"up in conversation — never guess or invent weather. Returns current temperature (°C), wind (m/s) and conditions, "
"plus a few forecast points.",
"parameters": {
"type": "object",
"properties": {
"location": {"type": "string", "description": "Place name to look up; omit for the default (first configured) place."},
},
"required": [],
},
}
def _reduce(data: Any, name: str) -> Dict[str, Any]:
"""Compact MET timeseries -> {location, now, forecast[]} (WEA-02). Nothing else reaches the prompt."""
series = data.get("properties", {}).get("timeseries", []) if isinstance(data, dict) else []
if not series:
return {"error": "weather data unavailable"}
def point(entry: Dict[str, Any]) -> Dict[str, Any]:
details = entry.get("data", {}).get("instant", {}).get("details", {})
hour = entry.get("data", {}).get("next_1_hours", {}) or entry.get("data", {}).get("next_6_hours", {})
out: Dict[str, Any] = {
"time": str(entry.get("time", "")),
"temp_c": details.get("air_temperature"),
"wind_ms": details.get("wind_speed"),
}
symbol = hour.get("summary", {}).get("symbol_code")
if symbol:
out["conditions"] = str(symbol)
precip = hour.get("details", {}).get("precipitation_amount")
if precip is not None:
out["precip_mm"] = precip
return out
forecast = [point(series[i]) for i in FORECAST_POINT_INDICES if i < len(series)]
return {"location": sanitize_external_text(name, 80), "now": point(series[0]), "forecast": forecast}
class Weather:
def __init__(self, config_getter: Callable[[], Dict[str, Any]]) -> None:
self._config = config_getter
def _locations(self) -> List[Tuple[str, float, float]]:
out: List[Tuple[str, float, float]] = []
for entry in self._config().get("weather-locations", []):
try:
name, lat, lon = entry[0], float(entry[1]), float(entry[2])
out.append((str(name), lat, lon))
except (TypeError, ValueError, IndexError):
logging.warning(f"weather: bad location entry {entry!r}")
return out
def enabled(self) -> bool:
return bool(self._config().get("enable-weather", False)) and bool(self._locations())
def _pick(self, location: Optional[str]) -> Optional[Tuple[str, float, float]]:
"""Case-insensitive substring match; unknown/absent = first configured (WEA-03)."""
entries = self._locations()
if not entries:
return None
wanted = (location or "").strip().casefold()
if wanted:
for entry in entries:
if wanted in entry[0].casefold():
return entry
return entries[0]
async def _fetch_json(self, lat: float, lon: float) -> Any:
timeout = aiohttp.ClientTimeout(total=FETCH_TIMEOUT_S)
params = {"lat": f"{lat:.4f}", "lon": f"{lon:.4f}"}
async with aiohttp.ClientSession(timeout=timeout, headers={"User-Agent": USER_AGENT}) as session:
async with session.get(MET_COMPACT_URL, params=params) as response:
response.raise_for_status()
return await response.json()
async def forecast(self, location: Optional[str] = None) -> Dict[str, Any]:
"""Return a compact forecast, or an error dict — never raise (WEA-04)."""
picked = self._pick(location)
if picked is None:
return {"error": "weather unavailable: no locations configured"}
name, lat, lon = picked
try:
data = await self._fetch_json(lat, lon)
except Exception as err:
logging.warning(f"weather fetch failed: {err!r}")
return {"error": "weather lookup failed"}
return _reduce(data, name)
+21
View File
@@ -61,3 +61,24 @@ Within `quiet-hours = "HH:MM-HH:MM"` (host-local, may wrap midnight)
`bot_initiated_allowed()` is false: no boreness, later no scheduler `bot_initiated_allowed()` is false: no boreness, later no scheduler
posts. Replies to users stay unaffected — a guest asking at 23:30 posts. Replies to users stay unaffected — a guest asking at 23:30
still gets an answer. still gets an answer.
### BEH-09 — Ignored channels are fully silent (coverage: test)
Channels matching `ignore-channels` get neither replies nor
classifier emoji reactions: the message handler returns before the
classifier gate, so no model call, no reaction, no history entry.
Entries are fnmatch patterns (`todo*` matches `todo`, `todo-lists`);
plain names keep matching exactly as before. DMs are never ignored.
`channel_by_name` resolution honors the same patterns. (Previously
the ignore check sat only in `respond()`, after the classifier —
emoji reactions leaked into ignored channels, and matching was
exact-name only.)
### BEH-10 — Factual questions may use a stronger model (coverage: test)
With `factual-model` configured, a message the classifier tagged
`factual` (BEH-05) is answered by that model instead of `model`
opening hours, release dates, news lookups get the stronger tier
while small talk stays on the cheap default. Unset = no change. The
`retry-model` override still wins on retry, and vision inputs keep
using `model-vision`.
+11
View File
@@ -58,3 +58,14 @@ Each fetch increments a per-user daily counter; over
`url-daily-per-user` (default 20) `fetch_url` refuses with an error `url-daily-per-user` (default 20) `fetch_url` refuses with an error
result. The budget gate (SAF-04) still applies to the surrounding result. The budget gate (SAF-04) still applies to the surrounding
model calls. model calls.
### URL-08 — Main-content extraction (coverage: test)
`fetch_url` text drops page chrome: content inside
`nav`/`header`/`footer`/`aside`/`form`/`select`/`button` is skipped
like scripts, and text blocks dominated by link text (over 60 % of a
block's characters inside `<a>` and the block shorter than 200 chars
— menus, related-article lists, tag clouds) are treated as
boilerplate and removed. Body paragraphs with inline links survive.
The default `url-max-chars` cap rises to 8000 now that the budget is
spent on content, not chrome.
+11
View File
@@ -98,3 +98,14 @@ Each `get_news` call increments a per-user daily counter; over
`news-daily-per-user` (default 30) the tool refuses with an error `news-daily-per-user` (default 30) the tool refuses with an error
result without touching the store. The budget gate (SAF-04) still result without touching the store. The budget gate (SAF-04) still
applies to the surrounding model calls. applies to the surrounding model calls.
### NEWS-13 — Topic misses degrade softly, never empty-handed (coverage: test)
A `topic` whose AND-match (NEWS-11) finds nothing falls back to an
any-term match, ranked by how many keywords hit (ties: newest first);
if that too is empty, the newest stored items are returned instead.
Both fallbacks set a `note` field naming the degradation so the model
can answer honestly ("nothing on that exactly, but…"). A model
passing a multi-word or wrong-language topic (the live
`"Nordland road accident"``[]` case) thus still gets usable
context. Exact matches return no `note`.
+35
View File
@@ -0,0 +1,35 @@
# SPEC-016 — Weather tool (get_weather)
Both personas talk about weather (the sea over the skerries, rain on
patch day) but had to guess it. `get_weather` grounds that in the
free MET Norway Locationforecast API (api.met.no, User-Agent
required, no key). Locations are host-configured coordinates — the
model picks by name, it never supplies raw URLs, so there is no SSRF
surface (one fixed API host).
### WEA-01 — Tool offered only when configured (coverage: test)
The chat call's tools include `get_weather` only when
`enable-weather` is true AND `weather-locations` (a list of
`[name, lat, lon]` entries) is non-empty. Otherwise it is absent.
### WEA-02 — Compact sanitized forecast (coverage: test)
The tool reduces the MET compact timeseries to: the named location,
current conditions (temperature °C, wind m/s, symbol), and a small
set of forecast points (next hours / tomorrow) with temperature,
symbol and precipitation. Location names pass
`sanitize_external_text`; numbers are numbers. Nothing else from the
API response reaches the prompt.
### WEA-03 — Location matched by name, defaults to first (coverage: test)
The `location` argument matches configured entries
case-insensitively by substring; no or unknown location = the first
configured entry. Coordinates never come from the model.
### WEA-04 — Errors return, never raise; calls are metered (coverage: test)
API/network failures return an `{error}` dict (the responder keeps
running). Each call counts against a per-user daily cap
(`weather-daily-per-user`, default 30) like the other tools.
+82
View File
@@ -65,6 +65,88 @@ class TestClassifierGate(ClassifierGateBase):
self.bot.respond.assert_not_awaited() self.bot.respond.assert_not_awaited()
class TestIgnoredChannels(ClassifierGateBase):
def ignored_msg(self, channel_name):
message = self.public_msg("hello there")
message.channel.name = channel_name
message.add_reaction = AsyncMock()
return message
async def test_pattern_match_suppresses_reaction_and_reply(self):
"""BEH-09: fnmatch pattern hit -> no classifier call, no emoji, no reply."""
self.gate_setup({"reply": False, "factual": False, "emoji": "👍"})
self.bot.config["ignore-channels"] = ["todo*"]
message = self.ignored_msg("todo-lists")
await self.bot.on_message(message)
self.bot.airesponder.classify.assert_not_awaited()
message.add_reaction.assert_not_awaited()
self.bot.respond.assert_not_awaited()
async def test_exact_name_still_matches(self):
"""BEH-09: plain names keep working as exact matches."""
self.gate_setup({"reply": True, "factual": False, "emoji": None})
self.bot.config["ignore-channels"] = ["blengon"]
await self.bot.on_message(self.ignored_msg("blengon"))
self.bot.respond.assert_not_awaited()
async def test_non_matching_channel_passes(self):
"""BEH-09: unmatched channels reach the responder as before."""
self.gate_setup({"reply": True, "factual": False, "emoji": None})
self.bot.config["ignore-channels"] = ["todo*"]
await self.bot.on_message(self.ignored_msg("chat"))
self.bot.respond.assert_awaited_once()
async def test_dm_never_ignored(self):
"""BEH-09: a DM whose recipient name matches a pattern is still answered."""
self.gate_setup({"reply": True, "factual": False, "emoji": None})
self.bot.config["ignore-channels"] = ["todo*"]
message = self.public_msg("hei bot")
message.channel = MagicMock(spec=DMChannel)
message.channel.recipient = MagicMock()
message.channel.recipient.name = "todo-fan"
await self.bot.on_message(message)
self.bot.respond.assert_awaited_once()
def test_channel_by_name_honors_patterns(self):
"""BEH-09: channel_by_name resolution skips pattern-ignored channels."""
self.bot.config["ignore-channels"] = ["todo*"]
fallback = MagicMock(spec=TextChannel)
self.assertIs(self.bot.channel_by_name("todo-lists", fallback), fallback)
class TestFactualModel(unittest.IsolatedAsyncioTestCase):
async def _model_used(self, config, factual):
from .test_spec_structured import ok_result
responder = OpenAIResponder(dict({"openai-token": "t", "model": "cheap", "system": "s", "history-limit": 5}, **config), "chat")
responder._factual = factual
with patch("fjerkroa_bot.openai_responder.openai_chat", new_callable=AsyncMock) as chat_mock:
chat_mock.return_value = ok_result()
await responder.chat([{"role": "user", "content": "hi"}], 10)
return chat_mock.await_args.kwargs["model"]
async def test_factual_uses_stronger_model(self):
"""BEH-10: factual verdict + factual-model config -> stronger tier."""
self.assertEqual(await self._model_used({"factual-model": "strong"}, True), "strong")
async def test_factual_without_config_stays_default(self):
"""BEH-10: no factual-model config -> default model, no behavior change."""
self.assertEqual(await self._model_used({}, True), "cheap")
async def test_small_talk_stays_default(self):
"""BEH-10: non-factual messages stay on the cheap default."""
self.assertEqual(await self._model_used({"factual-model": "strong"}, False), "cheap")
async def test_send_reads_flag_from_message(self):
"""BEH-10: send() picks the factual flag off the AIMessage."""
responder = FakeModelResponder({"system": "s", "history-limit": 5}, "chat")
responder.scripted = [envelope(answer="x", answer_needed=True)]
message = AIMessage("alice", "opening hours?", "chat")
message.factual = True
await responder.send(message)
self.assertTrue(responder._factual)
class TestTypingPacing(OpsBase): class TestTypingPacing(OpsBase):
async def send_with(self, answer, factual, cps=30): async def send_with(self, answer, factual, cps=30):
if cps is not None: if cps is not None:
+20
View File
@@ -303,6 +303,26 @@ class TestQueryNews(NewsStoreBase):
self.assertGreaterEqual(len(query_news(self.store, limit=0)["results"]), 1) self.assertGreaterEqual(len(query_news(self.store, limit=0)["results"]), 1)
self.assertIn("error", query_news(None)) self.assertIn("error", query_news(None))
def test_exact_match_has_no_note(self):
"""NEWS-13: a direct AND-match returns without a note field."""
self.seed()
self.assertNotIn("note", query_news(self.store, topic="storm"))
def test_partial_match_falls_back_ranked(self):
"""NEWS-13: AND-miss -> any-term match, most keyword hits first, with a note."""
self.seed()
res = query_news(self.store, topic="Nordland road accident")
self.assertEqual(res["results"][0]["title"], "Nordland storm")
self.assertIn("note", res)
def test_no_match_falls_back_to_recent(self):
"""NEWS-13: nothing matches any term -> newest items + note, never empty-handed."""
self.seed()
res = query_news(self.store, topic="quantum blockchain")
self.assertTrue(res["results"])
self.assertEqual(res["results"][0]["title"], "Sport result") # newest first
self.assertIn("note", res)
class TestNewsTool(unittest.IsolatedAsyncioTestCase): class TestNewsTool(unittest.IsolatedAsyncioTestCase):
def setUp(self): def setUp(self):
+25
View File
@@ -149,6 +149,31 @@ class TestTextExtraction(unittest.TestCase):
self.assertNotIn("evil", text) self.assertNotIn("evil", text)
self.assertNotIn("x{}", text) self.assertNotIn("x{}", text)
def test_chrome_and_link_boilerplate_dropped(self):
"""URL-08: nav/header/footer skipped; short link-dominated blocks (menus, related lists) removed."""
reader = URLReader(lambda: {}, None)
html = (
"<html><body>"
"<nav><a href='/a'>Home</a> <a href='/b'>Games</a></nav>"
"<header><a href='/login'>Login</a></header>"
"<ul><li><a href='/1'>Related article one</a></li><li><a href='/2'>Related article two</a></li></ul>"
"<article><p>The pop-up event runs from August 4 in Shibuya, with details "
"<a href='/x'>on the official page</a> for anyone attending the exhibition.</p></article>"
"<footer><a href='/imprint'>Imprint</a></footer>"
"</body></html>"
)
text = reader._to_text(html)
self.assertIn("pop-up event", text)
self.assertIn("on the official page", text) # inline link in a real paragraph survives
for chrome in ("Home", "Login", "Related article one", "Imprint"):
self.assertNotIn(chrome, text)
def test_default_cap_is_8000(self):
"""URL-08: the default url-max-chars budget is 8000."""
from fjerkroa_bot.url_reader import DEFAULT_MAX_CHARS
self.assertEqual(DEFAULT_MAX_CHARS, 8000)
class TestBodyReadCollectsAllChunks(unittest.IsolatedAsyncioTestCase): class TestBodyReadCollectsAllChunks(unittest.IsolatedAsyncioTestCase):
async def test_get_reads_past_first_chunk(self): async def test_get_reads_past_first_chunk(self):
+120
View File
@@ -0,0 +1,120 @@
"""Unit coverage for SPEC-016 weather tool (WEA-01..04)."""
import unittest
from unittest.mock import AsyncMock, patch
from fjerkroa_bot.openai_responder import OpenAIResponder
from fjerkroa_bot.weather import GET_WEATHER_TOOL, Weather, _reduce
CONFIG = {"openai-token": "t", "model": "m", "system": "s", "history-limit": 5}
LOCATIONS = [["Sleneset", 66.58, 12.68], ["Berlin", 52.52, 13.41]]
MET_DATA = {
"properties": {
"timeseries": [
{
"time": f"2026-07-17T{10 + i if 10 + i < 24 else 10 + i - 24:02d}:00:00Z",
"data": {
"instant": {"details": {"air_temperature": 14.0 + i, "wind_speed": 5.0}},
"next_1_hours": {"summary": {"symbol_code": "lightrain"}, "details": {"precipitation_amount": 0.3}},
},
}
for i in range(30)
]
}
}
def _tool_names(responder):
return [f["name"] for f in responder._available_tools()]
class TestToolOffered(unittest.TestCase):
def test_gate_needs_flag_and_locations(self):
"""WEA-01: get_weather offered only with enable-weather AND locations."""
self.assertNotIn("get_weather", _tool_names(OpenAIResponder(CONFIG, "chat")))
flag_only = OpenAIResponder(dict(CONFIG, **{"enable-weather": True}), "chat")
self.assertNotIn("get_weather", _tool_names(flag_only))
on = OpenAIResponder(dict(CONFIG, **{"enable-weather": True, "weather-locations": LOCATIONS}), "chat")
self.assertIn("get_weather", _tool_names(on))
self.assertEqual(GET_WEATHER_TOOL["name"], "get_weather")
class TestReduce(unittest.TestCase):
def test_compact_shape(self):
"""WEA-02: now + few forecast points; temperature/wind/conditions/precip only."""
out = _reduce(MET_DATA, "Sleneset")
self.assertEqual(out["location"], "Sleneset")
self.assertEqual(out["now"]["temp_c"], 14.0)
self.assertEqual(out["now"]["wind_ms"], 5.0)
self.assertEqual(out["now"]["conditions"], "lightrain")
self.assertEqual(out["now"]["precip_mm"], 0.3)
self.assertEqual(len(out["forecast"]), 3) # +6h, +12h, +24h
self.assertEqual(out["forecast"][0]["temp_c"], 20.0)
self.assertNotIn("error", out)
def test_location_name_sanitized_and_empty_series(self):
"""WEA-02: name passes sanitizer; empty timeseries -> error dict."""
out = _reduce(MET_DATA, "@everyone town")
self.assertNotIn("@everyone", out["location"])
self.assertIn("error", _reduce({"properties": {"timeseries": []}}, "x"))
class TestLocationPick(unittest.TestCase):
def setUp(self):
self.weather = Weather(lambda: {"enable-weather": True, "weather-locations": LOCATIONS})
def test_substring_case_insensitive(self):
"""WEA-03: case-insensitive substring match."""
self.assertEqual(self.weather._pick("berlin")[0], "Berlin")
self.assertEqual(self.weather._pick("slen")[0], "Sleneset")
def test_unknown_or_absent_defaults_to_first(self):
"""WEA-03: unknown/absent location -> first configured entry."""
self.assertEqual(self.weather._pick(None)[0], "Sleneset")
self.assertEqual(self.weather._pick("Atlantis")[0], "Sleneset")
def test_bad_entries_skipped(self):
"""WEA-03: malformed location entries are ignored, not fatal."""
weather = Weather(lambda: {"enable-weather": True, "weather-locations": [["broken"], ["OK", 1.0, 2.0]]})
self.assertEqual(weather._pick(None)[0], "OK")
class TestForecast(unittest.IsolatedAsyncioTestCase):
async def test_error_returned_not_raised(self):
"""WEA-04: network failure -> {error}, never an exception."""
weather = Weather(lambda: {"enable-weather": True, "weather-locations": LOCATIONS})
with patch.object(Weather, "_fetch_json", new_callable=AsyncMock, side_effect=RuntimeError("boom")):
out = await weather.forecast("Berlin")
self.assertIn("error", out)
async def test_forecast_happy_path(self):
"""WEA-02/03: full flow with mocked API."""
weather = Weather(lambda: {"enable-weather": True, "weather-locations": LOCATIONS})
with patch.object(Weather, "_fetch_json", new_callable=AsyncMock, return_value=MET_DATA):
out = await weather.forecast("berlin")
self.assertEqual(out["location"], "Berlin")
self.assertEqual(out["now"]["temp_c"], 14.0)
class TestMetering(unittest.IsolatedAsyncioTestCase):
async def test_daily_cap(self):
"""WEA-04: per-user daily cap refuses beyond weather-daily-per-user."""
import tempfile
with tempfile.TemporaryDirectory() as tmp:
config = dict(
CONFIG,
**{
"enable-weather": True,
"weather-locations": LOCATIONS,
"weather-daily-per-user": 1,
"history-directory": tmp,
},
)
responder = OpenAIResponder(config, "chat")
with patch.object(Weather, "_fetch_json", new_callable=AsyncMock, return_value=MET_DATA):
first = await responder._dispatch_tool("get_weather", {}, "alice")
second = await responder._dispatch_tool("get_weather", {}, "alice")
self.assertNotIn("error", first)
self.assertIn("error", second)