human behavior: classifier gate, pacing, splitting, quiet hours, stable prompt prefix
This commit is contained in:
@@ -54,6 +54,14 @@ Decisions inside the set architecture. D-NNN, never renumbered.
|
||||
fact-level erasure ships with FDB-007 structured memory — stated in
|
||||
the user-facing confirmation, not hidden. (Superseded by D-014:
|
||||
erasure now covers facts, observations and episode traces.)
|
||||
- **D-016** — The reply/ignore classifier fails open (BEH-03): a
|
||||
broken classifier must never mute the bot; the budget gate already
|
||||
bounds spend. Its verdict gates BEFORE the main call, the
|
||||
envelope's answer_needed still gates after — two independent nets.
|
||||
- **D-017** — All human-behavior knobs default to off/v3.0.0
|
||||
semantics; behavior changes are config rollouts per deployment, not
|
||||
code flips. The classifier's `factual` flag is the only coupling
|
||||
(delay bypass) and defaults to false without a classifier.
|
||||
- **D-015** — Deploys are push-based from the dev machine
|
||||
(`git archive <tag> | ssh`), not pull-based: no deploy keys or git
|
||||
state on the hosts, the artifact is exactly the tag tree, untracked
|
||||
|
||||
@@ -42,3 +42,11 @@ enable-game-info = true
|
||||
# memory-episodes-per-channel = 10 # episode decay cap
|
||||
# memory-fact-retention-days = 180 # GDPR storage limitation
|
||||
# Staff: !bot memory <user> | forget-fact <id> | pin <channel|global> <fact> | unpin <id>
|
||||
|
||||
# Human behavior (SPEC-010) — every knob unset = old behavior:
|
||||
# classifier-model = "gpt-5.6-luna" # reply/ignore + factual pre-pass (~100 tok)
|
||||
# typing-chars-per-second = 30 # reply pacing; factual answers skip it
|
||||
# typing-max-seconds = 8
|
||||
# split-threshold = 1200 # long answers split at paragraphs
|
||||
# split-max-parts = 3
|
||||
# quiet-hours = "21:00-09:00" # no bot-initiated posts in this window
|
||||
|
||||
@@ -154,17 +154,25 @@ class AIResponder(AIResponderBase):
|
||||
self.memory_manager = MemoryManager(self.store, lambda: self.config, self.consolidate, self.channel)
|
||||
logging.info(f"memmory:\n{self.memory}")
|
||||
|
||||
# Dynamic values move to a context suffix so the persona prefix
|
||||
# stays byte-stable for the prompt cache (ENV-20)
|
||||
DYNAMIC_PLACEHOLDERS = ("{date}", "{time}", "{news}", "{memory}")
|
||||
|
||||
def message(self, message: AIMessage, limit: Optional[int] = None) -> List[Dict[str, Any]]:
|
||||
messages = []
|
||||
system = self.config.get(self.channel, self.config["system"])
|
||||
system = system.replace("{date}", time.strftime("%Y-%m-%d")).replace("{time}", time.strftime("%H:%M:%S"))
|
||||
persona = self.config.get(self.channel, self.config["system"])
|
||||
for placeholder in self.DYNAMIC_PLACEHOLDERS:
|
||||
persona = persona.replace(placeholder, "")
|
||||
context = [f"date: {time.strftime('%Y-%m-%d')} ({time.strftime('%A')})", f"time: {time.strftime('%H:%M:%S')}"]
|
||||
news_feed = self.config.get("news")
|
||||
if news_feed and os.path.exists(news_feed):
|
||||
with open(news_feed) as fd:
|
||||
news_feed = fd.read().strip()
|
||||
system = system.replace("{news}", sanitize_external_text(news_feed))
|
||||
context.append("news:\n" + sanitize_external_text(fd.read().strip()))
|
||||
participants = [message.user] + [entry_user for entry_user in self._history_users(20)]
|
||||
system = system.replace("{memory}", self.memory_manager.memory_block(participants, self.memory))
|
||||
memory_block = self.memory_manager.memory_block(participants, self.memory)
|
||||
if memory_block:
|
||||
context.append("memory:\n" + memory_block)
|
||||
system = persona.rstrip() + "\n\n## Context\n" + "\n".join(context)
|
||||
messages.append({"role": "system", "content": system})
|
||||
if limit is not None:
|
||||
while len(self.history) > limit:
|
||||
@@ -238,6 +246,10 @@ class AIResponder(AIResponderBase):
|
||||
async def consolidate(self, observations: List[Dict[str, Any]], known_facts: List[Dict[str, Any]]) -> Optional[Dict[str, Any]]:
|
||||
raise NotImplementedError()
|
||||
|
||||
async def classify(self, message: AIMessage, history_tail: List[Dict[str, Any]]) -> Optional[Dict[str, Any]]:
|
||||
"""Cheap reply/factual/emoji pre-pass (BEH-01); None = fail open."""
|
||||
raise NotImplementedError()
|
||||
|
||||
async def translate(self, text: str, language: str = "english") -> str:
|
||||
raise NotImplementedError()
|
||||
|
||||
|
||||
+99
-20
@@ -24,6 +24,45 @@ DEFAULT_PRIVACY_NOTICE = (
|
||||
"Type !forgetme to remove your messages from my history. Questions: ask the staff."
|
||||
)
|
||||
|
||||
DISCORD_HARD_LIMIT = 1900 # margin under the 2000-char API limit
|
||||
|
||||
|
||||
def quiet_hours_active(spec: Optional[str], now_hhmm: str) -> bool:
|
||||
"""BEH-08: 'HH:MM-HH:MM' window, may wrap midnight; garbage = inactive."""
|
||||
if not spec or "-" not in str(spec):
|
||||
return False
|
||||
start, _, end = str(spec).partition("-")
|
||||
start, end = start.strip(), end.strip()
|
||||
if not (len(start) == 5 and len(end) == 5 and start[2] == ":" and end[2] == ":"):
|
||||
return False
|
||||
if start <= end:
|
||||
return start <= now_hhmm < end
|
||||
return now_hhmm >= start or now_hhmm < end
|
||||
|
||||
|
||||
def split_answer(text: str, threshold: int, max_parts: int) -> list:
|
||||
"""BEH-06: split at paragraph boundaries, hard-cap under the Discord limit."""
|
||||
if text is None:
|
||||
return [""]
|
||||
parts = [text]
|
||||
if len(text) > max(threshold, 1) and max_parts > 1:
|
||||
parts = []
|
||||
for paragraph in text.split("\n\n"):
|
||||
if parts and len(parts[-1]) + len(paragraph) + 2 <= threshold:
|
||||
parts[-1] = parts[-1] + "\n\n" + paragraph
|
||||
else:
|
||||
parts.append(paragraph)
|
||||
while len(parts) > max_parts:
|
||||
tail = parts.pop()
|
||||
parts[-1] = parts[-1] + "\n\n" + tail
|
||||
hard: list = []
|
||||
for part in parts:
|
||||
while len(part) > DISCORD_HARD_LIMIT:
|
||||
hard.append(part[:DISCORD_HARD_LIMIT])
|
||||
part = part[DISCORD_HARD_LIMIT:]
|
||||
hard.append(part)
|
||||
return hard
|
||||
|
||||
|
||||
class ConfigFileHandler(FileSystemEventHandler):
|
||||
def __init__(self, on_modified):
|
||||
@@ -172,12 +211,14 @@ class FjerkroaBot(commands.Bot):
|
||||
return self.replies_enabled and time.monotonic() >= self.quiet_until
|
||||
|
||||
def bot_initiated_allowed(self) -> bool:
|
||||
# Gate for boreness today, the FDB-011 scheduler later (OPS-09)
|
||||
# Gate for boreness today, the FDB-011 scheduler later (OPS-09, BEH-08)
|
||||
if quiet_hours_active(self.config.get("quiet-hours"), time.strftime("%H:%M")):
|
||||
return False
|
||||
return self.tasks_enabled and self.replies_allowed()
|
||||
|
||||
def _memory_command(self, args) -> Optional[str]:
|
||||
"""Staff memory review/edit (MEM-07)."""
|
||||
if args[:1] not in (["memory"], ["forget-fact"], ["pin"], ["unpin"]):
|
||||
if args[:1] not in (["memory"], ["forget-fact"], ["pin"], ["unpin"], ["pins"]):
|
||||
return None
|
||||
store = self.airesponder.store
|
||||
if store is None:
|
||||
@@ -193,6 +234,9 @@ class FjerkroaBot(commands.Bot):
|
||||
return f"Pinned for {args[1]}."
|
||||
if args[:1] == ["unpin"] and args[1:2] and args[1].isdigit():
|
||||
return f"Removed {store.delete_pinned(int(args[1]))} pin(s)."
|
||||
if args[:1] == ["pins"]:
|
||||
pins = store.pinned_all()
|
||||
return "\n".join(f"{pin['id']} [{pin['channel'] or 'global'}]: {pin['fact']}" for pin in pins) or "No pins."
|
||||
return None
|
||||
|
||||
async def handle_staff_command(self, message: Message) -> None:
|
||||
@@ -361,14 +405,7 @@ class FjerkroaBot(commands.Bot):
|
||||
message_content = f"> {reference_content}\n\n{message_content}"
|
||||
if len(message_content) < 1:
|
||||
return
|
||||
for ma_user in self._re_user.finditer(message_content):
|
||||
uid = int(ma_user.group(1))
|
||||
for guild in self.guilds:
|
||||
user = guild.get_member(uid)
|
||||
if user is not None:
|
||||
break
|
||||
if user is not None:
|
||||
message_content = re.sub(f"[<][@][!]? *{uid} *[>]", f"@{user.name}", message_content)
|
||||
message_content = self._resolve_mentions(message_content)
|
||||
channel_name = self.get_channel_name(message.channel)
|
||||
msg = AIMessage(
|
||||
message.author.name, message_content, channel_name, self.user in message.mentions or isinstance(message.channel, DMChannel)
|
||||
@@ -378,22 +415,63 @@ class FjerkroaBot(commands.Bot):
|
||||
if not msg.urls:
|
||||
msg.urls = []
|
||||
msg.urls.append(attachment.url)
|
||||
await self.respond(msg, message.channel)
|
||||
|
||||
# Reply/ignore classifier gate — direct messages bypass (BEH-01/02/03/07)
|
||||
airesponder = self.get_ai_responder(channel_name)
|
||||
handled, factual = await self._classifier_gate(message, msg, airesponder, channel_name)
|
||||
if handled:
|
||||
return
|
||||
await self.respond(msg, message.channel, factual=factual)
|
||||
|
||||
def _resolve_mentions(self, message_content: str) -> str:
|
||||
for ma_user in self._re_user.finditer(message_content):
|
||||
uid = int(ma_user.group(1))
|
||||
user = None
|
||||
for guild in self.guilds:
|
||||
user = guild.get_member(uid)
|
||||
if user is not None:
|
||||
break
|
||||
if user is not None:
|
||||
message_content = re.sub(f"[<][@][!]? *{uid} *[>]", f"@{user.name}", message_content)
|
||||
return message_content
|
||||
|
||||
async def _classifier_gate(self, message, msg: AIMessage, airesponder, channel_name: str):
|
||||
"""(handled, factual): handled=True = reply suppressed, maybe emoji (BEH-01/07)."""
|
||||
if "classifier-model" not in self.config or msg.direct:
|
||||
return False, False
|
||||
verdict = await airesponder.classify(msg, airesponder.history[-6:])
|
||||
if verdict is None:
|
||||
return False, False # fail open (BEH-03)
|
||||
if not verdict.get("reply", True):
|
||||
emoji = verdict.get("emoji")
|
||||
if emoji:
|
||||
try:
|
||||
await message.add_reaction(emoji)
|
||||
except Exception as err:
|
||||
logging.debug(f"reaction failed: {repr(err)}")
|
||||
self.log_message_action("classifier-skip", msg, channel_name)
|
||||
return True, False
|
||||
return False, bool(verdict.get("factual", False))
|
||||
|
||||
async def send_message_with_typing(self, airesponder, channel, message):
|
||||
"""Send the user message to the AI responder with typing animation in discord"""
|
||||
async with channel.typing():
|
||||
return await airesponder.send(message)
|
||||
|
||||
async def send_answer_with_typing(self, response, answer_channel, airesponder):
|
||||
"""Send an answer from AI to discord channel with typing animation"""
|
||||
async with answer_channel.typing():
|
||||
async def send_answer_with_typing(self, response, answer_channel, airesponder, factual: bool = False):
|
||||
"""Send the answer paced, split and with images on the last part (BEH-04/05/06)"""
|
||||
files = None
|
||||
if response.picture is not None:
|
||||
# Generate the image with the AI and send it with the answer
|
||||
images = [discord.File(fp=await airesponder.draw(response.picture), filename="image.png")]
|
||||
await answer_channel.send(response.answer, files=images, suppress_embeds=True)
|
||||
else:
|
||||
await answer_channel.send(response.answer, suppress_embeds=True)
|
||||
files = [discord.File(fp=await airesponder.draw(response.picture), filename="image.png")]
|
||||
parts = split_answer(response.answer, int(self.config.get("split-threshold", 1200)), int(self.config.get("split-max-parts", 3)))
|
||||
pace = float(self.config.get("typing-chars-per-second", 0) or 0)
|
||||
max_delay = float(self.config.get("typing-max-seconds", 8))
|
||||
for index, part in enumerate(parts):
|
||||
async with answer_channel.typing():
|
||||
if pace > 0 and not factual:
|
||||
await asyncio.sleep(min(len(part) / pace, max_delay))
|
||||
last = index == len(parts) - 1
|
||||
await answer_channel.send(part, files=files if last else None, suppress_embeds=True)
|
||||
self.last_activity_time = time.monotonic()
|
||||
|
||||
def _keyword_alert(self, message: AIMessage) -> Optional[str]:
|
||||
@@ -438,6 +516,7 @@ class FjerkroaBot(commands.Bot):
|
||||
self,
|
||||
message: AIMessage, # Incoming message object with user message and metadata
|
||||
channel: Union[TextChannel, DMChannel], # Channel (Text or Direct Message) the message is coming from
|
||||
factual: bool = False, # classifier verdict: skip the artificial typing delay (BEH-05)
|
||||
) -> None:
|
||||
"""Handle a message from a user with an AI responder"""
|
||||
|
||||
@@ -480,7 +559,7 @@ class FjerkroaBot(commands.Bot):
|
||||
return
|
||||
|
||||
# Send the AI's answer to the specified answer channel, with typing indicators
|
||||
await self.send_answer_with_typing(response, answer_channel, airesponder)
|
||||
await self.send_answer_with_typing(response, answer_channel, airesponder, factual=factual)
|
||||
|
||||
async def close(self):
|
||||
self.observer.stop()
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
import asyncio
|
||||
import hashlib
|
||||
import json
|
||||
import logging
|
||||
from io import BytesIO
|
||||
@@ -56,6 +57,29 @@ CONSOLIDATION_RESPONSE_FORMAT = {
|
||||
"type": "json_schema",
|
||||
"json_schema": {"name": "consolidation", "strict": True, "schema": CONSOLIDATION_SCHEMA},
|
||||
}
|
||||
# Reply/ignore + factual pre-pass (SPEC-010 BEH-01): one cheap call
|
||||
CLASSIFIER_SCHEMA = {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"reply": {"type": "boolean", "description": "Should the assistant answer this message?"},
|
||||
"factual": {"type": "boolean", "description": "Does the user want concrete information (hours, prices, availability)?"},
|
||||
"emoji": {"type": ["string", "null"], "description": "Optional single emoji reaction when not replying, else null."},
|
||||
},
|
||||
"required": ["reply", "factual", "emoji"],
|
||||
"additionalProperties": False,
|
||||
}
|
||||
CLASSIFIER_RESPONSE_FORMAT = {
|
||||
"type": "json_schema",
|
||||
"json_schema": {"name": "reply_verdict", "strict": True, "schema": CLASSIFIER_SCHEMA},
|
||||
}
|
||||
CLASSIFIER_SYSTEM = (
|
||||
"You watch a group chat that has an assistant bot. Decide whether the assistant should answer the LAST message:"
|
||||
" reply=true when it addresses the assistant, asks something the assistant can help with, or continues a conversation"
|
||||
" with the assistant; reply=false for human-to-human chatter the assistant should not butt into."
|
||||
" factual=true when the user wants concrete information (opening hours, prices, availability, addresses)."
|
||||
" When reply=false you may suggest one fitting emoji reaction, else null."
|
||||
)
|
||||
|
||||
CONSOLIDATION_SYSTEM = (
|
||||
"You maintain the long-term memory of a Discord assistant. From the observation log, extract NEW durable facts that users stated"
|
||||
" about THEMSELVES only (never record what one user claims about another user), and write one short episode summary of the"
|
||||
@@ -115,6 +139,16 @@ class OpenAIResponder(AIResponder, LeonardoAIDrawMixIn):
|
||||
logging.warning(f"Failed to generate image {repr(description)}: {repr(err)}")
|
||||
raise RuntimeError(f"Failed to generate image {repr(description)} after multiple retries")
|
||||
|
||||
@staticmethod
|
||||
def _last_author(messages: List[Dict[str, Any]]) -> Optional[str]:
|
||||
try:
|
||||
content = messages[-1]["content"]
|
||||
if not isinstance(content, str):
|
||||
content = content[0]["text"]
|
||||
return str(json.loads(content).get("user")) or None
|
||||
except Exception:
|
||||
return None
|
||||
|
||||
def _record_usage(self, result: Any) -> None:
|
||||
usage = getattr(result, "usage", None)
|
||||
prompt_tokens = getattr(usage, "prompt_tokens", None)
|
||||
@@ -164,6 +198,10 @@ class OpenAIResponder(AIResponder, LeonardoAIDrawMixIn):
|
||||
"messages": messages,
|
||||
"response_format": ENVELOPE_RESPONSE_FORMAT,
|
||||
}
|
||||
author = self._last_author(messages)
|
||||
if author:
|
||||
# hashed, never the raw Discord name (SAF-10)
|
||||
chat_kwargs["safety_identifier"] = "discord-" + hashlib.sha256(author.encode()).hexdigest()[:16]
|
||||
|
||||
if self.igdb and self.config.get("enable-game-info", False):
|
||||
try:
|
||||
@@ -326,6 +364,25 @@ class OpenAIResponder(AIResponder, LeonardoAIDrawMixIn):
|
||||
logging.warning(f"failed to translate the text: {repr(err)}")
|
||||
return text
|
||||
|
||||
async def classify(self, message: Any, history_tail: List[Dict[str, Any]]) -> Optional[Dict[str, Any]]:
|
||||
"""~100-token reply/factual/emoji verdict on classifier-model (BEH-01/03)."""
|
||||
if "classifier-model" not in self.config or not self.ledger.budget_ok():
|
||||
return None
|
||||
tail = "\n".join(str(entry.get("content", ""))[:300] for entry in history_tail[-6:])
|
||||
messages = [
|
||||
{"role": "system", "content": CLASSIFIER_SYSTEM},
|
||||
{"role": "user", "content": f"Recent chat:\n{tail}\n\nLAST message:\n{str(message)}"},
|
||||
]
|
||||
try:
|
||||
result = await openai_chat(
|
||||
self.client, model=self.config["classifier-model"], messages=messages, response_format=CLASSIFIER_RESPONSE_FORMAT
|
||||
)
|
||||
self._record_usage(result)
|
||||
return json.loads(result.choices[0].message.content)
|
||||
except Exception as err:
|
||||
logging.warning(f"classifier failed - failing open: {repr(err)}")
|
||||
return None
|
||||
|
||||
async def consolidate(self, observations: List[Dict[str, Any]], known_facts: List[Dict[str, Any]]) -> Optional[Dict[str, Any]]:
|
||||
"""Batched memory consolidation on memory-model (MEM-02)."""
|
||||
if "memory-model" not in self.config or not self.ledger.budget_ok():
|
||||
|
||||
@@ -163,6 +163,11 @@ class PersistentStore:
|
||||
).fetchall()
|
||||
return [{"id": row[0], "channel": row[1], "fact": row[2]} for row in rows]
|
||||
|
||||
def pinned_all(self) -> List[Dict[str, Any]]:
|
||||
with closing(self._connect()) as conn:
|
||||
rows = conn.execute("SELECT id, channel, fact FROM pinned_facts ORDER BY id").fetchall()
|
||||
return [{"id": row[0], "channel": row[1], "fact": row[2]} for row in rows]
|
||||
|
||||
def delete_pinned(self, pin_id: int) -> int:
|
||||
with closing(self._connect()) as conn, conn:
|
||||
return conn.execute("DELETE FROM pinned_facts WHERE id = ?", (pin_id,)).rowcount
|
||||
|
||||
@@ -66,13 +66,23 @@ link syntax from the model reads as noise.
|
||||
When the envelope `channel` is null/none/empty, the response channel
|
||||
is the channel the message came from.
|
||||
|
||||
### ENV-10 — System prompt template substitution (coverage: test)
|
||||
### ENV-10 — Dynamic context reaches the system prompt (coverage: test)
|
||||
|
||||
`message()` substitutes `{date}` (YYYY-MM-DD), `{time}`, `{memory}`
|
||||
(the assembled memory block — legacy memory string while the
|
||||
structured memory is inactive, see MEM-10) in the system prompt;
|
||||
`{news}` is replaced with the news file content when the configured
|
||||
file exists and stays literal when it does not.
|
||||
The system message carries the current date, time, news (when the
|
||||
configured file exists) and the memory block (legacy string while
|
||||
structured memory is inactive, see MEM-10). Since FDB-008 these live
|
||||
in a context suffix, not inline — see ENV-20; legacy `{date}`,
|
||||
`{time}`, `{news}`, `{memory}` placeholders in operator templates are
|
||||
stripped.
|
||||
|
||||
### ENV-20 — Persona prefix is byte-stable (coverage: test)
|
||||
|
||||
`message()` renders the system message as: static persona text
|
||||
(config template with all dynamic placeholders removed) followed by a
|
||||
`## Context` suffix holding date, time, news and memory. Two calls in
|
||||
the same channel produce byte-identical persona prefixes — the prompt
|
||||
cache can actually hit (the old inline `{date}`/`{time}` substitution
|
||||
invalidated it every minute).
|
||||
|
||||
### ENV-11 — Per-channel history shrink prefers busy channels (coverage: test)
|
||||
|
||||
|
||||
@@ -32,6 +32,12 @@ and game descriptions are attacker-influenced input.
|
||||
The `hack` envelope field remains as an advisory signal (logged,
|
||||
staff-notified) but is no longer the defense.
|
||||
|
||||
### SAF-10 — Model calls carry a hashed user identifier (coverage: test)
|
||||
|
||||
Chat calls pass `safety_identifier` = a short SHA-256 digest of the
|
||||
message author's name — OpenAI-side abuse tracing without shipping
|
||||
raw Discord identities (Codex review recommendation).
|
||||
|
||||
### SAF-04 — Hard daily budget, fail-closed (coverage: test)
|
||||
|
||||
When `daily-budget-usd` is configured and today's estimated spend
|
||||
|
||||
@@ -57,6 +57,11 @@ the alert text is written to the error log — never silently dropped.
|
||||
loop; later: the FDB-011 scheduler); `!bot tasks on` restores.
|
||||
Bot-initiated posts also respect pause/quiet.
|
||||
|
||||
### OPS-11 — Pins are listable (coverage: test)
|
||||
|
||||
`!bot pins` answers with all pinned facts and their ids (global +
|
||||
per-channel) — without it, `!bot unpin <id>` required guessing ids.
|
||||
|
||||
### OPS-10 — Spend report (coverage: test)
|
||||
|
||||
`!bot spend` answers in the staff channel with today's estimated
|
||||
|
||||
@@ -17,10 +17,11 @@ A responder bound to channel `X` uses `config["X"]` as its system
|
||||
prompt when that key exists, else `config["system"]`. One deployment
|
||||
can speak differently per channel.
|
||||
|
||||
### CFG-03 — Missing news file leaves the placeholder untouched (coverage: test)
|
||||
### CFG-03 — Missing news file degrades silently (coverage: test)
|
||||
|
||||
When `news` points to a non-existent file, the `{news}` placeholder
|
||||
stays literal in the system prompt (no crash, no empty substitution).
|
||||
When `news` points to a non-existent file, the context suffix simply
|
||||
carries no news section (no crash, no literal placeholder — revised
|
||||
with ENV-20; previously the `{news}` placeholder stayed literal).
|
||||
|
||||
### CFG-04 — Config hot-reload applies on the event loop (coverage: test)
|
||||
|
||||
|
||||
@@ -0,0 +1,63 @@
|
||||
# SPEC-010 — Human-behavior layer
|
||||
|
||||
The bot should feel like a considerate participant, not an instant
|
||||
wall of text: it decides *whether* to speak with a cheap classifier
|
||||
instead of trusting the main model's self-report, paces its replies,
|
||||
splits long answers, and sometimes just reacts. All knobs are
|
||||
per-deployment TOML; every feature degrades to the previous behavior
|
||||
when its knob is unset (config-off = v3.0.0 semantics).
|
||||
|
||||
### BEH-01 — Classifier gates non-direct replies (coverage: test)
|
||||
|
||||
With `classifier-model` configured, every non-direct user message
|
||||
first passes a cheap classification call (reply yes/no, factual
|
||||
yes/no, optional reaction emoji). `reply=false` means no main-model
|
||||
call happens at all — this is the boreness suppressor and the
|
||||
butting-into-conversations fix (replaces trusting `answer_needed`
|
||||
alone; the envelope flag still applies afterwards as second gate).
|
||||
|
||||
### BEH-02 — Direct messages bypass the gate (coverage: test)
|
||||
|
||||
Mentions and DMs never go through the classifier — someone addressing
|
||||
the bot always reaches the main model. Welcome and bot-initiated
|
||||
flows do not pass the gate either.
|
||||
|
||||
### BEH-03 — Classifier failure fails open (coverage: test)
|
||||
|
||||
A failed or unparseable classification (API error, budget refusal)
|
||||
falls through to the main model. Availability beats savings; the
|
||||
budget gate still protects spend.
|
||||
|
||||
### BEH-04 — Reply pacing is typing-proportional (coverage: test)
|
||||
|
||||
With `typing-chars-per-second` set (recommended 30), the typing
|
||||
indicator is held for `len(part) / cps` seconds per message part,
|
||||
capped at `typing-max-seconds` (default 8), before sending. Unset or
|
||||
0 = no pacing (v3.0.0 behavior).
|
||||
|
||||
### BEH-05 — Factual answers skip the artificial delay (coverage: test)
|
||||
|
||||
Messages the classifier tagged `factual` (opening hours, prices,
|
||||
addresses) are answered without the BEH-04 delay — utility beats
|
||||
theater exactly where users are waiting for information.
|
||||
|
||||
### BEH-06 — Long answers are split (coverage: test)
|
||||
|
||||
Answers longer than `split-threshold` chars (default 1200) are split
|
||||
at paragraph (then sentence) boundaries into at most
|
||||
`split-max-parts` (default 3) sequential messages, each under the
|
||||
Discord 2000-char limit (which unsplit answers would crash into
|
||||
today). Attached images go with the last part.
|
||||
|
||||
### BEH-07 — Sometimes a reaction is the reply (coverage: test)
|
||||
|
||||
When the classifier returns `reply=false` plus a reaction emoji, the
|
||||
bot adds that emoji to the user's message instead of staying fully
|
||||
silent. Zero main-model cost, human touch.
|
||||
|
||||
### BEH-08 — Quiet hours stop bot-initiated posts (coverage: test)
|
||||
|
||||
Within `quiet-hours = "HH:MM-HH:MM"` (host-local, may wrap midnight)
|
||||
`bot_initiated_allowed()` is false: no boreness, later no scheduler
|
||||
posts. Replies to users stay unaffected — a guest asking at 23:30
|
||||
still gets an answer.
|
||||
@@ -46,6 +46,9 @@ class FakeModelResponder(AIResponder):
|
||||
async def consolidate(self, observations, known_facts):
|
||||
return {"facts": [], "episode": None}
|
||||
|
||||
async def classify(self, message, history_tail):
|
||||
return getattr(self, "scripted_classification", None)
|
||||
|
||||
async def translate(self, text: str, language: str = "english") -> str:
|
||||
return text
|
||||
|
||||
|
||||
@@ -0,0 +1,210 @@
|
||||
"""Unit coverage for SPEC-010 human behavior (BEH-01..08) + ENV-20, SAF-10, OPS-11."""
|
||||
|
||||
import hashlib
|
||||
import tempfile
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
from unittest.mock import AsyncMock, MagicMock, Mock, patch
|
||||
|
||||
from discord import DMChannel, TextChannel
|
||||
|
||||
from fjerkroa_bot.ai_responder import AIMessage, AIResponse
|
||||
from fjerkroa_bot.discord_bot import quiet_hours_active, split_answer
|
||||
from fjerkroa_bot.openai_responder import OpenAIResponder
|
||||
from fjerkroa_bot.persistence import PersistentStore
|
||||
|
||||
from .test_bdd_envelope import FakeModelResponder, envelope
|
||||
from .test_spec_ops import OpsBase
|
||||
|
||||
|
||||
class ClassifierGateBase(OpsBase):
|
||||
def gate_setup(self, verdict):
|
||||
self.bot.config["classifier-model"] = "gpt-5.6-luna"
|
||||
self.bot.airesponder.classify = AsyncMock(return_value=verdict)
|
||||
self.bot.respond = AsyncMock()
|
||||
|
||||
|
||||
class TestClassifierGate(ClassifierGateBase):
|
||||
async def test_no_reply_means_no_model_call(self):
|
||||
"""BEH-01: classifier reply=false on a non-direct message -> respond() never runs."""
|
||||
self.gate_setup({"reply": False, "factual": False, "emoji": None})
|
||||
await self.bot.on_message(self.public_msg("just chatting with bob"))
|
||||
self.bot.airesponder.classify.assert_awaited_once()
|
||||
self.bot.respond.assert_not_awaited()
|
||||
|
||||
async def test_reply_true_passes_through_with_factual_flag(self):
|
||||
"""BEH-01: reply=true proceeds; factual flag is forwarded."""
|
||||
self.gate_setup({"reply": True, "factual": True, "emoji": None})
|
||||
await self.bot.on_message(self.public_msg("når har dere åpent?"))
|
||||
self.bot.respond.assert_awaited_once()
|
||||
self.assertTrue(self.bot.respond.await_args.kwargs.get("factual"))
|
||||
|
||||
async def test_direct_message_bypasses_gate(self):
|
||||
"""BEH-02: DMs never touch the classifier."""
|
||||
self.gate_setup({"reply": False, "factual": False, "emoji": None})
|
||||
message = self.public_msg("hei bot")
|
||||
message.channel = MagicMock(spec=DMChannel)
|
||||
message.channel.recipient = None
|
||||
await self.bot.on_message(message)
|
||||
self.bot.airesponder.classify.assert_not_awaited()
|
||||
self.bot.respond.assert_awaited_once()
|
||||
|
||||
async def test_classifier_failure_fails_open(self):
|
||||
"""BEH-03: classify() -> None falls through to the main model."""
|
||||
self.gate_setup(None)
|
||||
await self.bot.on_message(self.public_msg("hello?"))
|
||||
self.bot.respond.assert_awaited_once()
|
||||
|
||||
async def test_reaction_instead_of_reply(self):
|
||||
"""BEH-07: reply=false + emoji -> reaction on the message, no model call."""
|
||||
self.gate_setup({"reply": False, "factual": False, "emoji": "👍"})
|
||||
message = self.public_msg("gg everyone")
|
||||
message.add_reaction = AsyncMock()
|
||||
await self.bot.on_message(message)
|
||||
message.add_reaction.assert_awaited_once_with("👍")
|
||||
self.bot.respond.assert_not_awaited()
|
||||
|
||||
|
||||
class TestTypingPacing(OpsBase):
|
||||
async def send_with(self, answer, factual, cps=30):
|
||||
if cps is not None:
|
||||
self.bot.config["typing-chars-per-second"] = cps
|
||||
response = AIResponse(answer, True, "chat", None, None, False, False)
|
||||
channel = MagicMock(spec=TextChannel)
|
||||
channel.send = AsyncMock()
|
||||
with patch("fjerkroa_bot.discord_bot.asyncio.sleep", new_callable=AsyncMock) as sleep:
|
||||
await self.bot.send_answer_with_typing(response, channel, self.bot.airesponder, factual=factual)
|
||||
return sleep, channel
|
||||
|
||||
async def test_delay_proportional_and_capped(self):
|
||||
"""BEH-04: delay = len/cps capped at typing-max-seconds."""
|
||||
sleep, _ = await self.send_with("x" * 300, factual=False, cps=30)
|
||||
sleep.assert_awaited_once_with(8.0) # 300/30=10 -> cap 8
|
||||
|
||||
async def test_factual_skips_delay(self):
|
||||
"""BEH-05: factual answers go out instantly."""
|
||||
sleep, _ = await self.send_with("x" * 300, factual=True, cps=30)
|
||||
sleep.assert_not_awaited()
|
||||
|
||||
async def test_no_knob_no_delay(self):
|
||||
"""BEH-04: without typing-chars-per-second there is no pacing."""
|
||||
sleep, _ = await self.send_with("x" * 300, factual=False, cps=None)
|
||||
sleep.assert_not_awaited()
|
||||
|
||||
|
||||
class TestSplitting(unittest.TestCase):
|
||||
def test_split_at_paragraphs_under_limit(self):
|
||||
"""BEH-06: long answers split at paragraph boundaries, each under 2000."""
|
||||
text = "\n\n".join(["Avsnitt " + str(i) + " " + "x" * 700 for i in range(4)])
|
||||
parts = split_answer(text, threshold=1200, max_parts=3)
|
||||
self.assertGreaterEqual(len(parts), 2)
|
||||
self.assertLessEqual(len(parts), 3)
|
||||
for part in parts:
|
||||
self.assertLessEqual(len(part), 2000)
|
||||
self.assertEqual("\n\n".join(parts).replace("\n\n", ""), text.replace("\n\n", ""))
|
||||
|
||||
def test_short_answers_untouched(self):
|
||||
"""BEH-06: short answers stay a single message."""
|
||||
self.assertEqual(split_answer("kort svar", 1200, 3), ["kort svar"])
|
||||
|
||||
def test_oversized_single_block_hard_split(self):
|
||||
"""BEH-06: a single block over 2000 chars is hard-split under the Discord limit."""
|
||||
parts = split_answer("y" * 4500, 1200, 3)
|
||||
for part in parts:
|
||||
self.assertLessEqual(len(part), 2000)
|
||||
self.assertEqual(sum(len(p) for p in parts), 4500)
|
||||
|
||||
|
||||
class TestSplitSends(OpsBase):
|
||||
async def test_parts_sent_in_order_files_last(self):
|
||||
"""BEH-06: parts sent sequentially; image files ride on the last part."""
|
||||
self.bot.config["split-threshold"] = 50
|
||||
answer = "Første del.\n\nAndre del som også er ganske lang her."
|
||||
response = AIResponse(answer, True, "chat", None, "a cat", False, False)
|
||||
self.bot.airesponder.draw = AsyncMock(return_value=__import__("io").BytesIO(b"png"))
|
||||
channel = MagicMock(spec=TextChannel)
|
||||
channel.send = AsyncMock()
|
||||
await self.bot.send_answer_with_typing(response, channel, self.bot.airesponder, factual=True)
|
||||
self.assertEqual(channel.send.await_count, 2)
|
||||
first_kwargs = channel.send.await_args_list[0].kwargs
|
||||
last_kwargs = channel.send.await_args_list[1].kwargs
|
||||
self.assertIsNone(first_kwargs.get("files"))
|
||||
self.assertIsNotNone(last_kwargs.get("files"))
|
||||
|
||||
|
||||
class TestQuietHours(OpsBase):
|
||||
def test_quiet_hours_parsing(self):
|
||||
"""BEH-08: window logic incl. midnight wrap."""
|
||||
self.assertTrue(quiet_hours_active("23:00-08:00", "23:30"))
|
||||
self.assertTrue(quiet_hours_active("23:00-08:00", "07:59"))
|
||||
self.assertFalse(quiet_hours_active("23:00-08:00", "12:00"))
|
||||
self.assertTrue(quiet_hours_active("13:00-15:00", "14:00"))
|
||||
self.assertFalse(quiet_hours_active("13:00-15:00", "15:00"))
|
||||
self.assertFalse(quiet_hours_active(None, "14:00"))
|
||||
self.assertFalse(quiet_hours_active("garbage", "14:00"))
|
||||
|
||||
async def test_quiet_hours_block_bot_initiated(self):
|
||||
"""BEH-08: inside the window bot_initiated_allowed is false, replies unaffected."""
|
||||
self.bot.config["quiet-hours"] = "00:00-23:59"
|
||||
self.assertFalse(self.bot.bot_initiated_allowed())
|
||||
self.assertTrue(self.bot.replies_allowed())
|
||||
|
||||
|
||||
class TestPromptPrefixStability(unittest.IsolatedAsyncioTestCase):
|
||||
def test_persona_prefix_stable_and_context_suffix(self):
|
||||
"""ENV-20 + ENV-10: byte-stable persona prefix; date/memory in the context suffix."""
|
||||
config = {"system": "Du er Fjærkroa. I dag er {date} kl {time}. {news} {memory}", "history-limit": 5}
|
||||
responder = FakeModelResponder(config, "chat")
|
||||
responder.memory = "MEMSTR"
|
||||
first = responder.message(AIMessage("alice", "hei"))[0]["content"]
|
||||
second = responder.message(AIMessage("bob", "hallo"))[0]["content"]
|
||||
self.assertIn("## Context", first)
|
||||
prefix_one = first.split("## Context")[0]
|
||||
prefix_two = second.split("## Context")[0]
|
||||
self.assertEqual(prefix_one, prefix_two)
|
||||
self.assertNotIn("{date}", first)
|
||||
self.assertNotIn("{memory}", first)
|
||||
suffix = first.split("## Context")[1]
|
||||
self.assertIn("MEMSTR", suffix)
|
||||
import time as _time
|
||||
|
||||
self.assertIn(_time.strftime("%Y-%m-%d"), suffix)
|
||||
|
||||
def test_missing_news_file_no_placeholder(self):
|
||||
"""CFG-03 (revised): missing news file -> no literal placeholder, no crash."""
|
||||
config = {"system": "N: {news}", "history-limit": 5, "news": "/nonexistent/news.txt"}
|
||||
responder = FakeModelResponder(config, "chat")
|
||||
system = responder.message(AIMessage("alice", "hei"))[0]["content"]
|
||||
self.assertNotIn("{news}", system)
|
||||
self.assertNotIn("news:", system.split("## Context")[1])
|
||||
|
||||
|
||||
class TestSafetyIdentifier(unittest.IsolatedAsyncioTestCase):
|
||||
async def test_chat_carries_hashed_user(self):
|
||||
"""SAF-10: chat calls pass safety_identifier = sha256(user)[:16], never the raw name."""
|
||||
responder = OpenAIResponder({"openai-token": "t", "model": "m", "system": "s", "history-limit": 5}, "chat")
|
||||
message = Mock(content=envelope(answer="x", answer_needed=True), role="assistant", tool_calls=None, refusal=None)
|
||||
with patch("fjerkroa_bot.openai_responder.openai_chat", new_callable=AsyncMock) as chat_mock:
|
||||
chat_mock.return_value = Mock(choices=[Mock(message=message)], usage="usage")
|
||||
payload = '{"user": "alice", "message": "hei", "channel": "chat", "direct": false, "historise_question": true}'
|
||||
await responder.chat([{"role": "user", "content": payload}], 10)
|
||||
identifier = chat_mock.await_args.kwargs.get("safety_identifier")
|
||||
expected = "discord-" + hashlib.sha256(b"alice").hexdigest()[:16]
|
||||
self.assertEqual(identifier, expected)
|
||||
self.assertNotIn("alice", identifier)
|
||||
|
||||
|
||||
class TestPinsListing(OpsBase):
|
||||
async def test_pins_command_lists_ids(self):
|
||||
"""OPS-11: !bot pins lists pinned facts with ids."""
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
store = PersistentStore(Path(tmp) / "bot.db")
|
||||
self.bot.airesponder.store = store
|
||||
store.add_pinned(None, "Åpningstider: tirsdag-fredag 12-17")
|
||||
store.add_pinned("chat", "Kanalregel: norsk")
|
||||
await self.bot.on_message(self.staff_msg("!bot pins"))
|
||||
listing = self.bot.staff_channel.send.await_args.args[0]
|
||||
self.assertIn("Åpningstider", listing)
|
||||
self.assertIn("Kanalregel", listing)
|
||||
self.assertIn("1", listing)
|
||||
self.assertIn("2", listing)
|
||||
@@ -35,9 +35,10 @@ class TestPerChannelPrompt(unittest.TestCase):
|
||||
|
||||
|
||||
class TestNewsFileMissing(unittest.TestCase):
|
||||
def test_missing_news_file_keeps_placeholder(self):
|
||||
"""CFG-03: nonexistent news file -> {news} placeholder stays literal, no crash."""
|
||||
def test_missing_news_file_degrades_silently(self):
|
||||
"""CFG-03 (revised): nonexistent news file -> no news section, no literal, no crash."""
|
||||
config = {"system": "N: {news}", "history-limit": 5, "news": "/nonexistent/news.txt"}
|
||||
responder = AIResponder(config, "chat")
|
||||
system = responder.message(AIMessage("alice", "hei"))[0]["content"]
|
||||
self.assertIn("{news}", system)
|
||||
self.assertNotIn("{news}", system)
|
||||
self.assertNotIn("news:", system)
|
||||
|
||||
@@ -14,8 +14,8 @@ def entry(channel: str, text: str = "x"):
|
||||
|
||||
|
||||
class TestSystemPromptTemplate(unittest.TestCase):
|
||||
def test_template_substitution(self):
|
||||
"""ENV-10: {date}/{memory} substituted; missing news file leaves {news} literal."""
|
||||
def test_dynamic_context_in_suffix(self):
|
||||
"""ENV-10: date/memory reach the system message via the context suffix (ENV-20)."""
|
||||
config = {"system": "Date {date} memory {memory} news {news}", "history-limit": 5, "news": "/nonexistent/news.txt"}
|
||||
responder = AIResponder(config, "chat")
|
||||
responder.memory = "MEMSTR"
|
||||
@@ -23,7 +23,7 @@ class TestSystemPromptTemplate(unittest.TestCase):
|
||||
system = messages[0]["content"]
|
||||
self.assertIn(time.strftime("%Y-%m-%d"), system)
|
||||
self.assertIn("MEMSTR", system)
|
||||
self.assertIn("{news}", system) # left literal — file does not exist
|
||||
self.assertNotIn("{news}", system) # placeholders stripped since ENV-20
|
||||
|
||||
|
||||
class TestHistoryShrink(unittest.TestCase):
|
||||
|
||||
Reference in New Issue
Block a user