From f1578cbd99f882a34eb69d86c12b05a38753d11e Mon Sep 17 00:00:00 2001 From: Oleksandr Kozachuk Date: Mon, 13 Jul 2026 12:53:38 +0200 Subject: [PATCH] add spec system: sdd/bdd/tdd loops, trace enforcement, envelope+config specs --- DECISIONS.md | 38 +++++++ features/envelope.feature | 61 +++++++++++ manual-verification.md | 11 ++ specs/SPEC-000-process.md | 69 +++++++++++++ specs/SPEC-001-responder-envelope.md | 133 ++++++++++++++++++++++++ specs/SPEC-008-config.md | 23 +++++ tests/conftest.py | 15 +++ tests/test_bdd_envelope.py | 147 +++++++++++++++++++++++++++ tests/test_spec_cfg.py | 43 ++++++++ tests/test_spec_env.py | 56 ++++++++++ tools/trace.py | 86 ++++++++++++++++ 11 files changed, 682 insertions(+) create mode 100644 DECISIONS.md create mode 100644 features/envelope.feature create mode 100644 manual-verification.md create mode 100644 specs/SPEC-000-process.md create mode 100644 specs/SPEC-001-responder-envelope.md create mode 100644 specs/SPEC-008-config.md create mode 100644 tests/conftest.py create mode 100644 tests/test_bdd_envelope.py create mode 100644 tests/test_spec_cfg.py create mode 100644 tests/test_spec_env.py create mode 100644 tools/trace.py diff --git a/DECISIONS.md b/DECISIONS.md new file mode 100644 index 0000000..0a5876c --- /dev/null +++ b/DECISIONS.md @@ -0,0 +1,38 @@ +# DECISIONS.md — ADR log + +Decisions inside the set architecture. D-NNN, never renumbered. + +- **D-001** — SDD+BDD+TDD adopted (2026-07-13). SPEC-system port: + numbered requirements in `specs/`, coverage classes + feature/test/manual, `tools/trace.py` enforcement in `make check`. + Process is binding — see SPEC-000. +- **D-002** — BDD runs at the responder seam, not against live + services. `FakeModelResponder` scripts model output; Discord-event + behavior is unit-tested with mocked discord.py objects. Rationale: + Discord ToS forbids test-account automation and LLM output is + nondeterministic — a live-BDD lane would be flaky by construction. +- **D-003** — Packaging = uv + pyproject only (2026-07-13). setup.py, + requirements.txt and pytest.ini removed; single source of truth, + locked via uv.lock. Python `>=3.11` floor keeps the kitchen host + (py3.11) deployable until FDB-016 lands. +- **D-004** — openai SDK pinned `<2` (1.109.x). The v2 SDK migration + happens together with the FDB-005 envelope rewrite (structured + outputs / Responses API) — one breaking change, one test cycle, + instead of two. +- **D-005** — `--strict-markers` stays on; requirement tags from + `.feature` files are registered as pytest markers dynamically in + `tests/conftest.py`. +- **D-006** — FDB-005 stays on chat.completions + structured outputs; + the Responses API migration is deferred to FDB-007, where history + handling gets redesigned anyway — one conversation-state reshape + instead of two. +- **D-007** — openai SDK bumped to 2.x together with the envelope + rewrite (supersedes the D-004 pin). `multiline` dependency dropped — + strict schema output made relaxed-JSON parsing dead code. +- **D-008** — Operator runtime flags (pause/images/tasks/quiet) are + in-memory only; a restart resets to config defaults. Persistence + arrives with the FDB-011 task store if staff practice demands it. +- **D-009** — `translate()` still keys off `fix-model` although the + repair path is gone; the whole translate-before-draw step dies in + FDB-009 (gpt-image-2 is multilingual). Not worth a config rename + for one phase. diff --git a/features/envelope.feature b/features/envelope.feature new file mode 100644 index 0000000..4454723 --- /dev/null +++ b/features/envelope.feature @@ -0,0 +1,61 @@ +Feature: Response envelope handling + The responder parses the model's JSON envelope and decides what the + bot says, where, and whether staff is alerted. (SPEC-001) + + Background: + Given a responder with history limit 10 + + @ENV-01 + Scenario: Model answer reaches the user + Given the model answers with answer "Hei! Velkommen." and answer_needed "true" + When user "alice" sends "Hei bot" in channel "chat" + Then the response answer contains "Hei! Velkommen." + And the response is marked as needed + + @ENV-02 + Scenario: Suppressed answer stays silent + Given the model answers with answer "irrelevant musing" and answer_needed "false" + When user "alice" sends "talking to bob" in channel "chat" + Then the response is not marked as needed + + @ENV-03 + Scenario: Staff note forces delivery + Given the model answers with answer "Et oyeblikk!" and staff note "Guest at table 4 needs a waiter" + When user "guest" sends "Can somebody help us?" in channel "chat" + Then the response staff note is "Guest at table 4 needs a waiter" + And the response is marked as needed + + @ENV-04 + Scenario: Direct messages are always answered + Given the model answers with answer "Svar." and answer_needed "false" + When user "alice" sends "hei" directly to the bot + Then the response is marked as needed + + @ENV-05 + Scenario: Short-path rules skip the model + Given a short-path rule for channels "spam.*" and users "bob.*" + When user "bobby" sends "noise noise" in channel "spam-corner" + Then the model was not called + And the response is empty + And the history contains the message from "bobby" + + @ENV-07 + Scenario: History is trimmed to the limit + Given a responder with history limit 4 + And 6 prior history entries in channel "chat" + And the model answers with answer "ok" and answer_needed "true" + When user "alice" sends "hei" in channel "chat" + Then the history length is at most 4 + + @ENV-08 + Scenario: Markdown links are unwrapped + Given the model answers with answer "Se [menyen](https://fjerkroa.example/meny) her" and answer_needed "true" + When user "alice" sends "meny?" in channel "chat" + Then the response answer contains "https://fjerkroa.example/meny" + And the response answer does not contain "[menyen]" + + @ENV-09 + Scenario: Missing channel falls back to the message channel + Given the model answers with answer "ok" and no channel + When user "alice" sends "hei" in channel "kitchen-talk" + Then the response channel is "kitchen-talk" diff --git a/manual-verification.md b/manual-verification.md new file mode 100644 index 0000000..42ab789 --- /dev/null +++ b/manual-verification.md @@ -0,0 +1,11 @@ +# Manual verification log + +Rows for requirements with `coverage: manual` (see SPEC-000). Newest +first. A manual requirement is only "covered" when it has a row here +with date + result. + +| ID | Date | Result | +| --- | --- | --- | + +_No manual-coverage requirements declared yet — DEP-NN arrives with +FDB-017._ diff --git a/specs/SPEC-000-process.md b/specs/SPEC-000-process.md new file mode 100644 index 0000000..d8f7045 --- /dev/null +++ b/specs/SPEC-000-process.md @@ -0,0 +1,69 @@ +# SPEC-000 — Development process (binding) + +This repo is developed spec-driven + behaviour-driven + test-driven. + +## The three loops + +1. **Spec-driven**: every behavior exists first as a numbered + requirement (`-NN`) in `specs/SPEC-NNN-*.md`. No code without + a requirement. Requirements are never deleted — a dead requirement + is marked `(withdrawn: )` in its title line + and keeps its ID forever. +2. **Behaviour-driven**: every *user-visible* requirement gets at + least one Gherkin scenario tagged `@` in `features/*.feature`, + executed by pytest-bdd. +3. **Test-driven**: implementation starts at a red test. Unit tests + cover the non-visible requirements (arithmetic, invariants, + parsing). + +Change flow: spec change → feature/test red → implementation green → +refactor. + +## Requirement format + +``` +### ENV-01 — Model answer reaches the user (coverage: feature) + +Normative statement first. Rationale after. +``` + +Coverage classes: + +- `feature` — needs an `@` tag in some `.feature` file. +- `test` — the ID must appear in a test file (docstring or comment + of the covering test). +- `manual` — needs a row in `manual-verification.md` with date and + result before the increment demo. + +## Enforcement + +`tools/trace.py` (stdlib-only) runs in `make check` (and `make +trace`). It fails the build when a declared requirement lacks its +coverage class artifact, when a feature tag references an undeclared +ID, or when an ID is declared twice. Code fences in spec files are +ignored by trace (the example above does not declare ENV-01). + +## Test substrate + +BDD scenarios run against the **responder seam**: a +`FakeModelResponder` subclass with scripted model output — no live +Discord, no live OpenAI (see DECISIONS.md D-002). Discord-event +behavior is covered by unit tests with mocked discord.py objects. + +## ADRs + +Decisions inside the set architecture go to `DECISIONS.md` as +`D-NNN — title — one-paragraph rationale`. The stack itself is not +re-litigated there. + +## Spec map + +- SPEC-000 process (this file, no IDs) +- SPEC-001 responder + envelope — ENV-NN +- SPEC-002 memory — MEM-NN (lands with FDB-007) +- SPEC-003 safety/privacy/abuse — SAF-NN (lands with FDB-014) +- SPEC-004 images — IMG-NN (lands with FDB-009/010) +- SPEC-005 self-tasking — TSK-NN (lands with FDB-011) +- SPEC-006 staff/operator controls — OPS-NN (lands with FDB-005) +- SPEC-007 deploy/hosts — DEP-NN (lands with FDB-017, mostly manual) +- SPEC-008 config — CFG-NN diff --git a/specs/SPEC-001-responder-envelope.md b/specs/SPEC-001-responder-envelope.md new file mode 100644 index 0000000..5f5ad30 --- /dev/null +++ b/specs/SPEC-001-responder-envelope.md @@ -0,0 +1,133 @@ +# SPEC-001 — Responder + response envelope + +Characterization of the current (v2) responder contract in +`fjerkroa_bot/ai_responder.py`. The model answers with a JSON +envelope: `answer`, `answer_needed`, `channel`, `staff`, `picture`, +`picture_edit`, `hack`. FDB-005 will replace the transport of this +contract (structured outputs); the *behavioral* requirements below +survive that change unless marked otherwise. + +### ENV-01 — Model answer reaches the user (coverage: feature) + +When the model envelope carries a non-empty `answer` and +`answer_needed` is true, `AIResponder.send()` returns an `AIResponse` +with that answer text and `answer_needed == True`. This is the core +loop: user talks, bot answers. + +### ENV-02 — Suppressed answer stays silent (coverage: feature) + +When the model envelope sets `answer_needed` to false (and the +message is not direct, mentions nobody, and no staff note is set), +the returned `AIResponse` has `answer_needed == False`. The bot may +observe without butting in. + +### ENV-03 — Staff note forces delivery (coverage: feature) + +When the envelope carries a non-null `staff` text and a non-null +`answer`, the returned response preserves the staff text and has +`answer_needed == True`. Staff alerts must never be silently +dropped. + +### ENV-04 — Direct messages are always answered (coverage: feature) + +When the incoming `AIMessage` is marked `direct` and the model +returns a non-null answer, `answer_needed` is forced to `True` +regardless of the model's own `answer_needed`. A user addressing the +bot directly gets a reply. + +### ENV-05 — Short-path rules skip the model (coverage: feature) + +When a configured `short-path` `[channel-regex, user-regex]` pair +matches the message, `send()` appends the message to history, trims +history to the limit, persists it, and returns an empty response +(answer `None`, `answer_needed False`) **without calling the model**. +Cheap archival of noisy channels. + +### ENV-06 — Malformed model output is repaired (coverage: withdrawn — successor ENV-18) + +Withdrawn 2026-07-13 with FDB-005: strict structured outputs make the +repair model obsolete. Malformed output now counts as a failed +attempt — see ENV-18. + +### ENV-07 — History is trimmed to the limit (coverage: feature) + +After a completed exchange, `len(history) <= history-limit` holds. +Trimming happens both before the model call and after appending the +new question/answer pair. + +### ENV-08 — Markdown links are unwrapped (coverage: feature) + +In the final answer text, `[label](url)` becomes `url` and +`@[label](url)` becomes `label`. Discord renders raw URLs; markdown +link syntax from the model reads as noise. + +### ENV-09 — Missing channel falls back to the message channel (coverage: feature) + +When the envelope `channel` is null/none/empty, the response channel +is the channel the message came from. + +### ENV-10 — System prompt template substitution (coverage: test) + +`message()` substitutes `{date}` (YYYY-MM-DD), `{time}`, `{memory}` +(current memory string) in the system prompt; `{news}` is replaced +with the news file content when the configured file exists and stays +literal when it does not. + +### ENV-11 — Per-channel history shrink prefers busy channels (coverage: test) + +`shrink_history_by_one()` removes the oldest entry whose channel has +more than `history-per-channel` (default 3) entries; when no channel +exceeds the cap, the oldest entry overall is removed. + +### ENV-12 — Exhausted retries raise, attempts are spaced (coverage: test) + +When the model returns no usable answer three times in a row, +`send()` raises `RuntimeError`. Consecutive attempts are separated by +an exponential-backoff sleep — failures never hammer the API +back-to-back (D1). + +### ENV-13 — Reaction-clear events are recorded (coverage: test) + +`on_reaction_clear(message, reactions)` — the discord.py signature — +records the clearing in the channel's memory. (D7: the previous +handler declared `(reaction, user)` and crashed on dispatch.) + +### ENV-14 — update_memory persists its argument (coverage: test) + +`update_memory(memory)` sets the responder's memory to the passed +value and persists that value. (D12: the argument was ignored.) + +### ENV-15 — retry-model is used after a rate limit (coverage: test) + +After a rate-limited attempt, the next `chat()` attempt uses the +configured `retry-model` instead of `model`; a successful attempt +switches back. (D2: the fallback was assigned to a local and never +took effect.) + +### ENV-16 — Model calls are never served from a disk cache (coverage: test) + +`openai_chat`/`openai_image` call the client every time. The pickle +response cache (`openai_chat.dat`) is a test-era artifact and must +not exist in the production path (D3/D4). + +### ENV-17 — IGDB tool execution runs in a worker thread (coverage: test) + +`_execute_igdb_function` executes the synchronous IGDB library off +the event loop (worker thread), returning identical results. The +event loop keeps serving Discord events during lookups (D5). + +### ENV-18 — Malformed or refused output is a failed attempt (coverage: test) + +Model output is requested as schema-validated JSON (strict structured +outputs). Output that still fails to parse, or a model refusal, +counts as a failed attempt (backoff + retry per ENV-12) — there is no +repair model, no `fix()` path, no relaxed-JSON fallback. Replaces +ENV-06. + +### ENV-19 — The envelope schema is pinned (coverage: test) + +Every chat call carries `response_format` = strict JSON schema named +`envelope` with exactly the fields `answer`, `answer_needed`, +`channel`, `staff`, `picture`, `picture_edit`, `hack` — all required, +`additionalProperties: false`, nullable where the protocol allows +null. Tool-followup calls carry the same format. diff --git a/specs/SPEC-008-config.md b/specs/SPEC-008-config.md new file mode 100644 index 0000000..2c8e7ba --- /dev/null +++ b/specs/SPEC-008-config.md @@ -0,0 +1,23 @@ +# SPEC-008 — Configuration + +TOML config per deployment (`kroa.toml`, `ggg.toml` — both untracked; +`config.toml` in the repo is the placeholder sample). Loaded by +`FjerkroaBot.load_config`, hot-reloaded by a watchdog observer +(defect D9 — reload race — is tracked in FDB-004 and will refine +these requirements). + +### CFG-01 — TOML config loads into a plain dict (coverage: test) + +`FjerkroaBot.load_config(path)` parses the TOML file and returns its +top-level table as a dict; responders read raw keys from it. + +### CFG-02 — Per-channel system prompt override (coverage: test) + +A responder bound to channel `X` uses `config["X"]` as its system +prompt when that key exists, else `config["system"]`. One deployment +can speak differently per channel. + +### CFG-03 — Missing news file leaves the placeholder untouched (coverage: test) + +When `news` points to a non-existent file, the `{news}` placeholder +stays literal in the system prompt (no crash, no empty substitution). diff --git a/tests/conftest.py b/tests/conftest.py new file mode 100644 index 0000000..db03d97 --- /dev/null +++ b/tests/conftest.py @@ -0,0 +1,15 @@ +import re +from pathlib import Path + +FEATURES_DIR = Path(__file__).resolve().parent.parent / "features" +TAG_RE = re.compile(r"@([A-Z]{2,8}-\d{2,3})\b") + + +def pytest_configure(config): + # pytest-bdd turns @ENV-01 tags into markers; register them so + # --strict-markers stays enabled (D-005). + ids = set() + for feature in FEATURES_DIR.glob("**/*.feature"): + ids |= set(TAG_RE.findall(feature.read_text(encoding="utf-8"))) + for req_id in sorted(ids): + config.addinivalue_line("markers", f"{req_id}: spec requirement tag") diff --git a/tests/test_bdd_envelope.py b/tests/test_bdd_envelope.py new file mode 100644 index 0000000..ee41e44 --- /dev/null +++ b/tests/test_bdd_envelope.py @@ -0,0 +1,147 @@ +"""BDD steps for features/envelope.feature (SPEC-001, ENV-01..09). + +Scenarios drive AIResponder.send() through a FakeModelResponder with +scripted model output — no live OpenAI, no live Discord (D-002). +""" + +import asyncio +import json +from typing import Any, Dict, List, Optional, Tuple + +from pytest_bdd import given, parsers, scenarios, then, when + +from fjerkroa_bot.ai_responder import AIMessage, AIResponder + +scenarios("../features/envelope.feature") + + +def envelope(answer=None, answer_needed=False, channel="chat", staff=None, picture=None, picture_edit=False, hack=False) -> str: + return json.dumps( + { + "answer": answer, + "answer_needed": answer_needed, + "channel": channel, + "staff": staff, + "picture": picture, + "picture_edit": picture_edit, + "hack": hack, + } + ) + + +class FakeModelResponder(AIResponder): + """AIResponder with the model calls scripted away (D-002).""" + + def __init__(self, config: Dict[str, Any], channel: Optional[str] = None) -> None: + super().__init__(config, channel) + self.scripted: List[str] = [] + self.chat_calls = 0 + + async def chat(self, messages: List[Dict[str, Any]], limit: int) -> Tuple[Optional[Dict[str, Any]], int]: + self.chat_calls += 1 + if not self.scripted: + return None, limit + return {"role": "assistant", "content": self.scripted.pop(0)}, limit + + async def memory_rewrite(self, memory: str, message_user: str, answer_user: str, question: str, answer: str) -> str: + return memory + + async def translate(self, text: str, language: str = "english") -> str: + return text + + +@given(parsers.parse("a responder with history limit {limit:d}"), target_fixture="responder") +def responder(limit): + config = {"system": "You are a test bot. {memory}", "history-limit": limit} + return FakeModelResponder(config, "chat") + + +@given(parsers.parse('the model answers with answer "{answer}" and answer_needed "{needed}"')) +def script_answer(responder, answer, needed): + responder.scripted.append(envelope(answer=answer, answer_needed=needed == "true")) + + +@given(parsers.parse('the model answers with answer "{answer}" and staff note "{staff}"')) +def script_staff(responder, answer, staff): + responder.scripted.append(envelope(answer=answer, answer_needed=False, staff=staff)) + + +@given(parsers.parse('the model answers with answer "{answer}" and channel "{channel}"')) +def script_channel(responder, answer, channel): + responder.scripted.append(envelope(answer=answer, answer_needed=True, channel=channel)) + + +@given(parsers.parse('the model answers with answer "{answer}" and no channel')) +def script_channel_none(responder, answer): + responder.scripted.append(envelope(answer=answer, answer_needed=True, channel=None)) + + +@given(parsers.parse('a short-path rule for channels "{chan_re}" and users "{user_re}"')) +def short_path_rule(responder, chan_re, user_re): + responder.config["short-path"] = [[chan_re, user_re]] + + +@given(parsers.parse('{count:d} prior history entries in channel "{channel}"')) +def prior_history(responder, count, channel): + for i in range(count): + responder.history.append({"role": "user", "content": json.dumps({"message": f"old {i}", "channel": channel})}) + + +@when(parsers.parse('user "{user}" sends "{text}" in channel "{channel}"'), target_fixture="response") +def send_message(responder, user, text, channel): + return asyncio.run(responder.send(AIMessage(user, text, channel))) + + +@when(parsers.parse('user "{user}" sends "{text}" directly to the bot'), target_fixture="response") +def send_direct(responder, user, text): + return asyncio.run(responder.send(AIMessage(user, text, "chat", direct=True))) + + +@then(parsers.parse('the response answer contains "{text}"')) +def answer_contains(response, text): + assert response.answer is not None and text in response.answer + + +@then(parsers.parse('the response answer does not contain "{text}"')) +def answer_not_contains(response, text): + assert response.answer is not None and text not in response.answer + + +@then("the response is marked as needed") +def is_needed(response): + assert response.answer_needed is True + + +@then("the response is not marked as needed") +def not_needed(response): + assert response.answer_needed is False + + +@then(parsers.parse('the response staff note is "{text}"')) +def staff_note_is(response, text): + assert response.staff == text + + +@then(parsers.parse('the response channel is "{channel}"')) +def channel_is(response, channel): + assert response.channel == channel + + +@then("the model was not called") +def model_not_called(responder): + assert responder.chat_calls == 0 + + +@then("the response is empty") +def response_empty(response): + assert response.answer is None and response.answer_needed is False + + +@then(parsers.parse('the history contains the message from "{user}"')) +def history_has_user(responder, user): + assert any(f'"user": "{user}"' in item["content"] for item in responder.history) + + +@then(parsers.parse("the history length is at most {limit:d}")) +def history_at_most(responder, limit): + assert len(responder.history) <= limit diff --git a/tests/test_spec_cfg.py b/tests/test_spec_cfg.py new file mode 100644 index 0000000..6d6885e --- /dev/null +++ b/tests/test_spec_cfg.py @@ -0,0 +1,43 @@ +"""Unit coverage for SPEC-008 (CFG-01..03).""" + +import unittest +from unittest.mock import mock_open, patch + +import toml + +from fjerkroa_bot import FjerkroaBot +from fjerkroa_bot.ai_responder import AIMessage, AIResponder + + +class TestConfigLoad(unittest.TestCase): + def test_load_config_parses_toml(self): + """CFG-01: load_config parses the TOML file into a plain dict.""" + data = {"system": "prompt", "history-limit": 7, "additional-responders": []} + with patch("builtins.open", mock_open(read_data=toml.dumps(data))): + result = FjerkroaBot.load_config("config.toml") + self.assertEqual(result, data) + + +class TestPerChannelPrompt(unittest.TestCase): + def test_channel_override_wins(self): + """CFG-02: responder bound to a channel uses config[channel] over config['system'].""" + config = {"system": "Default prompt", "kitchen": "Kitchen prompt", "history-limit": 5} + responder = AIResponder(config, "kitchen") + system = responder.message(AIMessage("alice", "hei", "kitchen"))[0]["content"] + self.assertTrue(system.startswith("Kitchen prompt")) + + def test_fallback_to_system(self): + """CFG-02: without a channel key the shared system prompt is used.""" + config = {"system": "Default prompt", "history-limit": 5} + responder = AIResponder(config, "kitchen") + system = responder.message(AIMessage("alice", "hei", "kitchen"))[0]["content"] + self.assertTrue(system.startswith("Default prompt")) + + +class TestNewsFileMissing(unittest.TestCase): + def test_missing_news_file_keeps_placeholder(self): + """CFG-03: nonexistent news file -> {news} placeholder stays literal, no crash.""" + config = {"system": "N: {news}", "history-limit": 5, "news": "/nonexistent/news.txt"} + responder = AIResponder(config, "chat") + system = responder.message(AIMessage("alice", "hei"))[0]["content"] + self.assertIn("{news}", system) diff --git a/tests/test_spec_env.py b/tests/test_spec_env.py new file mode 100644 index 0000000..7383b2c --- /dev/null +++ b/tests/test_spec_env.py @@ -0,0 +1,56 @@ +"""Unit coverage for SPEC-001 test-class requirements (ENV-10..12).""" + +import json +import time +import unittest + +from fjerkroa_bot.ai_responder import AIMessage, AIResponder + +from .test_bdd_envelope import FakeModelResponder + + +def entry(channel: str, text: str = "x"): + return {"role": "user", "content": json.dumps({"message": text, "channel": channel})} + + +class TestSystemPromptTemplate(unittest.TestCase): + def test_template_substitution(self): + """ENV-10: {date}/{memory} substituted; missing news file leaves {news} literal.""" + config = {"system": "Date {date} memory {memory} news {news}", "history-limit": 5, "news": "/nonexistent/news.txt"} + responder = AIResponder(config, "chat") + responder.memory = "MEMSTR" + messages = responder.message(AIMessage("alice", "hei")) + system = messages[0]["content"] + self.assertIn(time.strftime("%Y-%m-%d"), system) + self.assertIn("MEMSTR", system) + self.assertIn("{news}", system) # left literal — file does not exist + + +class TestHistoryShrink(unittest.TestCase): + def test_shrink_prefers_busy_channel(self): + """ENV-11: entry from the channel exceeding history-per-channel is removed first.""" + config = {"system": "s", "history-limit": 4} + responder = AIResponder(config, "chat") + responder.history = [entry("a", "a0"), entry("a", "a1"), entry("a", "a2"), entry("a", "a3"), entry("b", "b0")] + responder.shrink_history_by_one() + self.assertEqual(len(responder.history), 4) + self.assertNotIn("a0", responder.history[0]["content"]) # oldest busy-channel entry gone + + def test_shrink_falls_back_to_oldest(self): + """ENV-11: when no channel exceeds the cap, the oldest entry overall is removed.""" + config = {"system": "s", "history-limit": 4} + responder = AIResponder(config, "chat") + responder.history = [entry("a", "a0"), entry("b", "b0"), entry("c", "c0")] + responder.shrink_history_by_one() + self.assertEqual(len(responder.history), 2) + self.assertNotIn("a0", responder.history[0]["content"]) + + +class TestRetriesExhausted(unittest.IsolatedAsyncioTestCase): + async def test_send_raises_after_three_failures(self): + """ENV-12: three model failures -> RuntimeError (revision pending in FDB-004/D1).""" + responder = FakeModelResponder({"system": "s", "history-limit": 5}, "chat") + # scripted list empty -> chat() returns None every attempt + with self.assertRaises(RuntimeError): + await responder.send(AIMessage("alice", "hei", "chat")) + self.assertEqual(responder.chat_calls, 3) diff --git a/tools/trace.py b/tools/trace.py new file mode 100644 index 0000000..cfe67aa --- /dev/null +++ b/tools/trace.py @@ -0,0 +1,86 @@ +#!/usr/bin/env python3 +"""trace.py — enforce spec requirement coverage (SPEC-000). + +Stdlib only. Collects declared requirement IDs from specs/SPEC-*.md +headers (code fences stripped), @ID tags from features/*.feature, ID +mentions from tests/**/*.py, and rows from manual-verification.md. +Fails when a declared requirement lacks its coverage artifact, when a +feature tag or manual row references an undeclared ID, or when an ID +is declared twice. +""" + +import re +import sys +from pathlib import Path + +ROOT = Path(__file__).resolve().parent.parent +ID_PATTERN = r"[A-Z]{2,8}-\d{2,3}" +HEADER_RE = re.compile(rf"^###\s+({ID_PATTERN})\s+[—–-]+\s+.*\(coverage:\s*(feature|test|manual|withdrawn[^)]*)\)", re.M) +FENCE_RE = re.compile(r"^```.*?^```", re.M | re.S) +TAG_RE = re.compile(rf"@({ID_PATTERN})\b") +MANUAL_ROW_RE = re.compile(rf"^\|\s*({ID_PATTERN})\s*\|", re.M) + + +def collect(): + declared = {} + errors = [] + for spec in sorted((ROOT / "specs").glob("SPEC-*.md")): + text = FENCE_RE.sub("", spec.read_text(encoding="utf-8")) + for req_id, coverage in HEADER_RE.findall(text): + if req_id in declared: + errors.append(f"{req_id}: declared twice") + declared[req_id] = coverage + + feature_tags = set() + features_dir = ROOT / "features" + if features_dir.exists(): + for feature in features_dir.glob("**/*.feature"): + feature_tags |= set(TAG_RE.findall(feature.read_text(encoding="utf-8"))) + + test_ids = set() + for test_file in (ROOT / "tests").glob("**/*.py"): + test_ids |= set(re.findall(ID_PATTERN, test_file.read_text(encoding="utf-8"))) + + manual_ids = set() + manual_file = ROOT / "manual-verification.md" + if manual_file.exists(): + manual_ids = set(MANUAL_ROW_RE.findall(manual_file.read_text(encoding="utf-8"))) + + return declared, feature_tags, test_ids, manual_ids, errors + + +def coverage_errors(declared, feature_tags, test_ids, manual_ids): + errors = [] + for req_id, coverage in sorted(declared.items()): + if "withdrawn" in coverage: + continue + if coverage == "feature" and req_id not in feature_tags: + errors.append(f"{req_id}: coverage 'feature' but no @{req_id} tag under features/") + elif coverage == "test" and req_id not in test_ids and req_id not in feature_tags: + errors.append(f"{req_id}: coverage 'test' but not mentioned under tests/ or features/") + elif coverage == "manual" and req_id not in manual_ids: + errors.append(f"{req_id}: coverage 'manual' but no row in manual-verification.md") + errors += [f"@{tag}: tagged under features/ but not declared in specs/" for tag in sorted(feature_tags - set(declared))] + errors += [f"{mid}: manual-verification.md row without spec declaration" for mid in sorted(manual_ids - set(declared))] + return errors + + +def main() -> int: + declared, feature_tags, test_ids, manual_ids, errors = collect() + errors += coverage_errors(declared, feature_tags, test_ids, manual_ids) + + if errors: + print("trace: FAIL") + for error in errors: + print(f" - {error}") + return 1 + + by_class = {c: sum(1 for v in declared.values() if v == c) for c in ("feature", "test", "manual")} + print( + f"trace: OK — {len(declared)} requirements ({by_class['feature']} feature / {by_class['test']} test / {by_class['manual']} manual)" + ) + return 0 + + +if __name__ == "__main__": + sys.exit(main())