From 41665209232e3f145fff1c0af70102765d3dbbe0 Mon Sep 17 00:00:00 2001 From: Oleksandr Kozachuk Date: Mon, 13 Jul 2026 18:25:01 +0200 Subject: [PATCH] ops: consistent rotated db backups + cron, consecutive-api-error staff alert --- Makefile | 3 + config.toml | 3 + deploy/backup_db.py | 80 +++++++++++++++++++++++++++ fjerkroa_bot/discord_bot.py | 20 ++++++- manual-verification.md | 1 + specs/SPEC-012-ops.md | 32 +++++++++++ tests/test_spec_ops_backup.py | 100 ++++++++++++++++++++++++++++++++++ 7 files changed, 237 insertions(+), 2 deletions(-) create mode 100644 deploy/backup_db.py create mode 100644 specs/SPEC-012-ops.md create mode 100644 tests/test_spec_ops_backup.py diff --git a/Makefile b/Makefile index 9503176..5c5e633 100644 --- a/Makefile +++ b/Makefile @@ -83,3 +83,6 @@ ci: install-dev all-checks ## Full CI pipeline (install deps and run all checks) # Deploy targets (SPEC-007) deploy: ## Deploy a tag to a host: make deploy HOST=ggg TAG=v3.0.0 bash deploy/deploy.sh $(HOST) $(TAG) + +backup: ## Back up a local bot.db: make backup DB=history/bot.db DIR=backups + uv run python deploy/backup_db.py $(DB) $(DIR) $(or $(KEEP),14) diff --git a/config.toml b/config.toml index fbade2e..2d45a12 100644 --- a/config.toml +++ b/config.toml @@ -73,3 +73,6 @@ enable-game-info = true # url-max-chars = 6000 # text handed to the model # url-max-images = 2 # page images into the vision cache # url-daily-per-user = 20 +# Ops (SPEC-012): consecutive OpenAI failures before a staff alert +# api-error-alert-threshold = 5 +# Backups: cron runs deploy/backup_db.py daily -> ~/backups// (keep 14) diff --git a/deploy/backup_db.py b/deploy/backup_db.py new file mode 100644 index 0000000..4658ef1 --- /dev/null +++ b/deploy/backup_db.py @@ -0,0 +1,80 @@ +#!/usr/bin/env python3 +"""Consistent, rotated bot.db backups (SPEC-012 OPS-13). + +Run from cron on each host. Uses the sqlite3 online-backup API so the +snapshot is consistent even while the bot writes (WAL-safe), gzips it, +and keeps the newest N. Stdlib only. + +Usage: python3 backup_db.py [keep] +""" + +import gzip +import os +import shutil +import sqlite3 +import sys +import tempfile +import time +from pathlib import Path + +DEFAULT_KEEP = 14 +BACKUP_GLOB = "bot-*.db.gz" + + +def snapshot(src: Path, dest_gz: Path) -> None: + """Write a consistent gzipped snapshot of src to dest_gz (OPS-13).""" + fd, tmp_path = tempfile.mkstemp(suffix=".db", dir=str(dest_gz.parent)) + os.close(fd) + tmp = Path(tmp_path) + try: + source = sqlite3.connect(str(src)) + try: + target = sqlite3.connect(str(tmp)) + try: + source.backup(target) # atomic, WAL-safe online backup + finally: + target.close() + finally: + source.close() + with open(tmp, "rb") as raw, gzip.open(str(dest_gz), "wb") as gz: + shutil.copyfileobj(raw, gz) + os.chmod(dest_gz, 0o600) # conversation data + finally: + tmp.unlink(missing_ok=True) + + +def victims(existing: list, keep: int) -> list: + """Given backup paths (any order), return the ones to delete, oldest first (OPS-14).""" + ordered = sorted(existing) # timestamped names sort chronologically + return ordered[: max(0, len(ordered) - keep)] + + +def rotate(backup_dir: Path, keep: int) -> int: + removed = 0 + for path in victims(list(backup_dir.glob(BACKUP_GLOB)), keep): + Path(path).unlink(missing_ok=True) + removed += 1 + return removed + + +def main() -> int: + if len(sys.argv) < 3: + print("usage: backup_db.py [keep]", file=sys.stderr) + return 2 + src = Path(sys.argv[1]).expanduser() + backup_dir = Path(sys.argv[2]).expanduser() + keep = int(sys.argv[3]) if len(sys.argv) > 3 else DEFAULT_KEEP + if not src.exists(): + print(f"backup: source {src} missing", file=sys.stderr) + return 1 + backup_dir.mkdir(parents=True, exist_ok=True) + stamp = time.strftime("%Y%m%d-%H%M%S", time.gmtime()) + dest = backup_dir / f"bot-{stamp}.db.gz" + snapshot(src, dest) + removed = rotate(backup_dir, keep) + print(f"backup: wrote {dest.name} ({dest.stat().st_size} bytes), rotated {removed} old") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/fjerkroa_bot/discord_bot.py b/fjerkroa_bot/discord_bot.py index baf240e..42a4def 100644 --- a/fjerkroa_bot/discord_bot.py +++ b/fjerkroa_bot/discord_bot.py @@ -89,6 +89,7 @@ class FjerkroaBot(commands.Bot): self.tasks_enabled = True self.quiet_until = 0.0 self._staff_alert_times: deque = deque() + self._consecutive_api_errors = 0 # OPS-16 self.init_observer() self.init_aichannels() @@ -498,6 +499,14 @@ class FjerkroaBot(commands.Bot): return True, False return False, bool(verdict.get("factual", False)) + async def _note_api_error(self, err: Exception) -> None: + """Count consecutive failures; alert staff once at threshold (OPS-16).""" + self._consecutive_api_errors += 1 + logging.warning(f"responder call failed ({self._consecutive_api_errors} in a row): {repr(err)}") + threshold = int(self.config.get("api-error-alert-threshold", 5)) + if self._consecutive_api_errors == threshold: + await self.send_staff_alert(f"⚠️ {threshold} consecutive API errors — the bot may be down. Last: {str(err)[:200]}") + async def send_message_with_typing(self, airesponder, channel, message): """Send the user message to the AI responder with typing animation in discord""" async with channel.typing(): @@ -603,8 +612,15 @@ class FjerkroaBot(commands.Bot): # Get the AI responder based on the channel name airesponder = self.get_ai_responder(channel_name) - # Send the user message to the AI responder, with typing indicators - response = await self.send_message_with_typing(airesponder, channel, message) + # Send the user message to the AI responder, with typing indicators. + # A raised call = a broken API path (cf. the gpt-5.6 tools incident): + # count it, alert staff at threshold, never crash the handler (OPS-16). + try: + response = await self.send_message_with_typing(airesponder, channel, message) + except Exception as err: + await self._note_api_error(err) + return + self._consecutive_api_errors = 0 # SAF/OPS gates between model proposal and delivery await self._apply_response_gates(message, response) diff --git a/manual-verification.md b/manual-verification.md index 5b90777..c971c58 100644 --- a/manual-verification.md +++ b/manual-verification.md @@ -12,3 +12,4 @@ with date + result. | DEP-04 | 2026-07-13 | Smoke gate exercised on ggg: RUNNING + fresh login line. | | DEP-05 | 2026-07-13 | Live-verified: kroa deploy attempt ~15h Oslo refused without DEPLOY_FORCE=1. | | DEP-06 | 2026-07-13 | Rollback documented (older tag + db backup restore); live drill pending — next release. | +| OPS-15 | 2026-07-13 | Backup cron installed on both hosts (daily 03:17 UTC → ~/backups//, keep 14); first snapshots written + verified 0600 (kroa 10965 B, luma 25871 B). | diff --git a/specs/SPEC-012-ops.md b/specs/SPEC-012-ops.md new file mode 100644 index 0000000..6c4991a --- /dev/null +++ b/specs/SPEC-012-ops.md @@ -0,0 +1,32 @@ +# SPEC-012 — Operations hardening + +Runtime + host operability (FDB-012). Backups and host wiring are +`manual` coverage; the in-process alerting is `test`. + +### OPS-13 — Consistent DB backups (coverage: test) + +`deploy/backup_db.py` writes a gzipped snapshot of `bot.db` using the +sqlite3 online-backup API — consistent even while the bot writes +(WAL-safe) — with 0600 permissions. Restoring a snapshot yields a +readable database with the same rows. + +### OPS-14 — Backups are rotated (coverage: test) + +The newest `backup-keep` (default 14) snapshots are kept; older ones +are deleted. Timestamped names sort chronologically so rotation is a +pure list operation. + +### OPS-15 — Backup cron on each host (coverage: manual) + +Each host runs `backup_db.py` daily via cron, writing to +`~/backups//` (outside `~/fjerkroa_bot`, so deploys and service +restarts never touch it). Verified by presence of the cron line and a +fresh snapshot. + +### OPS-16 — Repeated API errors alert staff (coverage: test) + +The responder counts consecutive OpenAI request failures; at +`api-error-alert-threshold` (default 5) in a row it fires one staff +alert (rate-limited like all staff alerts) so a silently-broken bot +(cf. the gpt-5.6 tools/reasoning incident) surfaces within minutes +instead of hours. A success resets the counter. diff --git a/tests/test_spec_ops_backup.py b/tests/test_spec_ops_backup.py new file mode 100644 index 0000000..5f76809 --- /dev/null +++ b/tests/test_spec_ops_backup.py @@ -0,0 +1,100 @@ +"""Unit coverage for SPEC-012 ops hardening (OPS-13/14/16).""" + +import gzip +import sqlite3 +import stat +import sys +import tempfile +import unittest +from pathlib import Path +from unittest.mock import AsyncMock, MagicMock + +from fjerkroa_bot.persistence import PersistentStore + +from .test_spec_ops import OpsBase + +sys.path.insert(0, str(Path(__file__).resolve().parent.parent / "deploy")) +import backup_db # noqa: E402 + + +class TestSnapshotConsistency(unittest.TestCase): + def test_snapshot_roundtrips(self): + """OPS-13: a gzipped snapshot restores to a readable DB with the same rows, 0600.""" + with tempfile.TemporaryDirectory() as tmp: + db = Path(tmp) / "bot.db" + store = PersistentStore(db) + store.save_history("chat", [{"role": "user", "content": "hei"}]) + store.add_user_fact("alice", "likes espresso", "self") + + dest = Path(tmp) / "snap.db.gz" + backup_db.snapshot(db, dest) + self.assertEqual(stat.S_IMODE(dest.stat().st_mode), 0o600) + + restored = Path(tmp) / "restored.db" + with gzip.open(dest, "rb") as gz, open(restored, "wb") as out: + out.write(gz.read()) + conn = sqlite3.connect(restored) + try: + rows = conn.execute("SELECT content FROM history WHERE channel='chat'").fetchall() + facts = conn.execute("SELECT fact FROM user_facts").fetchall() + finally: + conn.close() + self.assertEqual(rows, [("hei",)]) + self.assertEqual(facts, [("likes espresso",)]) + + def test_snapshot_during_writes(self): + """OPS-13: snapshot succeeds while another connection holds the DB open (WAL).""" + with tempfile.TemporaryDirectory() as tmp: + db = Path(tmp) / "bot.db" + store = PersistentStore(db) + store.save_history("chat", [{"role": "user", "content": "x"}]) + live = sqlite3.connect(db) # simulate the running bot's open handle + live.execute("PRAGMA journal_mode=WAL") + try: + dest = Path(tmp) / "snap.db.gz" + backup_db.snapshot(db, dest) # must not raise + self.assertTrue(dest.exists()) + finally: + live.close() + + +class TestRotation(unittest.TestCase): + def test_victims_keeps_newest(self): + """OPS-14: only the oldest beyond `keep` are selected for deletion.""" + names = [f"bot-2026070{d}-000000.db.gz" for d in range(1, 8)] # 7 chronological + victims = backup_db.victims(list(reversed(names)), keep=3) + self.assertEqual(victims, names[:4]) # oldest 4 removed, newest 3 kept + + def test_victims_under_keep_deletes_nothing(self): + """OPS-14: fewer than `keep` backups -> nothing deleted.""" + self.assertEqual(backup_db.victims(["bot-20260701-000000.db.gz"], keep=14), []) + + def test_rotate_on_disk(self): + """OPS-14: rotate removes the right files from a real dir.""" + with tempfile.TemporaryDirectory() as tmp: + for d in range(1, 6): + (Path(tmp) / f"bot-2026070{d}-000000.db.gz").write_bytes(b"x") + removed = backup_db.rotate(Path(tmp), keep=2) + self.assertEqual(removed, 3) + self.assertEqual(len(list(Path(tmp).glob("bot-*.db.gz"))), 2) + + +class TestApiErrorAlert(OpsBase): + async def test_threshold_alert_and_reset(self): + """OPS-16: N consecutive failures fire one staff alert; success resets.""" + self.bot.config["api-error-alert-threshold"] = 3 + self.bot.send_message_with_typing = AsyncMock(side_effect=RuntimeError("boom")) + origin = MagicMock() + from fjerkroa_bot.ai_responder import AIMessage + + for _ in range(3): + await self.bot.respond(AIMessage("alice", "hei", "chat"), origin) + self.assertEqual(self.bot.staff_channel.send.await_count, 1) # exactly one alert at threshold + self.assertEqual(self.bot._consecutive_api_errors, 3) + + # a success resets the counter + from fjerkroa_bot.ai_responder import AIResponse + + self.bot.send_message_with_typing = AsyncMock(return_value=AIResponse(None, False, "chat", None, None, False, False)) + await self.bot.respond(AIMessage("alice", "hei", "chat"), origin) + self.assertEqual(self.bot._consecutive_api_errors, 0)