structured memory: facts/pinned/episodes, batched consolidation, participant-scoped recall

2026-07-13 14:12:21 +02:00
parent 02c989946b
commit 3d2289496b
14 changed files with 608 additions and 90 deletions
@@ -13,7 +13,7 @@ from contextlib import closing
 from pathlib import Path
 from typing import Any, Dict, List, Optional

-SCHEMA_VERSION = 2
+SCHEMA_VERSION = 3


 class PersistentStore:
@@ -41,6 +41,25 @@ class PersistentStore:
                conn.execute(
                    "CREATE TABLE IF NOT EXISTS usage (day TEXT NOT NULL, key TEXT NOT NULL, value REAL NOT NULL, PRIMARY KEY (day, key))"
                )
+            if version < 3:
+                conn.execute(
+                    "CREATE TABLE IF NOT EXISTS user_facts (id INTEGER PRIMARY KEY, user TEXT NOT NULL, fact TEXT NOT NULL,"
+                    " source TEXT NOT NULL DEFAULT 'self', updated_at TEXT NOT NULL DEFAULT (datetime('now')))"
+                )
+                conn.execute(
+                    "CREATE TABLE IF NOT EXISTS pinned_facts (id INTEGER PRIMARY KEY, channel TEXT, fact TEXT NOT NULL,"
+                    " created_at TEXT NOT NULL DEFAULT (datetime('now')))"
+                )
+                conn.execute(
+                    "CREATE TABLE IF NOT EXISTS episodes (id INTEGER PRIMARY KEY, channel TEXT NOT NULL, summary TEXT NOT NULL,"
+                    " created_at TEXT NOT NULL DEFAULT (datetime('now')))"
+                )
+                conn.execute(
+                    "CREATE TABLE IF NOT EXISTS observations (id INTEGER PRIMARY KEY, channel TEXT NOT NULL, user TEXT NOT NULL,"
+                    " kind TEXT NOT NULL, content TEXT NOT NULL, created_at TEXT NOT NULL DEFAULT (datetime('now')))"
+                )
+                # Legacy single-string memories carry over as one episode each (MEM-08)
+                conn.execute("INSERT INTO episodes (channel, summary) SELECT channel, content FROM memory")
            if version < SCHEMA_VERSION:
                conn.execute(f"PRAGMA user_version = {SCHEMA_VERSION}")
        os.chmod(self.db_path, 0o600)  # conversation data (PER-04)
@@ -90,6 +109,90 @@ class PersistentStore:
            cursor = conn.execute("DELETE FROM history WHERE content LIKE ?", (f'%"user": "{user}"%',))
        return cursor.rowcount

+    # --- structured memory (SPEC-002) ---
+
+    def add_observation(self, channel: str, user: str, kind: str, content: str) -> None:
+        with closing(self._connect()) as conn, conn:
+            conn.execute("INSERT INTO observations (channel, user, kind, content) VALUES (?, ?, ?, ?)", (channel, user, kind, content))
+
+    def unconsumed_observations(self, channel: str) -> int:
+        with closing(self._connect()) as conn:
+            return int(conn.execute("SELECT COUNT(*) FROM observations WHERE channel = ?", (channel,)).fetchone()[0])
+
+    def peek_observations(self, channel: str) -> List[Dict[str, Any]]:
+        with closing(self._connect()) as conn:
+            rows = conn.execute("SELECT id, user, kind, content FROM observations WHERE channel = ? ORDER BY id", (channel,)).fetchall()
+        return [{"id": row[0], "user": row[1], "kind": row[2], "content": row[3]} for row in rows]
+
+    def consume_observations(self, channel: str, up_to_id: int) -> None:
+        with closing(self._connect()) as conn, conn:
+            conn.execute("DELETE FROM observations WHERE channel = ? AND id <= ?", (channel, up_to_id))
+
+    def add_user_fact(self, user: str, fact: str, source: str = "self") -> None:
+        with closing(self._connect()) as conn, conn:
+            conn.execute("INSERT INTO user_facts (user, fact, source) VALUES (?, ?, ?)", (user, fact, source))
+
+    def facts_for(self, users: List[str]) -> List[Dict[str, Any]]:
+        if not users:
+            return []
+        marks = ",".join("?" for _ in users)
+        with closing(self._connect()) as conn:
+            # marks is only "?" placeholders; user values stay parameterized
+            rows = conn.execute(
+                f"SELECT id, user, fact FROM user_facts WHERE user IN ({marks}) ORDER BY id", tuple(users)  # nosec B608
+            ).fetchall()
+        return [{"id": row[0], "user": row[1], "fact": row[2]} for row in rows]
+
+    def delete_fact(self, fact_id: int) -> int:
+        with closing(self._connect()) as conn, conn:
+            return conn.execute("DELETE FROM user_facts WHERE id = ?", (fact_id,)).rowcount
+
+    def purge_old_facts(self, retention_days: int) -> int:
+        with closing(self._connect()) as conn, conn:
+            cursor = conn.execute("DELETE FROM user_facts WHERE updated_at < datetime('now', ?)", (f"-{int(retention_days)} days",))
+        return cursor.rowcount
+
+    def add_pinned(self, channel: Optional[str], fact: str) -> None:
+        with closing(self._connect()) as conn, conn:
+            conn.execute("INSERT INTO pinned_facts (channel, fact) VALUES (?, ?)", (channel, fact))
+
+    def pinned_for(self, channel: str) -> List[Dict[str, Any]]:
+        with closing(self._connect()) as conn:
+            rows = conn.execute(
+                "SELECT id, channel, fact FROM pinned_facts WHERE channel IS NULL OR channel = ? ORDER BY id", (channel,)
+            ).fetchall()
+        return [{"id": row[0], "channel": row[1], "fact": row[2]} for row in rows]
+
+    def delete_pinned(self, pin_id: int) -> int:
+        with closing(self._connect()) as conn, conn:
+            return conn.execute("DELETE FROM pinned_facts WHERE id = ?", (pin_id,)).rowcount
+
+    def add_episode(self, channel: str, summary: str) -> None:
+        with closing(self._connect()) as conn, conn:
+            conn.execute("INSERT INTO episodes (channel, summary) VALUES (?, ?)", (channel, summary))
+
+    def recent_episodes(self, channel: str, count: int) -> List[str]:
+        with closing(self._connect()) as conn:
+            rows = conn.execute("SELECT summary FROM episodes WHERE channel = ? ORDER BY id DESC LIMIT ?", (channel, count)).fetchall()
+        return [row[0] for row in reversed(rows)]
+
+    def trim_episodes(self, channel: str, keep: int) -> int:
+        with closing(self._connect()) as conn, conn:
+            cursor = conn.execute(
+                "DELETE FROM episodes WHERE channel = ? AND id NOT IN (SELECT id FROM episodes WHERE channel = ? ORDER BY id DESC LIMIT ?)",
+                (channel, channel, keep),
+            )
+        return cursor.rowcount
+
+    def purge_user_memory(self, user: str) -> int:
+        """Facts, observations and episode traces of one user (MEM-09)."""
+        removed = 0
+        with closing(self._connect()) as conn, conn:
+            removed += conn.execute("DELETE FROM user_facts WHERE user = ?", (user,)).rowcount
+            removed += conn.execute("DELETE FROM observations WHERE user = ?", (user,)).rowcount
+            removed += conn.execute("DELETE FROM episodes WHERE summary LIKE ?", (f"%{user}%",)).rowcount
+        return removed
+
    def migrate_pickles(self, channel: str, history_file: Path, memory_file: Path) -> None:
        """Import legacy pickles once; rename them *.migrated (PER-03)."""
        if self.load_history(channel) or self.load_memory(channel) is not None:
@@ -105,7 +208,9 @@ class PersistentStore:
        if memory_file.exists():
            try:
                with open(memory_file, "rb") as fd:
-                    self.save_memory(channel, str(pickle.load(fd)))
+                    legacy_memory = str(pickle.load(fd))
+                self.save_memory(channel, legacy_memory)
+                self.add_episode(channel, legacy_memory)  # MEM-08
                memory_file.rename(memory_file.with_name(memory_file.name + ".migrated"))
                logging.info(f"migrated legacy memory pickle for {channel}")
            except Exception as err: