From 4a60171035ac08365691e659fe007c3569c2e9ce Mon Sep 17 00:00:00 2001 From: Joseph Doherty Date: Sun, 26 Apr 2026 11:23:38 -0400 Subject: [PATCH 01/45] feat: project skeleton with health endpoint --- .gitignore | 6 ++++++ .python-version | 1 + chat/__init__.py | 0 chat/app.py | 8 ++++++++ pyproject.toml | 23 +++++++++++++++++++++++ tests/__init__.py | 0 tests/test_health.py | 9 +++++++++ 7 files changed, 47 insertions(+) create mode 100644 .python-version create mode 100644 chat/__init__.py create mode 100644 chat/app.py create mode 100644 pyproject.toml create mode 100644 tests/__init__.py create mode 100644 tests/test_health.py diff --git a/.gitignore b/.gitignore index 92172f9..6bf8f70 100644 --- a/.gitignore +++ b/.gitignore @@ -2,3 +2,9 @@ # v1 runtime data (DB, backups, snapshots, exports, config with secrets) data/ + +# Python +.venv/ +__pycache__/ +*.pyc +.pytest_cache/ diff --git a/.python-version b/.python-version new file mode 100644 index 0000000..e4fba21 --- /dev/null +++ b/.python-version @@ -0,0 +1 @@ +3.12 diff --git a/chat/__init__.py b/chat/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/chat/app.py b/chat/app.py new file mode 100644 index 0000000..66990d8 --- /dev/null +++ b/chat/app.py @@ -0,0 +1,8 @@ +from fastapi import FastAPI + +app = FastAPI(title="chat") + + +@app.get("/health") +def health(): + return {"status": "ok"} diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..c0593a9 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,23 @@ +[project] +name = "chat" +version = "0.1.0" +requires-python = ">=3.11" +dependencies = [ + "fastapi>=0.110", + "uvicorn[standard]>=0.30", + "httpx>=0.27", + "pydantic>=2.6", + "pydantic-settings>=2.2", + "openai>=1.30", + "instructor>=1.3", + "tiktoken>=0.7", + "jinja2>=3.1", + "aiosqlite>=0.20", +] + +[project.optional-dependencies] +dev = ["pytest>=8", "pytest-asyncio>=0.23", "freezegun>=1.4"] + +[tool.pytest.ini_options] +pythonpath = ["."] +asyncio_mode = "auto" diff --git a/tests/__init__.py b/tests/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/test_health.py b/tests/test_health.py new file mode 100644 index 0000000..f44ad32 --- /dev/null +++ b/tests/test_health.py @@ -0,0 +1,9 @@ +from fastapi.testclient import TestClient +from chat.app import app + + +def test_health_endpoint_returns_ok(): + client = TestClient(app) + response = client.get("/health") + assert response.status_code == 200 + assert response.json() == {"status": "ok"} -- 2.52.0 From 01e6975d20c7cae035a3b7a6a868b03a0149d9b7 Mon Sep 17 00:00:00 2001 From: Joseph Doherty Date: Sun, 26 Apr 2026 11:28:40 -0400 Subject: [PATCH 02/45] feat: config loader with toml + env override --- chat/config.py | 40 ++++++++++++++++++++++++++++++++++++++++ data/config.example.toml | 6 ++++++ tests/test_config.py | 26 ++++++++++++++++++++++++++ 3 files changed, 72 insertions(+) create mode 100644 chat/config.py create mode 100644 data/config.example.toml create mode 100644 tests/test_config.py diff --git a/chat/config.py b/chat/config.py new file mode 100644 index 0000000..90d6175 --- /dev/null +++ b/chat/config.py @@ -0,0 +1,40 @@ +from __future__ import annotations +import os +import tomllib +from pathlib import Path +from pydantic import BaseModel, Field + +REPO_ROOT = Path(__file__).resolve().parent.parent +DEFAULT_CONFIG = REPO_ROOT / "data" / "config.toml" +DEFAULT_DB = REPO_ROOT / "data" / "chat.db" + +class Settings(BaseModel): + featherless_api_key: str + featherless_base_url: str = "https://api.featherless.ai/v1" + narrative_model: str = "dphn/Dolphin-Mistral-24B-Venice-Edition" + classifier_model: str = "NousResearch/Hermes-3-Llama-3.1-8B" + classifier_fallbacks: list[str] = Field( + default_factory=lambda: [ + "cognitivecomputations/dolphin-2.9.4-llama3-8b", + "mlabonne/Meta-Llama-3.1-8B-Instruct-abliterated", + ] + ) + ooc_marker: str = "((" + retrieval_k: int = 4 + narrative_budget_hard: int = 8000 + narrative_budget_soft: int = 6000 + classifier_budget_hard: int = 4000 + classifier_timeout_s: float = 10.0 + db_path: Path = DEFAULT_DB + data_dir: Path = REPO_ROOT / "data" + bind_host: str = "127.0.0.1" + bind_port: int = 8000 + +def load_settings() -> Settings: + config_path = Path(os.environ.get("CHAT_CONFIG_PATH", DEFAULT_CONFIG)) + raw: dict = {} + if config_path.exists(): + raw = tomllib.loads(config_path.read_text()) + if "CHAT_DB_PATH" in os.environ: + raw["db_path"] = Path(os.environ["CHAT_DB_PATH"]) + return Settings(**raw) diff --git a/data/config.example.toml b/data/config.example.toml new file mode 100644 index 0000000..1da91f2 --- /dev/null +++ b/data/config.example.toml @@ -0,0 +1,6 @@ +# Copy this file to data/config.toml and fill in your API key. +featherless_api_key = "REPLACE_ME" +narrative_model = "dphn/Dolphin-Mistral-24B-Venice-Edition" +classifier_model = "NousResearch/Hermes-3-Llama-3.1-8B" +ooc_marker = "((" +retrieval_k = 4 diff --git a/tests/test_config.py b/tests/test_config.py new file mode 100644 index 0000000..abffd57 --- /dev/null +++ b/tests/test_config.py @@ -0,0 +1,26 @@ +import os +from pathlib import Path +import pytest +from chat.config import load_settings + +def test_load_settings_reads_toml(tmp_path, monkeypatch): + cfg = tmp_path / "config.toml" + cfg.write_text(""" + featherless_api_key = "sk-test" + narrative_model = "dphn/Dolphin-Mistral-24B-Venice-Edition" + classifier_model = "NousResearch/Hermes-3-Llama-3.1-8B" + ooc_marker = "((" + retrieval_k = 4 + """) + monkeypatch.setenv("CHAT_CONFIG_PATH", str(cfg)) + s = load_settings() + assert s.featherless_api_key == "sk-test" + assert s.narrative_model.startswith("dphn/") + assert s.retrieval_k == 4 + +def test_chat_db_path_env_overrides_default(tmp_path, monkeypatch): + monkeypatch.setenv("CHAT_DB_PATH", str(tmp_path / "alt.db")) + monkeypatch.setenv("CHAT_CONFIG_PATH", str(tmp_path / "config.toml")) + (tmp_path / "config.toml").write_text('featherless_api_key = "x"\n') + s = load_settings() + assert s.db_path == tmp_path / "alt.db" -- 2.52.0 From 67517926aa65c5986aebdffdf2c4466e4d5a5f8f Mon Sep 17 00:00:00 2001 From: Joseph Doherty Date: Sun, 26 Apr 2026 11:32:32 -0400 Subject: [PATCH 03/45] feat: sqlite migration runner with meta version table --- chat/db/__init__.py | 0 chat/db/connection.py | 17 +++++++++++++++++ chat/db/migrate.py | 26 ++++++++++++++++++++++++++ chat/db/migrations/0001_init_meta.sql | 2 ++ tests/test_migrate.py | 22 ++++++++++++++++++++++ 5 files changed, 67 insertions(+) create mode 100644 chat/db/__init__.py create mode 100644 chat/db/connection.py create mode 100644 chat/db/migrate.py create mode 100644 chat/db/migrations/0001_init_meta.sql create mode 100644 tests/test_migrate.py diff --git a/chat/db/__init__.py b/chat/db/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/chat/db/connection.py b/chat/db/connection.py new file mode 100644 index 0000000..ad21a01 --- /dev/null +++ b/chat/db/connection.py @@ -0,0 +1,17 @@ +from __future__ import annotations +import sqlite3 +from contextlib import contextmanager +from pathlib import Path + + +@contextmanager +def open_db(path: Path): + path.parent.mkdir(parents=True, exist_ok=True) + conn = sqlite3.connect(path) + conn.execute("PRAGMA journal_mode=WAL") + conn.execute("PRAGMA foreign_keys=ON") + try: + yield conn + conn.commit() + finally: + conn.close() diff --git a/chat/db/migrate.py b/chat/db/migrate.py new file mode 100644 index 0000000..95d9ba3 --- /dev/null +++ b/chat/db/migrate.py @@ -0,0 +1,26 @@ +from __future__ import annotations +from pathlib import Path + +from chat.db.connection import open_db + +MIGRATIONS_DIR = Path(__file__).parent / "migrations" + + +def apply_migrations(db_path: Path) -> None: + with open_db(db_path) as conn: + conn.execute( + "CREATE TABLE IF NOT EXISTS meta (key TEXT PRIMARY KEY, value TEXT)" + ) + cur = conn.execute("SELECT value FROM meta WHERE key = 'schema_version'") + row = cur.fetchone() + current = int(row[0]) if row else 0 + for path in sorted(MIGRATIONS_DIR.glob("*.sql")): + version = int(path.stem.split("_", 1)[0]) + if version <= current: + continue + sql = path.read_text() + conn.executescript(sql) + conn.execute( + "INSERT OR REPLACE INTO meta (key, value) VALUES ('schema_version', ?)", + (str(version),), + ) diff --git a/chat/db/migrations/0001_init_meta.sql b/chat/db/migrations/0001_init_meta.sql new file mode 100644 index 0000000..c0020e4 --- /dev/null +++ b/chat/db/migrations/0001_init_meta.sql @@ -0,0 +1,2 @@ +-- meta table is created by the migrate runner; this migration is a marker. +SELECT 1; diff --git a/tests/test_migrate.py b/tests/test_migrate.py new file mode 100644 index 0000000..eaafcd3 --- /dev/null +++ b/tests/test_migrate.py @@ -0,0 +1,22 @@ +from chat.db.connection import open_db +from chat.db.migrate import apply_migrations + + +def test_apply_migrations_creates_meta_table(tmp_path): + db = tmp_path / "test.db" + apply_migrations(db) + with open_db(db) as conn: + row = conn.execute( + "SELECT value FROM meta WHERE key = 'schema_version'" + ).fetchone() + assert row is not None + assert int(row[0]) >= 1 + + +def test_apply_migrations_idempotent(tmp_path): + db = tmp_path / "test.db" + apply_migrations(db) + apply_migrations(db) # second call must be a no-op + with open_db(db) as conn: + count = conn.execute("SELECT COUNT(*) FROM meta").fetchone()[0] + assert count == 1 -- 2.52.0 From e627356168e1cf3119fc71d6d68832b8de78d5bc Mon Sep 17 00:00:00 2001 From: Joseph Doherty Date: Sun, 26 Apr 2026 11:35:57 -0400 Subject: [PATCH 04/45] feat: LLMClient protocol with Featherless and mock implementations --- chat/llm/__init__.py | 0 chat/llm/client.py | 14 ++++++++++++++ chat/llm/featherless.py | 29 +++++++++++++++++++++++++++++ chat/llm/mock.py | 16 ++++++++++++++++ tests/test_llm_mock.py | 21 +++++++++++++++++++++ 5 files changed, 80 insertions(+) create mode 100644 chat/llm/__init__.py create mode 100644 chat/llm/client.py create mode 100644 chat/llm/featherless.py create mode 100644 chat/llm/mock.py create mode 100644 tests/test_llm_mock.py diff --git a/chat/llm/__init__.py b/chat/llm/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/chat/llm/client.py b/chat/llm/client.py new file mode 100644 index 0000000..ca34a2d --- /dev/null +++ b/chat/llm/client.py @@ -0,0 +1,14 @@ +from __future__ import annotations +from dataclasses import dataclass +from typing import Protocol, AsyncIterator, Sequence + + +@dataclass +class Message: + role: str # "system" | "user" | "assistant" + content: str + + +class LLMClient(Protocol): + async def generate(self, messages: Sequence[Message], *, model: str, **params) -> str: ... + def stream(self, messages: Sequence[Message], *, model: str, **params) -> AsyncIterator[str]: ... diff --git a/chat/llm/featherless.py b/chat/llm/featherless.py new file mode 100644 index 0000000..3e1fbcf --- /dev/null +++ b/chat/llm/featherless.py @@ -0,0 +1,29 @@ +from __future__ import annotations +from typing import AsyncIterator, Sequence +from openai import AsyncOpenAI +from .client import Message + + +class FeatherlessClient: + def __init__(self, api_key: str, base_url: str = "https://api.featherless.ai/v1"): + self._client = AsyncOpenAI(api_key=api_key, base_url=base_url) + + async def generate(self, messages: Sequence[Message], *, model: str, **params) -> str: + resp = await self._client.chat.completions.create( + model=model, + messages=[{"role": m.role, "content": m.content} for m in messages], + **params, + ) + return resp.choices[0].message.content or "" + + async def stream(self, messages: Sequence[Message], *, model: str, **params) -> AsyncIterator[str]: + stream = await self._client.chat.completions.create( + model=model, + messages=[{"role": m.role, "content": m.content} for m in messages], + stream=True, + **params, + ) + async for chunk in stream: + delta = chunk.choices[0].delta.content or "" + if delta: + yield delta diff --git a/chat/llm/mock.py b/chat/llm/mock.py new file mode 100644 index 0000000..75ab786 --- /dev/null +++ b/chat/llm/mock.py @@ -0,0 +1,16 @@ +from __future__ import annotations +from typing import AsyncIterator, Sequence +from .client import Message + + +class MockLLMClient: + def __init__(self, canned: list[str]): + self._canned = list(canned) + + async def generate(self, messages: Sequence[Message], *, model: str, **params) -> str: + return self._canned.pop(0) + + async def stream(self, messages: Sequence[Message], *, model: str, **params) -> AsyncIterator[str]: + text = self._canned.pop(0) + for ch in text: + yield ch diff --git a/tests/test_llm_mock.py b/tests/test_llm_mock.py new file mode 100644 index 0000000..d56a783 --- /dev/null +++ b/tests/test_llm_mock.py @@ -0,0 +1,21 @@ +import pytest +from chat.llm.mock import MockLLMClient +from chat.llm.client import Message + + +@pytest.mark.asyncio +async def test_mock_returns_canned_response(): + client = MockLLMClient(canned=["Hello, world."]) + msgs = [Message(role="user", content="hi")] + out = await client.generate(msgs, model="any") + assert out == "Hello, world." + + +@pytest.mark.asyncio +async def test_mock_streams_tokens(): + client = MockLLMClient(canned=["abcd"]) + msgs = [Message(role="user", content="hi")] + chunks = [] + async for chunk in client.stream(msgs, model="any"): + chunks.append(chunk) + assert "".join(chunks) == "abcd" -- 2.52.0 From c2aceffda1daa198707857100ea12054872f72b3 Mon Sep 17 00:00:00 2001 From: Joseph Doherty Date: Sun, 26 Apr 2026 11:38:48 -0400 Subject: [PATCH 05/45] feat: classifier wrapper with retry, timeout, schema-default fallback --- .../migrations/0002_classifier_failures.sql | 8 ++++ chat/llm/classify.py | 41 +++++++++++++++++++ tests/test_classify.py | 24 +++++++++++ 3 files changed, 73 insertions(+) create mode 100644 chat/db/migrations/0002_classifier_failures.sql create mode 100644 chat/llm/classify.py create mode 100644 tests/test_classify.py diff --git a/chat/db/migrations/0002_classifier_failures.sql b/chat/db/migrations/0002_classifier_failures.sql new file mode 100644 index 0000000..7bfa23a --- /dev/null +++ b/chat/db/migrations/0002_classifier_failures.sql @@ -0,0 +1,8 @@ +CREATE TABLE classifier_failures ( + id INTEGER PRIMARY KEY, + kind TEXT NOT NULL, + model TEXT NOT NULL, + raw_text TEXT, + attempt_count INTEGER NOT NULL, + created_at TEXT NOT NULL DEFAULT (datetime('now')) +); diff --git a/chat/llm/classify.py b/chat/llm/classify.py new file mode 100644 index 0000000..66517c6 --- /dev/null +++ b/chat/llm/classify.py @@ -0,0 +1,41 @@ +from __future__ import annotations +import json +import asyncio +from typing import TypeVar +from pydantic import BaseModel, ValidationError +from .client import LLMClient, Message + +T = TypeVar("T", bound=BaseModel) + +REFUSAL_PATTERNS = ("i can't", "i cannot", "i'm sorry, but", "as an ai") + + +async def classify( + client: LLMClient, + *, + model: str, + system: str, + user: str, + schema: type[T], + default: T | None = None, + timeout_s: float = 10.0, +) -> T: + msgs = [ + Message(role="system", content=system + "\n\nRespond with JSON only matching the schema."), + Message(role="user", content=user), + ] + for attempt in range(2): + try: + text = await asyncio.wait_for( + client.generate(msgs, model=model, response_format={"type": "json_object"}), + timeout=timeout_s, + ) + if any(p in text.lower()[:80] for p in REFUSAL_PATTERNS) and not text.strip().startswith("{"): + raise ValueError("refusal-shaped response") + return schema.model_validate_json(text) + except (ValidationError, ValueError, json.JSONDecodeError, asyncio.TimeoutError): + msgs[0] = Message(role="system", content=system + "\n\nRespond with valid JSON ONLY. No prose.") + continue + if default is None: + raise RuntimeError(f"classify failed for schema {schema.__name__} with no default") + return default diff --git a/tests/test_classify.py b/tests/test_classify.py new file mode 100644 index 0000000..d059eaa --- /dev/null +++ b/tests/test_classify.py @@ -0,0 +1,24 @@ +import pytest +from pydantic import BaseModel +from chat.llm.mock import MockLLMClient +from chat.llm.classify import classify + + +class Verdict(BaseModel): + score: int + reason: str + + +@pytest.mark.asyncio +async def test_classify_parses_valid_json(): + mock = MockLLMClient(canned=['{"score": 2, "reason": "notable"}']) + result = await classify(mock, model="m", system="x", user="y", schema=Verdict) + assert result.score == 2 + + +@pytest.mark.asyncio +async def test_classify_falls_back_on_unparseable_after_retry(): + mock = MockLLMClient(canned=["nope", "still nope"]) + default = Verdict(score=1, reason="fallback") + result = await classify(mock, model="m", system="x", user="y", schema=Verdict, default=default) + assert result.reason == "fallback" -- 2.52.0 From 517fe49aefb26e2f823a40a11a04c6dadb466a0f Mon Sep 17 00:00:00 2001 From: Joseph Doherty Date: Sun, 26 Apr 2026 11:42:49 -0400 Subject: [PATCH 06/45] feat: append-only event log with projector skeleton --- chat/db/migrations/0003_event_log.sql | 10 +++++++ chat/eventlog/__init__.py | 0 chat/eventlog/log.py | 38 +++++++++++++++++++++++++++ chat/eventlog/projector.py | 27 +++++++++++++++++++ tests/test_eventlog.py | 15 +++++++++++ 5 files changed, 90 insertions(+) create mode 100644 chat/db/migrations/0003_event_log.sql create mode 100644 chat/eventlog/__init__.py create mode 100644 chat/eventlog/log.py create mode 100644 chat/eventlog/projector.py create mode 100644 tests/test_eventlog.py diff --git a/chat/db/migrations/0003_event_log.sql b/chat/db/migrations/0003_event_log.sql new file mode 100644 index 0000000..2b4c85e --- /dev/null +++ b/chat/db/migrations/0003_event_log.sql @@ -0,0 +1,10 @@ +CREATE TABLE event_log ( + id INTEGER PRIMARY KEY, + branch_id INTEGER NOT NULL DEFAULT 1, + ts TEXT NOT NULL DEFAULT (datetime('now')), + kind TEXT NOT NULL, + payload_json TEXT NOT NULL, + superseded_by INTEGER REFERENCES event_log(id), + hidden INTEGER NOT NULL DEFAULT 0 +); +CREATE INDEX idx_event_log_branch_kind ON event_log(branch_id, kind); diff --git a/chat/eventlog/__init__.py b/chat/eventlog/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/chat/eventlog/log.py b/chat/eventlog/log.py new file mode 100644 index 0000000..74919fb --- /dev/null +++ b/chat/eventlog/log.py @@ -0,0 +1,38 @@ +from __future__ import annotations +import json +from dataclasses import dataclass +from typing import Any, Iterator +from sqlite3 import Connection + + +@dataclass +class Event: + id: int + branch_id: int + ts: str + kind: str + payload: dict[str, Any] + superseded_by: int | None + hidden: bool + + +def append_event(conn: Connection, *, kind: str, payload: dict[str, Any], branch_id: int = 1) -> int: + cur = conn.execute( + "INSERT INTO event_log (branch_id, kind, payload_json) VALUES (?, ?, ?)", + (branch_id, kind, json.dumps(payload)), + ) + return cur.lastrowid + + +def read_events(conn: Connection, branch_id: int = 1, after_id: int = 0) -> Iterator[Event]: + cur = conn.execute( + "SELECT id, branch_id, ts, kind, payload_json, superseded_by, hidden " + "FROM event_log WHERE branch_id = ? AND id > ? AND hidden = 0 " + "AND superseded_by IS NULL ORDER BY id", + (branch_id, after_id), + ) + for row in cur: + yield Event( + id=row[0], branch_id=row[1], ts=row[2], kind=row[3], + payload=json.loads(row[4]), superseded_by=row[5], hidden=bool(row[6]), + ) diff --git a/chat/eventlog/projector.py b/chat/eventlog/projector.py new file mode 100644 index 0000000..fba406c --- /dev/null +++ b/chat/eventlog/projector.py @@ -0,0 +1,27 @@ +from __future__ import annotations +from collections.abc import Callable +from sqlite3 import Connection +from .log import Event, read_events + +Handler = Callable[[Connection, Event], None] +_REGISTRY: dict[str, Handler] = {} + + +def on(kind: str): + def deco(fn: Handler) -> Handler: + _REGISTRY[kind] = fn + return fn + return deco + + +def project(conn: Connection, branch_id: int = 1) -> None: + for event in read_events(conn, branch_id=branch_id): + h = _REGISTRY.get(event.kind) + if h: + h(conn, event) + + +def apply_event(conn: Connection, event: Event) -> None: + h = _REGISTRY.get(event.kind) + if h: + h(conn, event) diff --git a/tests/test_eventlog.py b/tests/test_eventlog.py new file mode 100644 index 0000000..c7c129f --- /dev/null +++ b/tests/test_eventlog.py @@ -0,0 +1,15 @@ +from chat.db.migrate import apply_migrations +from chat.db.connection import open_db +from chat.eventlog.log import append_event, read_events + + +def test_append_and_read(tmp_path): + db = tmp_path / "t.db" + apply_migrations(db) + with open_db(db) as conn: + eid = append_event(conn, kind="test_kind", payload={"a": 1}) + assert eid > 0 + rows = list(read_events(conn)) + assert len(rows) == 1 + assert rows[0].kind == "test_kind" + assert rows[0].payload["a"] == 1 -- 2.52.0 From 5e6bbb586c80e1d3d94c0bedaf29c3ea2fe95d28 Mon Sep 17 00:00:00 2001 From: Joseph Doherty Date: Sun, 26 Apr 2026 11:46:19 -0400 Subject: [PATCH 07/45] feat: bot and you entity schemas with projector handlers --- chat/db/migrations/0004_entities.sql | 18 ++++++++++ chat/state/__init__.py | 0 chat/state/entities.py | 54 ++++++++++++++++++++++++++++ tests/test_entities.py | 38 ++++++++++++++++++++ 4 files changed, 110 insertions(+) create mode 100644 chat/db/migrations/0004_entities.sql create mode 100644 chat/state/__init__.py create mode 100644 chat/state/entities.py create mode 100644 tests/test_entities.py diff --git a/chat/db/migrations/0004_entities.sql b/chat/db/migrations/0004_entities.sql new file mode 100644 index 0000000..ef7b9f3 --- /dev/null +++ b/chat/db/migrations/0004_entities.sql @@ -0,0 +1,18 @@ +CREATE TABLE bots ( + id TEXT PRIMARY KEY, + name TEXT NOT NULL, + persona TEXT NOT NULL, + voice_samples_json TEXT NOT NULL DEFAULT '[]', + traits_json TEXT NOT NULL DEFAULT '[]', + backstory TEXT NOT NULL DEFAULT '', + initial_relationship_to_you TEXT NOT NULL DEFAULT '', + kickoff_prose TEXT NOT NULL DEFAULT '', + created_at TEXT NOT NULL DEFAULT (datetime('now')) +); + +CREATE TABLE you_entity ( + id INTEGER PRIMARY KEY CHECK (id = 1), + name TEXT NOT NULL, + pronouns TEXT NOT NULL DEFAULT '', + persona TEXT NOT NULL DEFAULT '' +); diff --git a/chat/state/__init__.py b/chat/state/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/chat/state/entities.py b/chat/state/entities.py new file mode 100644 index 0000000..57d0cc3 --- /dev/null +++ b/chat/state/entities.py @@ -0,0 +1,54 @@ +from __future__ import annotations +import json +from sqlite3 import Connection +from chat.eventlog.projector import on +from chat.eventlog.log import Event + + +@on("bot_authored") +def _apply_bot_authored(conn: Connection, e: Event) -> None: + p = e.payload + conn.execute( + "INSERT OR REPLACE INTO bots " + "(id, name, persona, voice_samples_json, traits_json, backstory, " + " initial_relationship_to_you, kickoff_prose) " + "VALUES (?, ?, ?, ?, ?, ?, ?, ?)", + (p["id"], p["name"], p["persona"], + json.dumps(p.get("voice_samples", [])), + json.dumps(p.get("traits", [])), + p.get("backstory", ""), + p.get("initial_relationship_to_you", ""), + p.get("kickoff_prose", "")), + ) + + +@on("you_authored") +def _apply_you_authored(conn: Connection, e: Event) -> None: + p = e.payload + conn.execute( + "INSERT OR REPLACE INTO you_entity (id, name, pronouns, persona) VALUES (1, ?, ?, ?)", + (p["name"], p.get("pronouns", ""), p.get("persona", "")), + ) + + +def get_bot(conn: Connection, bot_id: str) -> dict | None: + row = conn.execute("SELECT * FROM bots WHERE id = ?", (bot_id,)).fetchone() + if not row: + return None + cols = [c[1] for c in conn.execute("PRAGMA table_info(bots)").fetchall()] + d = dict(zip(cols, row)) + d["voice_samples"] = json.loads(d.pop("voice_samples_json")) + d["traits"] = json.loads(d.pop("traits_json")) + return d + + +def list_bots(conn: Connection) -> list[dict]: + cur = conn.execute("SELECT id, name FROM bots ORDER BY name") + return [{"id": r[0], "name": r[1]} for r in cur] + + +def get_you(conn: Connection) -> dict | None: + row = conn.execute("SELECT name, pronouns, persona FROM you_entity WHERE id = 1").fetchone() + if not row: + return None + return {"name": row[0], "pronouns": row[1], "persona": row[2]} diff --git a/tests/test_entities.py b/tests/test_entities.py new file mode 100644 index 0000000..5ef6f17 --- /dev/null +++ b/tests/test_entities.py @@ -0,0 +1,38 @@ +from chat.db.migrate import apply_migrations +from chat.db.connection import open_db +from chat.eventlog.log import append_event +from chat.eventlog.projector import project +from chat.state.entities import get_bot, list_bots, get_you +import chat.state.entities # registers handlers + + +def test_bot_authored_creates_bot_row(tmp_path): + db = tmp_path / "t.db" + apply_migrations(db) + with open_db(db) as conn: + append_event(conn, kind="bot_authored", payload={ + "id": "bot_a", "name": "BotA", + "persona": "...", "voice_samples": ["sample"], "traits": ["shy"], + "backstory": "...", + "initial_relationship_to_you": "coworker", + "kickoff_prose": "you stay late", + }) + project(conn) + bot = get_bot(conn, "bot_a") + assert bot is not None + assert bot["name"] == "BotA" + assert bot["traits"] == ["shy"] + assert "bot_a" in [b["id"] for b in list_bots(conn)] + + +def test_you_authored_creates_you_singleton(tmp_path): + db = tmp_path / "t.db" + apply_migrations(db) + with open_db(db) as conn: + append_event(conn, kind="you_authored", payload={ + "name": "Me", "pronouns": "they/them", "persona": "engineer", + }) + project(conn) + you = get_you(conn) + assert you is not None + assert you["name"] == "Me" -- 2.52.0 From 7e6c2985dd817fd7c6393d8df9fd02826546cb3e Mon Sep 17 00:00:00 2001 From: Joseph Doherty Date: Sun, 26 Apr 2026 11:48:30 -0400 Subject: [PATCH 08/45] docs: fix Task 6 plan snippet: PRAGMA table_info name index is c[1] not c[0] --- docs/plans/2026-04-26-v1-phase1-implementation.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/plans/2026-04-26-v1-phase1-implementation.md b/docs/plans/2026-04-26-v1-phase1-implementation.md index 1a1ce16..0ad593d 100644 --- a/docs/plans/2026-04-26-v1-phase1-implementation.md +++ b/docs/plans/2026-04-26-v1-phase1-implementation.md @@ -884,7 +884,7 @@ def get_bot(conn: Connection, bot_id: str) -> dict | None: row = conn.execute("SELECT * FROM bots WHERE id = ?", (bot_id,)).fetchone() if not row: return None - cols = [c[0] for c in conn.execute("PRAGMA table_info(bots)").fetchall()] + cols = [c[1] for c in conn.execute("PRAGMA table_info(bots)").fetchall()] d = dict(zip(cols, row)) d["voice_samples"] = json.loads(d.pop("voice_samples_json")) d["traits"] = json.loads(d.pop("traits_json")) -- 2.52.0 From bc97d425ef267311162788c67a34f18006cc39b7 Mon Sep 17 00:00:00 2001 From: Joseph Doherty Date: Sun, 26 Apr 2026 11:51:15 -0400 Subject: [PATCH 09/45] feat: directed edges with per-turn delta projector --- chat/db/migrations/0005_edges.sql | 13 +++ chat/state/edges.py | 84 ++++++++++++++++ tests/test_edges.py | 162 ++++++++++++++++++++++++++++++ 3 files changed, 259 insertions(+) create mode 100644 chat/db/migrations/0005_edges.sql create mode 100644 chat/state/edges.py create mode 100644 tests/test_edges.py diff --git a/chat/db/migrations/0005_edges.sql b/chat/db/migrations/0005_edges.sql new file mode 100644 index 0000000..4a0969e --- /dev/null +++ b/chat/db/migrations/0005_edges.sql @@ -0,0 +1,13 @@ +CREATE TABLE edges ( + id INTEGER PRIMARY KEY, + chat_id TEXT, + source_id TEXT NOT NULL, + target_id TEXT NOT NULL, + affinity INTEGER NOT NULL DEFAULT 50, + trust INTEGER NOT NULL DEFAULT 50, + summary TEXT NOT NULL DEFAULT '', + knowledge_json TEXT NOT NULL DEFAULT '[]', + last_interaction_chat_id TEXT, + last_interaction_at TEXT, + UNIQUE (source_id, target_id) +); diff --git a/chat/state/edges.py b/chat/state/edges.py new file mode 100644 index 0000000..03be66b --- /dev/null +++ b/chat/state/edges.py @@ -0,0 +1,84 @@ +from __future__ import annotations +import json +from sqlite3 import Connection +from chat.eventlog.projector import on +from chat.eventlog.log import Event + + +def _clamp(value: int, lo: int = 0, hi: int = 100) -> int: + return max(lo, min(hi, value)) + + +@on("edge_update") +def _apply_edge_update(conn: Connection, e: Event) -> None: + p = e.payload + source_id = p["source_id"] + target_id = p["target_id"] + chat_id = p.get("chat_id") + + # Upsert: ensure a row exists with defaults, then apply deltas. + conn.execute( + "INSERT OR IGNORE INTO edges (chat_id, source_id, target_id) VALUES (?, ?, ?)", + (chat_id, source_id, target_id), + ) + + row = conn.execute( + "SELECT affinity, trust, knowledge_json, last_interaction_chat_id, last_interaction_at " + "FROM edges WHERE source_id = ? AND target_id = ?", + (source_id, target_id), + ).fetchone() + affinity, trust, knowledge_json, last_chat_id, last_at = row + + affinity_delta = int(p.get("affinity_delta", 0)) + trust_delta = int(p.get("trust_delta", 0)) + new_affinity = _clamp(affinity + affinity_delta) + new_trust = _clamp(trust + trust_delta) + + new_facts = p.get("knowledge_facts") or [] + if new_facts: + knowledge = json.loads(knowledge_json) + knowledge.extend(new_facts) + knowledge_json = json.dumps(knowledge) + + payload_at = p.get("last_interaction_at") + payload_chat_id = p.get("last_interaction_chat_id") + if payload_at is not None: + last_at = payload_at + if payload_chat_id is not None: + last_chat_id = payload_chat_id + + conn.execute( + "UPDATE edges SET affinity = ?, trust = ?, knowledge_json = ?, " + "last_interaction_chat_id = ?, last_interaction_at = ? " + "WHERE source_id = ? AND target_id = ?", + (new_affinity, new_trust, knowledge_json, last_chat_id, last_at, + source_id, target_id), + ) + + +def get_edge(conn: Connection, source_id: str, target_id: str) -> dict | None: + row = conn.execute( + "SELECT * FROM edges WHERE source_id = ? AND target_id = ?", + (source_id, target_id), + ).fetchone() + if not row: + return None + cols = [c[1] for c in conn.execute("PRAGMA table_info(edges)").fetchall()] + d = dict(zip(cols, row)) + d["knowledge"] = json.loads(d.pop("knowledge_json")) + return d + + +def list_edges_for(conn: Connection, source_id: str) -> list[dict]: + cur = conn.execute( + "SELECT * FROM edges WHERE source_id = ? ORDER BY target_id", + (source_id,), + ) + rows = cur.fetchall() + cols = [c[1] for c in conn.execute("PRAGMA table_info(edges)").fetchall()] + out: list[dict] = [] + for row in rows: + d = dict(zip(cols, row)) + d["knowledge"] = json.loads(d.pop("knowledge_json")) + out.append(d) + return out diff --git a/tests/test_edges.py b/tests/test_edges.py new file mode 100644 index 0000000..22746fc --- /dev/null +++ b/tests/test_edges.py @@ -0,0 +1,162 @@ +from chat.db.migrate import apply_migrations +from chat.db.connection import open_db +from chat.eventlog.log import append_event +from chat.eventlog.projector import project +from chat.state.edges import get_edge, list_edges_for +import chat.state.entities # registers bot/you handlers +import chat.state.edges # registers edge_update handler + + +def _seed_entities(conn) -> None: + append_event(conn, kind="bot_authored", payload={ + "id": "bot_a", "name": "BotA", "persona": "p", + "voice_samples": [], "traits": [], + "backstory": "", "initial_relationship_to_you": "", + "kickoff_prose": "", + }) + append_event(conn, kind="you_authored", payload={ + "name": "Me", "pronouns": "", "persona": "", + }) + + +def test_edge_update_upsert_applies_first_delta(tmp_path): + db = tmp_path / "t.db" + apply_migrations(db) + with open_db(db) as conn: + _seed_entities(conn) + append_event(conn, kind="edge_update", payload={ + "source_id": "bot_a", "target_id": "you", + "affinity_delta": 5, + }) + project(conn) + edge = get_edge(conn, "bot_a", "you") + assert edge is not None + assert edge["affinity"] == 55 + assert edge["trust"] == 50 + assert edge["knowledge"] == [] + assert edge["summary"] == "" + + +def test_edge_update_multiple_deltas_accumulate(tmp_path): + db = tmp_path / "t.db" + apply_migrations(db) + with open_db(db) as conn: + _seed_entities(conn) + append_event(conn, kind="edge_update", payload={ + "source_id": "bot_a", "target_id": "you", + "affinity_delta": 5, + }) + append_event(conn, kind="edge_update", payload={ + "source_id": "bot_a", "target_id": "you", + "affinity_delta": -3, + "trust_delta": 2, + "knowledge_facts": ["she has a sister"], + }) + project(conn) + edge = get_edge(conn, "bot_a", "you") + assert edge is not None + assert edge["affinity"] == 52 + assert edge["trust"] == 52 + assert edge["knowledge"] == ["she has a sister"] + + +def test_edge_update_clamps_affinity_at_max(tmp_path): + db = tmp_path / "t.db" + apply_migrations(db) + with open_db(db) as conn: + _seed_entities(conn) + append_event(conn, kind="edge_update", payload={ + "source_id": "bot_a", "target_id": "you", + "affinity_delta": 5, + }) + append_event(conn, kind="edge_update", payload={ + "source_id": "bot_a", "target_id": "you", + "affinity_delta": 100, + }) + project(conn) + edge = get_edge(conn, "bot_a", "you") + assert edge is not None + assert edge["affinity"] == 100 + + +def test_edge_update_clamps_trust_at_min(tmp_path): + db = tmp_path / "t.db" + apply_migrations(db) + with open_db(db) as conn: + _seed_entities(conn) + append_event(conn, kind="edge_update", payload={ + "source_id": "bot_a", "target_id": "you", + "trust_delta": -200, + }) + project(conn) + edge = get_edge(conn, "bot_a", "you") + assert edge is not None + assert edge["trust"] == 0 + + +def test_edges_are_directed_and_asymmetric(tmp_path): + db = tmp_path / "t.db" + apply_migrations(db) + with open_db(db) as conn: + _seed_entities(conn) + append_event(conn, kind="edge_update", payload={ + "source_id": "bot_a", "target_id": "you", + "affinity_delta": 5, + }) + append_event(conn, kind="edge_update", payload={ + "source_id": "you", "target_id": "bot_a", + "affinity_delta": 10, + }) + project(conn) + forward = get_edge(conn, "bot_a", "you") + reverse = get_edge(conn, "you", "bot_a") + assert forward is not None and reverse is not None + assert forward["affinity"] == 55 + assert reverse["affinity"] == 60 + # Independent rows + assert forward["affinity"] != reverse["affinity"] + + +def test_edge_update_bumps_last_interaction(tmp_path): + db = tmp_path / "t.db" + apply_migrations(db) + with open_db(db) as conn: + _seed_entities(conn) + append_event(conn, kind="edge_update", payload={ + "source_id": "bot_a", "target_id": "you", + "affinity_delta": 1, + "last_interaction_at": "2026-04-26T10:00:00", + "last_interaction_chat_id": "chat_bot_a", + }) + project(conn) + edge = get_edge(conn, "bot_a", "you") + assert edge is not None + assert edge["last_interaction_at"] == "2026-04-26T10:00:00" + assert edge["last_interaction_chat_id"] == "chat_bot_a" + + +def test_list_edges_for_returns_outgoing_only(tmp_path): + db = tmp_path / "t.db" + apply_migrations(db) + with open_db(db) as conn: + _seed_entities(conn) + append_event(conn, kind="bot_authored", payload={ + "id": "bot_b", "name": "BotB", "persona": "p", + "voice_samples": [], "traits": [], + "backstory": "", "initial_relationship_to_you": "", + "kickoff_prose": "", + }) + append_event(conn, kind="edge_update", payload={ + "source_id": "bot_a", "target_id": "you", "affinity_delta": 1, + }) + append_event(conn, kind="edge_update", payload={ + "source_id": "bot_a", "target_id": "bot_b", "affinity_delta": 2, + }) + append_event(conn, kind="edge_update", payload={ + "source_id": "bot_b", "target_id": "bot_a", "affinity_delta": 3, + }) + project(conn) + outgoing = list_edges_for(conn, "bot_a") + targets = [e["target_id"] for e in outgoing] + assert targets == sorted(targets) + assert set(targets) == {"you", "bot_b"} -- 2.52.0 From 30e664812216c688e8a6fd80e8299abd161fd653 Mon Sep 17 00:00:00 2001 From: Joseph Doherty Date: Sun, 26 Apr 2026 11:56:32 -0400 Subject: [PATCH 10/45] feat: memory schema with witness flags and FTS5 index --- chat/db/migrations/0006_memories.sql | 35 ++++ chat/state/memory.py | 96 +++++++++++ tests/test_memory.py | 229 +++++++++++++++++++++++++++ 3 files changed, 360 insertions(+) create mode 100644 chat/db/migrations/0006_memories.sql create mode 100644 chat/state/memory.py create mode 100644 tests/test_memory.py diff --git a/chat/db/migrations/0006_memories.sql b/chat/db/migrations/0006_memories.sql new file mode 100644 index 0000000..a9c9fa4 --- /dev/null +++ b/chat/db/migrations/0006_memories.sql @@ -0,0 +1,35 @@ +CREATE TABLE memories ( + id INTEGER PRIMARY KEY, + owner_id TEXT NOT NULL, + chat_id TEXT NOT NULL, + scene_id INTEGER, + pov_summary TEXT NOT NULL, + witness_you INTEGER NOT NULL, + witness_host INTEGER NOT NULL, + witness_guest INTEGER NOT NULL, + chat_clock_at TEXT, + source TEXT, + reliability REAL NOT NULL DEFAULT 1.0, + significance INTEGER NOT NULL DEFAULT 1, + pinned INTEGER NOT NULL DEFAULT 0, + auto_pinned INTEGER NOT NULL DEFAULT 0, + created_at TEXT NOT NULL DEFAULT (datetime('now')) +); +CREATE INDEX idx_memories_owner ON memories(owner_id); + +CREATE VIRTUAL TABLE memories_fts USING fts5( + pov_summary, content='memories', content_rowid='id' +); + +CREATE TRIGGER memories_ai AFTER INSERT ON memories BEGIN + INSERT INTO memories_fts(rowid, pov_summary) VALUES (new.id, new.pov_summary); +END; +CREATE TRIGGER memories_au AFTER UPDATE ON memories BEGIN + INSERT INTO memories_fts(memories_fts, rowid, pov_summary) + VALUES('delete', old.id, old.pov_summary); + INSERT INTO memories_fts(rowid, pov_summary) VALUES (new.id, new.pov_summary); +END; +CREATE TRIGGER memories_ad AFTER DELETE ON memories BEGIN + INSERT INTO memories_fts(memories_fts, rowid, pov_summary) + VALUES('delete', old.id, old.pov_summary); +END; diff --git a/chat/state/memory.py b/chat/state/memory.py new file mode 100644 index 0000000..b2655b0 --- /dev/null +++ b/chat/state/memory.py @@ -0,0 +1,96 @@ +from __future__ import annotations +from sqlite3 import Connection +from chat.eventlog.projector import on +from chat.eventlog.log import Event + +_VALID_WITNESS_ROLES = {"you", "host", "guest"} + + +def _row_to_dict(conn: Connection, row: tuple) -> dict: + cols = [c[1] for c in conn.execute("PRAGMA table_info(memories)").fetchall()] + return dict(zip(cols, row)) + + +@on("memory_written") +def _apply_memory_written(conn: Connection, e: Event) -> None: + p = e.payload + conn.execute( + "INSERT INTO memories (" + "owner_id, chat_id, scene_id, pov_summary, " + "witness_you, witness_host, witness_guest, " + "chat_clock_at, source, reliability, significance, pinned, auto_pinned" + ") VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)", + ( + p["owner_id"], + p["chat_id"], + p.get("scene_id"), + p["pov_summary"], + int(p["witness_you"]), + int(p["witness_host"]), + int(p["witness_guest"]), + p.get("chat_clock_at"), + p.get("source", "direct"), + float(p.get("reliability", 1.0)), + int(p.get("significance", 1)), + int(p.get("pinned", 0)), + int(p.get("auto_pinned", 0)), + ), + ) + + +def get_memory(conn: Connection, memory_id: int) -> dict | None: + row = conn.execute( + "SELECT * FROM memories WHERE id = ?", (memory_id,) + ).fetchone() + if not row: + return None + return _row_to_dict(conn, row) + + +def get_pinned(conn: Connection, owner_id: str) -> list[dict]: + cur = conn.execute( + "SELECT * FROM memories WHERE owner_id = ? AND pinned = 1 " + "ORDER BY created_at DESC, id DESC", + (owner_id,), + ) + rows = cur.fetchall() + cols = [c[1] for c in conn.execute("PRAGMA table_info(memories)").fetchall()] + return [dict(zip(cols, row)) for row in rows] + + +def search_memories( + conn: Connection, + owner_id: str, + witness_role: str, + query: str, + k: int = 4, +) -> list[dict]: + """FTS5 search over pov_summary, scoped by owner and witness role. + + witness_role must be one of {"you", "host", "guest"} per the witness flags + on each memory row. Returns up to k rows ordered by FTS5 bm25 rank. + """ + if witness_role not in _VALID_WITNESS_ROLES: + raise ValueError( + f"witness_role must be one of {sorted(_VALID_WITNESS_ROLES)}, " + f"got {witness_role!r}" + ) + witness_col = f"witness_{witness_role}" + cols = [c[1] for c in conn.execute("PRAGMA table_info(memories)").fetchall()] + select_list = ", ".join(f"m.{c}" for c in cols) + sql = ( + f"SELECT {select_list}, memories_fts.rank AS fts_rank " + "FROM memories_fts " + "JOIN memories m ON m.id = memories_fts.rowid " + f"WHERE m.owner_id = ? AND m.{witness_col} = 1 " + "AND memories_fts MATCH ? " + "ORDER BY memories_fts.rank " + "LIMIT ?" + ) + cur = conn.execute(sql, (owner_id, query, k)) + rows = cur.fetchall() + out: list[dict] = [] + result_cols = cols + ["fts_rank"] + for row in rows: + out.append(dict(zip(result_cols, row))) + return out diff --git a/tests/test_memory.py b/tests/test_memory.py new file mode 100644 index 0000000..3d174b0 --- /dev/null +++ b/tests/test_memory.py @@ -0,0 +1,229 @@ +from __future__ import annotations +import pytest + +from chat.db.connection import open_db +from chat.db.migrate import apply_migrations +from chat.eventlog.log import append_event +from chat.eventlog.projector import project +import chat.state.memory # registers memory_written handler +from chat.state.memory import get_memory, get_pinned, search_memories + + +def _base_memory(**overrides): + payload = { + "owner_id": "bot_a", + "chat_id": "chat_bot_a", + "scene_id": 1, + "pov_summary": "She laughed at his joke about owls.", + "witness_you": 1, + "witness_host": 1, + "witness_guest": 0, + "chat_clock_at": "2026-04-26T10:00:00", + "source": "direct", + "reliability": 1.0, + "significance": 1, + "pinned": 0, + "auto_pinned": 0, + } + payload.update(overrides) + return payload + + +def test_memory_written_is_projected_and_readable(tmp_path): + db = tmp_path / "t.db" + apply_migrations(db) + with open_db(db) as conn: + append_event(conn, kind="memory_written", payload=_base_memory()) + project(conn) + row = conn.execute("SELECT id FROM memories").fetchone() + assert row is not None + mem = get_memory(conn, row[0]) + assert mem is not None + assert mem["owner_id"] == "bot_a" + assert mem["chat_id"] == "chat_bot_a" + assert mem["scene_id"] == 1 + assert mem["pov_summary"] == "She laughed at his joke about owls." + assert mem["witness_you"] == 1 + assert mem["witness_host"] == 1 + assert mem["witness_guest"] == 0 + assert mem["chat_clock_at"] == "2026-04-26T10:00:00" + assert mem["source"] == "direct" + assert mem["reliability"] == 1.0 + assert mem["significance"] == 1 + assert mem["pinned"] == 0 + assert mem["auto_pinned"] == 0 + + +def test_get_memory_returns_none_for_missing_id(tmp_path): + db = tmp_path / "t.db" + apply_migrations(db) + with open_db(db) as conn: + assert get_memory(conn, 9999) is None + + +def test_search_memories_filters_out_non_witness(tmp_path): + db = tmp_path / "t.db" + apply_migrations(db) + with open_db(db) as conn: + append_event(conn, kind="memory_written", payload=_base_memory( + pov_summary="The cat sat on the mat.", + witness_you=1, witness_host=1, witness_guest=0, + )) + project(conn) + # guest did not witness => excluded + results = search_memories(conn, "bot_a", "guest", "cat") + assert results == [] + + +def test_search_memories_includes_witnesses(tmp_path): + db = tmp_path / "t.db" + apply_migrations(db) + with open_db(db) as conn: + append_event(conn, kind="memory_written", payload=_base_memory( + pov_summary="The cat sat on the mat.", + witness_you=1, witness_host=1, witness_guest=0, + )) + project(conn) + results = search_memories(conn, "bot_a", "host", "cat") + assert len(results) == 1 + assert results[0]["pov_summary"] == "The cat sat on the mat." + assert "fts_rank" in results[0] + + +def test_search_memories_fts_matches_only_relevant_text(tmp_path): + db = tmp_path / "t.db" + apply_migrations(db) + with open_db(db) as conn: + append_event(conn, kind="memory_written", payload=_base_memory( + pov_summary="She loves owls and stars.", + )) + append_event(conn, kind="memory_written", payload=_base_memory( + pov_summary="He fixed the broken kettle.", + )) + project(conn) + results = search_memories(conn, "bot_a", "you", "owls") + assert len(results) == 1 + assert results[0]["pov_summary"] == "She loves owls and stars." + + +def test_search_memories_filters_by_owner(tmp_path): + db = tmp_path / "t.db" + apply_migrations(db) + with open_db(db) as conn: + append_event(conn, kind="memory_written", payload=_base_memory( + owner_id="bot_a", + pov_summary="Owls hooted at midnight.", + )) + append_event(conn, kind="memory_written", payload=_base_memory( + owner_id="bot_b", + pov_summary="Owls hooted at midnight.", + )) + project(conn) + results_a = search_memories(conn, "bot_a", "you", "owls") + results_b = search_memories(conn, "bot_b", "you", "owls") + assert len(results_a) == 1 + assert results_a[0]["owner_id"] == "bot_a" + assert len(results_b) == 1 + assert results_b[0]["owner_id"] == "bot_b" + + +def test_search_memories_returns_empty_on_no_match(tmp_path): + db = tmp_path / "t.db" + apply_migrations(db) + with open_db(db) as conn: + append_event(conn, kind="memory_written", payload=_base_memory( + pov_summary="The cat sat on the mat.", + )) + project(conn) + results = search_memories(conn, "bot_a", "you", "spaceship") + assert results == [] + + +def test_search_memories_invalid_witness_role_raises(tmp_path): + db = tmp_path / "t.db" + apply_migrations(db) + with open_db(db) as conn: + with pytest.raises(ValueError): + search_memories(conn, "bot_a", "everyone", "cat") + + +def test_search_memories_respects_k_limit(tmp_path): + db = tmp_path / "t.db" + apply_migrations(db) + with open_db(db) as conn: + for i in range(6): + append_event(conn, kind="memory_written", payload=_base_memory( + pov_summary=f"Owls hooted at midnight number {i}.", + )) + project(conn) + results = search_memories(conn, "bot_a", "you", "owls", k=4) + assert len(results) == 4 + + +def test_get_pinned_returns_only_pinned(tmp_path): + db = tmp_path / "t.db" + apply_migrations(db) + with open_db(db) as conn: + append_event(conn, kind="memory_written", payload=_base_memory( + pov_summary="Pinned moment.", + pinned=1, + )) + append_event(conn, kind="memory_written", payload=_base_memory( + pov_summary="Unpinned moment.", + pinned=0, + )) + project(conn) + pinned = get_pinned(conn, "bot_a") + assert len(pinned) == 1 + assert pinned[0]["pov_summary"] == "Pinned moment." + assert pinned[0]["pinned"] == 1 + + +def test_get_pinned_filters_by_owner(tmp_path): + db = tmp_path / "t.db" + apply_migrations(db) + with open_db(db) as conn: + append_event(conn, kind="memory_written", payload=_base_memory( + owner_id="bot_a", pov_summary="A's pin.", pinned=1, + )) + append_event(conn, kind="memory_written", payload=_base_memory( + owner_id="bot_b", pov_summary="B's pin.", pinned=1, + )) + project(conn) + pinned_a = get_pinned(conn, "bot_a") + assert len(pinned_a) == 1 + assert pinned_a[0]["owner_id"] == "bot_a" + + +def test_memory_payload_defaults_when_optional_missing(tmp_path): + db = tmp_path / "t.db" + apply_migrations(db) + with open_db(db) as conn: + append_event(conn, kind="memory_written", payload={ + "owner_id": "bot_a", + "chat_id": "chat_bot_a", + "pov_summary": "Bare minimum memory.", + "witness_you": 1, + "witness_host": 1, + "witness_guest": 1, + }) + project(conn) + row = conn.execute("SELECT id FROM memories").fetchone() + mem = get_memory(conn, row[0]) + assert mem["scene_id"] is None + assert mem["chat_clock_at"] is None + assert mem["source"] == "direct" + assert mem["reliability"] == 1.0 + assert mem["significance"] == 1 + assert mem["pinned"] == 0 + assert mem["auto_pinned"] == 0 + + +def test_schema_version_after_migration_is_6(tmp_path): + db = tmp_path / "t.db" + apply_migrations(db) + with open_db(db) as conn: + row = conn.execute( + "SELECT value FROM meta WHERE key = 'schema_version'" + ).fetchone() + assert int(row[0]) == 6 -- 2.52.0 From ec344064f15c680bd177408a73fc7ea0b90708d3 Mon Sep 17 00:00:00 2001 From: Joseph Doherty Date: Sun, 26 Apr 2026 12:03:26 -0400 Subject: [PATCH 11/45] feat: chats, chat_state, containers, scenes, activity tables --- chat/db/migrations/0007_world.sql | 45 ++++ chat/state/world.py | 202 ++++++++++++++++++ tests/test_memory.py | 4 +- tests/test_world.py | 334 ++++++++++++++++++++++++++++++ 4 files changed, 583 insertions(+), 2 deletions(-) create mode 100644 chat/db/migrations/0007_world.sql create mode 100644 chat/state/world.py create mode 100644 tests/test_world.py diff --git a/chat/db/migrations/0007_world.sql b/chat/db/migrations/0007_world.sql new file mode 100644 index 0000000..74af990 --- /dev/null +++ b/chat/db/migrations/0007_world.sql @@ -0,0 +1,45 @@ +CREATE TABLE chats ( + id TEXT PRIMARY KEY, + host_bot_id TEXT NOT NULL, + guest_bot_id TEXT, + created_at TEXT NOT NULL DEFAULT (datetime('now')) +); + +CREATE TABLE chat_state ( + chat_id TEXT PRIMARY KEY, + time TEXT NOT NULL, + weather TEXT NOT NULL DEFAULT '', + active_scene_id INTEGER, + narrative_anchor TEXT +); + +CREATE TABLE containers ( + id INTEGER PRIMARY KEY, + chat_id TEXT NOT NULL, + name TEXT NOT NULL, + type TEXT NOT NULL, + properties_json TEXT NOT NULL DEFAULT '{}', + parent_id INTEGER REFERENCES containers(id) +); + +CREATE TABLE scenes ( + id INTEGER PRIMARY KEY, + chat_id TEXT NOT NULL, + container_id INTEGER REFERENCES containers(id), + started_at TEXT NOT NULL, + ended_at TEXT, + significance INTEGER NOT NULL DEFAULT 0, + participants_json TEXT NOT NULL DEFAULT '[]' +); + +CREATE TABLE activity ( + entity_id TEXT PRIMARY KEY, + container_id INTEGER REFERENCES containers(id), + slot TEXT, + posture TEXT NOT NULL DEFAULT '', + action_json TEXT NOT NULL DEFAULT '{}', + attention TEXT NOT NULL DEFAULT '', + holding_json TEXT NOT NULL DEFAULT '[]', + status_json TEXT NOT NULL DEFAULT '{}', + updated_at TEXT NOT NULL DEFAULT (datetime('now')) +); diff --git a/chat/state/world.py b/chat/state/world.py new file mode 100644 index 0000000..5966924 --- /dev/null +++ b/chat/state/world.py @@ -0,0 +1,202 @@ +from __future__ import annotations +import json +from sqlite3 import Connection +from chat.eventlog.projector import on +from chat.eventlog.log import Event + + +def _row_to_dict(conn: Connection, table: str, row: tuple) -> dict: + cols = [c[1] for c in conn.execute(f"PRAGMA table_info({table})").fetchall()] + return dict(zip(cols, row)) + + +@on("chat_created") +def _apply_chat_created(conn: Connection, e: Event) -> None: + p = e.payload + conn.execute( + "INSERT INTO chats (id, host_bot_id, guest_bot_id) VALUES (?, ?, ?)", + (p["id"], p["host_bot_id"], p.get("guest_bot_id")), + ) + conn.execute( + "INSERT INTO chat_state (chat_id, time, weather, active_scene_id, narrative_anchor) " + "VALUES (?, ?, ?, NULL, ?)", + ( + p["id"], + p["initial_time"], + p.get("weather", ""), + p.get("narrative_anchor", ""), + ), + ) + + +@on("container_created") +def _apply_container_created(conn: Connection, e: Event) -> None: + p = e.payload + conn.execute( + "INSERT INTO containers (chat_id, name, type, properties_json, parent_id) " + "VALUES (?, ?, ?, ?, ?)", + ( + p["chat_id"], + p["name"], + p["type"], + json.dumps(p.get("properties", {})), + p.get("parent_id"), + ), + ) + + +@on("activity_change") +def _apply_activity_change(conn: Connection, e: Event) -> None: + p = e.payload + conn.execute( + "INSERT OR REPLACE INTO activity (" + "entity_id, container_id, slot, posture, action_json, " + "attention, holding_json, status_json, updated_at" + ") VALUES (?, ?, ?, ?, ?, ?, ?, ?, datetime('now'))", + ( + p["entity_id"], + p.get("container_id"), + p.get("slot"), + p.get("posture", ""), + json.dumps(p.get("action", {})), + p.get("attention", ""), + json.dumps(p.get("holding", [])), + json.dumps(p.get("status", {})), + ), + ) + + +@on("scene_opened") +def _apply_scene_opened(conn: Connection, e: Event) -> None: + p = e.payload + cur = conn.execute( + "INSERT INTO scenes (chat_id, container_id, started_at, ended_at, " + "significance, participants_json) VALUES (?, ?, ?, NULL, 0, ?)", + ( + p["chat_id"], + p.get("container_id"), + p["started_at"], + json.dumps(p.get("participants", [])), + ), + ) + new_id = cur.lastrowid + conn.execute( + "UPDATE chat_state SET active_scene_id = ? WHERE chat_id = ?", + (new_id, p["chat_id"]), + ) + + +@on("scene_closed") +def _apply_scene_closed(conn: Connection, e: Event) -> None: + p = e.payload + scene_id = p["scene_id"] + significance = int(p.get("significance", 0)) + conn.execute( + "UPDATE scenes SET ended_at = ?, significance = ? WHERE id = ?", + (p["ended_at"], significance, scene_id), + ) + row = conn.execute( + "SELECT chat_id FROM scenes WHERE id = ?", (scene_id,) + ).fetchone() + if row is not None: + chat_id = row[0] + conn.execute( + "UPDATE chat_state SET active_scene_id = NULL WHERE chat_id = ?", + (chat_id,), + ) + + +def _chat_select_columns() -> str: + return ( + "c.id, c.host_bot_id, c.guest_bot_id, c.created_at, " + "s.time, s.weather, s.active_scene_id, s.narrative_anchor" + ) + + +def _chat_row_to_dict(row: tuple) -> dict: + return { + "id": row[0], + "host_bot_id": row[1], + "guest_bot_id": row[2], + "created_at": row[3], + "time": row[4], + "weather": row[5], + "active_scene_id": row[6], + "narrative_anchor": row[7], + } + + +def get_chat(conn: Connection, chat_id: str) -> dict | None: + row = conn.execute( + f"SELECT {_chat_select_columns()} FROM chats c " + "JOIN chat_state s ON s.chat_id = c.id WHERE c.id = ?", + (chat_id,), + ).fetchone() + if not row: + return None + return _chat_row_to_dict(row) + + +def list_chats(conn: Connection) -> list[dict]: + cur = conn.execute( + f"SELECT {_chat_select_columns()} FROM chats c " + "JOIN chat_state s ON s.chat_id = c.id ORDER BY c.id" + ) + return [_chat_row_to_dict(row) for row in cur.fetchall()] + + +def get_container(conn: Connection, container_id: int) -> dict | None: + row = conn.execute( + "SELECT * FROM containers WHERE id = ?", (container_id,) + ).fetchone() + if not row: + return None + d = _row_to_dict(conn, "containers", row) + d["properties"] = json.loads(d.pop("properties_json")) + return d + + +def find_container(conn: Connection, chat_id: str, name: str) -> dict | None: + row = conn.execute( + "SELECT * FROM containers WHERE chat_id = ? AND name = ?", + (chat_id, name), + ).fetchone() + if not row: + return None + d = _row_to_dict(conn, "containers", row) + d["properties"] = json.loads(d.pop("properties_json")) + return d + + +def get_activity(conn: Connection, entity_id: str) -> dict | None: + row = conn.execute( + "SELECT * FROM activity WHERE entity_id = ?", (entity_id,) + ).fetchone() + if not row: + return None + d = _row_to_dict(conn, "activity", row) + d["action"] = json.loads(d.pop("action_json")) + d["holding"] = json.loads(d.pop("holding_json")) + d["status"] = json.loads(d.pop("status_json")) + return d + + +def get_scene(conn: Connection, scene_id: int) -> dict | None: + row = conn.execute( + "SELECT * FROM scenes WHERE id = ?", (scene_id,) + ).fetchone() + if not row: + return None + d = _row_to_dict(conn, "scenes", row) + d["participants"] = json.loads(d.pop("participants_json")) + return d + + +def active_scene(conn: Connection, chat_id: str) -> dict | None: + row = conn.execute( + "SELECT active_scene_id FROM chat_state WHERE chat_id = ?", + (chat_id,), + ).fetchone() + if not row or row[0] is None: + return None + return get_scene(conn, row[0]) diff --git a/tests/test_memory.py b/tests/test_memory.py index 3d174b0..3675ce6 100644 --- a/tests/test_memory.py +++ b/tests/test_memory.py @@ -219,11 +219,11 @@ def test_memory_payload_defaults_when_optional_missing(tmp_path): assert mem["auto_pinned"] == 0 -def test_schema_version_after_migration_is_6(tmp_path): +def test_schema_version_after_migration_is_at_least_6(tmp_path): db = tmp_path / "t.db" apply_migrations(db) with open_db(db) as conn: row = conn.execute( "SELECT value FROM meta WHERE key = 'schema_version'" ).fetchone() - assert int(row[0]) == 6 + assert int(row[0]) >= 6 diff --git a/tests/test_world.py b/tests/test_world.py new file mode 100644 index 0000000..29a0bb0 --- /dev/null +++ b/tests/test_world.py @@ -0,0 +1,334 @@ +from __future__ import annotations + +from chat.db.connection import open_db +from chat.db.migrate import apply_migrations +from chat.eventlog.log import append_event +from chat.eventlog.projector import project +import chat.state.world # registers handlers +from chat.state.world import ( + active_scene, + find_container, + get_activity, + get_chat, + get_container, + get_scene, + list_chats, +) + + +def _chat_payload(**overrides): + payload = { + "id": "chat_bot_a", + "host_bot_id": "bot_a", + "guest_bot_id": None, + "initial_time": "2026-04-26T20:00:00+00:00", + "narrative_anchor": "Day 1 evening", + "weather": "clear", + } + payload.update(overrides) + return payload + + +def _container_payload(**overrides): + payload = { + "chat_id": "chat_bot_a", + "name": "office", + "type": "workplace", + "properties": { + "public": True, + "moving": False, + "audible_range": "normal", + "slots": [], + }, + "parent_id": None, + } + payload.update(overrides) + return payload + + +def test_chat_created_initializes_chats_and_chat_state(tmp_path): + db = tmp_path / "t.db" + apply_migrations(db) + with open_db(db) as conn: + append_event(conn, kind="chat_created", payload=_chat_payload()) + project(conn) + chat = get_chat(conn, "chat_bot_a") + assert chat is not None + assert chat["id"] == "chat_bot_a" + assert chat["host_bot_id"] == "bot_a" + assert chat["guest_bot_id"] is None + assert chat["time"] == "2026-04-26T20:00:00+00:00" + assert chat["weather"] == "clear" + assert chat["narrative_anchor"] == "Day 1 evening" + assert chat["active_scene_id"] is None + + +def test_get_chat_returns_none_for_missing_id(tmp_path): + db = tmp_path / "t.db" + apply_migrations(db) + with open_db(db) as conn: + assert get_chat(conn, "chat_missing") is None + + +def test_list_chats_returns_all_joined_rows(tmp_path): + db = tmp_path / "t.db" + apply_migrations(db) + with open_db(db) as conn: + append_event(conn, kind="chat_created", payload=_chat_payload()) + append_event(conn, kind="chat_created", payload=_chat_payload( + id="chat_bot_b", host_bot_id="bot_b", + )) + project(conn) + chats = list_chats(conn) + assert len(chats) == 2 + ids = [c["id"] for c in chats] + assert ids == sorted(ids) + assert all("time" in c for c in chats) + + +def test_chat_created_with_optional_fields_defaulted(tmp_path): + db = tmp_path / "t.db" + apply_migrations(db) + with open_db(db) as conn: + append_event(conn, kind="chat_created", payload={ + "id": "chat_bot_a", + "host_bot_id": "bot_a", + "initial_time": "2026-04-26T20:00:00+00:00", + }) + project(conn) + chat = get_chat(conn, "chat_bot_a") + assert chat is not None + assert chat["guest_bot_id"] is None + assert chat["weather"] == "" + assert chat["narrative_anchor"] == "" + assert chat["active_scene_id"] is None + + +def test_container_created_inserts_and_findable_by_name(tmp_path): + db = tmp_path / "t.db" + apply_migrations(db) + with open_db(db) as conn: + append_event(conn, kind="chat_created", payload=_chat_payload()) + append_event(conn, kind="container_created", payload=_container_payload()) + project(conn) + c = find_container(conn, "chat_bot_a", "office") + assert c is not None + assert c["name"] == "office" + assert c["type"] == "workplace" + assert c["chat_id"] == "chat_bot_a" + assert c["parent_id"] is None + assert c["properties"] == { + "public": True, + "moving": False, + "audible_range": "normal", + "slots": [], + } + # get_container by id also works + c2 = get_container(conn, c["id"]) + assert c2 is not None + assert c2["name"] == "office" + assert c2["properties"]["public"] is True + + +def test_find_container_returns_none_when_missing(tmp_path): + db = tmp_path / "t.db" + apply_migrations(db) + with open_db(db) as conn: + assert find_container(conn, "chat_bot_a", "nope") is None + assert get_container(conn, 999) is None + + +def test_activity_change_inserts_then_replaces(tmp_path): + db = tmp_path / "t.db" + apply_migrations(db) + with open_db(db) as conn: + append_event(conn, kind="chat_created", payload=_chat_payload()) + append_event(conn, kind="container_created", payload=_container_payload()) + # First activity event then a second for the same entity that supersedes it. + append_event(conn, kind="activity_change", payload={ + "entity_id": "bot_a", + "container_id": 1, + "slot": "desk_chair", + "posture": "sitting", + "action": { + "verb": "writing email", + "interruptible": True, + "required_attention": "medium", + "expected_duration": "a few minutes", + "started_at": "2026-04-26T20:00:00+00:00", + }, + "attention": "the screen", + "holding": [], + "status": {}, + }) + append_event(conn, kind="activity_change", payload={ + "entity_id": "bot_a", + "container_id": 1, + "posture": "standing", + "action": {"verb": "pacing"}, + "attention": "the window", + }) + project(conn) + a = get_activity(conn, "bot_a") + assert a is not None + assert a["entity_id"] == "bot_a" + assert a["container_id"] == 1 + # second event replaced first + assert a["posture"] == "standing" + assert a["action"]["verb"] == "pacing" + assert a["attention"] == "the window" + # only one row exists for that entity + count = conn.execute( + "SELECT COUNT(*) FROM activity WHERE entity_id = ?", ("bot_a",) + ).fetchone()[0] + assert count == 1 + + +def test_activity_change_initial_values_persist_when_only_one_event(tmp_path): + db = tmp_path / "t.db" + apply_migrations(db) + with open_db(db) as conn: + append_event(conn, kind="chat_created", payload=_chat_payload()) + append_event(conn, kind="container_created", payload=_container_payload()) + append_event(conn, kind="activity_change", payload={ + "entity_id": "bot_a", + "container_id": 1, + "slot": "desk_chair", + "posture": "sitting", + "action": {"verb": "writing email"}, + "attention": "the screen", + "holding": ["pen"], + "status": {"hungry": False}, + }) + project(conn) + a = get_activity(conn, "bot_a") + assert a is not None + assert a["slot"] == "desk_chair" + assert a["posture"] == "sitting" + assert a["action"]["verb"] == "writing email" + assert a["attention"] == "the screen" + assert a["holding"] == ["pen"] + assert a["status"] == {"hungry": False} + + +def test_activity_change_defaults_for_minimal_payload(tmp_path): + db = tmp_path / "t.db" + apply_migrations(db) + with open_db(db) as conn: + append_event(conn, kind="activity_change", payload={ + "entity_id": "you", + }) + project(conn) + a = get_activity(conn, "you") + assert a is not None + assert a["container_id"] is None + assert a["slot"] is None + assert a["posture"] == "" + assert a["action"] == {} + assert a["attention"] == "" + assert a["holding"] == [] + assert a["status"] == {} + + +def test_get_activity_returns_none_for_missing(tmp_path): + db = tmp_path / "t.db" + apply_migrations(db) + with open_db(db) as conn: + assert get_activity(conn, "ghost") is None + + +def test_scene_opened_marks_active_scene_id(tmp_path): + db = tmp_path / "t.db" + apply_migrations(db) + with open_db(db) as conn: + append_event(conn, kind="chat_created", payload=_chat_payload()) + append_event(conn, kind="container_created", payload=_container_payload()) + append_event(conn, kind="scene_opened", payload={ + "chat_id": "chat_bot_a", + "container_id": 1, + "started_at": "2026-04-26T20:00:00+00:00", + "participants": ["you", "bot_a"], + }) + project(conn) + + s = active_scene(conn, "chat_bot_a") + assert s is not None + assert s["chat_id"] == "chat_bot_a" + assert s["container_id"] == 1 + assert s["started_at"] == "2026-04-26T20:00:00+00:00" + assert s["ended_at"] is None + assert s["significance"] == 0 + assert s["participants"] == ["you", "bot_a"] + + chat = get_chat(conn, "chat_bot_a") + assert chat["active_scene_id"] == s["id"] + + s2 = get_scene(conn, s["id"]) + assert s2 is not None + assert s2["participants"] == ["you", "bot_a"] + + +def test_scene_closed_clears_active_scene_id_and_records_end(tmp_path): + db = tmp_path / "t.db" + apply_migrations(db) + with open_db(db) as conn: + append_event(conn, kind="chat_created", payload=_chat_payload()) + append_event(conn, kind="container_created", payload=_container_payload()) + append_event(conn, kind="scene_opened", payload={ + "chat_id": "chat_bot_a", + "container_id": 1, + "started_at": "2026-04-26T20:00:00+00:00", + "participants": ["you", "bot_a"], + }) + # The first scene insert will be id=1 (first row in scenes). + append_event(conn, kind="scene_closed", payload={ + "scene_id": 1, + "ended_at": "2026-04-26T21:00:00+00:00", + "significance": 2, + }) + project(conn) + + assert active_scene(conn, "chat_bot_a") is None + chat = get_chat(conn, "chat_bot_a") + assert chat["active_scene_id"] is None + s = get_scene(conn, 1) + assert s["ended_at"] == "2026-04-26T21:00:00+00:00" + assert s["significance"] == 2 + + +def test_scene_closed_significance_defaults_to_zero(tmp_path): + db = tmp_path / "t.db" + apply_migrations(db) + with open_db(db) as conn: + append_event(conn, kind="chat_created", payload=_chat_payload()) + append_event(conn, kind="scene_opened", payload={ + "chat_id": "chat_bot_a", + "started_at": "2026-04-26T20:00:00+00:00", + "participants": ["you", "bot_a"], + }) + append_event(conn, kind="scene_closed", payload={ + "scene_id": 1, + "ended_at": "2026-04-26T21:00:00+00:00", + }) + project(conn) + s = get_scene(conn, 1) + assert s["significance"] == 0 + assert s["ended_at"] == "2026-04-26T21:00:00+00:00" + + +def test_get_scene_returns_none_for_missing(tmp_path): + db = tmp_path / "t.db" + apply_migrations(db) + with open_db(db) as conn: + assert get_scene(conn, 999) is None + assert active_scene(conn, "chat_missing") is None + + +def test_schema_version_after_migration_is_7(tmp_path): + db = tmp_path / "t.db" + apply_migrations(db) + with open_db(db) as conn: + row = conn.execute( + "SELECT value FROM meta WHERE key = 'schema_version'" + ).fetchone() + assert int(row[0]) == 7 -- 2.52.0 From a5339fc1d2ce6aac391ea18d10213836e8293055 Mon Sep 17 00:00:00 2001 From: Joseph Doherty Date: Sun, 26 Apr 2026 12:09:17 -0400 Subject: [PATCH 12/45] feat: kickoff prose parser via classifier --- chat/services/__init__.py | 0 chat/services/kickoff.py | 121 +++++++++++++++++++++++++++++++++++ tests/test_kickoff.py | 131 ++++++++++++++++++++++++++++++++++++++ 3 files changed, 252 insertions(+) create mode 100644 chat/services/__init__.py create mode 100644 chat/services/kickoff.py create mode 100644 tests/test_kickoff.py diff --git a/chat/services/__init__.py b/chat/services/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/chat/services/kickoff.py b/chat/services/kickoff.py new file mode 100644 index 0000000..591b09c --- /dev/null +++ b/chat/services/kickoff.py @@ -0,0 +1,121 @@ +"""Kickoff prose parser. + +Service-layer function that converts a bot's authored kickoff prose into a +structured ``KickoffParse`` for the kickoff confirm-and-edit step (T13 will +wire this into the UI flow). + +The classifier prompt includes only the bot context that's load-bearing for +parsing the opening scene: name, persona, the authored +``initial_relationship_to_you`` blurb, the ``you`` entity name, and the +kickoff prose itself. Other identity fields (traits, backstory, ...) are +intentionally left out — they would be noise for this extraction. +""" + +from __future__ import annotations + +from pydantic import BaseModel, Field + +from chat.llm.classify import classify +from chat.llm.client import LLMClient + + +class ActivityShape(BaseModel): + """Per-entity activity at scene start. + + Maps onto Requirements §6.5: ``current_action.{verb,interruptible, + required_attention,expected_duration}`` plus posture, attention, holding. + ``action_required_attention`` is left as a free-form string ("low" / + "medium" / "high" expected) rather than a Literal so the classifier has + room to vary phrasing in v1. + """ + + posture: str + action_verb: str + action_interruptible: bool + action_required_attention: str # low | medium | high + action_expected_duration: str + attention: str = "" + holding: list[str] = Field(default_factory=list) + + +class KickoffParse(BaseModel): + """Structured opening-scene state extracted from kickoff prose. + + ``container_properties`` is loose ``dict``: the classifier may emit + ``moving`` / ``public`` / ``audible_range`` keys, but downstream + consumers (T13's confirm form) handle missing keys gracefully. + ``initial_time_iso`` is stored as text — not validated as a datetime + here; ``chat_state.time`` stores it as a plain string. + """ + + container_name: str + container_type: str + container_properties: dict + you_activity: ActivityShape + bot_activity: ActivityShape + initial_time_iso: str + edge_seed_summary: str + edge_seed_knowledge_facts: list[str] + + +_SYSTEM_PROMPT = ( + "You are extracting structured scene state from a roleplay kickoff " + "scene description. The user provides bot context and a prose " + "description of the opening scene; you output JSON conforming to the " + "schema. Be concrete: pick a single container, single activity per " + "entity, and a sensible initial in-fiction time. Anything not stated " + "explicitly should be inferred reasonably from the prose." +) + + +def _build_user_prompt( + *, + bot_name: str, + bot_persona: str, + initial_relationship_to_you: str, + kickoff_prose: str, + you_name: str, +) -> str: + return ( + f"BOT NAME: {bot_name}\n" + f"BOT PERSONA: {bot_persona}\n" + f"INITIAL RELATIONSHIP TO {you_name}: {initial_relationship_to_you}\n" + f"YOU NAME: {you_name}\n" + f"KICKOFF PROSE:\n{kickoff_prose}" + ) + + +async def parse_kickoff( + client: LLMClient, + *, + model: str, + bot_name: str, + bot_persona: str, + initial_relationship_to_you: str, + kickoff_prose: str, + you_name: str, + timeout_s: float = 10.0, +) -> KickoffParse: + """Parse authored kickoff prose into a structured ``KickoffParse``. + + Internally calls :func:`chat.llm.classify.classify` with a labeled + user prompt. Raises ``RuntimeError`` if the classifier fails twice in + a row — no default is supplied at this layer, since the caller (T13's + confirm form) is responsible for showing an error and letting the + user edit. + """ + user_prompt = _build_user_prompt( + bot_name=bot_name, + bot_persona=bot_persona, + initial_relationship_to_you=initial_relationship_to_you, + kickoff_prose=kickoff_prose, + you_name=you_name, + ) + return await classify( + client, + model=model, + system=_SYSTEM_PROMPT, + user=user_prompt, + schema=KickoffParse, + timeout_s=timeout_s, + ) diff --git a/tests/test_kickoff.py b/tests/test_kickoff.py new file mode 100644 index 0000000..c3dfc14 --- /dev/null +++ b/tests/test_kickoff.py @@ -0,0 +1,131 @@ +import json +import pytest + +from chat.llm.mock import MockLLMClient +from chat.services.kickoff import ( + ActivityShape, + KickoffParse, + parse_kickoff, +) + + +def _full_kickoff_json() -> str: + return json.dumps( + { + "container_name": "office bullpen, late evening", + "container_type": "office", + "container_properties": { + "moving": False, + "public": False, + "audible_range": "room", + }, + "you_activity": { + "posture": "sitting at your desk", + "action_verb": "finishing emails", + "action_interruptible": True, + "action_required_attention": "low", + "action_expected_duration": "15m", + "attention": "the screen", + "holding": ["coffee mug"], + }, + "bot_activity": { + "posture": "sitting at her desk", + "action_verb": "pretending to work", + "action_interruptible": True, + "action_required_attention": "low", + "action_expected_duration": "indefinite", + "attention": "you, in glances", + "holding": [], + }, + "initial_time_iso": "2026-04-26T19:42:00", + "edge_seed_summary": "coworkers; aware of each other; no shared history beyond the office", + "edge_seed_knowledge_facts": [ + "they work on the same floor", + "it is unusual to be the only two left", + ], + } + ) + + +@pytest.mark.asyncio +async def test_parse_kickoff_happy_path_populates_fields(): + mock = MockLLMClient(canned=[_full_kickoff_json()]) + result = await parse_kickoff( + mock, + model="m", + bot_name="BotA", + bot_persona="reserved colleague who quietly notices things", + initial_relationship_to_you="coworker, slight crush, never voiced", + kickoff_prose=( + "you stay late at the office; only you and BotA are there; " + "she's at her desk pretending to work" + ), + you_name="You", + ) + assert isinstance(result, KickoffParse) + assert result.container_name == "office bullpen, late evening" + assert result.container_type == "office" + assert isinstance(result.you_activity, ActivityShape) + assert result.you_activity.posture == "sitting at your desk" + assert result.bot_activity.action_verb == "pretending to work" + assert result.edge_seed_summary.startswith("coworkers") + assert "they work on the same floor" in result.edge_seed_knowledge_facts + assert result.initial_time_iso == "2026-04-26T19:42:00" + + +@pytest.mark.asyncio +async def test_parse_kickoff_applies_activity_defaults_for_missing_fields(): + minimal_payload = { + "container_name": "kitchen", + "container_type": "kitchen", + "container_properties": {}, + "you_activity": { + "posture": "standing", + "action_verb": "boiling water", + "action_interruptible": True, + "action_required_attention": "low", + "action_expected_duration": "5m", + }, + "bot_activity": { + "posture": "leaning on the counter", + "action_verb": "scrolling phone", + "action_interruptible": True, + "action_required_attention": "low", + "action_expected_duration": "10m", + }, + "initial_time_iso": "2026-04-26T08:00:00", + "edge_seed_summary": "roommates", + "edge_seed_knowledge_facts": [], + } + mock = MockLLMClient(canned=[json.dumps(minimal_payload)]) + result = await parse_kickoff( + mock, + model="m", + bot_name="BotA", + bot_persona="laid-back roommate", + initial_relationship_to_you="roommates of two years", + kickoff_prose="morning in the kitchen; you're making tea while BotA scrolls her phone", + you_name="You", + ) + assert result.you_activity.attention == "" + assert result.you_activity.holding == [] + assert result.bot_activity.attention == "" + assert result.bot_activity.holding == [] + # mutating one default must not leak into the other (default_factory check) + result.you_activity.holding.append("kettle") + assert result.bot_activity.holding == [] + + +@pytest.mark.asyncio +async def test_parse_kickoff_raises_when_classifier_fails_twice(): + mock = MockLLMClient(canned=["nope", "still nope"]) + with pytest.raises(RuntimeError): + await parse_kickoff( + mock, + model="m", + bot_name="BotA", + bot_persona="x", + initial_relationship_to_you="y", + kickoff_prose="z", + you_name="You", + ) -- 2.52.0 From 44ea627a8a1bbe45582ab2df598b9d736f6dc4e9 Mon Sep 17 00:00:00 2001 From: Joseph Doherty Date: Sun, 26 Apr 2026 12:17:06 -0400 Subject: [PATCH 13/45] feat: bot authoring form with bot_authored event --- chat/app.py | 35 +++++++++- chat/static/app.css | 37 ++++++++++ chat/templates/base.html | 18 +++++ chat/templates/bot_form.html | 57 +++++++++++++++ chat/templates/bot_list.html | 17 +++++ chat/web/__init__.py | 0 chat/web/bots.py | 116 ++++++++++++++++++++++++++++++ pyproject.toml | 1 + tests/test_bot_authoring.py | 132 +++++++++++++++++++++++++++++++++++ 9 files changed, 411 insertions(+), 2 deletions(-) create mode 100644 chat/static/app.css create mode 100644 chat/templates/base.html create mode 100644 chat/templates/bot_form.html create mode 100644 chat/templates/bot_list.html create mode 100644 chat/web/__init__.py create mode 100644 chat/web/bots.py create mode 100644 tests/test_bot_authoring.py diff --git a/chat/app.py b/chat/app.py index 66990d8..0ac28ef 100644 --- a/chat/app.py +++ b/chat/app.py @@ -1,6 +1,37 @@ -from fastapi import FastAPI +from __future__ import annotations +from contextlib import asynccontextmanager +from pathlib import Path -app = FastAPI(title="chat") +from fastapi import FastAPI +from fastapi.staticfiles import StaticFiles + +from chat.config import load_settings +from chat.db.migrate import apply_migrations + +# Trigger handler registration: +import chat.state.entities # noqa: F401 +import chat.state.edges # noqa: F401 +import chat.state.memory # noqa: F401 +import chat.state.world # noqa: F401 + +from chat.web.bots import router as bots_router + + +@asynccontextmanager +async def lifespan(app: FastAPI): + settings = load_settings() + settings.db_path.parent.mkdir(parents=True, exist_ok=True) + apply_migrations(settings.db_path) + app.state.settings = settings + yield + + +app = FastAPI(title="chat", lifespan=lifespan) + +STATIC_DIR = Path(__file__).resolve().parent / "static" +app.mount("/static", StaticFiles(directory=str(STATIC_DIR)), name="static") + +app.include_router(bots_router) @app.get("/health") diff --git a/chat/static/app.css b/chat/static/app.css new file mode 100644 index 0000000..96a0296 --- /dev/null +++ b/chat/static/app.css @@ -0,0 +1,37 @@ +* { box-sizing: border-box; } +body { + font: 15px/1.5 system-ui, -apple-system, "Segoe UI", sans-serif; + margin: 0; + color: #1c1c1c; + background: #fafafa; +} +.topbar { + padding: 12px 24px; + border-bottom: 1px solid #e5e5e5; + background: #fff; +} +.brand { font-weight: 600; text-decoration: none; color: inherit; } +.container { max-width: 720px; margin: 24px auto; padding: 0 16px; } +h1 { margin-top: 0; } +.page-header { display: flex; align-items: center; justify-content: space-between; } +.btn, button { + display: inline-block; padding: 8px 14px; + border: 1px solid #444; background: #1c1c1c; color: #fff; + border-radius: 4px; text-decoration: none; cursor: pointer; + font: inherit; +} +.bot-form label { display: block; margin-bottom: 14px; } +.bot-form label span { display: block; font-weight: 600; margin-bottom: 4px; } +.bot-form input[type=text], .bot-form textarea { + width: 100%; padding: 6px 8px; font: inherit; + border: 1px solid #ccc; border-radius: 3px; background: #fff; +} +.bot-form small { display: block; color: #666; margin-top: 2px; } +.bot-list { list-style: none; padding: 0; } +.bot-list li { padding: 8px 0; border-bottom: 1px solid #eee; } +.muted { color: #666; } +.error { + padding: 8px 12px; border: 1px solid #c33; background: #fdecea; + color: #a00; border-radius: 3px; +} +code { font-family: ui-monospace, "SF Mono", Menlo, monospace; } diff --git a/chat/templates/base.html b/chat/templates/base.html new file mode 100644 index 0000000..6c86104 --- /dev/null +++ b/chat/templates/base.html @@ -0,0 +1,18 @@ + + + + + + {% block title %}chat{% endblock %} + + + + +
+ chat +
+
+ {% block content %}{% endblock %} +
+ + diff --git a/chat/templates/bot_form.html b/chat/templates/bot_form.html new file mode 100644 index 0000000..e0d3331 --- /dev/null +++ b/chat/templates/bot_form.html @@ -0,0 +1,57 @@ +{% extends "base.html" %} +{% block title %}New bot - chat{% endblock %} +{% block content %} +

New bot

+{% if error %} +

{{ error }}

+{% endif %} +
+ + + + + + + + + + + + + + + + + +
+{% endblock %} diff --git a/chat/templates/bot_list.html b/chat/templates/bot_list.html new file mode 100644 index 0000000..f3b4562 --- /dev/null +++ b/chat/templates/bot_list.html @@ -0,0 +1,17 @@ +{% extends "base.html" %} +{% block title %}Bots - chat{% endblock %} +{% block content %} +
+

Bots

+ + New bot +
+{% if bots %} + +{% else %} +

No bots yet. Create your first bot.

+{% endif %} +{% endblock %} diff --git a/chat/web/__init__.py b/chat/web/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/chat/web/bots.py b/chat/web/bots.py new file mode 100644 index 0000000..bbf077d --- /dev/null +++ b/chat/web/bots.py @@ -0,0 +1,116 @@ +from __future__ import annotations +import sqlite3 +from pathlib import Path +from fastapi import APIRouter, Depends, Form, HTTPException, Request +from fastapi.responses import RedirectResponse, HTMLResponse +from fastapi.templating import Jinja2Templates + +from chat.eventlog.log import append_event +from chat.eventlog.projector import project +from chat.state.entities import list_bots + +TEMPLATES = Jinja2Templates(directory=str(Path(__file__).resolve().parent.parent / "templates")) + +router = APIRouter() + +REQUIRED_FIELDS = ("id", "name", "persona", "initial_relationship_to_you", "kickoff_prose") + + +def get_conn(request: Request): + settings = request.app.state.settings + db_path: Path = settings.db_path + db_path.parent.mkdir(parents=True, exist_ok=True) + conn = sqlite3.connect(db_path, check_same_thread=False) + conn.execute("PRAGMA journal_mode=WAL") + conn.execute("PRAGMA foreign_keys=ON") + try: + yield conn + conn.commit() + finally: + conn.close() + + +def _split_voice_samples(text: str) -> list[str]: + if not text or not text.strip(): + return [] + # Split on a line containing only "---" (with optional surrounding whitespace). + parts: list[str] = [] + buf: list[str] = [] + for line in text.splitlines(): + if line.strip() == "---": + if buf: + parts.append("\n".join(buf).strip()) + buf = [] + continue + buf.append(line) + if buf: + parts.append("\n".join(buf).strip()) + return [p for p in parts if p] + + +def _split_traits(text: str) -> list[str]: + if not text or not text.strip(): + return [] + items: list[str] = [] + for line in text.splitlines(): + line = line.strip() + if not line: + continue + if "," in line: + items.extend(p.strip() for p in line.split(",")) + else: + items.append(line) + return [t for t in items if t] + + +@router.get("/bots", response_class=HTMLResponse) +async def bots_list(request: Request, conn=Depends(get_conn)): + bots = list_bots(conn) + return TEMPLATES.TemplateResponse(request, "bot_list.html", {"bots": bots}) + + +@router.get("/bots/new", response_class=HTMLResponse) +async def bot_form(request: Request): + return TEMPLATES.TemplateResponse(request, "bot_form.html", {"values": {}, "error": None}) + + +@router.post("/bots/new") +async def bot_create( + request: Request, + id: str = Form(""), + name: str = Form(""), + persona: str = Form(""), + voice_samples: str = Form(""), + traits: str = Form(""), + backstory: str = Form(""), + initial_relationship_to_you: str = Form(""), + kickoff_prose: str = Form(""), + conn=Depends(get_conn), +): + values = { + "id": id, + "name": name, + "persona": persona, + "voice_samples": voice_samples, + "traits": traits, + "backstory": backstory, + "initial_relationship_to_you": initial_relationship_to_you, + "kickoff_prose": kickoff_prose, + } + missing = [f for f in REQUIRED_FIELDS if not values[f].strip()] + if missing: + raise HTTPException(status_code=400, detail=f"missing required: {', '.join(missing)}") + + payload = { + "id": id.strip(), + "name": name.strip(), + "persona": persona.strip(), + "voice_samples": _split_voice_samples(voice_samples), + "traits": _split_traits(traits), + "backstory": backstory.strip(), + "initial_relationship_to_you": initial_relationship_to_you.strip(), + "kickoff_prose": kickoff_prose.strip(), + } + append_event(conn, kind="bot_authored", payload=payload) + project(conn) + return RedirectResponse(url=f"/bots/{payload['id']}/kickoff", status_code=303) diff --git a/pyproject.toml b/pyproject.toml index c0593a9..b72bc90 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -13,6 +13,7 @@ dependencies = [ "tiktoken>=0.7", "jinja2>=3.1", "aiosqlite>=0.20", + "python-multipart>=0.0.9", ] [project.optional-dependencies] diff --git a/tests/test_bot_authoring.py b/tests/test_bot_authoring.py new file mode 100644 index 0000000..5b8a82d --- /dev/null +++ b/tests/test_bot_authoring.py @@ -0,0 +1,132 @@ +from __future__ import annotations + +import pytest +from fastapi.testclient import TestClient + +from chat.app import app + + +@pytest.fixture +def client(tmp_path, monkeypatch): + config_path = tmp_path / "config.toml" + config_path.write_text('featherless_api_key = "test"\n') + monkeypatch.setenv("CHAT_CONFIG_PATH", str(config_path)) + monkeypatch.setenv("CHAT_DB_PATH", str(tmp_path / "test.db")) + with TestClient(app) as c: + yield c + + +def test_get_new_bot_form_renders(client): + response = client.get("/bots/new") + assert response.status_code == 200 + body = response.text.lower() + assert " Date: Sun, 26 Apr 2026 12:22:00 -0400 Subject: [PATCH 14/45] feat: settings page with you-entity authoring --- chat/app.py | 2 + chat/static/app.css | 4 ++ chat/templates/settings.html | 29 +++++++++ chat/web/settings.py | 46 ++++++++++++++ tests/test_settings.py | 112 +++++++++++++++++++++++++++++++++++ 5 files changed, 193 insertions(+) create mode 100644 chat/templates/settings.html create mode 100644 chat/web/settings.py create mode 100644 tests/test_settings.py diff --git a/chat/app.py b/chat/app.py index 0ac28ef..82e49ad 100644 --- a/chat/app.py +++ b/chat/app.py @@ -15,6 +15,7 @@ import chat.state.memory # noqa: F401 import chat.state.world # noqa: F401 from chat.web.bots import router as bots_router +from chat.web.settings import router as settings_router @asynccontextmanager @@ -32,6 +33,7 @@ STATIC_DIR = Path(__file__).resolve().parent / "static" app.mount("/static", StaticFiles(directory=str(STATIC_DIR)), name="static") app.include_router(bots_router) +app.include_router(settings_router) @app.get("/health") diff --git a/chat/static/app.css b/chat/static/app.css index 96a0296..9c261a8 100644 --- a/chat/static/app.css +++ b/chat/static/app.css @@ -34,4 +34,8 @@ h1 { margin-top: 0; } padding: 8px 12px; border: 1px solid #c33; background: #fdecea; color: #a00; border-radius: 3px; } +.success { + padding: 8px 12px; border: 1px solid #2d7a3a; background: #eafaf0; + color: #1f5c2a; border-radius: 3px; +} code { font-family: ui-monospace, "SF Mono", Menlo, monospace; } diff --git a/chat/templates/settings.html b/chat/templates/settings.html new file mode 100644 index 0000000..e26fcf6 --- /dev/null +++ b/chat/templates/settings.html @@ -0,0 +1,29 @@ +{% extends "base.html" %} +{% block title %}Settings - chat{% endblock %} +{% block content %} +

Settings

+{% if saved %} +

Settings saved.

+{% endif %} +
+ + + + + + + +
+{% endblock %} diff --git a/chat/web/settings.py b/chat/web/settings.py new file mode 100644 index 0000000..cb04d58 --- /dev/null +++ b/chat/web/settings.py @@ -0,0 +1,46 @@ +from __future__ import annotations +from pathlib import Path +from fastapi import APIRouter, Depends, Form, HTTPException, Request +from fastapi.responses import HTMLResponse +from fastapi.templating import Jinja2Templates + +from chat.eventlog.log import append_event +from chat.eventlog.projector import project +from chat.state.entities import get_you +from chat.web.bots import get_conn + +TEMPLATES = Jinja2Templates(directory=str(Path(__file__).resolve().parent.parent / "templates")) + +router = APIRouter() + + +@router.get("/settings", response_class=HTMLResponse) +async def settings_get(request: Request, conn=Depends(get_conn)): + you = get_you(conn) or {"name": "", "pronouns": "", "persona": ""} + return TEMPLATES.TemplateResponse( + request, "settings.html", {"values": you, "saved": False} + ) + + +@router.post("/settings", response_class=HTMLResponse) +async def settings_post( + request: Request, + name: str = Form(""), + pronouns: str = Form(""), + persona: str = Form(""), + conn=Depends(get_conn), +): + if not name.strip(): + raise HTTPException(status_code=400, detail="name is required") + + payload = { + "name": name.strip(), + "pronouns": pronouns.strip(), + "persona": persona.strip(), + } + append_event(conn, kind="you_authored", payload=payload) + project(conn) + + return TEMPLATES.TemplateResponse( + request, "settings.html", {"values": payload, "saved": True} + ) diff --git a/tests/test_settings.py b/tests/test_settings.py new file mode 100644 index 0000000..18d2c4b --- /dev/null +++ b/tests/test_settings.py @@ -0,0 +1,112 @@ +from __future__ import annotations + +import pytest +from fastapi.testclient import TestClient + +from chat.app import app + + +@pytest.fixture +def client(tmp_path, monkeypatch): + config_path = tmp_path / "config.toml" + config_path.write_text('featherless_api_key = "test"\n') + monkeypatch.setenv("CHAT_CONFIG_PATH", str(config_path)) + monkeypatch.setenv("CHAT_DB_PATH", str(tmp_path / "test.db")) + with TestClient(app) as c: + yield c + + +def test_get_settings_renders_empty(client): + response = client.get("/settings") + assert response.status_code == 200 + body = response.text.lower() + assert " Date: Sun, 26 Apr 2026 12:28:05 -0400 Subject: [PATCH 15/45] feat: kickoff parse-and-confirm flow with chat creation --- chat/app.py | 2 + chat/templates/kickoff_confirm.html | 118 ++++++++++++ chat/web/kickoff.py | 284 ++++++++++++++++++++++++++++ tests/test_kickoff_confirm.py | 212 +++++++++++++++++++++ 4 files changed, 616 insertions(+) create mode 100644 chat/templates/kickoff_confirm.html create mode 100644 chat/web/kickoff.py create mode 100644 tests/test_kickoff_confirm.py diff --git a/chat/app.py b/chat/app.py index 82e49ad..4251290 100644 --- a/chat/app.py +++ b/chat/app.py @@ -15,6 +15,7 @@ import chat.state.memory # noqa: F401 import chat.state.world # noqa: F401 from chat.web.bots import router as bots_router +from chat.web.kickoff import router as kickoff_router from chat.web.settings import router as settings_router @@ -33,6 +34,7 @@ STATIC_DIR = Path(__file__).resolve().parent / "static" app.mount("/static", StaticFiles(directory=str(STATIC_DIR)), name="static") app.include_router(bots_router) +app.include_router(kickoff_router) app.include_router(settings_router) diff --git a/chat/templates/kickoff_confirm.html b/chat/templates/kickoff_confirm.html new file mode 100644 index 0000000..d830f39 --- /dev/null +++ b/chat/templates/kickoff_confirm.html @@ -0,0 +1,118 @@ +{% extends "base.html" %} +{% block title %}Confirm kickoff - chat{% endblock %} +{% block content %} +

Confirm kickoff

+

Review and edit the parsed opening scene for {{ values.bot_name }}, then confirm to start the chat.

+ +
+ +
+ Container + + + +
+ +
+ Initial in-fiction time + +
+ +
+ Your activity + + + + + + + +
+ +
+ {{ values.bot_name }}'s activity + + + + + + + +
+ +
+ Edge seed + + +
+ +
+ + Cancel +
+
+{% endblock %} diff --git a/chat/web/kickoff.py b/chat/web/kickoff.py new file mode 100644 index 0000000..4edfba9 --- /dev/null +++ b/chat/web/kickoff.py @@ -0,0 +1,284 @@ +"""Kickoff parse-and-confirm flow. + +After a bot is authored, the user lands on ``/bots//kickoff``. We call the +LLM-backed ``parse_kickoff`` to extract a structured opening scene from the +authored prose and render it as an editable form. On submit, the (possibly +edited) values are turned into a sequence of events that initialize the chat, +its container, the participants' activities, an open scene, and a seed edge. +""" + +from __future__ import annotations + +import json +from datetime import datetime +from pathlib import Path + +from fastapi import APIRouter, Depends, Form, HTTPException, Request +from fastapi.responses import HTMLResponse, RedirectResponse +from fastapi.templating import Jinja2Templates + +from chat.eventlog.log import append_event +from chat.eventlog.projector import project +from chat.llm.client import LLMClient +from chat.services.kickoff import parse_kickoff +from chat.state.entities import get_bot, get_you +from chat.web.bots import get_conn + +TEMPLATES = Jinja2Templates( + directory=str(Path(__file__).resolve().parent.parent / "templates") +) + +router = APIRouter() + + +def get_llm_client(request: Request) -> LLMClient: + """Production LLM client. Tests override this via ``app.dependency_overrides``.""" + settings = request.app.state.settings + from chat.llm.featherless import FeatherlessClient + + return FeatherlessClient( + api_key=settings.featherless_api_key, + base_url=settings.featherless_base_url, + ) + + +def _parse_holding(text: str) -> list[str]: + if not text or not text.strip(): + return [] + return [p.strip() for p in text.split(",") if p.strip()] + + +def _parse_facts(text: str) -> list[str]: + if not text or not text.strip(): + return [] + return [line.strip() for line in text.splitlines() if line.strip()] + + +def _parse_properties(text: str) -> dict: + """Parse the container_properties textarea as JSON. + + Returns ``{}`` on invalid JSON rather than raising — the form is editable + and a bad value should not block the user from confirming the rest. + """ + if not text or not text.strip(): + return {} + try: + loaded = json.loads(text) + return loaded if isinstance(loaded, dict) else {} + except (json.JSONDecodeError, ValueError): + return {} + + +@router.get("/bots/{bot_id}/kickoff", response_class=HTMLResponse) +async def kickoff_get( + bot_id: str, + request: Request, + conn=Depends(get_conn), + llm=Depends(get_llm_client), +): + bot = get_bot(conn, bot_id) + if bot is None: + raise HTTPException(status_code=404, detail=f"bot not found: {bot_id}") + + you = get_you(conn) + you_name = you["name"] if you else "You" + + settings = request.app.state.settings + parsed = await parse_kickoff( + llm, + model=settings.classifier_model, + bot_name=bot["name"], + bot_persona=bot["persona"], + initial_relationship_to_you=bot.get("initial_relationship_to_you", ""), + kickoff_prose=bot.get("kickoff_prose", ""), + you_name=you_name, + timeout_s=settings.classifier_timeout_s, + ) + + # Render values onto the form. ``container_properties`` is shown as JSON; + # ``holding`` lists are rendered as comma-separated text; the seed + # knowledge facts are rendered one-per-line. + values = { + "bot_id": bot_id, + "bot_name": bot["name"], + "container_name": parsed.container_name, + "container_type": parsed.container_type, + "container_properties": json.dumps(parsed.container_properties, indent=2), + "initial_time_iso": parsed.initial_time_iso, + "you_activity_posture": parsed.you_activity.posture, + "you_activity_action_verb": parsed.you_activity.action_verb, + "you_activity_action_interruptible": parsed.you_activity.action_interruptible, + "you_activity_action_required_attention": parsed.you_activity.action_required_attention, + "you_activity_action_expected_duration": parsed.you_activity.action_expected_duration, + "you_activity_attention": parsed.you_activity.attention, + "you_activity_holding": ", ".join(parsed.you_activity.holding), + "bot_activity_posture": parsed.bot_activity.posture, + "bot_activity_action_verb": parsed.bot_activity.action_verb, + "bot_activity_action_interruptible": parsed.bot_activity.action_interruptible, + "bot_activity_action_required_attention": parsed.bot_activity.action_required_attention, + "bot_activity_action_expected_duration": parsed.bot_activity.action_expected_duration, + "bot_activity_attention": parsed.bot_activity.attention, + "bot_activity_holding": ", ".join(parsed.bot_activity.holding), + "edge_seed_summary": parsed.edge_seed_summary, + "edge_seed_knowledge_facts": "\n".join(parsed.edge_seed_knowledge_facts), + } + return TEMPLATES.TemplateResponse(request, "kickoff_confirm.html", {"values": values}) + + +@router.post("/bots/{bot_id}/kickoff") +async def kickoff_post( + bot_id: str, + request: Request, + container_name: str = Form(""), + container_type: str = Form(""), + container_properties: str = Form(""), + initial_time_iso: str = Form(""), + you_activity_posture: str = Form(""), + you_activity_action_verb: str = Form(""), + you_activity_action_interruptible: str = Form(""), + you_activity_action_required_attention: str = Form("low"), + you_activity_action_expected_duration: str = Form(""), + you_activity_attention: str = Form(""), + you_activity_holding: str = Form(""), + bot_activity_posture: str = Form(""), + bot_activity_action_verb: str = Form(""), + bot_activity_action_interruptible: str = Form(""), + bot_activity_action_required_attention: str = Form("low"), + bot_activity_action_expected_duration: str = Form(""), + bot_activity_attention: str = Form(""), + bot_activity_holding: str = Form(""), + edge_seed_summary: str = Form(""), + edge_seed_knowledge_facts: str = Form(""), + conn=Depends(get_conn), +): + bot = get_bot(conn, bot_id) + if bot is None: + raise HTTPException(status_code=404, detail=f"bot not found: {bot_id}") + + # Loose ISO 8601 validation. ``datetime.fromisoformat`` accepts the offset + # form ``2026-04-26T20:00:00+00:00`` we use; reject anything it can't parse. + if initial_time_iso.strip(): + try: + datetime.fromisoformat(initial_time_iso.strip()) + except ValueError: + raise HTTPException( + status_code=400, + detail=f"invalid initial_time_iso: {initial_time_iso!r}", + ) + + chat_id = f"chat_{bot_id}" + + # Predict the next container id so we can reference it from later events + # without needing a mid-flow projection. Containers use AUTOINCREMENT-style + # rowid, so MAX(id)+1 is safe within this single-writer transaction. + next_container_row = conn.execute( + "SELECT COALESCE(MAX(id), 0) + 1 FROM containers" + ).fetchone() + container_id = next_container_row[0] + + # 1. chat_created + append_event( + conn, + kind="chat_created", + payload={ + "id": chat_id, + "host_bot_id": bot_id, + "initial_time": initial_time_iso, + "narrative_anchor": "Day 1", + "weather": "", + }, + ) + + # 2. container_created + append_event( + conn, + kind="container_created", + payload={ + "chat_id": chat_id, + "name": container_name, + "type": container_type, + "properties": _parse_properties(container_properties), + "parent_id": None, + }, + ) + + you_interruptible = bool(you_activity_action_interruptible) + bot_interruptible = bool(bot_activity_action_interruptible) + + # 3. activity_change for "you" + append_event( + conn, + kind="activity_change", + payload={ + "entity_id": "you", + "container_id": container_id, + "posture": you_activity_posture, + "action": { + "verb": you_activity_action_verb, + "interruptible": you_interruptible, + "required_attention": you_activity_action_required_attention, + "expected_duration": you_activity_action_expected_duration, + "started_at": initial_time_iso, + }, + "attention": you_activity_attention, + "holding": _parse_holding(you_activity_holding), + "status": {}, + }, + ) + + # 4. activity_change for bot + append_event( + conn, + kind="activity_change", + payload={ + "entity_id": bot_id, + "container_id": container_id, + "posture": bot_activity_posture, + "action": { + "verb": bot_activity_action_verb, + "interruptible": bot_interruptible, + "required_attention": bot_activity_action_required_attention, + "expected_duration": bot_activity_action_expected_duration, + "started_at": initial_time_iso, + }, + "attention": bot_activity_attention, + "holding": _parse_holding(bot_activity_holding), + "status": {}, + }, + ) + + # 5. scene_opened + append_event( + conn, + kind="scene_opened", + payload={ + "chat_id": chat_id, + "container_id": container_id, + "started_at": initial_time_iso, + "participants": ["you", bot_id], + }, + ) + + # 6. edge_update (seed). The seed summary is preserved as the first + # knowledge fact prefixed with ``[summary] `` — proper summary writes happen + # at scene-close (T27). + facts = _parse_facts(edge_seed_knowledge_facts) + if edge_seed_summary.strip(): + facts.insert(0, f"[summary] {edge_seed_summary.strip()}") + append_event( + conn, + kind="edge_update", + payload={ + "source_id": bot_id, + "target_id": "you", + "chat_id": chat_id, + "knowledge_facts": facts, + }, + ) + + # Project all events at once. ``bot_authored`` (already in log from prior + # POST) is idempotent (INSERT OR REPLACE); the new events project cleanly + # because they're being applied for the first time. + project(conn) + + return RedirectResponse(url=f"/chats/{chat_id}", status_code=303) diff --git a/tests/test_kickoff_confirm.py b/tests/test_kickoff_confirm.py new file mode 100644 index 0000000..ef308cb --- /dev/null +++ b/tests/test_kickoff_confirm.py @@ -0,0 +1,212 @@ +from __future__ import annotations + +import json +from pathlib import Path + +import pytest +from fastapi.testclient import TestClient + +from chat.app import app +from chat.eventlog.log import append_event +from chat.eventlog.projector import project +from chat.llm.mock import MockLLMClient + + +CANNED_PARSE = { + "container_name": "office", + "container_type": "workplace", + "container_properties": { + "public": True, + "moving": False, + "audible_range": "normal", + }, + "you_activity": { + "posture": "sitting", + "action_verb": "working late", + "action_interruptible": True, + "action_required_attention": "medium", + "action_expected_duration": "an hour", + "attention": "the screen", + "holding": [], + }, + "bot_activity": { + "posture": "sitting", + "action_verb": "writing email", + "action_interruptible": True, + "action_required_attention": "medium", + "action_expected_duration": "a few minutes", + "attention": "her keyboard", + "holding": [], + }, + "initial_time_iso": "2026-04-26T20:00:00+00:00", + "edge_seed_summary": "BotA is your coworker.", + "edge_seed_knowledge_facts": [ + "coworker", + "they sometimes stay late together", + ], +} + + +@pytest.fixture +def client(tmp_path, monkeypatch): + config_path = tmp_path / "config.toml" + config_path.write_text('featherless_api_key = "test"\n') + monkeypatch.setenv("CHAT_CONFIG_PATH", str(config_path)) + monkeypatch.setenv("CHAT_DB_PATH", str(tmp_path / "test.db")) + + # Import after env is set so dependency lookup uses MockLLMClient. + from chat.web.kickoff import get_llm_client + + mock = MockLLMClient(canned=[json.dumps(CANNED_PARSE)]) + app.dependency_overrides[get_llm_client] = lambda: mock + + with TestClient(app) as c: + c.mock_llm = mock # type: ignore[attr-defined] + yield c + + app.dependency_overrides.clear() + + +def _author_bot(db_path: Path, bot_id: str = "bot_a") -> None: + from chat.db.connection import open_db + + with open_db(db_path) as conn: + append_event( + conn, + kind="bot_authored", + payload={ + "id": bot_id, + "name": "BotA", + "persona": "thoughtful, observant", + "voice_samples": [], + "traits": ["shy"], + "backstory": "", + "initial_relationship_to_you": "coworker", + "kickoff_prose": "you stay late at the office; she's there too", + }, + ) + project(conn) + + +def test_get_kickoff_404_when_bot_missing(client): + response = client.get("/bots/no_such_bot/kickoff") + assert response.status_code == 404 + + +def test_get_kickoff_renders_parsed_form(client, tmp_path): + _author_bot(tmp_path / "test.db", "bot_a") + response = client.get("/bots/bot_a/kickoff") + assert response.status_code == 200 + body = response.text + assert "office" in body + assert "sitting" in body + assert "working late" in body + # Mock was consumed once. + assert len(client.mock_llm._canned) == 0 + + +def test_post_kickoff_creates_chat_and_redirects(client, tmp_path): + _author_bot(tmp_path / "test.db", "bot_a") + + form_data = { + "container_name": "office", + "container_type": "workplace", + "container_properties": json.dumps(CANNED_PARSE["container_properties"]), + "initial_time_iso": "2026-04-26T20:00:00+00:00", + "you_activity_posture": "sitting", + "you_activity_action_verb": "working late", + "you_activity_action_interruptible": "on", + "you_activity_action_required_attention": "medium", + "you_activity_action_expected_duration": "an hour", + "you_activity_attention": "the screen", + "you_activity_holding": "", + "bot_activity_posture": "sitting", + "bot_activity_action_verb": "writing email", + "bot_activity_action_interruptible": "on", + "bot_activity_action_required_attention": "medium", + "bot_activity_action_expected_duration": "a few minutes", + "bot_activity_attention": "her keyboard", + "bot_activity_holding": "", + "edge_seed_summary": "BotA is your coworker.", + "edge_seed_knowledge_facts": "coworker\nthey sometimes stay late together", + } + response = client.post( + "/bots/bot_a/kickoff", + data=form_data, + follow_redirects=False, + ) + assert response.status_code == 303 + assert response.headers["location"] == "/chats/chat_bot_a" + + from chat.db.connection import open_db + from chat.state.world import ( + active_scene, + find_container, + get_activity, + get_chat, + ) + from chat.state.edges import get_edge + + with open_db(tmp_path / "test.db") as conn: + chat = get_chat(conn, "chat_bot_a") + assert chat is not None + assert chat["host_bot_id"] == "bot_a" + assert chat["time"] == "2026-04-26T20:00:00+00:00" + + container = find_container(conn, "chat_bot_a", "office") + assert container is not None + assert container["type"] == "workplace" + + you_act = get_activity(conn, "you") + assert you_act is not None + assert you_act["posture"] == "sitting" + assert you_act["action"]["verb"] == "working late" + + bot_act = get_activity(conn, "bot_a") + assert bot_act is not None + assert bot_act["posture"] == "sitting" + assert bot_act["action"]["verb"] == "writing email" + + scene = active_scene(conn, "chat_bot_a") + assert scene is not None + assert scene["ended_at"] is None + assert "you" in scene["participants"] + assert "bot_a" in scene["participants"] + + edge = get_edge(conn, "bot_a", "you") + assert edge is not None + knowledge = edge["knowledge"] + assert "coworker" in knowledge + assert "they sometimes stay late together" in knowledge + # The seed summary should appear somewhere in knowledge as a v1 compromise. + assert any("BotA is your coworker" in k for k in knowledge) + + +def test_post_kickoff_404_when_bot_missing(client): + response = client.post( + "/bots/no_such/kickoff", + data={ + "container_name": "office", + "container_type": "workplace", + "container_properties": "{}", + "initial_time_iso": "2026-04-26T20:00:00+00:00", + "you_activity_posture": "", + "you_activity_action_verb": "", + "you_activity_action_interruptible": "on", + "you_activity_action_required_attention": "low", + "you_activity_action_expected_duration": "", + "you_activity_attention": "", + "you_activity_holding": "", + "bot_activity_posture": "", + "bot_activity_action_verb": "", + "bot_activity_action_interruptible": "on", + "bot_activity_action_required_attention": "low", + "bot_activity_action_expected_duration": "", + "bot_activity_attention": "", + "bot_activity_holding": "", + "edge_seed_summary": "", + "edge_seed_knowledge_facts": "", + }, + follow_redirects=False, + ) + assert response.status_code == 404 -- 2.52.0 From 0c08745194776c46b90082923c13f81754e4c2a7 Mon Sep 17 00:00:00 2001 From: Joseph Doherty Date: Sun, 26 Apr 2026 12:36:20 -0400 Subject: [PATCH 16/45] feat: top-level nav and chat list view --- chat/app.py | 2 + chat/static/app.css | 37 ++++++++++-- chat/templates/base.html | 7 +-- chat/templates/bot_form.html | 2 +- chat/templates/bot_list.html | 2 +- chat/templates/chat_list.html | 26 +++++++++ chat/templates/kickoff_confirm.html | 2 +- chat/templates/layout.html | 14 +++++ chat/templates/settings.html | 2 +- chat/web/bots.py | 8 ++- chat/web/kickoff.py | 4 +- chat/web/nav.py | 34 +++++++++++ chat/web/settings.py | 8 ++- tests/test_chat_list.py | 89 +++++++++++++++++++++++++++++ 14 files changed, 218 insertions(+), 19 deletions(-) create mode 100644 chat/templates/chat_list.html create mode 100644 chat/templates/layout.html create mode 100644 chat/web/nav.py create mode 100644 tests/test_chat_list.py diff --git a/chat/app.py b/chat/app.py index 4251290..a0924d4 100644 --- a/chat/app.py +++ b/chat/app.py @@ -16,6 +16,7 @@ import chat.state.world # noqa: F401 from chat.web.bots import router as bots_router from chat.web.kickoff import router as kickoff_router +from chat.web.nav import router as nav_router from chat.web.settings import router as settings_router @@ -36,6 +37,7 @@ app.mount("/static", StaticFiles(directory=str(STATIC_DIR)), name="static") app.include_router(bots_router) app.include_router(kickoff_router) app.include_router(settings_router) +app.include_router(nav_router) @app.get("/health") diff --git a/chat/static/app.css b/chat/static/app.css index 9c261a8..4645d19 100644 --- a/chat/static/app.css +++ b/chat/static/app.css @@ -4,11 +4,33 @@ body { margin: 0; color: #1c1c1c; background: #fafafa; + display: flex; + min-height: 100vh; } -.topbar { - padding: 12px 24px; - border-bottom: 1px solid #e5e5e5; - background: #fff; +.rail { + width: 200px; + background: #1c1c1c; + color: #fff; + padding: 16px; + flex-shrink: 0; +} +.rail a { color: #fff; text-decoration: none; } +.rail-brand { + font-weight: 600; + display: block; + padding-bottom: 16px; + border-bottom: 1px solid #333; + margin-bottom: 16px; +} +.rail ul { list-style: none; padding: 0; margin: 0; } +.rail li { margin: 4px 0; } +.rail li a { display: block; padding: 6px 8px; border-radius: 3px; } +.rail li a.active { background: #333; } +.content { + flex: 1; + padding: 24px; + background: #fafafa; + overflow: auto; } .brand { font-weight: 600; text-decoration: none; color: inherit; } .container { max-width: 720px; margin: 24px auto; padding: 0 16px; } @@ -29,6 +51,13 @@ h1 { margin-top: 0; } .bot-form small { display: block; color: #666; margin-top: 2px; } .bot-list { list-style: none; padding: 0; } .bot-list li { padding: 8px 0; border-bottom: 1px solid #eee; } +.chat-list { list-style: none; padding: 0; margin: 0; } +.chat-row { border-bottom: 1px solid #eee; } +.chat-row a { display: block; padding: 12px 0; text-decoration: none; color: inherit; } +.chat-row a:hover { background: #f0f0f0; } +.chat-row-name { font-weight: 600; } +.chat-row-snippet { font-size: 14px; } +.chat-row-meta { font-size: 12px; } .muted { color: #666; } .error { padding: 8px 12px; border: 1px solid #c33; background: #fdecea; diff --git a/chat/templates/base.html b/chat/templates/base.html index 6c86104..01d0abd 100644 --- a/chat/templates/base.html +++ b/chat/templates/base.html @@ -8,11 +8,6 @@ -
- chat -
-
- {% block content %}{% endblock %} -
+ {% block body %}{% endblock %} diff --git a/chat/templates/bot_form.html b/chat/templates/bot_form.html index e0d3331..6d44080 100644 --- a/chat/templates/bot_form.html +++ b/chat/templates/bot_form.html @@ -1,4 +1,4 @@ -{% extends "base.html" %} +{% extends "layout.html" %} {% block title %}New bot - chat{% endblock %} {% block content %}

New bot

diff --git a/chat/templates/bot_list.html b/chat/templates/bot_list.html index f3b4562..7a2a65b 100644 --- a/chat/templates/bot_list.html +++ b/chat/templates/bot_list.html @@ -1,4 +1,4 @@ -{% extends "base.html" %} +{% extends "layout.html" %} {% block title %}Bots - chat{% endblock %} {% block content %}
diff --git a/chat/templates/chat_list.html b/chat/templates/chat_list.html new file mode 100644 index 0000000..a16a1ed --- /dev/null +++ b/chat/templates/chat_list.html @@ -0,0 +1,26 @@ +{% extends "layout.html" %} +{% block title %}Chats - chat{% endblock %} +{% block content %} +
+

Chats

+ + New bot +
+{% if chats %} + +{% else %} +

No chats yet. Create a bot to start.

+{% endif %} +{% endblock %} diff --git a/chat/templates/kickoff_confirm.html b/chat/templates/kickoff_confirm.html index d830f39..56f6753 100644 --- a/chat/templates/kickoff_confirm.html +++ b/chat/templates/kickoff_confirm.html @@ -1,4 +1,4 @@ -{% extends "base.html" %} +{% extends "layout.html" %} {% block title %}Confirm kickoff - chat{% endblock %} {% block content %}

Confirm kickoff

diff --git a/chat/templates/layout.html b/chat/templates/layout.html new file mode 100644 index 0000000..7b1954b --- /dev/null +++ b/chat/templates/layout.html @@ -0,0 +1,14 @@ +{% extends "base.html" %} +{% block body %} + +
+ {% block content %}{% endblock %} +
+{% endblock %} diff --git a/chat/templates/settings.html b/chat/templates/settings.html index e26fcf6..26671f8 100644 --- a/chat/templates/settings.html +++ b/chat/templates/settings.html @@ -1,4 +1,4 @@ -{% extends "base.html" %} +{% extends "layout.html" %} {% block title %}Settings - chat{% endblock %} {% block content %}

Settings

diff --git a/chat/web/bots.py b/chat/web/bots.py index bbf077d..d06ba7b 100644 --- a/chat/web/bots.py +++ b/chat/web/bots.py @@ -66,12 +66,16 @@ def _split_traits(text: str) -> list[str]: @router.get("/bots", response_class=HTMLResponse) async def bots_list(request: Request, conn=Depends(get_conn)): bots = list_bots(conn) - return TEMPLATES.TemplateResponse(request, "bot_list.html", {"bots": bots}) + return TEMPLATES.TemplateResponse( + request, "bot_list.html", {"bots": bots, "active_nav": "bots"} + ) @router.get("/bots/new", response_class=HTMLResponse) async def bot_form(request: Request): - return TEMPLATES.TemplateResponse(request, "bot_form.html", {"values": {}, "error": None}) + return TEMPLATES.TemplateResponse( + request, "bot_form.html", {"values": {}, "error": None, "active_nav": "bots"} + ) @router.post("/bots/new") diff --git a/chat/web/kickoff.py b/chat/web/kickoff.py index 4edfba9..320b771 100644 --- a/chat/web/kickoff.py +++ b/chat/web/kickoff.py @@ -122,7 +122,9 @@ async def kickoff_get( "edge_seed_summary": parsed.edge_seed_summary, "edge_seed_knowledge_facts": "\n".join(parsed.edge_seed_knowledge_facts), } - return TEMPLATES.TemplateResponse(request, "kickoff_confirm.html", {"values": values}) + return TEMPLATES.TemplateResponse( + request, "kickoff_confirm.html", {"values": values, "active_nav": "bots"} + ) @router.post("/bots/{bot_id}/kickoff") diff --git a/chat/web/nav.py b/chat/web/nav.py new file mode 100644 index 0000000..842f2e0 --- /dev/null +++ b/chat/web/nav.py @@ -0,0 +1,34 @@ +from __future__ import annotations +from pathlib import Path +from fastapi import APIRouter, Depends, Request +from fastapi.responses import HTMLResponse, RedirectResponse +from fastapi.templating import Jinja2Templates + +from chat.web.bots import get_conn +from chat.state.world import list_chats +from chat.state.entities import get_bot + +TEMPLATES = Jinja2Templates(directory=str(Path(__file__).resolve().parent.parent / "templates")) + +router = APIRouter() + + +@router.get("/", include_in_schema=False) +async def home(): + return RedirectResponse(url="/chats", status_code=303) + + +@router.get("/chats", response_class=HTMLResponse) +async def chats_list(request: Request, conn=Depends(get_conn)): + chats = list_chats(conn) + # Annotate each chat with the host bot's name for display. + for ch in chats: + bot = get_bot(conn, ch["host_bot_id"]) + ch["host_bot_name"] = bot["name"] if bot else ch["host_bot_id"] + # Last-message snippet and last-played-at are blank in v1; T19 fills them. + ch["last_message_snippet"] = "" + ch["last_played_at"] = None + return TEMPLATES.TemplateResponse(request, "chat_list.html", { + "chats": chats, + "active_nav": "chats", + }) diff --git a/chat/web/settings.py b/chat/web/settings.py index cb04d58..8256843 100644 --- a/chat/web/settings.py +++ b/chat/web/settings.py @@ -18,7 +18,9 @@ router = APIRouter() async def settings_get(request: Request, conn=Depends(get_conn)): you = get_you(conn) or {"name": "", "pronouns": "", "persona": ""} return TEMPLATES.TemplateResponse( - request, "settings.html", {"values": you, "saved": False} + request, + "settings.html", + {"values": you, "saved": False, "active_nav": "settings"}, ) @@ -42,5 +44,7 @@ async def settings_post( project(conn) return TEMPLATES.TemplateResponse( - request, "settings.html", {"values": payload, "saved": True} + request, + "settings.html", + {"values": payload, "saved": True, "active_nav": "settings"}, ) diff --git a/tests/test_chat_list.py b/tests/test_chat_list.py new file mode 100644 index 0000000..1f9c998 --- /dev/null +++ b/tests/test_chat_list.py @@ -0,0 +1,89 @@ +from __future__ import annotations + +from pathlib import Path + +import pytest +from fastapi.testclient import TestClient + +from chat.app import app +from chat.eventlog.log import append_event +from chat.eventlog.projector import project + + +@pytest.fixture +def client(tmp_path, monkeypatch): + config_path = tmp_path / "config.toml" + config_path.write_text('featherless_api_key = "test"\n') + monkeypatch.setenv("CHAT_CONFIG_PATH", str(config_path)) + monkeypatch.setenv("CHAT_DB_PATH", str(tmp_path / "test.db")) + with TestClient(app) as c: + yield c + + +def _author_bot_and_chat(db_path: Path, bot_id: str = "bot_a") -> None: + """Insert a bot and a chat directly via the event log (skip kickoff route).""" + from chat.db.connection import open_db + + with open_db(db_path) as conn: + append_event( + conn, + kind="bot_authored", + payload={ + "id": bot_id, + "name": "BotA", + "persona": "thoughtful, observant", + "voice_samples": [], + "traits": ["shy"], + "backstory": "", + "initial_relationship_to_you": "coworker", + "kickoff_prose": "you stay late at the office; she's there too", + }, + ) + append_event( + conn, + kind="chat_created", + payload={ + "id": f"chat_{bot_id}", + "host_bot_id": bot_id, + "initial_time": "2026-04-26T20:00:00+00:00", + "narrative_anchor": "Day 1", + "weather": "", + }, + ) + project(conn) + + +def test_root_redirects_to_chats(client): + response = client.get("/", follow_redirects=False) + assert response.status_code == 303 + assert response.headers["location"] == "/chats" + + +def test_chats_list_empty_state(client): + response = client.get("/chats") + assert response.status_code == 200 + body = response.text.lower() + # Empty state should mention there are no chats yet. + assert "no chats yet" in body + + +def test_chats_list_renders_existing_chats(client, tmp_path): + _author_bot_and_chat(tmp_path / "test.db", "bot_a") + + response = client.get("/chats") + assert response.status_code == 200 + body = response.text + # The bot's display name should appear in the chat row. + assert "BotA" in body + # The chat's in-fiction time should appear in the meta. + assert "2026-04-26T20:00:00+00:00" in body + + +def test_existing_template_routes_still_work_with_new_layout(client): + # Smoke test the layout reshuffle didn't break the existing pages. + for path in ("/bots", "/bots/new", "/settings"): + response = client.get(path) + assert response.status_code == 200, f"{path} returned {response.status_code}" + body = response.text + # Each page should now show the persistent left-rail brand link. + assert 'class="rail"' in body or "rail-brand" in body -- 2.52.0 From e79f4d8d222ecc3695972e4496b937993d3a467e Mon Sep 17 00:00:00 2001 From: Joseph Doherty Date: Sun, 26 Apr 2026 12:39:15 -0400 Subject: [PATCH 17/45] feat: chat shell page rendering --- chat/app.py | 2 + chat/static/app.css | 11 ++++++ chat/templates/chat.html | 42 ++++++++++++++++++++ chat/web/chat.py | 54 ++++++++++++++++++++++++++ tests/test_chat_shell.py | 83 ++++++++++++++++++++++++++++++++++++++++ 5 files changed, 192 insertions(+) create mode 100644 chat/templates/chat.html create mode 100644 chat/web/chat.py create mode 100644 tests/test_chat_shell.py diff --git a/chat/app.py b/chat/app.py index a0924d4..a2033cf 100644 --- a/chat/app.py +++ b/chat/app.py @@ -15,6 +15,7 @@ import chat.state.memory # noqa: F401 import chat.state.world # noqa: F401 from chat.web.bots import router as bots_router +from chat.web.chat import router as chat_router from chat.web.kickoff import router as kickoff_router from chat.web.nav import router as nav_router from chat.web.settings import router as settings_router @@ -38,6 +39,7 @@ app.include_router(bots_router) app.include_router(kickoff_router) app.include_router(settings_router) app.include_router(nav_router) +app.include_router(chat_router) @app.get("/health") diff --git a/chat/static/app.css b/chat/static/app.css index 4645d19..f40bd40 100644 --- a/chat/static/app.css +++ b/chat/static/app.css @@ -68,3 +68,14 @@ h1 { margin-top: 0; } color: #1f5c2a; border-radius: 3px; } code { font-family: ui-monospace, "SF Mono", Menlo, monospace; } +.chat-shell { display: flex; flex-direction: column; height: 100%; max-width: 760px; margin: 0 auto; } +.chat-header { display: flex; align-items: center; gap: 16px; border-bottom: 1px solid #e5e5e5; padding-bottom: 8px; margin-bottom: 16px; } +.chat-header h1 { margin: 0; flex: 1; } +.chat-meta { font-size: 13px; } +.drawer-toggle { padding: 4px 10px; border: 1px solid #ccc; background: #fff; color: #1c1c1c; border-radius: 3px; cursor: pointer; } +.timeline { flex: 1; overflow-y: auto; min-height: 200px; padding: 8px 0; } +.turn { margin: 12px 0; } +.turn-input { display: flex; flex-direction: column; gap: 8px; padding-top: 12px; border-top: 1px solid #e5e5e5; } +.turn-input textarea { padding: 8px; font: inherit; border: 1px solid #ccc; border-radius: 3px; resize: vertical; } +.drawer { position: fixed; top: 0; right: 0; width: 360px; height: 100vh; background: #fff; border-left: 1px solid #e5e5e5; padding: 16px; overflow-y: auto; z-index: 10; } +.drawer[hidden] { display: none; } diff --git a/chat/templates/chat.html b/chat/templates/chat.html new file mode 100644 index 0000000..9aa04b4 --- /dev/null +++ b/chat/templates/chat.html @@ -0,0 +1,42 @@ +{% extends "layout.html" %} +{% block title %}{{ host_bot.name }} - chat{% endblock %} +{% block content %} +
+
+

{{ host_bot.name }}

+
{{ chat.time }}
+ +
+ +
+ {% if not turns %} +

No turns yet. Start typing below.

+ {% else %} + {% for turn in turns %} +
+ {{ turn.speaker }} +

{{ turn.text }}

+
+ {% endfor %} + {% endif %} +
+ +
+ + +
+ + +
+ +{% endblock %} diff --git a/chat/web/chat.py b/chat/web/chat.py new file mode 100644 index 0000000..35d12ef --- /dev/null +++ b/chat/web/chat.py @@ -0,0 +1,54 @@ +"""Chat detail (shell) page. + +Renders ``/chats/``: the title (host bot's name), a timeline placeholder, +the user-input form, and the drawer toggle. Turn handling lives in T19; this +module only sets up the structural shell. +""" + +from __future__ import annotations + +from pathlib import Path + +from fastapi import APIRouter, Depends, HTTPException, Request +from fastapi.responses import HTMLResponse +from fastapi.templating import Jinja2Templates + +from chat.state.entities import get_bot +from chat.state.world import get_chat +from chat.web.bots import get_conn + +TEMPLATES = Jinja2Templates( + directory=str(Path(__file__).resolve().parent.parent / "templates") +) + +router = APIRouter() + + +@router.get("/chats/{chat_id}", response_class=HTMLResponse) +async def chat_detail(chat_id: str, request: Request, conn=Depends(get_conn)): + chat = get_chat(conn, chat_id) + if chat is None: + raise HTTPException(status_code=404, detail=f"chat not found: {chat_id}") + + host_bot = get_bot(conn, chat["host_bot_id"]) + if host_bot is None: + # Defensive: chat row references a bot that doesn't exist. Treat as 404 + # rather than crashing the template render. + raise HTTPException( + status_code=404, detail=f"host bot not found: {chat['host_bot_id']}" + ) + + # Phase 1, T15: timeline starts empty. T19 will populate from event_log + # by reading user_turn / assistant_turn events for this chat. + turns: list[dict] = [] + + return TEMPLATES.TemplateResponse( + request, + "chat.html", + { + "chat": chat, + "host_bot": host_bot, + "turns": turns, + "active_nav": "chats", + }, + ) diff --git a/tests/test_chat_shell.py b/tests/test_chat_shell.py new file mode 100644 index 0000000..5cd3577 --- /dev/null +++ b/tests/test_chat_shell.py @@ -0,0 +1,83 @@ +from __future__ import annotations + +from pathlib import Path + +import pytest +from fastapi.testclient import TestClient + +from chat.app import app +from chat.db.connection import open_db +from chat.eventlog.log import append_event +from chat.eventlog.projector import project + + +@pytest.fixture +def client(tmp_path, monkeypatch): + cfg = tmp_path / "config.toml" + cfg.write_text('featherless_api_key = "test"\n') + monkeypatch.setenv("CHAT_CONFIG_PATH", str(cfg)) + db = tmp_path / "test.db" + monkeypatch.setenv("CHAT_DB_PATH", str(db)) + with TestClient(app) as c: + yield c + + +def _seed_chat(db_path: Path, bot_id: str = "bot_a", chat_id: str = "chat_bot_a") -> None: + """Author a bot, create a chat with default state.""" + with open_db(db_path) as conn: + append_event( + conn, + kind="bot_authored", + payload={ + "id": bot_id, + "name": "BotA", + "persona": "...", + "voice_samples": [], + "traits": [], + "backstory": "", + "initial_relationship_to_you": "", + "kickoff_prose": "...", + }, + ) + append_event( + conn, + kind="chat_created", + payload={ + "id": chat_id, + "host_bot_id": bot_id, + "initial_time": "2026-04-26T20:00:00+00:00", + "narrative_anchor": "Day 1", + "weather": "", + }, + ) + project(conn) + + +def test_get_chat_404_when_missing(client): + response = client.get("/chats/no_such_chat") + assert response.status_code == 404 + + +def test_get_chat_renders_shell_with_host_bot_name(client, tmp_path): + _seed_chat(tmp_path / "test.db") + response = client.get("/chats/chat_bot_a") + assert response.status_code == 200 + body = response.text + assert "BotA" in body + assert " Date: Sun, 26 Apr 2026 12:49:41 -0400 Subject: [PATCH 18/45] feat: per-chat SSE channel and pub/sub --- chat/app.py | 2 + chat/web/pubsub.py | 60 ++++++++++++++++ chat/web/sse.py | 72 +++++++++++++++++++ tests/test_sse.py | 174 +++++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 308 insertions(+) create mode 100644 chat/web/pubsub.py create mode 100644 chat/web/sse.py create mode 100644 tests/test_sse.py diff --git a/chat/app.py b/chat/app.py index a2033cf..02b936d 100644 --- a/chat/app.py +++ b/chat/app.py @@ -19,6 +19,7 @@ from chat.web.chat import router as chat_router from chat.web.kickoff import router as kickoff_router from chat.web.nav import router as nav_router from chat.web.settings import router as settings_router +from chat.web.sse import router as sse_router @asynccontextmanager @@ -40,6 +41,7 @@ app.include_router(kickoff_router) app.include_router(settings_router) app.include_router(nav_router) app.include_router(chat_router) +app.include_router(sse_router) @app.get("/health") diff --git a/chat/web/pubsub.py b/chat/web/pubsub.py new file mode 100644 index 0000000..533c975 --- /dev/null +++ b/chat/web/pubsub.py @@ -0,0 +1,60 @@ +"""In-process per-chat broadcast channel. + +Each ``chat_id`` has a list of subscriber ``asyncio.Queue`` instances. T16 +provides only the registry and fan-out mechanism; T19+ will publish events +(turn appends, streamed tokens, drawer updates, scene close, edge updates) +through this channel so all browser tabs viewing a chat stay in sync. + +The registry is process-local: appropriate for a single-user local server. +""" + +from __future__ import annotations + +import asyncio +from collections import defaultdict +from typing import Any + +# {chat_id: [queue, queue, ...]} +_subscribers: dict[str, list[asyncio.Queue]] = defaultdict(list) +_lock = asyncio.Lock() + + +async def subscribe(chat_id: str) -> asyncio.Queue: + """Subscribe to a chat's broadcast channel. + + Returns a fresh ``asyncio.Queue`` that will receive every event published + to ``chat_id`` while the subscription is active. Callers must invoke + :func:`unsubscribe` when finished (typically on client disconnect) to + avoid leaking queues into the registry. + """ + queue: asyncio.Queue = asyncio.Queue() + async with _lock: + _subscribers[chat_id].append(queue) + return queue + + +async def unsubscribe(chat_id: str, queue: asyncio.Queue) -> None: + """Remove ``queue`` from the registry; remove the chat key if empty.""" + async with _lock: + if chat_id in _subscribers: + if queue in _subscribers[chat_id]: + _subscribers[chat_id].remove(queue) + if not _subscribers[chat_id]: + del _subscribers[chat_id] + + +async def publish(chat_id: str, event: dict[str, Any]) -> None: + """Fan-out ``event`` to every subscriber of ``chat_id``. + + The same dict reference is enqueued to all subscribers. Callers should + treat published events as immutable. Queues are unbounded for v1. + """ + async with _lock: + queues = list(_subscribers.get(chat_id, [])) + for q in queues: + await q.put(event) + + +def subscriber_count(chat_id: str) -> int: + """Test helper. Returns the number of active subscribers for a chat.""" + return len(_subscribers.get(chat_id, [])) diff --git a/chat/web/sse.py b/chat/web/sse.py new file mode 100644 index 0000000..4e324c1 --- /dev/null +++ b/chat/web/sse.py @@ -0,0 +1,72 @@ +"""Server-Sent Events endpoint for per-chat live updates. + +Each browser tab on ``/chats/`` opens an SSE connection here. On connect: + +1. We verify the chat exists (404 otherwise). +2. We subscribe to the chat's pub/sub channel. +3. We emit a ``snapshot`` event with the current state. T16 only provides a + stub payload (``{"chat_id": , "ready": true}``) so the client can + confirm the channel is live; T19+ will populate it with real state. +4. We loop, awaiting events from the queue and yielding them as SSE frames. + A 15-second keepalive comment is emitted on idle to defeat intermediary + timeouts. +5. When the client disconnects we unsubscribe so the registry doesn't leak. +""" + +from __future__ import annotations + +import asyncio +import json + +from fastapi import APIRouter, Depends, HTTPException, Request +from fastapi.responses import StreamingResponse + +from chat.state.world import get_chat +from chat.web.bots import get_conn +from chat.web.pubsub import subscribe, unsubscribe + +router = APIRouter() + +# Heartbeat cadence. Long enough to avoid chattiness; short enough that most +# HTTP intermediaries won't close an idle connection. +_KEEPALIVE_SECONDS = 15.0 + + +def _format_sse(event: str, data: dict) -> bytes: + """Format a single SSE frame: ``event: \\ndata: \\n\\n``.""" + payload = json.dumps(data) + return f"event: {event}\ndata: {payload}\n\n".encode("utf-8") + + +@router.get("/chats/{chat_id}/events") +async def chat_events(chat_id: str, request: Request, conn=Depends(get_conn)): + chat = get_chat(conn, chat_id) + if chat is None: + raise HTTPException(status_code=404, detail="chat not found") + + async def stream(): + queue = await subscribe(chat_id) + try: + # Initial snapshot — T19 will fill in real state. + yield _format_sse("snapshot", {"chat_id": chat_id, "ready": True}) + while True: + if await request.is_disconnected(): + break + try: + event = await asyncio.wait_for( + queue.get(), timeout=_KEEPALIVE_SECONDS + ) + except asyncio.TimeoutError: + # SSE comment line (per spec, lines starting with ":" are + # ignored by the client) — keeps the connection warm. + yield b": keepalive\n\n" + continue + # Allow publishers to set the SSE event name via "event" key; + # default to "message" if omitted. + event = dict(event) # don't mutate the published dict + kind = event.pop("event", "message") + yield _format_sse(kind, event) + finally: + await unsubscribe(chat_id, queue) + + return StreamingResponse(stream(), media_type="text/event-stream") diff --git a/tests/test_sse.py b/tests/test_sse.py new file mode 100644 index 0000000..188af23 --- /dev/null +++ b/tests/test_sse.py @@ -0,0 +1,174 @@ +"""Tests for the per-chat SSE channel and pub/sub registry (T16). + +Covers: +- 404 from the SSE endpoint when the chat doesn't exist. +- The first event yielded on connect is the ``snapshot`` event with correct + framing (``event: snapshot\\ndata: {...}\\n\\n``) and includes the chat id. +- ``publish`` fans out to every subscriber of a chat. +- Events published to one chat don't leak to subscribers of another chat. + +Notes: +- Starlette's TestClient buffers the entire ASGI response before returning, + so it cannot read from an indefinitely-open SSE stream. The framing test + drives the route handler directly and pulls frames from the + ``StreamingResponse``'s body iterator. The 404 path is still validated via + TestClient since it returns synchronously without entering the stream. +""" + +from __future__ import annotations + +import asyncio +import json +from pathlib import Path + +import pytest +from fastapi import Request +from fastapi.testclient import TestClient + +from chat.app import app +from chat.db.connection import open_db +from chat.eventlog.log import append_event +from chat.eventlog.projector import project + + +@pytest.fixture +def client(tmp_path, monkeypatch): + cfg = tmp_path / "config.toml" + cfg.write_text('featherless_api_key = "test"\n') + monkeypatch.setenv("CHAT_CONFIG_PATH", str(cfg)) + db = tmp_path / "test.db" + monkeypatch.setenv("CHAT_DB_PATH", str(db)) + with TestClient(app) as c: + yield c + + +def _seed_chat( + db_path: Path, bot_id: str = "bot_a", chat_id: str = "chat_bot_a" +) -> None: + with open_db(db_path) as conn: + append_event( + conn, + kind="bot_authored", + payload={ + "id": bot_id, + "name": "BotA", + "persona": "...", + "voice_samples": [], + "traits": [], + "backstory": "", + "initial_relationship_to_you": "", + "kickoff_prose": "...", + }, + ) + append_event( + conn, + kind="chat_created", + payload={ + "id": chat_id, + "host_bot_id": bot_id, + "initial_time": "2026-04-26T20:00:00+00:00", + "narrative_anchor": "Day 1", + "weather": "", + }, + ) + project(conn) + + +def test_sse_endpoint_404_for_missing_chat(client): + response = client.get("/chats/nonexistent/events") + assert response.status_code == 404 + + +async def test_sse_streams_initial_snapshot(client, tmp_path): + """The first SSE frame is a correctly framed ``snapshot`` event.""" + _seed_chat(tmp_path / "test.db") + + # Drive the route directly. The TestClient can't iterate an open stream + # because Starlette's transport waits for the response to complete. + from chat.web.bots import get_conn + from chat.web.sse import chat_events + + disconnect = asyncio.Event() + + async def receive(): + await disconnect.wait() + return {"type": "http.disconnect"} + + scope = { + "type": "http", + "method": "GET", + "path": "/chats/chat_bot_a/events", + "headers": [], + "query_string": b"", + "app": client.app, + } + request = Request(scope, receive=receive) + + conn_gen = get_conn(request) + conn = next(conn_gen) + try: + response = await chat_events("chat_bot_a", request, conn=conn) + assert response.media_type == "text/event-stream" + + body_iter = response.body_iterator + first = await asyncio.wait_for(body_iter.__anext__(), timeout=2.0) + text = first.decode() + + # Framing: starts with the event line, ends with the blank-line + # terminator that delimits SSE frames. + assert text.startswith("event: snapshot\n") + assert "\ndata: " in text + assert text.endswith("\n\n") + + # Payload is JSON containing the chat id. + data_line = next( + line for line in text.splitlines() if line.startswith("data: ") + ) + payload = json.loads(data_line[len("data: "):]) + assert payload["chat_id"] == "chat_bot_a" + + # Cleanly tear down the generator's ``finally`` (unsubscribe). + disconnect.set() + try: + await asyncio.wait_for(body_iter.aclose(), timeout=2.0) + except (StopAsyncIteration, RuntimeError): + pass + finally: + try: + next(conn_gen) + except StopIteration: + pass + + +async def test_pubsub_publish_fanout(): + """Every active subscriber of a chat receives published events.""" + from chat.web.pubsub import publish, subscribe, unsubscribe + + q1 = await subscribe("chat_x") + q2 = await subscribe("chat_x") + try: + await publish("chat_x", {"event": "test", "value": 1}) + e1 = await asyncio.wait_for(q1.get(), timeout=1.0) + e2 = await asyncio.wait_for(q2.get(), timeout=1.0) + assert e1 == {"event": "test", "value": 1} + assert e2 == {"event": "test", "value": 1} + finally: + await unsubscribe("chat_x", q1) + await unsubscribe("chat_x", q2) + + +async def test_pubsub_isolated_per_chat(): + """Events published to one chat don't leak to a different chat.""" + from chat.web.pubsub import publish, subscribe, unsubscribe + + q_a = await subscribe("chat_a") + q_b = await subscribe("chat_b") + try: + await publish("chat_a", {"event": "for_a"}) + e = await asyncio.wait_for(q_a.get(), timeout=1.0) + assert e["event"] == "for_a" + with pytest.raises(asyncio.TimeoutError): + await asyncio.wait_for(q_b.get(), timeout=0.1) + finally: + await unsubscribe("chat_a", q_a) + await unsubscribe("chat_b", q_b) -- 2.52.0 From a0f5e818ecd1bee7a8178bd6214b8770bc3d588e Mon Sep 17 00:00:00 2001 From: Joseph Doherty Date: Sun, 26 Apr 2026 12:53:34 -0400 Subject: [PATCH 19/45] feat: turn input parser via classifier --- chat/services/turn_parse.py | 94 +++++++++++++++++++++++++++++++++++++ tests/test_turn_parse.py | 83 ++++++++++++++++++++++++++++++++ 2 files changed, 177 insertions(+) create mode 100644 chat/services/turn_parse.py create mode 100644 tests/test_turn_parse.py diff --git a/chat/services/turn_parse.py b/chat/services/turn_parse.py new file mode 100644 index 0000000..df721dc --- /dev/null +++ b/chat/services/turn_parse.py @@ -0,0 +1,94 @@ +"""Turn input parser. + +Service-layer function that splits a user's authored turn into typed +segments — ``dialogue``, ``action``, or ``ooc`` (out-of-character). + +Per Requirements §6.1 a turn is mixed prose with three conventions: + +- ``*action*`` (single asterisks around prose) → action segment. +- Quoted text, or bare prose between the conventions → dialogue. +- ``((double parens))`` → OOC, the author talking to the system rather + than the bot. Downstream (T19) strips OOC from the prompt sent to the + bot but keeps it in the transcript display. + +A regex-based splitter would brittle on edge cases (unclosed asterisks, +nested quotes, mixed punctuation), so v1 delegates the segmentation to +the classifier. The configurable ``Settings.ooc_marker`` is *not* read +here: the classifier figures OOC out from ``((`` ``))`` regardless of +config-time choice; marker-based stripping is a downstream concern. +""" + +from __future__ import annotations + +from pydantic import BaseModel + +from chat.llm.classify import classify +from chat.llm.client import LLMClient + + +class TurnSegment(BaseModel): + """One classified piece of a turn. + + ``kind`` is kept as a plain ``str`` (not a ``Literal``) so an + unexpected classifier output doesn't crash parsing — callers that + care about specific values can check defensively. + """ + + kind: str # "dialogue" | "action" | "ooc" + text: str + + +class ParsedTurn(BaseModel): + """A turn split into ordered, typed segments.""" + + segments: list[TurnSegment] + + +_SYSTEM_PROMPT = ( + "You are splitting a roleplay turn into typed segments. The input " + "is mixed prose with three conventions:\n" + "- *text in single asterisks* is an ACTION segment.\n" + "- \"quoted text\" or bare prose between conventions is a DIALOGUE segment.\n" + "- ((text in double parens)) is an OOC (out-of-character) segment — " + "the author talking to the system, not the in-fiction bot.\n\n" + "Output a JSON object with shape " + '{"segments": [{"kind": "...", "text": "..."}, ...]} ' + "where each ``kind`` is exactly one of: dialogue, action, ooc. " + "Preserve the original substring text as ``text``: do not rewrite, " + "translate, or normalize punctuation — strip only the marker " + "characters (asterisks, surrounding quotes, double parens) so " + "``text`` is the inner content. Emit segments in the order they " + "appear in the input." +) + + +async def parse_turn( + client: LLMClient, + *, + model: str, + prose: str, + timeout_s: float = 10.0, +) -> ParsedTurn: + """Parse a user turn into typed segments. + + Calls :func:`chat.llm.classify.classify` under the hood. Empty or + whitespace-only prose short-circuits to an empty ``ParsedTurn`` + without an LLM call (the classifier would error on empty input + anyway, and the result is unambiguous). + + Raises ``RuntimeError`` if the classifier fails twice — no default + is supplied, since the caller (T19's turn flow) is responsible for + surfacing the error to the user. + """ + if not prose.strip(): + return ParsedTurn(segments=[]) + + user_prompt = f"INPUT:\n{prose}" + return await classify( + client, + model=model, + system=_SYSTEM_PROMPT, + user=user_prompt, + schema=ParsedTurn, + timeout_s=timeout_s, + ) diff --git a/tests/test_turn_parse.py b/tests/test_turn_parse.py new file mode 100644 index 0000000..3246bd4 --- /dev/null +++ b/tests/test_turn_parse.py @@ -0,0 +1,83 @@ +import json + +import pytest + +from chat.llm.mock import MockLLMClient +from chat.services.turn_parse import ( + ParsedTurn, + TurnSegment, + parse_turn, +) + + +@pytest.mark.asyncio +async def test_parse_turn_three_segment_happy_path(): + canned = json.dumps( + { + "segments": [ + {"kind": "action", "text": "walks over"}, + {"kind": "dialogue", "text": "Hey."}, + {"kind": "ooc", "text": "player note"}, + ] + } + ) + mock = MockLLMClient(canned=[canned]) + result = await parse_turn( + mock, + model="m", + prose='*walks over* "Hey." ((player note))', + ) + assert isinstance(result, ParsedTurn) + assert len(result.segments) == 3 + kinds = [s.kind for s in result.segments] + assert kinds == ["action", "dialogue", "ooc"] + texts = [s.text for s in result.segments] + assert texts == ["walks over", "Hey.", "player note"] + assert all(isinstance(s, TurnSegment) for s in result.segments) + + +@pytest.mark.asyncio +async def test_parse_turn_pure_dialogue_single_segment(): + canned = json.dumps( + { + "segments": [ + {"kind": "dialogue", "text": "Hello there"}, + ] + } + ) + mock = MockLLMClient(canned=[canned]) + result = await parse_turn( + mock, + model="m", + prose='"Hello there"', + ) + assert isinstance(result, ParsedTurn) + assert len(result.segments) == 1 + assert result.segments[0].kind == "dialogue" + assert result.segments[0].text == "Hello there" + + +@pytest.mark.asyncio +async def test_parse_turn_empty_prose_short_circuits_without_classifier_call(): + # No canned responses provided — if classify() is invoked it will raise + # IndexError on the empty list. The short-circuit must prevent that. + mock = MockLLMClient(canned=[]) + result = await parse_turn(mock, model="m", prose="") + assert isinstance(result, ParsedTurn) + assert result.segments == [] + + # Whitespace-only prose must also short-circuit. + result_ws = await parse_turn(mock, model="m", prose=" \n\t ") + assert isinstance(result_ws, ParsedTurn) + assert result_ws.segments == [] + + +@pytest.mark.asyncio +async def test_parse_turn_raises_when_classifier_fails_twice(): + mock = MockLLMClient(canned=["nope", "still nope"]) + with pytest.raises(RuntimeError): + await parse_turn( + mock, + model="m", + prose='*shrugs* "whatever"', + ) -- 2.52.0 From 73d8b0c0921f622e48d4b624e687833f6549c80e Mon Sep 17 00:00:00 2001 From: Joseph Doherty Date: Sun, 26 Apr 2026 13:00:00 -0400 Subject: [PATCH 20/45] feat: prompt assembly with must/should/nice trim tiers --- chat/services/prompt.py | 554 ++++++++++++++++++++++++++++++++++++++++ tests/test_prompt.py | 255 ++++++++++++++++++ 2 files changed, 809 insertions(+) create mode 100644 chat/services/prompt.py create mode 100644 tests/test_prompt.py diff --git a/chat/services/prompt.py b/chat/services/prompt.py new file mode 100644 index 0000000..89b416e --- /dev/null +++ b/chat/services/prompt.py @@ -0,0 +1,554 @@ +"""Narrative-prompt assembly with must/should/nice trim tiers. + +Implements Task 18 (Phase 1D). See Requirements §3.2 (token budgets and +trim tiers) and §6.3 (speaker prompt assembly order). The function +:func:`assemble_narrative_prompt` returns a list of +:class:`chat.llm.client.Message` objects ready to feed to +``LLMClient.generate``. + +Trim policy when the assembled prompt exceeds the soft target: + +- **MUST-include** (never trimmed): system / speaker identity, the + speaker→addressee edge, the activity snapshot for all present + entities, the current scene description, and the last 4 turns of + dialogue. +- **SHOULD-include** (trim when over budget): other edges of the + speaker. (Group nodes, active threads, and active events / props are + Phase 3 — skipped here.) +- **NICE-include** (trim first): retrieved memories beyond top-2, + dialogue turns beyond the last 4 (replaced with a one-line elision + placeholder), per-POV summary of the previous scene. + +Token counting uses ``tiktoken.get_encoding("cl100k_base")`` per the +requirements. Mistral / Llama tokenizers diverge ~5%; we accept the +drift. + +The function is intentionally deterministic (no LLM call) so it is +testable with synthetic state and so T29's regenerate flow can rebuild +prompts without re-running classifiers. +""" + +from __future__ import annotations + +from sqlite3 import Connection + +import tiktoken + +from chat.llm.client import Message +from chat.state.edges import get_edge, list_edges_for +from chat.state.entities import get_bot, get_you +from chat.state.memory import search_memories +from chat.state.world import ( + active_scene, + get_activity, + get_chat, + get_container, + get_scene, +) + + +# Cache the encoder once at import-time. tiktoken's encoder load is +# non-trivial (~tens of ms) and the encoding is process-wide stable. +_ENCODER = tiktoken.get_encoding("cl100k_base") + + +# --------------------------------------------------------------------------- +# Helpers +# --------------------------------------------------------------------------- + + +def _count_tokens(text: str, encoding=_ENCODER) -> int: + """Return the cl100k_base token count for ``text`` (0 for falsy).""" + if not text: + return 0 + return len(encoding.encode(text)) + + +def _build_speaker_identity(bot: dict) -> str: + """Render the bot identity block. Skips empty optional fields.""" + lines = [f"You are {bot['name']}."] + if bot.get("persona"): + lines.append("") + lines.append("PERSONA:") + lines.append(bot["persona"]) + voice_samples = bot.get("voice_samples") or [] + if voice_samples: + lines.append("") + lines.append("VOICE REFERENCE:") + lines.append("\n---\n".join(voice_samples)) + traits = bot.get("traits") or [] + if traits: + lines.append("") + lines.append(f"TRAITS: {', '.join(traits)}") + if bot.get("backstory"): + lines.append("") + lines.append("BACKSTORY:") + lines.append(bot["backstory"]) + return "\n".join(lines) + + +def _build_edge_block(edge: dict | None, addressee_name: str) -> str | None: + """Render the speaker → addressee edge. Returns None when no edge exists.""" + if edge is None: + return None + lines = [f"YOUR EDGE TO {addressee_name}:"] + lines.append(f"- Affinity: {edge.get('affinity', 50)}/100") + lines.append(f"- Trust: {edge.get('trust', 50)}/100") + summary = edge.get("summary") or "" + if summary: + lines.append(f"- Summary: {summary}") + knowledge = edge.get("knowledge") or [] + if knowledge: + lines.append(f"- What you know about {addressee_name}:") + for fact in knowledge: + lines.append(f" * {fact}") + return "\n".join(lines) + + +def _build_activity_block(activities: list[dict]) -> str | None: + """Render the activity snapshot for all present entities.""" + rendered: list[str] = [] + for a in activities: + if a is None: + continue + label = a.get("_display_name") or a.get("entity_id", "?") + parts: list[str] = [] + posture = a.get("posture") or "" + if posture: + parts.append(posture) + action = a.get("action") or {} + verb = action.get("verb") if isinstance(action, dict) else None + if verb: + parts.append(verb) + attention = a.get("attention") or "" + if attention: + parts.append(f"attention: {attention}") + holding = a.get("holding") or [] + if holding: + parts.append(f"holding: {', '.join(holding)}") + if parts: + rendered.append(f"- {label}: " + ", ".join(parts)) + else: + rendered.append(f"- {label}: (no activity recorded)") + if not rendered: + return None + return "ACTIVITIES:\n" + "\n".join(rendered) + + +def _build_scene_block(chat: dict, container: dict | None, scene: dict | None) -> str | None: + """Render the current-scene block. Always present when chat exists.""" + lines = ["CURRENT SCENE:"] + if container is not None: + lines.append(f"- Container: {container['name']} ({container['type']})") + chat_time = chat.get("time") if chat else None + if chat_time: + lines.append(f"- Time: {chat_time}") + if scene is not None and scene.get("started_at"): + lines.append(f"- Active scene started: {scene['started_at']}") + if len(lines) == 1: + return None + return "\n".join(lines) + + +def _format_dialogue_turn(turn: dict) -> str: + speaker = turn.get("speaker") or "?" + text = turn.get("text") or "" + return f"{speaker}: {text}" + + +def _build_dialogue_block( + recent: list[dict], + earlier_summary: str | None, +) -> str | None: + """Render the recent-dialogue block. The ``recent`` list is the + *kept* tail of the dialogue (already trimmed to the last-N turns). + ``earlier_summary``, when non-None, is rendered as the first line as + ``earlier: `` to flag elided context. + """ + if not recent and not earlier_summary: + return None + lines = ["RECENT DIALOGUE:"] + if earlier_summary: + lines.append(f"earlier: {earlier_summary}") + for turn in recent: + lines.append(_format_dialogue_turn(turn)) + return "\n".join(lines) + + +def _build_memories_block(memory_summaries: list[str]) -> str | None: + if not memory_summaries: + return None + lines = ["RELEVANT MEMORIES:"] + for m in memory_summaries: + lines.append(f"- {m}") + return "\n".join(lines) + + +def _build_other_edges_block(edges: list[dict]) -> str | None: + """Render edges to entities other than the addressee.""" + if not edges: + return None + lines = ["OTHER EDGES:"] + for e in edges: + target = e.get("_display_name") or e.get("target_id", "?") + affinity = e.get("affinity", 50) + trust = e.get("trust", 50) + lines.append(f"- {target}: affinity {affinity}/100, trust {trust}/100") + summary = e.get("summary") or "" + if summary: + lines.append(f" summary: {summary}") + return "\n".join(lines) + + +def _build_previous_scene_block(pov_summary: str | None) -> str | None: + if not pov_summary: + return None + return "PREVIOUS SCENE SUMMARY:\n" + pov_summary + + +def _closing_instruction(speaker_name: str, addressee_name: str) -> str: + return ( + f"Continue the scene as {speaker_name}, in their voice, responding " + "naturally. Use *asterisks* for actions and quotes for dialogue. " + f"Stay in character. Do not narrate {addressee_name}'s actions or " + "thoughts." + ) + + +def _join_blocks(blocks: list[str | None]) -> str: + """Join non-empty blocks with double newlines.""" + return "\n\n".join(b for b in blocks if b) + + +def _earlier_summary_placeholder(elided_count: int) -> str: + """Phase 1 placeholder. Real summarization is a downstream concern.""" + plural = "turn" if elided_count == 1 else "turns" + return f"{elided_count} earlier {plural} elided for brevity" + + +def _resolve_previous_scene_summary( + conn: Connection, chat_id: str, speaker_bot_id: str +) -> str | None: + """Return ``pov_summary`` of the most recent ended scene, owned by + the speaker. None if no closed scene exists or no matching memory. + """ + row = conn.execute( + "SELECT id FROM scenes WHERE chat_id = ? AND ended_at IS NOT NULL " + "ORDER BY ended_at DESC LIMIT 1", + (chat_id,), + ).fetchone() + if not row: + return None + scene_id = row[0] + mem = conn.execute( + "SELECT pov_summary FROM memories WHERE scene_id = ? AND owner_id = ? " + "ORDER BY id DESC LIMIT 1", + (scene_id, speaker_bot_id), + ).fetchone() + if not mem: + return None + return mem[0] + + +def _resolve_addressee( + conn: Connection, addressee: str, you: dict | None +) -> tuple[str, str]: + """Return ``(addressee_id, addressee_display_name)``. + + The function is permissive: ``addressee="you"`` resolves to the + you-entity (display name is its authored name, falling back to + "you" if no entity exists yet). Other ids resolve as bot ids. + """ + if addressee == "you": + name = (you or {}).get("name") or "you" + return "you", name + bot = get_bot(conn, addressee) + if bot is not None: + return addressee, bot["name"] + return addressee, addressee + + +# --------------------------------------------------------------------------- +# Public API +# --------------------------------------------------------------------------- + + +def assemble_narrative_prompt( + conn: Connection, + *, + chat_id: str, + speaker_bot_id: str, + addressee: str = "you", + user_turn_prose: str | None = None, + recent_dialogue: list[dict] | None = None, + retrieved_memory_summaries: list[str] | None = None, + budget_soft: int = 6000, + budget_hard: int = 8000, + encoding_name: str = "cl100k_base", +) -> list[Message]: + """Assemble the narrative prompt for ``speaker_bot_id`` to respond. + + Returns a list of :class:`Message` objects: one ``system`` message + carrying the assembled context, optionally followed by a single + ``user`` message containing ``user_turn_prose`` (when provided). + + Trimming proceeds in tiers (NICE → SHOULD) once the total token + count exceeds ``budget_soft``; the function refuses to exceed + ``budget_hard``. If the MUST-include block alone is already over + ``budget_hard``, :class:`ValueError` is raised — the caller should + surface the failure rather than ship a malformed prompt. + """ + encoding = ( + _ENCODER if encoding_name == "cl100k_base" + else tiktoken.get_encoding(encoding_name) + ) + + bot = get_bot(conn, speaker_bot_id) + if bot is None: + raise ValueError(f"speaker_bot_id {speaker_bot_id!r} not found") + + chat = get_chat(conn, chat_id) + if chat is None: + raise ValueError(f"chat_id {chat_id!r} not found") + + you = get_you(conn) + addressee_id, addressee_name = _resolve_addressee(conn, addressee, you) + + # ---- Build all components as text strings ------------------------------ + + speaker_identity = _build_speaker_identity(bot) + + edge_to_addressee = _build_edge_block( + get_edge(conn, speaker_bot_id, addressee_id), + addressee_name, + ) + + # Activity for present entities. Phase 1: you + speaker bot. (When a + # guest is added in Phase 1+, callers that know about it can pass + # extra activities via a future hook; for now we keep it strict.) + activities: list[dict] = [] + you_act = get_activity(conn, "you") + if you_act is not None: + you_act = dict(you_act) + you_act["_display_name"] = (you or {}).get("name") or "you" + activities.append(you_act) + bot_act = get_activity(conn, speaker_bot_id) + if bot_act is not None: + bot_act = dict(bot_act) + bot_act["_display_name"] = bot["name"] + activities.append(bot_act) + activity_block = _build_activity_block(activities) + + container = None + if chat.get("active_scene_id"): + scene = get_scene(conn, chat["active_scene_id"]) + if scene and scene.get("container_id"): + container = get_container(conn, scene["container_id"]) + else: + scene = active_scene(conn, chat_id) + if container is None and scene and scene.get("container_id"): + container = get_container(conn, scene["container_id"]) + scene_block = _build_scene_block(chat, container, scene) + + # Other edges: speaker → non-addressee. + all_outgoing = list_edges_for(conn, speaker_bot_id) + other_edges_raw = [e for e in all_outgoing if e.get("target_id") != addressee_id] + for e in other_edges_raw: + tid = e.get("target_id") + if tid == "you": + e["_display_name"] = (you or {}).get("name") or "you" + else: + tb = get_bot(conn, tid) if tid else None + e["_display_name"] = tb["name"] if tb else (tid or "?") + other_edges_block = _build_other_edges_block(other_edges_raw) + + # Memories: caller override wins; otherwise FTS5 search keyed on the + # scene's container/posture as a coarse query proxy. + if retrieved_memory_summaries is not None: + memory_summaries = list(retrieved_memory_summaries) + else: + query = (container or {}).get("name") or chat.get("narrative_anchor") or "" + memory_summaries = [] + if query: + try: + hits = search_memories(conn, speaker_bot_id, "host", query, k=4) + memory_summaries = [h["pov_summary"] for h in hits] + except Exception: + memory_summaries = [] + + # Dialogue: caller override only (no event_log read in Phase 1). + dialogue_full = list(recent_dialogue or []) + + previous_scene_summary = _resolve_previous_scene_summary( + conn, chat_id, speaker_bot_id + ) + + closing = _closing_instruction(bot["name"], addressee_name) + + # ---- Build the MUST core ---------------------------------------------- + + last4 = dialogue_full[-4:] if dialogue_full else [] + must_dialogue_block = _build_dialogue_block(last4, earlier_summary=None) + + must_blocks: list[str | None] = [ + speaker_identity, + edge_to_addressee, + scene_block, + activity_block, + must_dialogue_block, + closing, + ] + must_text = _join_blocks(must_blocks) + must_tokens = _count_tokens(must_text, encoding) + if must_tokens > budget_hard: + raise ValueError( + f"MUST-include block ({must_tokens} tokens) exceeds budget_hard " + f"({budget_hard}). Cannot assemble prompt." + ) + + # ---- Stage SHOULD additions, then NICE additions ----------------------- + + # We carry a running "components" list and rebuild the body as we go + # so token accounting reflects join-overhead. Order in the final + # prompt follows §6.3: identity → edge → other edges → scene → + # activities → previous scene summary → memories → dialogue → close. + + def assemble( + *, + include_other_edges: bool, + include_previous_scene: bool, + include_memories_top_k: int, + dialogue_keep: int, + ) -> tuple[str, int, list[dict]]: + # dialogue: keep the last `dialogue_keep` turns verbatim; older + # turns become an "earlier:" placeholder line. + kept_dialogue = ( + dialogue_full[-dialogue_keep:] if dialogue_keep > 0 else [] + ) + elided = max(0, len(dialogue_full) - len(kept_dialogue)) + earlier_summary = ( + _earlier_summary_placeholder(elided) if elided > 0 else None + ) + dialogue_block = _build_dialogue_block(kept_dialogue, earlier_summary) + + memories_subset = memory_summaries[:include_memories_top_k] + memories_block = _build_memories_block(memories_subset) + + prev_block = ( + _build_previous_scene_block(previous_scene_summary) + if include_previous_scene else None + ) + + body = _join_blocks([ + speaker_identity, + edge_to_addressee, + other_edges_block if include_other_edges else None, + scene_block, + activity_block, + prev_block, + memories_block, + dialogue_block, + closing, + ]) + return body, _count_tokens(body, encoding), kept_dialogue + + # Start with the MUST baseline: last 4 turns of dialogue, no + # SHOULD/NICE extras. + baseline_keep = min(4, len(dialogue_full)) + + # Try the most generous configuration first; trim greedily. + nice_dialogue_keep = len(dialogue_full) # all turns, no elision + nice_memories_k = min(4, len(memory_summaries)) + include_prev = previous_scene_summary is not None + include_other = other_edges_block is not None + + body, total, _ = assemble( + include_other_edges=include_other, + include_previous_scene=include_prev, + include_memories_top_k=nice_memories_k, + dialogue_keep=nice_dialogue_keep, + ) + + # If under soft, we're done. + if total <= budget_soft: + return _emit(body, user_turn_prose) + + # Drop NICE in order: previous scene → memories beyond top-2 → + # older dialogue turns (collapse to 4). + if include_prev: + body, total, _ = assemble( + include_other_edges=include_other, + include_previous_scene=False, + include_memories_top_k=nice_memories_k, + dialogue_keep=nice_dialogue_keep, + ) + include_prev = False + if total <= budget_soft: + return _emit(body, user_turn_prose) + + if nice_memories_k > 2: + nice_memories_k = 2 + body, total, _ = assemble( + include_other_edges=include_other, + include_previous_scene=False, + include_memories_top_k=nice_memories_k, + dialogue_keep=nice_dialogue_keep, + ) + if total <= budget_soft: + return _emit(body, user_turn_prose) + + if nice_dialogue_keep > baseline_keep: + nice_dialogue_keep = baseline_keep + body, total, _ = assemble( + include_other_edges=include_other, + include_previous_scene=False, + include_memories_top_k=nice_memories_k, + dialogue_keep=nice_dialogue_keep, + ) + if total <= budget_soft: + return _emit(body, user_turn_prose) + + # Drop more NICE until we're under hard: memories all the way to 0. + while nice_memories_k > 0 and total > budget_hard: + nice_memories_k = max(0, nice_memories_k - 1) + body, total, _ = assemble( + include_other_edges=include_other, + include_previous_scene=False, + include_memories_top_k=nice_memories_k, + dialogue_keep=nice_dialogue_keep, + ) + + # Drop SHOULD: other edges. + if include_other and total > budget_hard: + include_other = False + body, total, _ = assemble( + include_other_edges=False, + include_previous_scene=False, + include_memories_top_k=nice_memories_k, + dialogue_keep=nice_dialogue_keep, + ) + + if total > budget_hard: + # We've stripped everything optional and we still overflow. + # MUST alone fits (we checked at the top), so this means our + # last-4 dialogue + must blocks together exceed hard. Fall back + # to the bare MUST core. + body = must_text + total = must_tokens + if total > budget_hard: + raise ValueError( + f"Prompt cannot fit budget_hard={budget_hard}; MUST core " + f"is {total} tokens" + ) + + return _emit(body, user_turn_prose) + + +def _emit(system_body: str, user_turn_prose: str | None) -> list[Message]: + msgs: list[Message] = [Message(role="system", content=system_body)] + if user_turn_prose is not None: + msgs.append(Message(role="user", content=user_turn_prose)) + return msgs + + +__all__ = ["assemble_narrative_prompt"] diff --git a/tests/test_prompt.py b/tests/test_prompt.py new file mode 100644 index 0000000..bef8dc0 --- /dev/null +++ b/tests/test_prompt.py @@ -0,0 +1,255 @@ +"""Tests for chat.services.prompt.assemble_narrative_prompt. + +Covers Task 18 — must/should/nice trim tiers (Requirements §3.2) and +the speaker prompt assembly order (§6.3). Tests use direct event-log +seeding so the projector populates state exactly the way the runtime +will at play-time. No LLM is invoked: prompt assembly is deterministic. +""" + +from __future__ import annotations + +import pytest + +from chat.db.connection import open_db +from chat.db.migrate import apply_migrations +from chat.eventlog.log import append_event +from chat.eventlog.projector import project +import chat.state.entities # noqa: F401 (registers handlers) +import chat.state.edges # noqa: F401 +import chat.state.memory # noqa: F401 +import chat.state.world # noqa: F401 +from chat.llm.client import Message +from chat.services.prompt import assemble_narrative_prompt + + +def _seed_basic(conn) -> None: + """Seed bot, you-entity, edge, chat, container, scene, activities.""" + append_event(conn, kind="bot_authored", payload={ + "id": "bot_a", + "name": "Aria", + "persona": "reserved coworker who notices things", + "voice_samples": ["I — sorry, I didn't mean to.", "Right. Of course."], + "traits": ["introverted", "observant"], + "backstory": "An archivist who joined the firm last spring.", + "initial_relationship_to_you": "coworker; mild crush; never voiced", + "kickoff_prose": "you stay late at the office", + }) + append_event(conn, kind="you_authored", payload={ + "name": "Sam", + "pronouns": "they/them", + "persona": "tired analyst", + }) + append_event(conn, kind="chat_created", payload={ + "id": "chat_bot_a", + "host_bot_id": "bot_a", + "guest_bot_id": None, + "initial_time": "2026-04-26T20:00:00+00:00", + "narrative_anchor": "Day 1 evening", + "weather": "clear", + }) + append_event(conn, kind="container_created", payload={ + "chat_id": "chat_bot_a", + "name": "office bullpen", + "type": "workplace", + "properties": {"public": False, "moving": False, "audible_range": "room"}, + }) + append_event(conn, kind="edge_update", payload={ + "source_id": "bot_a", + "target_id": "you", + "affinity_delta": 12, + "trust_delta": 5, + "knowledge_facts": [ + "they work on the same floor", + "they've stayed late twice this week", + ], + }) + append_event(conn, kind="activity_change", payload={ + "entity_id": "you", + "container_id": 1, + "posture": "sitting at your desk", + "action": {"verb": "finishing emails"}, + "attention": "the screen", + "holding": ["coffee mug"], + }) + append_event(conn, kind="activity_change", payload={ + "entity_id": "bot_a", + "container_id": 1, + "posture": "sitting at her desk", + "action": {"verb": "pretending to work"}, + "attention": "you, in glances", + }) + append_event(conn, kind="scene_opened", payload={ + "chat_id": "chat_bot_a", + "container_id": 1, + "started_at": "2026-04-26T20:00:00+00:00", + "participants": ["you", "bot_a"], + }) + project(conn) + + +def test_basic_assembly_returns_system_message_with_all_must_blocks(tmp_path): + db = tmp_path / "t.db" + apply_migrations(db) + with open_db(db) as conn: + _seed_basic(conn) + msgs = assemble_narrative_prompt( + conn, + chat_id="chat_bot_a", + speaker_bot_id="bot_a", + recent_dialogue=[], + retrieved_memory_summaries=[], + ) + assert isinstance(msgs, list) + assert len(msgs) == 1 + sys_msg = msgs[0] + assert isinstance(sys_msg, Message) + assert sys_msg.role == "system" + body = sys_msg.content + # Must-include markers + assert "Aria" in body + assert "PERSONA" in body + assert "ACTIVITIES" in body + assert "CURRENT SCENE" in body + # Edge to addressee — name + numeric values (default affinity 50, +12 = 62) + assert "Sam" in body + assert "62/100" in body + + +def test_user_turn_appended_as_user_message(tmp_path): + db = tmp_path / "t.db" + apply_migrations(db) + with open_db(db) as conn: + _seed_basic(conn) + msgs = assemble_narrative_prompt( + conn, + chat_id="chat_bot_a", + speaker_bot_id="bot_a", + user_turn_prose="*looks up* Hey.", + recent_dialogue=[], + retrieved_memory_summaries=[], + ) + assert len(msgs) == 2 + assert msgs[0].role == "system" + assert msgs[1].role == "user" + assert msgs[1].content == "*looks up* Hey." + + +def test_must_only_succeeds_with_empty_optional_blocks(tmp_path): + """No dialogue, memories, other edges, or previous scene summary — should not raise.""" + db = tmp_path / "t.db" + apply_migrations(db) + with open_db(db) as conn: + _seed_basic(conn) + msgs = assemble_narrative_prompt( + conn, + chat_id="chat_bot_a", + speaker_bot_id="bot_a", + recent_dialogue=None, # default → nothing + retrieved_memory_summaries=None, + user_turn_prose=None, + ) + assert len(msgs) == 1 + body = msgs[0].content + # Must blocks present + assert "PERSONA" in body + assert "ACTIVITIES" in body + # Optional blocks not in body (nothing to render) + assert "OTHER EDGES" not in body + assert "PREVIOUS SCENE SUMMARY" not in body + assert "RELEVANT MEMORIES" not in body + + +def test_long_dialogue_keeps_last_4_verbatim_and_summarizes_earlier(tmp_path): + """Stuff a huge dialogue history under budget pressure; older turns + must be elided to a placeholder, the last 4 verbatim, and earlier + unique markers gone. + """ + db = tmp_path / "t.db" + apply_migrations(db) + with open_db(db) as conn: + _seed_basic(conn) + dialogue = [] + for i in range(20): + speaker = "you" if i % 2 == 0 else "bot_a" + # Each line ~250 tokens of filler => 20 turns ≈ 5000 tokens, + # which together with MUST blocks pushes over soft (1500). + dialogue.append({ + "speaker": speaker, + "text": f"unique-line-marker-{i:02d} " + ("filler " * 200), + }) + msgs = assemble_narrative_prompt( + conn, + chat_id="chat_bot_a", + speaker_bot_id="bot_a", + recent_dialogue=dialogue, + retrieved_memory_summaries=[], + # Soft small enough to force NICE trim but hard fits MUST + 4. + budget_soft=1200, + budget_hard=8000, + ) + body = msgs[0].content + # The last 4 unique markers (16, 17, 18, 19) must be present verbatim. + for i in range(16, 20): + assert f"unique-line-marker-{i:02d}" in body, f"expected last-4 marker {i} in body" + # Older markers must be dropped (replaced by elision placeholder). + for i in range(0, 16): + assert f"unique-line-marker-{i:02d}" not in body + # An "earlier" summary line must be present. + assert "earlier" in body.lower() + # Token count of system message respects hard budget. + import tiktoken + enc = tiktoken.get_encoding("cl100k_base") + assert len(enc.encode(body)) <= 8000 + + +def test_memories_drop_to_top_2_under_budget_pressure(tmp_path): + """4 memory summaries, each large; under tight soft budget only 2 should appear.""" + db = tmp_path / "t.db" + apply_migrations(db) + with open_db(db) as conn: + _seed_basic(conn) + # Each ~1500 tokens of repeated text; drop tier should kick in. + long_chunk = "alpha beta gamma delta " * 400 + memories = [ + f"MEMORY-A {long_chunk}", + f"MEMORY-B {long_chunk}", + f"MEMORY-C {long_chunk}", + f"MEMORY-D {long_chunk}", + ] + msgs = assemble_narrative_prompt( + conn, + chat_id="chat_bot_a", + speaker_bot_id="bot_a", + recent_dialogue=[], + retrieved_memory_summaries=memories, + # Pressure: budgets that allow MUST + 2 memories but not 4. + budget_soft=4000, + budget_hard=5000, + ) + body = msgs[0].content + # MEMORY-A and MEMORY-B are the top-2 and should remain; C & D dropped. + assert "MEMORY-A" in body + assert "MEMORY-B" in body + assert "MEMORY-C" not in body + assert "MEMORY-D" not in body + # Token count fits the hard budget. + import tiktoken + enc = tiktoken.get_encoding("cl100k_base") + assert len(enc.encode(body)) <= 5000 + + +def test_must_exceeds_budget_hard_raises_value_error(tmp_path): + db = tmp_path / "t.db" + apply_migrations(db) + with open_db(db) as conn: + _seed_basic(conn) + with pytest.raises(ValueError): + assemble_narrative_prompt( + conn, + chat_id="chat_bot_a", + speaker_bot_id="bot_a", + recent_dialogue=[], + retrieved_memory_summaries=[], + budget_soft=5, + budget_hard=10, + ) -- 2.52.0 From 9b45710cb16da14bf2dc9958d8c6e15f9c497e37 Mon Sep 17 00:00:00 2001 From: Joseph Doherty Date: Sun, 26 Apr 2026 13:09:31 -0400 Subject: [PATCH 21/45] feat: narrative streaming via SSE with assistant_turn event --- chat/app.py | 2 + chat/templates/chat.html | 8 +- chat/web/chat.py | 15 ++- chat/web/sse.py | 26 ++++- chat/web/turns.py | 239 +++++++++++++++++++++++++++++++++++++++ tests/test_turn_flow.py | 177 +++++++++++++++++++++++++++++ 6 files changed, 458 insertions(+), 9 deletions(-) create mode 100644 chat/web/turns.py create mode 100644 tests/test_turn_flow.py diff --git a/chat/app.py b/chat/app.py index 02b936d..8417d1c 100644 --- a/chat/app.py +++ b/chat/app.py @@ -20,6 +20,7 @@ from chat.web.kickoff import router as kickoff_router from chat.web.nav import router as nav_router from chat.web.settings import router as settings_router from chat.web.sse import router as sse_router +from chat.web.turns import router as turns_router @asynccontextmanager @@ -42,6 +43,7 @@ app.include_router(settings_router) app.include_router(nav_router) app.include_router(chat_router) app.include_router(sse_router) +app.include_router(turns_router) @app.get("/health") diff --git a/chat/templates/chat.html b/chat/templates/chat.html index 9aa04b4..02f5c2d 100644 --- a/chat/templates/chat.html +++ b/chat/templates/chat.html @@ -1,14 +1,18 @@ {% extends "layout.html" %} {% block title %}{{ host_bot.name }} - chat{% endblock %} {% block content %} -
+

{{ host_bot.name }}

{{ chat.time }}
-
+
{% if not turns %}

No turns yet. Start typing below.

{% else %} diff --git a/chat/web/chat.py b/chat/web/chat.py index 35d12ef..b90b74c 100644 --- a/chat/web/chat.py +++ b/chat/web/chat.py @@ -16,6 +16,7 @@ from fastapi.templating import Jinja2Templates from chat.state.entities import get_bot from chat.state.world import get_chat from chat.web.bots import get_conn +from chat.web.turns import _read_recent_dialogue TEMPLATES = Jinja2Templates( directory=str(Path(__file__).resolve().parent.parent / "templates") @@ -38,9 +39,19 @@ async def chat_detail(chat_id: str, request: Request, conn=Depends(get_conn)): status_code=404, detail=f"host bot not found: {chat['host_bot_id']}" ) - # Phase 1, T15: timeline starts empty. T19 will populate from event_log - # by reading user_turn / assistant_turn events for this chat. + # T19: render the timeline from event_log. We pull both user_turn and + # assistant_turn events for this chat, in chronological order. Each row + # is shaped ``{"speaker": ..., "text": ...}`` and the template + # discriminates roles via the speaker id (the literal "you" vs. a bot id). + raw_turns = _read_recent_dialogue(conn, chat_id, limit=200) turns: list[dict] = [] + for t in raw_turns: + if t["speaker"] == "you": + turns.append({"role": "you", "speaker": "you", "text": t["text"]}) + else: + bot = get_bot(conn, t["speaker"]) + label = bot["name"] if bot else t["speaker"] + turns.append({"role": "bot", "speaker": label, "text": t["text"]}) return TEMPLATES.TemplateResponse( request, diff --git a/chat/web/sse.py b/chat/web/sse.py index 4e324c1..2b657b9 100644 --- a/chat/web/sse.py +++ b/chat/web/sse.py @@ -32,9 +32,17 @@ router = APIRouter() _KEEPALIVE_SECONDS = 15.0 -def _format_sse(event: str, data: dict) -> bytes: - """Format a single SSE frame: ``event: \\ndata: \\n\\n``.""" - payload = json.dumps(data) +def _format_sse(event: str, data: dict | str) -> bytes: + """Format a single SSE frame: ``event: \\ndata: \\n\\n``. + + ``data`` may be a dict (JSON-serialized) or a raw string. The string + form is used for HTMX SSE swaps where the payload is an HTML + fragment that the client splices into the DOM verbatim. + """ + if isinstance(data, str): + payload = data + else: + payload = json.dumps(data) return f"event: {event}\ndata: {payload}\n\n".encode("utf-8") @@ -62,10 +70,18 @@ async def chat_events(chat_id: str, request: Request, conn=Depends(get_conn)): yield b": keepalive\n\n" continue # Allow publishers to set the SSE event name via "event" key; - # default to "message" if omitted. + # default to "message" if omitted. When the remaining payload + # is a single ``data`` string, send it verbatim — that lets + # turn-flow publishers ship pre-rendered HTML fragments that + # HTMX's SSE extension can swap into the DOM directly. event = dict(event) # don't mutate the published dict kind = event.pop("event", "message") - yield _format_sse(kind, event) + if set(event.keys()) == {"data"} and isinstance( + event["data"], str + ): + yield _format_sse(kind, event["data"]) + else: + yield _format_sse(kind, event) finally: await unsubscribe(chat_id, queue) diff --git a/chat/web/turns.py b/chat/web/turns.py new file mode 100644 index 0000000..e2c0082 --- /dev/null +++ b/chat/web/turns.py @@ -0,0 +1,239 @@ +"""POST ``/chats//turns`` — narrative turn flow with SSE streaming. + +The turn flow strings together the pieces built in T17 (turn parser), T18 +(prompt assembler), and T16 (SSE channel): + +1. Parse the user's prose with the classifier into typed segments. +2. Append a ``user_turn`` event capturing both the original prose and the + parsed segments. +3. Append a placeholder ``assistant_turn_started`` marker so observers know + a response is in flight. +4. Build the narrative prompt, dropping OOC segments before they reach the + bot (per Requirements §6.1 the OOC convention is for the author to talk + to the system, not to the in-fiction bot). +5. Stream tokens from the LLM, broadcasting each chunk over the chat's SSE + channel as a ``token`` event so any subscribed browser tab sees them + arrive in real time. +6. On stream complete, append an ``assistant_turn`` event with the full + text and ``truncated=False``. Also publish a ``turn_html`` event with a + ready-to-swap HTML fragment so HTMX's SSE extension can append it to + the timeline without a page reload. +7. Return ``204 No Content`` — the SSE channel is the real conveyor of + state, not the POST response body. + +Errors during streaming flip the assistant_turn's ``truncated`` flag to +``True`` and we still commit what we received. ``asyncio.CancelledError`` +is treated identically and re-raised after recording the partial turn. +""" + +from __future__ import annotations + +import asyncio +import html +import json + +from fastapi import APIRouter, Depends, Form, HTTPException, Request +from fastapi.responses import Response + +from chat.eventlog.log import append_event +from chat.services.prompt import assemble_narrative_prompt +from chat.services.turn_parse import ParsedTurn, parse_turn +from chat.state.world import get_chat +from chat.state.entities import get_bot +from chat.web.bots import get_conn +from chat.web.kickoff import get_llm_client +from chat.web.pubsub import publish + +router = APIRouter() + + +def _strip_ooc_for_prompt(parsed: ParsedTurn) -> str: + """Concatenate non-OOC segments back to a prose string for the prompt. + + OOC segments (``((double parens))``) are kept in the user_turn payload + for transcript display but stripped before assembly so the bot never + sees author-to-system messages. + """ + keep = [s.text for s in parsed.segments if s.kind != "ooc"] + return " ".join(keep).strip() + + +def _read_recent_dialogue(conn, chat_id: str, limit: int = 200) -> list[dict]: + """Return ``user_turn`` and ``assistant_turn`` events for ``chat_id``. + + Ordered oldest-first. Skips superseded and hidden rows so regenerated + turns (T29) drop out of the rendered timeline. Each entry is shaped + ``{"speaker": , "text": }`` for the prompt + assembler and the chat-detail template. + """ + cur = conn.execute( + "SELECT id, kind, payload_json FROM event_log " + "WHERE kind IN ('user_turn', 'assistant_turn') " + " AND superseded_by IS NULL AND hidden = 0 " + "ORDER BY id DESC LIMIT ?", + (limit,), + ) + rows = cur.fetchall() + rows.reverse() # back to chronological order + out: list[dict] = [] + for _row_id, kind, payload_json in rows: + p = json.loads(payload_json) + if p.get("chat_id") != chat_id: + continue + if kind == "user_turn": + out.append({"speaker": "you", "text": p.get("prose", "")}) + else: + out.append( + { + "speaker": p.get("speaker_id", "bot"), + "text": p.get("text", ""), + } + ) + return out + + +def _render_turn_html(speaker_label: str, text: str, *, role: str) -> str: + """Render a single turn as a small HTML fragment (escaped).""" + return ( + f'
' + f"{html.escape(speaker_label)}" + f"

{html.escape(text)}

" + f"
" + ) + + +@router.post("/chats/{chat_id}/turns") +async def post_turn( + chat_id: str, + request: Request, + prose: str = Form(""), + conn=Depends(get_conn), + client=Depends(get_llm_client), +): + chat = get_chat(conn, chat_id) + if chat is None: + raise HTTPException(status_code=404, detail=f"chat not found: {chat_id}") + + host_bot = get_bot(conn, chat["host_bot_id"]) + if host_bot is None: + # Defensive: chat row references a missing bot. + raise HTTPException( + status_code=404, + detail=f"host bot not found: {chat['host_bot_id']}", + ) + + settings = request.app.state.settings + + # 1. Parse turn (classifier). + parsed = await parse_turn( + client, model=settings.classifier_model, prose=prose + ) + prompt_prose = _strip_ooc_for_prompt(parsed) + + # 2. Append user_turn event. + user_turn_event_id = append_event( + conn, + kind="user_turn", + payload={ + "chat_id": chat_id, + "prose": prose, + "segments": [s.model_dump() for s in parsed.segments], + }, + ) + + # 3. Append assistant_turn_started placeholder. ``user_turn``, + # ``assistant_turn_started``, and ``assistant_turn`` have no registered + # projector handlers — they live in the event_log purely for transcript + # rendering — so we don't call ``project`` here. (Re-projecting now would + # also re-run prior non-idempotent inserts like ``chat_created``.) + append_event( + conn, + kind="assistant_turn_started", + payload={ + "chat_id": chat_id, + "speaker_id": host_bot["id"], + "user_turn_id": user_turn_event_id, + }, + ) + + # 4. Build the narrative prompt. + recent = _read_recent_dialogue(conn, chat_id, limit=20) + # Drop the just-appended user turn from ``recent`` — it's passed as + # ``user_turn_prose`` to the assembler and would otherwise duplicate. + if recent and recent[-1].get("speaker") == "you": + recent = recent[:-1] + messages = assemble_narrative_prompt( + conn, + chat_id=chat_id, + speaker_bot_id=host_bot["id"], + user_turn_prose=prompt_prose if prompt_prose else None, + recent_dialogue=recent, + budget_soft=settings.narrative_budget_soft, + budget_hard=settings.narrative_budget_hard, + ) + + # 5. Stream and accumulate tokens. + accumulated: list[str] = [] + truncated = False + cancelled = False + try: + async for chunk in client.stream( + messages, model=settings.narrative_model + ): + accumulated.append(chunk) + await publish( + chat_id, + { + "event": "token", + "text": chunk, + "speaker_id": host_bot["id"], + }, + ) + except asyncio.CancelledError: + # Preserve the partial output before letting the cancellation + # propagate so the transcript reflects what the user actually saw. + truncated = True + cancelled = True + except Exception: + # Surface as a truncated turn rather than losing the partial output. + truncated = True + + full_text = "".join(accumulated) + + # 6. Append the assistant_turn with the final text. (See note above on + # why we skip ``project`` for these transcript-only event kinds.) + append_event( + conn, + kind="assistant_turn", + payload={ + "chat_id": chat_id, + "speaker_id": host_bot["id"], + "text": full_text, + "truncated": truncated, + "user_turn_id": user_turn_event_id, + }, + ) + + # 7. Broadcast a JSON completion event (for JS consumers) and an HTML + # fragment event (for HTMX SSE swap-into-timeline). + await publish( + chat_id, + { + "event": "assistant_turn_complete", + "speaker_id": host_bot["id"], + "text": full_text, + "truncated": truncated, + }, + ) + assistant_html = _render_turn_html( + host_bot["name"], full_text, role="bot" + ) + await publish( + chat_id, {"event": "turn_html", "data": assistant_html} + ) + + if cancelled: + # Re-raise after the partial-turn has been recorded. + raise asyncio.CancelledError + + return Response(status_code=204) diff --git a/tests/test_turn_flow.py b/tests/test_turn_flow.py new file mode 100644 index 0000000..b404fbb --- /dev/null +++ b/tests/test_turn_flow.py @@ -0,0 +1,177 @@ +"""End-to-end turn flow (T19): user POSTs prose, server parses, streams via SSE. + +Covers: +- POST ``/chats//turns`` returns 404 when the chat doesn't exist. +- A successful POST appends both a ``user_turn`` and an ``assistant_turn`` + event in chronological order. The assistant payload carries the full + streamed text and ``truncated=False``. +- After a turn lands, the chat detail GET renders the user prose and the + assistant text from the event log. +""" + +from __future__ import annotations + +import json +from pathlib import Path + +import pytest +from fastapi.testclient import TestClient + +from chat.app import app +from chat.db.connection import open_db +from chat.eventlog.log import append_event +from chat.eventlog.projector import project +from chat.llm.mock import MockLLMClient + + +@pytest.fixture +def client(tmp_path, monkeypatch): + cfg = tmp_path / "config.toml" + cfg.write_text('featherless_api_key = "test"\n') + monkeypatch.setenv("CHAT_CONFIG_PATH", str(cfg)) + db = tmp_path / "test.db" + monkeypatch.setenv("CHAT_DB_PATH", str(db)) + + canned_parse = json.dumps( + {"segments": [{"kind": "dialogue", "text": "hello"}]} + ) + canned_response = "Hi there." + + # Import here so env vars are visible to the dependency lookup. + from chat.web.kickoff import get_llm_client + + mock = MockLLMClient(canned=[canned_parse, canned_response]) + app.dependency_overrides[get_llm_client] = lambda: mock + + with TestClient(app) as c: + c.mock_llm = mock # type: ignore[attr-defined] + yield c + + app.dependency_overrides.clear() + + +def _seed(db_path: Path) -> None: + """Author a bot, create a chat, and seed enough state for prompt assembly.""" + with open_db(db_path) as conn: + append_event( + conn, + kind="bot_authored", + payload={ + "id": "bot_a", + "name": "BotA", + "persona": "thoughtful, observant", + "voice_samples": [], + "traits": [], + "backstory": "", + "initial_relationship_to_you": "", + "kickoff_prose": "...", + }, + ) + append_event( + conn, + kind="chat_created", + payload={ + "id": "chat_bot_a", + "host_bot_id": "bot_a", + "initial_time": "2026-04-26T20:00:00+00:00", + "narrative_anchor": "Day 1", + "weather": "", + }, + ) + # Seed an edge so the prompt assembler has something to render. + append_event( + conn, + kind="edge_update", + payload={ + "source_id": "bot_a", + "target_id": "you", + "chat_id": "chat_bot_a", + "knowledge_facts": ["coworker"], + }, + ) + # Activity for both speakers — required by the prompt assembler. + append_event( + conn, + kind="activity_change", + payload={ + "entity_id": "you", + "posture": "sitting", + "action": { + "verb": "talking", + "interruptible": True, + "required_attention": "low", + "expected_duration": "ongoing", + }, + "attention": "", + "holding": [], + "status": {}, + }, + ) + append_event( + conn, + kind="activity_change", + payload={ + "entity_id": "bot_a", + "posture": "sitting", + "action": { + "verb": "listening", + "interruptible": True, + "required_attention": "low", + "expected_duration": "ongoing", + }, + "attention": "", + "holding": [], + "status": {}, + }, + ) + project(conn) + + +def test_post_turn_404_when_chat_missing(client): + response = client.post("/chats/no_such/turns", data={"prose": "hello"}) + assert response.status_code == 404 + + +def test_post_turn_appends_user_and_assistant_events(client, tmp_path): + _seed(tmp_path / "test.db") + response = client.post( + "/chats/chat_bot_a/turns", data={"prose": "hello"} + ) + assert response.status_code == 204 + + with open_db(tmp_path / "test.db") as conn: + cur = conn.execute( + "SELECT kind, payload_json FROM event_log " + "WHERE kind IN ('user_turn', 'assistant_turn') ORDER BY id" + ) + rows = cur.fetchall() + assert len(rows) == 2 + assert rows[0][0] == "user_turn" + assert rows[1][0] == "assistant_turn" + + user_payload = json.loads(rows[0][1]) + assert user_payload["chat_id"] == "chat_bot_a" + assert user_payload["prose"] == "hello" + # Segments come from the canned classifier output. + assert any( + s.get("kind") == "dialogue" and s.get("text") == "hello" + for s in user_payload["segments"] + ) + + assistant_payload = json.loads(rows[1][1]) + assert assistant_payload["chat_id"] == "chat_bot_a" + assert assistant_payload["speaker_id"] == "bot_a" + assert assistant_payload["text"] == "Hi there." + assert assistant_payload["truncated"] is False + + +def test_get_chat_renders_existing_turns(client, tmp_path): + _seed(tmp_path / "test.db") + post = client.post("/chats/chat_bot_a/turns", data={"prose": "hello"}) + assert post.status_code == 204 + + response = client.get("/chats/chat_bot_a") + assert response.status_code == 200 + body = response.text + assert "hello" in body + assert "Hi there." in body -- 2.52.0 From e8d24a08753a3b2316b66a21209debd2767285b4 Mon Sep 17 00:00:00 2001 From: Joseph Doherty Date: Sun, 26 Apr 2026 13:17:07 -0400 Subject: [PATCH 22/45] feat: post-turn state-update pass per present entity --- chat/eventlog/log.py | 39 ++++++ chat/services/state_update.py | 144 +++++++++++++++++++++ chat/web/turns.py | 93 ++++++++++++- tests/test_state_update.py | 237 ++++++++++++++++++++++++++++++++++ tests/test_turn_flow.py | 16 ++- 5 files changed, 523 insertions(+), 6 deletions(-) create mode 100644 chat/services/state_update.py create mode 100644 tests/test_state_update.py diff --git a/chat/eventlog/log.py b/chat/eventlog/log.py index 74919fb..ad228da 100644 --- a/chat/eventlog/log.py +++ b/chat/eventlog/log.py @@ -24,6 +24,45 @@ def append_event(conn: Connection, *, kind: str, payload: dict[str, Any], branch return cur.lastrowid +def append_and_apply( + conn: Connection, + *, + kind: str, + payload: dict[str, Any], + branch_id: int = 1, +) -> int: + """Append an event AND immediately apply just that event's handler. + + Calling :func:`chat.eventlog.projector.project` after an append + re-runs every prior event, which is fine for idempotent inserts but + catastrophic for delta-shaped events like ``edge_update`` whose + handler is *not* replay-safe (each pass would re-add the same + ``affinity_delta``). This helper runs only the brand-new event + through the registered handler, leaving prior state untouched. + + No-ops cleanly when ``kind`` has no registered handler — useful for + transcript-only events like ``user_turn`` / ``assistant_turn`` where + callers may swap ``append_event`` for ``append_and_apply`` without + side effects. + """ + # Local import to avoid a circular dependency at module import: the + # projector imports from .log to define ``Event``. + from chat.eventlog.projector import apply_event + + eid = append_event(conn, kind=kind, payload=payload, branch_id=branch_id) + event = Event( + id=eid, + branch_id=branch_id, + ts="", + kind=kind, + payload=payload, + superseded_by=None, + hidden=False, + ) + apply_event(conn, event) + return eid + + def read_events(conn: Connection, branch_id: int = 1, after_id: int = 0) -> Iterator[Event]: cur = conn.execute( "SELECT id, branch_id, ts, kind, payload_json, superseded_by, hidden " diff --git a/chat/services/state_update.py b/chat/services/state_update.py new file mode 100644 index 0000000..c9408fc --- /dev/null +++ b/chat/services/state_update.py @@ -0,0 +1,144 @@ +"""Post-turn state-update pass. + +Per Requirements §3.4, after every utterance we run a classifier on each +present entity (silent witnesses included) to extract directed-edge +deltas — what changed in *source*'s view of *target*. The classifier +returns three signals: + +- ``affinity_delta`` — signed change in how warmly source feels (typical + range -3..+3; the edge handler clamps the running total to 0..100). +- ``trust_delta`` — signed change in source's trust of target (same + shape). +- ``knowledge_facts`` — concrete things source learned about target + during this exchange. Stored verbatim and appended to ``edge.knowledge``. + +The wrapper deliberately uses :func:`chat.llm.classify.classify` with a +``default=StateUpdate()`` so a flapping classifier never blocks the turn +flow — at worst the edge sits unchanged and the next turn tries again +(§3.3 "graceful degradation" rule). +""" + +from __future__ import annotations + +from pydantic import BaseModel, Field + +from chat.llm.classify import classify +from chat.llm.client import LLMClient + + +class StateUpdate(BaseModel): + """One directed-edge update from a single classifier call. + + Defaults are deliberately a no-op (zero deltas, empty facts) so a + failing classifier produces a benign event rather than a disruption. + """ + + affinity_delta: int = 0 + trust_delta: int = 0 + knowledge_facts: list[str] = Field(default_factory=list) + + +_SYSTEM_PROMPT = ( + "You are reading a recent slice of dialogue from a roleplay scene. " + "You assess how SOURCE's view of TARGET shifted based on what was " + "said — including silent witnessing (SOURCE may not have spoken).\n\n" + "Output a JSON object with exactly three fields:\n" + "- affinity_delta: signed integer in [-3, 3]. How much warmer " + "(positive) or cooler (negative) SOURCE now feels toward TARGET.\n" + "- trust_delta: signed integer in [-3, 3]. How much more (positive) " + "or less (negative) SOURCE now trusts TARGET.\n" + "- knowledge_facts: list of short strings. New, concrete facts " + "SOURCE learned about TARGET in this exchange. Use TARGET's actual " + "stated content; do not infer or interpret. Empty list is fine.\n\n" + "Be conservative. Most turns produce small deltas (-1, 0, +1). " + "Reserve +/-2 or +/-3 for moments that materially shift the " + "relationship. Knowledge_facts should be specific things stated in " + "dialogue (e.g. \"works at the bakery\"), not interpretations " + "(\"seems lonely\")." +) + + +def _format_dialogue(recent_dialogue: list[dict]) -> str: + """Render the recent-dialogue slice as plain ``Speaker: text`` lines.""" + if not recent_dialogue: + return "(no dialogue yet)" + lines = [] + for turn in recent_dialogue: + speaker = turn.get("speaker", "?") + text = turn.get("text", "") + lines.append(f"{speaker}: {text}") + return "\n".join(lines) + + +def _build_user_prompt( + *, + source_name: str, + source_persona: str, + target_name: str, + prior_affinity: int, + prior_trust: int, + prior_summary: str, + recent_dialogue: list[dict], +) -> str: + return ( + f"SOURCE: {source_name}\n" + f"SOURCE_PERSONA: {source_persona or '(none)'}\n" + f"TARGET: {target_name}\n" + f"PRIOR_AFFINITY (0-100): {prior_affinity}\n" + f"PRIOR_TRUST (0-100): {prior_trust}\n" + f"PRIOR_SUMMARY: {prior_summary or '(none)'}\n\n" + f"RECENT_DIALOGUE:\n{_format_dialogue(recent_dialogue)}\n\n" + "How did SOURCE's view of TARGET shift? Respond with JSON only." + ) + + +async def compute_state_update( + client: LLMClient, + *, + model: str, + source_id: str, + target_id: str, + source_name: str, + source_persona: str, + target_name: str, + prior_affinity: int, + prior_trust: int, + prior_summary: str, + recent_dialogue: list[dict], + timeout_s: float = 10.0, +) -> StateUpdate: + """Run a classifier pass and return the directed-edge update. + + On classifier failure (after retry) returns the schema default — a + no-op ``StateUpdate`` — so the turn flow can keep moving. The + ``source_id`` / ``target_id`` arguments are accepted for symmetry + with the caller (T20's POST flow uses them when emitting the + ``edge_update`` event); they're not currently embedded in the + prompt because the classifier reasons about names, not opaque ids. + """ + # ``source_id``/``target_id`` are kept on the signature even though + # the prompt only quotes the names: callers in turns.py thread the + # ids straight from this function's args into the appended event. + del source_id, target_id # silence unused-arg lint cleanly + + user_prompt = _build_user_prompt( + source_name=source_name, + source_persona=source_persona, + target_name=target_name, + prior_affinity=prior_affinity, + prior_trust=prior_trust, + prior_summary=prior_summary, + recent_dialogue=recent_dialogue, + ) + + return await classify( + client, + model=model, + system=_SYSTEM_PROMPT, + user=user_prompt, + schema=StateUpdate, + default=StateUpdate(), + timeout_s=timeout_s, + ) + + diff --git a/chat/web/turns.py b/chat/web/turns.py index e2c0082..5128ce3 100644 --- a/chat/web/turns.py +++ b/chat/web/turns.py @@ -15,9 +15,12 @@ The turn flow strings together the pieces built in T17 (turn parser), T18 channel as a ``token`` event so any subscribed browser tab sees them arrive in real time. 6. On stream complete, append an ``assistant_turn`` event with the full - text and ``truncated=False``. Also publish a ``turn_html`` event with a - ready-to-swap HTML fragment so HTMX's SSE extension can append it to - the timeline without a page reload. + text and ``truncated=False``. Then run a post-turn state-update pass + (Requirements §3.4): one classifier call per directed edge between + present entities, each producing an ``edge_update`` event with + affinity/trust/knowledge deltas. Finally publish a ``turn_html`` + event with a ready-to-swap HTML fragment so HTMX's SSE extension can + append it to the timeline without a page reload. 7. Return ``204 No Content`` — the SSE channel is the real conveyor of state, not the POST response body. @@ -35,11 +38,13 @@ import json from fastapi import APIRouter, Depends, Form, HTTPException, Request from fastapi.responses import Response -from chat.eventlog.log import append_event +from chat.eventlog.log import append_and_apply, append_event from chat.services.prompt import assemble_narrative_prompt +from chat.services.state_update import compute_state_update from chat.services.turn_parse import ParsedTurn, parse_turn +from chat.state.edges import get_edge +from chat.state.entities import get_bot, get_you from chat.state.world import get_chat -from chat.state.entities import get_bot from chat.web.bots import get_conn from chat.web.kickoff import get_llm_client from chat.web.pubsub import publish @@ -214,6 +219,84 @@ async def post_turn( }, ) + # 6b. Post-turn state-update pass (Requirements §3.4). For Phase 1 + # the only present entities are ``you`` and ``host_bot`` so we run + # two classifier calls — one per directed edge — and append the + # resulting ``edge_update`` events. The recent-dialogue slice is + # re-read here so the pass sees the just-appended assistant turn. + # We use ``append_and_apply`` (vs append + project) because the + # edge_update handler is *not* replay-safe: re-projecting prior + # events would re-apply their deltas on top of the live row. + recent_for_update = _read_recent_dialogue(conn, chat_id, limit=10) + you_entity = get_you(conn) or {"name": "you", "persona": ""} + last_at = chat.get("time") + + edge_b2y = get_edge(conn, host_bot["id"], "you") or { + "affinity": 50, + "trust": 50, + "summary": "", + } + update_b2y = await compute_state_update( + client, + model=settings.classifier_model, + source_id=host_bot["id"], + target_id="you", + source_name=host_bot["name"], + source_persona=host_bot.get("persona", ""), + target_name=you_entity.get("name", "you"), + prior_affinity=edge_b2y["affinity"], + prior_trust=edge_b2y["trust"], + prior_summary=edge_b2y.get("summary", "") or "", + recent_dialogue=recent_for_update, + ) + append_and_apply( + conn, + kind="edge_update", + payload={ + "source_id": host_bot["id"], + "target_id": "you", + "chat_id": chat_id, + "affinity_delta": update_b2y.affinity_delta, + "trust_delta": update_b2y.trust_delta, + "knowledge_facts": update_b2y.knowledge_facts, + "last_interaction_at": last_at, + "last_interaction_chat_id": chat_id, + }, + ) + + edge_y2b = get_edge(conn, "you", host_bot["id"]) or { + "affinity": 50, + "trust": 50, + "summary": "", + } + update_y2b = await compute_state_update( + client, + model=settings.classifier_model, + source_id="you", + target_id=host_bot["id"], + source_name=you_entity.get("name", "you"), + source_persona=you_entity.get("persona", "") or "", + target_name=host_bot["name"], + prior_affinity=edge_y2b["affinity"], + prior_trust=edge_y2b["trust"], + prior_summary=edge_y2b.get("summary", "") or "", + recent_dialogue=recent_for_update, + ) + append_and_apply( + conn, + kind="edge_update", + payload={ + "source_id": "you", + "target_id": host_bot["id"], + "chat_id": chat_id, + "affinity_delta": update_y2b.affinity_delta, + "trust_delta": update_y2b.trust_delta, + "knowledge_facts": update_y2b.knowledge_facts, + "last_interaction_at": last_at, + "last_interaction_chat_id": chat_id, + }, + ) + # 7. Broadcast a JSON completion event (for JS consumers) and an HTML # fragment event (for HTMX SSE swap-into-timeline). await publish( diff --git a/tests/test_state_update.py b/tests/test_state_update.py new file mode 100644 index 0000000..e7c19b3 --- /dev/null +++ b/tests/test_state_update.py @@ -0,0 +1,237 @@ +"""Post-turn state-update pass (T20). + +Per Requirements §3.4, after each utterance we run a classifier on every +present entity (silent witnesses too) to extract directed-edge deltas +(``affinity_delta``, ``trust_delta``, ``knowledge_facts``). The deltas +land as ``edge_update`` events and project into the ``edges`` table. + +These tests cover: +- The unit-level :func:`compute_state_update` happy path: classifier + returns valid JSON, the wrapper returns a populated ``StateUpdate``. +- The unit-level fallback path: classifier fails twice, the wrapper + returns a no-op ``StateUpdate`` (zeros + empty facts) per §3.3. +- The integration path: a successful POST appends two ``edge_update`` + events (one per direction) after the ``assistant_turn`` and the edge + projections reflect the deltas. +""" + +from __future__ import annotations + +import json +from pathlib import Path + +import pytest +from fastapi.testclient import TestClient + +from chat.app import app +from chat.db.connection import open_db +from chat.eventlog.log import append_event +from chat.eventlog.projector import project +from chat.llm.mock import MockLLMClient +from chat.services.state_update import StateUpdate, compute_state_update + + +@pytest.mark.asyncio +async def test_compute_state_update_parses_classifier_output(): + canned = json.dumps( + {"affinity_delta": 2, "trust_delta": 1, "knowledge_facts": ["likes coffee"]} + ) + mock = MockLLMClient(canned=[canned]) + result = await compute_state_update( + mock, + model="x", + source_id="bot_a", + target_id="you", + source_name="BotA", + source_persona="thoughtful", + target_name="Me", + prior_affinity=50, + prior_trust=50, + prior_summary="", + recent_dialogue=[ + {"speaker": "you", "text": "hi"}, + {"speaker": "BotA", "text": "Hello!"}, + ], + ) + assert isinstance(result, StateUpdate) + assert result.affinity_delta == 2 + assert result.trust_delta == 1 + assert result.knowledge_facts == ["likes coffee"] + + +@pytest.mark.asyncio +async def test_compute_state_update_returns_default_on_failure(): + """Two malformed classifier responses -> default StateUpdate (zeros).""" + mock = MockLLMClient(canned=["nope", "still nope"]) + result = await compute_state_update( + mock, + model="x", + source_id="bot_a", + target_id="you", + source_name="BotA", + source_persona="", + target_name="Me", + prior_affinity=50, + prior_trust=50, + prior_summary="", + recent_dialogue=[], + ) + assert result.affinity_delta == 0 + assert result.trust_delta == 0 + assert result.knowledge_facts == [] + + +# --- integration test -------------------------------------------------------- + + +@pytest.fixture +def client(tmp_path, monkeypatch): + cfg = tmp_path / "config.toml" + cfg.write_text('featherless_api_key = "test"\n') + monkeypatch.setenv("CHAT_CONFIG_PATH", str(cfg)) + db = tmp_path / "test.db" + monkeypatch.setenv("CHAT_DB_PATH", str(db)) + + canned_parse = json.dumps( + {"segments": [{"kind": "dialogue", "text": "hello"}]} + ) + canned_response = "Hi there." + canned_state_b2y = json.dumps( + {"affinity_delta": 2, "trust_delta": 1, "knowledge_facts": ["greets warmly"]} + ) + canned_state_y2b = json.dumps( + {"affinity_delta": 3, "trust_delta": 0, "knowledge_facts": []} + ) + + from chat.web.kickoff import get_llm_client + + mock = MockLLMClient( + canned=[canned_parse, canned_response, canned_state_b2y, canned_state_y2b] + ) + app.dependency_overrides[get_llm_client] = lambda: mock + + with TestClient(app) as c: + c.mock_llm = mock # type: ignore[attr-defined] + yield c + + app.dependency_overrides.clear() + + +def _seed(db_path: Path) -> None: + """Author a bot, create a chat, and seed enough state for prompt assembly.""" + with open_db(db_path) as conn: + append_event( + conn, + kind="bot_authored", + payload={ + "id": "bot_a", + "name": "BotA", + "persona": "thoughtful, observant", + "voice_samples": [], + "traits": [], + "backstory": "", + "initial_relationship_to_you": "", + "kickoff_prose": "...", + }, + ) + append_event( + conn, + kind="chat_created", + payload={ + "id": "chat_bot_a", + "host_bot_id": "bot_a", + "initial_time": "2026-04-26T20:00:00+00:00", + "narrative_anchor": "Day 1", + "weather": "", + }, + ) + append_event( + conn, + kind="edge_update", + payload={ + "source_id": "bot_a", + "target_id": "you", + "chat_id": "chat_bot_a", + "knowledge_facts": ["coworker"], + }, + ) + append_event( + conn, + kind="activity_change", + payload={ + "entity_id": "you", + "posture": "sitting", + "action": { + "verb": "talking", + "interruptible": True, + "required_attention": "low", + "expected_duration": "ongoing", + }, + "attention": "", + "holding": [], + "status": {}, + }, + ) + append_event( + conn, + kind="activity_change", + payload={ + "entity_id": "bot_a", + "posture": "sitting", + "action": { + "verb": "listening", + "interruptible": True, + "required_attention": "low", + "expected_duration": "ongoing", + }, + "attention": "", + "holding": [], + "status": {}, + }, + ) + project(conn) + + +def test_post_turn_appends_edge_updates_and_applies_deltas(client, tmp_path): + """After a turn, edge_update events fire for both directions and project.""" + db_path = tmp_path / "test.db" + _seed(db_path) + + response = client.post("/chats/chat_bot_a/turns", data={"prose": "hello"}) + assert response.status_code == 204 + + with open_db(db_path) as conn: + # Two new edge_update events should land *after* the assistant_turn. + cur = conn.execute( + "SELECT kind, payload_json FROM event_log " + "WHERE kind = 'edge_update' " + "AND id > (SELECT MAX(id) FROM event_log WHERE kind = 'assistant_turn') " + "ORDER BY id" + ) + rows = cur.fetchall() + assert len(rows) == 2 + kinds = [r[0] for r in rows] + assert kinds == ["edge_update", "edge_update"] + + # Inspect the two payloads — one per direction. + payloads = [json.loads(r[1]) for r in rows] + directions = {(p["source_id"], p["target_id"]) for p in payloads} + assert ("bot_a", "you") in directions + assert ("you", "bot_a") in directions + + # Edge bot_a -> you: seeded affinity=50, plus delta 2 -> 52. + from chat.state.edges import get_edge + + edge_b2y = get_edge(conn, "bot_a", "you") + assert edge_b2y is not None + assert edge_b2y["affinity"] == 52 + assert edge_b2y["trust"] == 51 + # Existing fact preserved, new fact appended. + assert "coworker" in edge_b2y["knowledge"] + assert "greets warmly" in edge_b2y["knowledge"] + + # Edge you -> bot_a: defaults (50/50) plus delta +3 affinity -> 53. + edge_y2b = get_edge(conn, "you", "bot_a") + assert edge_y2b is not None + assert edge_y2b["affinity"] == 53 + assert edge_y2b["trust"] == 50 diff --git a/tests/test_turn_flow.py b/tests/test_turn_flow.py index b404fbb..1f7228b 100644 --- a/tests/test_turn_flow.py +++ b/tests/test_turn_flow.py @@ -36,11 +36,25 @@ def client(tmp_path, monkeypatch): {"segments": [{"kind": "dialogue", "text": "hello"}]} ) canned_response = "Hi there." + # Two state-update classifier calls fire after the assistant_turn + # (one per directed edge: bot->you, you->bot). We feed them benign + # zero-delta JSON so the existing assertions about ``user_turn`` / + # ``assistant_turn`` are unaffected. + canned_state_update = json.dumps( + {"affinity_delta": 0, "trust_delta": 0, "knowledge_facts": []} + ) # Import here so env vars are visible to the dependency lookup. from chat.web.kickoff import get_llm_client - mock = MockLLMClient(canned=[canned_parse, canned_response]) + mock = MockLLMClient( + canned=[ + canned_parse, + canned_response, + canned_state_update, + canned_state_update, + ] + ) app.dependency_overrides[get_llm_client] = lambda: mock with TestClient(app) as c: -- 2.52.0 From a45dabb6ae3876ae6a5890668ea9a0171a15746d Mon Sep 17 00:00:00 2001 From: Joseph Doherty Date: Sun, 26 Apr 2026 13:20:43 -0400 Subject: [PATCH 23/45] feat: per-turn memory writes with witness flags --- chat/services/memory_write.py | 61 ++++++++ chat/web/turns.py | 18 ++- tests/test_memory_write.py | 283 ++++++++++++++++++++++++++++++++++ 3 files changed, 361 insertions(+), 1 deletion(-) create mode 100644 chat/services/memory_write.py create mode 100644 tests/test_memory_write.py diff --git a/chat/services/memory_write.py b/chat/services/memory_write.py new file mode 100644 index 0000000..ee27d3d --- /dev/null +++ b/chat/services/memory_write.py @@ -0,0 +1,61 @@ +"""Per-turn memory writes (T21). + +After ``assistant_turn`` lands, the turn flow records a ``memory_written`` +event for each present POV owner. Phase 1 single-bot turns only have the +host bot as a memory-store owner — ``you`` doesn't have a memory store in +v1 — so we write exactly one row per turn. + +Phase 1 simplifications (per plan §11.1, T27 will refine): + +- ``pov_summary`` is the assistant's raw narrative text. T27 rewrites at + scene close into per-POV summary form. +- ``significance`` defaults to ``1`` (Notable). T22's async significance + pass overwrites via a follow-up event. +- Witness flags are hard-coded ``[you=1, host=1, guest=0]``. Phase 2 will + derive them from ``chat.guest_bot_id`` once a guest can be present. +""" + +from __future__ import annotations + +from sqlite3 import Connection + +from chat.eventlog.log import append_and_apply + + +def record_turn_memory( + conn: Connection, + *, + chat_id: str, + host_bot_id: str, + narrative_text: str, + scene_id: int | None = None, + chat_clock_at: str | None = None, + source: str = "direct", + significance: int = 1, +) -> int: + """Append a ``memory_written`` event for the host bot's POV of this turn. + + Uses :func:`chat.eventlog.log.append_and_apply` (not raw + :func:`append_event`) so the new memory row is projected immediately + without re-running prior non-idempotent handlers (e.g. ``edge_update`` + deltas). Returns the new event id. + """ + payload: dict = { + "owner_id": host_bot_id, + "chat_id": chat_id, + "pov_summary": narrative_text, + "witness_you": 1, + "witness_host": 1, + "witness_guest": 0, + "source": source, + "reliability": 1.0, + "significance": significance, + "pinned": 0, + "auto_pinned": 0, + } + if scene_id is not None: + payload["scene_id"] = scene_id + if chat_clock_at is not None: + payload["chat_clock_at"] = chat_clock_at + + return append_and_apply(conn, kind="memory_written", payload=payload) diff --git a/chat/web/turns.py b/chat/web/turns.py index 5128ce3..ba678a6 100644 --- a/chat/web/turns.py +++ b/chat/web/turns.py @@ -39,12 +39,13 @@ from fastapi import APIRouter, Depends, Form, HTTPException, Request from fastapi.responses import Response from chat.eventlog.log import append_and_apply, append_event +from chat.services.memory_write import record_turn_memory from chat.services.prompt import assemble_narrative_prompt from chat.services.state_update import compute_state_update from chat.services.turn_parse import ParsedTurn, parse_turn from chat.state.edges import get_edge from chat.state.entities import get_bot, get_you -from chat.state.world import get_chat +from chat.state.world import active_scene, get_chat from chat.web.bots import get_conn from chat.web.kickoff import get_llm_client from chat.web.pubsub import publish @@ -219,6 +220,21 @@ async def post_turn( }, ) + # 6a. Per-turn memory write (Plan §11.1, T21). Phase 1 single-bot: + # only the host bot has a memory store, witness flags are + # ``[you=1, host=1, guest=0]``, and ``pov_summary`` is the raw + # narrative text (T27 will rewrite at scene close). Significance + # defaults to 1; T22's async classifier pass will overwrite it. + scene = active_scene(conn, chat_id) + record_turn_memory( + conn, + chat_id=chat_id, + host_bot_id=host_bot["id"], + narrative_text=full_text, + scene_id=scene["id"] if scene else None, + chat_clock_at=chat.get("time"), + ) + # 6b. Post-turn state-update pass (Requirements §3.4). For Phase 1 # the only present entities are ``you`` and ``host_bot`` so we run # two classifier calls — one per directed edge — and append the diff --git a/tests/test_memory_write.py b/tests/test_memory_write.py new file mode 100644 index 0000000..f25fb4f --- /dev/null +++ b/tests/test_memory_write.py @@ -0,0 +1,283 @@ +"""Per-turn memory writes (T21). + +After ``assistant_turn`` lands the turn flow records a ``memory_written`` +event for each present POV owner. Phase 1 single-bot turns only have the +host bot as a memory-store owner — ``you`` doesn't have a memory store in +v1 — so we write exactly one row per turn with witness flags +``[you=1, host=1, guest=0]``. The ``pov_summary`` is the assistant's raw +narrative text; T27 rewrites at scene close into per-POV summary form. +""" + +from __future__ import annotations + +import json +from pathlib import Path + +import pytest +from fastapi.testclient import TestClient + +from chat.app import app +from chat.db.connection import open_db +from chat.db.migrate import apply_migrations +from chat.eventlog.log import append_event +from chat.eventlog.projector import project +from chat.llm.mock import MockLLMClient +from chat.services.memory_write import record_turn_memory +import chat.state.entities # noqa: F401 - register handlers +import chat.state.memory # noqa: F401 +import chat.state.world # noqa: F401 + + +def _seed_minimal(db_path: Path) -> None: + """Author a bot and create a chat — bare minimum for memory writes.""" + with open_db(db_path) as conn: + append_event( + conn, + kind="bot_authored", + payload={ + "id": "bot_a", + "name": "BotA", + "persona": "...", + "voice_samples": [], + "traits": [], + "backstory": "", + "initial_relationship_to_you": "", + "kickoff_prose": "", + }, + ) + append_event( + conn, + kind="chat_created", + payload={ + "id": "chat_bot_a", + "host_bot_id": "bot_a", + "initial_time": "2026-04-26T20:00:00+00:00", + "narrative_anchor": "Day 1", + "weather": "", + }, + ) + project(conn) + + +def test_record_turn_memory_writes_event_and_projects(tmp_path): + db = tmp_path / "t.db" + apply_migrations(db) + _seed_minimal(db) + with open_db(db) as conn: + eid = record_turn_memory( + conn, + chat_id="chat_bot_a", + host_bot_id="bot_a", + narrative_text="BotA looks up. 'You're back late.'", + scene_id=None, + chat_clock_at="2026-04-26T20:00:00+00:00", + ) + assert eid > 0 + + rows = conn.execute( + "SELECT id, owner_id, chat_id, pov_summary, " + "witness_you, witness_host, witness_guest, " + "source, reliability, significance, pinned, auto_pinned, " + "chat_clock_at " + "FROM memories WHERE owner_id = ?", + ("bot_a",), + ).fetchall() + assert len(rows) == 1 + m = rows[0] + assert m[1] == "bot_a" + assert m[2] == "chat_bot_a" + assert "looks up" in m[3] + assert m[4] == 1 # witness_you + assert m[5] == 1 # witness_host + assert m[6] == 0 # witness_guest + assert m[7] == "direct" + assert m[8] == 1.0 # reliability default + assert m[9] == 1 # significance default + assert m[10] == 0 # pinned default + assert m[11] == 0 # auto_pinned default + assert m[12] == "2026-04-26T20:00:00+00:00" + + # And the underlying event_log row is the canonical source. + cur = conn.execute( + "SELECT COUNT(*) FROM event_log WHERE kind = 'memory_written'" + ) + assert cur.fetchone()[0] == 1 + + +def test_record_turn_memory_omits_optional_fields(tmp_path): + db = tmp_path / "t.db" + apply_migrations(db) + _seed_minimal(db) + with open_db(db) as conn: + # Call without scene_id/chat_clock_at — should default to None. + eid = record_turn_memory( + conn, + chat_id="chat_bot_a", + host_bot_id="bot_a", + narrative_text="A simple memory.", + ) + assert eid > 0 + + row = conn.execute( + "SELECT scene_id, chat_clock_at, source, reliability, " + "significance, pinned, auto_pinned " + "FROM memories WHERE owner_id = 'bot_a'" + ).fetchone() + assert row is not None + scene_id, chat_clock_at, source, reliability, significance, pinned, auto_pinned = row + assert scene_id is None + assert chat_clock_at is None + assert source == "direct" + assert reliability == 1.0 + assert significance == 1 + assert pinned == 0 + assert auto_pinned == 0 + + +# --------------------------------------------------------------------------- +# Integration: POST /chats//turns produces a memory_written event. +# --------------------------------------------------------------------------- + + +@pytest.fixture +def client(tmp_path, monkeypatch): + cfg = tmp_path / "config.toml" + cfg.write_text('featherless_api_key = "test"\n') + monkeypatch.setenv("CHAT_CONFIG_PATH", str(cfg)) + db = tmp_path / "test.db" + monkeypatch.setenv("CHAT_DB_PATH", str(db)) + + canned_parse = json.dumps( + {"segments": [{"kind": "dialogue", "text": "hello"}]} + ) + canned_response = "BotA nods. 'Hi there.'" + canned_state_update = json.dumps( + {"affinity_delta": 0, "trust_delta": 0, "knowledge_facts": []} + ) + + from chat.web.kickoff import get_llm_client + + mock = MockLLMClient( + canned=[ + canned_parse, + canned_response, + canned_state_update, + canned_state_update, + ] + ) + app.dependency_overrides[get_llm_client] = lambda: mock + + with TestClient(app) as c: + c.mock_llm = mock # type: ignore[attr-defined] + yield c + + app.dependency_overrides.clear() + + +def _seed_full(db_path: Path) -> None: + """Seed enough state for a full turn flow (matches test_turn_flow.py).""" + with open_db(db_path) as conn: + append_event( + conn, + kind="bot_authored", + payload={ + "id": "bot_a", + "name": "BotA", + "persona": "thoughtful, observant", + "voice_samples": [], + "traits": [], + "backstory": "", + "initial_relationship_to_you": "", + "kickoff_prose": "...", + }, + ) + append_event( + conn, + kind="chat_created", + payload={ + "id": "chat_bot_a", + "host_bot_id": "bot_a", + "initial_time": "2026-04-26T20:00:00+00:00", + "narrative_anchor": "Day 1", + "weather": "", + }, + ) + append_event( + conn, + kind="edge_update", + payload={ + "source_id": "bot_a", + "target_id": "you", + "chat_id": "chat_bot_a", + "knowledge_facts": ["coworker"], + }, + ) + append_event( + conn, + kind="activity_change", + payload={ + "entity_id": "you", + "posture": "sitting", + "action": { + "verb": "talking", + "interruptible": True, + "required_attention": "low", + "expected_duration": "ongoing", + }, + "attention": "", + "holding": [], + "status": {}, + }, + ) + append_event( + conn, + kind="activity_change", + payload={ + "entity_id": "bot_a", + "posture": "sitting", + "action": { + "verb": "listening", + "interruptible": True, + "required_attention": "low", + "expected_duration": "ongoing", + }, + "attention": "", + "holding": [], + "status": {}, + }, + ) + project(conn) + + +def test_post_turn_writes_memory_for_host_bot(client, tmp_path): + """After a POST turn, exactly one memory_written event is appended and + a corresponding memory row is projected for the host bot's POV.""" + _seed_full(tmp_path / "test.db") + response = client.post( + "/chats/chat_bot_a/turns", data={"prose": "hello"} + ) + assert response.status_code == 204 + + with open_db(tmp_path / "test.db") as conn: + cur = conn.execute( + "SELECT COUNT(*) FROM event_log WHERE kind = 'memory_written'" + ) + assert cur.fetchone()[0] == 1 + + cur = conn.execute( + "SELECT owner_id, chat_id, pov_summary, " + "witness_you, witness_host, witness_guest, source, significance " + "FROM memories" + ) + rows = cur.fetchall() + assert len(rows) == 1 + owner_id, chat_id, pov_summary, w_you, w_host, w_guest, source, sig = rows[0] + assert owner_id == "bot_a" + assert chat_id == "chat_bot_a" + # pov_summary is the assistant's narrative text (Phase 1 simplification). + assert pov_summary == "BotA nods. 'Hi there.'" + assert w_you == 1 + assert w_host == 1 + assert w_guest == 0 + assert source == "direct" + assert sig == 1 -- 2.52.0 From eb4cdf9cbb15685fe4cf0fcb5178cd6f3cc890d1 Mon Sep 17 00:00:00 2001 From: Joseph Doherty Date: Sun, 26 Apr 2026 13:27:25 -0400 Subject: [PATCH 24/45] feat: async significance pass with auto-pin on score 3 --- chat/app.py | 22 +++- chat/services/background.py | 173 +++++++++++++++++++++++++ chat/services/memory_write.py | 23 +++- chat/services/significance.py | 75 +++++++++++ chat/state/memory.py | 29 +++++ chat/web/turns.py | 20 ++- tests/test_memory_write.py | 11 +- tests/test_significance.py | 237 ++++++++++++++++++++++++++++++++++ tests/test_turn_flow.py | 5 + 9 files changed, 588 insertions(+), 7 deletions(-) create mode 100644 chat/services/background.py create mode 100644 chat/services/significance.py create mode 100644 tests/test_significance.py diff --git a/chat/app.py b/chat/app.py index 8417d1c..6233d58 100644 --- a/chat/app.py +++ b/chat/app.py @@ -7,6 +7,7 @@ from fastapi.staticfiles import StaticFiles from chat.config import load_settings from chat.db.migrate import apply_migrations +from chat.services.background import BackgroundWorker # Trigger handler registration: import chat.state.entities # noqa: F401 @@ -29,7 +30,26 @@ async def lifespan(app: FastAPI): settings.db_path.parent.mkdir(parents=True, exist_ok=True) apply_migrations(settings.db_path) app.state.settings = settings - yield + + # Background worker for the async significance pass (T22). Each job + # constructs a fresh FeatherlessClient via the factory; tests can + # disable enqueue by toggling ``app.state.background_worker.enabled``. + def _factory(): + from chat.llm.featherless import FeatherlessClient + + return FeatherlessClient( + api_key=settings.featherless_api_key, + base_url=settings.featherless_base_url, + ) + + worker = BackgroundWorker(settings, llm_client_factory=_factory) + await worker.start() + app.state.background_worker = worker + + try: + yield + finally: + await worker.stop() app = FastAPI(title="chat", lifespan=lifespan) diff --git a/chat/services/background.py b/chat/services/background.py new file mode 100644 index 0000000..0ad7f68 --- /dev/null +++ b/chat/services/background.py @@ -0,0 +1,173 @@ +"""Async background worker for post-turn jobs (T22). + +The turn flow records a ``memory_written`` event synchronously on the +request path so the timeline updates immediately. Significance scoring is +a separate classifier round-trip that we don't want to block on, so the +turn handler enqueues a :class:`SignificanceJob` here and the worker +drains the queue out-of-band. + +A single :class:`BackgroundWorker` is started/stopped via FastAPI lifespan +in :mod:`chat.app`. The worker owns its own ``asyncio.Queue`` and runs +exactly one task that pulls jobs off the queue, calls +:func:`chat.services.significance.compute_significance`, and writes +``memory_significance_set`` (and on score 3, ``memory_pin_changed``) +events. Each job opens its own DB connection — workers and request +handlers don't share connections. + +Failures inside ``_process`` are logged and swallowed: a flaky classifier +shouldn't take down the worker. Tests can disable enqueue() by setting +``BackgroundWorker.enabled = False`` (e.g. in the existing turn-flow +fixture, which doesn't have a usable LLM key for the lifespan-managed +factory). +""" + +from __future__ import annotations + +import asyncio +import logging +from dataclasses import dataclass +from typing import Callable + +from chat.config import Settings +from chat.db.connection import open_db +from chat.eventlog.log import append_and_apply +from chat.llm.client import LLMClient +from chat.services.significance import compute_significance + +log = logging.getLogger(__name__) + + +@dataclass +class SignificanceJob: + """One unit of work for the background worker. + + ``host_bot_id`` is the memory's owner — used both for the auto-pin + soft cap query and as the eventual scope for the soft-cap eviction. + """ + + memory_id: int + narrative_text: str + prior_dialogue: list[dict] + host_bot_id: str + + +class BackgroundWorker: + """asyncio.Queue-backed single-worker task. + + Started on app startup; ``stop()`` enqueues a sentinel and awaits the + task so any in-flight job has a chance to finish. Pending jobs after + the sentinel are dropped on shutdown — Phase 1 simplification. + """ + + def __init__( + self, + settings: Settings, + llm_client_factory: Callable[[], LLMClient], + *, + enabled: bool = True, + ) -> None: + self._settings = settings + self._llm_client_factory = llm_client_factory + self._queue: asyncio.Queue[SignificanceJob | None] = asyncio.Queue() + self._task: asyncio.Task | None = None + self.enabled = enabled + + async def start(self) -> None: + if self._task is not None: + return + self._task = asyncio.create_task(self._run()) + + async def stop(self) -> None: + if self._task is None: + return + await self._queue.put(None) # sentinel + await self._task + self._task = None + + def enqueue(self, job: SignificanceJob) -> None: + if not self.enabled: + return + self._queue.put_nowait(job) + + async def _run(self) -> None: + while True: + job = await self._queue.get() + if job is None: + return + try: + await self._process(job) + except Exception as exc: # noqa: BLE001 — worker must not die + log.exception("significance job failed: %s", exc) + + async def _process(self, job: SignificanceJob) -> None: + client = self._llm_client_factory() + score = await compute_significance( + client, + model=self._settings.classifier_model, + narrative_text=job.narrative_text, + prior_dialogue=job.prior_dialogue, + ) + with open_db(self._settings.db_path) as conn: + append_and_apply( + conn, + kind="memory_significance_set", + payload={ + "memory_id": job.memory_id, + "significance": score, + }, + ) + if score >= 3: + _auto_pin_with_cap( + conn, + owner_id=job.host_bot_id, + memory_id=job.memory_id, + ) + + +def _auto_pin_with_cap( + conn, + *, + owner_id: str, + memory_id: int, + cap: int = 8, +) -> None: + """Auto-pin ``memory_id`` and evict the oldest auto-pin if over ``cap``. + + Per §8.5: pivotal turns are auto-pinned, with a soft cap of 8 pins per + bot. When the cap is exceeded the oldest auto-pin is unpinned (manual + pins are never auto-evicted — we filter on ``auto_pinned = 1``). + """ + append_and_apply( + conn, + kind="memory_pin_changed", + payload={ + "memory_id": memory_id, + "pinned": 1, + "auto_pinned": 1, + }, + ) + cur = conn.execute( + "SELECT COUNT(*) FROM memories WHERE owner_id = ? AND pinned = 1", + (owner_id,), + ) + count = cur.fetchone()[0] + if count <= cap: + return + cur = conn.execute( + "SELECT id FROM memories " + "WHERE owner_id = ? AND pinned = 1 AND auto_pinned = 1 AND id != ? " + "ORDER BY created_at ASC, id ASC LIMIT 1", + (owner_id, memory_id), + ) + row = cur.fetchone() + if row is None: + return + append_and_apply( + conn, + kind="memory_pin_changed", + payload={ + "memory_id": row[0], + "pinned": 0, + "auto_pinned": 0, + }, + ) diff --git a/chat/services/memory_write.py b/chat/services/memory_write.py index ee27d3d..3ca40c5 100644 --- a/chat/services/memory_write.py +++ b/chat/services/memory_write.py @@ -32,13 +32,22 @@ def record_turn_memory( chat_clock_at: str | None = None, source: str = "direct", significance: int = 1, -) -> int: +) -> tuple[int, int | None]: """Append a ``memory_written`` event for the host bot's POV of this turn. Uses :func:`chat.eventlog.log.append_and_apply` (not raw :func:`append_event`) so the new memory row is projected immediately without re-running prior non-idempotent handlers (e.g. ``edge_update`` - deltas). Returns the new event id. + deltas). + + Returns ``(event_id, memory_id)``. ``event_id`` is the row id of the + just-appended ``memory_written`` event in ``event_log``. ``memory_id`` + is the autoincrement PK of the corresponding ``memories`` row — these + are *different* numbers (event_log and memories use independent + rowid sequences) so callers needing to update significance or pin + state must use ``memory_id``. Falls back to ``None`` if the projected + row can't be located, which shouldn't happen but keeps the return + shape stable. """ payload: dict = { "owner_id": host_bot_id, @@ -58,4 +67,12 @@ def record_turn_memory( if chat_clock_at is not None: payload["chat_clock_at"] = chat_clock_at - return append_and_apply(conn, kind="memory_written", payload=payload) + event_id = append_and_apply(conn, kind="memory_written", payload=payload) + row = conn.execute( + "SELECT id FROM memories " + "WHERE owner_id = ? AND chat_id = ? " + "ORDER BY id DESC LIMIT 1", + (host_bot_id, chat_id), + ).fetchone() + memory_id = row[0] if row else None + return event_id, memory_id diff --git a/chat/services/significance.py b/chat/services/significance.py new file mode 100644 index 0000000..eb3791c --- /dev/null +++ b/chat/services/significance.py @@ -0,0 +1,75 @@ +"""Turn-level significance scorer (T22). + +Per Requirements §11.1, each turn is scored on a 0-3 scale: + +- 0 = Routine: small talk, ordinary action. +- 1 = Notable: a specific detail or beat worth remembering. +- 2 = Significant: a scene-level moment, real disagreement, confided secret. +- 3 = Pivotal: a relationship-altering event (first kiss, betrayal, "I love + you"). + +The scorer is conservative: pivotal (3) requires a clear signal because the +auto-pin rule (§8.5) gives those memories permanent shelf space. The +classifier returns a strict-JSON ``SignificanceVerdict``; a malformed or +refusal-shaped response falls back to ``score=1`` (Notable) — a safe +middle-of-the-road default that won't trigger auto-pin. +""" + +from __future__ import annotations + +from pydantic import BaseModel, Field + +from chat.llm.classify import classify +from chat.llm.client import LLMClient + + +class SignificanceVerdict(BaseModel): + score: int = Field(ge=0, le=3) + reason: str = "" + + +_SYSTEM = """You score the significance of a roleplay turn 0-3: +0 = Routine: small talk, ordinary action. +1 = Notable: a specific detail or beat worth remembering. +2 = Significant: a scene-level moment, real disagreement, confided secret. +3 = Pivotal: a relationship-altering event (first kiss, betrayal, "I love you"). + +Be conservative — pivotal (3) requires a clear signal. Reply with JSON: {"score": int 0-3, "reason": str}.""" + + +async def compute_significance( + client: LLMClient, + *, + model: str, + narrative_text: str, + prior_dialogue: list[dict], + timeout_s: float = 10.0, +) -> int: + """Score the significance of ``narrative_text`` (the just-written turn). + + ``prior_dialogue`` is a list of ``{"speaker", "text"}`` dicts ordered + oldest-first; the last 6 entries are stitched into the user prompt as + context so the classifier can recognize escalation. Returns an int in + ``[0, 3]`` — clamped defensively in case the classifier slips a value + past the schema validator. + """ + user_prompt = "PRIOR DIALOGUE:\n" + for turn in prior_dialogue[-6:]: + speaker = turn.get("speaker", "?") + text = turn.get("text", "") + user_prompt += f"{speaker}: {text}\n" + user_prompt += ( + f"\nNEW TURN:\n{narrative_text}\n\n" + "Score the significance of the NEW TURN." + ) + + result = await classify( + client, + model=model, + system=_SYSTEM, + user=user_prompt, + schema=SignificanceVerdict, + default=SignificanceVerdict(score=1, reason="fallback"), + timeout_s=timeout_s, + ) + return max(0, min(3, result.score)) diff --git a/chat/state/memory.py b/chat/state/memory.py index b2655b0..f0420ca 100644 --- a/chat/state/memory.py +++ b/chat/state/memory.py @@ -38,6 +38,35 @@ def _apply_memory_written(conn: Connection, e: Event) -> None: ) +@on("memory_significance_set") +def _apply_memory_significance_set(conn: Connection, e: Event) -> None: + """Update an existing memory's significance score (T22). + + Emitted by the async significance worker after it scores the turn. + """ + p = e.payload + conn.execute( + "UPDATE memories SET significance = ? WHERE id = ?", + (int(p["significance"]), int(p["memory_id"])), + ) + + +@on("memory_pin_changed") +def _apply_memory_pin_changed(conn: Connection, e: Event) -> None: + """Toggle a memory's pin state (T22, §8.5). + + Used both for auto-pinning a pivotal turn and for evicting the oldest + auto-pin when the per-owner soft cap is exceeded. Manual pins use the + same handler; the ``auto_pinned`` flag distinguishes them so the + eviction query can leave manual pins alone. + """ + p = e.payload + conn.execute( + "UPDATE memories SET pinned = ?, auto_pinned = ? WHERE id = ?", + (int(p["pinned"]), int(p["auto_pinned"]), int(p["memory_id"])), + ) + + def get_memory(conn: Connection, memory_id: int) -> dict | None: row = conn.execute( "SELECT * FROM memories WHERE id = ?", (memory_id,) diff --git a/chat/web/turns.py b/chat/web/turns.py index ba678a6..b5cca0b 100644 --- a/chat/web/turns.py +++ b/chat/web/turns.py @@ -39,6 +39,7 @@ from fastapi import APIRouter, Depends, Form, HTTPException, Request from fastapi.responses import Response from chat.eventlog.log import append_and_apply, append_event +from chat.services.background import SignificanceJob from chat.services.memory_write import record_turn_memory from chat.services.prompt import assemble_narrative_prompt from chat.services.state_update import compute_state_update @@ -226,7 +227,7 @@ async def post_turn( # narrative text (T27 will rewrite at scene close). Significance # defaults to 1; T22's async classifier pass will overwrite it. scene = active_scene(conn, chat_id) - record_turn_memory( + _event_id, memory_id = record_turn_memory( conn, chat_id=chat_id, host_bot_id=host_bot["id"], @@ -313,6 +314,23 @@ async def post_turn( }, ) + # 6c. Enqueue the async significance pass (Plan §11.1, T22). The + # worker scores the just-written memory 0-3, updates significance, + # and auto-pins on score 3 with the §8.5 soft-cap eviction rule. + # Enqueued before the broadcast so it's outstanding by the time the + # client sees ``turn_html`` — but the worker is async, so the user + # never blocks on it. + worker = getattr(request.app.state, "background_worker", None) + if worker is not None and memory_id is not None: + worker.enqueue( + SignificanceJob( + memory_id=memory_id, + narrative_text=full_text, + prior_dialogue=recent_for_update, + host_bot_id=host_bot["id"], + ) + ) + # 7. Broadcast a JSON completion event (for JS consumers) and an HTML # fragment event (for HTMX SSE swap-into-timeline). await publish( diff --git a/tests/test_memory_write.py b/tests/test_memory_write.py index f25fb4f..feaf6f5 100644 --- a/tests/test_memory_write.py +++ b/tests/test_memory_write.py @@ -64,7 +64,7 @@ def test_record_turn_memory_writes_event_and_projects(tmp_path): apply_migrations(db) _seed_minimal(db) with open_db(db) as conn: - eid = record_turn_memory( + eid, mid = record_turn_memory( conn, chat_id="chat_bot_a", host_bot_id="bot_a", @@ -73,6 +73,7 @@ def test_record_turn_memory_writes_event_and_projects(tmp_path): chat_clock_at="2026-04-26T20:00:00+00:00", ) assert eid > 0 + assert mid is not None and mid > 0 rows = conn.execute( "SELECT id, owner_id, chat_id, pov_summary, " @@ -110,13 +111,14 @@ def test_record_turn_memory_omits_optional_fields(tmp_path): _seed_minimal(db) with open_db(db) as conn: # Call without scene_id/chat_clock_at — should default to None. - eid = record_turn_memory( + eid, mid = record_turn_memory( conn, chat_id="chat_bot_a", host_bot_id="bot_a", narrative_text="A simple memory.", ) assert eid > 0 + assert mid is not None and mid > 0 row = conn.execute( "SELECT scene_id, chat_clock_at, source, reliability, " @@ -168,6 +170,11 @@ def client(tmp_path, monkeypatch): app.dependency_overrides[get_llm_client] = lambda: mock with TestClient(app) as c: + # Disable the lifespan-managed background worker — it would try + # to call Featherless with the test API key. The unit tests in + # test_significance.py exercise the worker directly with a mock + # factory; here we only care about the synchronous turn flow. + app.state.background_worker.enabled = False c.mock_llm = mock # type: ignore[attr-defined] yield c diff --git a/tests/test_significance.py b/tests/test_significance.py new file mode 100644 index 0000000..e5a538c --- /dev/null +++ b/tests/test_significance.py @@ -0,0 +1,237 @@ +"""Async significance pass with auto-pin on score 3 (T22). + +After ``assistant_turn`` lands the turn flow enqueues a SignificanceJob on +a background asyncio worker. The worker calls a classifier (per §11.1, +score 0-3) and writes a ``memory_significance_set`` event. On score 3 the +memory is auto-pinned and a soft cap of 8 pins per owner is enforced — +when the cap is exceeded the oldest auto-pin (excluding the just-pinned +row) is unpinned via another ``memory_pin_changed`` event. +""" + +from __future__ import annotations + +import asyncio +import json + +import pytest + +from chat.config import load_settings +from chat.db.connection import open_db +from chat.db.migrate import apply_migrations +from chat.eventlog.log import append_event +from chat.eventlog.projector import project +from chat.llm.mock import MockLLMClient +from chat.services.background import BackgroundWorker, SignificanceJob +from chat.services.significance import compute_significance + +# Trigger handler registration for projection. +import chat.state.entities # noqa: F401 +import chat.state.memory # noqa: F401 +import chat.state.world # noqa: F401 + + +async def test_compute_significance_parses_score(): + canned = json.dumps({"score": 2, "reason": "notable"}) + mock = MockLLMClient(canned=[canned]) + score = await compute_significance( + mock, + model="x", + narrative_text="...", + prior_dialogue=[], + ) + assert score == 2 + + +async def test_compute_significance_default_on_failure(): + # Both attempts return non-JSON text; the classify wrapper falls back + # to the SignificanceVerdict default (score=1, "fallback"). + mock = MockLLMClient(canned=["nope", "still nope"]) + score = await compute_significance( + mock, + model="x", + narrative_text="...", + prior_dialogue=[], + ) + assert score == 1 + + +async def test_background_worker_processes_job_and_updates_significance( + tmp_path, monkeypatch +): + cfg = tmp_path / "config.toml" + cfg.write_text('featherless_api_key = "test"\n') + monkeypatch.setenv("CHAT_CONFIG_PATH", str(cfg)) + db = tmp_path / "test.db" + monkeypatch.setenv("CHAT_DB_PATH", str(db)) + apply_migrations(db) + settings = load_settings() + + # Seed bot, chat, memory. + with open_db(db) as conn: + append_event( + conn, + kind="bot_authored", + payload={ + "id": "bot_a", + "name": "BotA", + "persona": "...", + "voice_samples": [], + "traits": [], + "backstory": "", + "initial_relationship_to_you": "", + "kickoff_prose": "", + }, + ) + append_event( + conn, + kind="chat_created", + payload={ + "id": "chat_bot_a", + "host_bot_id": "bot_a", + "initial_time": "2026-04-26T20:00:00+00:00", + "narrative_anchor": "Day 1", + "weather": "", + }, + ) + append_event( + conn, + kind="memory_written", + payload={ + "owner_id": "bot_a", + "chat_id": "chat_bot_a", + "pov_summary": "Some scene", + "witness_you": 1, + "witness_host": 1, + "witness_guest": 0, + "source": "direct", + "reliability": 1.0, + "significance": 1, + "pinned": 0, + "auto_pinned": 0, + }, + ) + project(conn) + memory_id = conn.execute( + "SELECT id FROM memories WHERE owner_id = 'bot_a'" + ).fetchone()[0] + + # Worker with mock LLM that returns score=3 (pivotal). + canned = [json.dumps({"score": 3, "reason": "pivotal"})] + factory = lambda: MockLLMClient(canned=list(canned)) + worker = BackgroundWorker(settings, llm_client_factory=factory) + await worker.start() + worker.enqueue( + SignificanceJob( + memory_id=memory_id, + narrative_text="...", + prior_dialogue=[], + host_bot_id="bot_a", + ) + ) + # Drain via stop sentinel — guarantees the prior job completed. + await worker.stop() + + # Verify significance updated AND memory auto-pinned. + with open_db(db) as conn: + row = conn.execute( + "SELECT significance, pinned, auto_pinned FROM memories " + "WHERE id = ?", + (memory_id,), + ).fetchone() + assert row[0] == 3 + assert row[1] == 1 # pinned + assert row[2] == 1 # auto_pinned + + +async def test_auto_pin_evicts_oldest_when_over_cap(tmp_path, monkeypatch): + """Pin 9 memories with score 3; verify only 8 are pinned at the end.""" + cfg = tmp_path / "config.toml" + cfg.write_text('featherless_api_key = "test"\n') + monkeypatch.setenv("CHAT_CONFIG_PATH", str(cfg)) + db = tmp_path / "test.db" + monkeypatch.setenv("CHAT_DB_PATH", str(db)) + apply_migrations(db) + settings = load_settings() + + with open_db(db) as conn: + append_event( + conn, + kind="bot_authored", + payload={ + "id": "bot_a", + "name": "BotA", + "persona": "...", + "voice_samples": [], + "traits": [], + "backstory": "", + "initial_relationship_to_you": "", + "kickoff_prose": "", + }, + ) + append_event( + conn, + kind="chat_created", + payload={ + "id": "chat_bot_a", + "host_bot_id": "bot_a", + "initial_time": "2026-04-26T20:00:00+00:00", + "narrative_anchor": "Day 1", + "weather": "", + }, + ) + for i in range(9): + append_event( + conn, + kind="memory_written", + payload={ + "owner_id": "bot_a", + "chat_id": "chat_bot_a", + "pov_summary": f"memory {i}", + "witness_you": 1, + "witness_host": 1, + "witness_guest": 0, + "source": "direct", + "reliability": 1.0, + "significance": 1, + "pinned": 0, + "auto_pinned": 0, + }, + ) + project(conn) + memory_ids = [ + r[0] + for r in conn.execute( + "SELECT id FROM memories WHERE owner_id = 'bot_a' ORDER BY id" + ).fetchall() + ] + + # Each job runs through its own MockLLMClient with one canned response. + factory = lambda: MockLLMClient( + canned=[json.dumps({"score": 3, "reason": "pivotal"})] + ) + worker = BackgroundWorker(settings, llm_client_factory=factory) + await worker.start() + for mid in memory_ids: + worker.enqueue( + SignificanceJob( + memory_id=mid, + narrative_text="...", + prior_dialogue=[], + host_bot_id="bot_a", + ) + ) + await worker.stop() + + with open_db(db) as conn: + pinned_count = conn.execute( + "SELECT COUNT(*) FROM memories " + "WHERE owner_id = 'bot_a' AND pinned = 1" + ).fetchone()[0] + assert pinned_count == 8 + + # The oldest should have been evicted. + first_id = memory_ids[0] + first_pinned = conn.execute( + "SELECT pinned FROM memories WHERE id = ?", (first_id,) + ).fetchone()[0] + assert first_pinned == 0 diff --git a/tests/test_turn_flow.py b/tests/test_turn_flow.py index 1f7228b..44c3bfb 100644 --- a/tests/test_turn_flow.py +++ b/tests/test_turn_flow.py @@ -58,6 +58,11 @@ def client(tmp_path, monkeypatch): app.dependency_overrides[get_llm_client] = lambda: mock with TestClient(app) as c: + # Disable the lifespan-managed background worker — it would + # otherwise try to score significance through Featherless with + # a fake test API key. Worker behavior is exercised directly in + # tests/test_significance.py with a mock LLM factory. + app.state.background_worker.enabled = False c.mock_llm = mock # type: ignore[attr-defined] yield c -- 2.52.0 From 3995a8671bdf4dcf8fab54d5a8bf204a13eae76e Mon Sep 17 00:00:00 2001 From: Joseph Doherty Date: Sun, 26 Apr 2026 13:30:40 -0400 Subject: [PATCH 25/45] feat: FTS5 memory retrieval with witness filter and ranking boosts --- chat/state/memory.py | 51 +++++++++++++-- tests/test_memory_search.py | 127 ++++++++++++++++++++++++++++++++++++ 2 files changed, 173 insertions(+), 5 deletions(-) create mode 100644 tests/test_memory_search.py diff --git a/chat/state/memory.py b/chat/state/memory.py index f0420ca..0426067 100644 --- a/chat/state/memory.py +++ b/chat/state/memory.py @@ -87,6 +87,14 @@ def get_pinned(conn: Connection, owner_id: str) -> list[dict]: return [dict(zip(cols, row)) for row in rows] +# Composite-score weights used by ``search_memories`` (T23, §8 retrieval). +# FTS5 BM25 ``rank`` is *more negative* for better matches, so subtracting a +# positive boost from it drives stronger candidates further down (i.e. earlier +# in an ascending sort). Hardcoded for v1 — tunable in a later pass. +_SIGNIFICANCE_WEIGHT = 0.3 +_RECENCY_WEIGHT = 0.5 + + def search_memories( conn: Connection, owner_id: str, @@ -97,16 +105,32 @@ def search_memories( """FTS5 search over pov_summary, scoped by owner and witness role. witness_role must be one of {"you", "host", "guest"} per the witness flags - on each memory row. Returns up to k rows ordered by FTS5 bm25 rank. + on each memory row. Returns up to ``k`` rows ranked by a composite score + that combines the FTS5 BM25 rank with two boosts (§8 retrieval rules): + + * **significance boost** — ``0.3 * significance`` (0..3 per §11.1). + * **recency boost** — ``0.5 * (id / max_id)``, using the row id as a + monotonic recency proxy. Newer memories therefore tilt above older ones + when the BM25 rank and significance are otherwise tied. + + BM25 returns negative scores (lower = better). Both boosts are subtracted + so that stronger candidates yield smaller composite scores; the result is + sorted ascending and truncated to ``k``. The unmodified ``fts_rank`` and a + debug-friendly ``composite_score`` are kept on each returned dict. """ if witness_role not in _VALID_WITNESS_ROLES: raise ValueError( f"witness_role must be one of {sorted(_VALID_WITNESS_ROLES)}, " f"got {witness_role!r}" ) + if not query.strip(): + return [] witness_col = f"witness_{witness_role}" cols = [c[1] for c in conn.execute("PRAGMA table_info(memories)").fetchall()] select_list = ", ".join(f"m.{c}" for c in cols) + # Over-fetch from FTS so the Python-side re-rank has room to reorder + # results that BM25 alone would have demoted past the top-k boundary. + over_fetch = max(k * 4, 20) sql = ( f"SELECT {select_list}, memories_fts.rank AS fts_rank " "FROM memories_fts " @@ -116,10 +140,27 @@ def search_memories( "ORDER BY memories_fts.rank " "LIMIT ?" ) - cur = conn.execute(sql, (owner_id, query, k)) + cur = conn.execute(sql, (owner_id, query, over_fetch)) rows = cur.fetchall() - out: list[dict] = [] + if not rows: + return [] + + # Recency normalises against the current max id for this owner so the + # boost magnitude is bounded regardless of dataset size. + max_id_row = conn.execute( + "SELECT MAX(id) FROM memories WHERE owner_id = ?", (owner_id,) + ).fetchone() + max_id = max_id_row[0] if max_id_row and max_id_row[0] else 1 + result_cols = cols + ["fts_rank"] + enriched: list[dict] = [] for row in rows: - out.append(dict(zip(result_cols, row))) - return out + d = dict(zip(result_cols, row)) + fts_rank = d.get("fts_rank") or 0.0 + sig_boost = _SIGNIFICANCE_WEIGHT * (d.get("significance") or 0) + recency_boost = _RECENCY_WEIGHT * ((d.get("id") or 0) / max_id) + d["composite_score"] = fts_rank - sig_boost - recency_boost + enriched.append(d) + + enriched.sort(key=lambda x: x["composite_score"]) + return enriched[:k] diff --git a/tests/test_memory_search.py b/tests/test_memory_search.py new file mode 100644 index 0000000..dad7e84 --- /dev/null +++ b/tests/test_memory_search.py @@ -0,0 +1,127 @@ +"""Task 23: FTS5 memory retrieval with witness filter and ranking boosts. + +Verifies that ``search_memories`` applies recency + significance boosts on top +of the FTS5 BM25 rank so that newer / more significant memories surface above +older / less significant ones for the same match. Existing T8 behaviour +(witness filter, k limit, FTS match, role validation) is exercised again here +to lock the contract. +""" + +from __future__ import annotations +import pytest + +from chat.db.connection import open_db +from chat.db.migrate import apply_migrations +from chat.eventlog.log import append_event +from chat.eventlog.projector import project +from chat.state.memory import search_memories +import chat.state.memory # noqa: F401 (registers memory_written handler) + + +def _seed(db, *, memory_specs): + """Apply migrations + project a list of memory_written events. + + memory_specs: list of dicts. Required key: ``pov_summary``. Optional keys + override the defaults below. + """ + apply_migrations(db) + with open_db(db) as conn: + for spec in memory_specs: + payload = { + "owner_id": spec.get("owner_id", "bot_a"), + "chat_id": spec.get("chat_id", "chat_bot_a"), + "pov_summary": spec["pov_summary"], + "witness_you": spec.get("witness_you", 1), + "witness_host": spec.get("witness_host", 1), + "witness_guest": spec.get("witness_guest", 0), + "source": "direct", + "reliability": 1.0, + "significance": spec.get("significance", 1), + "pinned": 0, + "auto_pinned": 0, + } + append_event(conn, kind="memory_written", payload=payload) + project(conn) + + +def test_search_filters_by_witness_bit(tmp_path): + db = tmp_path / "t.db" + _seed( + db, + memory_specs=[ + { + "pov_summary": "BotA mentioned her sister", + "witness_you": 1, + "witness_host": 1, + "witness_guest": 0, + }, + ], + ) + with open_db(db) as conn: + # Witnessed by host -> returned. + out = search_memories(conn, "bot_a", "host", "sister", k=4) + assert len(out) == 1 + # NOT witnessed by guest -> filtered out. + out = search_memories(conn, "bot_a", "guest", "sister", k=4) + assert out == [] + + +def test_search_higher_significance_ranks_above_lower(tmp_path): + db = tmp_path / "t.db" + _seed( + db, + memory_specs=[ + # Both match "promise"; the third row carries significance 3 and + # should outrank the first two, which carry the default of 1. + {"pov_summary": "small promise"}, + {"pov_summary": "huge promise"}, + {"pov_summary": "tiny promise", "significance": 3}, + ], + ) + with open_db(db) as conn: + out = search_memories(conn, "bot_a", "host", "promise", k=3) + assert len(out) == 3 + assert out[0]["pov_summary"] == "tiny promise" + assert out[0]["significance"] == 3 + + +def test_search_newer_memory_ranks_above_older_when_same_match(tmp_path): + db = tmp_path / "t.db" + _seed( + db, + memory_specs=[ + {"pov_summary": "BotA said hello"}, + {"pov_summary": "BotA said hello again"}, + ], + ) + with open_db(db) as conn: + out = search_memories(conn, "bot_a", "host", "hello", k=2) + assert len(out) == 2 + # Newer (higher id, "again") wins on the recency boost when the BM25 + # rank and significance are otherwise comparable. + assert out[0]["pov_summary"] == "BotA said hello again" + + +def test_search_respects_k_limit(tmp_path): + db = tmp_path / "t.db" + _seed( + db, + memory_specs=[ + {"pov_summary": "the cat sat"}, + {"pov_summary": "the cat ran"}, + {"pov_summary": "the cat slept"}, + {"pov_summary": "the cat ate"}, + {"pov_summary": "the cat purred"}, + ], + ) + with open_db(db) as conn: + out = search_memories(conn, "bot_a", "host", "cat", k=2) + assert len(out) == 2 + + +def test_search_invalid_witness_role_raises(tmp_path): + db = tmp_path / "t.db" + apply_migrations(db) + with open_db(db) as conn: + with pytest.raises(ValueError): + search_memories(conn, "bot_a", "invalid_role", "anything", k=4) -- 2.52.0 From 5fc5b8ac23b2fe65ee3160dd45087f4255fcee77 Mon Sep 17 00:00:00 2001 From: Joseph Doherty Date: Sun, 26 Apr 2026 13:35:47 -0400 Subject: [PATCH 26/45] feat: read-only drawer with scene, activity, edges, memories --- chat/app.py | 2 + chat/static/app.css | 10 ++ chat/templates/_drawer.html | 94 ++++++++++++++++ chat/templates/chat.html | 7 +- chat/web/drawer.py | 106 ++++++++++++++++++ tests/test_drawer_render.py | 210 ++++++++++++++++++++++++++++++++++++ 6 files changed, 427 insertions(+), 2 deletions(-) create mode 100644 chat/templates/_drawer.html create mode 100644 chat/web/drawer.py create mode 100644 tests/test_drawer_render.py diff --git a/chat/app.py b/chat/app.py index 6233d58..1d47509 100644 --- a/chat/app.py +++ b/chat/app.py @@ -17,6 +17,7 @@ import chat.state.world # noqa: F401 from chat.web.bots import router as bots_router from chat.web.chat import router as chat_router +from chat.web.drawer import router as drawer_router from chat.web.kickoff import router as kickoff_router from chat.web.nav import router as nav_router from chat.web.settings import router as settings_router @@ -62,6 +63,7 @@ app.include_router(kickoff_router) app.include_router(settings_router) app.include_router(nav_router) app.include_router(chat_router) +app.include_router(drawer_router) app.include_router(sse_router) app.include_router(turns_router) diff --git a/chat/static/app.css b/chat/static/app.css index f40bd40..915aa1c 100644 --- a/chat/static/app.css +++ b/chat/static/app.css @@ -79,3 +79,13 @@ code { font-family: ui-monospace, "SF Mono", Menlo, monospace; } .turn-input textarea { padding: 8px; font: inherit; border: 1px solid #ccc; border-radius: 3px; resize: vertical; } .drawer { position: fixed; top: 0; right: 0; width: 360px; height: 100vh; background: #fff; border-left: 1px solid #e5e5e5; padding: 16px; overflow-y: auto; z-index: 10; } .drawer[hidden] { display: none; } +.drawer-content { display: flex; flex-direction: column; gap: 16px; } +.drawer-header { display: flex; align-items: center; justify-content: space-between; padding-bottom: 8px; border-bottom: 1px solid #e5e5e5; } +.drawer-close { border: none; background: transparent; color: #1c1c1c; font-size: 24px; padding: 0 4px; cursor: pointer; } +.drawer-section h3 { margin: 0 0 8px; font-size: 14px; text-transform: uppercase; letter-spacing: 0.5px; color: #666; } +.activity-row, .edge-row { margin-bottom: 12px; } +.activity-row strong, .edge-row strong { display: block; } +.memory-list { list-style: none; padding: 0; margin: 0; } +.memory-list li { padding: 4px 0; font-size: 13px; } +.sig { display: inline-block; min-width: 16px; } +.sig-3 { color: #d4af37; } diff --git a/chat/templates/_drawer.html b/chat/templates/_drawer.html new file mode 100644 index 0000000..637a8d3 --- /dev/null +++ b/chat/templates/_drawer.html @@ -0,0 +1,94 @@ +
+
+

{{ host_bot.name }}

+ +
+ +
+

Scene

+ {% if scene %} +

Started: {{ scene.started_at }}

+ {% endif %} + {% if container %} +

Container: {{ container.name }} ({{ container.type }})

+ {% else %} +

No active container.

+ {% endif %} +

Time: {{ chat.time }}

+
+ +
+

Activity

+ {% for label, act in [("you", you_activity), (host_bot.name, bot_activity)] %} +
+ {{ label }} + {% if act %} +

{{ act.posture or "—" }} / {{ (act.action or {}).verb or "—" }}

+ {% if act.attention %}

attention: {{ act.attention }}

{% endif %} + {% if act.holding %}

holding: {{ act.holding|join(", ") }}

{% endif %} + {% else %} +

No activity recorded.

+ {% endif %} +
+ {% endfor %} +
+ +
+

Edges

+ {% if edge_b2y %} +
+ {{ host_bot.name }} → you +

Affinity: {{ edge_b2y.affinity }}/100 · Trust: {{ edge_b2y.trust }}/100

+ {% if edge_b2y.summary %}

{{ edge_b2y.summary }}

{% endif %} + {% if edge_b2y.knowledge %} +
Knowledge ({{ edge_b2y.knowledge|length }}) +
    {% for fact in edge_b2y.knowledge %}
  • {{ fact }}
    • {% endfor %}
+
+ {% endif %} +
+ {% endif %} + {% if edge_y2b %} +
+ you → {{ host_bot.name }} +

Affinity: {{ edge_y2b.affinity }}/100 · Trust: {{ edge_y2b.trust }}/100

+ {% if edge_y2b.summary %}

{{ edge_y2b.summary }}

{% endif %} +
+ {% endif %} + {% if not edge_b2y and not edge_y2b %} +

No edges yet.

+ {% endif %} +
+ +
+

Pinned memories ({{ pinned|length }} / {{ pin_cap }})

+ {% if pinned %} +
    + {% for m in pinned %} +
  • + {{ ['·','•','★','★★'][m.significance|default(0)] }} + {{ m.pov_summary }} +
    • + {% endfor %} +
+ {% else %} +

No pinned memories.

+ {% endif %} +
+ +
+

Recent memories

+ {% if recent_memories %} +
    + {% for m in recent_memories %} +
  • + {{ ['·','•','★','★★'][m.significance|default(0)] }} + {{ m.pov_summary[:200] }}{% if m.pov_summary|length > 200 %}…{% endif %} +
    • + {% endfor %} +
+ {% else %} +

No memories yet.

+ {% endif %} +
+
diff --git a/chat/templates/chat.html b/chat/templates/chat.html index 02f5c2d..0e45b6b 100644 --- a/chat/templates/chat.html +++ b/chat/templates/chat.html @@ -30,8 +30,11 @@ -
") + assert " + {% endblock %} diff --git a/chat/web/turns.py b/chat/web/turns.py index 4105704..7a7db0e 100644 --- a/chat/web/turns.py +++ b/chat/web/turns.py @@ -58,6 +58,14 @@ from chat.web.render import render_turn_html as _render_turn_html router = APIRouter() +# Module-level registry of in-flight streaming tasks, keyed by chat_id. +# The POST /chats//turns/cancel route looks up the task and calls +# .cancel(); the streaming coroutine in post_turn catches the resulting +# CancelledError, commits the partial as truncated, and unregisters. +# Single-process v1 only — sufficient for one user with multiple tabs. +_in_flight_tasks: dict[str, asyncio.Task] = {} + + def _strip_ooc_for_prompt(parsed: ParsedTurn) -> str: """Concatenate non-OOC segments back to a prose string for the prompt. @@ -70,16 +78,17 @@ def _strip_ooc_for_prompt(parsed: ParsedTurn) -> str: def _read_recent_dialogue(conn, chat_id: str, limit: int = 200) -> list[dict]: - """Return ``user_turn`` and ``assistant_turn`` events for ``chat_id``. + """Return user-side and assistant_turn events for ``chat_id``. - Ordered oldest-first. Skips superseded and hidden rows so regenerated - turns (T29) drop out of the rendered timeline. Each entry is shaped - ``{"speaker": , "text": }`` for the prompt - assembler and the chat-detail template. + Includes ``user_turn``, ``user_turn_edit`` (T29 edited prose), and + ``assistant_turn``. Ordered oldest-first; superseded/hidden rows are + skipped so regenerated turns (T29) drop out of the rendered timeline. + Each entry is shaped ``{"speaker": , "text": }`` + for the prompt assembler and the chat-detail template. """ cur = conn.execute( "SELECT id, kind, payload_json FROM event_log " - "WHERE kind IN ('user_turn', 'assistant_turn') " + "WHERE kind IN ('user_turn', 'user_turn_edit', 'assistant_turn') " " AND superseded_by IS NULL AND hidden = 0 " "ORDER BY id DESC LIMIT ?", (limit,), @@ -91,7 +100,9 @@ def _read_recent_dialogue(conn, chat_id: str, limit: int = 200) -> list[dict]: p = json.loads(payload_json) if p.get("chat_id") != chat_id: continue - if kind == "user_turn": + if kind in ("user_turn", "user_turn_edit"): + # Edited prose substitutes for the original user_turn (the + # original is marked superseded_by and filtered above). out.append({"speaker": "you", "text": p.get("prose", "")}) else: out.append( @@ -173,11 +184,16 @@ async def post_turn( budget_hard=settings.narrative_budget_hard, ) - # 5. Stream and accumulate tokens. + # 5. Stream and accumulate tokens. The stream runs as a Task so the + # /turns/cancel route can invoke ``Task.cancel()`` to abort it + # mid-stream. ``accumulated`` is a closure over the inner coroutine, + # so when the await on ``stream_task`` raises CancelledError below + # we still see whatever tokens were appended before cancellation. accumulated: list[str] = [] truncated = False cancelled = False - try: + + async def _stream() -> None: async for chunk in client.stream( messages, model=settings.narrative_model ): @@ -190,6 +206,11 @@ async def post_turn( "speaker_id": host_bot["id"], }, ) + + stream_task = asyncio.create_task(_stream()) + _in_flight_tasks[chat_id] = stream_task + try: + await stream_task except asyncio.CancelledError: # Preserve the partial output before letting the cancellation # propagate so the transcript reflects what the user actually saw. @@ -198,6 +219,9 @@ async def post_turn( except Exception: # Surface as a truncated turn rather than losing the partial output. truncated = True + finally: + # Always unregister so a subsequent turn can register a fresh task. + _in_flight_tasks.pop(chat_id, None) full_text = "".join(accumulated) @@ -403,6 +427,29 @@ async def post_turn( return Response(status_code=204) +# --------------------------------------------------------------------------- +# Cancel route (Task 34). +# +# Fire-and-forget: the Stop button POSTs here, we mark the in-flight +# streaming Task as cancelled, and return 204 immediately. The cancel +# propagates into the streaming coroutine on its next await, the +# CancelledError handler in ``post_turn`` catches it, and the partial +# is committed with ``truncated=True``. No body is needed — the SSE +# channel is the conveyor of state. If no turn is in flight (or the +# task already completed), we 204 silently so the client can fire the +# Stop button without a precondition check. +# --------------------------------------------------------------------------- + + +@router.post("/chats/{chat_id}/turns/cancel") +async def cancel_turn(chat_id: str, request: Request): + task = _in_flight_tasks.get(chat_id) + if task is None or task.done(): + return Response(status_code=204) + task.cancel() + return Response(status_code=204) + + # --------------------------------------------------------------------------- # Rewind routes (Task 28). # diff --git a/tests/test_streaming_ux.py b/tests/test_streaming_ux.py new file mode 100644 index 0000000..45bf773 --- /dev/null +++ b/tests/test_streaming_ux.py @@ -0,0 +1,176 @@ +"""Streaming UX tests (T34): cancel route, recent-dialogue user_turn_edit +inclusion, and the chat-shell embeds the streaming JS hooks. + +The cancel route is exercised at the no-op level only — the full mid-stream +cancel path is covered indirectly by T19's CancelledError handling. We +verify here that the route itself is registered and silently 204s when no +in-flight task exists, since the JS Stop button fires unconditionally. + +The user_turn_edit inclusion test is the T29 follow-up fix: without it, +the original user_turn drops out of the timeline (correctly) but the +edited prose never lands (incorrectly), so the rendered chat detail is +missing the user's most recent words. +""" + +from __future__ import annotations + +from pathlib import Path + +import pytest +from fastapi.testclient import TestClient + +from chat.app import app +from chat.db.connection import open_db +from chat.eventlog.log import append_event +from chat.eventlog.projector import project + + +@pytest.fixture +def client(tmp_path, monkeypatch): + cfg = tmp_path / "config.toml" + cfg.write_text('featherless_api_key = "test"\n') + monkeypatch.setenv("CHAT_CONFIG_PATH", str(cfg)) + db = tmp_path / "test.db" + monkeypatch.setenv("CHAT_DB_PATH", str(db)) + with TestClient(app) as c: + # Disable the lifespan-managed background worker so it doesn't + # try to score significance through Featherless with the fake key. + worker = getattr(app.state, "background_worker", None) + if worker is not None: + worker.enabled = False + yield c + + +def _seed_chat( + db_path: Path, + bot_id: str = "bot_a", + chat_id: str = "chat_bot_a", +) -> None: + """Seed a bot + chat with the activity rows the prompt assembler expects.""" + with open_db(db_path) as conn: + append_event( + conn, + kind="bot_authored", + payload={ + "id": bot_id, + "name": "BotA", + "persona": "...", + "voice_samples": [], + "traits": [], + "backstory": "", + "initial_relationship_to_you": "", + "kickoff_prose": "...", + }, + ) + append_event( + conn, + kind="chat_created", + payload={ + "id": chat_id, + "host_bot_id": bot_id, + "initial_time": "2026-04-26T20:00:00+00:00", + "narrative_anchor": "Day 1", + "weather": "", + }, + ) + append_event( + conn, + kind="edge_update", + payload={ + "source_id": bot_id, + "target_id": "you", + "chat_id": chat_id, + }, + ) + append_event( + conn, + kind="edge_update", + payload={ + "source_id": "you", + "target_id": bot_id, + "chat_id": chat_id, + }, + ) + append_event( + conn, + kind="activity_change", + payload={ + "entity_id": "you", + "posture": "sitting", + "action": {"verb": "talking"}, + }, + ) + append_event( + conn, + kind="activity_change", + payload={ + "entity_id": bot_id, + "posture": "sitting", + "action": {"verb": "listening"}, + }, + ) + project(conn) + + +def test_cancel_route_no_op_when_no_in_flight(client, tmp_path): + """Hitting cancel with nothing streaming returns 204 silently.""" + _seed_chat(tmp_path / "test.db") + response = client.post("/chats/chat_bot_a/turns/cancel") + assert response.status_code == 204 + + +def test_user_turn_edit_appears_in_recent_dialogue(client, tmp_path): + """The chat-detail timeline includes a user_turn_edit's prose. + + Original user_turn is superseded by the edit, so it drops out, but + the edit's prose should render in its place. + """ + db_path = tmp_path / "test.db" + _seed_chat(db_path) + with open_db(db_path) as conn: + ut_id = append_event( + conn, + kind="user_turn", + payload={ + "chat_id": "chat_bot_a", + "prose": "OriginalUserText", + "segments": [], + }, + ) + edit_id = append_event( + conn, + kind="user_turn_edit", + payload={ + "chat_id": "chat_bot_a", + "prose": "EditedUserText", + "supersedes_user_turn_id": ut_id, + }, + ) + conn.execute( + "UPDATE event_log SET superseded_by = ? WHERE id = ?", + (edit_id, ut_id), + ) + conn.commit() + # No project() call — user_turn / user_turn_edit have no projector + # handlers (transcript-only kinds), and re-projecting would replay + # chat_created and trip its UNIQUE constraint. + + response = client.get("/chats/chat_bot_a") + assert response.status_code == 200 + body = response.text + assert "EditedUserText" in body + # The original (now-superseded) prose must not render. + assert "OriginalUserText" not in body + + +def test_chat_html_includes_stop_streaming_script(client, tmp_path): + """The chat shell embeds the streaming-JS hooks (Stop button + send-lock).""" + _seed_chat(tmp_path / "test.db") + response = client.get("/chats/chat_bot_a") + assert response.status_code == 200 + body = response.text + # Either the CSS class for the Stop button or the JS state flag must + # appear in the embedded script — both are load-bearing for T34. + assert "stop-streaming" in body or "isStreaming" in body + # Cancel route reference must be wired so the Stop button can call it. + assert "/turns/cancel" in body -- 2.52.0 From a302ed427a783097e7bf41f67ad967ec9fe09559 Mon Sep 17 00:00:00 2001 From: Joseph Doherty Date: Sun, 26 Apr 2026 14:33:28 -0400 Subject: [PATCH 37/45] feat: error banners and first-run navigation flow --- CLAUDE.md | 18 +++++ chat/app.py | 29 +++++++- chat/templates/errors.html | 9 +++ chat/web/middleware.py | 61 ++++++++++++++++ tests/test_chat_list.py | 39 +++++++++- tests/test_error_ux.py | 57 +++++++++++++++ tests/test_first_run.py | 146 +++++++++++++++++++++++++++++++++++++ 7 files changed, 354 insertions(+), 5 deletions(-) create mode 100644 chat/templates/errors.html create mode 100644 chat/web/middleware.py create mode 100644 tests/test_error_ux.py create mode 100644 tests/test_first_run.py diff --git a/CLAUDE.md b/CLAUDE.md index c4fb82c..2869b01 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -149,3 +149,21 @@ Don't jump phases. Phase 1 must work end-to-end before Phase 2 lands. - Inference hosting (start with a cloud API, re-evaluate later) - Character template format (during Phase 1) - Multi-session / multi-character casts: **out of scope for v1**. Leave cheap schema hooks only. + +## Phase 1 status + +Phase 1 shipped end-to-end across **35 tasks** (T0–T35). The single-bot core loop is functional: event log + projector, schema + migrations, settings/bot authoring, kickoff confirm, streaming turns, drawer rendering, regenerate/rewind, scene close + per-POV summaries, significance classifier, snapshots/backups, first-run navigation, and friendly 404/500 pages. **168 tests passing.** + +Deferred to Phase 2: second bot, group node, scene configurations, witness filtering across multi-entity scenes, activity/containers, scene-transition compression. Phase 3: event queue + triggers, time skips, active threads. Phase 4: vector retrieval, branching, surgical delete + regenerate, impact-preview UI. + +### Known v1 limitations (read before extending) + +- **Drawer edits scope**: only affinity, significance, and pin can be hand-edited from the drawer. Other v1 fields (knowledge, summary text, traits) are deferred to Phase 1.5. +- **Cold-load snapshot path** is wired and unit-tested but rarely exercised in dev — long-running sessions are the only realistic trigger. +- **WAL sidecar files** (`-wal`, `-shm`) are not captured in nightly backups; the nightly snapshot is a fresh `.backup()` so this is fine for restore but worth knowing if you copy the db file by hand. +- **HTMX SSE event names** may need a version check if you bump the htmx CDN URL in `base.html` — the swap targets are name-coupled. +- **"You" activity rows** can linger after `bot_reset` (the reset purges the bot's chats and the bot's own activity row but not the "you" row that was associated with those chats). Cosmetic, fixed in Phase 1.5. +- **Projector replay is non-idempotent** for plain `INSERT` events. After appending, call `apply_event(conn, event)` for the new row only — calling `project(conn)` re-runs every handler from scratch and will trip uniqueness or duplicate inserts. +- **8-pin auto-cap eviction** is FIFO over the auto-pinned set only. Manual pins survive the eviction; this is by design (manual intent > auto-pin signal). +- **Regenerate (T29) does not broadcast `turn_html` over SSE** — the page must refresh to show the regenerated turn. Acceptable for v1 single-tab usage; Phase 1.5 should wire the SSE event. +- **First-run middleware** fires only on bare `/` and `/chats`. Sub-paths like `/chats/` and `/chats//drawer` pass through (correct: HTMX partials should not page-redirect, and a deep-link to a missing chat should 404, not redirect mid-setup). diff --git a/chat/app.py b/chat/app.py index 709eeea..3e96347 100644 --- a/chat/app.py +++ b/chat/app.py @@ -4,8 +4,11 @@ import logging from contextlib import asynccontextmanager from pathlib import Path -from fastapi import FastAPI +from fastapi import FastAPI, Request +from fastapi.responses import JSONResponse from fastapi.staticfiles import StaticFiles +from fastapi.templating import Jinja2Templates +from starlette.exceptions import HTTPException as StarletteHTTPException from chat.config import load_settings from chat.db.connection import open_db @@ -26,6 +29,7 @@ from chat.web.bots import router as bots_router from chat.web.chat import router as chat_router from chat.web.drawer import router as drawer_router from chat.web.kickoff import router as kickoff_router +from chat.web.middleware import FirstRunRedirectMiddleware from chat.web.nav import router as nav_router from chat.web.settings import router as settings_router from chat.web.sse import router as sse_router @@ -83,10 +87,33 @@ async def lifespan(app: FastAPI): app = FastAPI(title="chat", lifespan=lifespan) +app.add_middleware(FirstRunRedirectMiddleware) STATIC_DIR = Path(__file__).resolve().parent / "static" app.mount("/static", StaticFiles(directory=str(STATIC_DIR)), name="static") +ERROR_TEMPLATES = Jinja2Templates( + directory=str(Path(__file__).resolve().parent / "templates") +) + + +@app.exception_handler(StarletteHTTPException) +async def http_exception_handler(request: Request, exc: StarletteHTTPException): + """Render a friendly HTML page for 404/500; JSON for everything else.""" + if exc.status_code in (404, 500): + return ERROR_TEMPLATES.TemplateResponse( + request, + "errors.html", + { + "status_code": exc.status_code, + "detail": exc.detail or "Something went wrong.", + "active_nav": "chats", + }, + status_code=exc.status_code, + ) + return JSONResponse(status_code=exc.status_code, content={"detail": exc.detail}) + + app.include_router(bots_router) app.include_router(kickoff_router) app.include_router(settings_router) diff --git a/chat/templates/errors.html b/chat/templates/errors.html new file mode 100644 index 0000000..85f44e0 --- /dev/null +++ b/chat/templates/errors.html @@ -0,0 +1,9 @@ +{% extends "layout.html" %} +{% block title %}Error - chat{% endblock %} +{% block content %} +
+

{{ status_code }}

+

{{ detail }}

+

Back to chats

+
+{% endblock %} diff --git a/chat/web/middleware.py b/chat/web/middleware.py new file mode 100644 index 0000000..0d4cb05 --- /dev/null +++ b/chat/web/middleware.py @@ -0,0 +1,61 @@ +from __future__ import annotations + +from fastapi import Request +from fastapi.responses import RedirectResponse +from starlette.middleware.base import BaseHTTPMiddleware + +from chat.db.connection import open_db +from chat.state.entities import get_you, list_bots + + +class FirstRunRedirectMiddleware(BaseHTTPMiddleware): + """Redirect users through the first-run flow (per requirements §16.2). + + Behavior on GET requests to landing routes (``/`` and ``/chats``): + + - No ``you_entity`` → ``/settings`` + - ``you_entity`` exists but no bots → ``/bots/new`` + - Otherwise pass through to the underlying handler. + + The middleware is a no-op for: + + - Non-GET requests (POST/PUT writes proceed and surface their own errors). + - Static assets, health checks, and any path under ``/settings``, + ``/bots``, ``/api``, ``/health``, ``/favicon`` — so the user can + actually complete setup once redirected. + - Sub-paths of ``/chats`` (e.g. ``/chats/``, ``/chats//drawer``); + only the bare landing pages get the redirect treatment. Sub-resources + either 404 cleanly or are HTMX partials that should not page-redirect. + """ + + SKIP_PREFIXES = ( + "/static", + "/settings", + "/bots", + "/health", + "/favicon", + "/api", + ) + + async def dispatch(self, request: Request, call_next): + if request.method != "GET": + return await call_next(request) + + path = request.url.path + if any(path.startswith(p) for p in self.SKIP_PREFIXES): + return await call_next(request) + + # Only fire on the landing routes themselves. + if path != "/" and path != "/chats": + return await call_next(request) + + settings = request.app.state.settings + with open_db(settings.db_path) as conn: + you = get_you(conn) + bots = list_bots(conn) + + if you is None: + return RedirectResponse(url="/settings", status_code=303) + if not bots: + return RedirectResponse(url="/bots/new", status_code=303) + return await call_next(request) diff --git a/tests/test_chat_list.py b/tests/test_chat_list.py index 1f9c998..f320503 100644 --- a/tests/test_chat_list.py +++ b/tests/test_chat_list.py @@ -20,11 +20,29 @@ def client(tmp_path, monkeypatch): yield c -def _author_bot_and_chat(db_path: Path, bot_id: str = "bot_a") -> None: - """Insert a bot and a chat directly via the event log (skip kickoff route).""" +def _author_you(db_path: Path) -> None: + """Author a ``you_entity`` so the first-run middleware doesn't redirect.""" from chat.db.connection import open_db with open_db(db_path) as conn: + append_event( + conn, + kind="you_authored", + payload={"name": "Me", "pronouns": "", "persona": ""}, + ) + project(conn) + + +def _author_bot_and_chat(db_path: Path, bot_id: str = "bot_a") -> None: + """Insert a you_entity, bot, and chat via the event log (skip kickoff route).""" + from chat.db.connection import open_db + + with open_db(db_path) as conn: + append_event( + conn, + kind="you_authored", + payload={"name": "Me", "pronouns": "", "persona": ""}, + ) append_event( conn, kind="bot_authored", @@ -53,13 +71,26 @@ def _author_bot_and_chat(db_path: Path, bot_id: str = "bot_a") -> None: project(conn) -def test_root_redirects_to_chats(client): +def test_root_redirects_to_chats_when_setup_complete(client, tmp_path): + # With both you_entity and a bot present, the first-run middleware + # passes through and the nav router sends "/" → "/chats". + _author_bot_and_chat(tmp_path / "test.db", "bot_a") response = client.get("/", follow_redirects=False) assert response.status_code == 303 assert response.headers["location"] == "/chats" -def test_chats_list_empty_state(client): +def test_chats_list_empty_state(client, tmp_path): + # Author you + a bot but NO chats — should render the empty-state + # chats list, not redirect. + _author_bot_and_chat(tmp_path / "test.db", "bot_a") + # Drop the chat row so we hit the empty-state branch (the helper + # creates a chat — undo it via a fresh seed without chat_created). + from chat.db.connection import open_db + + with open_db(tmp_path / "test.db") as conn: + conn.execute("DELETE FROM chats") + conn.commit() response = client.get("/chats") assert response.status_code == 200 body = response.text.lower() diff --git a/tests/test_error_ux.py b/tests/test_error_ux.py new file mode 100644 index 0000000..d30139f --- /dev/null +++ b/tests/test_error_ux.py @@ -0,0 +1,57 @@ +from __future__ import annotations + +import pytest +from fastapi.testclient import TestClient + +from chat.app import app + + +@pytest.fixture +def client(tmp_path, monkeypatch): + cfg = tmp_path / "config.toml" + cfg.write_text('featherless_api_key = "test"\n') + monkeypatch.setenv("CHAT_CONFIG_PATH", str(cfg)) + db = tmp_path / "test.db" + monkeypatch.setenv("CHAT_DB_PATH", str(db)) + with TestClient(app) as c: + if hasattr(app.state, "background_worker"): + app.state.background_worker.enabled = False + yield c + + +def _setup_minimal_state(db_path): + """Set up enough state so the first-run middleware doesn't redirect.""" + from chat.db.connection import open_db + from chat.eventlog.log import append_event + from chat.eventlog.projector import project + + with open_db(db_path) as conn: + append_event( + conn, + kind="you_authored", + payload={"name": "Me", "pronouns": "", "persona": ""}, + ) + append_event( + conn, + kind="bot_authored", + payload={ + "id": "bot_a", + "name": "BotA", + "persona": "", + "voice_samples": [], + "traits": [], + "backstory": "", + "initial_relationship_to_you": "", + "kickoff_prose": "", + }, + ) + project(conn) + + +def test_404_renders_friendly_page_for_html(client, tmp_path): + _setup_minimal_state(tmp_path / "test.db") + response = client.get("/chats/no_such_chat") + assert response.status_code == 404 + body = response.text + assert "404" in body + assert "back to" in body.lower() diff --git a/tests/test_first_run.py b/tests/test_first_run.py new file mode 100644 index 0000000..c7799aa --- /dev/null +++ b/tests/test_first_run.py @@ -0,0 +1,146 @@ +from __future__ import annotations + +import pytest +from fastapi.testclient import TestClient + +from chat.app import app +from chat.db.connection import open_db +from chat.eventlog.log import append_event +from chat.eventlog.projector import project + + +@pytest.fixture +def client(tmp_path, monkeypatch): + cfg = tmp_path / "config.toml" + cfg.write_text('featherless_api_key = "test"\n') + monkeypatch.setenv("CHAT_CONFIG_PATH", str(cfg)) + db = tmp_path / "test.db" + monkeypatch.setenv("CHAT_DB_PATH", str(db)) + with TestClient(app) as c: + if hasattr(app.state, "background_worker"): + app.state.background_worker.enabled = False + yield c + + +def test_root_redirects_to_settings_when_no_you(client): + response = client.get("/", follow_redirects=False) + assert response.status_code == 303 + assert response.headers["location"] == "/settings" + + +def test_chats_redirects_to_settings_when_no_you(client): + response = client.get("/chats", follow_redirects=False) + assert response.status_code == 303 + assert response.headers["location"] == "/settings" + + +def test_redirects_to_bots_new_when_you_exists_but_no_bots(client, tmp_path): + with open_db(tmp_path / "test.db") as conn: + append_event( + conn, + kind="you_authored", + payload={ + "name": "Me", + "pronouns": "they/them", + "persona": "engineer", + }, + ) + project(conn) + response = client.get("/chats", follow_redirects=False) + assert response.status_code == 303 + assert response.headers["location"] == "/bots/new" + + +def test_root_redirects_to_bots_new_when_you_exists_but_no_bots(client, tmp_path): + with open_db(tmp_path / "test.db") as conn: + append_event( + conn, + kind="you_authored", + payload={ + "name": "Me", + "pronouns": "they/them", + "persona": "engineer", + }, + ) + project(conn) + response = client.get("/", follow_redirects=False) + assert response.status_code == 303 + assert response.headers["location"] == "/bots/new" + + +def test_no_redirect_when_setup_complete(client, tmp_path): + with open_db(tmp_path / "test.db") as conn: + append_event( + conn, + kind="you_authored", + payload={ + "name": "Me", + "pronouns": "they/them", + "persona": "engineer", + }, + ) + append_event( + conn, + kind="bot_authored", + payload={ + "id": "bot_a", + "name": "BotA", + "persona": "...", + "voice_samples": [], + "traits": [], + "backstory": "", + "initial_relationship_to_you": "", + "kickoff_prose": "", + }, + ) + project(conn) + response = client.get("/chats", follow_redirects=False) + # /chats page renders normally (200) instead of redirecting. + assert response.status_code == 200 + + +def test_settings_page_accessible_without_you(client): + """Don't redirect FROM /settings — user needs to fill it out.""" + response = client.get("/settings", follow_redirects=False) + assert response.status_code == 200 + + +def test_bots_new_accessible_without_redirect(client, tmp_path): + with open_db(tmp_path / "test.db") as conn: + append_event( + conn, + kind="you_authored", + payload={"name": "Me", "pronouns": "", "persona": ""}, + ) + project(conn) + response = client.get("/bots/new", follow_redirects=False) + assert response.status_code == 200 + + +def test_bots_list_accessible_without_redirect_when_empty(client, tmp_path): + """The bot list page itself should never redirect — even when empty.""" + with open_db(tmp_path / "test.db") as conn: + append_event( + conn, + kind="you_authored", + payload={"name": "Me", "pronouns": "", "persona": ""}, + ) + project(conn) + response = client.get("/bots", follow_redirects=False) + assert response.status_code == 200 + + +def test_post_to_settings_not_redirected(client): + """POST should bypass middleware — it's a write, not a landing nav.""" + response = client.post( + "/settings", + data={"name": "Me", "pronouns": "", "persona": ""}, + follow_redirects=False, + ) + # Settings POST returns 200 with the saved page (no HTTPException raised). + assert response.status_code == 200 + + +def test_health_endpoint_not_redirected(client): + response = client.get("/health", follow_redirects=False) + assert response.status_code == 200 -- 2.52.0 From 365dacc0d0f58876ccf61ca59a31e7282b0b3ab0 Mon Sep 17 00:00:00 2001 From: Joseph Doherty Date: Sun, 26 Apr 2026 14:39:10 -0400 Subject: [PATCH 38/45] =?UTF-8?q?chore:=20post-Phase-1=20cleanup=20?= =?UTF-8?q?=E2=80=94=20gitignore,=20packaging,=20backlog?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - .gitignore: add *.egg-info/ so editable installs don't show in git status. - pyproject.toml: add [build-system] and [tool.setuptools.packages.find] scoped to chat*, fixing pip install -e . which was failing on data/ auto-discovery. - CLAUDE.md: add Phase 1.5 cleanup backlog section under Phase 1 status, capturing the small follow-ups surfaced in implementer reviews (open_db refactor, regenerate SSE broadcast, you-activity purge, drawer edits for deferred fields, NICE trim order). --- .gitignore | 1 + CLAUDE.md | 10 ++++++++++ pyproject.toml | 7 +++++++ 3 files changed, 18 insertions(+) diff --git a/.gitignore b/.gitignore index 6bf8f70..687ab3d 100644 --- a/.gitignore +++ b/.gitignore @@ -8,3 +8,4 @@ data/ __pycache__/ *.pyc .pytest_cache/ +*.egg-info/ diff --git a/CLAUDE.md b/CLAUDE.md index 2869b01..9a90f4a 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -167,3 +167,13 @@ Deferred to Phase 2: second bot, group node, scene configurations, witness filte - **8-pin auto-cap eviction** is FIFO over the auto-pinned set only. Manual pins survive the eviction; this is by design (manual intent > auto-pin signal). - **Regenerate (T29) does not broadcast `turn_html` over SSE** — the page must refresh to show the regenerated turn. Acceptable for v1 single-tab usage; Phase 1.5 should wire the SSE event. - **First-run middleware** fires only on bare `/` and `/chats`. Sub-paths like `/chats/` and `/chats//drawer` pass through (correct: HTMX partials should not page-redirect, and a deep-link to a missing chat should 404, not redirect mid-setup). + +### Phase 1.5 cleanup backlog + +Small follow-ups identified during Phase 1 reviews. Pick up at any time; none are blocking. + +- **`open_db` refactor.** `chat/web/bots.py:get_conn()` duplicates the context-manager body to add `check_same_thread=False`. Extend `open_db(path, *, check_same_thread=True)` and have `get_conn` call it directly — eliminates the duplicated PRAGMA setup and ensures any future PRAGMA tweak only happens in one place. +- **Regenerate broadcasts `turn_html` over SSE.** Currently a refresh is needed (see T29 limitation above). Mirror the broadcast logic from `chat/web/turns.py:post_turn` after the new `assistant_turn` lands. +- **`bot_reset` purges orphaned "you" activity rows** (see limitation above). Either delete `activity` rows by chat-membership or accept the noise indefinitely; the projection-layer fix is one extra `DELETE FROM activity WHERE entity_id='you' AND container_id IN (SELECT id FROM containers WHERE chat_id IN (...))` clause inside `_apply_bot_reset`. +- **Drawer edits for the deferred v1 fields**: edge_trust slider, edge_summary textarea, memory pov_summary textarea, knowledge_facts add/remove. The `manual_edit` projector already supports `edge_trust` / `edge_summary` / `memory_pov_summary` target_kinds — only the routes are missing. Knowledge_facts needs a new dispatch branch. +- **NICE trim order in prompt assembly** drops previous-scene first instead of last (T18 review). Greedy-cuts heuristic vs spec listing order; revisit if v1 play surfaces a real regression. diff --git a/pyproject.toml b/pyproject.toml index b72bc90..de06982 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -19,6 +19,13 @@ dependencies = [ [project.optional-dependencies] dev = ["pytest>=8", "pytest-asyncio>=0.23", "freezegun>=1.4"] +[build-system] +requires = ["setuptools>=61"] +build-backend = "setuptools.build_meta" + +[tool.setuptools.packages.find] +include = ["chat*"] + [tool.pytest.ini_options] pythonpath = ["."] asyncio_mode = "auto" -- 2.52.0 From 12502d6ec706993db2a9a43a809726851a0322d9 Mon Sep 17 00:00:00 2001 From: Joseph Doherty Date: Sun, 26 Apr 2026 14:50:06 -0400 Subject: [PATCH 39/45] chore: add scripts/seed_sample_bots.py MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Idempotent seeder for three sample bots (Maya — coworker slow-burn, Eli — live-in partner, Sam — bartender / new connection). Each is a distinct relational archetype to exercise the system from different angles. Run from repo root: .venv/bin/python scripts/seed_sample_bots.py Re-running skips ids that already exist. After seeding, walk each bot through kickoff parse-and-confirm at /bots//kickoff. --- scripts/seed_sample_bots.py | 253 ++++++++++++++++++++++++++++++++++++ 1 file changed, 253 insertions(+) create mode 100644 scripts/seed_sample_bots.py diff --git a/scripts/seed_sample_bots.py b/scripts/seed_sample_bots.py new file mode 100644 index 0000000..583cd09 --- /dev/null +++ b/scripts/seed_sample_bots.py @@ -0,0 +1,253 @@ +"""Seed three sample bots via direct event-log append. + +Idempotent: re-running skips bots whose ids already exist. + +Run from the repo root: + .venv/bin/python scripts/seed_sample_bots.py + +After running, walk each bot through kickoff parse-and-confirm at: + http://127.0.0.1:8000/bots//kickoff +""" + +from __future__ import annotations + +from chat.config import load_settings +from chat.db.connection import open_db +from chat.db.migrate import apply_migrations +from chat.eventlog.log import append_and_apply +from chat.state.entities import get_bot + +# Trigger handler registration. +import chat.state.entities # noqa: F401 +import chat.state.edges # noqa: F401 +import chat.state.memory # noqa: F401 +import chat.state.world # noqa: F401 +import chat.state.manual_edit # noqa: F401 + + +SAMPLES: list[dict] = [ + { + "id": "maya", + "name": "Maya Chen", + "persona": ( + "31, senior product designer at the same company you work for. " + "Sharp eye for what isn't said. Outwardly composed and dryly funny; " + "privately prone to overthinking and drafting texts she never sends. " + "Came out of a five-year relationship six months ago and is still " + "pretending she's fine." + ), + "voice_samples": [ + ( + "\"You look like someone who needs water more than another coffee. " + "I'm just saying.\"" + ), + ( + "She tilts her laptop screen toward you without looking up. " + "\"Tell me which one. Don't think about it. The first one your eye " + "lands on is the right one.\"" + ), + ( + "A pause. \"I'm going to head out. ...Unless you want company on the " + "elevator. Which is a weird sentence I just said out loud.\"" + ), + ], + "traits": [ + "dry humor", + "observant", + "perfectionist", + "quick to deflect compliments", + "late-night texter", + "runs on cold brew", + "slow to trust", + "draws in margins when bored", + "hates small talk", + "secretly sentimental", + ], + "backstory": ( + "Grew up in Vancouver, only child of immigrant parents. Design school " + "in Toronto. Three years at this company; took the senior role eight " + "months ago after the previous lead left abruptly. Her father died of " + "a stroke last fall — she flew home for the funeral and was back at " + "her desk on Monday. She has not really talked to anyone about it, " + "including her mother, including her therapist, including her best " + "friend. She works too much. She knows she works too much." + ), + "initial_relationship_to_you": ( + "Coworkers for about eighteen months. Two desks over. You've been on " + "the same product team for the last year. She thinks you're one of the " + "very few people at the company who actually thinks before speaking, " + "which she finds annoying and also relieving. The two of you have " + "lunch sometimes. You stayed late together once before the big launch " + "and she doesn't remember exactly what was said but she remembers the " + "feeling of the empty office. She has not admitted anything to " + "herself. You probably haven't either." + ), + "kickoff_prose": ( + "It's 9:14 on a Thursday and you and Maya are the only people left on " + "the floor. The deck is due in the morning. She has her shoes off " + "under her desk. The kitchen lights flicker once and then steady. She " + "slides her chair back and rubs her eyes with the heels of her hands. " + "\"Okay,\" she says, to no one in particular, \"tell me honestly. " + "Slide eleven — does that read as ambitious or as desperate.\"" + ), + }, + { + "id": "eli", + "name": "Eli Park", + "persona": ( + "34, freelance illustrator. Quiet, tactile, generous with attention " + "but stingy with words. Bakes when stressed. Falls asleep on the " + "couch with his glasses on. Loves you in the kind of way that doesn't " + "need to be announced." + ), + "voice_samples": [ + ( + "\"Hey.\" A pause, like he's deciding if it's worth saying. \"You " + "ate, right?\"" + ), + ( + "He kisses the top of your head and keeps walking, not breaking " + "stride. \"Don't fall asleep on the bathroom floor again. That's " + "all I'm saying.\"" + ), + ( + "\"You don't have to. I just.\" He looks at his hands. \"I just " + "like it when you're around when I'm working. It's stupid. It's " + "whatever.\"" + ), + ], + "traits": [ + "warm", + "present", + "distractible", + "terrible at confrontation", + "leaves coffee mugs in every room", + "draws on napkins", + "gets up at 6am to paint", + "owns far too many sweaters", + "never throws anything away", + "holds your hand without thinking about it", + ], + "backstory": ( + "Born and raised in Queens to Korean parents who ran a dry cleaner. " + "Older sister Lena died in a car accident when he was nineteen — the " + "year he left for art school. He won't talk about her on most days " + "but he keeps a small photo of her in his wallet, and on her birthday " + "he stops talking by 7pm and goes to bed early. He has been a " + "freelance illustrator for nine years. His work has appeared in The " + "New Yorker twice and he refuses to make this a personality trait. " + "He pays his bills on time. He loses his keys constantly." + ), + "initial_relationship_to_you": ( + "You've been together for four years, living together for two. He " + "proposed last summer, kind of — it was tentative and circular and " + "the question wasn't really a question, and you both laughed and " + "didn't really resolve it, and somewhere there is an unspent ring in " + "a sock drawer. You bicker about laundry and the right way to load a " + "dishwasher. He has seen you cry over genuinely stupid commercials. " + "You are each other's first call. You sleep on the left side." + ), + "kickoff_prose": ( + "Sunday morning, late. The blinds are still down. Eli is propped " + "against the headboard reading something on his phone, his glasses " + "pushed up into his hair. You've been awake for a while; he just " + "noticed. He sets the phone face-down on his chest and looks over at " + "you with the small private smile he only uses in this room. \"Hi,\" " + "he says, like it's a whole sentence." + ), + }, + { + "id": "sam", + "name": "Samira Reyes", + "persona": ( + "28, bartender at a small cocktail bar near where you live, doing a " + "part-time master's in psychology she refuses to talk about. " + "Confident posture, careful words. Reads people fast and shares the " + "readings only when she likes them. Single by deliberate choice for " + "the last two years." + ), + "voice_samples": [ + ( + "\"You're back.\" She says it without looking up from polishing " + "the glass. \"Same as last time, or are we trying something new " + "tonight.\"" + ), + ( + "A long look. \"I'm going to ask you a question and you're not " + "going to answer it carefully. The first thing that comes into " + "your head. Ready.\"" + ), + ( + "\"Don't tip me extra because we talked. I'm being serious. " + "That's a different transaction and I don't want it confused.\"" + ), + ], + "traits": [ + "observant", + "blunt", + "kind in unexpected ways", + "reads tarot for fun (doesn't believe in it)", + "drinks black coffee", + "runs at 5am", + "doesn't suffer fools", + "never forgets a face", + "occasional smoker when something is bothering her", + "owns three identical black t-shirts", + ], + "backstory": ( + "Born and raised in El Paso to a single mother who waitressed nights. " + "Came north five years ago for undergrad on a scholarship. Funded the " + "master's herself by bartending — she's careful about money in a way " + "that took being broke to learn. Her undergraduate thesis was on " + "attachment styles and she will not tell you what her own attachment " + "style is. Her mother passed away two years ago after a long illness, " + "and she went home for a month and came back different in ways she " + "can't articulate." + ), + "initial_relationship_to_you": ( + "You've talked at her bar maybe six times over the last month. She " + "knows your drink. The conversations have started lasting longer than " + "they should — you stay until close more often than you mean to. Last " + "week you walked her to her car at 1am because the lot is dim. " + "Nothing happened. You just talked, leaning on the hood of her old " + "Civic, longer than either of you intended. Neither of you has texted " + "the other since. Neither of you has stopped thinking about it." + ), + "kickoff_prose": ( + "It's 11:47 on a Tuesday — slow night. There's exactly one other " + "customer at the far end of the bar, finishing a beer he stopped " + "drinking ten minutes ago. Sam is wiping down the counter in long " + "unhurried passes. She glances up when the door chimes and the small " + "surprise on her face is gone before you'd swear it was there. " + "\"Look who it is,\" she says, even and unreadable, and pulls down a " + "glass without asking what you want." + ), + }, +] + + +def main() -> None: + settings = load_settings() + apply_migrations(settings.db_path) + + created: list[str] = [] + skipped: list[str] = [] + + with open_db(settings.db_path) as conn: + for spec in SAMPLES: + if get_bot(conn, spec["id"]) is not None: + skipped.append(spec["id"]) + continue + append_and_apply(conn, kind="bot_authored", payload=spec) + created.append(spec["id"]) + + print(f"created: {created}") + print(f"skipped (already existed): {skipped}") + print() + print("Walk each new bot through kickoff parse-and-confirm:") + for bot_id in created: + print(f" http://127.0.0.1:8000/bots/{bot_id}/kickoff") + + +if __name__ == "__main__": + main() -- 2.52.0 From 5aab98e4d72b26a35787adfb79345ec585faf45a Mon Sep 17 00:00:00 2001 From: Joseph Doherty Date: Sun, 26 Apr 2026 15:03:13 -0400 Subject: [PATCH 40/45] =?UTF-8?q?fix:=20classifier=20robustness=20?= =?UTF-8?q?=E2=80=94=20schema=20in=20prompt,=20retries,=20kickoff=20fallba?= =?UTF-8?q?ck?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The kickoff parse-and-confirm route was 500-ing intermittently because Hermes-3 + Featherless's response_format={"type":"json_object"} only guarantees JSON output, NOT a particular schema. The model was inventing its own field names (sceneTime, entities, settingDetails) instead of the KickoffParse fields, causing Pydantic validation to fail on both classify() retries. Three changes: 1. Include the Pydantic JSON schema in the system prompt so the model knows exactly which keys to produce. Affects every classify() call (kickoff parse, turn parse, scene-close detect, significance, state-update, scene summarize). Strip ```json fences if the model wraps its output. Bump retries 2 → 3 (model is stochastic; one extra attempt closes most of the remaining gap). 2. parse_kickoff() now passes a default empty KickoffParse so the route degrades to a fillable form instead of 500 when the classifier ultimately fails. The confirm form is the human-in-the-loop; an empty form is strictly better UX than a stack trace. 3. Tests updated: bumped canned-failure arrays from 2 → 3 entries to match the new attempt count; renamed kickoff test from "raises_when_classifier_fails_twice" to "falls_back_to_empty_when_classifier_fails" reflecting the new degraded-but-usable behavior. Verified live with all 3 sample bots (maya/eli/sam) — kickoff route returns 200 across multiple attempts. Full suite: 168 passed. --- chat/llm/classify.py | 31 +++++++++++++++++++++++----- chat/services/kickoff.py | 39 ++++++++++++++++++++++++++++++----- tests/test_classify.py | 2 +- tests/test_kickoff.py | 35 ++++++++++++++++++++----------- tests/test_per_pov_summary.py | 2 +- tests/test_scene_close.py | 2 +- tests/test_significance.py | 2 +- tests/test_state_update.py | 2 +- tests/test_turn_parse.py | 2 +- 9 files changed, 89 insertions(+), 28 deletions(-) diff --git a/chat/llm/classify.py b/chat/llm/classify.py index 66517c6..7074c3b 100644 --- a/chat/llm/classify.py +++ b/chat/llm/classify.py @@ -10,6 +10,18 @@ T = TypeVar("T", bound=BaseModel) REFUSAL_PATTERNS = ("i can't", "i cannot", "i'm sorry, but", "as an ai") +def _strip_json_fences(text: str) -> str: + """Strip ```json ... ``` markdown fences if the model wraps its JSON output.""" + s = text.strip() + if s.startswith("```"): + # Drop the first fence line (which may be ``` or ```json) + s = s.split("\n", 1)[1] if "\n" in s else s[3:] + # Drop the trailing fence + if s.rstrip().endswith("```"): + s = s.rstrip()[:-3] + return s.strip() + + async def classify( client: LLMClient, *, @@ -20,21 +32,30 @@ async def classify( default: T | None = None, timeout_s: float = 10.0, ) -> T: + schema_json = json.dumps(schema.model_json_schema(), indent=2) + schema_block = ( + f"\n\nRespond with a single JSON object matching this exact schema. " + f"Use these field names exactly; do not invent your own keys:\n```json\n{schema_json}\n```" + ) msgs = [ - Message(role="system", content=system + "\n\nRespond with JSON only matching the schema."), + Message(role="system", content=system + schema_block), Message(role="user", content=user), ] - for attempt in range(2): + for attempt in range(3): try: text = await asyncio.wait_for( client.generate(msgs, model=model, response_format={"type": "json_object"}), timeout=timeout_s, ) - if any(p in text.lower()[:80] for p in REFUSAL_PATTERNS) and not text.strip().startswith("{"): + cleaned = _strip_json_fences(text) + if any(p in cleaned.lower()[:80] for p in REFUSAL_PATTERNS) and not cleaned.lstrip().startswith("{"): raise ValueError("refusal-shaped response") - return schema.model_validate_json(text) + return schema.model_validate_json(cleaned) except (ValidationError, ValueError, json.JSONDecodeError, asyncio.TimeoutError): - msgs[0] = Message(role="system", content=system + "\n\nRespond with valid JSON ONLY. No prose.") + msgs[0] = Message( + role="system", + content=system + schema_block + "\n\nRespond with valid JSON ONLY. No prose, no markdown fences.", + ) continue if default is None: raise RuntimeError(f"classify failed for schema {schema.__name__} with no default") diff --git a/chat/services/kickoff.py b/chat/services/kickoff.py index 591b09c..a5540bf 100644 --- a/chat/services/kickoff.py +++ b/chat/services/kickoff.py @@ -85,6 +85,36 @@ def _build_user_prompt( ) +def _empty_activity() -> ActivityShape: + return ActivityShape( + posture="", + action_verb="", + action_interruptible=True, + action_required_attention="low", + action_expected_duration="brief", + ) + + +def _empty_kickoff_parse() -> KickoffParse: + """Default returned when the classifier can't produce a valid parse. + + The user gets a mostly-empty confirm form they can fill in by hand + instead of a 500. ``initial_time_iso`` is left as the current UTC. + """ + from datetime import datetime, timezone + + return KickoffParse( + container_name="", + container_type="", + container_properties={}, + you_activity=_empty_activity(), + bot_activity=_empty_activity(), + initial_time_iso=datetime.now(timezone.utc).isoformat(timespec="seconds"), + edge_seed_summary="", + edge_seed_knowledge_facts=[], + ) + + async def parse_kickoff( client: LLMClient, *, @@ -98,11 +128,9 @@ async def parse_kickoff( ) -> KickoffParse: """Parse authored kickoff prose into a structured ``KickoffParse``. - Internally calls :func:`chat.llm.classify.classify` with a labeled - user prompt. Raises ``RuntimeError`` if the classifier fails twice in - a row — no default is supplied at this layer, since the caller (T13's - confirm form) is responsible for showing an error and letting the - user edit. + Falls back to a mostly-empty default if the classifier fails — the + confirm-and-edit form is the human-in-the-loop, so a degraded form + that the user can fill in is preferable to a 500. """ user_prompt = _build_user_prompt( bot_name=bot_name, @@ -117,5 +145,6 @@ async def parse_kickoff( system=_SYSTEM_PROMPT, user=user_prompt, schema=KickoffParse, + default=_empty_kickoff_parse(), timeout_s=timeout_s, ) diff --git a/tests/test_classify.py b/tests/test_classify.py index d059eaa..c521c70 100644 --- a/tests/test_classify.py +++ b/tests/test_classify.py @@ -18,7 +18,7 @@ async def test_classify_parses_valid_json(): @pytest.mark.asyncio async def test_classify_falls_back_on_unparseable_after_retry(): - mock = MockLLMClient(canned=["nope", "still nope"]) + mock = MockLLMClient(canned=["nope", "still nope", "nope3"]) default = Verdict(score=1, reason="fallback") result = await classify(mock, model="m", system="x", user="y", schema=Verdict, default=default) assert result.reason == "fallback" diff --git a/tests/test_kickoff.py b/tests/test_kickoff.py index c3dfc14..4efa2a7 100644 --- a/tests/test_kickoff.py +++ b/tests/test_kickoff.py @@ -117,15 +117,26 @@ async def test_parse_kickoff_applies_activity_defaults_for_missing_fields(): @pytest.mark.asyncio -async def test_parse_kickoff_raises_when_classifier_fails_twice(): - mock = MockLLMClient(canned=["nope", "still nope"]) - with pytest.raises(RuntimeError): - await parse_kickoff( - mock, - model="m", - bot_name="BotA", - bot_persona="x", - initial_relationship_to_you="y", - kickoff_prose="z", - you_name="You", - ) +async def test_parse_kickoff_falls_back_to_empty_when_classifier_fails(): + """When the classifier fails three times, return an empty KickoffParse + instead of raising — the confirm form lets the user fill in by hand. + """ + mock = MockLLMClient(canned=["nope", "still nope", "still bad"]) + result = await parse_kickoff( + mock, + model="m", + bot_name="BotA", + bot_persona="x", + initial_relationship_to_you="y", + kickoff_prose="z", + you_name="You", + ) + assert isinstance(result, KickoffParse) + assert result.container_name == "" + assert result.container_type == "" + assert result.edge_seed_summary == "" + assert result.edge_seed_knowledge_facts == [] + # Activity defaults sane (action_interruptible defaults to True so the + # confirm form's checkbox is in a reasonable initial state). + assert result.you_activity.action_interruptible is True + assert result.bot_activity.action_interruptible is True diff --git a/tests/test_per_pov_summary.py b/tests/test_per_pov_summary.py index d6ce0f2..3e9d597 100644 --- a/tests/test_per_pov_summary.py +++ b/tests/test_per_pov_summary.py @@ -84,7 +84,7 @@ async def test_summarize_scene_default_on_failure(): """Two consecutive non-JSON returns trip the classifier's retry-then-default path; we should get the empty fallback rather than crashing the close flow.""" - mock = MockLLMClient(canned=["bad", "still bad"]) + mock = MockLLMClient(canned=["bad", "still bad", "bad3"]) result = await summarize_scene( mock, model="x", diff --git a/tests/test_scene_close.py b/tests/test_scene_close.py index 83a9701..41adf4b 100644 --- a/tests/test_scene_close.py +++ b/tests/test_scene_close.py @@ -58,7 +58,7 @@ async def test_detect_scene_close_default_on_failure(): """Two consecutive non-JSON returns trip the classifier's retry-then-default path; we should get the safe ``should_close=False`` fallback rather than crashing the turn flow.""" - mock = MockLLMClient(canned=["nope", "still nope"]) + mock = MockLLMClient(canned=["nope", "still nope", "nope3"]) decision = await detect_scene_close( mock, model="x", diff --git a/tests/test_significance.py b/tests/test_significance.py index e5a538c..97445ac 100644 --- a/tests/test_significance.py +++ b/tests/test_significance.py @@ -45,7 +45,7 @@ async def test_compute_significance_parses_score(): async def test_compute_significance_default_on_failure(): # Both attempts return non-JSON text; the classify wrapper falls back # to the SignificanceVerdict default (score=1, "fallback"). - mock = MockLLMClient(canned=["nope", "still nope"]) + mock = MockLLMClient(canned=["nope", "still nope", "nope3"]) score = await compute_significance( mock, model="x", diff --git a/tests/test_state_update.py b/tests/test_state_update.py index e7c19b3..8fad50a 100644 --- a/tests/test_state_update.py +++ b/tests/test_state_update.py @@ -62,7 +62,7 @@ async def test_compute_state_update_parses_classifier_output(): @pytest.mark.asyncio async def test_compute_state_update_returns_default_on_failure(): """Two malformed classifier responses -> default StateUpdate (zeros).""" - mock = MockLLMClient(canned=["nope", "still nope"]) + mock = MockLLMClient(canned=["nope", "still nope", "nope3"]) result = await compute_state_update( mock, model="x", diff --git a/tests/test_turn_parse.py b/tests/test_turn_parse.py index 3246bd4..026f9b1 100644 --- a/tests/test_turn_parse.py +++ b/tests/test_turn_parse.py @@ -74,7 +74,7 @@ async def test_parse_turn_empty_prose_short_circuits_without_classifier_call(): @pytest.mark.asyncio async def test_parse_turn_raises_when_classifier_fails_twice(): - mock = MockLLMClient(canned=["nope", "still nope"]) + mock = MockLLMClient(canned=["nope", "still nope", "nope3"]) with pytest.raises(RuntimeError): await parse_turn( mock, -- 2.52.0 From 5c039c8e564c419d063e6eb47eef4d678f4c236d Mon Sep 17 00:00:00 2001 From: Joseph Doherty Date: Sun, 26 Apr 2026 15:15:14 -0400 Subject: [PATCH 41/45] fix: classifier timeout + Featherless concurrency cap MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two related issues blocking real-world use of the kickoff parse: 1. Classifier calls take ~12s end-to-end on Featherless for the complex KickoffParse schema (Hermes-3-8B generating ~1.3KB of structured JSON). The 10s timeout was firing on most attempts, causing all 3 retries to time out and the empty-fallback to render with blank form values. Bumping the default classifier_timeout_s 10 → 30s gives generous headroom; measured p99 is ~13s, so 30s is comfortable. 2. Featherless caps concurrent connections per account (2 on free / lower paid tiers). Each turn flow can fire 4–5 calls (parse, scene-close detect, narrative stream, two state-update passes) plus the background significance worker. Without a gate, we'd exceed the cap and fail. Added a class-level ``asyncio.Semaphore`` to FeatherlessClient, shared across all instances, configured once in lifespan from ``Settings.featherless_max_concurrent`` (default 2). Both ``generate`` and ``stream`` acquire the semaphore for the duration of the call; the stream holds it until the async generator completes, so token streaming is correctly accounted for. Verified live: 4/4 sequential kickoff parses for the same bot all succeed with real parsed values (previously ~50% blank-fallback). Full suite: 168 passed. --- chat/app.py | 9 +++++-- chat/config.py | 5 +++- chat/llm/featherless.py | 58 +++++++++++++++++++++++++++++------------ 3 files changed, 53 insertions(+), 19 deletions(-) diff --git a/chat/app.py b/chat/app.py index 3e96347..c9daf90 100644 --- a/chat/app.py +++ b/chat/app.py @@ -65,12 +65,17 @@ async def lifespan(app: FastAPI): app.state.settings = settings + # Cap concurrent Featherless connections to the account's limit + # (free / lower paid tiers cap at 2). Shared across all + # FeatherlessClient instances in the process. + from chat.llm.featherless import FeatherlessClient + + FeatherlessClient.configure_concurrency(settings.featherless_max_concurrent) + # Background worker for the async significance pass (T22). Each job # constructs a fresh FeatherlessClient via the factory; tests can # disable enqueue by toggling ``app.state.background_worker.enabled``. def _factory(): - from chat.llm.featherless import FeatherlessClient - return FeatherlessClient( api_key=settings.featherless_api_key, base_url=settings.featherless_base_url, diff --git a/chat/config.py b/chat/config.py index b2aed74..935cb21 100644 --- a/chat/config.py +++ b/chat/config.py @@ -24,7 +24,10 @@ class Settings(BaseModel): narrative_budget_hard: int = 8000 narrative_budget_soft: int = 6000 classifier_budget_hard: int = 4000 - classifier_timeout_s: float = 10.0 + classifier_timeout_s: float = 30.0 + # Featherless free tier and lower paid tiers cap concurrent connections. + # Set this to your account's max-concurrent-connections limit. + featherless_max_concurrent: int = 2 db_path: Path = DEFAULT_DB data_dir: Path = REPO_ROOT / "data" bind_host: str = "127.0.0.1" diff --git a/chat/llm/featherless.py b/chat/llm/featherless.py index 3e1fbcf..cf1138b 100644 --- a/chat/llm/featherless.py +++ b/chat/llm/featherless.py @@ -1,29 +1,55 @@ from __future__ import annotations +import asyncio from typing import AsyncIterator, Sequence from openai import AsyncOpenAI from .client import Message class FeatherlessClient: + """Client for Featherless's OpenAI-compatible API. + + Featherless caps concurrent connections per account (2 on free / lower + paid tiers). A class-level semaphore gates every ``generate`` and + ``stream`` call so the orchestrator never exceeds the configured cap, + regardless of how many ``FeatherlessClient`` instances are alive. + + Configure once at app startup via :meth:`configure_concurrency`. The + default is 2. + """ + + _semaphore: asyncio.Semaphore | None = None + + @classmethod + def configure_concurrency(cls, max_concurrent: int) -> None: + cls._semaphore = asyncio.Semaphore(max(1, int(max_concurrent))) + + @classmethod + def _sem(cls) -> asyncio.Semaphore: + if cls._semaphore is None: + cls._semaphore = asyncio.Semaphore(2) + return cls._semaphore + def __init__(self, api_key: str, base_url: str = "https://api.featherless.ai/v1"): self._client = AsyncOpenAI(api_key=api_key, base_url=base_url) async def generate(self, messages: Sequence[Message], *, model: str, **params) -> str: - resp = await self._client.chat.completions.create( - model=model, - messages=[{"role": m.role, "content": m.content} for m in messages], - **params, - ) - return resp.choices[0].message.content or "" + async with self._sem(): + resp = await self._client.chat.completions.create( + model=model, + messages=[{"role": m.role, "content": m.content} for m in messages], + **params, + ) + return resp.choices[0].message.content or "" async def stream(self, messages: Sequence[Message], *, model: str, **params) -> AsyncIterator[str]: - stream = await self._client.chat.completions.create( - model=model, - messages=[{"role": m.role, "content": m.content} for m in messages], - stream=True, - **params, - ) - async for chunk in stream: - delta = chunk.choices[0].delta.content or "" - if delta: - yield delta + async with self._sem(): + stream = await self._client.chat.completions.create( + model=model, + messages=[{"role": m.role, "content": m.content} for m in messages], + stream=True, + **params, + ) + async for chunk in stream: + delta = chunk.choices[0].delta.content or "" + if delta: + yield delta -- 2.52.0 From 52555e04552d7f622a5fd380d949d0fabec66a62 Mon Sep 17 00:00:00 2001 From: Joseph Doherty Date: Sun, 26 Apr 2026 15:20:02 -0400 Subject: [PATCH 42/45] fix: reject empty prose on turn submit MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Empty submission was producing a blank user_turn event in the log and firing the LLM stream anyway — the bot would invent a response from the kickoff context alone, producing a monologue with no user input. Two- layer fix: - Browser: add `required` to the prose textarea in chat.html so the form refuses to submit empty. - Server: 400 in post_turn when prose.strip() is empty. Defense in depth — if a client bypasses the textarea attribute (custom UI, curl, etc.), the server still rejects. Verified live: POST with empty body returns 400; POST with whitespace- only returns 400; chat shell renders the textarea with required. Full suite: 168 passed. --- chat/templates/chat.html | 2 +- chat/web/turns.py | 3 +++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/chat/templates/chat.html b/chat/templates/chat.html index 4d9d24b..8833062 100644 --- a/chat/templates/chat.html +++ b/chat/templates/chat.html @@ -26,7 +26,7 @@
- +
diff --git a/chat/web/turns.py b/chat/web/turns.py index 7a7db0e..260bb21 100644 --- a/chat/web/turns.py +++ b/chat/web/turns.py @@ -122,6 +122,9 @@ async def post_turn( conn=Depends(get_conn), client=Depends(get_llm_client), ): + if not prose.strip(): + raise HTTPException(status_code=400, detail="prose cannot be empty") + chat = get_chat(conn, chat_id) if chat is None: raise HTTPException(status_code=404, detail=f"chat not found: {chat_id}") -- 2.52.0 From f0742dd4f988f26f78f3db93b2ab3090f55bf618 Mon Sep 17 00:00:00 2001 From: Joseph Doherty Date: Sun, 26 Apr 2026 15:23:06 -0400 Subject: [PATCH 43/45] fix: use readOnly (not disabled) to lock textarea during stream MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The form-submit handler in chat.html was setting ``textarea.disabled = true`` synchronously before the browser actually serialized the form. Disabled form fields are excluded from submission, so the request body contained ``prose=""`` even when the user had typed text — which the server (correctly) rejected with the new empty-prose 400. Net effect: typing "hello" + Send gave a "prose cannot be empty" error. Switched to ``readOnly``: same UX (user can't edit while streaming) but the field IS submitted. The unlock path now also clears the textarea and refocuses for the next turn. --- chat/templates/chat.html | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/chat/templates/chat.html b/chat/templates/chat.html index 8833062..c2f622c 100644 --- a/chat/templates/chat.html +++ b/chat/templates/chat.html @@ -78,7 +78,11 @@ document.querySelector('.drawer-toggle')?.addEventListener('click', (e) => { function unlock() { isStreaming = false; if (sendBtn) sendBtn.disabled = false; - if (textarea) textarea.disabled = false; + if (textarea) { + textarea.readOnly = false; + textarea.value = ''; + textarea.focus(); + } const stop = shell.querySelector('.stop-streaming'); if (stop) stop.remove(); } @@ -128,7 +132,10 @@ document.querySelector('.drawer-toggle')?.addEventListener('click', (e) => { form.addEventListener('submit', () => { isStreaming = true; if (sendBtn) sendBtn.disabled = true; - if (textarea) textarea.disabled = true; + // readOnly (not disabled) — disabled fields are excluded from the + // form submission, which would send prose="" and trigger the + // server's empty-prose 400. + if (textarea) textarea.readOnly = true; if (!shell.querySelector('.stop-streaming')) { const stopBtn = document.createElement('button'); stopBtn.type = 'button'; -- 2.52.0 From d161e7b8e9af93fff1f8645bbcbeb8ebd85c45fd Mon Sep 17 00:00:00 2001 From: Joseph Doherty Date: Sun, 26 Apr 2026 15:28:08 -0400 Subject: [PATCH 44/45] feat: cap narrative response length + tune sampling MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Bot replies were running long (4 paragraphs of action+dialogue beats per turn) because we never set max_tokens on the narrative call. Three tunable knobs now in Settings (set in data/config.toml to override): - narrative_max_tokens: int = 400 Hard cap on each generated response. ~400 tokens ≈ 1–2 short paragraphs. Drop to 200 for terse banter, bump to 800+ for longer scenes. - narrative_temperature: float = 0.85 Sampling temperature. 0.7 = grounded/consistent (slightly stiff), 0.85 = creative-but-in-character (default), 1.0 = wide variety, >1.0 = often off-the-rails. - prompt closing instruction now nudges: "Keep your response to a single beat — one or two short paragraphs at most. Don't monologue; leave room for the other person to react." Both turns.py (post_turn) and regenerate.py forward the params to client.stream(). FeatherlessClient already passes **params through to the OpenAI-compat endpoint. Note: temperature doesn't control length — that was a common misconception. max_tokens is the actual length cap. Lower temperature makes word choice more predictable (slightly stiffer voice), not shorter. Both knobs are useful for different goals. --- chat/config.py | 7 +++++++ chat/services/prompt.py | 4 +++- chat/services/regenerate.py | 5 ++++- chat/web/turns.py | 5 ++++- 4 files changed, 18 insertions(+), 3 deletions(-) diff --git a/chat/config.py b/chat/config.py index 935cb21..8eb19b6 100644 --- a/chat/config.py +++ b/chat/config.py @@ -23,6 +23,13 @@ class Settings(BaseModel): retrieval_k: int = 4 narrative_budget_hard: int = 8000 narrative_budget_soft: int = 6000 + # Cap on each generated bot response. ~400 tokens ≈ 1–2 short paragraphs. + # Bump if you want longer scenes; drop to 200 for terse banter. + narrative_max_tokens: int = 400 + # Sampling temperature for narrative generation. 0.7 = grounded / + # consistent; 0.85 = creative-but-in-character (default); 1.0 = wide + # variety, can drift; >1.0 = often off-the-rails. + narrative_temperature: float = 0.85 classifier_budget_hard: int = 4000 classifier_timeout_s: float = 30.0 # Featherless free tier and lower paid tiers cap concurrent connections. diff --git a/chat/services/prompt.py b/chat/services/prompt.py index 89b416e..1df074b 100644 --- a/chat/services/prompt.py +++ b/chat/services/prompt.py @@ -211,7 +211,9 @@ def _closing_instruction(speaker_name: str, addressee_name: str) -> str: f"Continue the scene as {speaker_name}, in their voice, responding " "naturally. Use *asterisks* for actions and quotes for dialogue. " f"Stay in character. Do not narrate {addressee_name}'s actions or " - "thoughts." + "thoughts. " + "Keep your response to a single beat — one or two short paragraphs " + "at most. Don't monologue; leave room for the other person to react." ) diff --git a/chat/services/regenerate.py b/chat/services/regenerate.py index d1e983f..c92c0f0 100644 --- a/chat/services/regenerate.py +++ b/chat/services/regenerate.py @@ -156,7 +156,10 @@ async def regenerate_assistant_turn( # 5. Stream the new narrative. accumulated: list[str] = [] async for chunk in client.stream( - messages, model=settings.narrative_model + messages, + model=settings.narrative_model, + max_tokens=settings.narrative_max_tokens, + temperature=settings.narrative_temperature, ): accumulated.append(chunk) await publish( diff --git a/chat/web/turns.py b/chat/web/turns.py index 260bb21..894dc72 100644 --- a/chat/web/turns.py +++ b/chat/web/turns.py @@ -198,7 +198,10 @@ async def post_turn( async def _stream() -> None: async for chunk in client.stream( - messages, model=settings.narrative_model + messages, + model=settings.narrative_model, + max_tokens=settings.narrative_max_tokens, + temperature=settings.narrative_temperature, ): accumulated.append(chunk) await publish( -- 2.52.0 From b8335895e14436f23d16b7565bbdfc415c95f47e Mon Sep 17 00:00:00 2001 From: Joseph Doherty Date: Sun, 26 Apr 2026 15:37:07 -0400 Subject: [PATCH 45/45] docs: add Phase 2 implementation plan with parallel-safe waves 13 tasks across 6 waves (1, 2, 3, 4a, 4b, 5). Designed for parallel subagent execution where file-disjointness allows. Waves 1, 2, 4a, and 5 each contain 2-3 tasks that touch disjoint files and can be dispatched concurrently via the Agent tool with isolation: "worktree". Waves 3 (drawer guest support) and 4b (multi- entity turn flow) are single-task because they touch hot files (_drawer.html, turns.py) that cannot be safely co-modified. Plan covers: - T36: group_node schema + handlers (new migration 0008) - T37: guest_added / guest_removed event handlers (modifies world.py) - T38: relationship-seed service ("have they met?") - T39: interjection classifier service - T40: multi-entity state-update coordinator (6 directed pairs) - T41: multi-witness memory write helper - T42: drawer guest add/remove UI + render - T43: multi-entity prompt assembly (extends T18) - T44: multi-entity turn flow (rewrites post_turn) - T45: multi-entity per-POV summaries on scene close - T46: witness filter cross-coverage tests - T47: bot_reset cascades to guest references - T48: Phase 2 documentation update Plan also documents: - Worktree-per-subagent dispatch pattern using Agent isolation flag - Merge ordering per wave (file-disjointness = conflict-free merges) - Failure recovery (cancel failed parallel task, re-dispatch as solo) - Conflict prevention checklist (verify Files sections disjoint per wave) Tasks file (.tasks.json) carries dependency DAG with `blockedBy` and `parallelGroup` so a future executing-plans run can dispatch correctly. NOT EXECUTING. Plan only. --- .../2026-04-26-v2-phase2-implementation.md | 910 ++++++++++++++++++ ...-26-v2-phase2-implementation.md.tasks.json | 20 + 2 files changed, 930 insertions(+) create mode 100644 docs/plans/2026-04-26-v2-phase2-implementation.md create mode 100644 docs/plans/2026-04-26-v2-phase2-implementation.md.tasks.json diff --git a/docs/plans/2026-04-26-v2-phase2-implementation.md b/docs/plans/2026-04-26-v2-phase2-implementation.md new file mode 100644 index 0000000..211199c --- /dev/null +++ b/docs/plans/2026-04-26-v2-phase2-implementation.md @@ -0,0 +1,910 @@ +# Roleplay Engine — Phase 2 Implementation Plan + +> **For Claude:** REQUIRED SUB-SKILL: Use `superpowers-extended-cc:executing-plans` to implement this plan task-by-task. Use `superpowers-extended-cc:dispatching-parallel-agents` for the parallel waves below. + +**Goal:** Add multi-entity (3-entity) scene support: guest bot can be added to a host's chat; up to 3 entities present simultaneously (you + host + guest); turn flow handles silent witnesses, interjections, per-pair edges, and per-witness memory; drawer reflects guest state; scene close writes per-POV summaries for each present witness. + +**Architecture:** Builds on Phase 1's event-sourced architecture. New event kinds (`guest_added`, `guest_removed`, `group_node_initialized`) carry the multi-entity state changes; existing handlers (`edge_update`, `memory_written`) already accept any `source_id`/`target_id` and witness mask, so most schema work is additive. The "have they met?" first-co-appearance prompt runs once per `(botA, botB)` pair and seeds initial inter-bot edges via existing `edge_update` events. + +**Tech Stack:** Same as Phase 1 (Python 3.11+, FastAPI, HTMX, SQLite, Featherless). No new dependencies. + +**Source-of-truth references:** +- Phase 2 scope: requirements doc §13 "Phase 2 — multi-entity" +- Behavioral details: requirements doc §6.2 (turn-taking with interjection), §7.5 (guest leaves), §8.5 (memory ownership), §11.2 (per-POV summaries on close) +- Conventions: [../../CLAUDE.md](../../CLAUDE.md) §"Behavioral defaults" +- Phase 1 plan (style, TDD pattern): [2026-04-26-v1-phase1-implementation.md](2026-04-26-v1-phase1-implementation.md) + +When a task says "see §X", that's the requirements doc unless stated otherwise. + +--- + +## Pre-flight + +**Branch:** Create `phase-2` from the latest `main` after Phase 1 has been merged. If Phase 1 is still in PR review, branch off `phase-1` directly: + +```bash +# Option A: after main has phase-1 merged +git checkout main && git pull && git checkout -b phase-2 + +# Option B: continue from phase-1 directly +git checkout phase-1 && git pull && git checkout -b phase-2 +``` + +**Schema baseline:** Phase 1 leaves the DB at version 7. Phase 2 adds **0008_group_node.sql**. No other migrations expected. + +**Pinned non-negotiables (carried forward from Phase 1):** +- State changes go through the event log. Use `append_and_apply(conn, kind, payload)` for the live path; `apply_event` only after a fresh `append_event` returning the new id. +- Witness filter every memory read at SQL level (hard `WHERE` constraint; never a soft signal). +- Edges are directed; `botA → botB` and `botB → botA` are independent records. +- Per-POV scene summaries — never write omniscient narration. +- TDD: every task starts with a failing test. +- One commit per task minimum, more if it splits naturally. + +**Verification before claiming done:** Use `superpowers-extended-cc:verification-before-completion` — run the test command, paste actual output. Don't assume green. + +--- + +## Parallel-Execution Strategy + +This plan is structured into **6 waves** of tasks. Within a wave, tasks are designed to touch disjoint files so they can be executed by parallel subagents safely. Between waves, the controller (you, the controlling Claude session) merges each subagent's commits and verifies the suite stays green before dispatching the next wave. + +### How to dispatch a wave in parallel + +Use the **Agent tool with `isolation: "worktree"`** so each subagent gets its own git worktree. The runtime cleans up the worktree automatically if no changes are made; otherwise it returns the path + branch for the controller to merge. + +In a single message, dispatch all tasks in the wave: + +``` +Agent({ + description: "Wave 1 — T36 group_node schema", + subagent_type: "general-purpose", + isolation: "worktree", + prompt: "", +}) +Agent({ + description: "Wave 1 — T37 guest events", + subagent_type: "general-purpose", + isolation: "worktree", + prompt: "", +}) +Agent({ + description: "Wave 1 — T38 relationship-seed service", + subagent_type: "general-purpose", + isolation: "worktree", + prompt: "", +}) +``` + +All three subagents start simultaneously, each working on a private worktree branched off `phase-2`. They cannot see each other's changes (no shared filesystem state) — that's the safety guarantee. + +### After a wave completes + +1. Each subagent returns its worktree path and commit SHA. +2. **Run a spec + quality reviewer subagent on each completed task** (same pattern as Phase 1 — see `superpowers-extended-cc:requesting-code-review`). +3. **Merge the wave into `phase-2`** in any order (file-disjointness guarantees no conflict). Use fast-forward if possible: + + ```bash + git checkout phase-2 + for branch in ; do + git merge --no-ff "$branch" -m "merge: " + done + ``` + +4. **Run the full test suite** on the merged `phase-2`. If it's red, the wave's mutual independence assumption was violated — bisect to find the offending pair, fix, re-merge. +5. **Push `phase-2` to gitea** so the work is durable before the next wave starts. +6. Optionally clean up worktrees: `git worktree remove .worktrees/`. + +### Conflict prevention checklist (apply before dispatch) + +For each parallel wave, verify the **Files** sections of all tasks in that wave have **no overlapping paths**. The waves below are designed to satisfy this; if you decide to add or merge tasks, re-check. + +If a hot file (`chat/web/turns.py`, `chat/services/prompt.py`, `chat/templates/_drawer.html`) needs changes from multiple tasks, do **not** parallelize them — serialize within the wave or split into separate waves. + +### Failure recovery + +If one subagent in a parallel wave fails (test failures, blocked, infinite loop): +- **Do not block the wave on a failure.** Cancel the failed subagent, merge the others' successful work, and re-dispatch the failed task as a single follow-up. +- If a failure exposes a bad assumption shared by multiple tasks (e.g. an event-payload schema mismatch), pause the wave and revisit the plan. + +### Why each wave is parallel-safe + +| Wave | Tasks | Hot files touched | Disjoint? | +|------|-------|-------------------|-----------| +| 1 | T36, T37, T38 | new files only + `chat/state/world.py` (T37 only) | ✅ | +| 2 | T39, T40, T41 | new files only + `chat/services/memory_write.py` (T41 only) | ✅ | +| 3 | T42 | `chat/web/drawer.py`, `chat/templates/_drawer.html` | (single task) | +| 4a | T43, T45 | `chat/services/prompt.py` (T43), `chat/services/scene_summarize.py` (T45) | ✅ | +| 4b | T44 | `chat/web/turns.py`, `chat/services/regenerate.py` | (single task; depends on 4a) | +| 5 | T46, T47, T48 | new tests + `chat/state/entities.py` (T47) + docs (T48) | ✅ | + +--- + +## Task overview + +``` +Wave 1 ─┬─ T36: group_node schema + handler + ├─ T37: guest_added / guest_removed events + └─ T38: relationship-seed service ("have they met?") + +Wave 2 ─┬─ T39: interjection classifier service + ├─ T40: multi-entity state-update coordinator + └─ T41: multi-witness memory write helper + +Wave 3 ─── T42: drawer guest support (add/remove + render guest state) + +Wave 4a ─┬─ T43: multi-entity prompt assembly (extends assemble_narrative_prompt) + └─ T45: multi-entity per-POV summaries on scene close + +Wave 4b ─── T44: multi-entity turn flow integration (post_turn rewrite) + +Wave 5 ─┬─ T46: witness filter test coverage (cross-witness scenarios) + ├─ T47: bot reset cascades to guest scenes + └─ T48: Phase 2 documentation update +``` + +Critical path: 6 sequential merge points. Total tasks: 13. Wall-clock parallelism advantage depends on subagent dispatch overhead, but in principle Wave 1's 3 tasks can run concurrently in ~the time of one task. + +--- + +## Wave 1 — Foundation + +These three tasks are **fully independent**: T36 adds new files only, T37 modifies `chat/state/world.py` (additive event handlers), T38 adds new files only. Dispatch all three in parallel. + +### Task 36: Group node schema + handler + +**Files:** +- Create: `chat/db/migrations/0008_group_node.sql` +- Create: `chat/state/group_node.py` +- Create: `tests/test_group_node.py` + +**Spec:** Adds the `group_node` table (one row per chat, populated when all three entities are present in a scene) and a projector handler for the `group_node_initialized` event. The group node carries the shared summary, group dynamic, inside jokes, and active threads (Phase 3 will populate `active_threads`; for Phase 2, just `summary` and `dynamic` matter). + +**Step 1: Write the failing test** + +```python +# tests/test_group_node.py +from chat.db.migrate import apply_migrations +from chat.db.connection import open_db +from chat.eventlog.log import append_event +from chat.eventlog.projector import project +from chat.state.group_node import get_group_node +import chat.state.entities # noqa +import chat.state.world # noqa +import chat.state.group_node # noqa: F401 - registers handlers + +def test_group_node_initialized_creates_row(tmp_path): + db = tmp_path / "t.db" + apply_migrations(db) + with open_db(db) as conn: + # Seed bot, you, chat (minimal world state — no scene yet) + append_event(conn, kind="bot_authored", payload={ + "id": "bot_a", "name": "BotA", "persona": "...", + "voice_samples": [], "traits": [], "backstory": "", + "initial_relationship_to_you": "", "kickoff_prose": "", + }) + append_event(conn, kind="chat_created", payload={ + "id": "chat_bot_a", "host_bot_id": "bot_a", + "initial_time": "2026-04-26T20:00:00+00:00", + "narrative_anchor": "Day 1", "weather": "", + }) + append_event(conn, kind="group_node_initialized", payload={ + "chat_id": "chat_bot_a", + "members": ["you", "bot_a", "bot_b"], + "summary": "", + "dynamic": "", + }) + project(conn) + gn = get_group_node(conn, "chat_bot_a") + assert gn is not None + assert gn["members"] == ["you", "bot_a", "bot_b"] + assert gn["summary"] == "" +``` + +**Step 2: Run test to verify it fails** + +```bash +.venv/bin/pytest tests/test_group_node.py -v +``` + +Expected: `ModuleNotFoundError: No module named 'chat.state.group_node'`. + +**Step 3: Write minimal implementation** + +`chat/db/migrations/0008_group_node.sql`: + +```sql +CREATE TABLE group_node ( + chat_id TEXT PRIMARY KEY, + members_json TEXT NOT NULL, + summary TEXT NOT NULL DEFAULT '', + dynamic TEXT NOT NULL DEFAULT '', + threads_json TEXT NOT NULL DEFAULT '[]', + updated_at TEXT NOT NULL DEFAULT (datetime('now')) +); +``` + +`chat/state/group_node.py`: + +```python +from __future__ import annotations +import json +from sqlite3 import Connection +from chat.eventlog.projector import on +from chat.eventlog.log import Event + + +@on("group_node_initialized") +def _apply_group_node_initialized(conn: Connection, e: Event) -> None: + p = e.payload + conn.execute( + "INSERT OR REPLACE INTO group_node " + "(chat_id, members_json, summary, dynamic, threads_json) " + "VALUES (?, ?, ?, ?, ?)", + ( + p["chat_id"], + json.dumps(p["members"]), + p.get("summary", ""), + p.get("dynamic", ""), + json.dumps(p.get("threads", [])), + ), + ) + + +@on("group_node_updated") +def _apply_group_node_updated(conn: Connection, e: Event) -> None: + """T45 calls this on scene close to rewrite summary + dynamic.""" + p = e.payload + conn.execute( + "UPDATE group_node SET summary = ?, dynamic = ?, updated_at = datetime('now') " + "WHERE chat_id = ?", + (p.get("summary", ""), p.get("dynamic", ""), p["chat_id"]), + ) + + +def get_group_node(conn: Connection, chat_id: str) -> dict | None: + row = conn.execute( + "SELECT chat_id, members_json, summary, dynamic, threads_json, updated_at " + "FROM group_node WHERE chat_id = ?", + (chat_id,), + ).fetchone() + if not row: + return None + return { + "chat_id": row[0], + "members": json.loads(row[1]), + "summary": row[2], + "dynamic": row[3], + "threads": json.loads(row[4]), + "updated_at": row[5], + } +``` + +**Step 4: Run test to verify it passes** + +```bash +.venv/bin/pytest tests/test_group_node.py -v +``` + +Expected: 1 passed. + +**Step 5: Commit** + +```bash +git add chat/db/migrations/0008_group_node.sql chat/state/group_node.py tests/test_group_node.py +git commit -m "feat: group_node schema + projector handlers" +``` + +**Notes for the implementer:** +- Add a second test for `group_node_updated`: append init then update, assert `summary` and `dynamic` change but `members` stays. +- Add a test for `get_group_node` returning `None` on a missing chat_id. +- Schema version after migration: 8. The migration runner handles this automatically; no test assertion on schema_version. + +--- + +### Task 37: Guest add / remove events + handlers + +**Files:** +- Modify: `chat/state/world.py` (add `_apply_guest_added`, `_apply_guest_removed` handlers; both update `chats.guest_bot_id`) +- Create: `tests/test_guest_events.py` + +**Spec:** Two new event kinds. +- `guest_added` payload: `{chat_id, guest_bot_id}`. Handler sets `chats.guest_bot_id = ?`. +- `guest_removed` payload: `{chat_id}`. Handler sets `chats.guest_bot_id = NULL`. + +These are pure state mutations — no related side effects. The kickoff parse-and-confirm flow (T13, Phase 1) and the new T39 interjection / T42 drawer routes will append these events. + +**Step 1: Write the failing test** + +```python +# tests/test_guest_events.py +from chat.db.migrate import apply_migrations +from chat.db.connection import open_db +from chat.eventlog.log import append_event +from chat.eventlog.projector import project +from chat.state.world import get_chat +import chat.state.entities # noqa +import chat.state.world # noqa + + +def test_guest_added_sets_guest_bot_id(tmp_path): + db = tmp_path / "t.db" + apply_migrations(db) + with open_db(db) as conn: + # Seed bot, chat + append_event(conn, kind="bot_authored", payload={ + "id": "bot_a", "name": "BotA", "persona": "...", + "voice_samples": [], "traits": [], "backstory": "", + "initial_relationship_to_you": "", "kickoff_prose": "", + }) + append_event(conn, kind="bot_authored", payload={ + "id": "bot_b", "name": "BotB", "persona": "...", + "voice_samples": [], "traits": [], "backstory": "", + "initial_relationship_to_you": "", "kickoff_prose": "", + }) + append_event(conn, kind="chat_created", payload={ + "id": "chat_bot_a", "host_bot_id": "bot_a", + "initial_time": "2026-04-26T20:00:00+00:00", + "narrative_anchor": "Day 1", "weather": "", + }) + append_event(conn, kind="guest_added", payload={ + "chat_id": "chat_bot_a", "guest_bot_id": "bot_b", + }) + project(conn) + chat = get_chat(conn, "chat_bot_a") + assert chat["guest_bot_id"] == "bot_b" + + +def test_guest_removed_clears_guest_bot_id(tmp_path): + # similar: add then remove, assert guest_bot_id is None + ... +``` + +**Step 3: Implementation** + +In `chat/state/world.py`, add the two handlers next to `_apply_chat_created`: + +```python +@on("guest_added") +def _apply_guest_added(conn: Connection, e: Event) -> None: + p = e.payload + conn.execute( + "UPDATE chats SET guest_bot_id = ? WHERE id = ?", + (p["guest_bot_id"], p["chat_id"]), + ) + + +@on("guest_removed") +def _apply_guest_removed(conn: Connection, e: Event) -> None: + p = e.payload + conn.execute( + "UPDATE chats SET guest_bot_id = NULL WHERE id = ?", + (p["chat_id"],), + ) +``` + +**Step 5: Commit** + +```bash +git add chat/state/world.py tests/test_guest_events.py +git commit -m "feat: guest_added / guest_removed event handlers" +``` + +**Notes:** +- 2 tests minimum (added, removed). Optional third: idempotent re-add (overwrites cleanly). +- Don't add any UI here — T42 handles UI. + +--- + +### Task 38: Relationship-seed service ("have they met?") + +**Files:** +- Create: `chat/services/relationship_seed.py` +- Create: `tests/test_relationship_seed.py` + +**Spec:** Per requirements §5.2: when two bots first co-appear in a chat, prompt the user with "Have they met before? If yes, write a short prose seed describing how." The seed is parsed via classifier into structured `botA ↔ botB` edge content (summary + initial knowledge facts). + +This task adds the **service layer** only. T39 (interjection) doesn't touch this; T42 (drawer guest UI) calls it via a route added there. So at the service level, we just expose: + +```python +async def seed_inter_bot_edges( + client: LLMClient, + *, + classifier_model: str, + bot_a_id: str, + bot_a_name: str, + bot_b_id: str, + bot_b_name: str, + relationship_prose: str, # user-supplied prose; empty = "they haven't met" + timeout_s: float = 30.0, +) -> RelationshipSeed: + """Parse user-supplied prose into structured edge content for both + directed pairs (bot_a → bot_b and bot_b → bot_a). Return the + RelationshipSeed; caller is responsible for emitting two edge_update + events.""" +``` + +`RelationshipSeed`: + +```python +class RelationshipSeed(BaseModel): + a_to_b_summary: str = "" + a_to_b_knowledge_facts: list[str] = Field(default_factory=list) + a_to_b_affinity_delta: int = 0 # signed, -10..+10 typical + a_to_b_trust_delta: int = 0 + b_to_a_summary: str = "" + b_to_a_knowledge_facts: list[str] = Field(default_factory=list) + b_to_a_affinity_delta: int = 0 + b_to_a_trust_delta: int = 0 +``` + +If `relationship_prose` is empty/whitespace, short-circuit and return an empty `RelationshipSeed` (they haven't met → fresh edges with default 50/50). + +**Step 1: Failing test** + +```python +import pytest, json +from chat.llm.mock import MockLLMClient +from chat.services.relationship_seed import seed_inter_bot_edges, RelationshipSeed + + +@pytest.mark.asyncio +async def test_seed_parses_canned_prose(): + canned = json.dumps({ + "a_to_b_summary": "BotA and BotB went to college together.", + "a_to_b_knowledge_facts": ["BotB has a younger brother."], + "a_to_b_affinity_delta": 5, + "a_to_b_trust_delta": 3, + "b_to_a_summary": "BotB sees BotA as the responsible one.", + "b_to_a_knowledge_facts": ["BotA was once a TA."], + "b_to_a_affinity_delta": 4, + "b_to_a_trust_delta": 5, + }) + mock = MockLLMClient(canned=[canned]) + seed = await seed_inter_bot_edges( + mock, classifier_model="x", + bot_a_id="bot_a", bot_a_name="BotA", + bot_b_id="bot_b", bot_b_name="BotB", + relationship_prose="They went to college together; BotB still sees BotA as the responsible one.", + ) + assert "college" in seed.a_to_b_summary + assert seed.a_to_b_affinity_delta == 5 + + +@pytest.mark.asyncio +async def test_seed_empty_prose_returns_empty(): + mock = MockLLMClient(canned=[]) # never called + seed = await seed_inter_bot_edges( + mock, classifier_model="x", + bot_a_id="bot_a", bot_a_name="BotA", + bot_b_id="bot_b", bot_b_name="BotB", + relationship_prose="", + ) + assert seed == RelationshipSeed() +``` + +**Step 3: Minimal impl** + +Wraps `classify()` from `chat.llm.classify` with a `RelationshipSeed` schema and a system prompt explaining the task. + +**Step 5: Commit** + +```bash +git add chat/services/relationship_seed.py tests/test_relationship_seed.py +git commit -m "feat: relationship-seed service for first-co-appearance prompt" +``` + +--- + +## Wave 2 — Services + +After Wave 1 merges, dispatch Wave 2 in parallel. T39 and T40 are new files; T41 modifies `chat/services/memory_write.py` (additive — adds a new function alongside existing `record_turn_memory`). + +### Task 39: Interjection classifier service + +**Files:** +- Create: `chat/services/interjection.py` +- Create: `tests/test_interjection.py` + +**Spec:** Per requirements §6.2: when a guest is present and the addressee bot has just spoken, decide whether the *non-addressee* bot interjects. Classifier returns `{should_interject: bool, reason: str}`. Caller (T44 turn flow) generates the interjection beat as a brief follow-on response if `should_interject`. + +**Public API:** + +```python +class InterjectionDecision(BaseModel): + should_interject: bool = False + reason: str = "" + + +async def detect_interjection( + client: LLMClient, + *, + classifier_model: str, + addressee_name: str, + addressee_just_said: str, + silent_witness_name: str, + silent_witness_persona: str, + silent_witness_edge_to_addressee: dict, # {affinity, trust, summary} + silent_witness_edge_to_you: dict, + you_just_said: str, + timeout_s: float = 30.0, +) -> InterjectionDecision: + """Decide whether the silent witness bot interjects after the addressee + finishes speaking. Conservative bias — most turns should NOT interject + (return False). Trigger only when the witness's character would + plausibly speak up: jealousy, surprise, agreement worth voicing, + correcting a falsehood, etc. + """ +``` + +Classifier system prompt should explicitly bias toward `should_interject=false` (per spec: "addressee gets the floor"; interjection is the exception). + +**Tests:** 3 minimum. +1. Mock returns `{should_interject: true, reason: "..."}` → result is True. +2. Mock returns `{should_interject: false}` → result is False. +3. Classifier failure → fallback default (`should_interject=false`, `reason="fallback"`). + +**Commit:** `feat: interjection classifier service` + +--- + +### Task 40: Multi-entity state-update coordinator + +**Files:** +- Create: `chat/services/multi_state_update.py` +- Create: `tests/test_multi_state_update.py` + +**Spec:** Wraps the existing `chat.services.state_update.compute_state_update` (single-pair) into a coordinator that runs state updates for **all directed pairs of present entities**. With 3 entities (you, host, guest), that's 6 pairs: + +``` +you → host, host → you +you → guest, guest → you +host → guest, guest → host +``` + +Returns a list of `(source_id, target_id, StateUpdate)` tuples; caller (T44) emits one `edge_update` event per tuple via `append_and_apply`. + +**Public API:** + +```python +async def compute_state_updates_for_present( + client: LLMClient, + *, + classifier_model: str, + present_ids: list[str], # e.g. ["you", "bot_a", "bot_b"] + present_names: dict[str, str], # id -> display name + personas: dict[str, str], # id -> persona blob + prior_edges: dict[tuple[str, str], dict], # (src, tgt) -> {affinity, trust, summary} + recent_dialogue: list[dict], # [{speaker, text}, ...] + timeout_s: float = 30.0, +) -> list[tuple[str, str, StateUpdate]]: + """Run compute_state_update for every directed pair where source != target. + Returns list of (source_id, target_id, update) tuples. Skips pairs + involving "you" with itself. + """ +``` + +Implementation: nested loops over `present_ids`, sequential calls to `compute_state_update` (parallel calls would exceed the Featherless 2-connection cap from the FeatherlessClient semaphore). + +**Tests:** 3 minimum. +1. With 2 present (you, host) → returns 2 updates (existing 1A/2D parity). +2. With 3 present (you, host, guest) → returns 6 updates, one per directed non-self pair. +3. Failures in one pair don't kill the whole batch (per-pair `compute_state_update` already has a default fallback). + +**Commit:** `feat: multi-entity state-update coordinator` + +--- + +### Task 41: Multi-witness memory write helper + +**Files:** +- Modify: `chat/services/memory_write.py` (add `record_turn_memory_for_present` alongside existing `record_turn_memory`; do NOT remove or change `record_turn_memory`) +- Add tests to: `tests/test_memory_write.py` + +**Spec:** Currently Phase 1's `record_turn_memory(conn, *, chat_id, host_bot_id, narrative_text, ...)` writes a single memory event for the host bot's POV. With a guest present, we need: +- One memory in the host's store (witness mask `[1, 1, 1]` if you/host/guest present) +- One memory in the guest's store (same witness mask, owner = guest_bot_id) + +"You" still doesn't have a memory store in v1 (per §5.4 / §11.2). + +**New helper:** + +```python +def record_turn_memory_for_present( + conn, + *, + chat_id: str, + host_bot_id: str, + guest_bot_id: str | None, + narrative_text: str, + scene_id: int | None = None, + chat_clock_at: str | None = None, + source: str = "direct", + significance: int = 1, +) -> dict[str, tuple[int, int]]: + """Write a memory_written event for each present bot witness (host + always; guest if guest_bot_id is not None). Returns {bot_id: + (event_id, memory_id)}. + + Witness mask is [1, 1, 1] when guest is present, [1, 1, 0] otherwise + (mirrors Phase 1 single-bot behavior when guest_bot_id is None). + """ +``` + +Implementation: appends one `memory_written` event per present bot, calling `append_and_apply` for each, and queries the resulting `memories.id` per owner+chat just like Phase 1's `record_turn_memory`. + +**Tests:** 3 minimum, added to `tests/test_memory_write.py`: +1. With `guest_bot_id=None`, behaves identically to `record_turn_memory` (one memory for host, witness `[1, 1, 0]`). +2. With `guest_bot_id="bot_b"`, writes two memories — one each for host and guest, both with witness `[1, 1, 1]`. +3. Returned dict keys match `{host_bot_id, guest_bot_id}` (or just `{host_bot_id}` when no guest). + +**Commit:** `feat: multi-witness memory write helper` + +--- + +## Wave 3 — Drawer guest support (single task) + +This wave is one task because all Phase 2 drawer work touches the same two files (`chat/web/drawer.py` and `chat/templates/_drawer.html`). Splitting would force serial execution with conflict resolution. Single-task wave runs alone. + +### Task 42: Drawer guest support (add/remove + render) + +**Files:** +- Modify: `chat/web/drawer.py` (add `POST /chats/{chat_id}/drawer/guest/add`, `POST /chats/{chat_id}/drawer/guest/remove`; extend `drawer` GET handler to query guest state when present) +- Modify: `chat/templates/_drawer.html` (render guest activity, guest edges, group node summary; add "Add guest" form and "Remove guest" button when applicable) +- Create: `tests/test_drawer_guest.py` + +**Spec:** + +**GET /chats/{chat_id}/drawer** (extend, don't replace): +- Read `chat["guest_bot_id"]` from the existing `get_chat` query. +- If guest present: also fetch `get_bot(conn, guest_bot_id)`, `get_activity(conn, guest_bot_id)`, edges in both `host ↔ guest` directions, edges in both `you ↔ guest` directions, and `get_group_node(conn, chat_id)`. +- Pass all of this to the template. + +**Template changes:** +- New section "Guest" rendering guest's name, activity, and the four edges involving the guest. +- New section "Group" rendering `group_node.summary` and `group_node.dynamic` when present. +- "Add guest" button → expands form with: bot selector (dropdown of authored bots not currently in this chat) + relationship prose textarea (the "have they met?" prompt). +- "Remove guest" button visible when a guest is present. + +**POST /chats/{chat_id}/drawer/guest/add** route: +1. Read form: `guest_bot_id`, `relationship_prose`. +2. 404 if chat or guest_bot is missing. +3. 400 if guest_bot_id == host_bot_id. +4. 400 if a guest is already present. +5. Call `seed_inter_bot_edges` (T38) with the prose. May produce empty seed if prose is blank. +6. Append events: `guest_added`, then up to 2 `edge_update` events (host ↔ guest deltas from the seed). Use `append_and_apply` for each. +7. If all 3 entities are now present and no `group_node` row exists for this chat, append `group_node_initialized` with members=[you, host, guest] and empty summary/dynamic. +8. Return refreshed drawer partial. + +**POST /chats/{chat_id}/drawer/guest/remove** route: +1. 404 if chat missing; 400 if no guest present. +2. Append `scene_closed` for the active scene (per §7.5: removing the guest closes the current scene). +3. Append `guest_removed`. +4. (Per §7.5 the host's chat then implicitly opens a new scene with you+host. For Phase 2, leave that as a manual "next user message creates the new scene" — same as Phase 1 mid-chat reset semantics. Phase 3 may auto-open.) +5. Return refreshed drawer partial. + +**Tests (`tests/test_drawer_guest.py`):** 6 minimum. +1. GET drawer with no guest → no "Guest" section in body. +2. POST add guest → 303-or-200 with refreshed drawer; chat.guest_bot_id is set; `group_node` row created; relationship-seed mock returns canned values; edges have the seeded values. +3. POST add guest with empty relationship_prose → guest added; `seed_inter_bot_edges` short-circuits; edges remain at default 50/50. +4. POST add guest when one is already present → 400. +5. POST remove guest → guest_bot_id NULL, scene_closed event written. +6. GET drawer with guest present → "Guest" section + group_node summary visible. + +**Commit:** `feat: drawer guest add/remove + render` + +**Notes for implementer:** +- The guest-bot-selector dropdown lists bots from `list_bots(conn)` minus the host. Don't filter for "bots not in any chat" — guests can be in multiple chats simultaneously (each chat has its own scene state). +- The "have they met?" prose textarea is the per-pair prompt. v1 only fires it on first co-appearance globally; for v2, fire it every time a `(host, guest)` pair has no existing `host → guest` edge. After the first add, the edge exists, so subsequent adds skip the prose (or render it disabled with "you've already met"). Treat this as Phase 2.5 polish if it gets fiddly — for T42 just always show the prose textarea, blank by default. +- The drawer route already uses `Depends(get_conn)` and templates; reuse the existing dependency and TEMPLATES instance. + +--- + +## Wave 4a — Multi-entity prompt + scene close (parallel) + +T43 and T45 touch different files (`prompt.py` and `scene_summarize.py`). Dispatch both in parallel. + +### Task 43: Multi-entity prompt assembly + +**Files:** +- Modify: `chat/services/prompt.py` (extend `assemble_narrative_prompt` to handle a `guest_id` parameter and fetch guest activity, guest edge, group node into the prompt blocks) +- Add tests to: `tests/test_prompt.py` + +**Spec:** The current `assemble_narrative_prompt(conn, *, chat_id, speaker_bot_id, addressee="you", ...)` only handles you+host. Extend: + +- Accept a `guest_id: str | None = None` parameter (auto-fetched from `chat.guest_bot_id` if not passed; explicit override for tests). +- When `guest_id` is provided: + - Activity block includes the guest's activity (`get_activity(conn, guest_id)`). + - If `speaker_bot_id == guest_id`, the addressee defaults to "you" but caller can override. + - "Speaker's other edges" SHOULD-tier block includes speaker → non-addressee (e.g., host → guest if speaker is host and addressee is you). + - MUST-tier identity block unchanged (still just speaker). + - Group-node summary becomes a SHOULD-tier block when all three are present (after MUST, before retrieved memories). +- Token budget tier ordering unchanged. + +**Tests:** 4 minimum, added to `tests/test_prompt.py`: +1. With `guest_id=None`, output matches existing 2-entity behavior (regression). +2. With `guest_id="bot_b"` present and group_node populated, the assembled system message contains: speaker identity, guest activity, group_node summary, host→guest edge for the speaker. +3. Speaker is the guest (`speaker_bot_id == guest_id`), addressee="you" → guest's edges and group node correctly oriented. +4. Tight budget forces NICE-trim of guest activity → MUST blocks (speaker identity, edge_to_addressee, last 4 turns) survive. + +**Commit:** `feat: multi-entity prompt assembly with guest activity, edges, group node` + +--- + +### Task 45: Multi-entity per-POV summaries on scene close + +**Files:** +- Modify: `chat/services/scene_summarize.py` (extend `apply_scene_close_summary` to write per-POV summaries for **each present witness** with a memory store, not just host) +- Modify: tests in `tests/test_per_pov_summary.py` + +**Spec:** Phase 1's `apply_scene_close_summary` only summarizes from the host bot's POV. For Phase 2: + +- Determine present witnesses with memory stores: host always; guest if `chat.guest_bot_id is not None`. +- For each, generate an independent per-POV summary via `summarize_scene` (the existing classifier wrapper). Each call uses **that bot's** persona, `you_name`, prior `bot → you` edge summary, and the same dialogue. +- Update each owner's memories of the closing scene with their per-POV summary. +- Update **all directed bot → you edges** with per-POV-derived `summary` content. +- If `group_node` exists for this chat, also append `group_node_updated` event with new `summary` and `dynamic` derived from the group view (run `summarize_scene` once with `bot_name="group"`, `bot_persona="all participants"` for a meta-summary). For v1 simplicity, the meta-summary can be naive concat of the host's per-POV summary + guest's per-POV summary; full LLM-merged group view is deferred to Phase 2.5. + +**Tests:** 4 minimum, added to `tests/test_per_pov_summary.py`: +1. With no guest, behavior matches Phase 1 (regression test). +2. With guest, `apply_scene_close_summary` calls `summarize_scene` twice (one per bot witness) — assert mock called 2x. +3. After close, each bot's memories of the closed scene have their respective per-POV summary (different text). +4. With group_node present, after close `get_group_node(conn, chat_id).summary` is updated. + +**Commit:** `feat: per-POV summaries on close for each present witness` + +--- + +## Wave 4b — Turn flow integration (single task; depends on 4a) + +T44 ties everything together. It modifies `chat/web/turns.py` (post_turn) and `chat/services/regenerate.py` to use the new multi-entity primitives. Must run after Wave 4a is merged so `assemble_narrative_prompt` accepts `guest_id` and `apply_scene_close_summary` handles guest. + +### Task 44: Multi-entity turn flow + +**Files:** +- Modify: `chat/web/turns.py` (rewrite `post_turn` to: parse turn → optionally close scene → assemble prompt with guest → narrative stream → write memories for ALL witnesses → state updates for ALL pairs → interjection check + interjection narrative if needed) +- Modify: `chat/services/regenerate.py` (mirror the changes — regenerated turn rebuilds with guest in scope) +- Modify: tests in `tests/test_turn_flow.py` (add multi-entity scenarios) + +**Spec:** Refactored `post_turn` flow: + +``` +1. Validate prose (existing 400 check). +2. Look up chat, host_bot, guest_bot (None if no guest). +3. Parse turn (existing parse_turn). +4. Append user_turn event. +5. Append assistant_turn_started. +6. Detect scene close (existing path; runs even with guest). +7. (Recent dialogue read with multi-witness in mind — same query.) +8. Determine ADDRESSEE: simplest v2 heuristic — addressee is host unless + prose explicitly names guest_bot.name. Pass to assemble_narrative_prompt. +9. Assemble narrative prompt with speaker=addressee, guest_id passed. +10. Stream narrative; broadcast tokens; commit assistant_turn (existing). +11. Write memories: record_turn_memory_for_present(host, guest). +12. State updates: compute_state_updates_for_present, then append_and_apply + one edge_update per pair. +13. INTERJECTION CHECK (only if guest present and addressee != silent witness): + a. Call detect_interjection with the silent witness as candidate. + b. If should_interject: assemble narrative prompt with speaker=silent_witness, + addressee=host (or whoever just spoke), and instruct briefly. + c. Stream second narrative; broadcast as second turn_html; commit second + assistant_turn event. + d. Run state updates + memory writes for the interjection turn too + (smaller scope — just the interjector's outgoing edges + memories). +14. Scene close summary (existing path; now multi-witness via T45). +15. Broadcast turn_html for primary + interjection (if any). +16. Return 204. +``` + +**Addressee heuristic (Phase 2 v1):** simple substring match on bot names. If both names appear or neither: addressee defaults to host. Phase 2.5 / Phase 3 may improve with a classifier call. + +**Cancel & truncated:** unchanged from Phase 1 — both halves of a streaming turn (primary + interjection) cancel together. + +**`regenerate.py` changes:** parallel to `turns.py` — multi-entity prompt assembly + multi-witness memory + multi-pair state update. Interjection regeneration is deferred to Phase 2.5 (regenerate only the addressee's turn for v2). + +**Tests added to `tests/test_turn_flow.py`:** 5 minimum. +1. Single-bot turn (no guest): full suite still passes (regression). +2. Multi-bot turn, no interjection: `post_turn` produces 1 user_turn + 1 assistant_turn + 6 edge_updates + 2 memory_written events. Mock interjection returns `should_interject=false`. +3. Multi-bot turn, with interjection: produces user_turn + 2 assistant_turns + 12 edge_updates + 4 memory_written events. +4. Multi-bot turn, scene close fires: `scene_closed` + multi-POV summaries written (per T45). +5. Addressee detection: prose `"BotB, what do you think?"` routes to BotB as speaker. + +**Commit:** `feat: multi-entity turn flow with interjection support` + +**Notes for implementer:** +- This task is the largest in Phase 2 by line count. Budget for ~150-300 lines of changes across `turns.py` and tests. The implementer should split commits if it helps clarity (one commit for primary turn, one for interjection, one for tests). +- Update the existing `_seed_chat` helper in `tests/test_turn_flow.py` to optionally seed a guest, and add `_seed_chat_with_guest` if cleaner. +- The fixture for the LLM mock now needs to provide canned responses for: parse_turn + scene_close_detect + narrative + state_updates×6 + interjection_decision + (optionally) interjection_narrative + state_updates×2 (interjection's outgoing only). + +--- + +## Wave 5 — Polish (parallel) + +Three independent tasks. Dispatch all three in parallel after Wave 4b merges. + +### Task 46: Witness filter test coverage + +**Files:** +- Create: `tests/test_witness_filter_multi.py` + +**Spec:** Phase 1 tested witness filtering with single-bot scenarios. Phase 2 needs explicit tests for the cross-witness cases: +- Memory with witness `[1, 1, 0]`: visible to host, not guest (when guest queries from their POV). +- Memory with witness `[0, 1, 1]`: visible to host and guest, not "you". +- Secondhand-source memories: `source: "told_by:bot_a"`, witness flag for bot_b set, reliability < 1.0. + +5 tests minimum. + +**Commit:** `test: witness filter coverage for multi-entity scenarios` + +--- + +### Task 47: Bot reset cascades to guest scenes + +**Files:** +- Modify: `chat/state/entities.py` (`_apply_bot_reset` extended to also remove the bot's `guest_bot_id` references in OTHER chats: `UPDATE chats SET guest_bot_id = NULL WHERE guest_bot_id = ?`; remove the bot's activity row in those chats too) +- Modify: tests in `tests/test_reset.py` (add scenario: bot is guest in another's chat; reset clears the guest reference) + +**Spec:** Currently `bot_reset` purges the bot's own chat state, memories, and edges. With Phase 2, a bot can be a guest in another bot's chat — that reference must also clear. Otherwise the host's chat sees a stale guest_bot_id pointing at a phantom bot. + +Update `_apply_bot_reset` handler: + +```python +# After existing purges: +conn.execute("UPDATE chats SET guest_bot_id = NULL WHERE guest_bot_id = ?", (bot_id,)) +conn.execute("DELETE FROM activity WHERE entity_id = ?", (bot_id,)) # already there; covers all chats +``` + +(Activity is keyed by entity_id, so the existing line handles cross-chat activity rows already.) + +**Tests:** 2 minimum, added to `tests/test_reset.py`. +1. BotB is guest in BotA's chat. Reset BotB. Assert `chat_bot_a.guest_bot_id` is NULL. +2. BotB has memories (witness flag set, owner=bot_b) from being guest in BotA's chat. Reset BotB. Assert those memories are gone. + +**Commit:** `fix: bot_reset cascades to guest references in other chats` + +--- + +### Task 48: Phase 2 documentation update + +**Files:** +- Modify: `CLAUDE.md` (add "Phase 2 status" section; update "Behavioral defaults" with multi-entity additions; add to "Phase 1.5 / 2 cleanup backlog" any v2 follow-ups discovered during execution) +- Modify: `docs/plans/2026-04-26-v1-requirements-design.md` (mark Phase 2 deliverables as "shipped" in the appendix decisions log) + +**Spec:** Documentation-only task. Run last in Phase 2 so it captures any deviations from the plan that emerged during execution. Reflect: +- Multi-entity scene support (you + host + guest). +- Interjection model (default false; explicit signals only). +- Per-POV summaries on close for all witnesses with memory stores. +- Group node populated on first 3-entity scene; updated on close. +- Phase 2 known limitations: + - "Meanwhile…" (scene config 4 — bot+bot without you) deferred to Phase 3. + - Interjection regeneration deferred (regenerate only acts on the addressee turn). + - Addressee detection is a simple name-match heuristic (no classifier call yet). + +**Commit:** `docs: phase 2 status, behavioral defaults, deferred items` + +--- + +## Wrap-up + +After Wave 5 lands: + +1. **Run full suite** on `phase-2`: should be ~210+ tests passing (168 from Phase 1 + ~45 new). +2. **Manual smoke**: + - Add a guest to one of the seeded bots' chats via the drawer. + - Verify "have they met?" prose seeds inter-bot edges. + - Play a few turns; verify host responds normally; verify guest occasionally interjects. + - Close the scene; check drawer for two distinct per-POV summaries. + - Remove guest mid-scene; check scene_closed fires. + - Reset a guest bot from another chat; verify guest_bot_id reference clears. +3. **Push `phase-2`** to gitea. +4. **Open PR** `phase-2 → main`. +5. **Phase 2.5 backlog candidates** (track in CLAUDE.md): interjection regenerate UI, classifier-based addressee detection, group-node LLM-merged meta-summary, drawer "first-meeting" gate vs "they already know each other" toggle, witness flag editing in drawer (currently read-only by spec). + +--- + +## Notes for the controller running this plan + +- **Don't dispatch Wave 4b until Wave 4a is merged AND tested green on `phase-2`.** Wave 4b's `turns.py` changes import the new `assemble_narrative_prompt` signature from Wave 4a's `prompt.py`; missing that produces import-time failures. +- **After each parallel wave**, the controller should run a code-review subagent (`subagent-driven-development` skill's two-stage review pattern) on each task before merging to `phase-2`. For purely mechanical tasks, a combined spec+quality review is acceptable. +- **If a parallel wave's merge produces a conflict**, the wave's file-disjointness assumption was violated. Bisect the affected pair, fix the offending task in a follow-up commit on `phase-2`, and proceed. +- **Token-spend rough estimate**: Phase 2 should be ~30-40% the size of Phase 1 (smaller scope; reuses Phase 1 patterns). Per-task token spend similar to Phase 1. +- **DO NOT modify Phase 1 code paths** unless explicitly required (e.g., Wave 5 T47 modifies `_apply_bot_reset` because the cascade is genuinely new behavior). The single-bot path must continue to work end-to-end after each wave. diff --git a/docs/plans/2026-04-26-v2-phase2-implementation.md.tasks.json b/docs/plans/2026-04-26-v2-phase2-implementation.md.tasks.json new file mode 100644 index 0000000..56d2934 --- /dev/null +++ b/docs/plans/2026-04-26-v2-phase2-implementation.md.tasks.json @@ -0,0 +1,20 @@ +{ + "planPath": "docs/plans/2026-04-26-v2-phase2-implementation.md", + "tasks": [ + {"id": 36, "subject": "T36: group_node schema + projector handlers", "status": "pending", "wave": 1, "parallelGroup": "wave-1"}, + {"id": 37, "subject": "T37: guest_added / guest_removed event handlers", "status": "pending", "wave": 1, "parallelGroup": "wave-1"}, + {"id": 38, "subject": "T38: relationship-seed service for first-co-appearance prompt", "status": "pending", "wave": 1, "parallelGroup": "wave-1"}, + {"id": 39, "subject": "T39: interjection classifier service", "status": "pending", "wave": 2, "parallelGroup": "wave-2", "blockedBy": [37]}, + {"id": 40, "subject": "T40: multi-entity state-update coordinator", "status": "pending", "wave": 2, "parallelGroup": "wave-2", "blockedBy": [37]}, + {"id": 41, "subject": "T41: multi-witness memory write helper", "status": "pending", "wave": 2, "parallelGroup": "wave-2", "blockedBy": [37]}, + {"id": 42, "subject": "T42: drawer guest add/remove + render", "status": "pending", "wave": 3, "parallelGroup": null, "blockedBy": [36, 37, 38]}, + {"id": 43, "subject": "T43: multi-entity prompt assembly with guest activity, edges, group node", "status": "pending", "wave": 4, "parallelGroup": "wave-4a", "blockedBy": [36, 37]}, + {"id": 45, "subject": "T45: per-POV summaries on close for each present witness", "status": "pending", "wave": 4, "parallelGroup": "wave-4a", "blockedBy": [36, 37]}, + {"id": 44, "subject": "T44: multi-entity turn flow with interjection support", "status": "pending", "wave": 4, "parallelGroup": null, "blockedBy": [39, 40, 41, 43, 45]}, + {"id": 46, "subject": "T46: witness filter test coverage for multi-entity scenarios", "status": "pending", "wave": 5, "parallelGroup": "wave-5", "blockedBy": [44]}, + {"id": 47, "subject": "T47: bot_reset cascades to guest references in other chats", "status": "pending", "wave": 5, "parallelGroup": "wave-5", "blockedBy": [37]}, + {"id": 48, "subject": "T48: Phase 2 documentation update", "status": "pending", "wave": 5, "parallelGroup": "wave-5", "blockedBy": [44]} + ], + "lastUpdated": "2026-04-26T00:00:00Z", + "notes": "13 tasks across 6 waves (1, 2, 3, 4a, 4b, 5). Waves 1, 2, 4a, 5 are parallel-safe (file-disjoint within each). Waves 3 and 4b are single-task. Use Agent tool with isolation: 'worktree' to dispatch parallel tasks. Merge each wave's worktrees back into phase-2 before dispatching the next wave. See plan §Parallel-Execution Strategy for full guidance." +} -- 2.52.0