Joseph Doherty
5aab98e4d7
The kickoff parse-and-confirm route was 500-ing intermittently because
Hermes-3 + Featherless's response_format={"type":"json_object"} only
guarantees JSON output, NOT a particular schema. The model was inventing
its own field names (sceneTime, entities, settingDetails) instead of
the KickoffParse fields, causing Pydantic validation to fail on both
classify() retries.
Three changes:
1. Include the Pydantic JSON schema in the system prompt so the model
knows exactly which keys to produce. Affects every classify() call
(kickoff parse, turn parse, scene-close detect, significance,
state-update, scene summarize). Strip ```json fences if the model
wraps its output. Bump retries 2 → 3 (model is stochastic; one extra
attempt closes most of the remaining gap).
2. parse_kickoff() now passes a default empty KickoffParse so the
route degrades to a fillable form instead of 500 when the classifier
ultimately fails. The confirm form is the human-in-the-loop; an
empty form is strictly better UX than a stack trace.
3. Tests updated: bumped canned-failure arrays from 2 → 3 entries to
match the new attempt count; renamed kickoff test from
"raises_when_classifier_fails_twice" to
"falls_back_to_empty_when_classifier_fails" reflecting the new
degraded-but-usable behavior.
Verified live with all 3 sample bots (maya/eli/sam) — kickoff route
returns 200 across multiple attempts. Full suite: 168 passed.
238 lines
7.7 KiB
Python
238 lines
7.7 KiB
Python
"""Post-turn state-update pass (T20).
|
|
|
|
Per Requirements §3.4, after each utterance we run a classifier on every
|
|
present entity (silent witnesses too) to extract directed-edge deltas
|
|
(``affinity_delta``, ``trust_delta``, ``knowledge_facts``). The deltas
|
|
land as ``edge_update`` events and project into the ``edges`` table.
|
|
|
|
These tests cover:
|
|
- The unit-level :func:`compute_state_update` happy path: classifier
|
|
returns valid JSON, the wrapper returns a populated ``StateUpdate``.
|
|
- The unit-level fallback path: classifier fails twice, the wrapper
|
|
returns a no-op ``StateUpdate`` (zeros + empty facts) per §3.3.
|
|
- The integration path: a successful POST appends two ``edge_update``
|
|
events (one per direction) after the ``assistant_turn`` and the edge
|
|
projections reflect the deltas.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import json
|
|
from pathlib import Path
|
|
|
|
import pytest
|
|
from fastapi.testclient import TestClient
|
|
|
|
from chat.app import app
|
|
from chat.db.connection import open_db
|
|
from chat.eventlog.log import append_event
|
|
from chat.eventlog.projector import project
|
|
from chat.llm.mock import MockLLMClient
|
|
from chat.services.state_update import StateUpdate, compute_state_update
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_compute_state_update_parses_classifier_output():
|
|
canned = json.dumps(
|
|
{"affinity_delta": 2, "trust_delta": 1, "knowledge_facts": ["likes coffee"]}
|
|
)
|
|
mock = MockLLMClient(canned=[canned])
|
|
result = await compute_state_update(
|
|
mock,
|
|
model="x",
|
|
source_id="bot_a",
|
|
target_id="you",
|
|
source_name="BotA",
|
|
source_persona="thoughtful",
|
|
target_name="Me",
|
|
prior_affinity=50,
|
|
prior_trust=50,
|
|
prior_summary="",
|
|
recent_dialogue=[
|
|
{"speaker": "you", "text": "hi"},
|
|
{"speaker": "BotA", "text": "Hello!"},
|
|
],
|
|
)
|
|
assert isinstance(result, StateUpdate)
|
|
assert result.affinity_delta == 2
|
|
assert result.trust_delta == 1
|
|
assert result.knowledge_facts == ["likes coffee"]
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_compute_state_update_returns_default_on_failure():
|
|
"""Two malformed classifier responses -> default StateUpdate (zeros)."""
|
|
mock = MockLLMClient(canned=["nope", "still nope", "nope3"])
|
|
result = await compute_state_update(
|
|
mock,
|
|
model="x",
|
|
source_id="bot_a",
|
|
target_id="you",
|
|
source_name="BotA",
|
|
source_persona="",
|
|
target_name="Me",
|
|
prior_affinity=50,
|
|
prior_trust=50,
|
|
prior_summary="",
|
|
recent_dialogue=[],
|
|
)
|
|
assert result.affinity_delta == 0
|
|
assert result.trust_delta == 0
|
|
assert result.knowledge_facts == []
|
|
|
|
|
|
# --- integration test --------------------------------------------------------
|
|
|
|
|
|
@pytest.fixture
|
|
def client(tmp_path, monkeypatch):
|
|
cfg = tmp_path / "config.toml"
|
|
cfg.write_text('featherless_api_key = "test"\n')
|
|
monkeypatch.setenv("CHAT_CONFIG_PATH", str(cfg))
|
|
db = tmp_path / "test.db"
|
|
monkeypatch.setenv("CHAT_DB_PATH", str(db))
|
|
|
|
canned_parse = json.dumps(
|
|
{"segments": [{"kind": "dialogue", "text": "hello"}]}
|
|
)
|
|
canned_response = "Hi there."
|
|
canned_state_b2y = json.dumps(
|
|
{"affinity_delta": 2, "trust_delta": 1, "knowledge_facts": ["greets warmly"]}
|
|
)
|
|
canned_state_y2b = json.dumps(
|
|
{"affinity_delta": 3, "trust_delta": 0, "knowledge_facts": []}
|
|
)
|
|
|
|
from chat.web.kickoff import get_llm_client
|
|
|
|
mock = MockLLMClient(
|
|
canned=[canned_parse, canned_response, canned_state_b2y, canned_state_y2b]
|
|
)
|
|
app.dependency_overrides[get_llm_client] = lambda: mock
|
|
|
|
with TestClient(app) as c:
|
|
c.mock_llm = mock # type: ignore[attr-defined]
|
|
yield c
|
|
|
|
app.dependency_overrides.clear()
|
|
|
|
|
|
def _seed(db_path: Path) -> None:
|
|
"""Author a bot, create a chat, and seed enough state for prompt assembly."""
|
|
with open_db(db_path) as conn:
|
|
append_event(
|
|
conn,
|
|
kind="bot_authored",
|
|
payload={
|
|
"id": "bot_a",
|
|
"name": "BotA",
|
|
"persona": "thoughtful, observant",
|
|
"voice_samples": [],
|
|
"traits": [],
|
|
"backstory": "",
|
|
"initial_relationship_to_you": "",
|
|
"kickoff_prose": "...",
|
|
},
|
|
)
|
|
append_event(
|
|
conn,
|
|
kind="chat_created",
|
|
payload={
|
|
"id": "chat_bot_a",
|
|
"host_bot_id": "bot_a",
|
|
"initial_time": "2026-04-26T20:00:00+00:00",
|
|
"narrative_anchor": "Day 1",
|
|
"weather": "",
|
|
},
|
|
)
|
|
append_event(
|
|
conn,
|
|
kind="edge_update",
|
|
payload={
|
|
"source_id": "bot_a",
|
|
"target_id": "you",
|
|
"chat_id": "chat_bot_a",
|
|
"knowledge_facts": ["coworker"],
|
|
},
|
|
)
|
|
append_event(
|
|
conn,
|
|
kind="activity_change",
|
|
payload={
|
|
"entity_id": "you",
|
|
"posture": "sitting",
|
|
"action": {
|
|
"verb": "talking",
|
|
"interruptible": True,
|
|
"required_attention": "low",
|
|
"expected_duration": "ongoing",
|
|
},
|
|
"attention": "",
|
|
"holding": [],
|
|
"status": {},
|
|
},
|
|
)
|
|
append_event(
|
|
conn,
|
|
kind="activity_change",
|
|
payload={
|
|
"entity_id": "bot_a",
|
|
"posture": "sitting",
|
|
"action": {
|
|
"verb": "listening",
|
|
"interruptible": True,
|
|
"required_attention": "low",
|
|
"expected_duration": "ongoing",
|
|
},
|
|
"attention": "",
|
|
"holding": [],
|
|
"status": {},
|
|
},
|
|
)
|
|
project(conn)
|
|
|
|
|
|
def test_post_turn_appends_edge_updates_and_applies_deltas(client, tmp_path):
|
|
"""After a turn, edge_update events fire for both directions and project."""
|
|
db_path = tmp_path / "test.db"
|
|
_seed(db_path)
|
|
|
|
response = client.post("/chats/chat_bot_a/turns", data={"prose": "hello"})
|
|
assert response.status_code == 204
|
|
|
|
with open_db(db_path) as conn:
|
|
# Two new edge_update events should land *after* the assistant_turn.
|
|
cur = conn.execute(
|
|
"SELECT kind, payload_json FROM event_log "
|
|
"WHERE kind = 'edge_update' "
|
|
"AND id > (SELECT MAX(id) FROM event_log WHERE kind = 'assistant_turn') "
|
|
"ORDER BY id"
|
|
)
|
|
rows = cur.fetchall()
|
|
assert len(rows) == 2
|
|
kinds = [r[0] for r in rows]
|
|
assert kinds == ["edge_update", "edge_update"]
|
|
|
|
# Inspect the two payloads — one per direction.
|
|
payloads = [json.loads(r[1]) for r in rows]
|
|
directions = {(p["source_id"], p["target_id"]) for p in payloads}
|
|
assert ("bot_a", "you") in directions
|
|
assert ("you", "bot_a") in directions
|
|
|
|
# Edge bot_a -> you: seeded affinity=50, plus delta 2 -> 52.
|
|
from chat.state.edges import get_edge
|
|
|
|
edge_b2y = get_edge(conn, "bot_a", "you")
|
|
assert edge_b2y is not None
|
|
assert edge_b2y["affinity"] == 52
|
|
assert edge_b2y["trust"] == 51
|
|
# Existing fact preserved, new fact appended.
|
|
assert "coworker" in edge_b2y["knowledge"]
|
|
assert "greets warmly" in edge_b2y["knowledge"]
|
|
|
|
# Edge you -> bot_a: defaults (50/50) plus delta +3 affinity -> 53.
|
|
edge_y2b = get_edge(conn, "you", "bot_a")
|
|
assert edge_y2b is not None
|
|
assert edge_y2b["affinity"] == 53
|
|
assert edge_y2b["trust"] == 50
|