Files
chat/chat/services/kickoff.py
T
Joseph Doherty 5aab98e4d7 fix: classifier robustness — schema in prompt, retries, kickoff fallback
The kickoff parse-and-confirm route was 500-ing intermittently because
Hermes-3 + Featherless's response_format={"type":"json_object"} only
guarantees JSON output, NOT a particular schema. The model was inventing
its own field names (sceneTime, entities, settingDetails) instead of
the KickoffParse fields, causing Pydantic validation to fail on both
classify() retries.

Three changes:

1. Include the Pydantic JSON schema in the system prompt so the model
   knows exactly which keys to produce. Affects every classify() call
   (kickoff parse, turn parse, scene-close detect, significance,
   state-update, scene summarize). Strip ```json fences if the model
   wraps its output. Bump retries 2 → 3 (model is stochastic; one extra
   attempt closes most of the remaining gap).

2. parse_kickoff() now passes a default empty KickoffParse so the
   route degrades to a fillable form instead of 500 when the classifier
   ultimately fails. The confirm form is the human-in-the-loop; an
   empty form is strictly better UX than a stack trace.

3. Tests updated: bumped canned-failure arrays from 2 → 3 entries to
   match the new attempt count; renamed kickoff test from
   "raises_when_classifier_fails_twice" to
   "falls_back_to_empty_when_classifier_fails" reflecting the new
   degraded-but-usable behavior.

Verified live with all 3 sample bots (maya/eli/sam) — kickoff route
returns 200 across multiple attempts. Full suite: 168 passed.
2026-04-26 15:03:13 -04:00

151 lines
4.7 KiB
Python

"""Kickoff prose parser.
Service-layer function that converts a bot's authored kickoff prose into a
structured ``KickoffParse`` for the kickoff confirm-and-edit step (T13 will
wire this into the UI flow).
The classifier prompt includes only the bot context that's load-bearing for
parsing the opening scene: name, persona, the authored
``initial_relationship_to_you`` blurb, the ``you`` entity name, and the
kickoff prose itself. Other identity fields (traits, backstory, ...) are
intentionally left out — they would be noise for this extraction.
"""
from __future__ import annotations
from pydantic import BaseModel, Field
from chat.llm.classify import classify
from chat.llm.client import LLMClient
class ActivityShape(BaseModel):
"""Per-entity activity at scene start.
Maps onto Requirements §6.5: ``current_action.{verb,interruptible,
required_attention,expected_duration}`` plus posture, attention, holding.
``action_required_attention`` is left as a free-form string ("low" /
"medium" / "high" expected) rather than a Literal so the classifier has
room to vary phrasing in v1.
"""
posture: str
action_verb: str
action_interruptible: bool
action_required_attention: str # low | medium | high
action_expected_duration: str
attention: str = ""
holding: list[str] = Field(default_factory=list)
class KickoffParse(BaseModel):
"""Structured opening-scene state extracted from kickoff prose.
``container_properties`` is loose ``dict``: the classifier may emit
``moving`` / ``public`` / ``audible_range`` keys, but downstream
consumers (T13's confirm form) handle missing keys gracefully.
``initial_time_iso`` is stored as text — not validated as a datetime
here; ``chat_state.time`` stores it as a plain string.
"""
container_name: str
container_type: str
container_properties: dict
you_activity: ActivityShape
bot_activity: ActivityShape
initial_time_iso: str
edge_seed_summary: str
edge_seed_knowledge_facts: list[str]
_SYSTEM_PROMPT = (
"You are extracting structured scene state from a roleplay kickoff "
"scene description. The user provides bot context and a prose "
"description of the opening scene; you output JSON conforming to the "
"schema. Be concrete: pick a single container, single activity per "
"entity, and a sensible initial in-fiction time. Anything not stated "
"explicitly should be inferred reasonably from the prose."
)
def _build_user_prompt(
*,
bot_name: str,
bot_persona: str,
initial_relationship_to_you: str,
kickoff_prose: str,
you_name: str,
) -> str:
return (
f"BOT NAME: {bot_name}\n"
f"BOT PERSONA: {bot_persona}\n"
f"INITIAL RELATIONSHIP TO {you_name}: {initial_relationship_to_you}\n"
f"YOU NAME: {you_name}\n"
f"KICKOFF PROSE:\n{kickoff_prose}"
)
def _empty_activity() -> ActivityShape:
return ActivityShape(
posture="",
action_verb="",
action_interruptible=True,
action_required_attention="low",
action_expected_duration="brief",
)
def _empty_kickoff_parse() -> KickoffParse:
"""Default returned when the classifier can't produce a valid parse.
The user gets a mostly-empty confirm form they can fill in by hand
instead of a 500. ``initial_time_iso`` is left as the current UTC.
"""
from datetime import datetime, timezone
return KickoffParse(
container_name="",
container_type="",
container_properties={},
you_activity=_empty_activity(),
bot_activity=_empty_activity(),
initial_time_iso=datetime.now(timezone.utc).isoformat(timespec="seconds"),
edge_seed_summary="",
edge_seed_knowledge_facts=[],
)
async def parse_kickoff(
client: LLMClient,
*,
model: str,
bot_name: str,
bot_persona: str,
initial_relationship_to_you: str,
kickoff_prose: str,
you_name: str,
timeout_s: float = 10.0,
) -> KickoffParse:
"""Parse authored kickoff prose into a structured ``KickoffParse``.
Falls back to a mostly-empty default if the classifier fails — the
confirm-and-edit form is the human-in-the-loop, so a degraded form
that the user can fill in is preferable to a 500.
"""
user_prompt = _build_user_prompt(
bot_name=bot_name,
bot_persona=bot_persona,
initial_relationship_to_you=initial_relationship_to_you,
kickoff_prose=kickoff_prose,
you_name=you_name,
)
return await classify(
client,
model=model,
system=_SYSTEM_PROMPT,
user=user_prompt,
schema=KickoffParse,
default=_empty_kickoff_parse(),
timeout_s=timeout_s,
)