Joseph Doherty
5aab98e4d7
The kickoff parse-and-confirm route was 500-ing intermittently because
Hermes-3 + Featherless's response_format={"type":"json_object"} only
guarantees JSON output, NOT a particular schema. The model was inventing
its own field names (sceneTime, entities, settingDetails) instead of
the KickoffParse fields, causing Pydantic validation to fail on both
classify() retries.
Three changes:
1. Include the Pydantic JSON schema in the system prompt so the model
knows exactly which keys to produce. Affects every classify() call
(kickoff parse, turn parse, scene-close detect, significance,
state-update, scene summarize). Strip ```json fences if the model
wraps its output. Bump retries 2 → 3 (model is stochastic; one extra
attempt closes most of the remaining gap).
2. parse_kickoff() now passes a default empty KickoffParse so the
route degrades to a fillable form instead of 500 when the classifier
ultimately fails. The confirm form is the human-in-the-loop; an
empty form is strictly better UX than a stack trace.
3. Tests updated: bumped canned-failure arrays from 2 → 3 entries to
match the new attempt count; renamed kickoff test from
"raises_when_classifier_fails_twice" to
"falls_back_to_empty_when_classifier_fails" reflecting the new
degraded-but-usable behavior.
Verified live with all 3 sample bots (maya/eli/sam) — kickoff route
returns 200 across multiple attempts. Full suite: 168 passed.
84 lines
2.4 KiB
Python
84 lines
2.4 KiB
Python
import json
|
|
|
|
import pytest
|
|
|
|
from chat.llm.mock import MockLLMClient
|
|
from chat.services.turn_parse import (
|
|
ParsedTurn,
|
|
TurnSegment,
|
|
parse_turn,
|
|
)
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_parse_turn_three_segment_happy_path():
|
|
canned = json.dumps(
|
|
{
|
|
"segments": [
|
|
{"kind": "action", "text": "walks over"},
|
|
{"kind": "dialogue", "text": "Hey."},
|
|
{"kind": "ooc", "text": "player note"},
|
|
]
|
|
}
|
|
)
|
|
mock = MockLLMClient(canned=[canned])
|
|
result = await parse_turn(
|
|
mock,
|
|
model="m",
|
|
prose='*walks over* "Hey." ((player note))',
|
|
)
|
|
assert isinstance(result, ParsedTurn)
|
|
assert len(result.segments) == 3
|
|
kinds = [s.kind for s in result.segments]
|
|
assert kinds == ["action", "dialogue", "ooc"]
|
|
texts = [s.text for s in result.segments]
|
|
assert texts == ["walks over", "Hey.", "player note"]
|
|
assert all(isinstance(s, TurnSegment) for s in result.segments)
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_parse_turn_pure_dialogue_single_segment():
|
|
canned = json.dumps(
|
|
{
|
|
"segments": [
|
|
{"kind": "dialogue", "text": "Hello there"},
|
|
]
|
|
}
|
|
)
|
|
mock = MockLLMClient(canned=[canned])
|
|
result = await parse_turn(
|
|
mock,
|
|
model="m",
|
|
prose='"Hello there"',
|
|
)
|
|
assert isinstance(result, ParsedTurn)
|
|
assert len(result.segments) == 1
|
|
assert result.segments[0].kind == "dialogue"
|
|
assert result.segments[0].text == "Hello there"
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_parse_turn_empty_prose_short_circuits_without_classifier_call():
|
|
# No canned responses provided — if classify() is invoked it will raise
|
|
# IndexError on the empty list. The short-circuit must prevent that.
|
|
mock = MockLLMClient(canned=[])
|
|
result = await parse_turn(mock, model="m", prose="")
|
|
assert isinstance(result, ParsedTurn)
|
|
assert result.segments == []
|
|
|
|
# Whitespace-only prose must also short-circuit.
|
|
result_ws = await parse_turn(mock, model="m", prose=" \n\t ")
|
|
assert isinstance(result_ws, ParsedTurn)
|
|
assert result_ws.segments == []
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_parse_turn_raises_when_classifier_fails_twice():
|
|
mock = MockLLMClient(canned=["nope", "still nope", "nope3"])
|
|
with pytest.raises(RuntimeError):
|
|
await parse_turn(
|
|
mock,
|
|
model="m",
|
|
prose='*shrugs* "whatever"',
|
|
)
|