fix: classifier robustness — schema in prompt, retries, kickoff fallback

The kickoff parse-and-confirm route was 500-ing intermittently because
Hermes-3 + Featherless's response_format={"type":"json_object"} only
guarantees JSON output, NOT a particular schema. The model was inventing
its own field names (sceneTime, entities, settingDetails) instead of
the KickoffParse fields, causing Pydantic validation to fail on both
classify() retries.

Three changes:

1. Include the Pydantic JSON schema in the system prompt so the model
   knows exactly which keys to produce. Affects every classify() call
   (kickoff parse, turn parse, scene-close detect, significance,
   state-update, scene summarize). Strip ```json fences if the model
   wraps its output. Bump retries 2 → 3 (model is stochastic; one extra
   attempt closes most of the remaining gap).

2. parse_kickoff() now passes a default empty KickoffParse so the
   route degrades to a fillable form instead of 500 when the classifier
   ultimately fails. The confirm form is the human-in-the-loop; an
   empty form is strictly better UX than a stack trace.

3. Tests updated: bumped canned-failure arrays from 2 → 3 entries to
   match the new attempt count; renamed kickoff test from
   "raises_when_classifier_fails_twice" to
   "falls_back_to_empty_when_classifier_fails" reflecting the new
   degraded-but-usable behavior.

Verified live with all 3 sample bots (maya/eli/sam) — kickoff route
returns 200 across multiple attempts. Full suite: 168 passed.
This commit is contained in:
Joseph Doherty
2026-04-26 15:03:13 -04:00
parent 12502d6ec7
commit 5aab98e4d7
9 changed files with 89 additions and 28 deletions
+1 -1
View File
@@ -18,7 +18,7 @@ async def test_classify_parses_valid_json():
@pytest.mark.asyncio
async def test_classify_falls_back_on_unparseable_after_retry():
mock = MockLLMClient(canned=["nope", "still nope"])
mock = MockLLMClient(canned=["nope", "still nope", "nope3"])
default = Verdict(score=1, reason="fallback")
result = await classify(mock, model="m", system="x", user="y", schema=Verdict, default=default)
assert result.reason == "fallback"
+23 -12
View File
@@ -117,15 +117,26 @@ async def test_parse_kickoff_applies_activity_defaults_for_missing_fields():
@pytest.mark.asyncio
async def test_parse_kickoff_raises_when_classifier_fails_twice():
mock = MockLLMClient(canned=["nope", "still nope"])
with pytest.raises(RuntimeError):
await parse_kickoff(
mock,
model="m",
bot_name="BotA",
bot_persona="x",
initial_relationship_to_you="y",
kickoff_prose="z",
you_name="You",
)
async def test_parse_kickoff_falls_back_to_empty_when_classifier_fails():
"""When the classifier fails three times, return an empty KickoffParse
instead of raising — the confirm form lets the user fill in by hand.
"""
mock = MockLLMClient(canned=["nope", "still nope", "still bad"])
result = await parse_kickoff(
mock,
model="m",
bot_name="BotA",
bot_persona="x",
initial_relationship_to_you="y",
kickoff_prose="z",
you_name="You",
)
assert isinstance(result, KickoffParse)
assert result.container_name == ""
assert result.container_type == ""
assert result.edge_seed_summary == ""
assert result.edge_seed_knowledge_facts == []
# Activity defaults sane (action_interruptible defaults to True so the
# confirm form's checkbox is in a reasonable initial state).
assert result.you_activity.action_interruptible is True
assert result.bot_activity.action_interruptible is True
+1 -1
View File
@@ -84,7 +84,7 @@ async def test_summarize_scene_default_on_failure():
"""Two consecutive non-JSON returns trip the classifier's retry-then-default
path; we should get the empty fallback rather than crashing the close
flow."""
mock = MockLLMClient(canned=["bad", "still bad"])
mock = MockLLMClient(canned=["bad", "still bad", "bad3"])
result = await summarize_scene(
mock,
model="x",
+1 -1
View File
@@ -58,7 +58,7 @@ async def test_detect_scene_close_default_on_failure():
"""Two consecutive non-JSON returns trip the classifier's retry-then-default
path; we should get the safe ``should_close=False`` fallback rather than
crashing the turn flow."""
mock = MockLLMClient(canned=["nope", "still nope"])
mock = MockLLMClient(canned=["nope", "still nope", "nope3"])
decision = await detect_scene_close(
mock,
model="x",
+1 -1
View File
@@ -45,7 +45,7 @@ async def test_compute_significance_parses_score():
async def test_compute_significance_default_on_failure():
# Both attempts return non-JSON text; the classify wrapper falls back
# to the SignificanceVerdict default (score=1, "fallback").
mock = MockLLMClient(canned=["nope", "still nope"])
mock = MockLLMClient(canned=["nope", "still nope", "nope3"])
score = await compute_significance(
mock,
model="x",
+1 -1
View File
@@ -62,7 +62,7 @@ async def test_compute_state_update_parses_classifier_output():
@pytest.mark.asyncio
async def test_compute_state_update_returns_default_on_failure():
"""Two malformed classifier responses -> default StateUpdate (zeros)."""
mock = MockLLMClient(canned=["nope", "still nope"])
mock = MockLLMClient(canned=["nope", "still nope", "nope3"])
result = await compute_state_update(
mock,
model="x",
+1 -1
View File
@@ -74,7 +74,7 @@ async def test_parse_turn_empty_prose_short_circuits_without_classifier_call():
@pytest.mark.asyncio
async def test_parse_turn_raises_when_classifier_fails_twice():
mock = MockLLMClient(canned=["nope", "still nope"])
mock = MockLLMClient(canned=["nope", "still nope", "nope3"])
with pytest.raises(RuntimeError):
await parse_turn(
mock,