From d161e7b8e9af93fff1f8645bbcbeb8ebd85c45fd Mon Sep 17 00:00:00 2001
From: Joseph Doherty <dohejw01@gmail.com>
Date: Sun, 26 Apr 2026 15:28:08 -0400
Subject: [PATCH] feat: cap narrative response length + tune sampling
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Bot replies were running long (4 paragraphs of action+dialogue beats
per turn) because we never set max_tokens on the narrative call. Three
tunable knobs now in Settings (set in data/config.toml to override):

- narrative_max_tokens: int = 400
  Hard cap on each generated response. ~400 tokens ≈ 1–2 short
  paragraphs. Drop to 200 for terse banter, bump to 800+ for longer
  scenes.

- narrative_temperature: float = 0.85
  Sampling temperature. 0.7 = grounded/consistent (slightly stiff),
  0.85 = creative-but-in-character (default), 1.0 = wide variety,
  >1.0 = often off-the-rails.

- prompt closing instruction now nudges: "Keep your response to a
  single beat — one or two short paragraphs at most. Don't monologue;
  leave room for the other person to react."

Both turns.py (post_turn) and regenerate.py forward the params to
client.stream(). FeatherlessClient already passes **params through to
the OpenAI-compat endpoint.

Note: temperature doesn't control length — that was a common
misconception. max_tokens is the actual length cap. Lower temperature
makes word choice more predictable (slightly stiffer voice), not
shorter. Both knobs are useful for different goals.
---
 chat/config.py              | 7 +++++++
 chat/services/prompt.py     | 4 +++-
 chat/services/regenerate.py | 5 ++++-
 chat/web/turns.py           | 5 ++++-
 4 files changed, 18 insertions(+), 3 deletions(-)

diff --git a/chat/config.py b/chat/config.py
index 935cb21..8eb19b6 100644
--- a/chat/config.py
+++ b/chat/config.py
@@ -23,6 +23,13 @@ class Settings(BaseModel):
     retrieval_k: int = 4
     narrative_budget_hard: int = 8000
     narrative_budget_soft: int = 6000
+    # Cap on each generated bot response. ~400 tokens ≈ 1–2 short paragraphs.
+    # Bump if you want longer scenes; drop to 200 for terse banter.
+    narrative_max_tokens: int = 400
+    # Sampling temperature for narrative generation. 0.7 = grounded /
+    # consistent; 0.85 = creative-but-in-character (default); 1.0 = wide
+    # variety, can drift; >1.0 = often off-the-rails.
+    narrative_temperature: float = 0.85
     classifier_budget_hard: int = 4000
     classifier_timeout_s: float = 30.0
     # Featherless free tier and lower paid tiers cap concurrent connections.
diff --git a/chat/services/prompt.py b/chat/services/prompt.py
index 89b416e..1df074b 100644
--- a/chat/services/prompt.py
+++ b/chat/services/prompt.py
@@ -211,7 +211,9 @@ def _closing_instruction(speaker_name: str, addressee_name: str) -> str:
         f"Continue the scene as {speaker_name}, in their voice, responding "
         "naturally. Use *asterisks* for actions and quotes for dialogue. "
         f"Stay in character. Do not narrate {addressee_name}'s actions or "
-        "thoughts."
+        "thoughts. "
+        "Keep your response to a single beat — one or two short paragraphs "
+        "at most. Don't monologue; leave room for the other person to react."
     )
 
 
diff --git a/chat/services/regenerate.py b/chat/services/regenerate.py
index d1e983f..c92c0f0 100644
--- a/chat/services/regenerate.py
+++ b/chat/services/regenerate.py
@@ -156,7 +156,10 @@ async def regenerate_assistant_turn(
     # 5. Stream the new narrative.
     accumulated: list[str] = []
     async for chunk in client.stream(
-        messages, model=settings.narrative_model
+        messages,
+        model=settings.narrative_model,
+        max_tokens=settings.narrative_max_tokens,
+        temperature=settings.narrative_temperature,
     ):
         accumulated.append(chunk)
         await publish(
diff --git a/chat/web/turns.py b/chat/web/turns.py
index 260bb21..894dc72 100644
--- a/chat/web/turns.py
+++ b/chat/web/turns.py
@@ -198,7 +198,10 @@ async def post_turn(
 
     async def _stream() -> None:
         async for chunk in client.stream(
-            messages, model=settings.narrative_model
+            messages,
+            model=settings.narrative_model,
+            max_tokens=settings.narrative_max_tokens,
+            temperature=settings.narrative_temperature,
         ):
             accumulated.append(chunk)
             await publish(