Add evaluation harness for memory/context-management experiments

Compare memory presets (full_history, prod, summarization/editing-only,
aggressive, profile_memory) on quality, memory retention, retrieval,
tokens, cost, and latency:

- app: memory presets (app/memory_presets.py), per-turn telemetry
independent of LangSmith (app/telemetry.py, context_stats event +
data-context-stats SSE part), long-term student-profile memory
(StudentProfileMiddleware + store), memoryPreset/studentId API fields
- evals/: battery runner (resume-safe, per-turn timeout, trace bundles),
offline grader (code checks + blinded hand-grading workbook), report
(side-by-side preset tables + token curves), trigger-verification gate
- docs: evals.md (datasets, metrics, Part B results, findings F1-F9),
evals_background.md (research notes), data/eval/README.md (schemas)
- data/scraping_scripts/annotate_discussion_posts.py: discussion-post
annotation used to build the eval batteries

Eval datasets and run results contain real student text and ship via the
private HF dataset (towardsai-tutors/ai-tutor-data), never git; gitignore
now covers review_batches/ and review_log explicitly.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>

.gitignore

@@ -206,3 +206,9 @@ frontend/out/
 !frontend/app/apple-icon.png
 
 .claude/
+runs/
+# Eval data with real student text: ships via the private HF dataset
+# (towardsai-tutors/ai-tutor-data), never via git — main gets force-pushed
+# to the public prod Space on deploys. (*.jsonl is already ignored globally.)
+data/eval/review_batches/
+data/eval/review_log_v1.md

app/api.py

@@ -25,6 +25,7 @@ from .chat_service import (
 )
 from .chat_types import ChatEvent, ChatRequest, ChatTurn
 from .kb_manifest import load_manifest_entries
+from .memory_presets import MEMORY_PRESETS
 from .config import (
     AVAILABLE_MODELS,
     AVAILABLE_SOURCES,
@@ -71,6 +72,11 @@ class ApiChatRequest(BaseModel):
     model: str | None = Field(default=None, max_length=200)
     includeReasoning: bool = True
     threadId: str = Field(default="", max_length=128)
+    # Memory/context-management preset (experiments + workshop toggles).
+    # Omitted means the server-side default resolution order.
+    memoryPreset: str | None = Field(default=None, max_length=64)
+    # Long-term memory key for profile-memory presets.
+    studentId: str = Field(default="", max_length=128)
 
     @field_validator("messages")
     @classmethod
@@ -233,6 +239,9 @@ def build_chat_request(payload: ApiChatRequest) -> ChatRequest:
     model_name = (payload.model or DEFAULT_MODEL_NAME).strip() or DEFAULT_MODEL_NAME
     if model_name not in {model["id"] for model in AVAILABLE_MODELS}:
         raise HTTPException(status_code=422, detail="Unknown model")
+    memory_preset = (payload.memoryPreset or "").strip()
+    if memory_preset and memory_preset not in MEMORY_PRESETS:
+        raise HTTPException(status_code=422, detail="Unknown memory preset")
     if payload.enabledTools is None:
         enabled_tools = tuple(
             tool["key"]
@@ -258,6 +267,8 @@ def build_chat_request(payload: ApiChatRequest) -> ChatRequest:
         include_reasoning=bool(payload.includeReasoning),
         thread_id=(payload.threadId or "").strip(),
         enabled_tools=enabled_tools,
+        memory_preset=memory_preset,
+        student_id=payload.studentId.strip(),
     )
 
 
@@ -472,6 +483,37 @@ class UIMessageStreamEncoder:
             )
             return parts
 
+        if event.type == "context_stats":
+            data = event.data
+            parts.append(
+                {
+                    "type": "data-context-stats",
+                    "data": {
+                        "messageId": str(data.get("message_id", "")),
+                        "memoryPreset": str(data.get("memory_preset", "")),
+                        "llmCalls": data.get("llm_calls"),
+                        "inputTokens": data.get("input_tokens"),
+                        "outputTokens": data.get("output_tokens"),
+                        "totalTokens": data.get("total_tokens"),
+                        "cacheReadTokens": data.get("cache_read_tokens"),
+                        "cacheCreationTokens": data.get("cache_creation_tokens"),
+                        # None when a used model has no price-table entry;
+                        # the client must render that as unknown, not $0.
+                        "estCostUsd": data.get("est_cost_usd"),
+                        "ttftMs": data.get("ttft_ms"),
+                        "totalMs": data.get("total_ms"),
+                        "contextMessages": data.get("context_messages"),
+                        "contextTokensApprox": data.get("context_tokens_approx"),
+                        "summaryMessages": data.get("summary_messages"),
+                        "clearedToolOutputs": data.get("cleared_tool_outputs"),
+                    },
+                    # Delivered to onData for a live meter; not stored in
+                    # message.parts (nothing renders it as message content).
+                    "transient": True,
+                }
+            )
+            return parts
+
         if event.type == "message_completed":
             parts.extend(self.close_reasoning_block())
             answer = str(event.data.get("answer", "")).strip()

app/chat_service.py

@@ -30,8 +30,21 @@ from langchain_core.messages import (
 )
 from langchain_openai import ChatOpenAI
 from langgraph.checkpoint.memory import InMemorySaver
+from langgraph.store.memory import InMemoryStore
 
 from .chat_types import ChatEvent, ChatRequest, ChatTurn, SourceMatch
+from .memory_presets import (
+    DEFAULT_MEMORY_PRESET,
+    MEMORY_PRESETS,
+    MemoryConfig,
+    resolve_memory_preset,
+)
+from .telemetry import (
+    TurnUsageHandler,
+    context_window_stats,
+    estimate_cost_usd,
+    usage_totals,
+)
 from .chroma_rag import LocalChromaRetriever, format_tool_payload, parse_tool_payload
 from .kb_shell import (
     KbCommandError,
@@ -66,6 +79,12 @@ from .config import (
 logger = logging.getLogger(__name__)
 
 CHECKPOINTER = InMemorySaver()
+# Long-term memory (student profiles), keyed by namespace ("student", <id>).
+# In-process like the checkpointer: profiles survive across threads/sessions
+# within one server lifetime, which is what the profile-memory experiments
+# need. Swap for a persistent LangGraph store to survive restarts.
+STORE = InMemoryStore()
+CLEARED_TOOL_OUTPUT_PLACEHOLDER = "[tool output cleared to save context]"
 # Visible transcript per thread, recorded as each turn finishes streaming.
 # The checkpointed messages stop mirroring the visible transcript once
 # SummarizationMiddleware rewrites thread state, so thread reuse is decided
@@ -98,6 +117,24 @@ class AppContext:
     allowed_sources: tuple[str, ...]
     kb_session_id: str = ""
     kb_command_limit: int = DEFAULT_KB_COMMAND_LIMIT
+    student_id: str = ""
+
+
+def get_student_profile(student_id: str) -> str:
+    if not student_id:
+        return ""
+    item = STORE.get(("student", student_id), "profile")
+    if item is None:
+        return ""
+    return str((item.value or {}).get("profile", "")).strip()
+
+
+def set_student_profile(student_id: str, profile: str) -> None:
+    """Write a student profile directly; eval runners use this to seed
+    personas without spending a turn on the on-the-fly write path."""
+    if not student_id:
+        return
+    STORE.put(("student", student_id), "profile", {"profile": profile.strip()})
 
 
 def _claim_kb_command_budget(session_id: str, limit: int) -> tuple[bool, int]:
@@ -812,6 +849,82 @@ class SourcePreferenceMiddleware(AgentMiddleware):
         return await handler(self._inject(request))
 
 
+class StudentProfileMiddleware(AgentMiddleware):
+    """Append the stored student profile to the system prompt.
+
+    Long-term semantic memory: the profile lives in the LangGraph store under
+    ``("student", <student_id>)`` and is updated after each turn by
+    ``stream_chat`` (see ``_update_student_profile``). InMemoryStore reads are
+    dict lookups, so the sync ``store.get`` is fine on the async path too.
+    """
+
+    def _inject(self, request):
+        runtime = getattr(request, "runtime", None)
+        ctx = getattr(runtime, "context", None) if runtime else None
+        student_id = getattr(ctx, "student_id", "") if ctx else ""
+        if not student_id:
+            return request
+        store = getattr(runtime, "store", None) or STORE
+        item = store.get(("student", student_id), "profile")
+        profile = ""
+        if item is not None:
+            profile = str((item.value or {}).get("profile", "")).strip()
+        if not profile:
+            return request
+        note = (
+            "## Student profile (long-term memory)\n\n"
+            f"{profile}\n\n"
+            "Use this profile to calibrate level, language, and examples. "
+            "Do not re-ask for information it already answers."
+        )
+        sys_msg = request.system_message
+        if sys_msg is None:
+            new_sys = SystemMessage(content=note)
+        else:
+            new_sys = SystemMessage(content=f"{sys_msg.content}\n\n{note}")
+        return request.override(system_message=new_sys)
+
+    def wrap_model_call(self, request, handler):
+        return handler(self._inject(request))
+
+    async def awrap_model_call(self, request, handler):
+        return await handler(self._inject(request))
+
+
+def build_agent_middleware(
+    model: Any, memory_config: MemoryConfig
+) -> list[AgentMiddleware]:
+    """Assemble the compaction/memory middleware stack for one preset."""
+    middleware: list[AgentMiddleware] = []
+    if memory_config.context_editing:
+        middleware.append(
+            ContextEditingMiddleware(
+                edits=[
+                    ClearToolUsesEdit(
+                        trigger=memory_config.context_editing_trigger_tokens,
+                        keep=memory_config.context_editing_keep,
+                        # Retrieval results stay; only shell outputs get cleared.
+                        exclude_tools=("retrieve_tutor_context",),
+                        placeholder=CLEARED_TOOL_OUTPUT_PLACEHOLDER,
+                    )
+                ],
+                token_count_method="approximate",
+            )
+        )
+    if memory_config.summarization:
+        middleware.append(
+            SummarizationMiddleware(
+                model=model,
+                trigger=("tokens", memory_config.summarization_trigger_tokens),
+                keep=("messages", memory_config.summarization_keep_messages),
+            )
+        )
+    if memory_config.longterm_memory:
+        middleware.append(StudentProfileMiddleware())
+    middleware.append(SourcePreferenceMiddleware())
+    return middleware
+
+
 @lru_cache(maxsize=32)
 def build_agent(
     model_name: str,
@@ -819,36 +932,19 @@ def build_agent(
     include_thoughts: bool = False,
     kb_agents_instructions: str | None = None,
     include_local_tools: bool = True,
+    memory_config: MemoryConfig = MEMORY_PRESETS[DEFAULT_MEMORY_PRESET],
 ):
     # kb_agents_instructions is part of the cache key on purpose: an agent
     # built before data/kb/AGENTS.md existed must not pin its degraded
-    # system prompt for the process lifetime.
+    # system prompt for the process lifetime. memory_config (frozen, hashable)
+    # is too: each preset gets its own agent.
     model = build_chat_model(model_name, include_thoughts=include_thoughts)
     # An explicit empty source selection turns the knowledge base off: no
     # retrieval, no KB browsing, and a system prompt that says so.
     tools: list[Any] = (
         [retrieve_tutor_context, run_kb_command] if include_local_tools else []
     )
-    middleware: list[AgentMiddleware] = [
-        ContextEditingMiddleware(
-            edits=[
-                ClearToolUsesEdit(
-                    trigger=5_000,
-                    keep=5,
-                    # Retrieval results stay; only shell outputs get cleared.
-                    exclude_tools=("retrieve_tutor_context",),
-                    placeholder="[tool output cleared to save context]",
-                )
-            ],
-            token_count_method="approximate",
-        ),
-        SummarizationMiddleware(
-            model=model,
-            trigger=("tokens", 30_000),
-            keep=("messages", 20),
-        ),
-        SourcePreferenceMiddleware(),
-    ]
+    middleware = build_agent_middleware(model, memory_config)
     enabled = set(enabled_tools)
     if is_google_genai_model(model_name):
         if "web_search" in enabled:
@@ -885,6 +981,7 @@ def build_agent(
         ),
         context_schema=AppContext,
         checkpointer=CHECKPOINTER,
+        store=STORE,
         middleware=middleware,
     )
 
@@ -954,6 +1051,7 @@ def agent_run_config(
     request: ChatRequest,
     active_thread_id: str,
     message_id: str,
+    memory_preset: str = "",
 ) -> dict[str, Any]:
     provider, actual_model = model_provider_and_name(request.model_name)
     tools = effective_tool_names(
@@ -965,6 +1063,7 @@ def agent_run_config(
         SOURCE_KEY_TO_LABEL.get(source_key, source_key)
         for source_key in request.source_keys
     ]
+    preset = memory_preset or DEFAULT_MEMORY_PRESET
     config = thread_config(active_thread_id)
     config.update(
         {
@@ -974,6 +1073,7 @@ def agent_run_config(
                 "knowledge-base-chatbot",
                 f"provider:{provider}",
                 f"model:{actual_model}",
+                f"memory:{preset}",
                 *(f"tool:{tool_name}" for tool_name in tools),
             ],
             "metadata": {
@@ -989,13 +1089,68 @@ def agent_run_config(
                 "source_keys": list(request.source_keys),
                 "source_labels": source_labels,
                 "include_reasoning": bool(request.include_reasoning),
+                "memory_preset": preset,
+                "student_id": request.student_id,
             },
         }
     )
     return config
 
 
+_PROFILE_UPDATE_PROMPT = """\
+You maintain a short profile of a student for an AI tutor.
+
+Current profile (may be empty):
+{profile}
+
+Latest exchange:
+Student: {query}
+Tutor: {answer}
+
+Rewrite the profile in at most 5 short lines. Keep only durable facts useful
+for future tutoring: skill level, goals, preferred language/tools, weak
+topics, current course/lesson. Drop one-off details. Return only the profile
+text; return NONE if nothing durable is known yet."""
+
+
+async def _update_student_profile(
+    request: ChatRequest, answer: str, usage_handler: TurnUsageHandler
+) -> None:
+    """Refresh the stored profile from this turn (one small extra LLM call).
+
+    The call reports into ``usage_handler`` so profile-memory presets pay for
+    their own upkeep in the turn's token/cost numbers.
+    """
+    model = build_chat_model(request.model_name)
+    prompt = _PROFILE_UPDATE_PROMPT.format(
+        profile=get_student_profile(request.student_id) or "(empty)",
+        query=request.query[:1500],
+        answer=answer[:3000],
+    )
+    response = await model.ainvoke(
+        [HumanMessage(content=prompt)],
+        config={
+            "run_name": "student-profile-update",
+            "callbacks": [usage_handler],
+            "metadata": {"lc_source": "student_profile_update"},
+        },
+    )
+    text = message_content_to_text(response.content).strip()
+    if not text or text.upper() == "NONE":
+        return
+    set_student_profile(request.student_id, text[:1200])
+
+
 async def stream_chat(request: ChatRequest) -> AsyncIterator[ChatEvent]:
+    turn_started = time.monotonic()
+    # Raises on unknown preset names: a mislabeled experiment run must fail,
+    # not silently fall back to prod. The API layer pre-validates to a 422.
+    memory_config = resolve_memory_preset(request.memory_preset)
+    # Token usage and call counts come from the model calls themselves
+    # (middleware summarization and profile updates included), so eval runs
+    # never depend on LangSmith being enabled or within plan limits.
+    usage_handler = TurnUsageHandler()
+    first_text_at: float | None = None
     normalized_history = normalize_history(request.history)
     retrieval_evidence: dict[str, SourceMatch] = {}
     shell_evidence: dict[str, SourceMatch] = {}
@@ -1036,6 +1191,7 @@ async def stream_chat(request: ChatRequest) -> AsyncIterator[ChatEvent]:
                 ensure_kb_agents_instructions() if include_local_tools else None
             ),
             include_local_tools=include_local_tools,
+            memory_config=memory_config,
         )
 
     agent = await asyncio.to_thread(_build_agent_for_request)
@@ -1047,7 +1203,10 @@ async def stream_chat(request: ChatRequest) -> AsyncIterator[ChatEvent]:
         normalized_history,
     )
     _touch_thread(active_thread_id)
-    run_config = agent_run_config(request, active_thread_id, message_id)
+    run_config = agent_run_config(
+        request, active_thread_id, message_id, memory_config.name
+    )
+    run_config["callbacks"] = [usage_handler]
     if fork_checkpoint_id:
         # Time travel: run from the checkpoint matching the history the
         # client kept; the turns after it become an abandoned branch.
@@ -1064,6 +1223,7 @@ async def stream_chat(request: ChatRequest) -> AsyncIterator[ChatEvent]:
                 allowed_sources=request.source_keys,
                 kb_session_id=message_id,
                 kb_command_limit=DEFAULT_KB_COMMAND_LIMIT,
+                student_id=request.student_id,
             ),
             stream_mode=["messages", "updates"],
             version="v2",
@@ -1116,6 +1276,8 @@ async def stream_chat(request: ChatRequest) -> AsyncIterator[ChatEvent]:
                 if not text_delta and token.content:
                     text_delta = message_content_to_text(token.content)
                 if text_delta:
+                    if first_text_at is None:
+                        first_text_at = time.monotonic()
                     answer_chunks.append(text_delta)
                     yield ChatEvent(
                         "text_delta",
@@ -1324,6 +1486,46 @@ async def stream_chat(request: ChatRequest) -> AsyncIterator[ChatEvent]:
             "source_match",
             source_match_payload(source_match, message_id=message_id),
         )
+
+    # Long-term memory upkeep happens before the stats event so the profile
+    # update's tokens land in this turn's bill.
+    if memory_config.longterm_memory and request.student_id and answer:
+        try:
+            await _update_student_profile(request, answer, usage_handler)
+        except Exception as exc:
+            logger.warning("Student profile update failed. error=%s", exc)
+
+    # Aborted streams never reach this event; eval runners consume the full
+    # stream, and the UI treats the meter as best-effort.
+    state_messages = agent.get_state(thread_config(active_thread_id)).values.get(
+        "messages", []
+    )
+    totals = usage_totals(usage_handler.usage_metadata)
+    total_ms = int((time.monotonic() - turn_started) * 1000)
+    yield ChatEvent(
+        "context_stats",
+        {
+            "message_id": message_id,
+            "thread_id": active_thread_id,
+            "memory_preset": memory_config.name,
+            "llm_calls": usage_handler.llm_calls,
+            # Per-model breakdown for trace bundles; the totals below are
+            # what the UI meter renders.
+            "usage_by_model": {
+                model_key: dict(usage)
+                for model_key, usage in usage_handler.usage_metadata.items()
+            },
+            **totals,
+            "est_cost_usd": estimate_cost_usd(usage_handler.usage_metadata),
+            "ttft_ms": (
+                int((first_text_at - turn_started) * 1000)
+                if first_text_at is not None
+                else None
+            ),
+            "total_ms": total_ms,
+            **context_window_stats(state_messages, CLEARED_TOOL_OUTPUT_PLACEHOLDER),
+        },
+    )
     yield ChatEvent(
         "message_completed",
         {

app/chat_types.py

@@ -33,6 +33,12 @@ class ChatRequest:
     include_reasoning: bool = False
     thread_id: str = ""
     enabled_tools: tuple[str, ...] = ()
+    # Memory/context-management preset name (see app/memory_presets.py).
+    # Empty means the env-var/default resolution order.
+    memory_preset: str = ""
+    # Long-term memory key: profile-memory presets read and update the stored
+    # student profile under this id. Empty disables profile memory I/O.
+    student_id: str = ""
 
 
 @dataclass(frozen=True, slots=True)

app/memory_presets.py

@@ -0,0 +1,69 @@
+"""Memory/context-management presets.
+
+A preset bundles every knob that changes how the agent manages conversation
+context: the compaction middlewares (summarization, tool-output clearing) and
+long-term student-profile memory. ``build_agent()`` assembles its middleware
+stack from the active preset, so experiment runs can compare configurations by
+name while the API default stays on ``prod``.
+
+Selection order: explicit request value > ``AI_TUTOR_MEMORY_PRESET`` env var >
+``DEFAULT_MEMORY_PRESET``. Unknown names raise instead of falling back; a
+mislabeled experiment run is worse than a failed one.
+"""
+
+from __future__ import annotations
+
+import os
+from dataclasses import dataclass
+
+
+@dataclass(frozen=True, slots=True)
+class MemoryConfig:
+    name: str
+    summarization: bool = True
+    summarization_trigger_tokens: int = 30_000
+    summarization_keep_messages: int = 20
+    context_editing: bool = True
+    context_editing_trigger_tokens: int = 5_000
+    context_editing_keep: int = 5
+    longterm_memory: bool = False
+
+
+MEMORY_PRESETS: dict[str, MemoryConfig] = {
+    # No compaction at all: the quality/memory upper bound and the
+    # token-cost worst case.
+    "full_history": MemoryConfig(
+        name="full_history", summarization=False, context_editing=False
+    ),
+    # What production runs today.
+    "prod": MemoryConfig(name="prod"),
+    "summarization_only": MemoryConfig(
+        name="summarization_only", context_editing=False
+    ),
+    "editing_only": MemoryConfig(name="editing_only", summarization=False),
+    # How bad can cheap get: compaction fires early and keeps little.
+    "aggressive": MemoryConfig(
+        name="aggressive",
+        summarization_trigger_tokens=8_000,
+        summarization_keep_messages=8,
+        context_editing_trigger_tokens=2_000,
+        context_editing_keep=2,
+    ),
+    # prod compaction + long-term semantic memory (student profile store).
+    "profile_memory": MemoryConfig(name="profile_memory", longterm_memory=True),
+}
+
+DEFAULT_MEMORY_PRESET = "prod"
+
+
+def resolve_memory_preset(name: str | None = None) -> MemoryConfig:
+    requested = (
+        (name or "").strip()
+        or os.environ.get("AI_TUTOR_MEMORY_PRESET", "").strip()
+        or DEFAULT_MEMORY_PRESET
+    )
+    config = MEMORY_PRESETS.get(requested)
+    if config is None:
+        known = ", ".join(sorted(MEMORY_PRESETS))
+        raise ValueError(f"Unknown memory preset {requested!r}. Known presets: {known}")
+    return config

app/telemetry.py

@@ -0,0 +1,148 @@
+"""Per-turn token/cost/latency telemetry, independent of LangSmith.
+
+Token counts come from the ``usage_metadata`` every chat-model call reports,
+collected by a callback handler attached to the agent run. That includes
+SummarizationMiddleware's internal summary calls and the student-profile
+update call, so a preset's overhead is part of its own bill. Nothing here
+talks to LangSmith: eval runs can stream thousands of turns with tracing
+disabled and still get complete numbers.
+
+Costs are estimates from the local price table below. Raw token counts are
+always emitted alongside, so costs can be recomputed offline whenever prices
+change; an unknown model yields ``est_cost_usd=None`` rather than a wrong
+number.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+from typing import Any
+
+from langchain_core.callbacks.usage import UsageMetadataCallbackHandler
+from langchain_core.messages import BaseMessage
+from langchain_core.messages.utils import count_tokens_approximately
+from langchain_core.outputs import LLMResult
+
+
+class TurnUsageHandler(UsageMetadataCallbackHandler):
+    """Aggregate per-model usage for one turn and count chat-model calls."""
+
+    def __init__(self) -> None:
+        super().__init__()
+        self.llm_calls = 0
+
+    def on_llm_end(self, response: LLMResult, **kwargs: Any) -> None:
+        with self._lock:
+            self.llm_calls += 1
+        super().on_llm_end(response, **kwargs)
+
+
+@dataclass(frozen=True, slots=True)
+class ModelPricing:
+    """USD per million tokens. ``cache_write`` of None bills cache-creation
+    tokens at the plain input rate (providers without a write surcharge)."""
+
+    input: float
+    output: float
+    cache_read: float
+    cache_write: float | None = None
+
+
+# Verify against the provider price sheets before quoting cost numbers
+# anywhere public; prices move. Keys match by longest prefix so dated
+# variants ("claude-haiku-4-5-20251001") hit their family entry.
+MODEL_PRICING: dict[str, ModelPricing] = {
+    # TODO verify: seeded from the gemini-2.5-flash sheet.
+    "gemini-3.5-flash": ModelPricing(input=0.30, output=2.50, cache_read=0.075),
+    "claude-haiku-4-5": ModelPricing(
+        input=1.00, output=5.00, cache_read=0.10, cache_write=1.25
+    ),
+}
+
+
+def pricing_for_model(model_key: str) -> ModelPricing | None:
+    best: tuple[int, ModelPricing] | None = None
+    for prefix, pricing in MODEL_PRICING.items():
+        if model_key.startswith(prefix) and (best is None or len(prefix) > best[0]):
+            best = (len(prefix), pricing)
+    return best[1] if best else None
+
+
+def usage_totals(usage_by_model: dict[str, Any]) -> dict[str, int]:
+    """Sum usage across models into the fields the stats event reports."""
+    totals = {
+        "input_tokens": 0,
+        "output_tokens": 0,
+        "total_tokens": 0,
+        "cache_read_tokens": 0,
+        "cache_creation_tokens": 0,
+    }
+    for usage in usage_by_model.values():
+        totals["input_tokens"] += int(usage.get("input_tokens", 0) or 0)
+        totals["output_tokens"] += int(usage.get("output_tokens", 0) or 0)
+        totals["total_tokens"] += int(usage.get("total_tokens", 0) or 0)
+        details = usage.get("input_token_details") or {}
+        totals["cache_read_tokens"] += int(details.get("cache_read", 0) or 0)
+        totals["cache_creation_tokens"] += int(details.get("cache_creation", 0) or 0)
+    return totals
+
+
+def estimate_cost_usd(usage_by_model: dict[str, Any]) -> float | None:
+    """Estimated turn cost, or None when any used model has no price entry.
+
+    Cached input tokens are billed at the cache-read rate; ``input_tokens``
+    includes them (LangChain's UsageMetadata convention), so they are carved
+    out of the plain-input bucket rather than added on top.
+    """
+    total = 0.0
+    for model_key, usage in usage_by_model.items():
+        pricing = pricing_for_model(model_key)
+        if pricing is None:
+            return None
+        input_tokens = int(usage.get("input_tokens", 0) or 0)
+        output_tokens = int(usage.get("output_tokens", 0) or 0)
+        details = usage.get("input_token_details") or {}
+        cache_read = int(details.get("cache_read", 0) or 0)
+        cache_creation = int(details.get("cache_creation", 0) or 0)
+        plain_input = max(0, input_tokens - cache_read - cache_creation)
+        write_rate = (
+            pricing.cache_write if pricing.cache_write is not None else pricing.input
+        )
+        total += (
+            plain_input * pricing.input
+            + cache_read * pricing.cache_read
+            + cache_creation * write_rate
+            + output_tokens * pricing.output
+        ) / 1_000_000
+    return total
+
+
+def context_window_stats(
+    messages: list[BaseMessage], cleared_placeholder: str
+) -> dict[str, int]:
+    """Describe the checkpointed context after a turn.
+
+    ``summary_messages`` counts SummarizationMiddleware's summary insertions
+    (tagged ``lc_source: summarization``); ``cleared_tool_outputs`` counts
+    tool results ContextEditingMiddleware replaced with the placeholder. A
+    runner diffs these across turns to verify compaction actually fired.
+    """
+    summary_messages = 0
+    cleared_tool_outputs = 0
+    for message in messages:
+        additional = getattr(message, "additional_kwargs", None) or {}
+        if additional.get("lc_source") == "summarization":
+            summary_messages += 1
+        if getattr(message, "type", "") == "tool":
+            content = message.content
+            text = content if isinstance(content, str) else str(content)
+            if text.startswith(cleared_placeholder):
+                cleared_tool_outputs += 1
+    return {
+        "context_messages": len(messages),
+        "context_tokens_approx": (
+            int(count_tokens_approximately(messages)) if messages else 0
+        ),
+        "summary_messages": summary_messages,
+        "cleared_tool_outputs": cleared_tool_outputs,
+    }

data/eval/README.md

@@ -0,0 +1,122 @@
+# Evaluation batteries (v1, frozen 2026-06-11)
+
+The datasets for evaluating the AI tutor (see `evals.md` at the repo root for the overall effort and results). **This README is the single reference for what every file, field, and term means.**
+
+Built from `data/academy_discussion_eval.jsonl` — real student posts from the academy discussion boards with real staff answers — plus authored content (sessions, personas).
+
+## Ground rules
+
+- **Never edit a battery in place.** To change cases: edit the verdict/source files, rerun assembly, write `_v2`. Results are only comparable within one battery version.
+- **All `*.jsonl` here is gitignored** (repo-wide rule) and contains real student text. Treat like the private HF dataset; never publish or paste into slides.
+- **Inputs only.** No file contains LLM-generated answers to grade against. Ground truth is either real staff replies (distilled into key points) or facts we authored ourselves (sessions, personas).
+
+## What each battery is for
+
+| File | Tests | Main metrics it feeds | Graded by |
+|---|---|---|---|
+| `battery_singleturn_v1.jsonl` | One-shot answer quality and behavior routing. Memory presets should score ~equal here (sanity tier); differences indicate bugs, not memory effects. | key-point coverage · behavior accuracy · retrieval recall@k · citation validity · all ops metrics (tokens/cost/latency) | code + judge/hand-grade |
+| `battery_sessions_v1.jsonl` | **Where memory presets separate.** Long sessions inflate context past compression triggers, then probes test what survived. | memory probe accuracy (per probe type) · cumulative tokens per session · trigger firings · late-turn retrieval recall | code (string checks) + judge for nuanced probes |
+| `battery_personas_v1.jsonl` | The `profile_memory` preset specifically: does a stored student profile actually shape answers across sessions? | personalization pass rate | mostly code (regex), few judge checks |
+| `replay_n1_v1.jsonl` | Multi-turn answering on *real* conversation prefixes (no synthetic anything). Secondary battery. | reply quality vs the real staff reply | judge or hand-grade |
+
+"Metrics" here map to the metric layers defined in `evals.md` (ops, trajectory, retrieval, behavior, quality, memory).
+
+## Glossary
+
+- **Case / session / probe**: one gradeable unit. A *case* is a single question (single-turn, persona, replay). A *session* is an ordered multi-turn conversation run on one `thread_id`. A *probe* is a turn inside a session that gets graded (other turns are ungraded context).
+- **eval_quality — `gold` / `usable` / `weak` / `exclude`**: reviewer confidence that the case is a fair, gradeable test. `gold` = clear question, verifiable expectation, safe to treat a failure as the system's fault. `usable` = gradeable but with a caveat (thin reference answer, fragile specifics) — failures deserve a second look before blaming the system. `weak`/`exclude` never enter a battery. "60 cases: gold 49 / usable 11" means 49 high-confidence + 11 caveated cases.
+- **key_points**: 1–5 atomic, binary-checkable claims that a good answer SHOULD contain, distilled from the real staff answer (only claims still true today). Each is graded independently pass/fail ("does the answer correctly convey this point?") → the **key-point coverage** metric (fraction of points passed). This decomposition is what makes grading reliable: small binary checks instead of a holistic 1–5 score. Empty list is intentional for redirect/feedback cases — those are graded on behavior, not content.
+- **expected_behavior**: which of the tutor's four correct moves this case demands. `answer_from_corpus` = retrieve and ground the answer in course/docs content. `answer_general` = legitimate AI/programming question outside the corpus; answer from general knowledge without fake citations. `redirect_to_support` = platform/billing/submission issue; empathize and point to human support, never invent platform answers. `acknowledge_feedback` = course feedback; thank and acknowledge, don't promise fixes. Graded as **behavior accuracy** (did the tutor do the right *kind* of thing), separate from answer content.
+- **standalone question** (`question` field): the student's post rewritten so it works without seeing the discussion page — names the course/lesson, keeps code and error text, strips student names. This is the text actually sent to the tutor.
+- **reference_answer / reference_reply / reference_links**: the real staff reply. **Grader context only — never shown to the tutor.** Key points are derived from it; graders read it to resolve ambiguity.
+- **time_bound**: the staff answer depended on a point in time (broken link since fixed, "we'll add this soon", old library version). Time-bound cases were excluded unless durable behavior can still be graded — the 9 included `time_bound: true` cases are all platform issues where the *redirect behavior* is what's graded, not the stale facts.
+- **requires_notebook**: fully answering needs the Colab notebook contents (cells/outputs/files), which our KB does not contain — verified against lesson markdown with file/line evidence (`notebook_evidence`), not guessed. All 60 included cases are `false`; the one true case in review was excluded. If retrieval misses on a flagged-adjacent case, check `notebook_evidence` before blaming the retriever.
+- **requires_media**: the post referenced a screenshot we don't have. The 2 included cases remain gradeable without it.
+- **planted facts**: personal facts (environment, goal, level, weak topic, preference/constraint) a session's student states in turn 0, which later probes test. The session-level analog of a profile.
+- **Probe types** (`probe_type`): `fact_recall` — answer must use a planted fact from many turns ago. `preference_compliance` — answer must honor a stated preference (OS, tooling, language, style). `anaphora` — question refers to "the X you explained earlier"; unanswerable without resolving it to an earlier turn. `anaphora_consistency` — answer must stay consistent with the tutor's own earlier recommendation. `fact_update` — the student CHANGED a fact mid-session; using the old value = fail (catches summaries that freeze stale state — the signature compaction failure). `behavior_routing` — a platform issue dropped mid-session; the tutor must still redirect properly under context pressure.
+- **expected_facts / check_note**: per probe, the specific facts a correct answer must use, and a one-line pass/fail rule for the grader. Grade probes binary → **memory probe accuracy**, reported overall and per probe type per memory preset.
+- **profile_seed / facts**: the persona's stored profile. Seed it with `app.chat_service.set_student_profile(student_id, profile_seed)` before running that persona's questions; `facts` is the same content structured, for slicing results by fact type.
+- **Self-grading checks** (`checks` / `anti_patterns`): persona questions grade themselves because we authored the ground truth. `{"type": "regex_any", "patterns": [...]}` = case-insensitive regex over the answer; any match passes that check. `{"type": "llm", "instruction": ...}` = a judge call with that instruction (use a strong Anthropic model; binary verdict). `anti_patterns` = case-insensitive regexes whose match FAILS the question (e.g. bash `export` advice for a Windows persona). A question passes when every check passes and no anti-pattern matches.
+- **N-1 replay**: take a real thread, feed turns 1..N-1 as history, have the tutor produce turn N, grade against what the staff actually replied. Tests multi-turn behavior on fully real data (Hamel's recommended alternative to simulated users).
+- **Retrieval ground truth**: every discussion-derived case carries `source_key` + `lesson_url` — the course/lesson the question came from. Computing whether retrieval surfaced that source/lesson gives **recall@k / MRR** with zero extra labeling.
+- **review / verdict / review log**: every single-turn case carries the reviewer's `changes` (diff vs the original Gemini annotation) and `notes`. `review_log_v1.md` is the human-readable audit: systematic findings, the 11 judgment-call cases, all 92 verdicts, all 32 exclusions. Machine-readable verdicts: `review_batches/verdicts_batch_*.jsonl`.
+- **Memory preset**: a named memory/context configuration (`full_history`, `prod`, `aggressive`, `profile_memory`, ...) — see `app/memory_presets.py`. The experiment variable these batteries measure.
+
+## File schemas
+
+### `battery_singleturn_v1.jsonl` — 60 cases, one JSON object per line
+
+| Field | Meaning |
+|---|---|
+| `case_id` | Stable id (`st_<post_id>`). Use in run logs/bundles. |
+| `post_id` | Provenance: the original academy discussion post (`academy.towardsai.net/manage/discussion/posts/<post_id>`). |
+| `course`, `source_key`, `lesson_name`, `lesson_url` | Where the question came from; `source_key`+`lesson_url` double as retrieval ground truth. |
+| `asked_at` | Original post date (for staleness debugging). |
+| `question` | The standalone question — the prompt sent to the tutor. |
+| `category` | `conceptual` / `debugging` / `platform_issue` / `course_feedback` / `other` — for slicing results. |
+| `expected_behavior` | See glossary. Drives the behavior-accuracy check. |
+| `key_points` | See glossary. Empty for behavior-only cases (13 of 60). |
+| `eval_quality` | `gold` (49) or `usable` (11) — see glossary. |
+| `time_bound` | 9 true: grade behavior only, ignore stale specifics. |
+| `requires_notebook`, `notebook_evidence` | All false; evidence records what was checked in the KB. |
+| `requires_media` | 2 true; still gradeable. |
+| `reference_answer`, `reference_links` | Real staff reply — grader-only. |
+| `review.{changes,notes,batch}` | Reviewer audit trail per case. |
+
+Distribution to keep in mind when reporting: behaviors 37 corpus / 10 redirect / 7 general / 6 feedback; courses 41 full-stack / 16 agentic / 3 beginner-python (skewed — say so on slides).
+
+### `battery_sessions_v1.jsonl` — 32 sessions (337 turns, 113 probes)
+
+| Field | Meaning |
+|---|---|
+| `session_id` | e.g. `s08_fullstack_gamedev_update_12t` (s01–s05 hand-written, s06–s32 generated + spot-checked). |
+| `course`, `source_key` | Course context; pass `source_keys=(source_key,)` to the tutor. |
+| `persona` | One-line description of the student (for readers, not the tutor). |
+| `planted_facts` | The facts turn 0 establishes (and mid-session updates, marked "UPDATED at turn k"). |
+| `turns` | Ordered user messages. Run ALL sequentially on ONE `thread_id`. Middle turns are real corpus questions (post_ids in `notes`) — ungraded context whose job is to inflate tokens past compression triggers. Some reference since-changed content; that's realistic, leave it. |
+| `probes` | The graded turns: `turn_index` (0-based into `turns` — the probe IS that turn), `probe_type` (glossary), `expected_facts`, `check_note`. |
+| `notes` | Provenance: which post_ids / lesson names the turns use. |
+
+Probe mix: fact_recall 55 · preference_compliance 27 · anaphora 12 · anaphora_consistency 9 · fact_update 7 (sessions s08, s11, s18, s21, s24, s31) · behavior_routing 3.
+**Before the bake-off**: smoke-run a few sessions and confirm compression actually fired before the probe turns (`context_stats.summary_messages` / `cleared_tool_outputs` > 0); lengthen sessions if not — a memory eval where compaction never triggered measures nothing.
+
+### `battery_personas_v1.jsonl` — 10 personas × 4 questions
+
+| Field | Meaning |
+|---|---|
+| `persona_id` | Also use as the `student_id` for seeding/runs. |
+| `course`, `source_key` | Course context. |
+| `profile_seed` | Text to write into the store via `set_student_profile()` before the run. |
+| `facts` | Same profile, structured (level/goal/environment/weak_topic/preference) for slicing. |
+| `questions[].question_id` | e.g. `p01_q3`. |
+| `questions[].question` | Sent on a FRESH `thread_id` per question — isolates long-term memory from working memory. Deliberately unanswerable-correctly without the profile. |
+| `questions[].expected_facts_used` | Which profile facts a correct answer must draw on. |
+| `questions[].checks`, `anti_patterns`, `check_note` | Self-grading rules — see glossary. |
+
+Run each persona under `profile_memory` AND under a no-memory preset: the gap between the two pass rates is the personalization effect.
+
+### `replay_n1_v1.jsonl` — 30 cases from 24 real threads
+
+| Field | Meaning |
+|---|---|
+| `replay_id` | `replay_<post_id>_t<k>`: replays up to staff turn k. |
+| `post_id`, `course`, `source_key`, `lesson_url`, `category` | Provenance + retrieval ground truth. |
+| `eval_quality`, `time_bound` | 17 gold / 13 usable; 8 time_bound — filter those for most uses. |
+| `history` | Real conversation prefix: original question + thread turns, mapped student→`user`, staff→`assistant`. Feed as the request history; the last entry is always a user message. |
+| `reference_reply` | The real staff reply to grade against (grader-only). |
+
+Contains real names inside message text (inherent to replay) — private.
+
+### Provenance files
+
+- `review_batches/batch_*.json` — the 92 raw cases as sent to reviewers; `review_batches/verdicts_batch_*.jsonl` — machine-readable verdicts. To overturn a verdict: edit the line, rerun assembly, bump the battery to v2.
+- `review_log_v1.md` — the human audit document for the single-turn battery.
+- `sessions_handwritten.jsonl`, `sessions_generated_{a,b,c}.jsonl` — raw session inputs merged into `battery_sessions_v1.jsonl`.
+
+## How grading will work (Part A3/B of the experiments plan)
+
+- **Single-turn**: behavior routing + citation validity as code assertions; key-point coverage hand-graded for the workshop (≈60–80 binary readings), later automated by a judge validated against those hand labels (product plan P4). Judges: strong Anthropic model via API, binary verdict + one-line critique per check.
+- **Sessions**: probe `expected_facts` checked by string match where unambiguous, judge with `check_note` as the rubric otherwise; trigger-firing verified from `context_stats` telemetry.
+- **Personas**: fully programmatic except `type: "llm"` checks.
+- **Replay**: judge or hand-grade against `reference_reply`.
+- Every reported number should name its battery version (e.g. "key-point coverage, singleturn_v1") and, for judge-graded metrics, the judge's measured agreement with human labels (TPR/TNR).

data/scraping_scripts/annotate_discussion_posts.py

@@ -0,0 +1,433 @@
+"""Annotate academy discussion posts into an eval-ready dataset.
+
+Reads ``data/academy_discussion_posts.jsonl`` (scraped course discussion
+threads) and emits ``data/academy_discussion_eval.jsonl``, one annotated
+record per thread, combining:
+
+- deterministic fields extracted in code: the cleaned question, the earliest
+  reply written by course staff (the reference answer), the full thread in
+  chronological order, and the corpus source key for the course; and
+- LLM annotations (Gemini structured output): category, expected tutor
+  behavior, a self-contained rewrite of the question, evergreen key points
+  distilled from the staff answer, and an overall eval-quality grade.
+
+The output is the raw material for eval datasets: filter on
+``annotation.eval_quality`` / ``annotation.category`` to build the Q&A set
+(judge against ``key_points``), the out-of-scope behavior set
+(``expected_behavior == "redirect_to_support"``), and retrieval ground truth
+(``source_key`` + ``lesson_url``).
+
+Resumable: post_ids already present in the output file are skipped unless
+``--force`` is passed.
+
+Usage:
+    uv run -m data.scraping_scripts.annotate_discussion_posts [--limit N] [--force]
+"""
+
+import argparse
+import asyncio
+import json
+import os
+import re
+from datetime import UTC, datetime
+from pathlib import Path
+from typing import Literal
+
+from dotenv import load_dotenv
+from google import genai
+from google.genai import types
+from google.genai.errors import APIError
+from pydantic import BaseModel, Field
+from tenacity import (
+    retry,
+    retry_if_exception,
+    stop_after_attempt,
+    wait_random_exponential,
+)
+from tqdm.asyncio import tqdm
+
+load_dotenv(".env")
+
+DEFAULT_INPUT = Path("data/academy_discussion_posts.jsonl")
+DEFAULT_OUTPUT = Path("data/academy_discussion_eval.jsonl")
+ANNOTATION_MODEL = os.getenv("ANNOTATION_MODEL", "gemini-3.5-flash")
+ANNOTATION_CONCURRENCY = int(os.getenv("ANNOTATION_CONCURRENCY", "8"))
+RETRYABLE_GENAI_STATUS_CODES = {408, 429, 500, 502, 503, 504}
+
+# People who answer threads in an official capacity. Replies from anyone else
+# (students helping each other, the original poster following up) are kept in
+# the thread but never used as the reference answer. Derived from responders
+# who reply in threads they did not start; edit as the team changes.
+STAFF_RESPONDERS = {
+    "Jaiganesan N",
+    "Louis-François Bouchard",
+    "Paul Iusztin",
+    "Samridhi",
+    "Louie Peters",
+    "Omar Solano",
+}
+
+# Dataset course_name -> source key in source_registry.SOURCE_CONFIGS.
+# Courses absent from the corpus map to None (the tutor has no material for
+# them, which the annotator is told about).
+COURSE_TO_SOURCE_KEY: dict[str, str | None] = {
+    "Full Stack AI Engineering": "full_stack_ai_engineering",
+    "Agent Engineering: Building Multi-Agent Systems": "agentic_ai_engineering",
+    "Beginner Python for AI Engineering": "beginner_python_for_ai_engineering",
+    "Master AI For Work": "master_ai_for_work",
+    "8-hour Generative AI Primer": None,
+}
+
+
+class PostAnnotation(BaseModel):
+    """LLM-produced annotation for one discussion thread."""
+
+    category: Literal[
+        "conceptual", "debugging", "course_feedback", "platform_issue", "other"
+    ] = Field(
+        description=(
+            "conceptual: technical/theory question about course material. "
+            "debugging: an error or unexpected result in the student's code, "
+            "notebook, or environment. course_feedback: typo/errata reports or "
+            "suggestions about course content. platform_issue: academy website, "
+            "videos, quizzes, certificates, accounts, billing, community access. "
+            "other: greetings, kudos, or unintelligible posts."
+        )
+    )
+    expected_behavior: Literal[
+        "answer_from_corpus",
+        "answer_general",
+        "redirect_to_support",
+        "acknowledge_feedback",
+    ] = Field(
+        description=(
+            "What an ideal tutor response does. answer_from_corpus: answer "
+            "grounded in course/docs material. answer_general: sound technical "
+            "answer that likely goes beyond the corpus. redirect_to_support: "
+            "the tutor cannot act on this; it should say so briefly and point "
+            "to the support team, without inventing troubleshooting steps. "
+            "acknowledge_feedback: thank the student and suggest reporting the "
+            "issue to the team."
+        )
+    )
+    self_contained: bool = Field(
+        description=(
+            "True if the question can be fully understood without seeing the "
+            "lesson page, a screenshot, a quiz, or other context the tutor "
+            "does not have."
+        )
+    )
+    standalone_question: str | None = Field(
+        description=(
+            "A minimally rewritten, self-contained version of the question "
+            "suitable to send to the tutor: resolve references like 'this "
+            "lesson', 'Image 8', or 'the notebook' using the course and "
+            "lesson names; keep the student's wording and code verbatim "
+            "otherwise. Null if the question cannot be made self-contained."
+        )
+    )
+    key_points: list[str] = Field(
+        description=(
+            "1-4 evergreen technical claims distilled from STAFF replies, "
+            "each independently checkable against a candidate answer (the "
+            "diagnosis, the fix, the concept). Exclude insider or time-bound "
+            "content: notebook-update promises, platform actions, version "
+            "numbers true only at the time. Empty if there is no staff reply "
+            "or it contains nothing technical."
+        )
+    )
+    excluded_from_key_points: str | None = Field(
+        description=(
+            "One short note on staff-reply content deliberately left out of "
+            "key_points (e.g. 'promise to update the notebook'). Null if "
+            "nothing was excluded."
+        )
+    )
+    time_bound: bool = Field(
+        description=(
+            "True if the correct answer depends on a point-in-time state "
+            "(package version, notebook revision, platform status) that may "
+            "have changed since."
+        )
+    )
+    eval_quality: Literal["gold", "usable", "weak", "exclude"] = Field(
+        description=(
+            "Suitability as an eval case for the eval its expected_behavior "
+            "implies. gold: clear self-contained question with a solid staff "
+            "answer yielding key points, or an unambiguous redirect case. "
+            "usable: good question with a partial/thin reference or needing "
+            "the rewrite. weak: marginal; heavily time-bound or barely "
+            "intelligible. exclude: no eval value (greetings, kudos, empty)."
+        )
+    )
+    notes: str = Field(description="One-line rationale for the grades above.")
+
+
+ANNOTATION_PROMPT = """\
+You are annotating a student discussion thread from an online AI course so it \
+can be turned into an evaluation case for an AI tutor chatbot.
+
+About the tutor being evaluated:
+- It answers questions grounded in a corpus of course materials and AI library \
+docs (Transformers, PEFT, TRL, LlamaIndex, LangChain, LangGraph, OpenAI docs, \
+and the courses themselves), and can optionally search the web.
+- It CANNOT: access the academy platform (videos, quizzes, assignments, \
+certificates, accounts, billing, Slack/Discord), see screenshots or images, \
+run or update course notebooks, or change course content.
+
+Thread metadata:
+- Course: {course_name} ({corpus_note})
+- Lesson: {lesson_name}
+- The original post {media_note}.
+
+Thread (chronological; STAFF = official course staff, STUDENT = learner):
+
+{thread_text}
+
+Annotate the thread per the response schema. Additional rules:
+- key_points must come only from STAFF replies. If there is no staff reply, \
+key_points must be empty and eval_quality is at most "usable".
+- standalone_question preserves the student's voice and any code verbatim; it \
+only resolves references the tutor could not see. If the post fundamentally \
+depends on unseen content (a screenshot, quiz option text, a specific page), \
+set self_contained to false and standalone_question to null.
+- A clear platform_issue is still a valuable eval case (the tutor should \
+redirect gracefully), so grade it on that basis rather than excluding it.
+"""
+
+_genai_client: genai.Client | None = None
+
+
+def get_genai_client() -> genai.Client:
+    global _genai_client
+    if _genai_client is None:
+        api_key = os.getenv("GEMINI_API_KEY") or os.getenv("GOOGLE_API_KEY")
+        if not api_key:
+            raise RuntimeError("GEMINI_API_KEY or GOOGLE_API_KEY must be set.")
+        _genai_client = genai.Client(api_key=api_key)
+    return _genai_client
+
+
+def is_retryable_genai_error(exc: BaseException) -> bool:
+    return isinstance(exc, APIError) and exc.code in RETRYABLE_GENAI_STATUS_CODES
+
+
+def parse_post_date(text: str | None) -> str | None:
+    if not text:
+        return None
+    cleaned = " ".join(text.split())
+    for fmt in ("%B %d, %Y", "%b %d, %Y"):
+        try:
+            return datetime.strptime(cleaned, fmt).date().isoformat()
+        except ValueError:
+            continue
+    return None
+
+
+def parse_byline(byline: str | None) -> tuple[str | None, str | None]:
+    """Split 'Posted by NAME on\\nDATE' into (name, ISO date)."""
+    if not byline:
+        return None, None
+    match = re.search(r"Posted by\s+(.+?)\s+on\b\s*(.*)$", byline, re.DOTALL)
+    if not match:
+        return " ".join(byline.split()) or None, None
+    name = " ".join(match.group(1).split())
+    return name or None, parse_post_date(match.group(2))
+
+
+def build_thread(record: dict) -> list[dict]:
+    """Replies in chronological order (the source lists newest first)."""
+    thread = []
+    for resp in reversed(record.get("responses") or []):
+        author = (resp.get("responded_by") or "").strip()
+        content = (resp.get("content") or "").strip()
+        if not content:
+            continue
+        thread.append(
+            {
+                "role": "staff" if author in STAFF_RESPONDERS else "student",
+                "author": author,
+                "date": parse_post_date(resp.get("responded_at")),
+                "content": content,
+                "links": resp.get("links") or [],
+            }
+        )
+    return thread
+
+
+def first_staff_reply(thread: list[dict]) -> dict | None:
+    return next((r for r in thread if r["role"] == "staff"), None)
+
+
+def render_thread(
+    question: str, asked_by: str | None, asked_at: str | None, thread: list[dict]
+) -> str:
+    parts = [
+        f"[1] STUDENT {asked_by or 'Unknown'} ({asked_at or 'unknown date'}) — original post:\n{question}"
+    ]
+    for i, reply in enumerate(thread, start=2):
+        parts.append(
+            f"[{i}] {reply['role'].upper()} {reply['author']} "
+            f"({reply['date'] or 'unknown date'}):\n{reply['content']}"
+        )
+    return "\n\n".join(parts)
+
+
+def build_record(raw: dict) -> dict:
+    """Deterministic part of the output record (everything but `annotation`)."""
+    asked_by, asked_at = parse_byline((raw.get("original_post") or {}).get("byline"))
+    title = (raw.get("discussion_title") or "").strip()
+    body = ((raw.get("original_post") or {}).get("content") or "").strip()
+    question = f"{title}\n\n{body}".strip() if title else body
+    thread = build_thread(raw)
+    reference = first_staff_reply(thread)
+    course_name = raw.get("course_name") or ""
+
+    return {
+        "post_id": raw.get("post_id"),
+        "discussion_url": raw.get("discussion_url"),
+        "discussion_title": title,
+        "course_name": course_name,
+        "source_key": COURSE_TO_SOURCE_KEY.get(course_name),
+        "lesson_name": raw.get("lesson_name"),
+        "lesson_url": raw.get("lesson_url"),
+        "asked_by": asked_by,
+        "asked_at": asked_at,
+        "question": question,
+        "question_has_media": bool((raw.get("original_post") or {}).get("media")),
+        "reference_answer": reference["content"] if reference else None,
+        "reference_answer_by": reference["author"] if reference else None,
+        "reference_answer_at": reference["date"] if reference else None,
+        "reference_links": reference["links"] if reference else [],
+        "thread": thread,
+    }
+
+
+@retry(
+    retry=retry_if_exception(is_retryable_genai_error),
+    stop=stop_after_attempt(6),
+    wait=wait_random_exponential(multiplier=1, max=60),
+    reraise=True,
+)
+async def annotate_record(record: dict, model: str) -> PostAnnotation:
+    source_key = record["source_key"]
+    corpus_note = (
+        f"in the tutor's corpus as source '{source_key}'"
+        if source_key
+        else "NOT in the tutor's corpus"
+    )
+    media_note = (
+        "includes an image/screenshot the tutor cannot see"
+        if record["question_has_media"]
+        else "has no attached media"
+    )
+    prompt = ANNOTATION_PROMPT.format(
+        course_name=record["course_name"],
+        corpus_note=corpus_note,
+        lesson_name=record["lesson_name"] or "unknown",
+        media_note=media_note,
+        thread_text=render_thread(
+            record["question"],
+            record["asked_by"],
+            record["asked_at"],
+            record["thread"],
+        ),
+    )
+    response = await get_genai_client().aio.models.generate_content(
+        model=model,
+        contents=prompt,
+        config=types.GenerateContentConfig(
+            temperature=0.0,
+            response_mime_type="application/json",
+            response_schema=PostAnnotation,
+        ),
+    )
+    if isinstance(response.parsed, PostAnnotation):
+        return response.parsed
+    if isinstance(response.parsed, dict):
+        return PostAnnotation.model_validate(response.parsed)
+    if response.text:
+        return PostAnnotation.model_validate_json(response.text)
+    raise ValueError(f"Gemini returned no annotation for post {record['post_id']}.")
+
+
+def load_jsonl(path: Path) -> list[dict]:
+    if not path.exists():
+        return []
+    with path.open() as fh:
+        return [json.loads(line) for line in fh if line.strip()]
+
+
+def write_jsonl(path: Path, records: list[dict]) -> None:
+    path.parent.mkdir(parents=True, exist_ok=True)
+    with path.open("w") as fh:
+        for record in records:
+            fh.write(json.dumps(record, ensure_ascii=False) + "\n")
+
+
+async def annotate_all(records: list[dict], model: str) -> list[dict]:
+    semaphore = asyncio.Semaphore(ANNOTATION_CONCURRENCY)
+    annotated_at = datetime.now(UTC).isoformat(timespec="seconds")
+
+    async def worker(record: dict) -> dict:
+        async with semaphore:
+            annotation = await annotate_record(record, model)
+        return {
+            **record,
+            "annotation": annotation.model_dump(),
+            "annotation_model": model,
+            "annotated_at": annotated_at,
+        }
+
+    return await tqdm.gather(*(worker(r) for r in records), desc="Annotating")
+
+
+def main() -> None:
+    parser = argparse.ArgumentParser(description=__doc__.splitlines()[0])
+    parser.add_argument("--input", type=Path, default=DEFAULT_INPUT)
+    parser.add_argument("--output", type=Path, default=DEFAULT_OUTPUT)
+    parser.add_argument("--model", default=ANNOTATION_MODEL)
+    parser.add_argument(
+        "--limit", type=int, default=None, help="Annotate at most N pending posts."
+    )
+    parser.add_argument(
+        "--force",
+        action="store_true",
+        help="Re-annotate posts already present in the output file.",
+    )
+    args = parser.parse_args()
+
+    raw_records = load_jsonl(args.input)
+    if not raw_records:
+        raise SystemExit(f"No records found in {args.input}.")
+
+    existing = {r["post_id"]: r for r in load_jsonl(args.output)}
+    records = [build_record(raw) for raw in raw_records]
+    pending = [r for r in records if args.force or r["post_id"] not in existing]
+    if args.limit is not None:
+        pending = pending[: args.limit]
+
+    print(
+        f"{len(records)} threads in input, {len(existing)} already annotated, "
+        f"{len(pending)} to annotate with {args.model}."
+    )
+    if pending:
+        for record in asyncio.run(annotate_all(pending, args.model)):
+            existing[record["post_id"]] = record
+
+    merged = sorted(existing.values(), key=lambda r: int(r["post_id"]))
+    write_jsonl(args.output, merged)
+
+    counts: dict[str, int] = {}
+    for record in merged:
+        quality = record["annotation"]["eval_quality"]
+        category = record["annotation"]["category"]
+        counts[f"quality={quality}"] = counts.get(f"quality={quality}", 0) + 1
+        counts[f"category={category}"] = counts.get(f"category={category}", 0) + 1
+    print(f"Wrote {len(merged)} annotated threads to {args.output}.")
+    for key in sorted(counts):
+        print(f"  {key}: {counts[key]}")
+
+
+if __name__ == "__main__":
+    main()

evals.md

@@ -0,0 +1,133 @@
+# Evaluating the AI tutor
+
+We compare memory and context-management techniques on the tutor with measured numbers — answer quality, memory retention, retrieval accuracy, tokens, cost, latency — for the June 2026 workshop, and keep the same harness as the ongoing quality program afterwards. Research sources and design rationale: `evals_background.md`.
+
+## What we evaluate, and what varies
+
+The system under test is the production tutor (`app/`): a LangGraph agent over the course corpus with hybrid retrieval (`retrieve_tutor_context`), a sandboxed KB shell (`run_kb_command`), and per-thread conversation memory. One variable changes between experiment arms — the **memory preset** (`app/memory_presets.py`):
+
+| Preset | Configuration |
+|---|---|
+| `full_history` | no compaction (baseline: quality ceiling, token worst case) |
+| `prod` | production settings: summarization @30k tokens keep 20 msgs + tool-output clearing @5k keep 5 |
+| `summarization_only` / `editing_only` | each compaction technique isolated |
+| `aggressive` | summarize @8k keep 8, clear @2k keep 2 |
+| `profile_memory` | prod + long-term memory: stored student profile injected into the system prompt, updated after each turn (`StudentProfileMiddleware`) |
+
+Held constant per experiment: model (`gemini-3.5-flash`), system prompt, retrieval configuration, source selection, web tools off, temperature. Presets are selectable per request (`ChatRequest.memory_preset`, API `memoryPreset`) and are part of the agent cache key.
+
+## What we measure
+
+| Layer | Metrics | Source |
+|---|---|---|
+| Ops | TTFT, turn latency, input/output tokens, cache-read/creation tokens, est. cost, LLM calls, compaction-trigger firings | `context_stats` event each turn (`app/telemetry.py`) — self-contained, no LangSmith dependency |
+| Trajectory | tool calls per turn by tool, redundant calls, KB-budget use | tool-call events (code) |
+| Retrieval | recall@shown / MRR of the case's true `source_key` + `lesson_url` in retrieval results | ground truth carried by every discussion-derived case (code) |
+| Behavior | called retrieval on corpus questions, citations present and resolving, redirect/feedback routing | code assertions + heuristics; authoritative number from hand grades |
+| Quality | key-point coverage, behavior correctness, probe pass/fail | **hand-graded binary** (blinded workbook); an LLM judge only after validation against those labels (TPR/TNR, target >90%) |
+| Memory | probe accuracy by type (fact recall, preference, anaphora, fact-update, behavior-under-pressure), personalization pass rate | session probes + persona checks |
+
+Cost is reported both as tokens and as cached dollars — they rank presets differently (finding F2).
+
+## The dataset
+
+Source: `data/academy_discussion_eval.jsonl` — 151 real student posts from the academy discussion boards with real staff answers, annotated by gemini-3.5-flash. All eval inputs are real or hand-authored; **we never generate reference answers** — ground truth is staff replies (distilled to `key_points`) or facts we wrote ourselves.
+
+**Review.** All 62 gold + 30 not-time-bound usable annotations (92) were re-reviewed against the live KB with file/line evidence. 60 kept, 32 excluded. Main corrections: the annotator badly under-flagged staleness (course content was updated after many posts — ~14 exclusions had premises no longer true in today's corpus), 5 near-duplicates, 1 fabricated URL, 1 misaligned question/answer pair, and key points rewritten to claims that are atomic, still true, and binary-checkable. Colab-notebook dependency was checked per case: **1 of 92** truly required the notebook (excluded); lessons embed the cells everywhere else. Full audit trail: `data/eval/review_log_v1.md` (11 judgment-call cases flagged for human review).
+
+**Batteries** (`data/eval/`, schemas and glossary in its `README.md`; files are gitignored — real student text):
+
+| Battery | Contents | Tests |
+|---|---|---|
+| `battery_singleturn_v1` | 60 reviewed real questions (37 corpus / 10 redirect / 7 general / 6 feedback; 49 gold) | one-shot answer quality + behavior routing; presets should roughly tie here |
+| `battery_sessions_v1` | 32 authored study sessions, 337 turns, 113 probes (5 hand-written, 27 generated and spot-checked; middle turns are verbatim real questions) | where memory methods separate: facts planted early, context inflated past compression triggers, probes late. Probe types: fact_recall (55), preference_compliance (27), anaphora (12), anaphora_consistency (9), **fact_update** (7 — the student changes a fact mid-session; answering with the stale value fails), behavior_routing (3) |
+| `battery_personas_v1` | 10 authored student profiles × 4 questions each, unanswerable correctly without the profile; self-grading regex/anti-pattern checks | long-term profile memory |
+| `replay_n1_v1` | 30 prefixes of real multi-turn threads, graded against the real next staff reply | multi-turn behavior on fully real data (secondary) |
+
+## How it runs
+
+```bash
+uv run -m evals.run_battery --battery data/eval/battery_singleturn_v1.jsonl --preset prod --trials 2 --out runs/exp
+uv run -m evals.grade  --run runs/exp                      # code checks + handgrade_sheet.csv
+uv run -m evals.check_triggers --runs runs/exp             # gate: compaction fired where probes assume
+uv run -m evals.report --runs runs/expA runs/expB          # side-by-side tables + token curves
+uv run -m evals.handgrade_workbook build|merge ...         # blinded human-grading workbook
+```
+
+Every turn persists a JSON trace bundle, so grading and reporting re-run offline forever without touching the API. Runs are resume-safe (re-run the same command after any interruption), have a 10-minute per-turn timeout, and keep LangSmith off by default. Needs `GEMINI_API_KEY` + `COHERE_API_KEY`; the full 4-preset bake-off below cost $73.
+
+**Setup for collaborators.** Code and docs are in git; the datasets and run results contain real student text and live only in the private HF dataset (`towardsai-tutors/ai-tutor-data` — git is force-pushed to the public prod Space on deploys, so student data never enters git). With an `HF_TOKEN` that can read it:
+
+```bash
+uv run python -c "from huggingface_hub import snapshot_download as d; d(repo_id='towardsai-tutors/ai-tutor-data', repo_type='dataset', allow_patterns=['eval/**','eval_runs/**'], local_dir='.')"
+mv eval/* data/eval/ && mv eval_runs/part_b/* runs/   # restore working paths
+```
+
+## Results — Part B bake-off, 2026-06-12
+
+4 presets × 2 trials over the full single-turn battery, full persona battery, and 3 sessions (s01 13-turn, s02 agentic, s08 fact-update). 1,232 turns, zero API errors. Tables: `runs/b_report/report.md`; token curves: `tokens_by_turn.csv`. **Auto-graded only — quality and probe-accuracy columns land after the hand-grading pass.**
+
+| | full_history | prod | profile_memory | aggressive |
+|---|---|---|---|---|
+| personalization pass (personas, n=70) | 63% | 67% | **94%** | 56% |
+| sessions: est. cost/turn | **$0.034** | $0.051 | $0.047 | $0.066 |
+| sessions: TTFT p50 | **17s** | 21s | 22s | 43s |
+| sessions: tool calls/turn | **2.8** | 3.5 | 3.9 | 8.3 |
+| sessions: cumulative input tokens | 2.9M | 1.7M | 1.6M | 1.75M |
+| single-turn: behavior heuristic (n=106) | **88%** | 86% | 86% | 78% |
+| single-turn: retrieval recall@shown source | 51% | 50% | 49% | 47% |
+
+Row definitions:
+- **personalization pass** — % of persona question-runs whose answer passed every authored check (expected regex matched, e.g. `conda` for the conda persona) with no anti-pattern hit (e.g. bash `export` for a Windows persona). n=70: 40 questions × 2 trials, minus 10 runs whose checks need human judgment.
+- **est. cost/turn** — mean estimated $ per conversation turn: token counts × `MODEL_PRICING`, cached input billed at the cache-read discount (our table, not the invoice).
+- **TTFT p50** — median seconds from user message to first answer text, including all tool calls and internal LLM rounds before the answer starts (perceived wait).
+- **tool calls/turn** — mean retrieval + KB-command invocations per turn; here it's the re-work signal (compaction presets re-search for evidence their compressed history lost).
+- **cumulative input tokens** — all input tokens billed across an entire session (every turn, every internal call), mean over the 6 session-runs per preset.
+- **behavior heuristic** — % of single-turn case-runs where a programmatic proxy confirms the right *kind* of response (corpus → called retrieval/KB; support issue → answer points to support; feedback → acknowledges). n=106 of 120: `answer_general` has no heuristic. Proxy only — authoritative behavior accuracy comes from the hand grades.
+- **retrieval recall@shown source** — % of case-runs where the reranked retrieval results the agent saw contained at least one chunk from the correct course. Stricter `recall@lesson` (right lesson page, 30–36%) and MRR are in the full report.
+
+### Observations
+
+Numbered findings; each states what it was tested on. Convention: entries are never edited, only superseded.
+
+- **F1 — Retrieval payloads dominate input tokens, not conversation history.** Each retrieval call may return up to `DEFAULT_CONTEXT_TOKEN_BUDGET = 100_000` tokens; turns average ~200k input. (All runs; high confidence.) → retrieval budget became a Part C variant dimension.
+- **F2 — Compaction saves tokens but not necessarily dollars.** Summarization rewrites the prompt prefix and invalidates Gemini's implicit cache: on the same session, `full_history` had 86.8% of input billed at the ~4x cache discount; `aggressive` used 44% fewer tokens yet cost 68% more. (s03 × 3 presets, then confirmed n=24 session-runs; provider-specific — Anthropic caching is explicit.)
+- **F3 — Context editing never fires on this workload.** `ClearToolUsesEdit` excludes retrieval results — where the tokens are; 0 clears across all session runs. → `editing_only` dropped from Part B; "clear retrieval results too" variant queued for Part C.
+- **F4 — Aggressive compaction degrades even single-turn behavior.** 18.0 vs 9.6 LLM calls/turn, 57s vs 39s median, behavior heuristic −10pts vs full_history (60 cases × 2 trials). Mid-turn compaction churn changes agent behavior, it doesn't just trim.
+- **F5 — Gemini reasoning tokens are ~90%+ of output even with reasoning display off.** Billed as output; dominates latency; recorded per model in `usage_by_model`.
+- **F6 — Long context costs latency, not correctness.** Zero API errors in 1,018+ turns at any size (max 6.06M tokens across one turn's calls; largest single context 274k). Median TTFT scales 22s → 76s from <100k to >800k input tokens/turn. Compaction's value here is responsiveness and spend, not keeping the model functional.
+- **F7 — ~0.8% of turns produce no answer text** despite tool calls and billed reasoning tokens; preset- and size-independent. Candidate golden-case assertion ("answer non-empty").
+- **F8 — Profile memory wins on quality AND cost.** 94% personalization vs 56–67% without, while the cheapest persona preset ($0.049 vs $0.081–0.095/turn, 7.6 vs 10–11.2 tool calls): the stored profile saves the agent from re-searching for user context. (40 questions × 2 trials × 4 presets, auto checks; LLM-check rows pending hand grades.)
+- **F9 — Full history is cheapest AND fastest up to 13 turns; compaction causes re-work.** Sessions: $0.034/turn, TTFT 17s, 2.8 tool calls for `full_history` vs $0.051–0.066, 21–43s, 3.5–8.3 elsewhere — despite ~2x the tokens (F2's cache mechanism plus a second one: raw history lets the agent re-use earlier retrieval evidence; summaries force re-retrieval). (3 sessions × 4 presets × 2 trials, n=24, zero errors.) The conventional pitch inverts: under modern prompt caching, the naive baseline wins short-to-medium sessions; compaction must justify itself on quality (pending hand grades) and long horizons. Where the crossover actually sits is a Part C question.
+
+Harness corrections (bugs in our measurement, not findings): the overnight 06-12 stall was machine sleep hanging API streams (all four pipelines stopped the same minute; fixed with the per-turn timeout); battery lesson-URLs carried a `/discussions/` suffix that silently zeroed recall@lesson until normalized (caught by the A3 smoke, re-graded from bundles without re-running).
+
+Talk-outline correction: the planned "skills / just-in-time KB instructions" change (Change 4) was never built, and its premise is off — the KB instructions block measures **458 tokens** (not ~4k) inside a 1,655-token system prompt; max saving ~2% of a turn, mostly cache-discounted. Recommended reframe: demo progressive disclosure on tool *outputs* (F1), where our numbers actually are.
+
+## Remaining work
+
+**Omar**
+- [ ] Grade `runs/b_report/workbook.csv` — 284 blinded rows, priority-ordered (96 session probes ≈ 25 min is the critical tier). Don't open `workbook_keymap.csv` until done. Then: `uv run -m evals.handgrade_workbook merge --workbook <filled.csv>` and re-run grade/report for the final table.
+- [ ] Audit `data/eval/review_log_v1.md` (the 11 flagged judgment calls first).
+- [ ] Verify the `gemini-3.5-flash` entry in `app/telemetry.py:MODEL_PRICING` against the current price sheet.
+
+**Workshop**
+- [ ] Replace placeholder slide numbers with measured ones; pick 2–3 failure traces from the graded probes for the failure→fix→number beats (a `fact_update` failure under summarization is the target demo).
+- [ ] Token-vs-turn plot: CSV is ready; add matplotlib (or plot elsewhere) for the PNG.
+- [ ] Next.js meter component over the `data-context-stats` SSE part, if the live demo needs it.
+- [ ] Skills section: reframe per the 458-token measurement above.
+
+**Part C (post-workshop, ~$300–400, after the judge is validated)**
+- [ ] Variants: `summarization_only`, `aggressive`, clear-retrieval-results editing, retrieval budget 100k/30k/10k, 1–2 SOTA methods (retrieval-over-history, structured note-taking, LangMem-style consolidation), optionally lazy KB instructions.
+- [ ] Full batteries (60 single-turn + 32 sessions + 30 replay) × 3 trials; paired stats (McNemar/bootstrap), pass^3 consistency; failure-taxonomy diffs per variant.
+
+**Product quality track (post-workshop)**
+Golden cases as a CI gate (5–10 critical-path cases, deterministic assertions, incl. the F7 non-empty check) → error-analysis cycles (expert reads 50–100 traces in a small viewer, binary + first-failure note, stop at saturation) → failure taxonomy → code assertions for recurring modes → LLM judges built from the hand-grade critiques and validated to >90% TPR/TNR on held-out labels before any judge-graded number is reported → nightly battery + weekly trace sampling. Full methodology behind each step: `evals_background.md`.
+
+## Files
+
+- `evals.md` — this file: the what, the data, the results, the queue.
+- `evals_background.md` — research sources (Hamel, howtoeval, OpenAI macro-evals) and design rationale.
+- `data/eval/README.md` — battery schemas + glossary of terms; `review_log_v1.md` — dataset audit trail.
+- `evals/` — the harness code; `app/memory_presets.py`, `app/telemetry.py` — the app-side hooks.
+- `runs/b_report/` — generated: final tables, token curves, the blinded grading workbook.

evals/__init__.py

@@ -0,0 +1,10 @@
+"""Eval harness: run batteries against the tutor, grade bundles, report.
+
+Pipeline (see evals.md):
+  uv run -m evals.run_battery  -> runs/<exp>/bundles.jsonl   (talks to the app)
+  uv run -m evals.grade        -> grades_auto.jsonl + handgrade_sheet.csv (pure)
+  uv run -m evals.report       -> report.md + tokens_by_turn.csv (pure)
+
+Only run_battery imports app code; grade/report operate on JSON alone so they
+can re-grade old bundles forever without a running system.
+"""

evals/check_triggers.py

@@ -0,0 +1,127 @@
+"""Verify compression triggers fired where the session battery assumes.
+
+  uv run -m evals.check_triggers --runs runs/a4_sessions_prod
+  uv run -m evals.check_triggers --runs runs/a4_s03_fullhist --expect-none
+
+A memory eval where compaction never activated measures nothing (see
+data/eval/README.md), so this is the gate before any bake-off:
+- default: every probe turn must have compaction active (summary_messages or
+  cleared_tool_outputs > 0 in context_stats); exit 1 otherwise.
+- --expect-none (for full_history): NO turn may show compaction; exit 1 if
+  the baseline ever compressed — that would invalidate the comparison.
+
+Also prints per-session cost/token totals for Part B/C budget projection.
+"""
+
+from __future__ import annotations
+
+import argparse
+import sys
+from collections import defaultdict
+from pathlib import Path
+from typing import Any
+
+from .common import load_jsonl
+
+
+def session_table(bundles: list[dict[str, Any]]) -> dict[tuple[str, int], list[dict]]:
+    sessions: dict[tuple[str, int], list[dict]] = defaultdict(list)
+    for bundle in bundles:
+        if bundle.get("turn_index") is None:
+            continue
+        sessions[(bundle["unit_id"], bundle["trial"])].append(bundle)
+    for rows in sessions.values():
+        rows.sort(key=lambda b: b["turn_index"])
+    return sessions
+
+
+def probe_indices(battery_path: str, session_id: str) -> list[int]:
+    for record in load_jsonl(battery_path):
+        if record.get("session_id") == session_id:
+            return [probe["turn_index"] for probe in record.get("probes", [])]
+    return []
+
+
+def compaction_active(bundle: dict[str, Any]) -> bool:
+    stats = bundle.get("context_stats") or {}
+    return bool(
+        (stats.get("summary_messages") or 0) or (stats.get("cleared_tool_outputs") or 0)
+    )
+
+
+def check_run(run_dir: Path, expect_none: bool) -> bool:
+    bundles = load_jsonl(run_dir / "bundles.jsonl")
+    ok = True
+    for (session_id, trial), rows in sorted(session_table(bundles).items()):
+        preset = rows[0]["preset"]
+        probes = set(probe_indices(rows[0]["battery_path"], session_id))
+        first_summary = next(
+            (
+                b["turn_index"]
+                for b in rows
+                if (b.get("context_stats") or {}).get("summary_messages")
+            ),
+            None,
+        )
+        first_clear = next(
+            (
+                b["turn_index"]
+                for b in rows
+                if (b.get("context_stats") or {}).get("cleared_tool_outputs")
+            ),
+            None,
+        )
+        tokens = sum(
+            (b.get("context_stats") or {}).get("input_tokens") or 0 for b in rows
+        )
+        cost = sum(
+            (b.get("context_stats") or {}).get("est_cost_usd") or 0 for b in rows
+        )
+        errors = [b["turn_index"] for b in rows if b.get("error")]
+        bad_probes = []
+        compressed_turns = [b["turn_index"] for b in rows if compaction_active(b)]
+        if expect_none:
+            if compressed_turns:
+                ok = False
+                verdict = f"FAIL compaction at turns {compressed_turns} (expected none)"
+            else:
+                verdict = "OK no compaction"
+        else:
+            bad_probes = [
+                b["turn_index"]
+                for b in rows
+                if b["turn_index"] in probes and not compaction_active(b)
+            ]
+            if bad_probes:
+                ok = False
+                verdict = f"FAIL probes without compaction: {bad_probes}"
+            elif not probes:
+                verdict = "OK (no probes found?)"
+            else:
+                verdict = "OK all probes under compaction"
+        if errors:
+            ok = False
+            verdict += f" | ERRORS at turns {errors}"
+        print(
+            f"{session_id} t{trial} [{preset}] turns={len(rows)} "
+            f"first_summary@{first_summary} first_clear@{first_clear} "
+            f"input_tok={tokens:,} est_cost=${cost:.2f} -> {verdict}"
+        )
+    return ok
+
+
+def main() -> None:
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument("--runs", nargs="+", required=True)
+    parser.add_argument(
+        "--expect-none",
+        action="store_true",
+        help="Assert NO compaction anywhere (full_history baseline).",
+    )
+    args = parser.parse_args()
+    ok = all(check_run(Path(r), args.expect_none) for r in args.runs)
+    sys.exit(0 if ok else 1)
+
+
+if __name__ == "__main__":
+    main()

evals/common.py

@@ -0,0 +1,68 @@
+"""Shared helpers for the eval runner/grader/report (no app imports)."""
+
+from __future__ import annotations
+
+import json
+from pathlib import Path
+from typing import Any, Iterable
+
+BATTERY_TYPES = ("singleturn", "sessions", "personas", "replay")
+# Identifying key per battery type (see data/eval/README.md schemas).
+_TYPE_KEYS = {
+    "case_id": "singleturn",
+    "session_id": "sessions",
+    "persona_id": "personas",
+    "replay_id": "replay",
+}
+
+
+def load_jsonl(path: str | Path) -> list[dict[str, Any]]:
+    with open(path, encoding="utf-8") as f:
+        return [json.loads(line) for line in f if line.strip()]
+
+
+def append_jsonl(path: str | Path, rows: Iterable[dict[str, Any]]) -> None:
+    with open(path, "a", encoding="utf-8") as f:
+        for row in rows:
+            f.write(json.dumps(row, ensure_ascii=False) + "\n")
+
+
+def write_jsonl(path: str | Path, rows: Iterable[dict[str, Any]]) -> None:
+    with open(path, "w", encoding="utf-8") as f:
+        for row in rows:
+            f.write(json.dumps(row, ensure_ascii=False) + "\n")
+
+
+def detect_battery_type(records: list[dict[str, Any]]) -> str:
+    if not records:
+        raise ValueError("Battery file is empty.")
+    for key, battery_type in _TYPE_KEYS.items():
+        if key in records[0]:
+            return battery_type
+    raise ValueError(
+        f"Unrecognized battery schema; expected one of {sorted(_TYPE_KEYS)} "
+        f"in the first record, got {sorted(records[0])}"
+    )
+
+
+def normalize_url(url: str | None) -> str:
+    """Comparable form for lesson-URL matching (recall ground truth).
+
+    Battery `lesson_url`s point at the discussion on the lesson page
+    (`.../<lesson>/discussions/<post_id>`); retrieval matches carry the bare
+    lesson URL, so the discussion suffix is stripped before comparing.
+    """
+    if not url:
+        return ""
+    url = url.strip().lower().split("#", 1)[0].split("?", 1)[0]
+    url = url.split("/discussions/", 1)[0]
+    return url.rstrip("/")
+
+
+def percentile(values: list[float], pct: float) -> float | None:
+    """Nearest-rank percentile; None on empty input."""
+    if not values:
+        return None
+    ordered = sorted(values)
+    rank = max(0, min(len(ordered) - 1, round(pct / 100 * (len(ordered) - 1))))
+    return ordered[rank]

evals/grade.py

@@ -0,0 +1,342 @@
+"""Grade trace bundles: code checks now, a hand-grade sheet for the rest.
+
+  uv run -m evals.grade --run runs/<exp>                 # auto grades + sheet
+  uv run -m evals.grade --run runs/<exp> --handgrades runs/<exp>/handgrade_filled.csv
+
+Pure JSON/CSV in and out — no app imports — so old bundles re-grade forever.
+
+Outputs in the run dir:
+- grades_auto.jsonl  — one row per bundle row with every code-computable check
+  (retrieval recall, behavior heuristics, citation presence, persona regex
+  checks, probe trigger context). See data/eval/README.md for term definitions.
+- handgrade_sheet.csv — one row per pending human judgment (key points, session
+  probes, replay replies, persona llm-checks). Fill `grade` with pass/fail
+  (optionally a note), save as handgrade_filled.csv, re-run with --handgrades.
+- grades_merged.jsonl — auto + human grades joined, ready for evals.report.
+
+Behavior checks here are HEURISTIC proxies (regex/trajectory); the reportable
+behavior-accuracy number comes from the hand grades.
+"""
+
+from __future__ import annotations
+
+import argparse
+import csv
+import re
+from pathlib import Path
+from typing import Any
+
+from .common import load_jsonl, normalize_url, write_jsonl
+
+BEHAVIOR_HEURISTICS = {
+    "redirect_to_support": r"support|academy team|reach out|contact",
+    "acknowledge_feedback": r"thank|feedback|appreciate|noted|passed (this|it) (on|along)",
+}
+RETRIEVAL_TOOLS = {"retrieve_tutor_context", "run_kb_command"}
+
+
+def index_battery(battery: list[dict[str, Any]]) -> dict[str, dict[str, Any]]:
+    index: dict[str, dict[str, Any]] = {}
+    for record in battery:
+        for key in ("case_id", "session_id", "replay_id"):
+            if key in record:
+                index[record[key]] = record
+        if "persona_id" in record:
+            for question in record["questions"]:
+                index[question["question_id"]] = {**question, "_persona": record}
+    return index
+
+
+def retrieval_metrics(
+    bundle: dict[str, Any], source_key: str | None, lesson_url: str | None
+) -> dict[str, Any]:
+    """Recall/MRR over the (post-rerank) matches of retrieval tool calls.
+
+    Matches are the reranked top-k the agent actually saw, so 'recall' here is
+    recall@top-k-shown. MRR ranks across calls in order.
+    """
+    matches: list[dict[str, Any]] = []
+    called_retrieval = False
+    for call in bundle.get("tool_calls") or []:
+        if call.get("tool_name") in RETRIEVAL_TOOLS:
+            called_retrieval = True
+        if call.get("tool_name") == "retrieve_tutor_context":
+            matches.extend(call.get("matches") or [])
+    lesson = normalize_url(lesson_url)
+    source_hit = any(m.get("source_key") == source_key for m in matches)
+    lesson_rank = next(
+        (
+            rank
+            for rank, m in enumerate(matches, start=1)
+            if lesson and normalize_url(m.get("url")) == lesson
+        ),
+        0,
+    )
+    return {
+        "called_retrieval": called_retrieval,
+        "retrieved_matches": len(matches),
+        "recall_source": source_hit if matches or called_retrieval else None,
+        "recall_lesson": bool(lesson_rank) if lesson else None,
+        "mrr_lesson": (1.0 / lesson_rank) if lesson_rank else 0.0 if lesson else None,
+    }
+
+
+def behavior_heuristic(expected: str, bundle: dict[str, Any]) -> bool | None:
+    """Cheap proxy for behavior routing; None = no heuristic for this class."""
+    answer = bundle.get("answer") or ""
+    if expected == "answer_from_corpus":
+        return any(
+            call.get("tool_name") in RETRIEVAL_TOOLS
+            for call in bundle.get("tool_calls") or []
+        )
+    pattern = BEHAVIOR_HEURISTICS.get(expected)
+    if pattern is None:
+        return None
+    return re.search(pattern, answer, re.IGNORECASE) is not None
+
+
+def grade_persona_question(question: dict[str, Any], answer: str) -> dict[str, Any]:
+    """Apply the battery's self-grading checks (see data/eval/README.md)."""
+    results, needs_judgment = [], False
+    for check in question.get("checks") or []:
+        if check["type"] == "regex_any":
+            passed = any(
+                re.search(pattern, answer, re.IGNORECASE)
+                for pattern in check["patterns"]
+            )
+            results.append({"type": "regex_any", "passed": passed})
+        elif check["type"] == "llm":
+            needs_judgment = True
+            results.append({"type": "llm", "passed": None})
+    anti_hits = [
+        pattern
+        for pattern in question.get("anti_patterns") or []
+        if re.search(pattern, answer, re.IGNORECASE)
+    ]
+    decided = [r["passed"] for r in results if r["passed"] is not None]
+    auto_pass: bool | None
+    if anti_hits:
+        auto_pass = False
+    elif needs_judgment:
+        auto_pass = None  # resolved by the hand/judge grade
+    else:
+        auto_pass = all(decided) if decided else None
+    return {
+        "checks": results,
+        "anti_pattern_hits": anti_hits,
+        "needs_judgment": needs_judgment,
+        "auto_pass": auto_pass,
+    }
+
+
+def sheet_row(
+    *,
+    bundle: dict[str, Any],
+    item_type: str,
+    criterion: str,
+    reference: str = "",
+) -> dict[str, str]:
+    return {
+        "sheet_row_id": f"{bundle['run_id']}|{item_type}|{abs(hash(criterion)) % 10**8}",
+        "run_id": bundle["run_id"],
+        "battery_type": bundle["battery_type"],
+        "preset": bundle["preset"],
+        "item_type": item_type,
+        "question": bundle["query"][:600],
+        "answer": (bundle.get("answer") or "")[:4000],
+        "criterion": criterion,
+        "reference": reference[:2000],
+        "grade": "",
+        "note": "",
+    }
+
+
+def grade_run(run_dir: Path) -> tuple[list[dict[str, Any]], list[dict[str, str]]]:
+    bundles = load_jsonl(run_dir / "bundles.jsonl")
+    if not bundles:
+        raise SystemExit(f"No bundles in {run_dir}")
+    battery = load_jsonl(bundles[0]["battery_path"])
+    by_id = index_battery(battery)
+
+    grades: list[dict[str, Any]] = []
+    sheet: list[dict[str, str]] = []
+    for bundle in bundles:
+        record = by_id.get(bundle["unit_id"], {})
+        stats = bundle.get("context_stats") or {}
+        row: dict[str, Any] = {
+            "run_id": bundle["run_id"],
+            "unit_id": bundle["unit_id"],
+            "battery_type": bundle["battery_type"],
+            "preset": bundle["preset"],
+            "model": bundle["model"],
+            "trial": bundle["trial"],
+            "turn_index": bundle.get("turn_index"),
+            "error": bundle.get("error"),
+            "answer_chars": len(bundle.get("answer") or ""),
+            "tool_call_count": len(bundle.get("tool_calls") or []),
+            "has_citation": bool(bundle.get("resolved_sources")),
+            "ttft_ms": stats.get("ttft_ms"),
+            "total_ms": stats.get("total_ms"),
+            "input_tokens": stats.get("input_tokens"),
+            "output_tokens": stats.get("output_tokens"),
+            "est_cost_usd": stats.get("est_cost_usd"),
+            "llm_calls": stats.get("llm_calls"),
+            "context_tokens_approx": stats.get("context_tokens_approx"),
+            "summary_messages": stats.get("summary_messages"),
+            "cleared_tool_outputs": stats.get("cleared_tool_outputs"),
+        }
+        if bundle.get("error"):
+            grades.append(row)
+            continue
+
+        if bundle["battery_type"] == "singleturn":
+            row.update(
+                retrieval_metrics(
+                    bundle, record.get("source_key"), record.get("lesson_url")
+                )
+            )
+            expected = record.get("expected_behavior", "")
+            row["expected_behavior"] = expected
+            row["behavior_heuristic"] = behavior_heuristic(expected, bundle)
+            for point in record.get("key_points") or []:
+                sheet.append(
+                    sheet_row(
+                        bundle=bundle,
+                        item_type="key_point",
+                        criterion=point,
+                        reference=record.get("reference_answer") or "",
+                    )
+                )
+            if expected:
+                sheet.append(
+                    sheet_row(
+                        bundle=bundle,
+                        item_type="behavior",
+                        criterion=f"Did the tutor do the right thing for "
+                        f"'{expected}'? (see README definitions)",
+                        reference=record.get("reference_answer") or "",
+                    )
+                )
+
+        elif bundle["battery_type"] == "sessions":
+            probe = next(
+                (
+                    p
+                    for p in record.get("probes", [])
+                    if p["turn_index"] == bundle.get("turn_index")
+                ),
+                None,
+            )
+            row["is_probe"] = probe is not None
+            if probe:
+                row["probe_type"] = probe["probe_type"]
+                # Compression context at probe time: a memory eval where the
+                # triggers never fired measures nothing (README warning).
+                row["compaction_active"] = bool(
+                    (stats.get("summary_messages") or 0)
+                    or (stats.get("cleared_tool_outputs") or 0)
+                )
+                sheet.append(
+                    sheet_row(
+                        bundle=bundle,
+                        item_type=f"probe:{probe['probe_type']}",
+                        criterion=f"Expected facts: {probe['expected_facts']} | "
+                        f"Rule: {probe['check_note']}",
+                    )
+                )
+
+        elif bundle["battery_type"] == "personas":
+            result = grade_persona_question(record, bundle.get("answer") or "")
+            row.update(
+                {
+                    "persona_id": (record.get("_persona") or {}).get("persona_id"),
+                    "auto_pass": result["auto_pass"],
+                    "anti_pattern_hits": result["anti_pattern_hits"],
+                    "needs_judgment": result["needs_judgment"],
+                }
+            )
+            if result["needs_judgment"] and not result["anti_pattern_hits"]:
+                llm_checks = [
+                    c["instruction"]
+                    for c in record.get("checks", [])
+                    if c["type"] == "llm"
+                ]
+                sheet.append(
+                    sheet_row(
+                        bundle=bundle,
+                        item_type="persona_llm_check",
+                        criterion=" | ".join(llm_checks),
+                    )
+                )
+
+        elif bundle["battery_type"] == "replay":
+            sheet.append(
+                sheet_row(
+                    bundle=bundle,
+                    item_type="replay_reply",
+                    criterion="Is this reply as helpful and correct as the real "
+                    "staff reply? (binary)",
+                    reference=record.get("reference_reply") or "",
+                )
+            )
+        grades.append(row)
+    return grades, sheet
+
+
+def merge_handgrades(
+    grades: list[dict[str, Any]], filled_csv: Path
+) -> list[dict[str, Any]]:
+    human: dict[str, list[dict[str, str]]] = {}
+    with open(filled_csv, encoding="utf-8") as f:
+        for row in csv.DictReader(f):
+            if row.get("grade", "").strip():
+                human.setdefault(row["run_id"], []).append(row)
+    for grade in grades:
+        rows = human.get(grade["run_id"], [])
+        if not rows:
+            continue
+        key_points = [r for r in rows if r["item_type"] == "key_point"]
+        if key_points:
+            passed = sum(1 for r in key_points if r["grade"].lower() == "pass")
+            grade["key_points_passed"] = passed
+            grade["key_points_total"] = len(key_points)
+        for row in rows:
+            verdict = row["grade"].strip().lower() == "pass"
+            if row["item_type"] == "behavior":
+                grade["behavior_pass"] = verdict
+            elif row["item_type"].startswith("probe:"):
+                grade["probe_pass"] = verdict
+            elif row["item_type"] == "persona_llm_check":
+                # Combines with the regex auto result (both must pass).
+                grade["auto_pass"] = verdict and grade.get("auto_pass") is not False
+            elif row["item_type"] == "replay_reply":
+                grade["replay_pass"] = verdict
+    return grades
+
+
+def main() -> None:
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument("--run", required=True, help="Run dir with bundles.jsonl")
+    parser.add_argument("--handgrades", default="", help="Filled handgrade CSV")
+    args = parser.parse_args()
+    run_dir = Path(args.run)
+
+    grades, sheet = grade_run(run_dir)
+    write_jsonl(run_dir / "grades_auto.jsonl", grades)
+    if sheet:
+        with open(run_dir / "handgrade_sheet.csv", "w", encoding="utf-8") as f:
+            writer = csv.DictWriter(f, fieldnames=list(sheet[0]))
+            writer.writeheader()
+            writer.writerows(sheet)
+    print(
+        f"{len(grades)} rows auto-graded; {len(sheet)} human judgments pending "
+        f"in handgrade_sheet.csv"
+    )
+    if args.handgrades:
+        merged = merge_handgrades(grades, Path(args.handgrades))
+        write_jsonl(run_dir / "grades_merged.jsonl", merged)
+        print(f"Merged human grades -> {run_dir}/grades_merged.jsonl")
+
+
+if __name__ == "__main__":
+    main()

evals/handgrade_workbook.py

@@ -0,0 +1,191 @@
+"""Build and merge the blinded hand-grading workbook for a bake-off.
+
+  uv run -m evals.handgrade_workbook build --dirs runs/b_st_* runs/b_pe_* \
+      runs/b_se_* --out runs/b_report/workbook.csv
+  uv run -m evals.handgrade_workbook merge --workbook runs/b_report/workbook_filled.csv
+
+Build collects the human-judgment rows from each run dir's handgrade_sheet.csv
+into ONE shuffled, BLINDED workbook: the grader sees question/answer/criterion
+but never the preset, so grades can't favor a method. A key map (written next
+to the workbook) links each blinded row back to its run dir for the merge.
+
+Row selection (keeps the pass to a half-day):
+- priority 1: ALL session probe rows (the memory metric).
+- priority 2: ALL persona llm-check rows.
+- priority 3: single-turn key_point + behavior rows for a stratified subset of
+  cases (default 12: 6 corpus / 2 redirect / 2 feedback / 2 general), trial 1.
+
+Grade with `pass` or `fail` in the grade column (notes optional). Merge splits
+the filled workbook into per-run-dir handgrade_filled.csv files and prints the
+evals.grade commands that fold them into grades_merged.jsonl.
+"""
+
+from __future__ import annotations
+
+import argparse
+import csv
+import random
+from collections import defaultdict
+from pathlib import Path
+
+from .common import load_jsonl
+
+VISIBLE_FIELDS = (
+    "key",
+    "priority",
+    "item_type",
+    "question",
+    "answer",
+    "criterion",
+    "reference",
+    "grade",
+    "note",
+)
+
+
+def read_sheet(run_dir: Path) -> list[dict[str, str]]:
+    path = run_dir / "handgrade_sheet.csv"
+    if not path.exists():
+        return []
+    with open(path, encoding="utf-8") as f:
+        return [dict(row, _dir=str(run_dir)) for row in csv.DictReader(f)]
+
+
+def singleturn_subset(battery_path: str, per_behavior: dict[str, int]) -> set[str]:
+    by_behavior: dict[str, list[str]] = defaultdict(list)
+    for case in load_jsonl(battery_path):
+        by_behavior[case["expected_behavior"]].append(case["case_id"])
+    subset: set[str] = set()
+    for behavior, count in per_behavior.items():
+        subset.update(sorted(by_behavior.get(behavior, []))[:count])
+    return subset
+
+
+def build(args: argparse.Namespace) -> None:
+    per_behavior = {
+        "answer_from_corpus": 6,
+        "redirect_to_support": 2,
+        "acknowledge_feedback": 2,
+        "answer_general": 2,
+    }
+    selected: list[dict[str, str]] = []
+    st_subset: set[str] | None = None
+    for run_dir in map(Path, args.dirs):
+        for row in read_sheet(run_dir):
+            item_type = row["item_type"]
+            if item_type.startswith("probe:"):
+                row["priority"] = "1"
+            elif item_type == "persona_llm_check":
+                row["priority"] = "2"
+            elif item_type in ("key_point", "behavior"):
+                if st_subset is None:
+                    bundles = load_jsonl(run_dir / "bundles.jsonl")
+                    st_subset = singleturn_subset(
+                        bundles[0]["battery_path"], per_behavior
+                    )
+                unit_id, _, rest = row["run_id"].partition("|")
+                if unit_id not in st_subset or not rest.endswith("t1"):
+                    continue
+                row["priority"] = "3"
+            else:  # replay rows etc. — not part of this pass
+                continue
+            selected.append(row)
+
+    random.Random(42).shuffle(selected)
+    out = Path(args.out)
+    out.parent.mkdir(parents=True, exist_ok=True)
+    keymap_path = out.with_name(out.stem + "_keymap.csv")
+    with (
+        open(out, "w", encoding="utf-8") as wf,
+        open(keymap_path, "w", encoding="utf-8") as kf,
+    ):
+        writer = csv.DictWriter(wf, fieldnames=list(VISIBLE_FIELDS))
+        writer.writeheader()
+        key_writer = csv.writer(kf)
+        key_writer.writerow(["key", "run_dir", "sheet_row_id"])
+        for index, row in enumerate(selected):
+            key = f"g{index:04d}"
+            writer.writerow(
+                {
+                    "key": key,
+                    "priority": row["priority"],
+                    "item_type": row["item_type"],
+                    "question": row["question"],
+                    "answer": row["answer"],
+                    "criterion": row["criterion"],
+                    "reference": row["reference"],
+                    "grade": "",
+                    "note": "",
+                }
+            )
+            key_writer.writerow([key, row["_dir"], row["sheet_row_id"]])
+    counts = defaultdict(int)
+    for row in selected:
+        counts[row["priority"]] += 1
+    print(f"{len(selected)} blinded rows -> {out}")
+    print(f"  priority 1 (session probes): {counts['1']}")
+    print(f"  priority 2 (persona llm checks): {counts['2']}")
+    print(f"  priority 3 (single-turn subset): {counts['3']}")
+    print(f"key map (do not open while grading): {keymap_path}")
+
+
+def merge(args: argparse.Namespace) -> None:
+    workbook = Path(args.workbook)
+    keymap_path = Path(args.keymap) if args.keymap else None
+    if keymap_path is None:
+        stem = workbook.stem.replace("_filled", "")
+        keymap_path = workbook.with_name(stem + "_keymap.csv")
+    with open(keymap_path, encoding="utf-8") as f:
+        key_map = {row["key"]: row for row in csv.DictReader(f)}
+
+    by_dir: dict[str, dict[str, dict[str, str]]] = defaultdict(dict)
+    graded = 0
+    with open(workbook, encoding="utf-8") as f:
+        for row in csv.DictReader(f):
+            if not row.get("grade", "").strip():
+                continue
+            mapping = key_map[row["key"]]
+            by_dir[mapping["run_dir"]][mapping["sheet_row_id"]] = row
+            graded += 1
+
+    for run_dir, grades in by_dir.items():
+        sheet = read_sheet(Path(run_dir))
+        out_rows = []
+        for sheet_row in sheet:
+            filled = grades.get(sheet_row["sheet_row_id"])
+            if filled:
+                sheet_row["grade"] = filled["grade"].strip().lower()
+                sheet_row["note"] = filled.get("note", "")
+            sheet_row.pop("_dir", None)
+            out_rows.append(sheet_row)
+        out_path = Path(run_dir) / "handgrade_filled.csv"
+        with open(out_path, "w", encoding="utf-8") as f:
+            writer = csv.DictWriter(f, fieldnames=list(out_rows[0]))
+            writer.writeheader()
+            writer.writerows(out_rows)
+        print(
+            f"{run_dir}: {len(grades)} grade(s) -> {out_path}\n"
+            f"  next: uv run -m evals.grade --run {run_dir} "
+            f"--handgrades {out_path}"
+        )
+    print(f"total grades merged: {graded}")
+
+
+def main() -> None:
+    parser = argparse.ArgumentParser(description=__doc__)
+    sub = parser.add_subparsers(dest="cmd", required=True)
+    build_p = sub.add_parser("build")
+    build_p.add_argument("--dirs", nargs="+", required=True)
+    build_p.add_argument("--out", required=True)
+    merge_p = sub.add_parser("merge")
+    merge_p.add_argument("--workbook", required=True)
+    merge_p.add_argument("--keymap", default="")
+    args = parser.parse_args()
+    if args.cmd == "build":
+        build(args)
+    else:
+        merge(args)
+
+
+if __name__ == "__main__":
+    main()

evals/report.py

@@ -0,0 +1,276 @@
+"""Build the comparison report from one or more graded runs.
+
+  uv run -m evals.report --runs runs/bake1_singleturn_prod \
+      runs/bake1_singleturn_full_history --out runs/bake1_report
+
+Each run dir = one battery x preset x model. Runs sharing a battery_type are
+shown side by side (columns = presets) — the bake-off table. Also emits
+tokens_by_turn.csv (cumulative input tokens per session turn per preset: the
+signature plot) and tokens_by_turn.png when matplotlib is available.
+
+Reads grades_merged.jsonl when present (hand grades included), else
+grades_auto.jsonl; quality rows (key-point coverage, probe accuracy, behavior
+accuracy) appear only once hand grades are merged.
+"""
+
+from __future__ import annotations
+
+import argparse
+import csv
+from collections import defaultdict
+from pathlib import Path
+from statistics import mean
+from typing import Any
+
+from .common import load_jsonl, percentile
+
+
+def load_run(run_dir: Path) -> dict[str, Any]:
+    grades_path = run_dir / "grades_merged.jsonl"
+    if not grades_path.exists():
+        grades_path = run_dir / "grades_auto.jsonl"
+    grades = load_jsonl(grades_path)
+    if not grades:
+        raise SystemExit(f"No grades in {run_dir}; run evals.grade first.")
+    return {
+        "dir": run_dir,
+        "label": f"{grades[0]['preset']}",
+        "battery_type": grades[0]["battery_type"],
+        "model": grades[0]["model"],
+        "grades": grades,
+        "merged": grades_path.name == "grades_merged.jsonl",
+    }
+
+
+def rate(values: list[Any]) -> str:
+    """pass-rate over boolean-like values, ignoring None."""
+    decided = [v for v in values if v is not None]
+    if not decided:
+        return "—"
+    return f"{sum(1 for v in decided if v) / len(decided):.0%} (n={len(decided)})"
+
+
+def fmt_ms(values: list[Any]) -> str:
+    nums = [v for v in values if isinstance(v, (int, float))]
+    if not nums:
+        return "—"
+    return f"{percentile(nums, 50):.0f} / {percentile(nums, 95):.0f}"
+
+
+def fmt_mean(values: list[Any], spec: str = "{:.0f}") -> str:
+    nums = [v for v in values if isinstance(v, (int, float))]
+    return spec.format(mean(nums)) if nums else "—"
+
+
+def col(grades: list[dict[str, Any]], key: str) -> list[Any]:
+    return [g.get(key) for g in grades]
+
+
+def metric_rows(battery_type: str, runs: list[dict[str, Any]]) -> list[list[str]]:
+    """One row per metric, one column per run (preset)."""
+    rows: list[list[str]] = []
+
+    def add(name: str, fn) -> None:
+        rows.append([name] + [fn(run["grades"]) for run in runs])
+
+    add("cases run", lambda g: str(len({x["unit_id"] for x in g})))
+    add("errors", lambda g: str(sum(1 for x in g if x.get("error"))))
+    add("ttft ms p50/p95", lambda g: fmt_ms(col(g, "ttft_ms")))
+    add("turn ms p50/p95", lambda g: fmt_ms(col(g, "total_ms")))
+    add("input tok/turn", lambda g: fmt_mean(col(g, "input_tokens")))
+    add("output tok/turn", lambda g: fmt_mean(col(g, "output_tokens")))
+    add(
+        "est cost/turn $",
+        lambda g: fmt_mean(col(g, "est_cost_usd"), "{:.4f}"),
+    )
+    add("llm calls/turn", lambda g: fmt_mean(col(g, "llm_calls"), "{:.1f}"))
+    add("tool calls/turn", lambda g: fmt_mean(col(g, "tool_call_count"), "{:.1f}"))
+
+    if battery_type == "singleturn":
+        add(
+            "retrieval called (corpus)",
+            lambda g: rate(
+                [
+                    x.get("called_retrieval")
+                    for x in g
+                    if x.get("expected_behavior") == "answer_from_corpus"
+                ]
+            ),
+        )
+        add("recall@shown source", lambda g: rate(col(g, "recall_source")))
+        add("recall@shown lesson", lambda g: rate(col(g, "recall_lesson")))
+        add(
+            "MRR lesson", lambda g: fmt_mean([x.get("mrr_lesson") for x in g], "{:.2f}")
+        )
+        add(
+            "citation present",
+            lambda g: rate(
+                [
+                    x.get("has_citation")
+                    for x in g
+                    if x.get("expected_behavior") == "answer_from_corpus"
+                ]
+            ),
+        )
+        add("behavior heuristic", lambda g: rate(col(g, "behavior_heuristic")))
+        add("behavior (hand)", lambda g: rate(col(g, "behavior_pass")))
+        add("key-point coverage (hand)", lambda g: _kp_coverage(g))
+
+    if battery_type == "sessions":
+        add("cumulative input tok (last turn, mean)", _final_cumulative)
+        add(
+            "compaction active at probes",
+            lambda g: rate(
+                [x.get("compaction_active") for x in g if x.get("is_probe")]
+            ),
+        )
+        add(
+            "probe accuracy (hand)",
+            lambda g: rate([x.get("probe_pass") for x in g if x.get("is_probe")]),
+        )
+        for probe_type in sorted(
+            {
+                x.get("probe_type")
+                for run in runs
+                for x in run["grades"]
+                if x.get("probe_type")
+            }
+        ):
+            rows.append(
+                [f"  └ {probe_type}"]
+                + [
+                    rate(
+                        [
+                            x.get("probe_pass")
+                            for x in run["grades"]
+                            if x.get("probe_type") == probe_type
+                        ]
+                    )
+                    for run in runs
+                ]
+            )
+
+    if battery_type == "personas":
+        add("personalization pass (auto)", lambda g: rate(col(g, "auto_pass")))
+        add(
+            "anti-pattern failures",
+            lambda g: str(sum(1 for x in g if x.get("anti_pattern_hits"))),
+        )
+
+    if battery_type == "replay":
+        add("replay reply pass (hand)", lambda g: rate(col(g, "replay_pass")))
+    return rows
+
+
+def _kp_coverage(grades: list[dict[str, Any]]) -> str:
+    passed = sum(g.get("key_points_passed", 0) for g in grades)
+    total = sum(g.get("key_points_total", 0) for g in grades)
+    return f"{passed / total:.0%} ({passed}/{total})" if total else "—"
+
+
+def _final_cumulative(grades: list[dict[str, Any]]) -> str:
+    finals: dict[tuple[str, int], int] = {}
+    for g in grades:
+        tokens = g.get("input_tokens")
+        if g.get("turn_index") is None or not isinstance(tokens, (int, float)):
+            continue
+        key = (g["unit_id"], g["trial"])
+        finals[key] = finals.get(key, 0) + int(tokens)
+    return fmt_mean(list(finals.values()))
+
+
+def write_token_curves(runs: list[dict[str, Any]], out_dir: Path) -> None:
+    session_runs = [r for r in runs if r["battery_type"] == "sessions"]
+    if not session_runs:
+        return
+    rows = []
+    for run in session_runs:
+        cumulative: dict[tuple[str, int], int] = defaultdict(int)
+        for g in sorted(
+            run["grades"],
+            key=lambda x: (x["unit_id"], x["trial"], x["turn_index"] or 0),
+        ):
+            if g.get("turn_index") is None:
+                continue
+            key = (g["unit_id"], g["trial"])
+            cumulative[key] += int(g.get("input_tokens") or 0)
+            rows.append(
+                {
+                    "preset": run["label"],
+                    "session_id": g["unit_id"],
+                    "trial": g["trial"],
+                    "turn_index": g["turn_index"],
+                    "turn_input_tokens": g.get("input_tokens") or 0,
+                    "cumulative_input_tokens": cumulative[key],
+                }
+            )
+    path = out_dir / "tokens_by_turn.csv"
+    with open(path, "w", encoding="utf-8") as f:
+        writer = csv.DictWriter(f, fieldnames=list(rows[0]))
+        writer.writeheader()
+        writer.writerows(rows)
+    try:
+        _plot_token_curves(rows, out_dir)
+    except ImportError:
+        print("matplotlib not installed; wrote CSV only.")
+
+
+def _plot_token_curves(rows: list[dict[str, Any]], out_dir: Path) -> None:
+    import matplotlib
+
+    matplotlib.use("Agg")
+    import matplotlib.pyplot as plt
+
+    by_preset: dict[str, dict[int, list[int]]] = defaultdict(lambda: defaultdict(list))
+    for row in rows:
+        by_preset[row["preset"]][row["turn_index"]].append(
+            row["cumulative_input_tokens"]
+        )
+    fig, ax = plt.subplots(figsize=(8, 5))
+    for preset, by_turn in sorted(by_preset.items()):
+        turns = sorted(by_turn)
+        ax.plot(turns, [mean(by_turn[t]) for t in turns], marker="o", label=preset)
+    ax.set_xlabel("turn")
+    ax.set_ylabel("cumulative input tokens (mean across sessions)")
+    ax.set_title("Context cost per turn by memory preset")
+    ax.legend()
+    fig.tight_layout()
+    fig.savefig(out_dir / "tokens_by_turn.png", dpi=150)
+
+
+def main() -> None:
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument("--runs", nargs="+", required=True)
+    parser.add_argument("--out", default="", help="Report dir (default: first run)")
+    args = parser.parse_args()
+    runs = [load_run(Path(r)) for r in args.runs]
+    out_dir = Path(args.out or runs[0]["dir"])
+    out_dir.mkdir(parents=True, exist_ok=True)
+
+    lines = ["# Eval report", ""]
+    models = {run["model"] for run in runs}
+    lines.append(
+        f"Model(s): {', '.join(sorted(models))}. Runs missing hand grades show "
+        "— for quality rows (fill handgrade_sheet.csv, re-run evals.grade "
+        "with --handgrades)."
+    )
+    by_type: dict[str, list[dict[str, Any]]] = defaultdict(list)
+    for run in runs:
+        by_type[run["battery_type"]].append(run)
+    for battery_type, type_runs in by_type.items():
+        lines.append(f"\n## {battery_type}\n")
+        header = ["metric"] + [
+            run["label"] + ("" if run["merged"] else " (auto only)")
+            for run in type_runs
+        ]
+        lines.append("| " + " | ".join(header) + " |")
+        lines.append("|" + "---|" * len(header))
+        for row in metric_rows(battery_type, type_runs):
+            lines.append("| " + " | ".join(row) + " |")
+    (out_dir / "report.md").write_text("\n".join(lines) + "\n")
+    write_token_curves(runs, out_dir)
+    print(f"Report written to {out_dir}/report.md")
+
+
+if __name__ == "__main__":
+    main()

evals/run_battery.py

@@ -0,0 +1,454 @@
+"""Run an eval battery against the tutor and persist one trace bundle per turn.
+
+Examples:
+  uv run -m evals.run_battery --battery data/eval/battery_singleturn_v1.jsonl \
+      --preset prod --out runs/bake1_singleturn_prod
+  uv run -m evals.run_battery --battery data/eval/battery_sessions_v1.jsonl \
+      --preset full_history --ids s01_fullstack_beginner_13t --trials 2
+
+Notes:
+- LangSmith tracing is OFF by default (free-plan quota); --langsmith enables it.
+- Web tools are OFF by default for reproducibility; --enable-tools to add.
+- Re-running with the same --out resumes: completed cases (all trials of all
+  turns) are kept; incomplete sessions are re-run whole, because thread state
+  lives in process memory and cannot be resumed across runs.
+- Bundles are the durable artifact: grade/report re-run offline against them.
+"""
+
+from __future__ import annotations
+
+import argparse
+import asyncio
+import datetime
+import json
+import logging
+import os
+import time
+from collections import defaultdict
+from pathlib import Path
+from typing import Any
+
+from .common import append_jsonl, detect_battery_type, load_jsonl, write_jsonl
+
+logger = logging.getLogger("evals.run_battery")
+
+# Full tool outputs can be 40k chars each; keep bundles browsable. The full
+# size is preserved in output_chars so truncation is visible.
+TOOL_OUTPUT_MAX_CHARS = 6_000
+# Slowest observed turn is ~2.5 min; anything past this is a wedged stream
+# (e.g. laptop sleep killed the connection mid-turn). The turn records a
+# TimeoutError and the unit re-runs on resume instead of hanging forever.
+TURN_TIMEOUT_SECONDS = 600
+
+
+def parse_args() -> argparse.Namespace:
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument("--battery", required=True, help="Path to a battery JSONL.")
+    parser.add_argument("--preset", default="prod", help="Memory preset name.")
+    parser.add_argument("--model", default="", help="Model id (default: app default).")
+    parser.add_argument("--trials", type=int, default=1)
+    parser.add_argument("--out", default="", help="Output dir (default: derived).")
+    parser.add_argument("--limit", type=int, default=0, help="First N cases only.")
+    parser.add_argument(
+        "--ids", nargs="*", default=[], help="Run only these case/session/persona ids."
+    )
+    parser.add_argument("--concurrency", type=int, default=4)
+    parser.add_argument(
+        "--scope-sources",
+        action="store_true",
+        help="Restrict the tutor to the case's own course (default: all sources, "
+        "like production).",
+    )
+    parser.add_argument(
+        "--enable-tools", nargs="*", default=[], help="e.g. web_search url_context"
+    )
+    parser.add_argument("--langsmith", action="store_true", help="Enable tracing.")
+    return parser.parse_args()
+
+
+def record_id(record: dict[str, Any]) -> str:
+    for key in ("case_id", "session_id", "persona_id", "replay_id"):
+        if key in record:
+            return str(record[key])
+    raise KeyError("record has no id")
+
+
+class BundleSink:
+    """Append-only bundle store with resume bookkeeping."""
+
+    def __init__(self, path: Path):
+        self.path = path
+        self.lock = asyncio.Lock()
+
+    async def write(self, rows: list[dict[str, Any]]) -> None:
+        async with self.lock:
+            await asyncio.to_thread(append_jsonl, self.path, rows)
+
+
+def prune_incomplete(
+    path: Path, battery_type: str, expected_turns: dict[str, int], trials: int
+) -> set[tuple[str, int]]:
+    """Drop partial work units from a previous run; return completed (id, trial).
+
+    A work unit is one trial of one case (or one whole session). Sessions with
+    missing turns are pruned entirely: their thread state died with the old
+    process, so they must re-run from turn 0.
+    """
+    if not path.exists():
+        return set()
+    rows = load_jsonl(path)
+    by_unit: dict[tuple[str, int], list[dict[str, Any]]] = {}
+    for row in rows:
+        by_unit.setdefault((row["unit_id"], row["trial"]), []).append(row)
+    completed = set()
+    for (unit_id, trial), unit_rows in by_unit.items():
+        wanted = expected_turns.get(unit_id, 1)
+        if len(unit_rows) >= wanted and not any(r.get("error") for r in unit_rows):
+            completed.add((unit_id, trial))
+    kept = [r for r in rows if (r["unit_id"], r["trial"]) in completed]
+    if len(kept) != len(rows):
+        write_jsonl(path, kept)
+        logger.info(
+            "Resume: kept %d completed unit(s), pruned %d partial row(s).",
+            len(completed),
+            len(rows) - len(kept),
+        )
+    return completed
+
+
+def make_bundle(
+    *,
+    args: argparse.Namespace,
+    battery_type: str,
+    unit_id: str,
+    trial: int,
+    turn_index: int | None,
+    query: str,
+    result: dict[str, Any],
+) -> dict[str, Any]:
+    return {
+        "run_id": f"{unit_id}|turn{turn_index if turn_index is not None else 0}"
+        f"|t{trial}",
+        "unit_id": unit_id,
+        "battery_path": args.battery,
+        "battery_type": battery_type,
+        "preset": args.preset,
+        "model": result["model"],
+        "trial": trial,
+        "turn_index": turn_index,
+        "started_at": result["started_at"],
+        "duration_ms": result["duration_ms"],
+        "query": query,
+        "answer": result["answer"],
+        "thread_id": result["thread_id"],
+        "tool_calls": result["tool_calls"],
+        "resolved_sources": result["resolved_sources"],
+        "context_stats": result["context_stats"],
+        "error": result["error"],
+    }
+
+
+async def run_turn(request: Any) -> dict[str, Any]:
+    """Drive one stream_chat turn and collect everything the graders need."""
+    from app.chat_service import stream_chat
+    from app.config import DEFAULT_MODEL_NAME
+
+    started_at = datetime.datetime.now(datetime.UTC).isoformat(timespec="seconds")
+    started = time.monotonic()
+    tool_calls: list[dict[str, Any]] = []
+    resolved_sources: list[dict[str, Any]] = []
+    answer, thread_id, error = "", "", None
+    context_stats: dict[str, Any] | None = None
+    try:
+        async with asyncio.timeout(TURN_TIMEOUT_SECONDS):
+            async for event in stream_chat(request):
+                if event.type == "thread_started":
+                    thread_id = str(event.data.get("thread_id", ""))
+                elif event.type == "tool_call_completed":
+                    data = event.data
+                    output_text = str(data.get("output_text") or "")
+                    tool_calls.append(
+                        {
+                            "tool_name": data.get("tool_name"),
+                            "args_text": data.get("args_text", ""),
+                            "output_text": output_text[:TOOL_OUTPUT_MAX_CHARS],
+                            "output_chars": len(output_text),
+                            "matches": [
+                                {
+                                    key: match.get(key)
+                                    for key in ("source_key", "url", "title", "score")
+                                }
+                                for match in data.get("matches") or []
+                            ],
+                        }
+                    )
+                elif event.type == "source_match":
+                    resolved_sources.append(
+                        {
+                            key: event.data.get(key)
+                            for key in ("source_key", "url", "title", "group")
+                        }
+                    )
+                elif event.type == "context_stats":
+                    context_stats = dict(event.data)
+                elif event.type == "message_completed":
+                    answer = str(event.data.get("answer", ""))
+    except Exception as exc:  # noqa: BLE001 - record and continue the battery
+        error = f"{type(exc).__name__}: {exc}"
+        logger.warning("Turn failed: %s", error)
+    return {
+        "model": request.model_name or DEFAULT_MODEL_NAME,
+        "started_at": started_at,
+        "duration_ms": int((time.monotonic() - started) * 1000),
+        "answer": answer,
+        "thread_id": thread_id,
+        "tool_calls": tool_calls,
+        "resolved_sources": resolved_sources,
+        "context_stats": context_stats,
+        "error": error,
+    }
+
+
+def build_request(
+    args: argparse.Namespace,
+    *,
+    query: str,
+    source_key: str | None,
+    history: tuple[Any, ...] = (),
+    thread_id: str = "",
+    student_id: str = "",
+) -> Any:
+    from app.chat_types import ChatRequest
+    from app.config import DEFAULT_MODEL_NAME, DEFAULT_SELECTED_SOURCE_KEYS
+
+    source_keys = (
+        (source_key,)
+        if (args.scope_sources and source_key)
+        else tuple(DEFAULT_SELECTED_SOURCE_KEYS)
+    )
+    return ChatRequest(
+        query=query,
+        history=history,
+        source_keys=source_keys,
+        model_name=args.model or DEFAULT_MODEL_NAME,
+        include_reasoning=False,
+        thread_id=thread_id,
+        enabled_tools=tuple(args.enable_tools),
+        memory_preset=args.preset,
+        student_id=student_id,
+    )
+
+
+async def run_single_case(
+    args: argparse.Namespace,
+    battery_type: str,
+    record: dict[str, Any],
+    trial: int,
+    sink: BundleSink,
+) -> None:
+    """singleturn / personas-question / replay: one independent turn."""
+    from app.chat_types import ChatTurn
+
+    if record.get("_profile_seed"):
+        # Re-seed right before every persona question: under profile_memory
+        # the post-turn write-back would otherwise drift the profile between
+        # questions (and across concurrent trials), making results
+        # order-dependent. Each question grades the canonical seeded profile.
+        from app.chat_service import set_student_profile
+
+        set_student_profile(record["_student_id"], record["_profile_seed"])
+
+    unit_id = record["_unit_id"]
+    query = record["_query"]
+    history = tuple(
+        ChatTurn(role=turn["role"], content=turn["content"])
+        for turn in record.get("_history", [])
+    )
+    request = build_request(
+        args,
+        query=query,
+        source_key=record.get("source_key"),
+        history=history,
+        student_id=record.get("_student_id", ""),
+    )
+    result = await run_turn(request)
+    await sink.write(
+        [
+            make_bundle(
+                args=args,
+                battery_type=battery_type,
+                unit_id=unit_id,
+                trial=trial,
+                turn_index=None,
+                query=query,
+                result=result,
+            )
+        ]
+    )
+
+
+async def run_session(
+    args: argparse.Namespace,
+    session: dict[str, Any],
+    trial: int,
+    sink: BundleSink,
+) -> None:
+    """All turns sequentially on one thread, passing the visible transcript
+    back each turn exactly like the real frontend does."""
+    from app.chat_types import ChatTurn
+
+    history: list[ChatTurn] = []
+    thread_id = ""
+    rows = []
+    for turn_index, query in enumerate(session["turns"]):
+        request = build_request(
+            args,
+            query=query,
+            source_key=session.get("source_key"),
+            history=tuple(history),
+            thread_id=thread_id,
+        )
+        result = await run_turn(request)
+        thread_id = result["thread_id"] or thread_id
+        rows.append(
+            make_bundle(
+                args=args,
+                battery_type="sessions",
+                unit_id=session["session_id"],
+                trial=trial,
+                turn_index=turn_index,
+                query=query,
+                result=result,
+            )
+        )
+        if result["error"]:
+            logger.warning(
+                "Session %s trial %d aborted at turn %d.",
+                session["session_id"],
+                trial,
+                turn_index,
+            )
+            break
+        history.append(ChatTurn("user", query.strip()))
+        history.append(ChatTurn("assistant", result["answer"]))
+    await sink.write(rows)
+
+
+def prepare_units(
+    battery_type: str, records: list[dict[str, Any]]
+) -> list[dict[str, Any]]:
+    """Flatten batteries into uniform work units with _unit_id/_query fields."""
+    if battery_type == "singleturn":
+        for record in records:
+            record["_unit_id"] = record["case_id"]
+            record["_query"] = record["question"]
+        return records
+    if battery_type == "replay":
+        for record in records:
+            record["_unit_id"] = record["replay_id"]
+            record["_query"] = record["history"][-1]["content"]
+            record["_history"] = record["history"][:-1]
+        return records
+    if battery_type == "personas":
+        units = []
+        for persona in records:
+            for question in persona["questions"]:
+                units.append(
+                    {
+                        "_unit_id": question["question_id"],
+                        "_query": question["question"],
+                        "_student_id": persona["persona_id"],
+                        "_profile_seed": persona["profile_seed"],
+                        "source_key": persona.get("source_key"),
+                    }
+                )
+        return units
+    return records  # sessions keep their own shape
+
+
+async def run_all(args: argparse.Namespace) -> Path:
+    records = load_jsonl(args.battery)
+    battery_type = detect_battery_type(records)
+    if args.ids:
+        wanted = set(args.ids)
+        key = {"personas": "persona_id"}.get(battery_type)
+        records = [
+            r
+            for r in records
+            if record_id(r) in wanted or (key and r.get(key) in wanted)
+        ]
+    if args.limit:
+        records = records[: args.limit]
+    if not records:
+        raise SystemExit("No records selected.")
+
+    units = prepare_units(battery_type, records)
+    out_dir = Path(
+        args.out or f"runs/{datetime.date.today():%Y%m%d}_{battery_type}_{args.preset}"
+    )
+    out_dir.mkdir(parents=True, exist_ok=True)
+    (out_dir / "run_config.json").write_text(
+        json.dumps(vars(args), indent=1, default=str)
+    )
+    sink = BundleSink(out_dir / "bundles.jsonl")
+
+    expected_turns = {
+        unit["session_id"] if battery_type == "sessions" else unit["_unit_id"]: (
+            len(unit["turns"]) if battery_type == "sessions" else 1
+        )
+        for unit in units
+    }
+    completed = prune_incomplete(sink.path, battery_type, expected_turns, args.trials)
+
+    semaphore = asyncio.Semaphore(args.concurrency)
+    # Same-persona questions are serialized: each one re-seeds the canonical
+    # profile (run_single_case) and profile_memory's post-turn write-back must
+    # not land mid-way through a sibling question's turn. Different personas
+    # still run concurrently.
+    student_locks: dict[str, asyncio.Lock] = defaultdict(asyncio.Lock)
+    pending = 0
+
+    async def guarded(unit: dict[str, Any], trial: int) -> None:
+        async with semaphore:
+            if battery_type == "sessions":
+                await run_session(args, unit, trial, sink)
+            elif unit.get("_student_id"):
+                async with student_locks[unit["_student_id"]]:
+                    await run_single_case(args, battery_type, unit, trial, sink)
+            else:
+                await run_single_case(args, battery_type, unit, trial, sink)
+
+    tasks = []
+    for unit in units:
+        unit_id = unit["session_id"] if battery_type == "sessions" else unit["_unit_id"]
+        for trial in range(1, args.trials + 1):
+            if (unit_id, trial) in completed:
+                continue
+            pending += 1
+            tasks.append(asyncio.create_task(guarded(unit, trial)))
+    logger.info(
+        "Running %d work unit(s) (%d already complete) -> %s",
+        pending,
+        len(completed),
+        out_dir,
+    )
+    if tasks:
+        await asyncio.gather(*tasks)
+    return out_dir
+
+
+def main() -> None:
+    logging.basicConfig(level=logging.INFO, format="%(levelname)s %(message)s")
+    args = parse_args()
+    # Must happen before any app import: app.config enables tracing at import
+    # time when LANGSMITH_API_KEY is set, and batch runs would eat the
+    # free-plan trace quota.
+    if not args.langsmith:
+        os.environ["LANGSMITH_TRACING"] = "false"
+    out_dir = asyncio.run(run_all(args))
+    print(f"Bundles written to {out_dir}/bundles.jsonl")
+    print(f"Next: uv run -m evals.grade --run {out_dir}")
+
+
+if __name__ == "__main__":
+    main()

evals_background.md

@@ -0,0 +1,233 @@
+# Evals background: research notes & rationale
+
+Reference material behind `evals.md`: source attributions, metric design rationale, methodology, pitfalls.
+
+Goal: build an eval harness that lets us say, with evidence, "memory/context method A got these results, method B regressed quality but cut cost 40%, method C improved both" — across latency, tokens, cost, retrieval accuracy, and answer quality — and turn that into a workshop on comparing these methods.
+
+Sources read: Hamel Husain's evals FAQ + related posts, howtoeval.com (Ben Hylak), and the OpenAI cookbook "Macro Evals for Agentic Systems". Codebase and `data/academy_discussion_eval.jsonl` inspected directly.
+
+---
+
+## 1. The single most important framing: you are doing TWO different things
+
+Everything in the sources clicks into place once you separate two activities that look similar but have different methodologies:
+
+1. **Product evals (quality assurance).** "Is our tutor good? Where does it fail?" This is Hamel's world: error analysis → failure taxonomy → targeted binary checks and validated LLM judges. The output is a list of failure modes with frequencies, and automated checks that catch regressions.
+
+2. **Controlled experiments (benchmarking variants).** "Does summarization middleware beat full history?" This is classic experiment design: freeze everything (model, prompt, dataset, retrieval), vary exactly one component (the memory/context strategy), run the **identical** scenario battery through each variant, compare metrics with paired statistics. This is what the workshop is actually about, and it's closest to the OpenAI cookbook's approach.
+
+**Activity 2 depends on activity 1.** You can't compare variants until you have (a) a trustworthy dataset, (b) metrics you believe, and (c) a judge validated against human labels. And when variant B scores worse, only error analysis tells you *why* (the macro-evals point: a regression number without failure-pattern analysis is a dead end). Hamel gives you the machinery for (a)–(c); the cookbook gives you the cross-population analysis; howtoeval gives you the harness philosophy (assert on trajectories, golden cases gate shipping, small high-signal datasets).
+
+This two-activity distinction is also a great workshop narrative: *most teams try to jump straight to activity 2 and get numbers they can't trust.*
+
+---
+
+## 2. What the three sources say (distilled)
+
+### Hamel Husain (hamel.dev/blog/posts/evals-faq + evals + llm-judge + field-guide)
+- **Error analysis first, always.** Read 100+ real traces, take free-form notes ("open coding"), group into a failure taxonomy ("axial coding"), count frequencies. Stop when ~20 consecutive traces yield no new failure category ("theoretical saturation"). Re-run on 100+ fresh traces each cycle. This is "the single highest-ROI activity"; expect 60–80% of dev time on evals overall.
+- **Binary pass/fail + written critique, never 1–5 Likert scores.** Decompose nuance into multiple binary checks (e.g., each expected fact = one binary check). One domain-expert "benevolent dictator" labels; don't outsource, don't committee.
+- **LLM-as-judge via "critique shadowing":** expert labels ~30 diverse examples binary + critique → few-shot judge prompt built from the critiques → iterate until >90% agreement with expert on a held-out set → measure **TPR/TNR**, not raw agreement (classes are imbalanced). Use the strongest model you can afford as judge.
+- **Generic metric dashboards (hallucination score, helpfulness 1–5) are an anti-pattern** — "waste time and create false confidence." Build checks for failures you observed, not failures you imagine. Don't do "eval-driven development."
+- **RAG:** evaluate retrieval and generation **separately**. Retrieval = classic IR metrics (Recall@k, Precision@k, MRR) against query→document ground truth. Generation = error analysis + targeted checks (faithfulness to context, answers the question).
+- **Multi-turn:** start with one conversation-level pass/fail ("did the conversation meet the user's goal?"); annotate only the **first** failure (downstream failures cascade). For test generation, **N-1 replay** (feed real turns 1..N-1, evaluate the model's turn N) beats fully simulated users.
+- **Agents:** two phases — (1) end-to-end black-box success rate, (2) step-level diagnostics: tool choice, argument quality, error recovery, context retention across steps, efficiency (steps, seconds, tokens). Build **transition failure matrices** (last good step × first failed step) and compare them **across experiments** — directly applicable to our variant comparison.
+- **CI:** small purpose-built suites (often ~100 examples), prefer cheap code assertions over judges; 100% pass means your evals are too easy (~70% is healthier).
+
+### howtoeval.com (Ben Hylak, Raindrop — one long guide, May 2026)
+- Frame: **floor-raising** (eliminate worst-case failures on critical paths) over benchmark-maxxing. For agents, "the path matters as much as the answer" — offline evals should be **code-aware tests** that run the real agent and assert on outputs *and* tool-call sequences (pytest/vitest style), not prompt-scoring.
+- **Golden cases:** start with 5–10 critical-path scenarios; "if your agent starts failing your golden cases, you do not ship." Datasets come from real production failures; "20 high-signal cases beats 200 low-signal ones"; prune cases that haven't failed in 3 months.
+- **Skeptical of LLM-as-judge** (calibration brittleness, Goodhart's law) and of hosted eval dashboards; weights production monitoring + A/B on real traffic for unverifiable hypotheses. Useful tension to present in the workshop against Hamel's pro-judge stance — the resolution is Hamel's validation discipline: a judge is only as good as its measured agreement with a human expert.
+- Budget ~10–20% of dev time (vs Hamel's 60–80% — present both, the truth depends on maturity/stakes).
+
+### OpenAI cookbook — "Macro Evals for Agentic Systems" (May 2026, w/ Promptfoo)
+- **Micro evals** grade one run against a rubric; **macro evals** look *across* hundreds of graded runs to find which failure patterns repeat, where they concentrate, and what to inspect first. Rationale: "a final answer is only the last event in a longer workflow" — final-answer inspection alone misses upstream failures.
+- Four labels per run: **case_type** (the scenario setup) → **run_outcome** (how it ended) → **eval_finding** (which rubric check failed) → **behavior_pattern** (cluster discovered across the population). Slice analysis via lift = pattern share in slice ÷ overall share.
+- **Trace bundles**: persist a complete evidence packet per run (events, tool calls, outcomes, judge labels) so you can **re-grade offline without re-running the agent**. "The quality of the trace document is part of the evaluation design."
+- Tracks per-trace counters (tool calls, handoffs, loops, retries) as severity/complexity inputs. Notably it does **not** cover latency/token/cost — none of the three sources do. We have to define that layer ourselves (it's the easy, objective part).
+
+---
+
+## 3. What we should measure (the metric stack)
+
+Five layers, cheapest/most-objective first. Layers 1–3 are code; only layer 4 needs a judge; layer 5 is the one that actually differentiates memory methods.
+
+### Layer 1 — Operational telemetry (free, fully objective)
+Per turn and cumulative per conversation:
+- **Latency**: time-to-first-token and total wall-clock per turn (we stream, so TTFT is what users feel).
+- **Tokens**: input/output per LLM call, summed per turn and per conversation. Input tokens are where memory methods differ most — full history grows O(n²) cumulative input tokens over a conversation; summarization/editing flattens that curve. *The cumulative-input-token curve per conversation turn is the workshop's signature plot.*
+- **Cost**: tokens × provider price sheet (account for cache-read pricing if applicable; summarization itself costs extra LLM calls — count them, that's part of the honest comparison).
+- **Trajectory counters**: number of LLM calls, tool calls per turn (retrieve vs kb_command vs web), KB-command budget consumption, summarization/context-edit trigger firings.
+
+LangSmith already captures token usage and latency per run (runs are tagged with model + metadata; add a `memory_variant` tag). The `stream_chat` ChatEvent stream gives tool-call-level detail.
+
+### Layer 2 — Retrieval accuracy (objective, we have ground truth)
+- The dataset has `source_key` and `lesson_url` per question → **Recall@k / MRR**: did the right source/lesson appear in `retrieve_tutor_context` results? (Hook into the SearchResult list on `tool_call_completed` events; results carry source + URL.)
+- **Tool-routing correctness**: when `expected_behavior == answer_from_corpus`, did the agent call retrieval/KB at all? When `redirect_to_support`, did it correctly *not* go down a retrieval rabbit hole?
+- Why this matters for memory comparisons: compressed/summarized context can degrade the agent's *query formulation* on later turns (it lost the details needed to write a good retrieval query). Retrieval recall on late turns is a sensitive early indicator of memory damage.
+
+### Layer 3 — Behavioral correctness (programmatic binary checks)
+- **Expected-behavior match**: dataset annotates `expected_behavior` ∈ {answer_from_corpus, answer_general, redirect_to_support, acknowledge_feedback}. A small classifier-judge or keyword check per behavior → binary pass.
+- **Citation validity**: we already resolve inline citations against current-turn evidence + KB manifest (`app/kb_manifest.py`) — assert citations resolve, and that corpus answers carry ≥1 citation.
+- **Format/safety assertions**: answered in scope, didn't fabricate course logistics, didn't promise human follow-up, etc. (grow this list from error analysis, not imagination).
+
+### Layer 4 — Answer quality (LLM-as-judge, validated)
+- **Key-point coverage** (primary): annotations include 1–4 `key_points` extracted from real staff answers. For each key point, one binary judge call: "Does the answer correctly convey this point? pass/fail + one-line critique." Report coverage = fraction of key points passed. This is exactly Hamel's "decompose gradations into separate binary checks" and is far more reliable than holistic scoring.
+- **Faithfulness**: binary — is every substantive claim in the answer supported by the retrieved evidence / KB content in the trace? (Catches hallucination specifically on corpus questions.)
+- **Holistic pass/fail** (secondary): "Would the course staff member have approved sending this answer? pass/fail + critique" — few-shot prompt built from *our own labeled critiques*, validated to >90% TPR/TNR agreement against ~30–50 human-labeled traces before we trust it. The team member who actually answers academy questions is the "benevolent dictator" labeler.
+
+### Layer 5 — Memory-specific probes (the differentiator — this is OUR design; no source covers it)
+Memory methods only differ when context pressure exists. Single-turn evals will show **zero difference** between variants (our SummarizationMiddleware triggers at 30k tokens, ContextEditing at 5k tool-tokens — a one-shot question never trips them). So:
+- **Recall-after-compression probes**: plant a fact early (turn 1–2: "I'm on the Agentic AI course, lesson 4, using Python 3.13 on Windows, my API key is set via .env"), drive 5–10 heavy turns (each invoking retrieval/KB to inflate tool tokens past the triggers), then probe: "given my setup, why might X fail?" Binary: does the answer use the planted facts?
+- **Consistency**: does turn N contradict what the tutor said in turn 2? (Judge check on conversation pairs.)
+- **No re-asking**: does the agent ask for information the user already provided? (Strong signal of memory loss; cheap judge check.)
+- **Instruction persistence**: user says "explain everything assuming I'm a beginner / always show code in Python" at turn 1 — is turn 8 still complying?
+- **Anaphora resolution under compression**: late-turn question that says "the second approach you mentioned" — resolvable only if earlier assistant content survived.
+
+Score each probe binary, report **memory probe accuracy** per variant. This is the column where full-history wins and aggressive compression loses — the interesting result is *how much* quality each method trades for its token savings.
+
+---
+
+## 4. Agents vs one-turn chatbots — and tutors vs coding agents
+
+How agent evals differ from single-turn chatbot evals (all three sources agree):
+1. **The trajectory is a first-class eval target.** A correct answer reached via 14 redundant KB commands is a different (worse) result than the same answer in 2 tool calls. Assert on tool-call sequences and counts, not just final text.
+2. **Failures cascade.** Annotate the *first* failure per trace; build transition matrices (where in the pipeline do failures start — query formulation? retrieval? synthesis? citation?). Fixing late-stage symptoms of early-stage failures is wasted work.
+3. **Final-answer inspection alone is insufficient** — an answer can look fine while the trace shows the agent ignored retrieved evidence and answered from parametric knowledge (a faithfulness failure that will bite on corpus-specific content).
+4. **Two-phase evaluation**: end-to-end success rate first (cheap, comparable), step-level diagnostics second (explains the deltas).
+5. **Multi-turn adds a conversation-level unit of analysis**: "did the session meet the student's goal" is judged over the whole conversation, with per-turn metrics underneath.
+
+How a **tutor** differs from a **coding agent** (your instinct is right):
+- Coding agents have **executable ground truth** (tests pass / build compiles). We don't — there is no compiler for "good pedagogical answer." That pushes us toward (a) human-aligned LLM judges and (b) squeezing every drop out of the *programmatic* ground truth we *do* have: retrieval ground truth (`source_key`/`lesson_url`), citation resolvability, expected-behavior routing, and key-point coverage against real staff answers. We're actually unusually well-positioned: most chatbot teams have no reference answers; we have 135 real staff answers.
+- Tutor-specific quality dimensions worth tracking once error analysis confirms they occur: grounding in *our* course material vs generic internet answers (a student asks about lesson 4's notebook; a generically-correct-but-course-ignorant answer is a failure), level-appropriateness, and scope discipline (redirect platform issues instead of hallucinating refund policies).
+- The agentic surface we *do* share with coding agents: tool selection (retrieve vs browse KB vs web), query formulation quality, and budget efficiency — evaluate those the same way coding-agent evals do (trajectory assertions + efficiency counters).
+
+---
+
+## 5. The dataset: what we have, and the gap we must fill
+
+`data/academy_discussion_eval.jsonl` — 151 real student questions, LLM-annotated (gemini-3.5-flash, 2026-06-10):
+
+| Dimension | Distribution |
+|---|---|
+| eval_quality | gold 62 · usable 61 · weak 9 · exclude 19 |
+| category | debugging 47 · course_feedback 39 · conceptual 31 · platform_issue 20 · other 14 |
+| expected_behavior | answer_general 49 · acknowledge_feedback 41 · answer_from_corpus 40 · redirect_to_support 21 |
+| has reference_answer | 135 yes · 16 no |
+| key_points | 91 cases have ≥1 (60 have none) |
+| time_bound | **52 true** · 99 false |
+| source | full_stack 98 · agentic 37 · python 15 · none 1 |
+
+**Strengths**: real user distribution (not synthetic), real staff reference answers, behavior labels that map directly to binary checks, **81 cases that are gold/usable + key_points + reference answer** — that's the core single-turn battery, right in Hamel's "~100-example purpose-built suite" range.
+
+**Caveats**:
+- **Verify the annotations.** They're LLM-generated and unreviewed. Before trusting them as ground truth, the domain expert should review at least the 62 gold cases (Hamel: never delegate ground-truth labels). This doubles as your first error-analysis session. Expect to demote some.
+- **52 time_bound cases**: answers reference point-in-time state (broken links, current platform behavior). Exclude from the comparison battery or re-verify; they'll produce noise, not signal.
+- **Skew**: 65% from one course; fine for now, note it when reporting.
+- **It's almost entirely single-turn** (82 threads have 1 reply, 14 have none; the few longer threads are discussion back-and-forth, not chat sessions). **This dataset alone cannot differentiate memory methods** (§3 layer 5). 
+
+**The gap-filler: a synthetic multi-turn "study session" suite.** Following Hamel's synthetic-inputs guidance (generate *inputs* only; run them through the real system):
+1. Take related gold cases from the same course/lesson; chain 4–8 of them into a plausible study session script (student persona working through a lesson, hitting issues).
+2. Insert planted facts early (persona, environment, constraints) and 1–3 **memory probes** late (§3 layer 5), positioned *after* enough tool-heavy turns that the 5k/30k triggers have demonstrably fired (log trigger events to verify — a memory eval where compression never activated measures nothing).
+3. Hand-write ~5 session scripts first (Hamel: hand-write ~20 seed tuples before LLM-generating more), then LLM-generate variants along dimensions: course × persona (beginner/advanced) × session length × probe type. Target ~30–50 sessions.
+4. Also use **N-1 replay** on the 9 real threads with ≥3 turns — replay real turns, evaluate the next one. Small but real.
+
+---
+
+## 6. Error analysis and LLM-as-judge, in plain terms (and exactly what to do)
+
+**Error analysis** = systematically reading your system's transcripts and writing down what went wrong, then grouping the notes into a counted taxonomy. No math, no models — structured journaling. It is unanimously (all three sources) the highest-ROI step and the one everyone skips. Concretely for us:
+1. Run the current system (default config) over the ~80-case single-turn battery. Persist full trace bundles (every ChatEvent: tool calls, retrieval results, reasoning, final answer) as JSONL — the cookbook's "grade offline, re-grade cheaply" pattern.
+2. Omar (or whoever answers academy questions) reads every trace next to the staff reference answer, marks binary good/bad, writes a one-to-three-sentence note on the *first* thing that went wrong.
+3. Group notes into a failure taxonomy (LLM can help cluster; human validates) and count. Expect things like: retrieval missed the lesson → answered generically; ignored retrieved evidence; over-eager web search; wrong behavior routing (answered a platform issue instead of redirecting); citation errors.
+4. **Those observed failure modes — not a generic metric list — become the automated checks in layers 3–4.** Hamel's strongest warning is against bolting on prefab "hallucination/helpfulness" scores; build checks for what actually breaks.
+5. A tiny annotation viewer (Streamlit, one screen: trace + reference answer + pass/fail button + notes box) is "the single most impactful investment" — buildable in an afternoon, ~10x labeling speed.
+
+**LLM-as-judge** = using a strong LLM to grade outputs at a scale humans can't. The catch: an unvalidated judge is a random-ish number generator with confident prose. The discipline (Hamel's critique-shadowing): human-label 30–50 diverse traces binary+critique → build the judge prompt with those critiques as few-shot examples → run judge on held-out labeled traces → measure TPR/TNR → iterate the prompt until agreement >90% → only then run it at scale, and re-spot-check periodically. Our `key_points` decomposition makes the judge's job nearly mechanical ("does the answer say X?"), which is exactly where judges are most reliable. Hylak's skepticism (Goodhart, calibration drift) is the reason for the validation step, not a reason to skip judges — we have no executable ground truth, so a validated judge is the only scalable quality measure available.
+
+---
+
+## 7. The controlled experiment: "method A vs B vs C"
+
+### Variants (all parameterizable from our codebase — see §8 for the needed refactor)
+| Variant | Description |
+|---|---|
+| **A. Full history** (baseline) | No summarization, no context editing. Upper bound on quality/memory, worst tokens/cost. |
+| **B. Current prod** | ContextEditing(trigger 5k tool-tokens, keep 5) + Summarization(trigger 30k, keep last 20 msgs). |
+| **C. Summarization only** | Isolate the summarizer's contribution. |
+| **D. Context-editing only** | Isolate tool-result pruning. |
+| **E. Aggressive compression** | e.g. Summarization @ 8k keep 8, editing @ 2k keep 2. The "how bad can cheap get" point. |
+| **F+. SOTA methods (workshop highlights)** | One or more of: **long-term memory store** (LangMem/LangGraph Store: extract salient facts per turn into a store, retrieve-into-context on later turns — MemGPT/Letta lineage); **retrieval-over-history** (embed past turns, RAG over your own conversation instead of carrying it); **structured note-taking / compaction** (agent maintains a running scratchpad of session state, à la Anthropic's context-management work and Claude Code's compaction); **observation offloading** (truncate tool outputs to references, re-fetch on demand). |
+
+### Protocol
+- **Hold constant**: model (run the full matrix on one model first; model×memory interaction is a separate, second experiment), system prompt, retrieval config, dataset, tool config, temperature.
+- **Vary**: the memory/context configuration only. Tag every run/trace with `memory_variant` (LangSmith metadata + trace bundle field).
+- **Repeat trials**: LLM nondeterminism is real. 3 trials per case per variant minimum; report mean ± and also **consistency** (pass^3: passed all 3 trials — workshops love seeing that a method is not just better on average but more *reliable*).
+- **Paired comparison**: every variant sees the identical cases, so compare per-case (paired bootstrap for rates, McNemar for binary). With ~80 single-turn cases + ~40 sessions × 3 trials, ~10-point pass-rate differences will be clearly resolvable (none of the sources give significance guidance — this is standard stats we add ourselves).
+- **Two-tier reporting**: single-turn battery (sanity tier: variants should be ~equal here — if a memory method hurts single-turn answers, that's a bug, and it cleanly isolates *memory* effects to the multi-turn tier) + multi-turn session battery (where the methods actually separate).
+
+### The results matrix (the workshop money-slide)
+Rows = variants A–F. Columns = answer pass rate · key-point coverage · faithfulness · memory-probe accuracy · retrieval Recall@5 (late-turn) · expected-behavior accuracy · cumulative input tokens/session · cost/session · p50/p95 TTFT · summarizer overhead tokens. Plus the signature plot: **cumulative input tokens vs turn number, one line per variant**, annotated with where each variant's quality started dropping.
+
+### Explaining the deltas (macro layer)
+When a variant regresses, don't stop at the number: diff the failure taxonomies between variants (which failure modes did compression *create*?), and use first-failure/transition analysis (did failures move upstream into query formulation?). With a few hundred graded traces per variant, even the cookbook's lightweight version — group failed traces by eval_finding × case_type, look for concentration — is enough; full clustering machinery is optional.
+
+---
+
+## 8. Implementation notes (grounded in our code)
+
+1. **Runner**: a Python script calling `stream_chat(ChatRequest(...))` directly (no HTTP needed) — `app/chat_service.py` is the single entry point. Multi-turn sessions: reuse one `thread_id` across turns (InMemorySaver is in-process, so the runner keeps state naturally). Persist one **trace bundle** JSONL per run: case_id, variant, trial, every ChatEvent, per-call token usage, timings, final answer.
+2. **Parameterize the middlewares**: `build_agent` hard-codes ContextEditing/Summarization params (`app/chat_service.py:832–851`). Add a memory-config parameter (env var or ChatRequest field) selecting variant presets, and include it in the agent cache key. This is the one refactor the harness requires.
+3. **Token/latency capture**: LangChain returns `usage_metadata` per model call; also already in LangSmith runs (tagged with model/thread; add `memory_variant` to the metadata dict at `app/chat_service.py:979–992`). Capture both — bundles for offline analysis, LangSmith for trace browsing during error analysis.
+4. **Retrieval ground truth hook**: `tool_call_completed` events carry SearchResult matches (source, URL, score) — compute Recall@k/MRR in the runner, no retriever changes needed.
+5. **Verify triggers fire**: log summarization/context-edit activations per session; assert the multi-turn suite actually trips them (else lengthen sessions).
+6. **LangSmith vs roll-your-own**: use LangSmith for trace inspection + telemetry; keep the runner/dataset/judging in-repo (plain JSONL + pytest-style checks, per howtoeval) so the workshop materials are self-contained and reproducible without a SaaS dependency.
+
+### Phased build (each phase ships something usable alone)
+1. **Curate** (≈1 day): expert-review the 62 gold annotations; freeze a v1 battery (~80 single-turn cases, time_bound excluded); pick 5–10 **golden cases** that gate any future ship.
+2. **Runner + bundles** (≈1–2 days): batch runner, trace persistence, telemetry. Layer-1 metrics work immediately.
+3. **Error analysis** (≈1 day of expert time): run baseline, read all traces in a small Streamlit viewer, build the failure taxonomy. *Do this before writing any judge.*
+4. **Programmatic checks** (≈1 day): retrieval recall, behavior routing, citation validity, + checks derived from step 3.
+5. **Judge** (≈2 days incl. labeling): key-point coverage + faithfulness + holistic, validated to >90% TPR/TNR against expert labels.
+6. **Multi-turn suite** (≈2 days): 5 hand-written sessions → ~30–50 generated; memory probes; N-1 replay of the 9 real multi-turn threads.
+7. **Experiment matrix** (compute-bound): variants × battery × 3 trials; results matrix; failure-taxonomy diff per variant.
+
+---
+
+## 9. Workshop framing suggestions
+
+- The build order **is** the talk: why generic metrics fail → error analysis on real student questions (show real traces!) → decomposed binary checks → judge validation (show the TPR/TNR table — audiences rarely see *judge* evals) → controlled comparison → results matrix → "why did E fail" trace autopsy.
+- The honest tension between sources is great material: Hamel (judges, 60–80% time) vs Hylak (judge-skeptic, golden cases, 10–20%) vs OpenAI (population-scale macro analysis). Resolution: maturity and stakes determine the dose; validation discipline determines whether judges are trustworthy at all.
+- The memory-probe design (plant facts → inflate context past compression triggers → probe) is the novel, reusable artifact attendees take home — none of the public sources cover it.
+- Have one **counterintuitive result** ready (e.g., "aggressive compression cut cost 70% and memory-probe accuracy only dropped 8 points" or the reverse) — that's what gets shared.
+
+## 10. The two-week workshop cut (what to actually do before the talk)
+
+The full plan above is research-grade; the talk needs **demo-grade numbers on a meter plus one honest bake-off table**. The cut:
+
+**Keep (the talk literally requires these):**
+1. **Layer-1 telemetry = the meter.** Build the `context_stats` instrumentation once so both the UI meter and a headless runner consume it. Capture usage from the raw stream / `aget_state` *before* display filtering (compaction-ON runs undercount otherwise).
+2. **Demo session scripts = the multi-turn suite, miniaturized.** The 13-turn compaction session and the 5-fact memory session ARE recall-after-compression probes. Write them deliberately: plant facts early, verify the 30k/5k triggers actually fire (log trigger events), probe late. 2–3 scripted sessions, rehearsed.
+3. **Rehearsal = error analysis lite.** Read every trace from rehearsal runs. The failures found become the "failure first" demo beats — the talk format (failure → fix → number) *requires* harvesting real failures, which is exactly error analysis.
+4. **Mini bake-off**: 3–4 configs (no-memory RAG baseline · compaction OFF/ON · + profile memory) × ~15–20 expert-verified gold single-turn cases × 2–3 scripted sessions, 1–2 trials. **Hand-grade** answers binary against key_points (≈60–80 readings, a half-day) instead of building a validated judge. Report counts honestly ("20 real student questions, 2 trials") — small-n with real data beats large-n synthetic for a talk.
+5. **Replace every placeholder number before slides freeze** (4k→45k→8k, 0/5→5/5, 4k→800) with measured ones, and cache/pre-record fallback runs for the live demo.
+
+**Cut / defer to post-workshop (and pitch as course content):** judge validation to >90% TPR/TNR, the 6-variant × 80-case × 3-trial matrix, N-1 replay, paired statistics, macro failure-pattern analysis, variants D/E/F beyond profile memory.
+
+**Three fixes the talk outline needs:**
+- **Stale code paths**: the outline targets `scripts/chat_service.py` + Gradio (`main.py`, `gradio_presenter.py`), but the repo is now `app/chat_service.py` + Next.js (Gradio was removed). The agent-side changes (toggles in `build_agent`, `MemoryMiddleware` via the `SourcePreferenceMiddleware` pattern, `InMemoryStore`, `context_stats` event) port ~1:1; the UI meter must be a Next.js component fed by the SSE stream (or the workshop branch resurrects the old Gradio snapshot — decide which, it changes the estimate).
+- **The bake-off as written can't show a memory effect.** "No-memory RAG vs profile memory on the same questions" yields identical results if the questions are single-turn and profile-independent (§3 layer 5). The bake-off questions must be personalization-sensitive: phrased against the stored profile ("given my level / my project / my language preference…") or run as session turn-N probes. Otherwise the table shows memory ≈ baseline and undermines the demo.
+- **Show compaction's tradeoff, not only the win.** 45k→8k tokens is the headline, but include one probe where summarization *loses* a planted detail (and full history doesn't). That's the credible, memorable beat — and it sets up "this is why you measure" for the course upsell.
+
+**Rough schedule (10 working days):** D1–3 workshop-branch code (meter, toggles, memory, skills) · D4–5 curate ~20 gold cases + write session scripts + headless runner · D6–8 bake-off runs, hand-grading, build the table, harvest failure beats, lock real numbers · D9–10 rehearse live paths + cached fallbacks.
+
+## 11. Pitfalls checklist
+
+- ☐ Don't compare variants on single-turn cases only — memory methods are indistinguishable there.
+- ☐ Don't trust the LLM-generated annotations without expert review (esp. key_points and eval_quality).
+- ☐ Exclude/handle the 52 time_bound cases.
+- ☐ Don't use an unvalidated judge; report its TPR/TNR alongside results.
+- ☐ Count the *full* cost of each method (summarizer LLM calls, memory-store writes) — not just the saved prompt tokens.
+- ☐ One trial per case is noise; pair the comparisons.
+- ☐ No prefab metric dashboards; every automated check traces back to an observed failure or a dataset ground truth.
+- ☐ Verify compression triggers actually fired in the multi-turn suite.

tests/test_api.py

@@ -710,6 +710,66 @@ class ApiTestCase(unittest.TestCase):
         omitted = build_chat_request(ApiChatRequest(query="What is RAG?"))
         self.assertEqual(omitted.source_keys, tuple(DEFAULT_SELECTED_SOURCE_KEYS))
 
+    def test_memory_preset_and_student_id_mapping(self) -> None:
+        from fastapi import HTTPException
+
+        from app.api import ApiChatRequest, build_chat_request
+
+        request = build_chat_request(
+            ApiChatRequest(
+                query="What is RAG?",
+                memoryPreset="full_history",
+                studentId=" student-1 ",
+            )
+        )
+        self.assertEqual(request.memory_preset, "full_history")
+        self.assertEqual(request.student_id, "student-1")
+
+        # Omitted preset stays empty so the server-side default resolution
+        # (env var, then "prod") applies at stream time.
+        omitted = build_chat_request(ApiChatRequest(query="What is RAG?"))
+        self.assertEqual(omitted.memory_preset, "")
+
+        with self.assertRaises(HTTPException) as raised:
+            build_chat_request(
+                ApiChatRequest(query="What is RAG?", memoryPreset="typo")
+            )
+        self.assertEqual(raised.exception.status_code, 422)
+
+    def test_encoder_emits_transient_context_stats_part(self) -> None:
+        encoder = UIMessageStreamEncoder()
+        parts = encoder.encode(
+            ChatEvent(
+                "context_stats",
+                {
+                    "message_id": "msg_1",
+                    "memory_preset": "prod",
+                    "llm_calls": 3,
+                    "input_tokens": 1200,
+                    "output_tokens": 250,
+                    "total_tokens": 1450,
+                    "cache_read_tokens": 400,
+                    "cache_creation_tokens": 0,
+                    "est_cost_usd": None,
+                    "ttft_ms": 850,
+                    "total_ms": 4200,
+                    "context_messages": 9,
+                    "context_tokens_approx": 5400,
+                    "summary_messages": 1,
+                    "cleared_tool_outputs": 2,
+                },
+            )
+        )
+        self.assertEqual(len(parts), 1)
+        part = parts[0]
+        self.assertEqual(part["type"], "data-context-stats")
+        self.assertTrue(part["transient"])
+        self.assertEqual(part["data"]["memoryPreset"], "prod")
+        self.assertEqual(part["data"]["inputTokens"], 1200)
+        self.assertEqual(part["data"]["summaryMessages"], 1)
+        # Unknown pricing must surface as null, never $0.
+        self.assertIsNone(part["data"]["estCostUsd"])
+
 
 LIVE_API_E2E = pytest.mark.skipif(
     os.getenv("RUN_LIVE_API_E2E") != "1",

tests/test_evals_grade.py

@@ -0,0 +1,151 @@
+from __future__ import annotations
+
+import unittest
+
+from evals.common import detect_battery_type, normalize_url, percentile
+from evals.grade import (
+    behavior_heuristic,
+    grade_persona_question,
+    retrieval_metrics,
+)
+
+
+class CommonTests(unittest.TestCase):
+    def test_detect_battery_type(self) -> None:
+        self.assertEqual(detect_battery_type([{"case_id": "x"}]), "singleturn")
+        self.assertEqual(detect_battery_type([{"session_id": "x"}]), "sessions")
+        self.assertEqual(detect_battery_type([{"persona_id": "x"}]), "personas")
+        self.assertEqual(detect_battery_type([{"replay_id": "x"}]), "replay")
+        with self.assertRaises(ValueError):
+            detect_battery_type([{"foo": 1}])
+
+    def test_normalize_url(self) -> None:
+        self.assertEqual(
+            normalize_url("https://X.com/Lessons/abc/?q=1#frag"),
+            "https://x.com/lessons/abc",
+        )
+        self.assertEqual(normalize_url(None), "")
+
+    def test_normalize_url_strips_discussion_suffix(self) -> None:
+        # Battery lesson_urls point at the discussion; matches carry the bare
+        # lesson URL. Both must normalize to the same key.
+        discussion = (
+            "https://academy.towardsai.net/courses/take/agent-engineering/"
+            "multimedia/67469688-lesson-1/discussions/12758677"
+        )
+        lesson = (
+            "https://academy.towardsai.net/courses/take/agent-engineering/"
+            "multimedia/67469688-lesson-1"
+        )
+        self.assertEqual(normalize_url(discussion), normalize_url(lesson))
+
+    def test_percentile(self) -> None:
+        # Nearest-rank: even-length p50 rounds to the upper of the two middles.
+        self.assertEqual(percentile([1, 2, 3, 4], 50), 3)
+        self.assertEqual(percentile([1, 2, 3], 50), 2)
+        self.assertEqual(percentile([5], 95), 5)
+        self.assertIsNone(percentile([], 50))
+
+
+def bundle_with_matches(matches, tool_name="retrieve_tutor_context"):
+    return {"tool_calls": [{"tool_name": tool_name, "matches": matches}]}
+
+
+class RetrievalMetricsTests(unittest.TestCase):
+    LESSON = "https://academy.towardsai.net/courses/take/x/lessons/123-foo"
+
+    def test_hit_source_and_lesson_with_mrr(self) -> None:
+        bundle = bundle_with_matches(
+            [
+                {"source_key": "other", "url": "https://elsewhere"},
+                {"source_key": "full_stack_ai_engineering", "url": self.LESSON + "/"},
+            ]
+        )
+        metrics = retrieval_metrics(bundle, "full_stack_ai_engineering", self.LESSON)
+        self.assertTrue(metrics["called_retrieval"])
+        self.assertTrue(metrics["recall_source"])
+        self.assertTrue(metrics["recall_lesson"])
+        self.assertEqual(metrics["mrr_lesson"], 0.5)
+
+    def test_miss_lesson(self) -> None:
+        bundle = bundle_with_matches(
+            [{"source_key": "full_stack_ai_engineering", "url": "https://other"}]
+        )
+        metrics = retrieval_metrics(bundle, "full_stack_ai_engineering", self.LESSON)
+        self.assertFalse(metrics["recall_lesson"])
+        self.assertEqual(metrics["mrr_lesson"], 0.0)
+
+    def test_kb_command_counts_as_retrieval_but_adds_no_matches(self) -> None:
+        bundle = bundle_with_matches([], tool_name="run_kb_command")
+        metrics = retrieval_metrics(bundle, "x", self.LESSON)
+        self.assertTrue(metrics["called_retrieval"])
+        self.assertEqual(metrics["retrieved_matches"], 0)
+
+    def test_no_ground_truth_yields_none(self) -> None:
+        bundle = bundle_with_matches([])
+        metrics = retrieval_metrics(bundle, None, None)
+        self.assertIsNone(metrics["recall_lesson"])
+        self.assertIsNone(metrics["mrr_lesson"])
+
+
+class BehaviorHeuristicTests(unittest.TestCase):
+    def test_corpus_requires_tool_use(self) -> None:
+        used = {"answer": "...", "tool_calls": [{"tool_name": "run_kb_command"}]}
+        bare = {"answer": "...", "tool_calls": []}
+        self.assertTrue(behavior_heuristic("answer_from_corpus", used))
+        self.assertFalse(behavior_heuristic("answer_from_corpus", bare))
+
+    def test_redirect_and_feedback_regexes(self) -> None:
+        self.assertTrue(
+            behavior_heuristic(
+                "redirect_to_support",
+                {"answer": "Please reach out to the academy team.", "tool_calls": []},
+            )
+        )
+        self.assertTrue(
+            behavior_heuristic(
+                "acknowledge_feedback",
+                {"answer": "Thank you for the suggestion!", "tool_calls": []},
+            )
+        )
+        self.assertIsNone(
+            behavior_heuristic("answer_general", {"answer": "x", "tool_calls": []})
+        )
+
+
+class PersonaGradingTests(unittest.TestCase):
+    QUESTION = {
+        "checks": [{"type": "regex_any", "patterns": ["conda"]}],
+        "anti_patterns": ["uv sync", "python -m venv"],
+    }
+
+    def test_pass(self) -> None:
+        result = grade_persona_question(self.QUESTION, "Use conda env create.")
+        self.assertTrue(result["auto_pass"])
+
+    def test_anti_pattern_fails_even_when_check_passes(self) -> None:
+        result = grade_persona_question(
+            self.QUESTION, "conda works, or run `uv sync` instead."
+        )
+        self.assertFalse(result["auto_pass"])
+        self.assertEqual(result["anti_pattern_hits"], ["uv sync"])
+
+    def test_case_insensitive(self) -> None:
+        result = grade_persona_question(self.QUESTION, "CONDA is fine")
+        self.assertTrue(result["auto_pass"])
+
+    def test_llm_check_defers(self) -> None:
+        question = {
+            "checks": [
+                {"type": "regex_any", "patterns": ["conda"]},
+                {"type": "llm", "instruction": "is it beginner-level?"},
+            ],
+            "anti_patterns": [],
+        }
+        result = grade_persona_question(question, "conda activate course")
+        self.assertIsNone(result["auto_pass"])
+        self.assertTrue(result["needs_judgment"])
+
+
+if __name__ == "__main__":
+    unittest.main()

tests/test_memory_presets.py

@@ -0,0 +1,146 @@
+from __future__ import annotations
+
+import types
+import unittest
+from unittest.mock import patch
+
+from app.chat_service import (
+    SourcePreferenceMiddleware,
+    StudentProfileMiddleware,
+    build_agent,
+    build_agent_middleware,
+)
+from app.memory_presets import (
+    DEFAULT_MEMORY_PRESET,
+    MEMORY_PRESETS,
+    resolve_memory_preset,
+)
+from langchain.agents.middleware import (
+    ContextEditingMiddleware,
+    SummarizationMiddleware,
+)
+
+
+def _stub_model():
+    # SummarizationMiddleware reads model._llm_type at init.
+    return types.SimpleNamespace(_llm_type="fake-chat-model")
+
+
+class MemoryPresetResolutionTests(unittest.TestCase):
+    def test_prod_preset_matches_production_constants(self) -> None:
+        prod = MEMORY_PRESETS["prod"]
+        self.assertTrue(prod.summarization)
+        self.assertEqual(prod.summarization_trigger_tokens, 30_000)
+        self.assertEqual(prod.summarization_keep_messages, 20)
+        self.assertTrue(prod.context_editing)
+        self.assertEqual(prod.context_editing_trigger_tokens, 5_000)
+        self.assertEqual(prod.context_editing_keep, 5)
+        self.assertFalse(prod.longterm_memory)
+
+    def test_default_resolution(self) -> None:
+        with patch.dict("os.environ", {}, clear=False):
+            import os
+
+            os.environ.pop("AI_TUTOR_MEMORY_PRESET", None)
+            self.assertEqual(resolve_memory_preset("").name, DEFAULT_MEMORY_PRESET)
+            self.assertEqual(resolve_memory_preset(None).name, DEFAULT_MEMORY_PRESET)
+
+    def test_env_var_default_and_explicit_override(self) -> None:
+        with patch.dict("os.environ", {"AI_TUTOR_MEMORY_PRESET": "full_history"}):
+            self.assertEqual(resolve_memory_preset("").name, "full_history")
+            # An explicit request value beats the env default.
+            self.assertEqual(resolve_memory_preset("aggressive").name, "aggressive")
+
+    def test_unknown_preset_raises(self) -> None:
+        with self.assertRaises(ValueError):
+            resolve_memory_preset("does_not_exist")
+        with patch.dict("os.environ", {"AI_TUTOR_MEMORY_PRESET": "typo"}):
+            with self.assertRaises(ValueError):
+                resolve_memory_preset("")
+
+
+class MiddlewareAssemblyTests(unittest.TestCase):
+    def test_full_history_disables_compaction(self) -> None:
+        middleware = build_agent_middleware(
+            _stub_model(), MEMORY_PRESETS["full_history"]
+        )
+        self.assertEqual(len(middleware), 1)
+        self.assertIsInstance(middleware[0], SourcePreferenceMiddleware)
+
+    def test_prod_has_editing_then_summarization(self) -> None:
+        middleware = build_agent_middleware(_stub_model(), MEMORY_PRESETS["prod"])
+        self.assertIsInstance(middleware[0], ContextEditingMiddleware)
+        self.assertIsInstance(middleware[1], SummarizationMiddleware)
+        self.assertIsInstance(middleware[-1], SourcePreferenceMiddleware)
+        self.assertEqual(len(middleware), 3)
+
+    def test_single_technique_presets(self) -> None:
+        summarization_only = build_agent_middleware(
+            _stub_model(), MEMORY_PRESETS["summarization_only"]
+        )
+        self.assertFalse(
+            any(isinstance(m, ContextEditingMiddleware) for m in summarization_only)
+        )
+        self.assertTrue(
+            any(isinstance(m, SummarizationMiddleware) for m in summarization_only)
+        )
+        editing_only = build_agent_middleware(
+            _stub_model(), MEMORY_PRESETS["editing_only"]
+        )
+        self.assertTrue(
+            any(isinstance(m, ContextEditingMiddleware) for m in editing_only)
+        )
+        self.assertFalse(
+            any(isinstance(m, SummarizationMiddleware) for m in editing_only)
+        )
+
+    def test_profile_memory_adds_student_profile_middleware(self) -> None:
+        middleware = build_agent_middleware(
+            _stub_model(), MEMORY_PRESETS["profile_memory"]
+        )
+        self.assertTrue(
+            any(isinstance(m, StudentProfileMiddleware) for m in middleware)
+        )
+
+    def test_build_agent_cache_keys_include_memory_config(self) -> None:
+        build_agent.cache_clear()
+        created = []
+
+        def fake_create_agent(**kwargs):
+            agent = types.SimpleNamespace(kwargs=kwargs)
+            created.append(agent)
+            return agent
+
+        try:
+            with (
+                patch(
+                    "app.chat_service.build_chat_model",
+                    return_value=_stub_model(),
+                ),
+                patch("app.chat_service.build_system_prompt", return_value="prompt"),
+                patch("app.chat_service.create_agent", side_effect=fake_create_agent),
+            ):
+                prod = build_agent(
+                    "google-genai:gemini-3.5-flash",
+                    memory_config=MEMORY_PRESETS["prod"],
+                )
+                full_history = build_agent(
+                    "google-genai:gemini-3.5-flash",
+                    memory_config=MEMORY_PRESETS["full_history"],
+                )
+                full_history_again = build_agent(
+                    "google-genai:gemini-3.5-flash",
+                    memory_config=MEMORY_PRESETS["full_history"],
+                )
+        finally:
+            build_agent.cache_clear()
+
+        self.assertIsNot(prod, full_history)
+        self.assertIs(full_history, full_history_again)
+        self.assertEqual(len(created), 2)
+        # Long-term memory needs the store wired into every agent.
+        self.assertIn("store", created[0].kwargs)
+
+
+if __name__ == "__main__":
+    unittest.main()

tests/test_telemetry.py

@@ -0,0 +1,143 @@
+from __future__ import annotations
+
+import unittest
+
+from langchain_core.messages import AIMessage, HumanMessage, ToolMessage
+from langchain_core.outputs import ChatGeneration, LLMResult
+
+from app.chat_service import CLEARED_TOOL_OUTPUT_PLACEHOLDER
+from app.telemetry import (
+    TurnUsageHandler,
+    context_window_stats,
+    estimate_cost_usd,
+    pricing_for_model,
+    usage_totals,
+)
+
+
+class UsageTotalsTests(unittest.TestCase):
+    def test_sums_across_models_including_cache_details(self) -> None:
+        usage = {
+            "gemini-3.5-flash": {
+                "input_tokens": 1_000,
+                "output_tokens": 200,
+                "total_tokens": 1_200,
+                "input_token_details": {"cache_read": 400},
+            },
+            "claude-haiku-4-5-20251001": {
+                "input_tokens": 500,
+                "output_tokens": 100,
+                "total_tokens": 600,
+                "input_token_details": {"cache_read": 50, "cache_creation": 25},
+            },
+        }
+        totals = usage_totals(usage)
+        self.assertEqual(totals["input_tokens"], 1_500)
+        self.assertEqual(totals["output_tokens"], 300)
+        self.assertEqual(totals["total_tokens"], 1_800)
+        self.assertEqual(totals["cache_read_tokens"], 450)
+        self.assertEqual(totals["cache_creation_tokens"], 25)
+
+    def test_missing_details_default_to_zero(self) -> None:
+        totals = usage_totals({"m": {"input_tokens": 10, "output_tokens": 5}})
+        self.assertEqual(totals["cache_read_tokens"], 0)
+        self.assertEqual(totals["total_tokens"], 0)
+
+
+class CostEstimateTests(unittest.TestCase):
+    def test_dated_model_name_matches_family_pricing(self) -> None:
+        self.assertIsNotNone(pricing_for_model("claude-haiku-4-5-20251001"))
+        self.assertIsNone(pricing_for_model("some-unknown-model"))
+
+    def test_cache_tokens_priced_separately(self) -> None:
+        # claude-haiku-4-5: input $1, output $5, cache read $0.10, write $1.25
+        # per MTok. input_tokens includes the cached buckets, so the plain
+        # bucket is 1M - 400k - 100k = 500k.
+        usage = {
+            "claude-haiku-4-5-20251001": {
+                "input_tokens": 1_000_000,
+                "output_tokens": 200_000,
+                "input_token_details": {
+                    "cache_read": 400_000,
+                    "cache_creation": 100_000,
+                },
+            }
+        }
+        expected = 0.5 * 1.00 + 0.4 * 0.10 + 0.1 * 1.25 + 0.2 * 5.00
+        self.assertAlmostEqual(estimate_cost_usd(usage), expected)
+
+    def test_cache_creation_without_write_rate_bills_as_input(self) -> None:
+        # gemini-3.5-flash has no cache_write entry.
+        usage = {
+            "gemini-3.5-flash": {
+                "input_tokens": 1_000_000,
+                "output_tokens": 0,
+                "input_token_details": {"cache_creation": 200_000},
+            }
+        }
+        expected = 0.8 * 0.30 + 0.2 * 0.30
+        self.assertAlmostEqual(estimate_cost_usd(usage), expected)
+
+    def test_unknown_model_yields_none_not_zero(self) -> None:
+        self.assertIsNone(estimate_cost_usd({"mystery-model": {"input_tokens": 10}}))
+        mixed = {
+            "gemini-3.5-flash": {"input_tokens": 10, "output_tokens": 1},
+            "mystery-model": {"input_tokens": 10, "output_tokens": 1},
+        }
+        self.assertIsNone(estimate_cost_usd(mixed))
+
+
+class ContextWindowStatsTests(unittest.TestCase):
+    def test_counts_summaries_and_cleared_tool_outputs(self) -> None:
+        messages = [
+            HumanMessage(
+                content="Here is a summary of the conversation to date: ...",
+                additional_kwargs={"lc_source": "summarization"},
+            ),
+            HumanMessage(content="What is RAG?"),
+            AIMessage(content="RAG is retrieval-augmented generation."),
+            ToolMessage(
+                content=CLEARED_TOOL_OUTPUT_PLACEHOLDER,
+                tool_call_id="call_1",
+            ),
+            ToolMessage(content="$ ls\nwiki", tool_call_id="call_2"),
+        ]
+        stats = context_window_stats(messages, CLEARED_TOOL_OUTPUT_PLACEHOLDER)
+        self.assertEqual(stats["context_messages"], 5)
+        self.assertEqual(stats["summary_messages"], 1)
+        self.assertEqual(stats["cleared_tool_outputs"], 1)
+        self.assertGreater(stats["context_tokens_approx"], 0)
+
+    def test_empty_context(self) -> None:
+        stats = context_window_stats([], CLEARED_TOOL_OUTPUT_PLACEHOLDER)
+        self.assertEqual(stats["context_messages"], 0)
+        self.assertEqual(stats["context_tokens_approx"], 0)
+
+
+class TurnUsageHandlerTests(unittest.TestCase):
+    def test_counts_calls_and_aggregates_usage(self) -> None:
+        handler = TurnUsageHandler()
+
+        def result(input_tokens: int, output_tokens: int) -> LLMResult:
+            message = AIMessage(
+                content="ok",
+                usage_metadata={
+                    "input_tokens": input_tokens,
+                    "output_tokens": output_tokens,
+                    "total_tokens": input_tokens + output_tokens,
+                },
+                response_metadata={"model_name": "gemini-3.5-flash"},
+            )
+            return LLMResult(generations=[[ChatGeneration(message=message)]])
+
+        handler.on_llm_end(result(100, 20))
+        handler.on_llm_end(result(50, 10))
+
+        self.assertEqual(handler.llm_calls, 2)
+        usage = handler.usage_metadata["gemini-3.5-flash"]
+        self.assertEqual(usage["input_tokens"], 150)
+        self.assertEqual(usage["output_tokens"], 30)
+
+
+if __name__ == "__main__":
+    unittest.main()