Enhance chat service and local retriever functionality

- Updated chat_service.py to preserve retrieval results throughout the conversation, ensuring citation-bearing summaries are retained for grounding final outputs.
- Added filtering in stream_chat to prevent structured summary templates from leaking into user-facing responses.
- Modified chroma_rag.py to handle special tokens during encoding, preventing crashes when processing certain content.

scripts/chat_service.py

@@ -618,6 +618,13 @@ def build_agent(
                 ClearToolUsesEdit(
                     trigger=5_000,
                     keep=3,
+                    # Preserve retrieval results through the whole turn. They
+                    # are pre-ranked, citation-bearing summaries that the
+                    # model uses to ground the final synthesis. KB shell
+                    # outputs (raw file content) get aggressively cleared
+                    # past the last 3, which is correct because they're
+                    # large and easy to re-fetch with another `cat`.
+                    exclude_tools=("retrieve_tutor_context",),
                     placeholder="[tool output cleared to save context]",
                 )
             ],
@@ -953,6 +960,17 @@ async def stream_chat(request: ChatRequest) -> AsyncIterator[ChatEvent]:
                 if not isinstance(token, AIMessageChunk):
                     continue
 
+                # SummarizationMiddleware fires its own LLM call to compress
+                # older history. LangChain tags those calls with
+                # `lc_source="summarization"` (see langchain.agents.middleware
+                # SummarizationMiddleware._create_summary). Without this
+                # filter, the structured summary template ("## SESSION INTENT
+                # ...") leaks into the user-facing answer stream. The summary
+                # is supposed to be model-internal — like Codex's compact —
+                # so we drop those chunks here.
+                if metadata.get("lc_source") == "summarization":
+                    continue
+
                 step = str(metadata.get("langgraph_step", ""))
                 if include_reasoning:
                     thought_text = "\n\n".join(

scripts/chroma_rag.py

@@ -1339,7 +1339,12 @@ class LocalChromaRetriever:
             if result.score < 0.10:
                 continue
 
-            result_tokens = len(self._encoding.encode(result.content))
+            # disallowed_special=() so chunks containing literal special-token
+            # strings like "<|endoftext|>" (which sometimes appear verbatim in
+            # tokenizer/training docs) don't crash the encode call.
+            result_tokens = len(
+                self._encoding.encode(result.content, disallowed_special=())
+            )
             if total_tokens + result_tokens > self._token_budget:
                 break