Add LangSmith tracing support and enhance documentation

- Introduced LangSmith tracing capabilities by adding environment variables in .env.example and implementing related functions in agent_tracing.py.
- Updated README.md to include instructions for enabling LangSmith tracing and its integration with the chatbot.
- Refactored chat_service.py to support tracing metadata in agent runs and added warm-up functionality for the retriever.
- Enhanced setup.py to configure LangSmith environment variables automatically.
- Added unit tests for agent tracing functionality to ensure proper behavior and configuration.
- Updated main.py to invoke retriever warm-up during application startup.

.env.example

@@ -4,6 +4,11 @@ ANTHROPIC_API_KEY=...
 GEMINI_API_KEY=...
 COHERE_API_KEY=...
 
+# Optional: trace agent/model/tool runs in LangSmith
+LANGSMITH_API_KEY=...
+LANGSMITH_TRACING=true
+LANGSMITH_PROJECT=ai-tutor-app
+
 # To update documentation/add new course
 HF_TOKEN=... https://huggingface.co/settings/tokens to read or write to private DATASET repos e.g. https://huggingface.co/datasets/towardsai-tutors/ai-tutor-data/tree/main
 GITHUB_TOKEN=...https://github.com/settings/tokens to use the Github API

README.md

@@ -31,6 +31,7 @@ The Gradio demo is deployed on Hugging Face Spaces at: [AI Tutor Chatbot on Hugg
    ```
 
    The chat model is provider-agnostic. Use the UI field in `provider:model` format, for example `openai:gpt-5.4-mini`. Optional provider keys include `OPENAI_API_KEY`, `ANTHROPIC_API_KEY`, and `GOOGLE_API_KEY`. Anthropic support is wired through the `anthropic` SDK, and Gemini support is wired through Google’s `google-genai` SDK.
+   To trace requests in LangSmith, set `LANGSMITH_API_KEY`. The app enables tracing automatically when that key is present unless `LANGSMITH_TRACING=false` is set.
 
 3. Run:
 
@@ -40,6 +41,20 @@ The Gradio demo is deployed on Hugging Face Spaces at: [AI Tutor Chatbot on Hugg
 
    Starts the Gradio AI Tutor interface.
 
+### LangSmith Agent Tracing
+
+The chatbot is built with `langchain.agents.create_agent()`, so LangSmith can trace the LangGraph/LangChain run tree without extra dependencies. Add these values to `.env`:
+
+```bash
+LANGSMITH_API_KEY=ls_...
+LANGSMITH_TRACING=true
+LANGSMITH_PROJECT=ai-tutor-app
+```
+
+Each chat turn is traced as `ai-tutor-agent-turn` with metadata for the backend thread id, message id, selected sources, requested model, and effective tool list. Child runs capture the model calls and `retrieve_tutor_context` tool executions, including tool inputs and outputs. Provider-side tools such as Gemini `google_search` / `url_context` and Claude `web_search` / `web_fetch` are visible through the model request/response metadata that LangChain receives from those providers.
+
+Tracing sends prompts, retrieved snippets, tool inputs, tool outputs, and model responses to LangSmith. Set `LANGSMITH_TRACING=false` to keep tracing disabled while leaving the key in your environment.
+
 ### Next.js Frontend — Quick Start
 
 The repo now also includes a separate Next.js frontend in [frontend](./frontend) that talks to the FastAPI backend instead of the Gradio transport.

scripts/agent_tracing.py

@@ -0,0 +1,39 @@
+from __future__ import annotations
+
+import os
+
+DEFAULT_LANGSMITH_PROJECT = "ai-tutor-app"
+
+TRUE_VALUES = {"1", "true", "t", "yes", "y", "on"}
+FALSE_VALUES = {"0", "false", "f", "no", "n", "off"}
+
+
+def parse_env_bool(value: str | None) -> bool | None:
+    if value is None:
+        return None
+
+    normalized = value.strip().lower()
+    if normalized in TRUE_VALUES:
+        return True
+    if normalized in FALSE_VALUES:
+        return False
+    return None
+
+
+def langsmith_tracing_enabled() -> bool:
+    explicit = parse_env_bool(os.getenv("LANGSMITH_TRACING"))
+    if explicit is not None:
+        return explicit
+    return bool(os.getenv("LANGSMITH_API_KEY"))
+
+
+def configure_langsmith_environment() -> None:
+    """Apply app defaults for LangSmith without requiring code changes in deploys."""
+    if os.getenv("LANGSMITH_API_KEY") and os.getenv("LANGSMITH_TRACING") is None:
+        os.environ["LANGSMITH_TRACING"] = "true"
+
+    if not langsmith_tracing_enabled():
+        return
+
+    if not os.getenv("LANGSMITH_PROJECT"):
+        os.environ["LANGSMITH_PROJECT"] = DEFAULT_LANGSMITH_PROJECT

scripts/api.py

@@ -16,11 +16,11 @@ from fastapi.staticfiles import StaticFiles
 from pydantic import BaseModel, Field
 
 from .chat_service import (
-    get_retriever,
     is_anthropic_model,
     is_google_genai_model,
     message_content_to_text,
     stream_chat,
+    warm_up_retriever,
 )
 from .chat_types import ChatEvent, ChatRequest, ChatTurn
 from .setup import (
@@ -78,14 +78,7 @@ async def lifespan(_app: FastAPI) -> AsyncIterator[None]:
     `Could not connect to tenant default_tenant`. Touching the retriever
     at startup forces single-threaded tenant init.
     """
-    if os.environ.get("COHERE_API_KEY"):
-        try:
-            get_retriever()
-        except Exception as exc:  # pragma: no cover - diagnostic logging only
-            logfire.warn(
-                "Retriever warm-up failed; first retrieval call may retry.",
-                error=str(exc),
-            )
+    warm_up_retriever()
     yield
 
 

scripts/chat_service.py

@@ -4,6 +4,7 @@ import json
 import os
 from dataclasses import dataclass
 from functools import lru_cache
+from threading import Lock
 from typing import Any, AsyncIterator
 from uuid import uuid4
 
@@ -37,6 +38,7 @@ ANSWER_HEADER = "**Answer**"
 LEGACY_THOUGHTS_DETAILS_OPEN = "<details><summary>Gemini thoughts</summary>"
 LEGACY_THOUGHTS_DETAILS_OPEN_EXPANDED = "<details open><summary>Gemini thoughts</summary>"
 CHECKPOINTER = InMemorySaver()
+_RETRIEVER_INIT_LOCK = Lock()
 
 
 @dataclass(frozen=True)
@@ -45,7 +47,7 @@ class AppContext:
 
 
 @lru_cache(maxsize=1)
-def get_retriever() -> LocalChromaRetriever:
+def _build_retriever() -> LocalChromaRetriever:
     ensure_local_vector_db()
     cohere_api_key = os.environ["COHERE_API_KEY"]
     return LocalChromaRetriever(
@@ -57,6 +59,23 @@ def get_retriever() -> LocalChromaRetriever:
     )
 
 
+def get_retriever() -> LocalChromaRetriever:
+    with _RETRIEVER_INIT_LOCK:
+        return _build_retriever()
+
+
+def warm_up_retriever() -> None:
+    if not os.environ.get("COHERE_API_KEY"):
+        return
+    try:
+        get_retriever()
+    except Exception as exc:  # pragma: no cover - diagnostic logging only
+        logfire.warn(
+            "Retriever warm-up failed; first retrieval call may retry.",
+            error=str(exc),
+        )
+
+
 RETRIEVE_TUTOR_CONTEXT_SCHEMA = {
     "title": "retrieve_tutor_context",
     "description": "Retrieve relevant course and documentation context for an AI tutor question.",
@@ -431,6 +450,72 @@ def build_agent(
     )
 
 
+def model_provider_and_name(model_name: str) -> tuple[str, str]:
+    provider_model = normalize_model_name(model_name)
+    provider, _, actual_model = provider_model.partition(":")
+    return provider or "unknown", actual_model or provider_model
+
+
+def effective_tool_names(
+    model_name: str,
+    enabled_tools: tuple[str, ...],
+) -> tuple[str, ...]:
+    names = ["retrieve_tutor_context"]
+    enabled = set(enabled_tools)
+    if is_google_genai_model(model_name):
+        if "web_search" in enabled:
+            names.append("google_search")
+        if "url_context" in enabled:
+            names.append("url_context")
+    elif is_anthropic_model(model_name):
+        if "web_search" in enabled:
+            names.append("web_search")
+        if "web_fetch" in enabled:
+            names.append("web_fetch")
+    return tuple(names)
+
+
+def agent_run_config(
+    request: ChatRequest,
+    active_thread_id: str,
+    message_id: str,
+) -> dict[str, Any]:
+    provider, actual_model = model_provider_and_name(request.model_name)
+    tools = effective_tool_names(request.model_name, request.enabled_tools)
+    source_labels = [
+        SOURCE_KEY_TO_LABEL.get(source_key, source_key)
+        for source_key in request.source_keys
+    ]
+    config = thread_config(active_thread_id)
+    config.update(
+        {
+            "run_name": "ai-tutor-agent-turn",
+            "tags": [
+                "ai-tutor-app",
+                "knowledge-base-chatbot",
+                f"provider:{provider}",
+                f"model:{actual_model}",
+                *(f"tool:{tool_name}" for tool_name in tools),
+            ],
+            "metadata": {
+                "app": "ai-tutor-app",
+                "thread_id": active_thread_id,
+                "conversation_id": active_thread_id,
+                "message_id": message_id,
+                "model_provider": provider,
+                "model_name": actual_model,
+                "requested_model": request.model_name,
+                "available_tools": list(tools),
+                "enabled_tool_toggles": list(request.enabled_tools),
+                "source_keys": list(request.source_keys),
+                "source_labels": source_labels,
+                "include_reasoning": bool(request.include_reasoning),
+            },
+        }
+    )
+    return config
+
+
 def extract_web_search_queries(response_metadata: Any) -> list[str]:
     """Pull the queries Gemini ran against google_search from grounding metadata."""
     if not isinstance(response_metadata, dict):
@@ -643,7 +728,7 @@ async def stream_chat(request: ChatRequest) -> AsyncIterator[ChatEvent]:
 
     async for chunk in agent.astream(
         {"messages": [{"role": "user", "content": request.query}]},
-        thread_config(active_thread_id),
+        agent_run_config(request, active_thread_id, message_id),
         context=AppContext(allowed_sources=request.source_keys),
         stream_mode=["messages", "updates"],
         version="v2",

scripts/main.py

@@ -2,7 +2,12 @@ from __future__ import annotations
 
 import gradio as gr
 
-from .chat_service import ChatRequest, normalize_history, stream_chat
+from .chat_service import (
+    ChatRequest,
+    normalize_history,
+    stream_chat,
+    warm_up_retriever,
+)
 from .gradio_presenter import GradioPresenterState
 from .setup import (
     AVAILABLE_SOURCES_UI,
@@ -63,7 +68,7 @@ model = gr.Textbox(
     label="Model (provider:model)",
     value=DEFAULT_MODEL_NAME,
     interactive=False,
-    placeholder="openai:gpt-5.4-mini | anthropic:claude-opus-4-6 | google-genai:gemini-3.1-pro-preview",
+    placeholder="openai:gpt-5.4-mini | anthropic:claude-opus-4-6 | google-genai:gemini-3.5-flash",
 )
 enable_web_search = gr.Checkbox(
     label="Web search",
@@ -128,7 +133,13 @@ with gr.Blocks(
     gr.ChatInterface(
         fn=generate_completion,
         chatbot=chatbot,
-        additional_inputs=[sources, model, thread_id, enable_web_search, enable_url_read],
+        additional_inputs=[
+            sources,
+            model,
+            thread_id,
+            enable_web_search,
+            enable_url_read,
+        ],
         additional_outputs=[thread_id],
         additional_inputs_accordion=accordion,
         api_name="chat",
@@ -136,4 +147,5 @@ with gr.Blocks(
 
 
 if __name__ == "__main__":
+    warm_up_retriever()
     demo.launch(server_name="0.0.0.0", server_port=7860, debug=False, share=False)

scripts/setup.py

@@ -12,22 +12,31 @@ from data.scraping_scripts.source_registry import (
     SOURCE_KEY_TO_LABEL,
     SOURCE_UI_TO_KEY,
 )
+
+from .agent_tracing import configure_langsmith_environment, langsmith_tracing_enabled
 from .utils import init_mongo_db
 
 load_dotenv(override=True)
+configure_langsmith_environment()
 try:
     logfire.configure()
 except Exception:
     pass
 
+if langsmith_tracing_enabled():
+    logfire.info(
+        "LangSmith tracing enabled.",
+        project=os.getenv("LANGSMITH_PROJECT", "default"),
+    )
+
 VECTOR_DB_DIR = "data/chroma-db-all_sources"
 VECTOR_COLLECTION_NAME = "chroma-db-all_sources"
 DOCUMENT_DICT_PATH = f"{VECTOR_DB_DIR}/document_dict_all_sources.pkl"
 BM25_INDEX_PATH = f"{VECTOR_DB_DIR}/bm25_index_all_sources.pkl"
-DEFAULT_MODEL_NAME = "google-genai:gemini-3-flash-preview"
+DEFAULT_MODEL_NAME = "google-genai:gemini-3.5-flash"
 
 AVAILABLE_MODELS: tuple[dict[str, str], ...] = (
-    {"id": "google-genai:gemini-3-flash-preview", "label": "Gemini 3 Flash Preview"},
+    {"id": "google-genai:gemini-3.5-flash", "label": "Gemini 3.5 Flash"},
     {"id": "anthropic:claude-haiku-4-5", "label": "Claude Haiku 4.5"},
 )
 

tests/test_agent_tracing.py

@@ -0,0 +1,59 @@
+from __future__ import annotations
+
+import os
+import unittest
+from unittest.mock import patch
+
+from scripts.agent_tracing import (
+    DEFAULT_LANGSMITH_PROJECT,
+    configure_langsmith_environment,
+    langsmith_tracing_enabled,
+    parse_env_bool,
+)
+
+
+class AgentTracingTestCase(unittest.TestCase):
+    def test_parse_env_bool(self) -> None:
+        self.assertIs(parse_env_bool(None), None)
+        self.assertTrue(parse_env_bool("true"))
+        self.assertTrue(parse_env_bool("1"))
+        self.assertFalse(parse_env_bool("false"))
+        self.assertFalse(parse_env_bool("0"))
+        self.assertIs(parse_env_bool("maybe"), None)
+
+    def test_api_key_enables_langsmith_by_default(self) -> None:
+        with patch.dict(os.environ, {"LANGSMITH_API_KEY": "ls_test"}, clear=True):
+            configure_langsmith_environment()
+
+            self.assertTrue(langsmith_tracing_enabled())
+            self.assertEqual(os.environ["LANGSMITH_TRACING"], "true")
+            self.assertEqual(os.environ["LANGSMITH_PROJECT"], DEFAULT_LANGSMITH_PROJECT)
+
+    def test_explicit_disable_wins_over_api_key(self) -> None:
+        with patch.dict(
+            os.environ,
+            {"LANGSMITH_API_KEY": "ls_test", "LANGSMITH_TRACING": "false"},
+            clear=True,
+        ):
+            configure_langsmith_environment()
+
+            self.assertFalse(langsmith_tracing_enabled())
+            self.assertNotIn("LANGSMITH_PROJECT", os.environ)
+
+    def test_project_is_preserved(self) -> None:
+        with patch.dict(
+            os.environ,
+            {
+                "LANGSMITH_API_KEY": "ls_test",
+                "LANGSMITH_TRACING": "true",
+                "LANGSMITH_PROJECT": "custom-project",
+            },
+            clear=True,
+        ):
+            configure_langsmith_environment()
+
+            self.assertEqual(os.environ["LANGSMITH_PROJECT"], "custom-project")
+
+
+if __name__ == "__main__":
+    unittest.main()

tests/test_chat_service.py

@@ -6,8 +6,13 @@ from unittest.mock import patch
 
 from langchain_core.messages import AIMessage, HumanMessage
 
-from scripts.chat_service import sync_thread_with_history
-from scripts.chat_types import ChatTurn
+from scripts.chat_service import (
+    agent_run_config,
+    build_agent,
+    effective_tool_names,
+    sync_thread_with_history,
+)
+from scripts.chat_types import ChatRequest, ChatTurn
 
 
 class FakeAgent:
@@ -23,6 +28,99 @@ class FakeAgent:
 
 
 class ChatServiceTestCase(unittest.TestCase):
+    def test_effective_tool_names_follow_provider(self) -> None:
+        self.assertEqual(
+            effective_tool_names(
+                "google-genai:gemini-3.5-flash",
+                ("web_search", "url_context", "web_fetch"),
+            ),
+            ("retrieve_tutor_context", "google_search", "url_context"),
+        )
+        self.assertEqual(
+            effective_tool_names(
+                "anthropic:claude-haiku-4-5",
+                ("web_search", "url_context", "web_fetch"),
+            ),
+            ("retrieve_tutor_context", "web_search", "web_fetch"),
+        )
+
+    def test_agent_run_config_adds_langsmith_metadata(self) -> None:
+        request = ChatRequest(
+            query="What is RAG?",
+            source_keys=("langchain", "transformers"),
+            model_name="google-genai:gemini-3.5-flash",
+            include_reasoning=True,
+            enabled_tools=("web_search",),
+        )
+
+        config = agent_run_config(request, "thread_123", "message_456")
+
+        self.assertEqual(config["configurable"], {"thread_id": "thread_123"})
+        self.assertEqual(config["run_name"], "ai-tutor-agent-turn")
+        self.assertIn("provider:google-genai", config["tags"])
+        self.assertIn("tool:retrieve_tutor_context", config["tags"])
+        self.assertIn("tool:google_search", config["tags"])
+        self.assertEqual(config["metadata"]["thread_id"], "thread_123")
+        self.assertEqual(config["metadata"]["conversation_id"], "thread_123")
+        self.assertEqual(config["metadata"]["message_id"], "message_456")
+        self.assertEqual(
+            config["metadata"]["available_tools"],
+            ["retrieve_tutor_context", "google_search"],
+        )
+        self.assertEqual(
+            config["metadata"]["source_keys"],
+            ["langchain", "transformers"],
+        )
+
+    def test_build_agent_cache_keys_include_tool_toggles(self) -> None:
+        build_agent.cache_clear()
+        created_agents = []
+
+        def fake_create_agent(**kwargs):
+            agent = types.SimpleNamespace(kwargs=kwargs)
+            created_agents.append(agent)
+            return agent
+
+        try:
+            with (
+                patch("scripts.chat_service.build_chat_model", return_value=object()),
+                patch(
+                    "scripts.chat_service.build_system_prompt", return_value="prompt"
+                ),
+                patch(
+                    "scripts.chat_service.create_agent", side_effect=fake_create_agent
+                ),
+            ):
+                with_web_tools = build_agent(
+                    "google-genai:gemini-3.5-flash",
+                    enabled_tools=("web_search", "url_context"),
+                    include_thoughts=True,
+                )
+                without_web_tools = build_agent(
+                    "google-genai:gemini-3.5-flash",
+                    enabled_tools=(),
+                    include_thoughts=True,
+                )
+                without_web_tools_again = build_agent(
+                    "google-genai:gemini-3.5-flash",
+                    enabled_tools=(),
+                    include_thoughts=True,
+                )
+        finally:
+            build_agent.cache_clear()
+
+        self.assertIsNot(with_web_tools, without_web_tools)
+        self.assertIs(without_web_tools, without_web_tools_again)
+        self.assertEqual(len(created_agents), 2)
+
+        enabled_tool_defs = created_agents[0].kwargs["tools"]
+        disabled_tool_defs = created_agents[1].kwargs["tools"]
+        self.assertIn({"google_search": {}}, enabled_tool_defs)
+        self.assertIn({"url_context": {}}, enabled_tool_defs)
+        self.assertEqual(disabled_tool_defs, [enabled_tool_defs[0]])
+        self.assertEqual(len(created_agents[0].kwargs["middleware"]), 1)
+        self.assertEqual(created_agents[1].kwargs["middleware"], [])
+
     def test_shorter_history_branches_to_fresh_thread(self) -> None:
         agent = FakeAgent(
             [