add knowledge base wiki exploration tool

.gitignore

@@ -164,6 +164,12 @@ cython_debug/
 .vscode/
 data/chroma-db**/
 evaluation_data/chroma-db**/
+# `data/kb/` is built by data.scraping_scripts.build_kb_artifacts and
+# uploaded to HuggingFace by data.scraping_scripts.upload_dbs_to_hf.
+# The runtime downloads it on first start (see scripts/setup.ensure_local_vector_db).
+# Treat it as a build artifact, not a source file. The canonical AGENTS.md
+# content lives in data/scraping_scripts/kb_agents_template.md.
+data/kb/
 data/*_md_files/
 data/full_stack_ai_engineering/
 data/beginner_python_for_ai_engineering/

README.md

@@ -181,11 +181,13 @@ Notes:
 After updating the JSONL corpus, rebuild the local Chroma index with:
 
 ```bash
+uv run -m data.scraping_scripts.build_kb_artifacts
+uv run -m data.scraping_scripts.update_kb_wiki --changed-only
 uv run -m data.scraping_scripts.add_context_to_nodes
 uv run -m data.scraping_scripts.create_vector_stores all_sources
 ```
 
-The first command now builds the chunk manifest used by the workflows, and the second command writes dense embeddings into the local Chroma database.
+The KB commands generate browseable markdown, indexes, and wiki navigation pages for the agent. The context command builds the chunk manifest used by the workflows, and the vector command writes dense embeddings into the local Chroma database.
 The vector-store build now shows embedding and Chroma upsert progress in the terminal.
 
 ### Updating Data Sources

data/.gitattributes

@@ -0,0 +1,65 @@
+*.7z filter=lfs diff=lfs merge=lfs -text
+*.arrow filter=lfs diff=lfs merge=lfs -text
+*.bin filter=lfs diff=lfs merge=lfs -text
+*.bz2 filter=lfs diff=lfs merge=lfs -text
+*.ckpt filter=lfs diff=lfs merge=lfs -text
+*.ftz filter=lfs diff=lfs merge=lfs -text
+*.gz filter=lfs diff=lfs merge=lfs -text
+*.h5 filter=lfs diff=lfs merge=lfs -text
+*.joblib filter=lfs diff=lfs merge=lfs -text
+*.lfs.* filter=lfs diff=lfs merge=lfs -text
+*.lz4 filter=lfs diff=lfs merge=lfs -text
+*.mlmodel filter=lfs diff=lfs merge=lfs -text
+*.model filter=lfs diff=lfs merge=lfs -text
+*.msgpack filter=lfs diff=lfs merge=lfs -text
+*.npy filter=lfs diff=lfs merge=lfs -text
+*.npz filter=lfs diff=lfs merge=lfs -text
+*.onnx filter=lfs diff=lfs merge=lfs -text
+*.ot filter=lfs diff=lfs merge=lfs -text
+*.parquet filter=lfs diff=lfs merge=lfs -text
+*.pb filter=lfs diff=lfs merge=lfs -text
+*.pickle filter=lfs diff=lfs merge=lfs -text
+*.pkl filter=lfs diff=lfs merge=lfs -text
+*.pt filter=lfs diff=lfs merge=lfs -text
+*.pth filter=lfs diff=lfs merge=lfs -text
+*.rar filter=lfs diff=lfs merge=lfs -text
+*.safetensors filter=lfs diff=lfs merge=lfs -text
+saved_model/**/* filter=lfs diff=lfs merge=lfs -text
+*.tar.* filter=lfs diff=lfs merge=lfs -text
+*.tar filter=lfs diff=lfs merge=lfs -text
+*.tflite filter=lfs diff=lfs merge=lfs -text
+*.tgz filter=lfs diff=lfs merge=lfs -text
+*.wasm filter=lfs diff=lfs merge=lfs -text
+*.xz filter=lfs diff=lfs merge=lfs -text
+*.zip filter=lfs diff=lfs merge=lfs -text
+*.zst filter=lfs diff=lfs merge=lfs -text
+*tfevents* filter=lfs diff=lfs merge=lfs -text
+# Audio files - uncompressed
+*.pcm filter=lfs diff=lfs merge=lfs -text
+*.sam filter=lfs diff=lfs merge=lfs -text
+*.raw filter=lfs diff=lfs merge=lfs -text
+# Audio files - compressed
+*.aac filter=lfs diff=lfs merge=lfs -text
+*.flac filter=lfs diff=lfs merge=lfs -text
+*.mp3 filter=lfs diff=lfs merge=lfs -text
+*.ogg filter=lfs diff=lfs merge=lfs -text
+*.wav filter=lfs diff=lfs merge=lfs -text
+# Image files - uncompressed
+*.bmp filter=lfs diff=lfs merge=lfs -text
+*.gif filter=lfs diff=lfs merge=lfs -text
+*.png filter=lfs diff=lfs merge=lfs -text
+*.tiff filter=lfs diff=lfs merge=lfs -text
+# Image files - compressed
+*.jpg filter=lfs diff=lfs merge=lfs -text
+*.jpeg filter=lfs diff=lfs merge=lfs -text
+*.webp filter=lfs diff=lfs merge=lfs -text
+ai-tutor-vector-db/chroma.sqlite3 filter=lfs diff=lfs merge=lfs -text
+chroma.sqlite3 filter=lfs diff=lfs merge=lfs -text
+ai_for_business_professionals/AI[[:space:]]for[[:space:]]Business[[:space:]]Professionals[[:space:]]190f9b6f42708087863df100e9b4b556/(Video)[[:space:]]AI[[:space:]]Apps[[:space:]]and[[:space:]]AI[[:space:]]Audio[[:space:]]Using[[:space:]]NotebookLM[[:space:]]to[[:space:]]L[[:space:]]1b0f9b6f4270800f8164d5a4a365196a/Screen_Recording_2025-03-14_at_10.05.04.mov filter=lfs diff=lfs merge=lfs -text
+ai_for_business_professionals/AI[[:space:]]for[[:space:]]Business[[:space:]]Professionals[[:space:]]190f9b6f42708087863df100e9b4b556/(Video)[[:space:]]AI[[:space:]]Apps[[:space:]]and[[:space:]]AI[[:space:]]Audio[[:space:]]Using[[:space:]]NotebookLM[[:space:]]to[[:space:]]L[[:space:]]1b0f9b6f4270800f8164d5a4a365196a/Screen_Recording_2025-03-14_at_10.15.13.mov filter=lfs diff=lfs merge=lfs -text
+ai_for_business_professionals/AI[[:space:]]for[[:space:]]Business[[:space:]]Professionals[[:space:]]190f9b6f42708087863df100e9b4b556/(video)[[:space:]]Making[[:space:]]Prompts[[:space:]]Robust[[:space:]]and[[:space:]]Ready[[:space:]]for[[:space:]]Re-Use[[:space:]]190f9b6f42708042bed5e17583e364a9/1.mov filter=lfs diff=lfs merge=lfs -text
+ai_for_business_professionals/AI[[:space:]]for[[:space:]]Business[[:space:]]Professionals[[:space:]]190f9b6f42708087863df100e9b4b556/(video)[[:space:]]Making[[:space:]]Prompts[[:space:]]Robust[[:space:]]and[[:space:]]Ready[[:space:]]for[[:space:]]Re-Use[[:space:]]190f9b6f42708042bed5e17583e364a9/2.mov filter=lfs diff=lfs merge=lfs -text
+ai_for_business_professionals/AI[[:space:]]for[[:space:]]Business[[:space:]]Professionals[[:space:]]190f9b6f42708087863df100e9b4b556/(video)[[:space:]]Making[[:space:]]Prompts[[:space:]]Robust[[:space:]]and[[:space:]]Ready[[:space:]]for[[:space:]]Re-Use[[:space:]]190f9b6f42708042bed5e17583e364a9/3.mov filter=lfs diff=lfs merge=lfs -text
+ai_for_business_professionals/AI[[:space:]]for[[:space:]]Business[[:space:]]Professionals[[:space:]]190f9b6f42708087863df100e9b4b556/(video)[[:space:]]Making[[:space:]]Prompts[[:space:]]Robust[[:space:]]and[[:space:]]Ready[[:space:]]for[[:space:]]Re-Use[[:space:]]190f9b6f42708042bed5e17583e364a9/4.mov filter=lfs diff=lfs merge=lfs -text
+ai_for_business_professionals/AI[[:space:]]for[[:space:]]Business[[:space:]]Professionals[[:space:]]190f9b6f42708087863df100e9b4b556/Using[[:space:]]LLMs[[:space:]]for[[:space:]]Research[[:space:]](Perplexity)[[:space:]]-[[:space:]]Video[[:space:]]190f9b6f42708069954bd75cc797a378/Screen_Recording_2024-11-27_at_17.10.38.mov filter=lfs diff=lfs merge=lfs -text
+chroma-db-all_sources/chroma.sqlite3 filter=lfs diff=lfs merge=lfs -text

data/scraping_scripts/README.md

@@ -138,6 +138,79 @@ To update library documentation from GitHub repositories:
 uv run -m data.scraping_scripts.update_docs_workflow
 ```
 
+### Rebuilding the KB from scratch
+
+If you've blown away local KB state (e.g. `rm -rf data/kb data/chroma-db-all_sources`)
+and want everything back, **just run the docs workflow** — it now handles
+fresh-build cases automatically.
+
+```bash
+uv run -m data.scraping_scripts.update_docs_workflow
+```
+
+What happens under the hood when local state is missing:
+
+1. `ensure_required_files_exist` downloads `all_sources_data.jsonl`,
+   `all_sources_contextual_nodes.pkl`, and every per-source JSONL (including
+   courses) from the private HF dataset `towardsai-tutors/ai-tutor-data`.
+2. Fresh docs are re-fetched from GitHub / llms.txt indexes.
+3. `build_kb_artifacts.py` regenerates `data/kb/raw/` and `data/kb/generated/`
+   from the JSONL. Course raw pages are reconstructed from the JSONL too —
+   you do **not** need the original Notion exports in `data/<course_name>/`.
+4. `update_kb_wiki.py --changed-only` rebuilds `data/kb/wiki/`. When it
+   detects an empty `wiki/` it auto-promotes to a full seed (topic pages,
+   recipes/errors index pages), so a clean rebuild produces the full wiki.
+5. `data/kb/AGENTS.md` is overwritten from
+   `data/scraping_scripts/kb_agents_template.md` (the canonical template,
+   tracked in git so it survives any `rm -rf data/`).
+6. Chroma vector store is rebuilt by `create_vector_stores.py`.
+
+Prerequisites for a from-scratch rebuild:
+
+- `HF_TOKEN` with access to `towardsai-tutors/ai-tutor-data` (so course
+  JSONLs and the contextual nodes PKL can be downloaded).
+- `GITHUB_TOKEN` (rate-limit headroom for docs downloads).
+- `GEMINI_API_KEY` or `GOOGLE_API_KEY` (used by `add_context_to_nodes.py`).
+- `COHERE_API_KEY` (used by `create_vector_stores.py`).
+
+What this workflow **cannot** restore: the original Notion-exported course
+markdown in `data/<course_name>/`. You don't need it for the runtime — the
+chunked content lives in the JSONL on HF — but if you want to re-run
+`add_course_workflow.py` against fresh course content, you'd need a new
+Notion export.
+
+### Editing KB agent guidance permanently
+
+Do **not** edit `data/kb/AGENTS.md` directly. The file is overwritten in two
+places, both reading from the canonical template at
+`data/scraping_scripts/kb_agents_template.md`:
+
+- `data.scraping_scripts.update_kb_wiki.write_agents_md` — rewrites it during
+  every `update_docs_workflow.py` run, before uploading to HuggingFace.
+- `scripts.setup.ensure_kb_agents_md` — rewrites it on every runtime startup,
+  after `ensure_local_vector_db` downloads the snapshot. This catches the
+  case where the HF snapshot has a stale AGENTS.md (e.g. uploaded before a
+  template change landed in git) and ensures the live file always matches
+  the version checked into the current branch.
+
+Edit the template, commit it, and either run the docs workflow (to refresh
+the HF snapshot) or just restart the chatbot (the template gets copied into
+place on startup either way).
+
+### Where `data/kb/` actually lives
+
+`data/kb/` is gitignored (it's a 100MB+ build artifact). It's:
+
+- **Built** by `data.scraping_scripts.build_kb_artifacts` (from
+  `data/all_sources_data.jsonl`) and `data.scraping_scripts.update_kb_wiki`.
+- **Uploaded** to `towardsai-tutors/ai-tutor-vector-db` by
+  `data.scraping_scripts.upload_dbs_to_hf` (already includes `kb/**`).
+- **Downloaded** by `scripts.setup.ensure_local_vector_db` on the first
+  chatbot start (or any start where the local KB is missing).
+
+Treat it the same way as `data/chroma-db-all_sources/`: never commit it,
+never edit it by hand, regenerate or re-download when you need it.
+
 This will update all supported documentation sources. You can also specify specific sources:
 
 ```bash
@@ -225,3 +298,9 @@ leaving the source configured as active for a future rebuild.
 6. Vector-store creation:
    - `create_vector_stores.py` now shows terminal progress for embedding generation and Chroma upserts
    - `all_sources_contextual_nodes.pkl` stores chunk records for embedding; legacy pickles containing LlamaIndex nodes are still accepted by the runtime compatibility helpers
+
+7. KB wiki artifact creation:
+   - `build_kb_artifacts.py` converts `all_sources_data.jsonl` into generated markdown under `data/kb/raw/`
+   - `update_kb_wiki.py --changed-only` refreshes wiki navigation pages under `data/kb/wiki/`
+   - The docs and course workflows run these steps automatically unless `--skip-kb` is passed
+   - `upload_dbs_to_hf.py` uploads `data/kb/**` with the vector database so the runtime can download one KB bundle

data/scraping_scripts/add_course_workflow.py

@@ -381,6 +381,21 @@ def create_vector_stores() -> None:
     logger.info("Successfully created vector stores")
 
 
+def build_kb_artifacts() -> None:
+    """Build generated markdown corpus, indexes, and refreshed wiki pages."""
+    logger.info("Building KB artifacts")
+    result = run_module("data.scraping_scripts.build_kb_artifacts")
+    if result.returncode != 0:
+        logger.error("Error building KB artifacts - check output above")
+        sys.exit(1)
+
+    result = run_module("data.scraping_scripts.update_kb_wiki", "--changed-only")
+    if result.returncode != 0:
+        logger.error("Error updating KB wiki - check output above")
+        sys.exit(1)
+    logger.info("Successfully built KB artifacts")
+
+
 def upload_to_huggingface(upload_jsonl: bool = False) -> None:
     """Upload databases to HuggingFace."""
     logger.info("Uploading databases to HuggingFace")
@@ -465,6 +480,11 @@ def main():
     parser.add_argument(
         "--skip-vectors", action="store_true", help="Skip vector store creation"
     )
+    parser.add_argument(
+        "--skip-kb",
+        action="store_true",
+        help="Skip generated KB markdown/wiki artifact creation",
+    )
     parser.add_argument(
         "--skip-upload", action="store_true", help="Skip uploading to HuggingFace"
     )
@@ -514,6 +534,9 @@ def main():
     if not args.skip_merge:
         rebuild_all_sources(courses)
 
+    if not args.skip_kb:
+        build_kb_artifacts()
+
     if args.purge_sources:
         purge_sources_from_pkl(args.purge_sources)
 

data/scraping_scripts/build_kb_artifacts.py

@@ -0,0 +1,546 @@
+"""Build agent-browsable KB artifacts from the normalized JSONL corpus."""
+
+from __future__ import annotations
+
+import argparse
+import csv
+import json
+import logging
+import re
+import shutil
+import uuid
+from dataclasses import dataclass
+from pathlib import Path
+from typing import Any, Iterable
+
+try:
+    from data.scraping_scripts.process_md_files import (
+        clean_document_content,
+        content_sha256,
+        extract_title,
+        generate_url,
+        load_source_extension_manifest,
+        load_source_url_manifest,
+        should_include_file,
+        split_frontmatter,
+        slugify_identifier,
+        stable_doc_id,
+    )
+    from data.scraping_scripts.source_registry import COURSE_SOURCE_KEYS, SOURCE_CONFIGS
+except ModuleNotFoundError:
+    from process_md_files import (
+        clean_document_content,
+        content_sha256,
+        extract_title,
+        generate_url,
+        load_source_extension_manifest,
+        load_source_url_manifest,
+        should_include_file,
+        split_frontmatter,
+        slugify_identifier,
+        stable_doc_id,
+    )
+    from source_registry import COURSE_SOURCE_KEYS, SOURCE_CONFIGS
+
+
+DEFAULT_INPUT_FILE = Path("data/all_sources_data.jsonl")
+DEFAULT_OUTPUT_DIR = Path("data/kb")
+RAW_DIR_NAME = "raw"
+GENERATED_DIR_NAME = "generated"
+MARKDOWN_EXTENSIONS = {".md", ".mdx"}
+HEADING_RE = re.compile(r"^(#{1,6})\s+(.+?)\s*#*\s*$")
+FENCE_RE = re.compile(r"^\s*(`{3,}|~{3,})")
+INLINE_CODE_RE = re.compile(r"`([^`\n]{2,160})`")
+SYMBOL_RE = re.compile(
+    r"\b[A-Za-z_][A-Za-z0-9_]*(?:\.[A-Za-z_][A-Za-z0-9_]*)+\b|"
+    r"\b[a-z_]+_[a-z0-9_]+\b|"
+    r"\b[A-Z][A-Za-z0-9_]{2,}\b"
+)
+logger = logging.getLogger(__name__)
+
+
+@dataclass(frozen=True)
+class CorpusRecord:
+    doc_id: str
+    legacy_doc_id: str
+    title: str
+    source: str
+    source_group: str
+    source_path: str
+    url: str
+    tokens: int
+    content_hash: str
+    content: str
+
+
+@dataclass(frozen=True)
+class OriginalMarkdown:
+    source_path: str
+    path: Path
+    url: str
+    title: str
+    content_hash: str
+    content: str
+
+
+@dataclass
+class OriginalMarkdownIndex:
+    by_source_path: dict[str, OriginalMarkdown]
+    by_url: dict[str, list[OriginalMarkdown]]
+    by_content_hash: dict[str, list[OriginalMarkdown]]
+    by_title: dict[str, list[OriginalMarkdown]]
+
+
+def load_jsonl(path: Path) -> list[dict[str, Any]]:
+    rows: list[dict[str, Any]] = []
+    with path.open("r", encoding="utf-8") as handle:
+        for line in handle:
+            if line.strip():
+                rows.append(json.loads(line))
+    return rows
+
+
+def looks_like_uuid(value: str) -> bool:
+    try:
+        uuid.UUID(value)
+    except (TypeError, ValueError):
+        return False
+    return True
+
+
+def normalize_source_path(row: dict[str, Any]) -> str:
+    source_path = str(row.get("source_path") or "").replace("\\", "/").strip("/")
+    path = Path(source_path)
+    if path.is_absolute() or ".." in path.parts:
+        return ""
+    return source_path
+
+
+def normalize_record(row: dict[str, Any]) -> CorpusRecord:
+    content = str(row.get("content") or "")
+    source = str(row.get("source") or "unknown")
+    title = str(row.get("name") or row.get("title") or "Untitled")
+    url = str(row.get("url") or "")
+    source_path = normalize_source_path(row)
+    if source in COURSE_SOURCE_KEYS:
+        source_path = ""
+    content_hash = str(row.get("content_hash") or content_sha256(content))
+    raw_doc_id = str(row.get("doc_id") or "")
+    if raw_doc_id and not looks_like_uuid(raw_doc_id) and ":" in raw_doc_id:
+        doc_id = raw_doc_id
+    else:
+        doc_id = stable_doc_id(
+            source=source,
+            source_path=source_path,
+            title=title,
+            url=url,
+            content_hash=content_hash,
+        )
+    try:
+        tokens = int(row.get("tokens") or 0)
+    except (TypeError, ValueError):
+        tokens = 0
+    return CorpusRecord(
+        doc_id=doc_id,
+        legacy_doc_id=raw_doc_id if raw_doc_id != doc_id else "",
+        title=title,
+        source=source,
+        source_group="courses" if source in COURSE_SOURCE_KEYS else "docs",
+        source_path=source_path,
+        url=url,
+        tokens=tokens,
+        content_hash=content_hash,
+        content=content,
+    )
+
+
+def quote_frontmatter(value: str) -> str:
+    return json.dumps(value, ensure_ascii=False)
+
+
+def strip_existing_frontmatter(content: str) -> str:
+    frontmatter, body = split_frontmatter(content)
+    return body if frontmatter is not None else content
+
+
+def markdown_with_frontmatter(
+    record: CorpusRecord,
+    *,
+    generated_from: Path,
+    content: str,
+    source_path: str = "",
+    original_path: str = "",
+) -> str:
+    frontmatter_source_path = source_path or record.source_path
+    lines = [
+        "---",
+        f"doc_id: {quote_frontmatter(record.doc_id)}",
+        f"legacy_doc_id: {quote_frontmatter(record.legacy_doc_id)}",
+        f"source: {quote_frontmatter(record.source)}",
+        f"source_group: {quote_frontmatter(record.source_group)}",
+        f"title: {quote_frontmatter(record.title)}",
+        f"url: {quote_frontmatter(record.url)}",
+        f"source_path: {quote_frontmatter(frontmatter_source_path)}",
+        f"tokens: {record.tokens}",
+        f"content_hash: {quote_frontmatter(record.content_hash)}",
+        f"generated_from: {quote_frontmatter(str(generated_from))}",
+    ]
+    if original_path:
+        lines.append(f"original_path: {quote_frontmatter(original_path)}")
+    lines.extend(["---", ""])
+    body = strip_existing_frontmatter(content).strip()
+    return "\n".join(lines) + body + "\n"
+
+
+def record_slug(record: CorpusRecord) -> str:
+    basis = record.source_path or record.title or record.doc_id
+    slug = slugify_identifier(basis)
+    if slug == "untitled":
+        slug = record.content_hash.removeprefix("sha256:")[:12]
+    return slug
+
+
+def safe_relative_path(value: str) -> Path:
+    path = Path(value.replace("\\", "/"))
+    if path.is_absolute() or ".." in path.parts:
+        raise ValueError(f"Unsafe source path: {value}")
+    return path
+
+
+def unique_path(candidate: Path, used_paths: set[Path], content_hash: str) -> Path:
+    if candidate not in used_paths:
+        used_paths.add(candidate)
+        return candidate
+
+    suffix = content_hash.removeprefix("sha256:")[:8]
+    stem = candidate.stem
+    candidate = candidate.with_name(f"{stem}-{suffix}{candidate.suffix}")
+    counter = 2
+    while candidate in used_paths:
+        candidate = candidate.with_name(
+            f"{stem}-{suffix}-{counter}{candidate.suffix}"
+        )
+        counter += 1
+    used_paths.add(candidate)
+    return candidate
+
+
+def unique_output_path(
+    record: CorpusRecord,
+    used_paths: set[Path],
+    raw_dir: Path,
+) -> Path:
+    slug = record_slug(record)
+    base_dir = raw_dir / record.source_group / record.source
+    candidate = base_dir / f"{slug}.md"
+    if candidate not in used_paths:
+        used_paths.add(candidate)
+        return candidate
+
+    suffix = record.content_hash.removeprefix("sha256:")[:8]
+    candidate = base_dir / f"{slug}-{suffix}.md"
+    counter = 2
+    while candidate in used_paths:
+        candidate = base_dir / f"{slug}-{suffix}-{counter}.md"
+        counter += 1
+    used_paths.add(candidate)
+    return candidate
+
+
+def add_index_value(
+    index: dict[str, list[OriginalMarkdown]],
+    key: str,
+    item: OriginalMarkdown,
+) -> None:
+    if key:
+        index.setdefault(key, []).append(item)
+
+
+def build_original_markdown_index(source: str) -> OriginalMarkdownIndex | None:
+    config = SOURCE_CONFIGS.get(source)
+    if not config:
+        return None
+
+    directory = Path(str(config.get("input_directory") or ""))
+    if not directory.exists() or not directory.is_dir():
+        return None
+
+    source_extension_manifest = load_source_extension_manifest(str(directory))
+    source_url_manifest = load_source_url_manifest(str(directory))
+    by_source_path: dict[str, OriginalMarkdown] = {}
+    by_url: dict[str, list[OriginalMarkdown]] = {}
+    by_content_hash: dict[str, list[OriginalMarkdown]] = {}
+    by_title: dict[str, list[OriginalMarkdown]] = {}
+
+    for path in sorted(directory.rglob("*")):
+        if not path.is_file() or path.suffix not in MARKDOWN_EXTENSIONS:
+            continue
+        source_path = path.relative_to(directory).as_posix()
+        if not should_include_file(source_path, config):
+            continue
+        raw_content = path.read_text(encoding="utf-8")
+        content = clean_document_content(raw_content)
+        item = OriginalMarkdown(
+            source_path=source_path,
+            path=path,
+            url=generate_url(
+                source_path,
+                config,
+                source_extension_manifest.get(source_path),
+                source_url_manifest.get(source_path),
+            ),
+            title=extract_title(content) or path.name,
+            content_hash=content_sha256(content),
+            content=content,
+        )
+        by_source_path[source_path] = item
+        add_index_value(by_url, item.url, item)
+        add_index_value(by_content_hash, item.content_hash, item)
+        add_index_value(by_title, item.title, item)
+
+    return OriginalMarkdownIndex(
+        by_source_path=by_source_path,
+        by_url=by_url,
+        by_content_hash=by_content_hash,
+        by_title=by_title,
+    )
+
+
+def unique_match(items: list[OriginalMarkdown] | None) -> OriginalMarkdown | None:
+    return items[0] if items and len(items) == 1 else None
+
+
+def find_original_markdown(
+    record: CorpusRecord,
+    indexes: dict[str, OriginalMarkdownIndex | None],
+) -> OriginalMarkdown | None:
+    if record.source_group != "docs":
+        return None
+    if record.source not in indexes:
+        indexes[record.source] = build_original_markdown_index(record.source)
+    index = indexes.get(record.source)
+    if index is None:
+        return None
+
+    if record.source_path and (match := index.by_source_path.get(record.source_path)):
+        return match
+    if match := unique_match(index.by_url.get(record.url)):
+        return match
+    if match := unique_match(index.by_content_hash.get(record.content_hash)):
+        return match
+    if match := unique_match(index.by_title.get(record.title)):
+        return match
+    return None
+
+
+def manifest_row(
+    record: CorpusRecord,
+    *,
+    output_path: Path,
+    source_path: str = "",
+    original_path: str = "",
+) -> dict[str, Any]:
+    return {
+        "doc_id": record.doc_id,
+        "legacy_doc_id": record.legacy_doc_id,
+        "title": record.title,
+        "source": record.source,
+        "source_group": record.source_group,
+        "source_path": source_path or record.source_path,
+        "url": record.url,
+        "tokens": record.tokens,
+        "content_hash": record.content_hash,
+        "path": output_path.as_posix(),
+        "original_path": original_path,
+    }
+
+
+def write_raw_markdown(
+    records: list[CorpusRecord],
+    *,
+    input_file: Path,
+    output_dir: Path,
+) -> list[dict[str, Any]]:
+    raw_dir = output_dir / RAW_DIR_NAME
+    if raw_dir.exists():
+        shutil.rmtree(raw_dir)
+    raw_dir.mkdir(parents=True, exist_ok=True)
+
+    used_paths: set[Path] = set()
+    original_indexes: dict[str, OriginalMarkdownIndex | None] = {}
+    fallback_counts: dict[str, int] = {}
+    manifest: list[dict[str, Any]] = []
+    for record in records:
+        original = find_original_markdown(record, original_indexes)
+        if original:
+            output_path = raw_dir / "docs" / record.source / safe_relative_path(
+                original.source_path
+            )
+            output_path = unique_path(output_path, used_paths, record.content_hash)
+            source_path = original.source_path
+            original_path = original.path.as_posix()
+            content = original.content
+        else:
+            output_path = unique_output_path(record, used_paths, raw_dir)
+            source_path = record.source_path
+            original_path = ""
+            content = record.content
+            if record.source_group == "docs":
+                fallback_counts[record.source] = fallback_counts.get(record.source, 0) + 1
+
+        output_path.parent.mkdir(parents=True, exist_ok=True)
+        markdown = markdown_with_frontmatter(
+            record,
+            generated_from=input_file,
+            content=content,
+            source_path=source_path,
+            original_path=original_path,
+        )
+        output_path.write_text(markdown, encoding="utf-8")
+        manifest.append(
+            manifest_row(
+                record,
+                output_path=output_path,
+                source_path=source_path,
+                original_path=original_path,
+            )
+        )
+    for source, count in sorted(fallback_counts.items()):
+        logger.warning(
+            "Generated %s docs KB page(s) from JSONL because original markdown was not found.",
+            count,
+            extra={"source": source},
+        )
+    return manifest
+
+
+def iter_markdown_headings(path: Path) -> Iterable[dict[str, Any]]:
+    in_fence = False
+    fence_char = ""
+    for line_number, line in enumerate(path.read_text(encoding="utf-8").splitlines(), 1):
+        fence_match = FENCE_RE.match(line)
+        if fence_match:
+            current = fence_match.group(1)[0]
+            if not in_fence:
+                in_fence = True
+                fence_char = current
+            elif current == fence_char:
+                in_fence = False
+                fence_char = ""
+            continue
+        if in_fence:
+            continue
+        match = HEADING_RE.match(line)
+        if match:
+            yield {
+                "line": line_number,
+                "level": len(match.group(1)),
+                "heading": match.group(2).strip(),
+            }
+
+
+def heading_path_for_line(headings: list[dict[str, Any]], line_number: int) -> str:
+    stack: list[dict[str, Any]] = []
+    for heading in headings:
+        if int(heading["line"]) > line_number:
+            break
+        level = int(heading["level"])
+        stack = [item for item in stack if int(item["level"]) < level]
+        stack.append(heading)
+    return " > ".join(str(item["heading"]) for item in stack)
+
+
+def extract_symbols(text: str) -> set[str]:
+    symbols: set[str] = set()
+    for inline_match in INLINE_CODE_RE.finditer(text):
+        for match in SYMBOL_RE.finditer(inline_match.group(1)):
+            symbols.add(match.group(0))
+    for match in SYMBOL_RE.finditer(text):
+        symbols.add(match.group(0))
+    return {
+        symbol
+        for symbol in symbols
+        if len(symbol) >= 3 and not symbol.lower().startswith(("http", "www"))
+    }
+
+
+def write_generated_indexes(manifest: list[dict[str, Any]], generated_dir: Path) -> None:
+    if generated_dir.exists():
+        shutil.rmtree(generated_dir)
+    generated_dir.mkdir(parents=True, exist_ok=True)
+
+    manifest_path = generated_dir / "corpus_manifest.jsonl"
+    with manifest_path.open("w", encoding="utf-8") as handle:
+        for item in manifest:
+            handle.write(json.dumps(item, ensure_ascii=False, sort_keys=True) + "\n")
+
+    heading_rows: list[dict[str, Any]] = []
+    symbol_rows: list[dict[str, str]] = []
+    for item in manifest:
+        path = Path(str(item["path"]))
+        text = path.read_text(encoding="utf-8")
+        headings = list(iter_markdown_headings(path))
+        for heading in headings:
+            heading_rows.append(
+                {
+                    "doc_id": item["doc_id"],
+                    "source": item["source"],
+                    "title": item["title"],
+                    "path": item["path"],
+                    **heading,
+                }
+            )
+        for symbol in sorted(extract_symbols(text)):
+            first_line = 1
+            for line_number, line in enumerate(text.splitlines(), 1):
+                if symbol in line:
+                    first_line = line_number
+                    break
+            symbol_rows.append(
+                {
+                    "symbol": symbol,
+                    "source": str(item["source"]),
+                    "title": str(item["title"]),
+                    "path": str(item["path"]),
+                    "heading": heading_path_for_line(headings, first_line),
+                    "doc_id": str(item["doc_id"]),
+                }
+            )
+
+    with (generated_dir / "headings.jsonl").open("w", encoding="utf-8") as handle:
+        for row in heading_rows:
+            handle.write(json.dumps(row, ensure_ascii=False, sort_keys=True) + "\n")
+
+    with (generated_dir / "symbols.tsv").open("w", encoding="utf-8", newline="") as handle:
+        writer = csv.DictWriter(
+            handle,
+            fieldnames=["symbol", "source", "title", "path", "heading", "doc_id"],
+            delimiter="\t",
+        )
+        writer.writeheader()
+        writer.writerows(symbol_rows)
+
+
+def build_kb_artifacts(input_file: Path, output_dir: Path) -> dict[str, int]:
+    rows = load_jsonl(input_file)
+    records = [normalize_record(row) for row in rows if str(row.get("content") or "").strip()]
+    manifest = write_raw_markdown(records, input_file=input_file, output_dir=output_dir)
+    write_generated_indexes(manifest, output_dir / GENERATED_DIR_NAME)
+    return {"documents": len(records), "manifest_rows": len(manifest)}
+
+
+def main() -> None:
+    parser = argparse.ArgumentParser(description="Build local KB markdown artifacts.")
+    parser.add_argument("--input-file", default=str(DEFAULT_INPUT_FILE))
+    parser.add_argument("--output-dir", default=str(DEFAULT_OUTPUT_DIR))
+    args = parser.parse_args()
+
+    summary = build_kb_artifacts(Path(args.input_file), Path(args.output_dir))
+    print(
+        "Built KB artifacts: "
+        f"{summary['documents']} documents, {summary['manifest_rows']} manifest rows"
+    )
+
+
+if __name__ == "__main__":
+    main()

data/scraping_scripts/kb_agents_template.md

@@ -0,0 +1,50 @@
+# AI Tutor KB Instructions
+
+## Ground Truth
+
+- `raw/` contains generated markdown mirrors of the normalized JSONL corpus. Treat these pages as the local source authority.
+- `wiki/` contains navigation and synthesis pages. Use them to orient, not as final authority.
+- `generated/` contains machine-generated indexes for manifests, headings, and symbols.
+
+## Standard Workflow
+
+**First command rule.** Unless the user gives you a specific file path or a
+known symbol/class name to find, your FIRST `run_kb_command` must be one of:
+
+- `cat wiki/index.md` — for any "tell me about X" / "what is X" / comparison
+  / recipe question where X is a broad topic.
+- `cat wiki/frameworks/<source>.md` or `cat wiki/courses/<source>.md` — when
+  the question is already scoped to one known source.
+
+Do NOT start with `rg`, `grep`, `find`, or `ls` over `raw/` as the first
+command. The wiki pages already give you the source map, so doing your own
+discovery first wastes budget and pulls noisy hits into context.
+
+After the wiki orientation step:
+
+1. Use `rg`, `find`, `sed`, `head`, and `cat` to verify and read raw pages.
+2. Use generated indexes for exact symbols, headings, source paths, and URLs.
+3. Read `raw/` pages to verify exact claims, code, and version-sensitive behavior.
+4. Mention uncertainty when the knowledge base does not cover the answer.
+
+## Exploration Budget
+
+- Prefer 3-6 KB commands for normal answers. Stop browsing once you have one clearly relevant source page and enough detail to answer.
+- Use more commands only when the user asks for exhaustive coverage, a comparison across sources, or debugging that requires multiple files.
+- Do not read a long file from top to bottom. Scan headings first, then read only targeted ranges.
+- Avoid broad searches that return thousands of lines. Add `-m 20` or narrow the path before searching `raw/`.
+
+## Efficient Command Patterns
+
+- Orientation (always first for broad questions): `cat wiki/index.md`
+- Source map: `cat wiki/courses/<source>.md` or `cat wiki/frameworks/<source>.md`
+- Follow-up search (after orientation): `rg -n -m 20 "query terms" raw/<scoped-path>`
+- Heading scan: `rg -n "^#{1,3} " raw/.../<page>.md`
+- Targeted read: `sed -n 'START,ENDp' raw/.../<page>.md`
+- Exact symbols: `rg -n -m 20 "ClassName|function_name" generated/symbols.tsv raw/`
+
+## Answering From KB Browsing
+
+- Default to a concise synthesis rather than rewriting the markdown content verbatim.
+- For "how do I build X?" questions, give the minimal architecture and a small runnable skeleton; expand only if the user asks for a full implementation.
+- Cite the raw/source page inline for key claims. Do not cite every code comment or repeat the same citation after every sentence.

data/scraping_scripts/process_md_files.py

@@ -30,12 +30,13 @@ To add, modify, or retire sources, update data/scraping_scripts/source_registry.
 """
 
 import argparse
+import hashlib
 import json
 import logging
 import os
 import re
-import uuid
 from typing import Dict, List
+from urllib.parse import urlparse
 
 import tiktoken
 
@@ -262,6 +263,34 @@ def clean_document_content(content: str) -> str:
     return content.strip()
 
 
+def content_sha256(content: str) -> str:
+    return "sha256:" + hashlib.sha256(content.encode("utf-8")).hexdigest()
+
+
+def slugify_identifier(value: str) -> str:
+    value = os.path.splitext(value.replace("\\", "/"))[0]
+    value = value.strip().lower()
+    value = re.sub(r"[^a-z0-9]+", "-", value)
+    value = value.strip("-")
+    return value or "untitled"
+
+
+def stable_doc_id(
+    *,
+    source: str,
+    source_path: str,
+    title: str | None,
+    url: str,
+    content_hash: str,
+) -> str:
+    if source_path:
+        basis = source_path
+    else:
+        parsed_path = urlparse(url).path.strip("/") if url else ""
+        basis = parsed_path or title or content_hash.removeprefix("sha256:")[:12]
+    return f"{source}:{slugify_identifier(basis)}"
+
+
 def process_md_files(directory: str, config: Dict) -> List[Dict]:
     jsonl_data = []
     source_extension_manifest = load_source_extension_manifest(directory)
@@ -280,6 +309,14 @@ def process_md_files(directory: str, config: Dict) -> List[Dict]:
                     cleaned_content = clean_document_content(content)
                     title = extract_title(cleaned_content)
                     token_count = num_tokens_from_string(cleaned_content, "cl100k_base")
+                    source_path = relative_path.replace("\\", "/")
+                    url = generate_url(
+                        relative_path,
+                        config,
+                        source_extension_manifest.get(source_path),
+                        source_url_manifest.get(source_path),
+                    )
+                    hash_value = content_sha256(cleaned_content)
 
                     # Skip very small or extremely large files
                     if token_count < 100 or token_count > 200_000:
@@ -290,18 +327,19 @@ def process_md_files(directory: str, config: Dict) -> List[Dict]:
 
                     json_object = {
                         "tokens": token_count,
-                        "doc_id": str(uuid.uuid4()),
-                        "name": (title if title else file),
-                        "url": generate_url(
-                            relative_path,
-                            config,
-                            source_extension_manifest.get(
-                                relative_path.replace("\\", "/")
-                            ),
-                            source_url_manifest.get(relative_path.replace("\\", "/")),
+                        "doc_id": stable_doc_id(
+                            source=config["source_name"],
+                            source_path=source_path,
+                            title=title,
+                            url=url,
+                            content_hash=hash_value,
                         ),
+                        "name": (title if title else file),
+                        "url": url,
                         "retrieve_doc": (token_count <= 8000),
                         "source": config["source_name"],
+                        "source_path": source_path,
+                        "content_hash": hash_value,
                         "content": cleaned_content,
                     }
 

data/scraping_scripts/update_docs_workflow.py

@@ -374,6 +374,36 @@ def create_vector_stores() -> None:
     logger.info("Successfully created vector stores")
 
 
+def build_kb_artifacts() -> None:
+    """Build generated markdown corpus, indexes, and refreshed wiki pages.
+
+    Notes for a from-scratch rebuild (e.g. after `rm -rf data/kb`):
+
+    - `build_kb_artifacts.py` rebuilds `kb/raw/` and `kb/generated/` from
+      `data/all_sources_data.jsonl` (which `ensure_required_files_exist`
+      downloaded from HuggingFace if it was missing). Course raw pages come
+      from the JSONL too, so missing Notion exports don't matter at runtime.
+    - `update_kb_wiki.py` is called with `--changed-only` here, which only
+      forces overwrites on `index.md`, `log.md`, and per-source pages. The
+      topic pages (`wiki/topics/*.md`) and the recipes/errors index pages
+      need `--seed-defaults`. The script auto-promotes to `seed_defaults=True`
+      when it sees an empty `wiki/` (see `wiki_is_empty` in update_kb_wiki.py),
+      so the workflow's `--changed-only` is safe for both fresh and
+      incremental rebuilds.
+    """
+    logger.info("Building KB artifacts")
+    result = run_module("data.scraping_scripts.build_kb_artifacts")
+    if result.returncode != 0:
+        logger.error("Error building KB artifacts - check output above")
+        sys.exit(1)
+
+    result = run_module("data.scraping_scripts.update_kb_wiki", "--changed-only")
+    if result.returncode != 0:
+        logger.error("Error updating KB wiki - check output above")
+        sys.exit(1)
+    logger.info("Successfully built KB artifacts")
+
+
 def upload_to_huggingface(upload_jsonl: bool = False) -> None:
     """Upload databases to HuggingFace."""
     logger.info("Uploading databases to HuggingFace")
@@ -432,6 +462,11 @@ def main():
     parser.add_argument(
         "--skip-vectors", action="store_true", help="Skip vector store creation"
     )
+    parser.add_argument(
+        "--skip-kb",
+        action="store_true",
+        help="Skip generated KB markdown/wiki artifact creation",
+    )
     parser.add_argument(
         "--skip-upload", action="store_true", help="Skip uploading to HuggingFace"
     )
@@ -458,6 +493,9 @@ def main():
     if not args.skip_process:
         process_markdown_files(args.sources)
 
+    if not args.skip_kb:
+        build_kb_artifacts()
+
     if not args.skip_context:
         add_context_to_nodes(not args.process_all_context)
 

data/scraping_scripts/update_kb_wiki.py

@@ -0,0 +1,302 @@
+"""Seed and refresh the agent-facing KB wiki."""
+
+from __future__ import annotations
+
+import argparse
+import json
+from collections import Counter, defaultdict
+from datetime import UTC, datetime
+from pathlib import Path
+from typing import Any
+
+DEFAULT_KB_DIR = Path("data/kb")
+# Template file lives next to this script (inside `data/scraping_scripts/`) so
+# it survives `rm -rf data/kb` and stays in git. Edit the template here when
+# you want a permanent change to the KB agent guidance; the live
+# `data/kb/AGENTS.md` is overwritten from it on every run of this script.
+AGENTS_TEMPLATE_PATH = Path(__file__).resolve().parent / "kb_agents_template.md"
+AUTO_START = "<!-- AUTO-GENERATED:START -->"
+AUTO_END = "<!-- AUTO-GENERATED:END -->"
+TOPIC_KEYWORDS: dict[str, tuple[str, ...]] = {
+    "rag": ("rag", "retrieval", "vector", "embedding", "chunk"),
+    "agents": ("agent", "tool", "langgraph", "workflow", "mcp"),
+    "retrieval": ("retrieval", "retriever", "bm25", "rerank", "search"),
+    "lora": ("lora", "qlora", "adapter", "peft", "loraconfig"),
+    "quantization": ("quantization", "quantized", "bitsandbytes", "nf4", "4-bit"),
+    "fine-tuning": ("fine-tuning", "finetuning", "sft", "trainer", "trl"),
+    "model-loading": ("from_pretrained", "automodel", "tokenizer", "device_map"),
+    "tool-calling": ("tool calling", "function calling", "tools", "schema"),
+    "evaluation": ("evaluation", "eval", "benchmark", "metrics"),
+}
+
+
+def load_manifest(kb_dir: Path) -> list[dict[str, Any]]:
+    manifest_path = kb_dir / "generated" / "corpus_manifest.jsonl"
+    if not manifest_path.exists():
+        return []
+    rows: list[dict[str, Any]] = []
+    with manifest_path.open("r", encoding="utf-8") as handle:
+        for line in handle:
+            if line.strip():
+                rows.append(json.loads(line))
+    return rows
+
+
+def wiki_path(kb_dir: Path, *parts: str) -> Path:
+    return kb_dir / "wiki" / Path(*parts)
+
+
+def write_page(path: Path, content: str, *, overwrite: bool) -> bool:
+    path.parent.mkdir(parents=True, exist_ok=True)
+    if path.exists() and not overwrite:
+        return False
+    path.write_text(content.rstrip() + "\n", encoding="utf-8")
+    return True
+
+
+def shell_path(row: dict[str, Any], kb_dir: Path) -> str:
+    path = str(row.get("path") or "")
+    if not path:
+        return ""
+    try:
+        return Path(path).resolve().relative_to(kb_dir.resolve()).as_posix()
+    except ValueError:
+        pass
+    prefix = "data/kb/"
+    if path.startswith(prefix):
+        return path[len(prefix) :]
+    parts = Path(path).parts
+    if "raw" in parts:
+        return "/".join(parts[parts.index("raw") :])
+    return path
+
+
+def generated_block(content: str) -> str:
+    return f"{AUTO_START}\n{content.rstrip()}\n{AUTO_END}"
+
+
+def write_generated_section(path: Path, header: str, generated: str, *, overwrite: bool) -> None:
+    path.parent.mkdir(parents=True, exist_ok=True)
+    block = generated_block(generated)
+    if not path.exists() or overwrite:
+        path.write_text(f"{header.rstrip()}\n\n{block}\n", encoding="utf-8")
+        return
+
+    existing = path.read_text(encoding="utf-8")
+    start = existing.find(AUTO_START)
+    end = existing.find(AUTO_END, start)
+    if start != -1 and end != -1:
+        updated = existing[:start] + block + existing[end + len(AUTO_END) :]
+        path.write_text(updated.rstrip() + "\n", encoding="utf-8")
+        return
+    path.write_text(existing.rstrip() + "\n\n" + block + "\n", encoding="utf-8")
+
+
+def top_sources(manifest: list[dict[str, Any]]) -> list[tuple[str, int]]:
+    counts = Counter(str(row.get("source") or "unknown") for row in manifest)
+    return sorted(counts.items(), key=lambda item: (-item[1], item[0]))
+
+
+def seed_index(kb_dir: Path, manifest: list[dict[str, Any]], *, overwrite: bool) -> None:
+    group_by_source = {
+        str(row.get("source") or "unknown"): str(row.get("source_group") or "docs")
+        for row in manifest
+    }
+    source_lines = []
+    for source, count in top_sources(manifest):
+        folder = "courses" if group_by_source.get(source) == "courses" else "frameworks"
+        source_lines.append(f"- {source}: `wiki/{folder}/{source}.md` - {count} corpus pages")
+    topic_lines = [f"- {topic}: `wiki/topics/{topic}.md`" for topic in TOPIC_KEYWORDS]
+    content = f"""# AI Tutor KB Index
+
+This wiki is a navigation layer over the generated corpus markdown.
+
+## Sources
+
+{chr(10).join(source_lines) if source_lines else "- No corpus sources indexed yet."}
+
+## Topics
+
+{chr(10).join(topic_lines)}
+
+## Generated Indexes
+
+- Corpus manifest: `generated/corpus_manifest.jsonl`
+- Headings: `generated/headings.jsonl`
+- Symbols: `generated/symbols.tsv`
+"""
+    write_page(wiki_path(kb_dir, "index.md"), content, overwrite=overwrite)
+
+
+def seed_log(kb_dir: Path, manifest: list[dict[str, Any]], *, overwrite: bool) -> None:
+    now = datetime.now(UTC).isoformat(timespec="seconds")
+    path = wiki_path(kb_dir, "log.md")
+    entry = f"""## {now}
+
+- Generated or refreshed the KB wiki scaffold/source maps.
+- Corpus documents indexed: {len(manifest)}
+"""
+    path.parent.mkdir(parents=True, exist_ok=True)
+    if not path.exists():
+        path.write_text(f"# KB Log\n\n{entry.rstrip()}\n", encoding="utf-8")
+        return
+    existing = path.read_text(encoding="utf-8").rstrip()
+    path.write_text(f"{existing}\n\n{entry.rstrip()}\n", encoding="utf-8")
+
+
+def seed_source_pages(kb_dir: Path, manifest: list[dict[str, Any]], *, overwrite: bool) -> None:
+    by_source: dict[str, list[dict[str, Any]]] = defaultdict(list)
+    for row in manifest:
+        by_source[str(row.get("source") or "unknown")].append(row)
+
+    for source, rows in sorted(by_source.items()):
+        rows = sorted(rows, key=lambda item: str(item.get("title") or ""))
+        sample = rows[:20]
+        page_links = [
+            f"- {row.get('title')}: `{shell_path(row, kb_dir)}`"
+            for row in sample
+        ]
+        group = str(rows[0].get("source_group") or "docs")
+        folder = "courses" if group == "courses" else "frameworks"
+        content = f"""# {source}
+
+Use this page to orient inside the `{source}` corpus before reading source pages directly.
+
+## Corpus Pages
+
+{chr(10).join(page_links) if page_links else "- No pages indexed."}
+
+## Verification
+
+Treat generated corpus markdown as the source of truth for this source. Use exact search for symbols, error strings, and code identifiers.
+"""
+        write_page(wiki_path(kb_dir, folder, f"{source}.md"), content, overwrite=overwrite)
+
+
+def matching_topic_rows(
+    manifest: list[dict[str, Any]],
+    keywords: tuple[str, ...],
+    *,
+    limit: int = 20,
+) -> list[dict[str, Any]]:
+    scored: list[tuple[int, dict[str, Any]]] = []
+    lowered = tuple(keyword.lower() for keyword in keywords)
+    for row in manifest:
+        title = str(row.get("title") or "")
+        path = Path(str(row.get("path") or ""))
+        text = title.lower()
+        if path.exists():
+            text += "\n" + path.read_text(encoding="utf-8").lower()[:20_000]
+        score = sum(text.count(keyword) for keyword in lowered)
+        if score:
+            scored.append((score, row))
+    scored.sort(key=lambda item: (-item[0], str(item[1].get("title") or "")))
+    return [row for _score, row in scored[:limit]]
+
+
+def seed_topic_pages(kb_dir: Path, manifest: list[dict[str, Any]], *, overwrite: bool) -> None:
+    for topic, keywords in TOPIC_KEYWORDS.items():
+        rows = matching_topic_rows(manifest, keywords)
+        links = [
+            f"- {row.get('title')}: `{shell_path(row, kb_dir)}`"
+            for row in rows
+        ]
+        header = f"""# {topic.replace("-", " ").title()}
+
+Use this page as a starting map for questions involving: {", ".join(keywords)}.
+"""
+        generated = f"""## Starting Corpus Pages
+
+{chr(10).join(links) if links else "- No strong corpus matches found yet."}
+
+## Agent Notes
+
+- Start here for orientation.
+- Verify exact factual claims in raw corpus markdown pages.
+- Use `rg` over `raw/` and `generated/symbols.tsv` for API/class/function names.
+"""
+        write_generated_section(
+            wiki_path(kb_dir, "topics", f"{topic}.md"),
+            header,
+            generated,
+            overwrite=overwrite,
+        )
+
+
+def seed_index_pages(kb_dir: Path, *, overwrite: bool) -> None:
+    for folder, title in (("recipes", "Recipes"), ("errors", "Errors")):
+        header = f"""# {title}
+
+This index starts empty. Add pages here when repeated recipes or recurring errors emerge from the corpus.
+"""
+        write_generated_section(
+            wiki_path(kb_dir, folder, "index.md"),
+            header,
+            "- No generated entries yet.",
+            overwrite=overwrite,
+        )
+
+
+def write_agents_md(kb_dir: Path) -> None:
+    # The canonical AGENTS.md content lives in `kb_agents_template.md` next to
+    # this script. Editing the template is the right way to change the agent's
+    # KB guidance for good — direct edits to `data/kb/AGENTS.md` would be
+    # clobbered on the next workflow run.
+    if not AGENTS_TEMPLATE_PATH.exists():
+        raise FileNotFoundError(
+            f"KB AGENTS.md template missing at {AGENTS_TEMPLATE_PATH}. "
+            "Restore it from git (it's tracked alongside this script)."
+        )
+    content = AGENTS_TEMPLATE_PATH.read_text(encoding="utf-8")
+    (kb_dir / "AGENTS.md").parent.mkdir(parents=True, exist_ok=True)
+    (kb_dir / "AGENTS.md").write_text(content, encoding="utf-8")
+
+
+def wiki_is_empty(kb_dir: Path) -> bool:
+    """Return True when the wiki has no real content yet (fresh build).
+
+    Used by both this script and `update_docs_workflow.py` to decide when to
+    auto-promote to `seed_defaults=True` (which seeds topic pages and the
+    recipes/errors index pages, not just the source pages). Treats a missing
+    or empty `wiki/` directory, or one with no `.md` files, as empty.
+    """
+    wiki_dir = kb_dir / "wiki"
+    if not wiki_dir.exists():
+        return True
+    return not any(wiki_dir.rglob("*.md"))
+
+
+def update_kb_wiki(kb_dir: Path, *, seed_defaults: bool, changed_only: bool) -> None:
+    manifest = load_manifest(kb_dir)
+    # Fresh-build safety net: if the wiki dir is empty (e.g. after a full
+    # `rm -rf data/kb`), promote to seed_defaults so topic/recipe/error pages
+    # are also written. Without this, only `index.md`, `log.md`, and source
+    # pages would be created and the wiki would feel half-empty to the agent.
+    if not seed_defaults and wiki_is_empty(kb_dir):
+        seed_defaults = True
+    overwrite = seed_defaults or changed_only
+    write_agents_md(kb_dir)
+    seed_index(kb_dir, manifest, overwrite=overwrite)
+    seed_log(kb_dir, manifest, overwrite=overwrite)
+    seed_source_pages(kb_dir, manifest, overwrite=overwrite)
+    seed_topic_pages(kb_dir, manifest, overwrite=seed_defaults)
+    seed_index_pages(kb_dir, overwrite=seed_defaults)
+
+
+def main() -> None:
+    parser = argparse.ArgumentParser(description="Seed or refresh the KB wiki.")
+    parser.add_argument("--kb-dir", default=str(DEFAULT_KB_DIR))
+    parser.add_argument("--seed-defaults", action="store_true")
+    parser.add_argument("--changed-only", action="store_true")
+    args = parser.parse_args()
+
+    update_kb_wiki(
+        Path(args.kb_dir),
+        seed_defaults=bool(args.seed_defaults),
+        changed_only=bool(args.changed_only),
+    )
+    print(f"Updated KB wiki in {args.kb_dir}")
+
+
+if __name__ == "__main__":
+    main()

data/scraping_scripts/upload_dbs_to_hf.py

@@ -31,8 +31,9 @@ def upload_vector_db(repo_id: str = DEFAULT_REPO_ID) -> None:
         allow_patterns=[
             "chroma-db-all_sources/**",
             "all_sources_contextual_nodes.pkl",
+            "kb/**",
         ],
-        ignore_patterns=["*.jsonl", "*.py", "*.txt", "*.ipynb", "*.md", "*.pyc", "*.mdx"],
+        ignore_patterns=["*.py", "*.ipynb", "*.pyc", "__pycache__/**"],
     )
 
 

scripts/chat_service.py

@@ -2,6 +2,7 @@ from __future__ import annotations
 
 import json
 import os
+import shlex
 from dataclasses import dataclass
 from functools import lru_cache
 from threading import Lock
@@ -10,7 +11,12 @@ from uuid import uuid4
 
 import logfire
 from langchain.agents import create_agent
-from langchain.agents.middleware import AgentMiddleware
+from langchain.agents.middleware import (
+    AgentMiddleware,
+    ClearToolUsesEdit,
+    ContextEditingMiddleware,
+    SummarizationMiddleware,
+)
 from langchain.tools import ToolRuntime, tool
 from langchain_core.messages import AIMessage, AIMessageChunk, BaseMessage, HumanMessage
 from langchain_openai import ChatOpenAI
@@ -18,6 +24,18 @@ from langgraph.checkpoint.memory import InMemorySaver
 
 from .chat_types import ChatEvent, ChatRequest, ChatTurn, SourceMatch
 from .chroma_rag import LocalChromaRetriever, format_tool_payload, parse_tool_payload
+from .kb_shell import (
+    KbCommandError,
+    format_command_payload,
+    run_kb_command as execute_kb_command,
+)
+from .kb_manifest import (
+    extract_raw_paths,
+    parse_markdown_citations,
+    resolve_manifest_reference,
+    source_match_key,
+    source_match_payload,
+)
 from .prompts import build_system_prompt
 from .setup import (
     BM25_INDEX_PATH,
@@ -39,11 +57,36 @@ LEGACY_THOUGHTS_DETAILS_OPEN = "<details><summary>Gemini thoughts</summary>"
 LEGACY_THOUGHTS_DETAILS_OPEN_EXPANDED = "<details open><summary>Gemini thoughts</summary>"
 CHECKPOINTER = InMemorySaver()
 _RETRIEVER_INIT_LOCK = Lock()
+KB_TOOL_NAMES = ("run_kb_command",)
+DEFAULT_KB_COMMAND_LIMIT = 20
+_KB_COMMAND_COUNTS: dict[str, int] = {}
+_KB_COMMAND_COUNT_LOCK = Lock()
 
 
 @dataclass(frozen=True)
 class AppContext:
     allowed_sources: tuple[str, ...]
+    kb_session_id: str = ""
+    kb_command_limit: int = DEFAULT_KB_COMMAND_LIMIT
+
+
+def _claim_kb_command_budget(session_id: str, limit: int) -> tuple[bool, int]:
+    if not session_id:
+        return True, 1
+    with _KB_COMMAND_COUNT_LOCK:
+        used = _KB_COMMAND_COUNTS.get(session_id, 0)
+        if used >= limit:
+            return False, used
+        used += 1
+        _KB_COMMAND_COUNTS[session_id] = used
+        return True, used
+
+
+def _clear_kb_command_budget(session_id: str) -> None:
+    if not session_id:
+        return
+    with _KB_COMMAND_COUNT_LOCK:
+        _KB_COMMAND_COUNTS.pop(session_id, None)
 
 
 @lru_cache(maxsize=1)
@@ -100,6 +143,68 @@ def retrieve_tutor_context(query: str, runtime: ToolRuntime[AppContext]) -> str:
     return format_tool_payload(query, results)
 
 
+RUN_KB_COMMAND_SCHEMA = {
+    "title": "run_kb_command",
+    "description": "Run a safe, read-only terminal-style command inside the local KB.",
+    "type": "object",
+    "properties": {
+        "command": {
+            "type": "string",
+            "description": (
+                "Single read-only command to run under data/kb. Supported commands: "
+                "rg, grep, find, ls, sed, head, cat, wc. Pipes, redirects, command "
+                "chaining, network commands, and writes are not allowed."
+            ),
+        },
+        "timeout_seconds": {
+            "type": "integer",
+            "description": "Command timeout in seconds, capped by the runtime.",
+            "default": 8,
+        },
+        "max_output_chars": {
+            "type": "integer",
+            "description": "Maximum stdout/stderr characters to return, capped by the runtime.",
+            "default": 40000,
+        },
+    },
+    "required": ["command"],
+}
+
+
+@tool(args_schema=RUN_KB_COMMAND_SCHEMA)
+def run_kb_command(
+    command: str,
+    runtime: ToolRuntime[AppContext],
+    timeout_seconds: int = 8,
+    max_output_chars: int = 40000,
+) -> str:
+    """Run a safe, read-only terminal-style command inside the local KB."""
+    allowed, used = _claim_kb_command_budget(
+        runtime.context.kb_session_id,
+        runtime.context.kb_command_limit,
+    )
+    if not allowed:
+        return (
+            f"$ {command}\n"
+            "error: KB command budget exceeded for this turn "
+            f"({used}/{runtime.context.kb_command_limit}). "
+            "Use the evidence already collected to answer now."
+        )
+    try:
+        ensure_local_vector_db()
+    except Exception as exc:  # pragma: no cover - diagnostic only
+        logfire.warn("KB artifact download/check failed.", error=str(exc))
+    try:
+        result = execute_kb_command(
+            command,
+            timeout_seconds=timeout_seconds,
+            max_output_chars=max_output_chars,
+        )
+        return format_command_payload(result)
+    except (KbCommandError, OSError) as exc:
+        return f"$ {command}\nerror: {exc}"
+
+
 def message_content_to_text(content: Any) -> str:
     if isinstance(content, str):
         return content
@@ -260,27 +365,114 @@ def sync_thread_with_history(
     return branched_thread_id
 
 
-def collect_updated_source_matches(
-    matches_by_doc_id: dict[str, SourceMatch],
-    payload: str,
-) -> list[SourceMatch]:
-    updated: list[SourceMatch] = []
+def collect_retrieval_source_matches(payload: str) -> list[SourceMatch]:
+    matches: list[SourceMatch] = []
     for match in parse_tool_payload(payload):
-        source_match = SourceMatch(
-            doc_id=match.doc_id,
-            title=match.title,
-            url=match.url,
-            source_key=match.source,
-            source_label=SOURCE_KEY_TO_LABEL.get(match.source, match.source),
-            score=match.score,
-            group="courses" if match.source in COURSE_SOURCE_KEYS else "docs",
+        matches.append(
+            SourceMatch(
+                doc_id=match.doc_id,
+                title=match.title,
+                url=match.url,
+                source_key=match.source,
+                source_label=SOURCE_KEY_TO_LABEL.get(match.source, match.source),
+                score=match.score,
+                group="courses" if match.source in COURSE_SOURCE_KEYS else "docs",
+            )
         )
-        existing = matches_by_doc_id.get(match.doc_id)
-        if existing and existing.score >= source_match.score:
+    return matches
+
+
+def _record_evidence(target: dict[str, SourceMatch], matches: list[SourceMatch]) -> None:
+    for match in matches:
+        key = source_match_key(match)
+        existing = target.get(key)
+        if existing and existing.score >= match.score:
             continue
-        matches_by_doc_id[match.doc_id] = source_match
-        updated.append(source_match)
-    return updated
+        target[key] = match
+
+
+def _index_evidence(matches: dict[str, SourceMatch]) -> dict[str, SourceMatch]:
+    index: dict[str, SourceMatch] = {}
+    for match in matches.values():
+        for key in (match.doc_id, match.url, match.title.strip().lower()):
+            if key:
+                index[key] = match
+    return index
+
+
+def resolve_answer_citations(
+    answer: str,
+    *,
+    retrieval_evidence: dict[str, SourceMatch],
+    shell_evidence: dict[str, SourceMatch],
+    web_evidence: dict[str, SourceMatch],
+) -> list[SourceMatch]:
+    evidence_indexes = [
+        _index_evidence(retrieval_evidence),
+        _index_evidence(shell_evidence),
+        _index_evidence(web_evidence),
+    ]
+    resolved: list[SourceMatch] = []
+    seen: set[str] = set()
+    for label, reference in parse_markdown_citations(answer):
+        candidates = [reference, label.strip().lower()]
+        match: SourceMatch | None = None
+        for index in evidence_indexes:
+            for candidate in candidates:
+                if candidate and candidate in index:
+                    match = index[candidate]
+                    break
+            if match:
+                break
+        if not match:
+            manifest_match = resolve_manifest_reference(reference, label=label)
+            if manifest_match:
+                shell_index = _index_evidence(shell_evidence)
+                if (
+                    manifest_match.doc_id in shell_index
+                    or manifest_match.url in shell_index
+                    or manifest_match.title.strip().lower() in shell_index
+                ):
+                    match = shell_index.get(manifest_match.doc_id) or shell_index.get(manifest_match.url) or shell_index.get(manifest_match.title.strip().lower())
+        if not match:
+            continue
+        key = source_match_key(match)
+        if key in seen:
+            continue
+        seen.add(key)
+        resolved.append(match)
+    return resolved
+
+
+def extract_shell_source_matches(command: str, output_text: str) -> list[SourceMatch]:
+    raw_paths = extract_raw_paths(output_text)
+    try:
+        tokens = shlex.split(command)
+    except ValueError:
+        tokens = []
+    if tokens:
+        executable = tokens[0]
+        if executable == "cat":
+            raw_paths.extend(tokens[1:])
+        elif executable == "sed" and len(tokens) >= 4:
+            raw_paths.append(tokens[-1])
+        elif executable == "head":
+            start = 1
+            if len(tokens) > 2 and tokens[1] == "-n":
+                start = 3
+            raw_paths.extend(tokens[start:])
+    matches: list[SourceMatch] = []
+    seen: set[str] = set()
+    for path in raw_paths:
+        match = resolve_manifest_reference(path)
+        if not match:
+            continue
+        key = source_match_key(match)
+        if key in seen:
+            continue
+        seen.add(key)
+        matches.append(match)
+    return matches
 
 
 def normalize_model_name(model_name: str) -> str:
@@ -335,6 +527,9 @@ def format_tool_args(args: Any) -> str:
         query = str(args.get("query", "")).strip()
         if query:
             return query
+        command = str(args.get("command", "")).strip()
+        if command:
+            return command
         if args:
             return json.dumps(args, ensure_ascii=False, sort_keys=True)
         return ""
@@ -413,8 +608,27 @@ def build_agent(
     include_thoughts: bool = False,
 ):
     model = build_chat_model(model_name, include_thoughts=include_thoughts)
-    tools: list[Any] = [retrieve_tutor_context]
-    middleware: list[AgentMiddleware] = []
+    tools: list[Any] = [
+        retrieve_tutor_context,
+        run_kb_command,
+    ]
+    middleware: list[AgentMiddleware] = [
+        ContextEditingMiddleware(
+            edits=[
+                ClearToolUsesEdit(
+                    trigger=5_000,
+                    keep=3,
+                    placeholder="[tool output cleared to save context]",
+                )
+            ],
+            token_count_method="approximate",
+        ),
+        SummarizationMiddleware(
+            model=model,
+            trigger=("tokens", 30_000),
+            keep=("messages", 20),
+        ),
+    ]
     enabled = set(enabled_tools)
     if is_google_genai_model(model_name):
         if "web_search" in enabled:
@@ -460,7 +674,7 @@ def effective_tool_names(
     model_name: str,
     enabled_tools: tuple[str, ...],
 ) -> tuple[str, ...]:
-    names = ["retrieve_tutor_context"]
+    names = ["retrieve_tutor_context", *KB_TOOL_NAMES]
     enabled = set(enabled_tools)
     if is_google_genai_model(model_name):
         if "web_search" in enabled:
@@ -693,7 +907,9 @@ def extract_anthropic_source_matches(
 
 async def stream_chat(request: ChatRequest) -> AsyncIterator[ChatEvent]:
     normalized_history = normalize_history(request.history)
-    matches_by_doc_id: dict[str, SourceMatch] = {}
+    retrieval_evidence: dict[str, SourceMatch] = {}
+    shell_evidence: dict[str, SourceMatch] = {}
+    web_evidence: dict[str, SourceMatch] = {}
     tool_calls_by_id: dict[str, dict[str, Any]] = {}
     answer_chunks: list[str] = []
     completed_answer = ""
@@ -717,260 +933,234 @@ async def stream_chat(request: ChatRequest) -> AsyncIterator[ChatEvent]:
         normalized_history,
     )
 
-    yield ChatEvent(
-        "thread_started",
-        {"thread_id": active_thread_id},
-    )
-    yield ChatEvent(
-        "message_started",
-        {"message_id": message_id},
-    )
+    yield ChatEvent("thread_started", {"thread_id": active_thread_id})
+    yield ChatEvent("message_started", {"message_id": message_id})
 
-    async for chunk in agent.astream(
-        {"messages": [{"role": "user", "content": request.query}]},
-        agent_run_config(request, active_thread_id, message_id),
-        context=AppContext(allowed_sources=request.source_keys),
-        stream_mode=["messages", "updates"],
-        version="v2",
-    ):
-        if chunk["type"] == "messages":
-            token, metadata = chunk["data"]
-            if not isinstance(token, AIMessageChunk):
-                continue
+    try:
+        async for chunk in agent.astream(
+            {"messages": [{"role": "user", "content": request.query}]},
+            agent_run_config(request, active_thread_id, message_id),
+            context=AppContext(
+                allowed_sources=request.source_keys,
+                kb_session_id=message_id,
+                kb_command_limit=DEFAULT_KB_COMMAND_LIMIT,
+            ),
+            stream_mode=["messages", "updates"],
+            version="v2",
+        ):
+            if chunk["type"] == "messages":
+                token, metadata = chunk["data"]
+                if not isinstance(token, AIMessageChunk):
+                    continue
 
-            step = str(metadata.get("langgraph_step", ""))
-            if include_reasoning:
-                thought_text = "\n\n".join(extract_thought_summaries(token.content))
-                if thought_text:
+                step = str(metadata.get("langgraph_step", ""))
+                if include_reasoning:
+                    thought_text = "\n\n".join(
+                        extract_thought_summaries(token.content)
+                    )
+                    if thought_text:
+                        yield ChatEvent(
+                            "reasoning_delta",
+                            {
+                                "message_id": message_id,
+                                "step": step,
+                                "text": thought_text,
+                            },
+                        )
+
+                for tool_call in getattr(token, "tool_calls", []) or []:
+                    tool_call_id = str(tool_call.get("id") or uuid4().hex)
+                    tool_calls_by_id[tool_call_id] = tool_call
                     yield ChatEvent(
-                        "reasoning_delta",
+                        "tool_call_started",
                         {
                             "message_id": message_id,
-                            "step": step,
-                            "text": thought_text,
+                            "call_id": tool_call_id,
+                            "tool_name": str(tool_call.get("name", "tool")),
+                            "args": tool_call.get("args"),
+                            "args_text": format_tool_args(tool_call.get("args")),
                         },
                     )
 
-            for tool_call in getattr(token, "tool_calls", []) or []:
-                tool_call_id = str(tool_call.get("id") or uuid4().hex)
-                tool_calls_by_id[tool_call_id] = tool_call
-                yield ChatEvent(
-                    "tool_call_started",
-                    {
-                        "message_id": message_id,
-                        "call_id": tool_call_id,
-                        "tool_name": str(tool_call.get("name", "tool")),
-                        "args": tool_call.get("args"),
-                        "args_text": format_tool_args(tool_call.get("args")),
-                    },
-                )
+                text_delta = token.text or ""
+                if not text_delta and token.content:
+                    text_delta = message_content_to_text(token.content)
+                if text_delta:
+                    answer_chunks.append(text_delta)
+                    yield ChatEvent(
+                        "text_delta",
+                        {
+                            "message_id": message_id,
+                            "text": text_delta,
+                        },
+                    )
 
-            text_delta = token.text or ""
-            if not text_delta and token.content:
-                text_delta = message_content_to_text(token.content)
-            if text_delta:
-                answer_chunks.append(text_delta)
-                yield ChatEvent(
-                    "text_delta",
-                    {
-                        "message_id": message_id,
-                        "text": text_delta,
-                    },
+                token_metadata = getattr(token, "response_metadata", None)
+                new_queries = [
+                    q
+                    for q in extract_web_search_queries(token_metadata)
+                    if q not in google_search_queries
+                ]
+                new_grounding = extract_grounding_source_matches(
+                    token_metadata,
+                    web_evidence,
                 )
+                if (new_queries or new_grounding) and not google_search_call_id:
+                    google_search_call_id = uuid4().hex
+                    yield ChatEvent(
+                        "tool_call_started",
+                        {
+                            "message_id": message_id,
+                            "call_id": google_search_call_id,
+                            "tool_name": GOOGLE_SEARCH_TOOL_NAME,
+                            "args": {
+                                "query": "; ".join(new_queries)
+                                if new_queries
+                                else ""
+                            },
+                            "args_text": "; ".join(new_queries),
+                        },
+                    )
+                if new_queries:
+                    google_search_queries.extend(new_queries)
+                google_search_match_count += len(new_grounding)
+                continue
 
-            token_metadata = getattr(token, "response_metadata", None)
-            new_queries = [
-                q
-                for q in extract_web_search_queries(token_metadata)
-                if q not in google_search_queries
-            ]
-            new_grounding = extract_grounding_source_matches(
-                token_metadata,
-                matches_by_doc_id,
-            )
-            if (new_queries or new_grounding) and not google_search_call_id:
-                google_search_call_id = uuid4().hex
-                yield ChatEvent(
-                    "tool_call_started",
-                    {
-                        "message_id": message_id,
-                        "call_id": google_search_call_id,
-                        "tool_name": GOOGLE_SEARCH_TOOL_NAME,
-                        "args": {"query": "; ".join(new_queries) if new_queries else ""},
-                        "args_text": "; ".join(new_queries),
-                    },
-                )
-            if new_queries:
-                google_search_queries.extend(new_queries)
-            for source_match in new_grounding:
-                google_search_match_count += 1
-                yield ChatEvent(
-                    "source_match",
-                    {
-                        "message_id": message_id,
-                        "doc_id": source_match.doc_id,
-                        "title": source_match.title,
-                        "url": source_match.url,
-                        "source_key": source_match.source_key,
-                        "source_label": source_match.source_label,
-                        "score": source_match.score,
-                        "group": source_match.group,
-                        "call_id": google_search_call_id,
-                    },
-                )
-            continue
+            if chunk["type"] != "updates":
+                continue
 
-        if chunk["type"] != "updates":
-            continue
+            for step, update in chunk["data"].items():
+                if not isinstance(update, dict):
+                    continue
+                messages = update.get("messages")
+                if not messages:
+                    continue
+                message = messages[-1]
+                if getattr(message, "type", None) == "tool":
+                    payload = message_content_to_text(message.content)
+                    tool_call_id = str(
+                        getattr(message, "tool_call_id", "") or uuid4().hex
+                    )
+                    tool_name = str(getattr(message, "name", "tool"))
+                    if tool_name == "retrieve_tutor_context":
+                        _record_evidence(
+                            retrieval_evidence,
+                            collect_retrieval_source_matches(payload),
+                        )
 
-        for step, update in chunk["data"].items():
-            message = update["messages"][-1]
-            if (
-                getattr(message, "type", None) == "tool"
-                and getattr(message, "name", "") == "retrieve_tutor_context"
-            ):
-                payload = message_content_to_text(message.content)
-                tool_call_id = str(getattr(message, "tool_call_id", "") or uuid4().hex)
-                for source_match in collect_updated_source_matches(
-                    matches_by_doc_id,
-                    payload,
-                ):
+                    tool_call = tool_calls_by_id.get(
+                        tool_call_id,
+                        {"name": getattr(message, "name", "tool"), "args": None},
+                    )
+                    if tool_name == "run_kb_command":
+                        args = tool_call.get("args")
+                        command = str(
+                            args.get("command") if isinstance(args, dict) else ""
+                        )
+                        _record_evidence(
+                            shell_evidence,
+                            extract_shell_source_matches(command, payload),
+                        )
                     yield ChatEvent(
-                        "source_match",
+                        "tool_call_completed",
                         {
                             "message_id": message_id,
-                            "doc_id": source_match.doc_id,
-                            "title": source_match.title,
-                            "url": source_match.url,
-                            "source_key": source_match.source_key,
-                            "source_label": source_match.source_label,
-                            "score": source_match.score,
-                            "group": source_match.group,
+                            "step": step,
                             "call_id": tool_call_id,
+                            "tool_name": str(
+                                tool_call.get(
+                                    "name",
+                                    getattr(message, "name", "tool"),
+                                )
+                            ),
+                            "args": tool_call.get("args"),
+                            "args_text": format_tool_args(tool_call.get("args")),
+                            "output_text": payload,
                         },
                     )
+                    continue
 
-                tool_call = tool_calls_by_id.get(
-                    tool_call_id,
-                    {"name": getattr(message, "name", "tool"), "args": None},
-                )
-                yield ChatEvent(
-                    "tool_call_completed",
-                    {
-                        "message_id": message_id,
-                        "step": step,
-                        "call_id": tool_call_id,
-                        "tool_name": str(tool_call.get("name", getattr(message, "name", "tool"))),
-                        "args": tool_call.get("args"),
-                        "args_text": format_tool_args(tool_call.get("args")),
-                        "output_text": payload,
-                    },
-                )
-                continue
-
-            if step != "model" or getattr(message, "type", None) != "ai":
-                continue
-
-            message_metadata = getattr(message, "response_metadata", None)
-            new_queries = [
-                q
-                for q in extract_web_search_queries(message_metadata)
-                if q not in google_search_queries
-            ]
-            new_grounding = extract_grounding_source_matches(
-                message_metadata,
-                matches_by_doc_id,
-            )
-            if (new_queries or new_grounding) and not google_search_call_id:
-                google_search_call_id = uuid4().hex
-                yield ChatEvent(
-                    "tool_call_started",
-                    {
-                        "message_id": message_id,
-                        "call_id": google_search_call_id,
-                        "tool_name": GOOGLE_SEARCH_TOOL_NAME,
-                        "args": {"query": "; ".join(new_queries) if new_queries else ""},
-                        "args_text": "; ".join(new_queries),
-                    },
-                )
-            if new_queries:
-                google_search_queries.extend(new_queries)
-            for source_match in new_grounding:
-                google_search_match_count += 1
-                yield ChatEvent(
-                    "source_match",
-                    {
-                        "message_id": message_id,
-                        "doc_id": source_match.doc_id,
-                        "title": source_match.title,
-                        "url": source_match.url,
-                        "source_key": source_match.source_key,
-                        "source_label": source_match.source_label,
-                        "score": source_match.score,
-                        "group": source_match.group,
-                        "call_id": google_search_call_id,
-                    },
-                )
+                if step != "model" or getattr(message, "type", None) != "ai":
+                    continue
 
-            if is_anthropic_model(request.model_name):
-                anthropic_updates, anthropic_tool_uses = extract_anthropic_source_matches(
-                    message.content,
-                    matches_by_doc_id,
+                message_metadata = getattr(message, "response_metadata", None)
+                new_queries = [
+                    q
+                    for q in extract_web_search_queries(message_metadata)
+                    if q not in google_search_queries
+                ]
+                new_grounding = extract_grounding_source_matches(
+                    message_metadata,
+                    web_evidence,
                 )
-                for tool_use_id, tool_use in anthropic_tool_uses.items():
-                    if tool_use_id in tool_calls_by_id:
-                        continue
-                    tool_calls_by_id[tool_use_id] = tool_use
+                if (new_queries or new_grounding) and not google_search_call_id:
+                    google_search_call_id = uuid4().hex
                     yield ChatEvent(
                         "tool_call_started",
                         {
                             "message_id": message_id,
-                            "call_id": tool_use_id,
-                            "tool_name": tool_use["name"],
-                            "args": tool_use.get("args"),
-                            "args_text": format_tool_args(tool_use.get("args")),
+                            "call_id": google_search_call_id,
+                            "tool_name": GOOGLE_SEARCH_TOOL_NAME,
+                            "args": {
+                                "query": "; ".join(new_queries)
+                                if new_queries
+                                else ""
+                            },
+                            "args_text": "; ".join(new_queries),
                         },
                     )
-                for tool_use_id, new_matches in anthropic_updates.items():
-                    for source_match in new_matches:
+                if new_queries:
+                    google_search_queries.extend(new_queries)
+                google_search_match_count += len(new_grounding)
+
+                if is_anthropic_model(request.model_name):
+                    anthropic_updates, anthropic_tool_uses = (
+                        extract_anthropic_source_matches(
+                            message.content,
+                            web_evidence,
+                        )
+                    )
+                    for tool_use_id, tool_use in anthropic_tool_uses.items():
+                        if tool_use_id in tool_calls_by_id:
+                            continue
+                        tool_calls_by_id[tool_use_id] = tool_use
                         yield ChatEvent(
-                            "source_match",
+                            "tool_call_started",
                             {
                                 "message_id": message_id,
-                                "doc_id": source_match.doc_id,
-                                "title": source_match.title,
-                                "url": source_match.url,
-                                "source_key": source_match.source_key,
-                                "source_label": source_match.source_label,
-                                "score": source_match.score,
-                                "group": source_match.group,
                                 "call_id": tool_use_id,
+                                "tool_name": tool_use["name"],
+                                "args": tool_use.get("args"),
+                                "args_text": format_tool_args(tool_use.get("args")),
+                            },
+                        )
+                    for tool_use_id, new_matches in anthropic_updates.items():
+                        if not tool_use_id:
+                            continue
+                        tool_call = tool_calls_by_id.get(tool_use_id, {})
+                        tool_name = str(tool_call.get("name") or "web_search")
+                        plural = "" if len(new_matches) == 1 else "s"
+                        output_text = (
+                            f"{tool_name} returned {len(new_matches)} result{plural}."
+                        )
+                        yield ChatEvent(
+                            "tool_call_completed",
+                            {
+                                "message_id": message_id,
+                                "step": step,
+                                "call_id": tool_use_id,
+                                "tool_name": tool_name,
+                                "args": tool_call.get("args"),
+                                "args_text": format_tool_args(tool_call.get("args")),
+                                "output_text": output_text,
                             },
                         )
-                    if not tool_use_id:
-                        continue
-                    tool_call = tool_calls_by_id.get(tool_use_id, {})
-                    tool_name = str(tool_call.get("name") or "web_search")
-                    plural = "" if len(new_matches) == 1 else "s"
-                    output_text = (
-                        f"{tool_name} returned {len(new_matches)} result{plural}."
-                    )
-                    yield ChatEvent(
-                        "tool_call_completed",
-                        {
-                            "message_id": message_id,
-                            "step": step,
-                            "call_id": tool_use_id,
-                            "tool_name": tool_name,
-                            "args": tool_call.get("args"),
-                            "args_text": format_tool_args(tool_call.get("args")),
-                            "output_text": output_text,
-                        },
-                    )
 
-            if getattr(message, "tool_calls", None):
-                continue
-            completed_answer = message_content_to_text(message.content)
+                if getattr(message, "tool_calls", None):
+                    continue
+                completed_answer = message_content_to_text(message.content)
+    finally:
+        _clear_kb_command_budget(message_id)
 
     if google_search_call_id:
         joined_queries = "; ".join(google_search_queries)
@@ -994,6 +1184,16 @@ async def stream_chat(request: ChatRequest) -> AsyncIterator[ChatEvent]:
         )
 
     answer = "".join(answer_chunks).strip() or completed_answer.strip()
+    for source_match in resolve_answer_citations(
+        answer,
+        retrieval_evidence=retrieval_evidence,
+        shell_evidence=shell_evidence,
+        web_evidence=web_evidence,
+    ):
+        yield ChatEvent(
+            "source_match",
+            source_match_payload(source_match, message_id=message_id),
+        )
     yield ChatEvent(
         "message_completed",
         {

scripts/gradio_presenter.py

@@ -71,9 +71,15 @@ def upsert_activity_event(
 def format_tool_args(args: Any, args_text: str = "") -> str:
     query = ""
     url = ""
+    command = ""
     if isinstance(args, dict):
         query = str(args.get("query", "")).strip()
         url = str(args.get("url", "")).strip()
+        command = str(args.get("command", "")).strip()
+
+    if command:
+        trimmed = command[:157] + "..." if len(command) > 160 else command
+        return f"Command: `{trimmed}`"
 
     if not query and args_text.strip():
         query = args_text.strip()
@@ -115,12 +121,25 @@ def summarize_tool_result(event: ChatEvent) -> str:
     match_count = len(matches)
 
     if tool_name == "retrieve_tutor_context":
+        if not matches:
+            try:
+                payload = json.loads(str(event.data.get("output_text", "")))
+                matches = payload.get("matches", []) if isinstance(payload, dict) else []
+                match_count = len(matches)
+            except json.JSONDecodeError:
+                matches = []
+                match_count = 0
         if not matches:
             return "_No matching sources found in the selected sources._"
         ordered_sources: list[str] = []
         seen_sources: set[str] = set()
         for match in matches:
-            source_label = str(match.get("source_label", match.get("source_key", "unknown")))
+            source_label = str(
+                match.get("source_label")
+                or match.get("source_key")
+                or match.get("source")
+                or "unknown"
+            )
             if source_label in seen_sources:
                 continue
             seen_sources.add(source_label)
@@ -129,6 +148,23 @@ def summarize_tool_result(event: ChatEvent) -> str:
         match_label = "match" if match_count == 1 else "matches"
         return f"_Found {match_count} {match_label} from {source_summary}._"
 
+    if tool_name == "run_kb_command":
+        output_text = str(event.data.get("output_text", "")).strip()
+        command = ""
+        exit_code = ""
+        for line in output_text.splitlines():
+            if line.startswith("$ "):
+                command = line[2:].strip()
+            elif line.startswith("exit_code:"):
+                exit_code = line.partition(":")[2].strip()
+        if "error:" in output_text:
+            return f"_KB command failed: {output_text.split('error:', 1)[1].strip()[:160]}_"
+        if command and exit_code:
+            return f"_Ran `{command}` with exit code {exit_code}._"
+        if command:
+            return f"_Ran `{command}`._"
+        return "_Ran KB command._"
+
     output_text = str(event.data.get("output_text", "")).strip()
     if output_text:
         return f"_{output_text}_"

scripts/kb_manifest.py

@@ -0,0 +1,172 @@
+from __future__ import annotations
+
+import json
+import re
+from dataclasses import dataclass
+from functools import lru_cache
+from pathlib import Path
+from typing import Any
+
+from .chat_types import SourceMatch
+from .setup import COURSE_SOURCE_KEYS, KB_DIR, SOURCE_KEY_TO_LABEL
+
+KB_DOC_SCHEME_RE = re.compile(r"^kb://doc/(?P<doc_id>[^)\]\s]+)$")
+RAW_PATH_RE = re.compile(r"(?:data/kb/)?raw/[^\s)\]>,:]+?\.(?:mdx|md)")
+
+
+@dataclass(frozen=True, slots=True)
+class KbManifestEntry:
+    doc_id: str
+    title: str
+    url: str
+    source: str
+    source_group: str
+    path: str
+
+
+def _normalize_path(value: str) -> str:
+    value = value.strip().strip("<>`'\"")
+    if value.startswith("./"):
+        value = value[2:]
+    if value.startswith(f"{KB_DIR}/"):
+        return value
+    if value.startswith("raw/"):
+        return f"{KB_DIR}/{value}"
+    return value
+
+
+@lru_cache(maxsize=4)
+def load_manifest_entries(kb_dir: str = KB_DIR) -> tuple[KbManifestEntry, ...]:
+    manifest_path = Path(kb_dir) / "generated" / "corpus_manifest.jsonl"
+    if not manifest_path.exists():
+        return ()
+
+    entries: list[KbManifestEntry] = []
+    with manifest_path.open("r", encoding="utf-8") as handle:
+        for line in handle:
+            if not line.strip():
+                continue
+            row = json.loads(line)
+            entries.append(
+                KbManifestEntry(
+                    doc_id=str(row.get("doc_id") or ""),
+                    title=str(row.get("title") or ""),
+                    url=str(row.get("url") or ""),
+                    source=str(row.get("source") or ""),
+                    source_group=str(row.get("source_group") or ""),
+                    path=str(row.get("path") or ""),
+                )
+            )
+    return tuple(entries)
+
+
+def manifest_indexes(
+    kb_dir: str = KB_DIR,
+) -> tuple[dict[str, KbManifestEntry], dict[str, KbManifestEntry], dict[str, KbManifestEntry], dict[str, KbManifestEntry]]:
+    by_doc_id: dict[str, KbManifestEntry] = {}
+    by_url: dict[str, KbManifestEntry] = {}
+    by_path: dict[str, KbManifestEntry] = {}
+    by_title: dict[str, KbManifestEntry] = {}
+    for entry in load_manifest_entries(kb_dir):
+        if entry.doc_id:
+            by_doc_id[entry.doc_id] = entry
+        if entry.url:
+            by_url[entry.url] = entry
+        if entry.path:
+            by_path[_normalize_path(entry.path)] = entry
+            if entry.path.startswith(f"{KB_DIR}/"):
+                by_path[entry.path[len(KB_DIR) + 1 :]] = entry
+        if entry.title:
+            by_title[entry.title.strip().lower()] = entry
+    return by_doc_id, by_url, by_path, by_title
+
+
+def source_match_from_manifest(entry: KbManifestEntry, *, score: float = 1.0) -> SourceMatch:
+    return SourceMatch(
+        doc_id=entry.doc_id,
+        title=entry.title,
+        url=entry.url or entry.path,
+        source_key=entry.source,
+        source_label=SOURCE_KEY_TO_LABEL.get(entry.source, entry.source),
+        score=score,
+        group="courses" if entry.source in COURSE_SOURCE_KEYS else entry.source_group or "docs",
+    )
+
+
+def resolve_manifest_reference(
+    reference: str,
+    *,
+    label: str = "",
+    kb_dir: str = KB_DIR,
+) -> SourceMatch | None:
+    value = reference.strip()
+    by_doc_id, by_url, by_path, by_title = manifest_indexes(kb_dir)
+
+    if match := KB_DOC_SCHEME_RE.match(value):
+        entry = by_doc_id.get(match.group("doc_id"))
+        return source_match_from_manifest(entry) if entry else None
+
+    entry = by_url.get(value)
+    if entry:
+        return source_match_from_manifest(entry)
+
+    entry = by_path.get(_normalize_path(value))
+    if entry:
+        return source_match_from_manifest(entry)
+
+    if label:
+        entry = by_title.get(label.strip().lower())
+        if entry:
+            return source_match_from_manifest(entry)
+    return None
+
+
+def extract_raw_paths(text: str) -> list[str]:
+    paths: list[str] = []
+    seen: set[str] = set()
+    for match in RAW_PATH_RE.finditer(text):
+        value = match.group(0).rstrip(":.,;")
+        key = _normalize_path(value)
+        if key in seen:
+            continue
+        seen.add(key)
+        paths.append(value)
+    return paths
+
+
+def parse_markdown_citations(text: str) -> list[tuple[str, str]]:
+    citations: list[tuple[str, str]] = []
+    link_re = re.compile(r"\[([^\]]+)\]\(([^)]+)\)")
+    for match in link_re.finditer(text):
+        citations.append((match.group(1).strip(), match.group(2).strip()))
+
+    linked_refs = {ref for _label, ref in citations}
+    bare_re = re.compile(r"(?<!\()(?P<ref>https?://[^\s<>()]+|kb://doc/[^\s<>()]+)")
+    for match in bare_re.finditer(text):
+        ref = match.group("ref").rstrip(".,;")
+        if ref not in linked_refs:
+            citations.append(("", ref))
+    for path in extract_raw_paths(text):
+        if path not in linked_refs:
+            citations.append(("", path))
+    return citations
+
+
+def source_match_key(match: SourceMatch) -> str:
+    return match.doc_id or match.url or match.title
+
+
+def source_match_payload(match: SourceMatch, *, message_id: str, call_id: str = "") -> dict[str, Any]:
+    payload: dict[str, Any] = {
+        "message_id": message_id,
+        "doc_id": match.doc_id,
+        "title": match.title,
+        "url": match.url,
+        "source_key": match.source_key,
+        "source_label": match.source_label,
+        "score": match.score,
+        "group": match.group,
+    }
+    if call_id:
+        payload["call_id"] = call_id
+    return payload

scripts/kb_shell.py

@@ -0,0 +1,394 @@
+from __future__ import annotations
+
+import os
+import re
+import shlex
+import shutil
+import subprocess
+from dataclasses import dataclass
+from pathlib import Path
+
+DEFAULT_KB_DIR = Path(os.getenv("AI_TUTOR_KB_DIR", "data/kb"))
+DEFAULT_TIMEOUT_SECONDS = 8
+DEFAULT_MAX_OUTPUT_CHARS = 40_000
+SUPPORTED_COMMANDS = frozenset({"rg", "grep", "find", "ls", "sed", "head", "cat", "wc"})
+SHELL_TOKENS = frozenset({"|", "||", "&", "&&", ";", ">", ">>", "<", "<<", "`"})
+RG_VALUE_OPTIONS = frozenset(
+    {
+        "-g",
+        "--glob",
+        "-t",
+        "--type",
+        "-T",
+        "--type-not",
+        "-m",
+        "--max-count",
+    }
+)
+RG_FLAG_OPTIONS = frozenset(
+    {
+        "-n",
+        "--line-number",
+        "-i",
+        "--ignore-case",
+        "-S",
+        "--smart-case",
+        "-F",
+        "--fixed-strings",
+        "-w",
+        "--word-regexp",
+        "--hidden",
+        "--no-ignore",
+    }
+)
+GREP_FLAG_OPTIONS = frozenset({"-i", "--ignore-case", "-w", "--word-regexp", "-F", "-E"})
+LS_FLAG_OPTIONS = frozenset({"-1", "-a", "-l", "-la", "-al"})
+WC_FLAG_OPTIONS = frozenset({"-l", "-w", "-c", "-m"})
+
+
+@dataclass(frozen=True, slots=True)
+class KbCommandResult:
+    command: str
+    argv: list[str]
+    cwd: str
+    exit_code: int
+    stdout: str
+    stderr: str
+    timed_out: bool
+    truncated: bool
+
+
+class KbCommandError(ValueError):
+    """Raised when a KB command violates the read-only command policy."""
+
+
+def _resolve_root(root: Path | None) -> Path:
+    resolved = (root or DEFAULT_KB_DIR).resolve()
+    if not resolved.exists():
+        raise KbCommandError(f"KB root does not exist: {resolved}")
+    if not resolved.is_dir():
+        raise KbCommandError(f"KB root is not a directory: {resolved}")
+    return resolved
+
+
+def _resolve_inside_root(value: str, root: Path, *, must_exist: bool = True) -> Path:
+    path = Path(value)
+    if path.is_absolute():
+        resolved = path.resolve()
+    else:
+        resolved = (root / value).resolve()
+    if resolved != root and root not in resolved.parents:
+        raise KbCommandError("KB command paths must stay inside data/kb")
+    if must_exist and not resolved.exists():
+        raise KbCommandError(f"KB command path does not exist: {value}")
+    return resolved
+
+
+def _relative_path_arg(value: str, root: Path) -> str:
+    resolved = _resolve_inside_root(value, root)
+    return "." if resolved == root else resolved.relative_to(root).as_posix()
+
+
+def _reject_shell_syntax(command: str, tokens: list[str]) -> None:
+    if "\n" in command or "\r" in command:
+        raise KbCommandError("KB command must be a single command, not a script")
+    if "$(" in command or "${" in command:
+        raise KbCommandError("Shell expansion is not allowed in KB commands")
+    if any(token in SHELL_TOKENS for token in tokens):
+        raise KbCommandError("Pipes, redirects, and command chaining are not allowed")
+
+
+def _numeric_value(token: str, option: str) -> str:
+    if not token.isdigit():
+        raise KbCommandError(f"{option} expects a positive integer")
+    return token
+
+
+def _safe_pattern_value(token: str, option: str) -> str:
+    if "\x00" in token:
+        raise KbCommandError(f"{option} contains an invalid value")
+    if Path(token).is_absolute() or ".." in Path(token).parts:
+        raise KbCommandError(f"{option} values must stay inside data/kb")
+    return token
+
+
+def _is_broad_raw_path(path: str) -> bool:
+    normalized = path.rstrip("/")
+    return normalized in {".", "raw", "raw/courses", "raw/docs"}
+
+
+def _reject_unbounded_raw_search(command: str, paths: list[str], has_max_count: bool) -> None:
+    if has_max_count:
+        return
+    if any(_is_broad_raw_path(path) for path in paths):
+        raise KbCommandError(
+            f"`{command}` over broad raw paths requires -m/--max-count "
+            "or a narrower source/file path"
+        )
+
+
+def _build_rg(tokens: list[str], root: Path) -> list[str]:
+    if shutil.which("rg") is None:
+        raise KbCommandError("`rg` is not installed in this runtime")
+    options: list[str] = []
+    positionals: list[str] = []
+    has_max_count = False
+    idx = 1
+    parsing_options = True
+    while idx < len(tokens):
+        token = tokens[idx]
+        if parsing_options and token == "--":
+            parsing_options = False
+            idx += 1
+            continue
+        if parsing_options and token in RG_FLAG_OPTIONS:
+            options.append(token)
+            idx += 1
+            continue
+        if parsing_options and token in RG_VALUE_OPTIONS:
+            if idx + 1 >= len(tokens):
+                raise KbCommandError(f"{token} requires a value")
+            value = tokens[idx + 1]
+            if token in {"-m", "--max-count"}:
+                value = _numeric_value(value, token)
+                has_max_count = True
+            else:
+                value = _safe_pattern_value(value, token)
+            options.extend([token, value])
+            idx += 2
+            continue
+        positionals.append(token)
+        parsing_options = False
+        idx += 1
+
+    if not positionals:
+        raise KbCommandError("`rg` requires a search pattern")
+    pattern = positionals[0]
+    paths = [_relative_path_arg(value, root) for value in positionals[1:]] or ["."]
+    _reject_unbounded_raw_search("rg", paths, has_max_count)
+    return [
+        "rg",
+        "--color=never",
+        "--line-number",
+        "--no-heading",
+        *options,
+        pattern,
+        *paths,
+    ]
+
+
+def _build_grep(tokens: list[str], root: Path) -> list[str]:
+    options: list[str] = []
+    positionals: list[str] = []
+    idx = 1
+    parsing_options = True
+    while idx < len(tokens):
+        token = tokens[idx]
+        if parsing_options and token == "--":
+            parsing_options = False
+            idx += 1
+            continue
+        if parsing_options and token in GREP_FLAG_OPTIONS:
+            options.append(token)
+            idx += 1
+            continue
+        positionals.append(token)
+        parsing_options = False
+        idx += 1
+    if not positionals:
+        raise KbCommandError("`grep` requires a search pattern")
+    pattern = positionals[0]
+    paths = [_relative_path_arg(value, root) for value in positionals[1:]] or ["."]
+    return ["grep", "-R", "-n", "--color=never", *options, pattern, *paths]
+
+
+def _build_find(tokens: list[str], root: Path) -> list[str]:
+    path = "."
+    idx = 1
+    if idx < len(tokens) and not tokens[idx].startswith("-"):
+        path = _relative_path_arg(tokens[idx], root)
+        idx += 1
+
+    argv = ["find", path]
+    while idx < len(tokens):
+        token = tokens[idx]
+        if token in {"-maxdepth", "-mindepth"}:
+            if idx + 1 >= len(tokens):
+                raise KbCommandError(f"{token} requires a value")
+            argv.extend([token, _numeric_value(tokens[idx + 1], token)])
+            idx += 2
+            continue
+        if token == "-type":
+            if idx + 1 >= len(tokens) or tokens[idx + 1] not in {"f", "d"}:
+                raise KbCommandError("-type supports only `f` or `d`")
+            argv.extend([token, tokens[idx + 1]])
+            idx += 2
+            continue
+        if token in {"-name", "-iname"}:
+            if idx + 1 >= len(tokens):
+                raise KbCommandError(f"{token} requires a value")
+            argv.extend([token, _safe_pattern_value(tokens[idx + 1], token)])
+            idx += 2
+            continue
+        raise KbCommandError("`find` supports only path, -maxdepth, -mindepth, -type, -name, and -iname")
+    return argv
+
+
+def _build_ls(tokens: list[str], root: Path) -> list[str]:
+    argv = ["ls"]
+    paths: list[str] = []
+    for token in tokens[1:]:
+        if token in LS_FLAG_OPTIONS:
+            argv.append(token)
+            continue
+        paths.append(_relative_path_arg(token, root))
+    return [*argv, *(paths or ["."])]
+
+
+def _build_sed(tokens: list[str], root: Path) -> list[str]:
+    if len(tokens) != 4 or tokens[1] != "-n":
+        raise KbCommandError("`sed` supports only: sed -n START,ENDp FILE")
+    expression = tokens[2]
+    if not re.fullmatch(r"\d+,\d+p", expression):
+        raise KbCommandError("`sed` supports only START,ENDp read expressions")
+    path = _relative_path_arg(tokens[3], root)
+    return ["sed", "-n", expression, path]
+
+
+def _build_head(tokens: list[str], root: Path) -> list[str]:
+    argv = ["head"]
+    paths: list[str] = []
+    idx = 1
+    if idx < len(tokens) and tokens[idx] == "-n":
+        if idx + 1 >= len(tokens):
+            raise KbCommandError("-n requires a value")
+        argv.extend(["-n", _numeric_value(tokens[idx + 1], "-n")])
+        idx += 2
+    while idx < len(tokens):
+        paths.append(_relative_path_arg(tokens[idx], root))
+        idx += 1
+    if not paths:
+        raise KbCommandError("`head` requires at least one file path")
+    return [*argv, *paths]
+
+
+def _build_cat(tokens: list[str], root: Path) -> list[str]:
+    if len(tokens) < 2:
+        raise KbCommandError("`cat` requires at least one file path")
+    return ["cat", *[_relative_path_arg(token, root) for token in tokens[1:]]]
+
+
+def _build_wc(tokens: list[str], root: Path) -> list[str]:
+    argv = ["wc"]
+    paths: list[str] = []
+    for token in tokens[1:]:
+        if token in WC_FLAG_OPTIONS:
+            argv.append(token)
+            continue
+        paths.append(_relative_path_arg(token, root))
+    if not paths:
+        raise KbCommandError("`wc` requires at least one file path")
+    return [*argv, *paths]
+
+
+def build_kb_command_argv(command: str, *, root: Path | None = None) -> tuple[list[str], Path]:
+    resolved_root = _resolve_root(root)
+    try:
+        tokens = shlex.split(command)
+    except ValueError as exc:
+        raise KbCommandError(f"Invalid KB command syntax: {exc}") from exc
+    if not tokens:
+        raise KbCommandError("KB command cannot be empty")
+    _reject_shell_syntax(command, tokens)
+
+    executable = tokens[0]
+    if executable not in SUPPORTED_COMMANDS:
+        supported = ", ".join(sorted(SUPPORTED_COMMANDS))
+        raise KbCommandError(f"Unsupported KB command `{executable}`. Supported: {supported}")
+
+    builders = {
+        "rg": _build_rg,
+        "grep": _build_grep,
+        "find": _build_find,
+        "ls": _build_ls,
+        "sed": _build_sed,
+        "head": _build_head,
+        "cat": _build_cat,
+        "wc": _build_wc,
+    }
+    return builders[executable](tokens, resolved_root), resolved_root
+
+
+def _truncate(text: str, max_chars: int) -> tuple[str, bool]:
+    if len(text) <= max_chars:
+        return text, False
+    marker = f"\n... output truncated to {max_chars} characters ..."
+    return text[: max(0, max_chars - len(marker))] + marker, True
+
+
+def run_kb_command(
+    command: str,
+    *,
+    root: Path | None = None,
+    timeout_seconds: int = DEFAULT_TIMEOUT_SECONDS,
+    max_output_chars: int = DEFAULT_MAX_OUTPUT_CHARS,
+) -> KbCommandResult:
+    argv, resolved_root = build_kb_command_argv(command, root=root)
+    timeout = max(1, min(int(timeout_seconds), 30))
+    output_limit = max(1_000, min(int(max_output_chars), 80_000))
+    env = {
+        "HOME": os.getenv("HOME", ""),
+        "LANG": os.getenv("LANG", "C.UTF-8"),
+        "LC_ALL": os.getenv("LC_ALL", "C.UTF-8"),
+        "PATH": os.getenv("PATH", ""),
+    }
+    timed_out = False
+    try:
+        completed = subprocess.run(
+            argv,
+            cwd=resolved_root,
+            capture_output=True,
+            text=True,
+            timeout=timeout,
+            env=env,
+            check=False,
+        )
+        exit_code = int(completed.returncode)
+        stdout = completed.stdout or ""
+        stderr = completed.stderr or ""
+    except subprocess.TimeoutExpired as exc:
+        timed_out = True
+        exit_code = 124
+        stdout = exc.stdout if isinstance(exc.stdout, str) else ""
+        stderr = exc.stderr if isinstance(exc.stderr, str) else ""
+        stderr = (stderr + "\n" if stderr else "") + f"Command timed out after {timeout}s."
+
+    stdout, stdout_truncated = _truncate(stdout, output_limit)
+    stderr, stderr_truncated = _truncate(stderr, min(output_limit, 8_000))
+    return KbCommandResult(
+        command=command,
+        argv=argv,
+        cwd=resolved_root.as_posix(),
+        exit_code=exit_code,
+        stdout=stdout,
+        stderr=stderr,
+        timed_out=timed_out,
+        truncated=stdout_truncated or stderr_truncated,
+    )
+
+
+def format_command_payload(result: KbCommandResult) -> str:
+    lines = [
+        f"$ {result.command}",
+        f"cwd: {result.cwd}",
+        f"exit_code: {result.exit_code}",
+    ]
+    if result.timed_out:
+        lines.append("timed_out: true")
+    if result.truncated:
+        lines.append("truncated: true")
+    if result.stdout:
+        lines.extend(["stdout:", result.stdout.rstrip()])
+    if result.stderr:
+        lines.extend(["stderr:", result.stderr.rstrip()])
+    return "\n".join(lines)

scripts/probe_gemini_cache.py

@@ -0,0 +1,82 @@
+"""Send the same prefix to Gemini twice and report cached_content_token_count.
+
+Run with:
+    uv run dotenv -f .env run -- python -m scripts.probe_gemini_cache
+"""
+
+from __future__ import annotations
+
+import os
+import time
+
+from google import genai
+from google.genai import types
+
+MODEL = "gemini-3.5-flash"
+
+# Build a stable prefix that comfortably exceeds the 1024-token Flash minimum
+# for implicit caching. Repeat a paragraph until we have ~3k tokens worth.
+SYSTEM = (
+    "You are an AI teacher for applied AI, LLM, RAG, and Python topics. "
+    "Your job is to answer student questions clearly and accurately. "
+    "Ground claims in the provided context and cite sources inline."
+)
+PARAGRAPH = (
+    "Retrieval-Augmented Generation (RAG) systems combine a retrieval step "
+    "over a corpus with a generation step using a language model. The "
+    "retrieval step selects the most relevant passages from a vector index, "
+    "and the generation step conditions on those passages to produce a "
+    "grounded answer. Common components include a chunker, an embedding "
+    "model, a vector store, a retriever, a reranker, and a generation model. "
+    "Evaluation typically considers faithfulness, answer relevance, and "
+    "context precision and recall.\n\n"
+)
+LONG_CONTEXT = PARAGRAPH * 12  # ~3-4k tokens
+
+
+def call(client: genai.Client, user_text: str) -> dict:
+    start = time.perf_counter()
+    response = client.models.generate_content(
+        model=MODEL,
+        config=types.GenerateContentConfig(system_instruction=SYSTEM),
+        contents=[
+            {
+                "role": "user",
+                "parts": [
+                    {"text": LONG_CONTEXT},
+                    {"text": user_text},
+                ],
+            }
+        ],
+    )
+    elapsed = time.perf_counter() - start
+    usage = response.usage_metadata
+    return {
+        "elapsed_s": round(elapsed, 2),
+        "prompt_tokens": getattr(usage, "prompt_token_count", None),
+        "cached_tokens": getattr(usage, "cached_content_token_count", None),
+        "output_tokens": getattr(usage, "candidates_token_count", None),
+        "thoughts_tokens": getattr(usage, "thoughts_token_count", None),
+        "text_preview": (response.text or "")[:120],
+    }
+
+
+def main() -> None:
+    api_key = os.environ.get("GEMINI_API_KEY") or os.environ.get("GOOGLE_API_KEY")
+    if not api_key:
+        raise SystemExit("Set GEMINI_API_KEY in .env")
+
+    client = genai.Client(api_key=api_key)
+
+    print(f"Model: {MODEL}")
+    print(f"Stable prefix (system + LONG_CONTEXT) repeats. Only the final user_text differs.")
+    print()
+
+    for i in range(3):
+        result = call(client, user_text=f"Question {i+1}: what is RAG in one sentence?")
+        print(f"Call {i+1}: {result}")
+        time.sleep(0.5)
+
+
+if __name__ == "__main__":
+    main()

scripts/prompts.py

@@ -1,15 +1,25 @@
 from __future__ import annotations
 
+import os
+from pathlib import Path
+
 
 BASE_PROMPT_HEADER = """You are an AI teacher for applied AI, LLM, RAG, and Python topics.
 
 Your job is to answer student questions clearly and accurately."""
 
+DEFAULT_KB_AGENTS_PATH = "data/kb/AGENTS.md"
+
 RETRIEVAL_TOOL_LINE = (
     "- `retrieve_tutor_context` — retrieval over the course and documentation\n"
     "  corpus. Use this for anything that depends on course content."
 )
 
+KB_TOOL_LINES = (
+    "- `run_kb_command` — run safe, read-only terminal-style KB inspection\n"
+    "  commands such as `rg`, `grep`, `find`, `ls`, `sed`, `head`, `cat`, and `wc`.",
+)
+
 WEB_TOOL_LINES = {
     "google-genai": {
         "web_search": (
@@ -74,6 +84,21 @@ DO NOT use retrieval for:
 - Questions about your own role or capabilities in this app (answer directly).
 - Questions you can answer fully from general knowledge with no corpus dependency."""
 
+KB_USAGE_SECTION = """## When to use the KB browsing tools
+
+USE KB browsing for:
+- Exact API names, class names, function names, config keys, or error strings.
+- Broad questions where a wiki page can identify the right source pages.
+- Comparisons, recipes, debugging, and verification-heavy answers.
+- Course questions where reading the generated lesson markdown is useful.
+- Targeted terminal-style inspection when `rg`, `grep`, `find`, or `sed`
+  can verify a specific string/path faster than top-k retrieval.
+
+`run_kb_command` is not a general shell: no pipes, redirects, command chaining,
+network commands, or writes. Follow the Local KB Instructions below for the
+current wiki structure, the first-command rule, and the efficient command
+patterns to use."""
+
 RETRIEVAL_CALL_STRATEGY = """## How to call the retrieval tool
 
 You have two strategies. Pick one per turn:
@@ -97,6 +122,26 @@ You have two strategies. Pick one per turn:
 ANSWERING_RULES = """## Answering rules
 
 - Ground factual claims about the corpus in the retrieved results.
+- When using KB browsing, ground factual claims in raw source pages inspected
+  with `run_kb_command`, not only in wiki navigation pages.
+- If the user explicitly restricts you to one available local knowledge tool,
+  obey that restriction. If the restriction prevents a complete answer, say so.
+- Cite retrieved context using the original source URL or title shown by
+  `retrieve_tutor_context`.
+- Cite files explored with `run_kb_command` using the raw file frontmatter URL,
+  `kb://doc/<doc_id>`, or a KB-root path like
+  `raw/docs/peft/developer_guides/lora.md`.
+- Use inline Markdown links as citations sentence-by-sentence: place a
+  `[short source title](url-or-kb-reference)` citation in the same sentence,
+  immediately after the claim it supports.
+- Do not use bare bracketed URLs or paths like `[https://...]` or
+  `[raw/docs/...]`; the bracket text should be a short source title.
+- Do not place citations inside code blocks. Cite the prose sentence that
+  introduces or explains the code instead.
+- Do not put all citations only in a final "Sources" section. A short sources
+  recap is okay only if the answer already has inline citations.
+- Do not cite wiki pages unless the wiki page itself is the source of the
+  claim. Prefer raw docs and course lessons.
 - If retrieval results are weak or missing, say the topic is not well
   covered by the current knowledge base rather than guessing.
 - Synthesize retrieved material into a clear teaching explanation. Do not
@@ -122,10 +167,24 @@ def _provider_key(model_name: str) -> str:
     return ""
 
 
+def kb_agents_path() -> Path:
+    return Path(os.getenv("AI_TUTOR_KB_AGENTS_PATH", DEFAULT_KB_AGENTS_PATH))
+
+
+def load_kb_agents_instructions(path: Path | None = None) -> str:
+    resolved = path or kb_agents_path()
+    try:
+        if not resolved.exists() or not resolved.is_file():
+            return ""
+        return resolved.read_text(encoding="utf-8").strip()
+    except OSError:
+        return ""
+
+
 def build_system_prompt(model_name: str, enabled_tools: tuple[str, ...]) -> str:
     provider = _provider_key(model_name)
     enabled = set(enabled_tools)
-    tool_lines = [RETRIEVAL_TOOL_LINE]
+    tool_lines = [RETRIEVAL_TOOL_LINE, *KB_TOOL_LINES]
     usage_sections: list[str] = []
     provider_web_tools = WEB_TOOL_LINES.get(provider, {})
     provider_web_usage = WEB_USAGE_SECTIONS.get(provider, {})
@@ -143,6 +202,7 @@ def build_system_prompt(model_name: str, enabled_tools: tuple[str, ...]) -> str:
         BASE_PROMPT_HEADER,
         f"{intro}\n\n" + "\n".join(tool_lines),
         RETRIEVAL_USAGE_SECTION,
+        KB_USAGE_SECTION,
     ]
     if usage_sections:
         parts.append("## When to use web search / URL reading\n\n" + "\n\n".join(usage_sections))
@@ -151,6 +211,14 @@ def build_system_prompt(model_name: str, enabled_tools: tuple[str, ...]) -> str:
             "course material. Combine tools when it helps (e.g. retrieve corpus\n"
             "context, then search the web for the latest update)."
         )
+    kb_agents_instructions = load_kb_agents_instructions()
+    if kb_agents_instructions:
+        parts.append(
+            "## Local KB Instructions\n\n"
+            "The following instructions are loaded from `data/kb/AGENTS.md` and "
+            "describe the current local KB schema and workflow.\n\n"
+            f"{kb_agents_instructions}"
+        )
     parts.append(RETRIEVAL_CALL_STRATEGY)
     parts.append(ANSWERING_RULES)
     return "\n\n".join(parts) + "\n"

scripts/setup.py

@@ -1,4 +1,5 @@
 import os
+from pathlib import Path
 
 import logfire
 from dotenv import load_dotenv
@@ -33,6 +34,14 @@ VECTOR_DB_DIR = "data/chroma-db-all_sources"
 VECTOR_COLLECTION_NAME = "chroma-db-all_sources"
 DOCUMENT_DICT_PATH = f"{VECTOR_DB_DIR}/document_dict_all_sources.pkl"
 BM25_INDEX_PATH = f"{VECTOR_DB_DIR}/bm25_index_all_sources.pkl"
+KB_DIR = "data/kb"
+KB_MANIFEST_PATH = f"{KB_DIR}/generated/corpus_manifest.jsonl"
+KB_INDEX_PATH = f"{KB_DIR}/wiki/index.md"
+KB_AGENTS_PATH = f"{KB_DIR}/AGENTS.md"
+# Canonical AGENTS.md content for the KB. Tracked in git alongside the
+# scraping scripts so it survives `rm -rf data/kb` and a stale HF snapshot.
+# `ensure_kb_agents_md()` copies it into `data/kb/AGENTS.md` on every startup.
+KB_AGENTS_TEMPLATE_PATH = "data/scraping_scripts/kb_agents_template.md"
 DEFAULT_MODEL_NAME = "google-genai:gemini-3.5-flash"
 
 AVAILABLE_MODELS: tuple[dict[str, str], ...] = (
@@ -44,20 +53,55 @@ CONCURRENCY_COUNT = int(os.getenv("CONCURRENCY_COUNT", 64))
 MONGODB_URI = os.getenv("MONGODB_URI")
 
 
-def ensure_local_vector_db() -> None:
-    if os.path.exists(VECTOR_DB_DIR) and os.path.exists(DOCUMENT_DICT_PATH):
+def ensure_kb_agents_md() -> None:
+    """Write `data/kb/AGENTS.md` from the canonical template if it exists.
+
+    Why: `data/kb/` is gitignored and downloaded from HuggingFace, so the
+    `AGENTS.md` that arrives in the snapshot might be stale (whoever uploaded
+    last is whoever wrote it). The template at
+    `data/scraping_scripts/kb_agents_template.md` is tracked in git and is the
+    single source of truth — we overwrite the live file from it on every
+    startup so a fresh `git pull` always propagates KB guidance changes to
+    the model, regardless of what's in the HF snapshot.
+    """
+    template_path = Path(KB_AGENTS_TEMPLATE_PATH)
+    if not template_path.exists():
+        # If the template is missing (partial checkout, etc.), don't clobber
+        # whatever AGENTS.md the HF snapshot provided — at least the agent
+        # will see *some* guidance.
         return
+    target = Path(KB_AGENTS_PATH)
+    target.parent.mkdir(parents=True, exist_ok=True)
+    target.write_text(template_path.read_text(encoding="utf-8"), encoding="utf-8")
 
-    logfire.warn(
-        "Vector database does not exist locally, downloading from Hugging Face"
-    )
-    from huggingface_hub import snapshot_download
 
-    snapshot_download(
-        repo_id="towardsai-tutors/ai-tutor-vector-db",
-        local_dir="data",
-        repo_type="dataset",
+def ensure_local_vector_db() -> None:
+    # On first start (or after `rm -rf data/{kb,chroma-db-all_sources}`),
+    # pull the whole bundle from HuggingFace. The repo
+    # `towardsai-tutors/ai-tutor-vector-db` contains both
+    # `chroma-db-all_sources/` AND `kb/`, both uploaded by
+    # `data.scraping_scripts.upload_dbs_to_hf`.
+    needs_download = not (
+        os.path.exists(VECTOR_DB_DIR)
+        and os.path.exists(DOCUMENT_DICT_PATH)
+        and os.path.exists(KB_MANIFEST_PATH)
+        and os.path.exists(KB_INDEX_PATH)
     )
+    if needs_download:
+        logfire.warn(
+            "Vector database does not exist locally, downloading from Hugging Face"
+        )
+        from huggingface_hub import snapshot_download
+
+        snapshot_download(
+            repo_id="towardsai-tutors/ai-tutor-vector-db",
+            local_dir="data",
+            repo_type="dataset",
+        )
+
+    # Always refresh AGENTS.md from the local template, regardless of whether
+    # we just downloaded or short-circuited. See `ensure_kb_agents_md` docstring.
+    ensure_kb_agents_md()
 
 
 mongo_db = (
@@ -77,10 +121,16 @@ __all__ = [
     "DEFAULT_MODEL_NAME",
     "BM25_INDEX_PATH",
     "DOCUMENT_DICT_PATH",
+    "KB_AGENTS_PATH",
+    "KB_AGENTS_TEMPLATE_PATH",
+    "KB_DIR",
+    "KB_INDEX_PATH",
+    "KB_MANIFEST_PATH",
     "SOURCE_KEY_TO_LABEL",
     "SOURCE_UI_TO_KEY",
     "VECTOR_COLLECTION_NAME",
     "VECTOR_DB_DIR",
+    "ensure_kb_agents_md",
     "ensure_local_vector_db",
     "mongo_db",
 ]

tests/manual_e2e_langsmith.md

@@ -0,0 +1,345 @@
+# Manual End-to-End LangSmith Runbook
+
+This runbook verifies the live chatbot path end to end and then looks up the
+matching LangSmith trace/run from the CLI.
+
+Use it when changing agent tools, KB browsing, citation handling, streaming API
+parts, or model/provider settings.
+
+## Prerequisites
+
+Run commands from the repository root.
+
+Required local artifacts and environment:
+
+- `.env` contains `COHERE_API_KEY`
+- `.env` contains `GEMINI_API_KEY` or `GOOGLE_API_KEY`
+- `.env` contains `LANGSMITH_API_KEY`
+- `.env` has `LANGSMITH_TRACING=true`
+- `.env` has `LANGSMITH_PROJECT=ai-tutor-app`
+- `data/chroma-db-all_sources/` exists
+- `data/kb/wiki/index.md` exists
+- `curl`, `jq`, and the `langsmith` CLI are available through `uv run`
+
+Quick check:
+
+```bash
+uv run dotenv -f .env run -- python - <<'PY'
+import os
+for key in [
+    "COHERE_API_KEY",
+    "GEMINI_API_KEY",
+    "GOOGLE_API_KEY",
+    "LANGSMITH_API_KEY",
+    "LANGSMITH_TRACING",
+    "LANGSMITH_PROJECT",
+]:
+    value = os.getenv(key)
+    print(f"{key}={'set' if value else 'missing'}")
+PY
+```
+
+## Fast Automated Smoke Tests
+
+These tests run a live model-backed request and check the frontend/API stream
+shape.
+
+API stream contract:
+
+```bash
+uv run dotenv -f .env run -- env RUN_LIVE_API_E2E=1 \
+  pytest tests/test_api.py::test_live_api_stream_exposes_frontend_parts -q
+```
+
+Gradio curl path:
+
+```bash
+uv run dotenv -f .env run -- env RUN_LIVE_GRADIO_CURL_E2E=1 \
+  pytest tests/test_gradio_kb_e2e.py::test_live_gradio_curl_can_use_retrieval_and_shell -q
+```
+
+## Manual API Test
+
+Start the FastAPI app in one terminal:
+
+```bash
+uv run dotenv -f .env run -- env \
+  AI_TUTOR_API_HOST=127.0.0.1 \
+  AI_TUTOR_API_PORT=8000 \
+  python -m scripts.api
+```
+
+From another terminal, confirm the server is ready:
+
+```bash
+curl -s http://127.0.0.1:8000/healthz
+```
+
+Create a request payload. This prompt intentionally asks for both structured
+retrieval and KB shell browsing so both citation paths are exercised.
+
+```bash
+cat >/tmp/ai_tutor_e2e_payload.json <<'JSON'
+{
+  "messages": [
+    {
+      "role": "user",
+      "parts": [
+        {
+          "type": "text",
+          "text": "Use run_kb_command to answer: can you tell me about codex?"
+        }
+      ]
+    }
+  ],
+  "sourceKeys": [
+    "agentic_ai_engineering",
+    "langchain",
+    "langgraph",
+    "peft",
+    "transformers"
+  ],
+  "enabledTools": [],
+  "model": "google-genai:gemini-3.5-flash",
+  "includeReasoning": true,
+  "threadId": ""
+}
+JSON
+```
+
+Send the request and save the server-sent event stream:
+
+```bash
+curl -sN --max-time 240 http://127.0.0.1:8000/api/chat \
+  -H "Content-Type: application/json" \
+  -d @/tmp/ai_tutor_e2e_payload.json \
+  -o /tmp/ai_tutor_e2e_api.sse
+```
+
+Convert the stream to JSONL parts:
+
+```bash
+grep '^data: ' /tmp/ai_tutor_e2e_api.sse \
+  | sed 's/^data: //' \
+  | grep -v '^\[DONE\]$' \
+  > /tmp/ai_tutor_e2e_api.parts.jsonl
+```
+
+Inspect what the frontend received:
+
+```bash
+jq -r '.type' /tmp/ai_tutor_e2e_api.parts.jsonl | sort | uniq -c
+```
+
+Inspect tool calls:
+
+```bash
+jq -r '
+  select(.type == "tool-input-available" or .type == "tool-input-start")
+  | [.type, .toolName, ((.input // .inputText // "") | tostring | gsub("\n"; " ") | .[0:220])]
+  | @tsv
+' /tmp/ai_tutor_e2e_api.parts.jsonl
+```
+
+Inspect final answer text:
+
+```bash
+jq -r 'select(.type == "text-delta") | .delta' \
+  /tmp/ai_tutor_e2e_api.parts.jsonl \
+  | tr -d '\n'
+echo
+```
+
+Inspect source cards:
+
+```bash
+jq -r '
+  select(.type == "data-source")
+  | [.data.title, .data.url, .data.sourceKey, (.data.score // "")]
+  | @tsv
+' /tmp/ai_tutor_e2e_api.parts.jsonl
+```
+
+Expected result:
+
+- The stream includes `tool-input-*`, `tool-output-available`, `text-delta`,
+  `source-url`, `source-document`, `data-source`, and `finish` parts.
+- Tool calls include `retrieve_tutor_context` and `run_kb_command`.
+- The answer has inline citations, not only a final sources list.
+- `data-source` parts include the source cards the frontend will render.
+
+## Manual Gradio Test
+
+Start the Gradio app in one terminal:
+
+```bash
+uv run dotenv -f .env run -- python -m scripts.main
+```
+
+Submit a Gradio job:
+
+```bash
+cat >/tmp/ai_tutor_gradio_payload.json <<'JSON'
+{
+  "data": [
+    "Use run_kb_command to answer: can you tell me about codex?",
+    [],
+    [
+      "Agentic AI Engineering",
+      "LangChain Docs",
+      "LangGraph Docs",
+      "PEFT Docs",
+      "Transformers Docs"
+    ],
+    "google-genai:gemini-3.5-flash",
+    "",
+    false,
+    false
+  ]
+}
+JSON
+
+curl -s http://127.0.0.1:7860/gradio_api/call/chat \
+  -H "Content-Type: application/json" \
+  -d @/tmp/ai_tutor_gradio_payload.json \
+  -o /tmp/ai_tutor_gradio_post.json
+```
+
+Stream the result:
+
+```bash
+EVENT_ID="$(jq -r '.event_id' /tmp/ai_tutor_gradio_post.json)"
+
+curl -N --max-time 240 \
+  "http://127.0.0.1:7860/gradio_api/call/chat/${EVENT_ID}" \
+  -o /tmp/ai_tutor_gradio_stream.sse
+```
+
+Expected result:
+
+- The rendered text contains `Using \`retrieve_tutor_context\``.
+- The rendered text contains `Using \`run_kb_command\``.
+- The rendered text contains inline citations and source cards.
+
+## Find The LangSmith Trace
+
+List recent traces in the project:
+
+```bash
+uv run dotenv -f .env run -- langsmith trace list \
+  --project ai-tutor-app \
+  --limit 10 \
+  --full \
+  --format json \
+  -o /tmp/ai_tutor_langsmith_traces.json
+```
+
+Print the likely match. Look for the newest trace whose input contains the
+manual prompt.
+
+```bash
+jq -r '
+  .[]
+  | [
+      (.trace_id // .id),
+      (.status // ""),
+      (.duration_ms // .latency_ms // .latency // ""),
+      ((.inputs.query // .inputs.input // .inputs.messages[-1].content // .inputs // "") | tostring | gsub("\n"; " ") | .[0:220])
+    ]
+  | @tsv
+' /tmp/ai_tutor_langsmith_traces.json
+```
+
+Save the trace ID:
+
+```bash
+TRACE_ID="paste-trace-id-here"
+```
+
+Fetch the full trace:
+
+```bash
+uv run dotenv -f .env run -- langsmith trace get "$TRACE_ID" \
+  --project ai-tutor-app \
+  --full \
+  --format json \
+  -o "/tmp/ai_tutor_trace_${TRACE_ID}.json"
+```
+
+Fetch the corresponding run. For the root run, the trace ID is usually also
+the run ID. If this fails, open the trace JSON and use the root run ID shown
+there.
+
+```bash
+uv run dotenv -f .env run -- langsmith run get "$TRACE_ID" \
+  --project ai-tutor-app \
+  --full \
+  --format json \
+  -o "/tmp/ai_tutor_run_${TRACE_ID}.json"
+```
+
+## Inspect Trace/Run Details
+
+Overall status and latency:
+
+```bash
+jq '{
+  id: (.trace_id // .id),
+  name,
+  status,
+  duration_ms: (.duration_ms // .latency_ms // .latency),
+  run_count: (.run_count // ((.runs // .child_runs // []) | length))
+}' "/tmp/ai_tutor_trace_${TRACE_ID}.json"
+```
+
+Tool calls:
+
+```bash
+jq -r '
+  .. | objects
+  | select((.name? // "") | test("retrieve_tutor_context|run_kb_command"))
+  | [
+      (.name // ""),
+      (.status // ""),
+      (.duration_ms // .latency_ms // .latency // ""),
+      ((.inputs.command // .inputs.query // .inputs.input // "") | tostring | gsub("\n"; " ") | .[0:220])
+    ]
+  | @tsv
+' "/tmp/ai_tutor_trace_${TRACE_ID}.json"
+```
+
+LLM calls:
+
+```bash
+jq -r '
+  .. | objects
+  | select((.run_type? // "") | test("llm|chat_model"))
+  | [
+      (.name // ""),
+      (.status // ""),
+      (.duration_ms // .latency_ms // .latency // ""),
+      (.prompt_tokens // .usage.prompt_tokens // .outputs.llm_output.token_usage.prompt_tokens // ""),
+      (.completion_tokens // .usage.completion_tokens // .outputs.llm_output.token_usage.completion_tokens // "")
+    ]
+  | @tsv
+' "/tmp/ai_tutor_trace_${TRACE_ID}.json"
+```
+
+Search the raw trace for the KB shell command sequence:
+
+```bash
+rg -n "run_kb_command|retrieve_tutor_context|sed -n|rg -n|cat wiki" \
+  "/tmp/ai_tutor_trace_${TRACE_ID}.json"
+```
+
+For latency debugging, compare these numbers:
+
+- Root trace duration.
+- Count of `run_kb_command` calls.
+- Total shell/tool duration.
+- Count of LLM/chat model calls.
+- Longest LLM call duration.
+- Whether the trace ended in `success`, `error`, or cancellation.
+
+If shell/tool duration is tiny but total trace duration is large, the bottleneck
+is model inference or repeated model turns, not filesystem access.

tests/test_api.py

@@ -1,9 +1,18 @@
 from __future__ import annotations
 
 import json
+import os
+import re
+import shutil
+import socket
+import subprocess
+import threading
+import time
 import unittest
 from unittest.mock import patch
 
+import pytest
+import uvicorn
 from fastapi.testclient import TestClient
 
 from scripts.api import app
@@ -18,6 +27,12 @@ def parse_sse_payloads(raw_text: str) -> list[str]:
     ]
 
 
+def free_port() -> int:
+    with socket.socket() as sock:
+        sock.bind(("127.0.0.1", 0))
+        return int(sock.getsockname()[1])
+
+
 class ApiTestCase(unittest.TestCase):
     def test_healthcheck(self) -> None:
         with TestClient(app) as client:
@@ -230,5 +245,176 @@ class ApiTestCase(unittest.TestCase):
         self.assertLess(part_types.index("tool-input-start"), part_types.index("text-start"))
 
 
+LIVE_API_E2E = pytest.mark.skipif(
+    os.getenv("RUN_LIVE_API_E2E") != "1",
+    reason="Set RUN_LIVE_API_E2E=1 to run the live frontend API smoke test.",
+)
+
+
+def require_live_api_e2e_prereqs() -> None:
+    if shutil.which("curl") is None:
+        pytest.skip("curl is required for the live API smoke test.")
+    if not os.getenv("COHERE_API_KEY"):
+        pytest.skip("COHERE_API_KEY is required for live retrieval.")
+    if not (os.getenv("GEMINI_API_KEY") or os.getenv("GOOGLE_API_KEY")):
+        pytest.skip("GEMINI_API_KEY or GOOGLE_API_KEY is required for the live model.")
+    if not os.path.exists("data/kb/wiki/index.md"):
+        pytest.skip("data/kb artifacts must exist before running the live smoke test.")
+
+
+def run_live_api_chat(prompt: str) -> list[dict]:
+    port = free_port()
+    config = uvicorn.Config(
+        app,
+        host="127.0.0.1",
+        port=port,
+        log_level="warning",
+        lifespan="on",
+    )
+    server = uvicorn.Server(config)
+    thread = threading.Thread(target=server.run, daemon=True)
+    thread.start()
+    try:
+        deadline = time.time() + 30
+        while time.time() < deadline:
+            health = subprocess.run(
+                ["curl", "-s", f"http://127.0.0.1:{port}/healthz"],
+                capture_output=True,
+                text=True,
+                check=False,
+                timeout=5,
+            )
+            if health.returncode == 0 and "ok" in health.stdout:
+                break
+            time.sleep(0.25)
+        else:
+            pytest.fail("API server did not become healthy")
+
+        payload = {
+            "messages": [
+                {
+                    "role": "user",
+                    "parts": [
+                        {
+                            "type": "text",
+                            "text": prompt,
+                        }
+                    ],
+                }
+            ],
+            "sourceKeys": ["peft", "transformers"],
+            "enabledTools": [],
+            "model": os.getenv("LIVE_API_E2E_MODEL", "google-genai:gemini-3.5-flash"),
+            "includeReasoning": False,
+            "threadId": "",
+        }
+        response = subprocess.run(
+            [
+                "curl",
+                "-sN",
+                f"http://127.0.0.1:{port}/api/chat",
+                "-H",
+                "Content-Type: application/json",
+                "-d",
+                json.dumps(payload),
+            ],
+            capture_output=True,
+            text=True,
+            check=True,
+            timeout=240,
+        )
+    finally:
+        server.should_exit = True
+        thread.join(timeout=10)
+
+    payloads = parse_sse_payloads(response.stdout)
+    assert payloads[-1] == "[DONE]"
+    return [json.loads(item) for item in payloads[:-1]]
+
+
+def live_part_types(parts: list[dict]) -> list[str]:
+    return [part["type"] for part in parts]
+
+
+def live_tool_names(parts: list[dict]) -> set[str]:
+    return {
+        part.get("toolName")
+        for part in parts
+        if part["type"] in {"tool-input-start", "tool-input-available"}
+    }
+
+
+def live_answer_text(parts: list[dict]) -> str:
+    return "\n".join(
+        part.get("delta", "") for part in parts if part["type"] == "text-delta"
+    )
+
+
+def live_sources(parts: list[dict]) -> list[dict]:
+    return [part["data"] for part in parts if part["type"] == "data-source"]
+
+
+@LIVE_API_E2E
+def test_live_api_stream_exposes_frontend_parts() -> None:
+    require_live_api_e2e_prereqs()
+
+    parts = run_live_api_chat(
+        "Use both retrieve_tutor_context and run_kb_command to answer: "
+        "how does PEFT configure LoRA with LoraConfig?"
+    )
+    part_types = [part["type"] for part in parts]
+    assert "data-thread" in part_types
+    assert "tool-input-start" in part_types
+    assert "tool-output-available" in part_types
+    assert "text-delta" in part_types
+    assert "source-url" in part_types
+    assert "source-document" in part_types
+    assert "data-source" in part_types
+    assert "finish" in part_types
+    tool_names = live_tool_names(parts)
+    assert {"retrieve_tutor_context", "run_kb_command"} <= tool_names
+    final_text = live_answer_text(parts)
+    assert "LoRA" in final_text
+    assert re.search(r"\[[^\]]+\]\((?:https://|raw/|kb://doc/)", final_text)
+    assert "### Sources" not in final_text
+    sources = live_sources(parts)
+    assert any(source["sourceKey"] in {"peft", "transformers"} for source in sources)
+
+
+@LIVE_API_E2E
+def test_live_api_stream_obeys_retrieval_only_instruction() -> None:
+    require_live_api_e2e_prereqs()
+
+    parts = run_live_api_chat(
+        "Use only the retrieve_tutor_context tool. Do not call run_kb_command. "
+        "Answer: how does PEFT configure LoRA with LoraConfig?"
+    )
+
+    assert "data-source" in live_part_types(parts)
+    tool_names = live_tool_names(parts)
+    assert "retrieve_tutor_context" in tool_names
+    assert "run_kb_command" not in tool_names
+    assert "LoRA" in live_answer_text(parts)
+    assert live_sources(parts)
+
+
+@LIVE_API_E2E
+def test_live_api_stream_obeys_shell_only_instruction() -> None:
+    require_live_api_e2e_prereqs()
+
+    parts = run_live_api_chat(
+        "Use only the run_kb_command tool. Do not call retrieve_tutor_context. "
+        "Inspect raw PEFT docs and answer: how does PEFT configure LoRA with "
+        "LoraConfig?"
+    )
+
+    assert "data-source" in live_part_types(parts)
+    tool_names = live_tool_names(parts)
+    assert "run_kb_command" in tool_names
+    assert "retrieve_tutor_context" not in tool_names
+    assert "LoRA" in live_answer_text(parts)
+    assert live_sources(parts)
+
+
 if __name__ == "__main__":
     unittest.main()

tests/test_build_kb_artifacts.py

@@ -0,0 +1,106 @@
+from __future__ import annotations
+
+import json
+from pathlib import Path
+
+from data.scraping_scripts import build_kb_artifacts as builder
+
+
+def write_jsonl(path: Path, rows: list[dict]) -> None:
+    with path.open("w", encoding="utf-8") as handle:
+        for row in rows:
+            handle.write(json.dumps(row) + "\n")
+
+
+def test_build_kb_artifacts_writes_markdown_and_indexes(
+    tmp_path: Path,
+    monkeypatch,
+) -> None:
+    input_file = tmp_path / "all_sources_data.jsonl"
+    output_dir = tmp_path / "kb"
+    docs_dir = tmp_path / "temp_docs_md_files"
+    docs_page = docs_dir / "package_reference" / "lora.mdx"
+    docs_page.parent.mkdir(parents=True)
+    docs_content = """---
+sidebarTitle: LoRA Guide
+---
+# LoRA Guide
+
+Use `LoraConfig` with `get_peft_model`.
+"""
+    docs_page.write_text(docs_content, encoding="utf-8")
+    monkeypatch.setitem(
+        builder.SOURCE_CONFIGS,
+        "temp_docs",
+        {
+            "base_url": "https://example.com/docs/",
+            "input_directory": str(docs_dir),
+            "output_file": str(tmp_path / "temp_docs_data.jsonl"),
+            "source_name": "temp_docs",
+            "use_include_list": False,
+            "included_dirs": [],
+            "excluded_dirs": [],
+            "excluded_root_files": [],
+            "included_root_files": [],
+            "url_extension": "",
+        },
+    )
+    write_jsonl(
+        input_file,
+        [
+            {
+                "doc_id": "legacy-random-id",
+                "name": "LoRA Guide",
+                "url": "https://example.com/docs/package_reference/lora",
+                "source": "temp_docs",
+                "tokens": 400,
+                "retrieve_doc": True,
+                "content": docs_content,
+            },
+            {
+                "doc_id": "0a4fe6fa-928c-4cbf-951c-d0fd9dce01f3",
+                "name": "Lesson 18: Research Loop",
+                "url": "https://academy.towardsai.net/courses/take/agent-engineering/multimedia/70289117-lesson-18-the-research-loop",
+                "source": "agentic_ai_engineering",
+                "tokens": 800,
+                "retrieve_doc": True,
+                "content": "# Lesson 18: Research Loop\n\nCall `generate_next_queries_tool`.",
+            },
+        ],
+    )
+
+    summary = builder.build_kb_artifacts(input_file, output_dir)
+
+    assert summary == {"documents": 2, "manifest_rows": 2}
+    manifest_path = output_dir / "generated" / "corpus_manifest.jsonl"
+    headings_path = output_dir / "generated" / "headings.jsonl"
+    symbols_path = output_dir / "generated" / "symbols.tsv"
+    assert manifest_path.exists()
+    assert headings_path.exists()
+    assert symbols_path.exists()
+
+    manifest_rows = [
+        json.loads(line)
+        for line in manifest_path.read_text(encoding="utf-8").splitlines()
+    ]
+    assert manifest_rows[0]["doc_id"] == "temp_docs:docs-package-reference-lora"
+    assert manifest_rows[0]["content_hash"].startswith("sha256:")
+    assert manifest_rows[0]["source_path"] == "package_reference/lora.mdx"
+    assert manifest_rows[0]["original_path"] == docs_page.as_posix()
+
+    markdown_path = Path(manifest_rows[0]["path"])
+    assert markdown_path == output_dir / "raw" / "docs" / "temp_docs" / "package_reference" / "lora.mdx"
+    markdown = markdown_path.read_text(encoding="utf-8")
+    assert 'doc_id: "temp_docs:docs-package-reference-lora"' in markdown
+    assert 'source_path: "package_reference/lora.mdx"' in markdown
+    assert "sidebarTitle" not in markdown
+    assert "# LoRA Guide" in markdown
+    assert "`LoraConfig`" in markdown
+
+    course_row = manifest_rows[1]
+    assert course_row["source_group"] == "courses"
+    assert course_row["source_path"] == ""
+
+    symbols = symbols_path.read_text(encoding="utf-8")
+    assert "LoraConfig" in symbols
+    assert "generate_next_queries_tool" in symbols

tests/test_chat_service.py

@@ -1,18 +1,23 @@
 from __future__ import annotations
 
+import asyncio
 import types
 import unittest
 from unittest.mock import patch
 
-from langchain_core.messages import AIMessage, HumanMessage
+from langchain_core.messages import AIMessage, AIMessageChunk, HumanMessage, ToolMessage
 
 from scripts.chat_service import (
+    _claim_kb_command_budget,
+    _clear_kb_command_budget,
     agent_run_config,
     build_agent,
     effective_tool_names,
+    resolve_answer_citations,
     sync_thread_with_history,
+    stream_chat,
 )
-from scripts.chat_types import ChatRequest, ChatTurn
+from scripts.chat_types import ChatRequest, ChatTurn, SourceMatch
 
 
 class FakeAgent:
@@ -27,6 +32,61 @@ class FakeAgent:
         self.updated_states.append((config, payload))
 
 
+class FakeStreamingAgent(FakeAgent):
+    async def astream(self, *_args, **_kwargs):
+        yield {
+            "type": "messages",
+            "data": (
+                AIMessageChunk(
+                    content="",
+                    tool_calls=[
+                        {
+                            "id": "call_rg",
+                            "name": "run_kb_command",
+                            "args": {"command": "rg LoraConfig raw"},
+                        }
+                    ],
+                ),
+                {"langgraph_step": "model"},
+            ),
+        }
+        yield {
+            "type": "updates",
+            "data": {
+                "tools": {
+                    "messages": [
+                        ToolMessage(
+                            content=(
+                                "$ rg LoraConfig raw\n"
+                                "cwd: /tmp/kb\n"
+                                "exit_code: 0\n"
+                                "stdout:\n"
+                                "raw/docs/peft/lora.md:3:LoraConfig"
+                            ),
+                            name="run_kb_command",
+                            tool_call_id="call_rg",
+                        )
+                    ]
+                }
+            },
+        }
+        yield {
+            "type": "updates",
+            "data": {
+                "model": {
+                    "messages": [
+                        AIMessage(
+                            content=(
+                                "LoraConfig is documented in LoRA "
+                                "[LoRA](raw/docs/peft/lora.md)."
+                            )
+                        )
+                    ]
+                }
+            },
+        }
+
+
 class ChatServiceTestCase(unittest.TestCase):
     def test_effective_tool_names_follow_provider(self) -> None:
         self.assertEqual(
@@ -34,14 +94,24 @@ class ChatServiceTestCase(unittest.TestCase):
                 "google-genai:gemini-3.5-flash",
                 ("web_search", "url_context", "web_fetch"),
             ),
-            ("retrieve_tutor_context", "google_search", "url_context"),
+            (
+                "retrieve_tutor_context",
+                "run_kb_command",
+                "google_search",
+                "url_context",
+            ),
         )
         self.assertEqual(
             effective_tool_names(
                 "anthropic:claude-haiku-4-5",
                 ("web_search", "url_context", "web_fetch"),
             ),
-            ("retrieve_tutor_context", "web_search", "web_fetch"),
+            (
+                "retrieve_tutor_context",
+                "run_kb_command",
+                "web_search",
+                "web_fetch",
+            ),
         )
 
     def test_agent_run_config_adds_langsmith_metadata(self) -> None:
@@ -59,13 +129,18 @@ class ChatServiceTestCase(unittest.TestCase):
         self.assertEqual(config["run_name"], "ai-tutor-agent-turn")
         self.assertIn("provider:google-genai", config["tags"])
         self.assertIn("tool:retrieve_tutor_context", config["tags"])
+        self.assertIn("tool:run_kb_command", config["tags"])
         self.assertIn("tool:google_search", config["tags"])
         self.assertEqual(config["metadata"]["thread_id"], "thread_123")
         self.assertEqual(config["metadata"]["conversation_id"], "thread_123")
         self.assertEqual(config["metadata"]["message_id"], "message_456")
         self.assertEqual(
             config["metadata"]["available_tools"],
-            ["retrieve_tutor_context", "google_search"],
+            [
+                "retrieve_tutor_context",
+                "run_kb_command",
+                "google_search",
+            ],
         )
         self.assertEqual(
             config["metadata"]["source_keys"],
@@ -117,10 +192,117 @@ class ChatServiceTestCase(unittest.TestCase):
         disabled_tool_defs = created_agents[1].kwargs["tools"]
         self.assertIn({"google_search": {}}, enabled_tool_defs)
         self.assertIn({"url_context": {}}, enabled_tool_defs)
-        self.assertEqual(disabled_tool_defs, [enabled_tool_defs[0]])
+        self.assertEqual(len(disabled_tool_defs), 2)
+        disabled_tool_names = {tool.name for tool in disabled_tool_defs}
+        self.assertEqual(
+            disabled_tool_names,
+            {
+                "retrieve_tutor_context",
+                "run_kb_command",
+            },
+        )
         self.assertEqual(len(created_agents[0].kwargs["middleware"]), 1)
         self.assertEqual(created_agents[1].kwargs["middleware"], [])
 
+    def test_kb_command_budget_blocks_after_limit(self) -> None:
+        session_id = "test_budget_session"
+        _clear_kb_command_budget(session_id)
+        try:
+            self.assertEqual(_claim_kb_command_budget(session_id, 2), (True, 1))
+            self.assertEqual(_claim_kb_command_budget(session_id, 2), (True, 2))
+            self.assertEqual(_claim_kb_command_budget(session_id, 2), (False, 2))
+        finally:
+            _clear_kb_command_budget(session_id)
+
+    def test_resolve_answer_citations_uses_current_turn_evidence(self) -> None:
+        retrieval = SourceMatch(
+            doc_id="peft:lora",
+            title="LoRA",
+            url="https://example.com/lora",
+            source_key="peft",
+            source_label="PEFT Docs",
+            score=12.0,
+            group="docs",
+        )
+
+        resolved = resolve_answer_citations(
+            "See [LoRA](https://example.com/lora) and [Other](https://example.com/other).",
+            retrieval_evidence={"peft:lora": retrieval},
+            shell_evidence={},
+            web_evidence={},
+        )
+
+        self.assertEqual(resolved, [retrieval])
+
+    def test_resolve_answer_citations_ignores_unseen_kb_paths(self) -> None:
+        resolved = resolve_answer_citations(
+            "See [LoRA](raw/docs/peft/lora.md).",
+            retrieval_evidence={},
+            shell_evidence={},
+            web_evidence={},
+        )
+
+        self.assertEqual(resolved, [])
+
+    def test_stream_chat_resolves_shell_citation_after_final_answer(self) -> None:
+        agent = FakeStreamingAgent([])
+        request = ChatRequest(
+            query="Use rg to find LoraConfig",
+            source_keys=("peft",),
+            model_name="google-genai:gemini-3.5-flash",
+            include_reasoning=False,
+            enabled_tools=(),
+        )
+
+        async def collect_events():
+            return [event async for event in stream_chat(request)]
+
+        shell_match = SourceMatch(
+            doc_id="peft:lora",
+            title="LoRA",
+            url="https://example.com/lora",
+            source_key="peft",
+            source_label="PEFT Docs",
+            score=1.0,
+            group="docs",
+        )
+
+        def fake_resolve_manifest_reference(reference, **_kwargs):
+            return shell_match if "raw/docs/peft/lora.md" in reference else None
+
+        with (
+            patch("scripts.chat_service.build_agent", return_value=agent),
+            patch("scripts.chat_service.new_thread_id", return_value="thread_rg"),
+            patch(
+                "scripts.chat_service.resolve_manifest_reference",
+                side_effect=fake_resolve_manifest_reference,
+            ),
+        ):
+            events = asyncio.run(collect_events())
+
+        started = [
+            event
+            for event in events
+            if event.type == "tool_call_started"
+            and event.data.get("tool_name") == "run_kb_command"
+        ]
+        completed = [
+            event
+            for event in events
+            if event.type == "tool_call_completed"
+            and event.data.get("tool_name") == "run_kb_command"
+        ]
+        source_matches = [
+            event
+            for event in events
+            if event.type == "source_match"
+        ]
+
+        self.assertEqual(started[0].data["args_text"], "rg LoraConfig raw")
+        self.assertIn("rg LoraConfig raw", completed[0].data["output_text"])
+        self.assertEqual(source_matches[0].data["source_key"], "peft")
+        self.assertNotIn("call_id", source_matches[0].data)
+
     def test_shorter_history_branches_to_fresh_thread(self) -> None:
         agent = FakeAgent(
             [

tests/test_gradio_kb_e2e.py

@@ -0,0 +1,207 @@
+from __future__ import annotations
+
+import socket
+import os
+
+import pytest
+from gradio_client import Client
+
+from scripts.chat_types import ChatEvent
+
+
+def free_port() -> int:
+    with socket.socket() as sock:
+        sock.bind(("127.0.0.1", 0))
+        return int(sock.getsockname()[1])
+
+
+def test_gradio_chat_api_surfaces_shell_and_retrieval_activity(monkeypatch) -> None:
+    import scripts.main as main
+
+    async def fake_stream_chat(_request):
+        yield ChatEvent("thread_started", {"thread_id": "thread_kb"})
+        yield ChatEvent("message_started", {"message_id": "message_kb"})
+        yield ChatEvent(
+            "tool_call_started",
+            {
+                "message_id": "message_kb",
+                "call_id": "call_search",
+                "tool_name": "retrieve_tutor_context",
+                "args": {"query": "generate_next_queries_tool"},
+                "args_text": "generate_next_queries_tool",
+            },
+        )
+        yield ChatEvent(
+            "tool_call_completed",
+            {
+                "message_id": "message_kb",
+                "call_id": "call_search",
+                "tool_name": "retrieve_tutor_context",
+                "args": {"query": "generate_next_queries_tool"},
+                "args_text": "generate_next_queries_tool",
+                "output_text": (
+                    '{"matches": [{"doc_id": "agentic_ai_engineering:lesson-18", '
+                    '"title": "Lesson 18: Research Loop", '
+                    '"url": "https://academy.towardsai.net/lesson-18", '
+                    '"source": "agentic_ai_engineering", "score": 10.0}]}'
+                ),
+            },
+        )
+        yield ChatEvent(
+            "tool_call_started",
+            {
+                "message_id": "message_kb",
+                "call_id": "call_rg",
+                "tool_name": "run_kb_command",
+                "args": {"command": "rg generate_next_queries_tool raw"},
+                "args_text": "rg generate_next_queries_tool raw",
+            },
+        )
+        yield ChatEvent(
+            "tool_call_completed",
+            {
+                "message_id": "message_kb",
+                "call_id": "call_rg",
+                "tool_name": "run_kb_command",
+                "args": {"command": "rg generate_next_queries_tool raw"},
+                "args_text": "rg generate_next_queries_tool raw",
+                "output_text": "$ rg generate_next_queries_tool raw\ncwd: data/kb\nexit_code: 0\nstdout:\nraw/courses/agentic_ai_engineering/lesson-18.md:4:Call `generate_next_queries_tool`.",
+            },
+        )
+        yield ChatEvent(
+            "text_delta",
+            {
+                "message_id": "message_kb",
+                "text": "The symbol appears in Lesson 18: Research Loop.",
+            },
+        )
+        yield ChatEvent(
+            "source_match",
+            {
+                "message_id": "message_kb",
+                "doc_id": "agentic_ai_engineering:lesson-18",
+                "title": "Lesson 18: Research Loop",
+                "url": "https://academy.towardsai.net/lesson-18",
+                "source_key": "agentic_ai_engineering",
+                "source_label": "Agentic AI Engineering",
+                "score": 10.0,
+                "group": "courses",
+            },
+        )
+        yield ChatEvent(
+            "message_completed",
+            {
+                "message_id": "message_kb",
+                "thread_id": "thread_kb",
+                "answer": "The symbol appears in Lesson 18: Research Loop.",
+            },
+        )
+
+    monkeypatch.setattr(main, "stream_chat", fake_stream_chat)
+    port = free_port()
+    main.demo.launch(
+        server_name="127.0.0.1",
+        server_port=port,
+        prevent_thread_lock=True,
+        quiet=True,
+    )
+    try:
+        client = Client(f"http://127.0.0.1:{port}")
+        result = client.predict(
+            "Where is generate_next_queries_tool discussed?",
+            [],
+            ["Agentic AI Engineering"],
+            "google-genai:gemini-3.5-flash",
+            "",
+            False,
+            False,
+            api_name="/chat",
+        )
+    finally:
+        main.demo.close()
+
+    rendered = result[0] if isinstance(result, tuple) else str(result)
+    assert "Using `retrieve_tutor_context`" in rendered
+    assert "Using `run_kb_command`" in rendered
+    assert "rg generate_next_queries_tool raw" in rendered
+    assert "Lesson 18: Research Loop" in rendered
+    assert "Agentic AI Engineering" in rendered
+
+
+@pytest.mark.skipif(
+    os.getenv("RUN_LIVE_GRADIO_CURL_E2E") != "1",
+    reason="Set RUN_LIVE_GRADIO_CURL_E2E=1 to run the live model-backed Gradio curl smoke test.",
+)
+def test_live_gradio_curl_can_use_retrieval_and_shell() -> None:
+    if os.getenv("RUN_LIVE_GRADIO_CURL_E2E") != "1":
+        pytest.skip("Set RUN_LIVE_GRADIO_CURL_E2E=1 to run this test.")
+    import scripts.main as main
+
+    import json
+    import shutil
+    import subprocess
+
+    if shutil.which("curl") is None:
+        pytest.skip("curl is required for the live Gradio curl smoke test.")
+    if not os.getenv("COHERE_API_KEY"):
+        pytest.skip("COHERE_API_KEY is required for live retrieval fallback.")
+    if not (os.getenv("GEMINI_API_KEY") or os.getenv("GOOGLE_API_KEY")):
+        pytest.skip("GEMINI_API_KEY or GOOGLE_API_KEY is required for the live model.")
+    if not os.path.exists("data/kb/wiki/index.md"):
+        pytest.skip("data/kb artifacts must exist before running the live smoke test.")
+
+    port = free_port()
+    main.demo.launch(
+        server_name="127.0.0.1",
+        server_port=port,
+        prevent_thread_lock=True,
+        quiet=True,
+    )
+    try:
+        post = subprocess.run(
+            [
+                "curl",
+                "-s",
+                f"http://127.0.0.1:{port}/gradio_api/call/chat",
+                "-H",
+                "Content-Type: application/json",
+                "-d",
+                json.dumps(
+                    {
+                        "data": [
+                            "Use both retrieve_tutor_context and run_kb_command to answer: how does PEFT configure LoRA with LoraConfig?",
+                            [],
+                            ["PEFT Docs", "Transformers Docs"],
+                            os.getenv("LIVE_GRADIO_E2E_MODEL", "google-genai:gemini-3.5-flash"),
+                            "",
+                            False,
+                            False,
+                        ]
+                    }
+                ),
+            ],
+            check=True,
+            capture_output=True,
+            text=True,
+            timeout=30,
+        )
+        event_id = json.loads(post.stdout)["event_id"]
+        stream = subprocess.run(
+            [
+                "curl",
+                "-N",
+                f"http://127.0.0.1:{port}/gradio_api/call/chat/{event_id}",
+            ],
+            check=True,
+            capture_output=True,
+            text=True,
+            timeout=180,
+        )
+    finally:
+        main.demo.close()
+
+    rendered = stream.stdout
+    assert "Using `retrieve_tutor_context`" in rendered
+    assert "Using `run_kb_command`" in rendered
+    assert "LoRA" in rendered
+    assert "PEFT" in rendered or "Transformers" in rendered

tests/test_gradio_presenter.py

@@ -3,7 +3,11 @@ from __future__ import annotations
 import unittest
 
 from scripts.chat_types import ChatEvent
-from scripts.gradio_presenter import GradioPresenterState, SOURCES_HEADER
+from scripts.gradio_presenter import (
+    GradioPresenterState,
+    SOURCES_HEADER,
+    summarize_tool_result,
+)
 
 
 class GradioPresenterTestCase(unittest.TestCase):
@@ -46,6 +50,43 @@ class GradioPresenterTestCase(unittest.TestCase):
         self.assertIn(SOURCES_HEADER, completed)
         self.assertIn("RAG overview", completed)
 
+    def test_retrieval_summary_uses_tool_payload_matches(self) -> None:
+        event = ChatEvent(
+            "tool_call_completed",
+            {
+                "tool_name": "retrieve_tutor_context",
+                "output_text": (
+                    '{"matches": [{"title": "LoRA", '
+                    '"source_label": "PEFT Docs"}]}'
+                ),
+            },
+        )
+
+        self.assertEqual(
+            summarize_tool_result(event),
+            "_Found 1 match from PEFT Docs._",
+        )
+
+    def test_run_kb_command_summary_confirms_command_execution(self) -> None:
+        event = ChatEvent(
+            "tool_call_completed",
+            {
+                "tool_name": "run_kb_command",
+                "output_text": (
+                    "$ rg LoraConfig raw\n"
+                    "cwd: data/kb\n"
+                    "exit_code: 0\n"
+                    "stdout:\n"
+                    "raw/docs/peft/lora.md:3:LoraConfig"
+                ),
+            },
+        )
+
+        self.assertEqual(
+            summarize_tool_result(event),
+            "_Ran `rg LoraConfig raw` with exit code 0._",
+        )
+
 
 if __name__ == "__main__":
     unittest.main()

tests/test_kb_shell.py

@@ -0,0 +1,102 @@
+from __future__ import annotations
+
+import json
+import shutil
+from pathlib import Path
+
+import pytest
+
+from data.scraping_scripts.build_kb_artifacts import build_kb_artifacts
+from data.scraping_scripts.update_kb_wiki import update_kb_wiki
+from scripts.kb_shell import KbCommandError, format_command_payload, run_kb_command
+
+
+def write_jsonl(path: Path, rows: list[dict]) -> None:
+    with path.open("w", encoding="utf-8") as handle:
+        for row in rows:
+            handle.write(json.dumps(row) + "\n")
+
+
+@pytest.fixture()
+def kb_dir(tmp_path: Path) -> Path:
+    input_file = tmp_path / "all_sources_data.jsonl"
+    output_dir = tmp_path / "kb"
+    write_jsonl(
+        input_file,
+        [
+            {
+                "doc_id": "peft:lora",
+                "name": "LoRA",
+                "url": "https://example.com/peft/lora",
+                "source": "peft",
+                "source_path": "package_reference/lora.md",
+                "tokens": 300,
+                "retrieve_doc": True,
+                "content": "# LoRA\n\nUse `LoraConfig` with `get_peft_model`.",
+            },
+            {
+                "doc_id": "agentic_ai_engineering:lesson-18",
+                "name": "Lesson 18: Research Loop",
+                "url": "https://academy.towardsai.net/lesson-18",
+                "source": "agentic_ai_engineering",
+                "tokens": 500,
+                "retrieve_doc": True,
+                "content": "# Lesson 18\n\nCall `generate_next_queries_tool`.",
+            },
+        ],
+    )
+    build_kb_artifacts(input_file, output_dir)
+    update_kb_wiki(output_dir, seed_defaults=True, changed_only=False)
+    return output_dir
+
+
+@pytest.mark.skipif(shutil.which("rg") is None, reason="ripgrep is required")
+def test_run_kb_command_runs_rg_and_returns_plain_transcript(kb_dir: Path) -> None:
+    result = run_kb_command(
+        "rg LoraConfig raw/docs/peft",
+        root=kb_dir,
+    )
+
+    assert result.command == "rg LoraConfig raw/docs/peft"
+    assert result.argv[:4] == ["rg", "--color=never", "--line-number", "--no-heading"]
+    assert result.exit_code == 0
+    assert not result.timed_out
+    assert "LoraConfig" in result.stdout
+
+    payload = format_command_payload(result)
+    assert "$ rg LoraConfig raw/docs/peft" in payload
+    assert "exit_code: 0" in payload
+    assert "stdout:" in payload
+    assert "matches" not in payload
+
+
+@pytest.mark.skipif(shutil.which("rg") is None, reason="ripgrep is required")
+def test_run_kb_command_can_read_all_raw_sources(kb_dir: Path) -> None:
+    result = run_kb_command(
+        "rg generate_next_queries_tool raw/courses/agentic_ai_engineering",
+        root=kb_dir,
+    )
+
+    assert result.exit_code == 0
+    assert "generate_next_queries_tool" in result.stdout
+    assert "agentic_ai_engineering" in result.stdout
+
+
+def test_run_kb_command_rejects_shell_chaining(kb_dir: Path) -> None:
+    with pytest.raises(KbCommandError):
+        run_kb_command("rg LoraConfig | head", root=kb_dir)
+
+
+def test_run_kb_command_rejects_path_traversal(kb_dir: Path) -> None:
+    with pytest.raises(KbCommandError):
+        run_kb_command("cat ../outside.md", root=kb_dir)
+
+
+def test_run_kb_command_rejects_unbounded_broad_raw_search(kb_dir: Path) -> None:
+    with pytest.raises(KbCommandError, match="requires -m"):
+        run_kb_command("rg LoraConfig raw", root=kb_dir)
+
+    result = run_kb_command("rg -m 20 LoraConfig raw", root=kb_dir)
+
+    assert result.exit_code == 0
+    assert "LoraConfig" in result.stdout

tests/test_process_md_files.py

@@ -1,4 +1,8 @@
-from data.scraping_scripts.process_md_files import extract_title
+from data.scraping_scripts.process_md_files import (
+    content_sha256,
+    extract_title,
+    stable_doc_id,
+)
 
 
 def test_extract_title_prefers_frontmatter_title() -> None:
@@ -56,3 +60,36 @@ First paragraph fallback
 """
 
     assert extract_title(content) == "First paragraph fallback"
+
+
+def test_stable_doc_id_prefers_source_path() -> None:
+    doc_id = stable_doc_id(
+        source="transformers",
+        source_path="main_classes/model.md",
+        title="Model",
+        url="https://example.com/ignored",
+        content_hash=content_sha256("content"),
+    )
+
+    assert doc_id == "transformers:main-classes-model"
+
+
+def test_stable_doc_id_uses_url_when_source_path_missing() -> None:
+    doc_id = stable_doc_id(
+        source="agentic_ai_engineering",
+        source_path="",
+        title="Lesson 18",
+        url="https://academy.towardsai.net/courses/take/agent-engineering/multimedia/70289117-lesson-18-the-research-loop",
+        content_hash=content_sha256("content"),
+    )
+
+    assert doc_id == (
+        "agentic_ai_engineering:courses-take-agent-engineering-multimedia-"
+        "70289117-lesson-18-the-research-loop"
+    )
+
+
+def test_content_sha256_is_stable_and_prefixed() -> None:
+    assert content_sha256("same") == content_sha256("same")
+    assert content_sha256("same").startswith("sha256:")
+    assert content_sha256("same") != content_sha256("different")

tests/test_prompts.py

@@ -0,0 +1,43 @@
+from __future__ import annotations
+
+from pathlib import Path
+
+from scripts.prompts import build_system_prompt, load_kb_agents_instructions
+
+
+def test_build_system_prompt_includes_local_kb_agents_instructions(
+    tmp_path: Path,
+    monkeypatch,
+) -> None:
+    agents_path = tmp_path / "AGENTS.md"
+    agents_path.write_text(
+        "# Test KB Instructions\n\n"
+        "- Read `wiki/index.md` before broad questions.\n"
+        "- Treat `raw/` as source authority.\n",
+        encoding="utf-8",
+    )
+    monkeypatch.setenv("AI_TUTOR_KB_AGENTS_PATH", str(agents_path))
+
+    prompt = build_system_prompt("google-genai:gemini-3.5-flash", ())
+
+    assert "## Local KB Instructions" in prompt
+    assert "loaded from `data/kb/AGENTS.md`" in prompt
+    assert "# Test KB Instructions" in prompt
+    assert "Treat `raw/` as source authority." in prompt
+
+
+def test_build_system_prompt_omits_local_kb_section_when_file_missing(
+    tmp_path: Path,
+    monkeypatch,
+) -> None:
+    monkeypatch.setenv("AI_TUTOR_KB_AGENTS_PATH", str(tmp_path / "missing.md"))
+
+    prompt = build_system_prompt("google-genai:gemini-3.5-flash", ())
+
+    assert "## Local KB Instructions" not in prompt
+
+
+def test_load_kb_agents_instructions_returns_empty_for_missing_path(
+    tmp_path: Path,
+) -> None:
+    assert load_kb_agents_instructions(tmp_path / "missing.md") == ""

tests/test_update_kb_wiki.py

@@ -0,0 +1,94 @@
+from __future__ import annotations
+
+import json
+from pathlib import Path
+
+from data.scraping_scripts.build_kb_artifacts import build_kb_artifacts
+from data.scraping_scripts.update_kb_wiki import update_kb_wiki
+
+
+def write_jsonl(path: Path, rows: list[dict]) -> None:
+    with path.open("w", encoding="utf-8") as handle:
+        for row in rows:
+            handle.write(json.dumps(row) + "\n")
+
+
+def test_update_kb_wiki_seeds_navigation_pages(tmp_path: Path) -> None:
+    input_file = tmp_path / "all_sources_data.jsonl"
+    kb_dir = tmp_path / "kb"
+    write_jsonl(
+        input_file,
+        [
+            {
+                "doc_id": "peft:lora",
+                "name": "LoRA",
+                "url": "https://example.com/lora",
+                "source": "peft",
+                "source_path": "package_reference/lora.md",
+                "tokens": 300,
+                "retrieve_doc": True,
+                "content": "# LoRA\n\nUse `LoraConfig` for adapter fine-tuning.",
+            },
+            {
+                "doc_id": "agentic_ai_engineering:lesson-1",
+                "name": "Lesson 1",
+                "url": "https://example.com/lesson-1",
+                "source": "agentic_ai_engineering",
+                "tokens": 300,
+                "retrieve_doc": True,
+                "content": "# Lesson 1\n\nAgent lesson.",
+            }
+        ],
+    )
+    build_kb_artifacts(input_file, kb_dir)
+
+    update_kb_wiki(kb_dir, seed_defaults=True, changed_only=False)
+
+    assert (kb_dir / "AGENTS.md").exists()
+    assert (kb_dir / "wiki" / "index.md").exists()
+    assert (kb_dir / "wiki" / "frameworks" / "peft.md").exists()
+    assert (kb_dir / "wiki" / "courses" / "agentic_ai_engineering.md").exists()
+    agents = (kb_dir / "AGENTS.md").read_text(encoding="utf-8")
+    assert "## Exploration Budget" in agents
+    assert 'rg -n -m 20 "query terms"' in agents
+    assert "Stop browsing once you have one clearly relevant source page" in agents
+    index = (kb_dir / "wiki" / "index.md").read_text(encoding="utf-8")
+    assert "`wiki/courses/agentic_ai_engineering.md`" in index
+    assert "`wiki/frameworks/peft.md`" in index
+    topic = (kb_dir / "wiki" / "topics" / "lora.md").read_text(encoding="utf-8")
+    assert "LoRA" in topic
+    assert "`raw/docs/peft/package_reference/lora.md`" in topic
+    assert "lookup_tutor_symbol" not in topic
+
+
+def test_update_kb_wiki_preserves_authored_topic_content_and_appends_log(tmp_path: Path) -> None:
+    input_file = tmp_path / "all_sources_data.jsonl"
+    kb_dir = tmp_path / "kb"
+    write_jsonl(
+        input_file,
+        [
+            {
+                "doc_id": "peft:lora",
+                "name": "LoRA",
+                "url": "https://example.com/lora",
+                "source": "peft",
+                "source_path": "package_reference/lora.md",
+                "tokens": 300,
+                "retrieve_doc": True,
+                "content": "# LoRA\n\nUse `LoraConfig` for adapter fine-tuning.",
+            }
+        ],
+    )
+    build_kb_artifacts(input_file, kb_dir)
+    topic_path = kb_dir / "wiki" / "topics" / "lora.md"
+    topic_path.parent.mkdir(parents=True)
+    topic_path.write_text("# Lora\n\nAuthored synthesis stays.\n", encoding="utf-8")
+
+    update_kb_wiki(kb_dir, seed_defaults=False, changed_only=True)
+    update_kb_wiki(kb_dir, seed_defaults=False, changed_only=True)
+
+    topic = topic_path.read_text(encoding="utf-8")
+    assert "Authored synthesis stays." in topic
+    assert "`raw/docs/peft/package_reference/lora.md`" in topic
+    log = (kb_dir / "wiki" / "log.md").read_text(encoding="utf-8")
+    assert log.count("Generated or refreshed") == 2