Deploy fail-closed helper 436d652

Refresh live offer evidence, validate exact retrieved claims, and hand unsupported questions to the canonical contact form.

.env.example

@@ -1,11 +1,21 @@
 # Public domains where the helper widget is allowed to run.
-HELPER_ALLOWED_ORIGINS=https://academy.towardsai.net,https://towardsai.net,https://www.towardsai.net
-HELPER_ALLOWED_HOSTS=academy.towardsai.net,towardsai.net,www.towardsai.net
+HELPER_ALLOWED_ORIGINS=https://towardsai.com,https://www.towardsai.com,https://academy.towardsai.net,https://towardsai.net,https://www.towardsai.net
+HELPER_ALLOWED_HOSTS=towardsai.com,www.towardsai.com,academy.towardsai.net,towardsai.net,www.towardsai.net
 
-# Gemini.
+# These public hosts allow the widget on every path except private/admin routes.
+HELPER_SITE_WIDE_HOSTS=towardsai.com,www.towardsai.com
+
+# Primary model: DeepSeek V4 Flash through the DeepSeek API.
+DEEPSEEK_API_KEY=
+DEEPSEEK_BASE_URL=https://api.deepseek.com
+HELPER_PRIMARY_MODEL=deepseek-v4-flash
+HELPER_DEEPSEEK_THINKING=disabled
+
+# Backup model: Gemini 2.5 Flash.
 GEMINI_API_KEY=
-HELPER_MODEL=gemini-2.5-flash
+HELPER_FALLBACK_MODEL=gemini-2.5-flash
 HELPER_MAX_OUTPUT_TOKENS=420
+HELPER_LLM_REQUEST_TIMEOUT_SECONDS=20
 
 # Hard public rate limits.
 HELPER_RATE_LIMIT_PER_MINUTE=3
@@ -17,6 +27,9 @@ HELPER_MAX_BODY_BYTES=65536
 HELPER_MAX_QUERY_CHARS=600
 HELPER_MAX_HISTORY_TURNS=8
 
+# Fail closed when the local public-page catalog has not been refreshed recently.
+HELPER_CATALOG_MAX_AGE_DAYS=14
+
 # Optional Opik monitoring.
 OPIK_ENABLED=false
 OPIK_PROJECT_NAME=towards-ai-helper

.github/workflows/ci.yml

@@ -55,6 +55,7 @@ jobs:
 
     env:
       LIVE_SPACE_BASE_URL: ${{ vars.LIVE_SPACE_BASE_URL || 'https://towardsai-tutors-tai-helper.hf.space' }}
+      LIVE_HELPER_WIDGET_PATH: ${{ vars.LIVE_HELPER_WIDGET_PATH || '/helper-widget.js' }}
       RUN_LIVE_CHAT_SMOKE: ${{ github.event.inputs.run_live_chat || 'false' }}
 
     steps:

.github/workflows/keepalive.yml

@@ -21,4 +21,4 @@ jobs:
 
       - name: Widget check
         run: |
-          curl --fail --show-error --silent --max-time 60 "$SPACE_BASE_URL/widget.js" | grep -q "Towards AI Helper"
+          curl --fail --show-error --silent --max-time 60 "$SPACE_BASE_URL/helper-widget.js" | grep -q "Towards AI Helper"

README.md

@@ -7,35 +7,49 @@ pinned: false
 
 # Towards AI Helper
 
-Public sales helper for anonymous visitors on Towards AI Academy and
-TowardsAI.net. It helps prospective students choose courses, bundles,
-mentorship, free resources, the book, or B2B training/consulting.
+Public sales helper for anonymous visitors on `towardsai.com`, the legacy
+`towardsai.net` domain, and Towards AI Academy. It helps prospective students
+choose courses, bundles, mentorship, free resources, the book, or B2B
+training/consulting.
 
 It is deliberately separate from the Thinkific lesson tutor:
 
-- This helper appears only on public, sitemap-discoverable pages.
+- This helper appears on public `towardsai.com` pages and selected public
+  Academy/legacy `.net` pages.
 - It hides when a visitor appears signed in.
 - The first message must be one of the fixed prompt buttons.
-- It uses Gemini 2.5 Flash with strict rate limits and concise answers.
+- It uses DeepSeek V4 Flash through the DeepSeek API first, then falls back to
+  Gemini 2.5 Flash if the primary provider fails.
 - It does not answer general AI questions or give away course lesson content.
+- It returns offer facts only after every sentence has passed exact-quote and
+  retrieved-chunk validation. Missing, stale, conflicting, or invalid evidence
+  produces an explicit “I couldn't verify that” response instead of a guess.
 
 ## Local Setup
 
 ```bash
-cd /Users/louis/Documents/GitHub/tai-helper
+cd /path/to/tai-helper
 uv sync
 cp .env.example .env
 uv run uvicorn tai_helper.api:app --host 0.0.0.0 --port 8001
 ```
 
+Required model secrets:
+
+```bash
+DEEPSEEK_API_KEY=...
+GEMINI_API_KEY=...
+```
+
 ## Public Widget Snippet
 
-Add this to public site footer code on `academy.towardsai.net` and
-`towardsai.net`:
+Add this to the global footer code on `towardsai.com`. Add it separately to
+`academy.towardsai.net` if the helper should also appear on its signed-out
+public pages:
 
 ```html
 <script
-  src="https://YOUR-HF-SPACE/widget.js"
+  src="https://YOUR-HF-SPACE/helper-widget.js"
   data-api-base="https://YOUR-HF-SPACE"
   defer
 ></script>
@@ -44,6 +58,60 @@ Add this to public site footer code on `academy.towardsai.net` and
 The widget fetches `/api/helper/config`, checks the current public URL, hides on
 signed-in sessions, and starts as a bottom-right `Ask the helper` bubble.
 
+The `.net` website redirects to `.com` before page JavaScript runs, so installing
+the snippet only on `.net` will not make the helper appear on `.com`.
+
+## Deployment Domain Settings
+
+The defaults include both `.com` and `.net`. If the deployed Space already has
+these environment variables set, update them because deployed values override
+the defaults:
+
+```bash
+HELPER_ALLOWED_ORIGINS=https://towardsai.com,https://www.towardsai.com,https://academy.towardsai.net,https://towardsai.net,https://www.towardsai.net
+HELPER_ALLOWED_HOSTS=towardsai.com,www.towardsai.com,academy.towardsai.net,towardsai.net,www.towardsai.net
+HELPER_SITE_WIDE_HOSTS=towardsai.com,www.towardsai.com
+```
+
+`HELPER_SITE_WIDE_HOSTS` makes the widget available on current and future public
+`.com` paths. Signed-in sessions, checkout/account paths, WordPress admin paths,
+API paths, and previews remain blocked.
+
+## Knowledge Catalog
+
+`data/towardsai_com_pages.json` is generated from every URL in the current
+`towardsai.com` page sitemap. `data/pages.json` is generated from every public
+Academy sitemap URL. Each eligible page records its canonical URL, successful
+fetch time, content hash, source authority, heading-aware chunks, and atomic
+evidence spans (complete source sentences, DOM blocks, or table rows). A second
+hash binds the canonical URL, page text, chunks, and span definitions together.
+Known staging pages, legacy duplicates, conflicting catalog summaries, failed
+fetches, and manually described links remain in the inventory but are explicitly
+ineligible as evidence.
+
+Refresh both catalogs after public pages change:
+
+```bash
+python scripts/build_towardsai_com_catalog.py
+```
+
+The API accepts evidence only while each page's successful fetch is within
+`HELPER_CATALOG_MAX_AGE_DAYS` (14 days by default). Canonical `.com` offer pages
+supersede lower-authority Academy mirrors for the same offer. Routing notes are
+never treated as factual evidence.
+
+For factual answers, the model must return structured sentence-level claims.
+Every claim needs a valid retrieved chunk ID and must copy one complete
+server-defined evidence span. Arbitrary substrings are forbidden, so a model
+cannot turn “No code required” into “code required” or cross table-row
+boundaries. The server independently verifies catalog freshness, both hashes,
+canonical URLs, chunk/span identity, numbers, prices, percentages, URLs,
+negation, the named offer, the requested fact type, and output schema before
+anything is shown to a visitor. If exact target-qualified evidence is missing or
+generation fails validation, the API returns no sources and directs the visitor
+to the canonical [Towards AI contact form](https://towardsai.com/academy/contact/#contact)
+instead of guessing.
+
 ## Monitoring
 
 Enable Opik:

data/assistant_notes.json

@@ -15,50 +15,75 @@
     "Keep answers concise and useful to control costs.",
     "For B2B, company training, and consulting leads, ask for company/team context and recommend emailing louis@towardsai.net.",
     "For coupon code requests, do not provide a code. If the user insists, tell them to email louis@towardsai.net with context and what they need.",
-    "For eager learners who want broad access, recommend the Get it all bundle as the strongest value option."
+    "For eager learners who want broad access, recommend the Get it all bundle as the strongest value option.",
+    "Never state an exact price, discount, product count, included course, access duration, refund term, or other offer detail unless that fact is explicitly present in the retrieved source text. If it is not present, say you cannot confirm it from the current public pages and link to the relevant page or contact."
+  ],
+  "catalog_hubs": [
+    {
+      "name": "Towards AI Academy",
+      "url": "https://towardsai.com/academy/",
+      "when_to_recommend": "Use this as the central overview of Towards AI learning options. For current inclusions, prices, discounts, and terms, rely only on exact text retrieved from the relevant offer page."
+    }
   ],
   "course_routing": [
     {
-      "name": "Get it all! From Novice to Expert",
-      "url": "https://academy.towardsai.net/bundles/get-it-all",
+      "name": "Get It All: Every Course, One Bundle",
+      "url": "https://towardsai.com/academy/bundles/get-it-all/",
       "when_to_recommend": "Best value for motivated learners who want broad access across the academy or are serious about going from fundamentals to advanced AI engineering."
     },
     {
-      "name": "Python for GenAI / Become a LLM Developer and Python Pro",
-      "url": "https://academy.towardsai.net/courses/python-for-genai",
+      "name": "Beginner Python for AI Engineering",
+      "url": "https://towardsai.com/academy/python-for-ai-engineering/",
       "when_to_recommend": "Best first step for beginners, non-coders, or people who need Python foundations before building LLM products."
     },
     {
-      "name": "Full Stack AI Engineering / Beginner to Advanced LLM Dev",
-      "url": "https://academy.towardsai.net/courses/beginner-to-advanced-llm-dev",
+      "name": "Full Stack AI Engineering",
+      "url": "https://towardsai.com/academy/full-stack-ai-engineering/",
       "when_to_recommend": "Best for builders who want to ship LLM products with RAG, context engineering, deployment, and product-oriented AI engineering."
     },
     {
-      "name": "Agent Engineering: Building Multi-Agent Systems",
-      "url": "https://academy.towardsai.net/courses/agent-engineering",
+      "name": "Agent Engineering",
+      "url": "https://towardsai.com/academy/agent-engineering/",
       "when_to_recommend": "Best for intermediate to advanced engineers who know Python/APIs and want production-grade autonomous agents, evaluation, monitoring, and deployment."
     },
     {
-      "name": "No-Code AI Mastery for Businesses and Professionals",
-      "url": "https://academy.towardsai.net/courses/ai-business-professionals",
+      "name": "Master AI for Work",
+      "url": "https://towardsai.com/academy/ai-for-work/",
       "when_to_recommend": "Best for business users, operators, founders, and professionals who want practical AI workflows without heavy coding."
     },
     {
       "name": "10-Hours LLM Fundamentals",
-      "url": "https://academy.towardsai.net/courses/llm-primer",
+      "url": "https://towardsai.com/academy/llm-primer/",
       "when_to_recommend": "Best quick primer for LLM fundamentals before taking a deeper course."
     },
     {
       "name": "Building LLMs for Production",
-      "url": "https://academy.towardsai.net/courses/buildingllmsforproduction",
+      "url": "https://towardsai.com/academy/building-llms-for-production/",
       "when_to_recommend": "Best for learners who want production LLM engineering concepts connected to the Towards AI book/resources."
     },
     {
       "name": "Towards AI Mentorship",
-      "url": "https://academy.towardsai.net/bundles/tai-mentorship",
+      "url": "https://towardsai.com/academy/mentorship/",
       "when_to_recommend": "Best when the user asks for mentors, personalized guidance, accountability, project feedback, or career/course direction."
     }
   ],
+  "bundle_routing": [
+    {
+      "name": "From Developer to Advanced AI Engineer",
+      "url": "https://towardsai.com/academy/bundles/10-hour-crash-course-into-llm-developer-expert/",
+      "when_to_recommend": "Best for developers who already code and want a path from LLM fundamentals into advanced AI engineering."
+    },
+    {
+      "name": "From Non-Coder to AI Engineer",
+      "url": "https://towardsai.com/academy/bundles/from-coding-novice-to-advanced-llm-developer/",
+      "when_to_recommend": "Best for learners who need Python foundations before progressing through the AI engineering curriculum."
+    },
+    {
+      "name": "Get It All",
+      "url": "https://towardsai.com/academy/bundles/get-it-all/",
+      "when_to_recommend": "Best-value choice for learners who want every course and the broadest path from novice to expert."
+    }
+  ],
   "external_resources": [
     {
       "name": "What's AI YouTube",
@@ -67,12 +92,12 @@
     },
     {
       "name": "Towards AI Resource Library",
-      "url": "https://towardsai.net/towards-ai-resource-library",
+      "url": "https://towardsai.com/towards-ai-resource-library",
       "when_to_recommend": "Free articles, guides, and learning resources."
     },
     {
       "name": "Towards AI book resources",
-      "url": "https://towardsai.net/book",
+      "url": "https://towardsai.com/academy/book/",
       "when_to_recommend": "Book companion resources and production LLM learning path."
     },
     {
@@ -83,10 +108,12 @@
   ],
   "b2b": {
     "urls": [
-      "https://towardsai.net/b2b",
-      "https://towardsai.net/let-us-transform-your-team-into-ai-first-employees-to-stay-ahead-of-competitors"
+      "https://towardsai.com/enterpriseenablement/",
+      "https://towardsai.com/valuecreation/",
+      "https://towardsai.com/enterprise/software-developer-to-ai-engineer/",
+      "https://towardsai.com/enterprise/agentic-developer-conversion/"
     ],
     "email": "louis@towardsai.net",
-    "instruction": "Ask for company size, team background, use cases, desired training outcome, and timeline. Suggest emailing louis@towardsai.net with details or continuing in chat."
+    "instruction": "Ask for company size, team background, use cases, desired outcome, and timeline. Route software developer conversion to the AI Engineer programme; Claude Code, Codex, or coding-agent adoption to Agentic Developer Conversion; broader deployment or custom development to Value Creation; and general team training to Enterprise Enablement. Suggest emailing louis@towardsai.net with details or continuing in chat."
   }
-}
+}

data/discovered_urls.json

@@ -1,6 +1,7 @@
 {
   "academy_sitemap_url": "https://academy.towardsai.net/sitemap.xml",
   "towardsai_page_sitemap_url": "https://towardsai.net/page-sitemap1.xml",
+  "towardsai_com_page_sitemap_url": "https://towardsai.com/pages-sitemap.xml",
   "academy_urls": [
     "https://academy.towardsai.net/",
     "https://academy.towardsai.net/collections",
@@ -39,10 +40,42 @@
     "https://towardsai.net/b2b",
     "https://towardsai.net/let-us-transform-your-team-into-ai-first-employees-to-stay-ahead-of-competitors"
   ],
+  "reviewed_towardsai_com_pages": [
+    "https://towardsai.com/",
+    "https://towardsai.com/academy/",
+    "https://towardsai.com/academy/full-stack-ai-engineering/",
+    "https://towardsai.com/academy/agent-engineering/",
+    "https://towardsai.com/academy/llm-primer/",
+    "https://towardsai.com/academy/python-for-ai-engineering/",
+    "https://towardsai.com/academy/ai-for-work/",
+    "https://towardsai.com/academy/building-llms-for-production/",
+    "https://towardsai.com/academy/bundles/",
+    "https://towardsai.com/academy/mentorship/",
+    "https://towardsai.com/academy/about/",
+    "https://towardsai.com/academy/contact/",
+    "https://towardsai.com/contribute/",
+    "https://towardsai.com/enterprise/software-developer-to-ai-engineer/",
+    "https://towardsai.com/enterprise/agentic-developer-conversion/",
+    "https://towardsai.com/academy/affiliate/",
+    "https://towardsai.com/academy/agent-engineering-free-preview/",
+    "https://towardsai.com/academy/full-stack-ai-engineering-free-preview/",
+    "https://towardsai.com/academy/book/",
+    "https://towardsai.com/academy/bundles/get-it-all/",
+    "https://towardsai.com/academy/bundles/from-coding-novice-to-advanced-llm-developer/",
+    "https://towardsai.com/academy/bundles/10-hour-crash-course-into-llm-developer-expert/",
+    "https://towardsai.com/enterpriseenablement/",
+    "https://towardsai.com/valuecreation/",
+    "https://towardsai.com/valuecreation/careers/",
+    "https://towardsai.com/valuecreation/deployment-strategist/",
+    "https://towardsai.com/valuecreation/junior-ai-engineer/",
+    "https://towardsai.com/valuecreation/senior-ai-engineer/",
+    "https://towardsai.com/theaitastegap/",
+    "https://towardsai.com/webinars/agentengineering/"
+  ],
   "extra_external_urls": [
     "https://www.youtube.com/channel/UCQNjFuhOJM1YqFTPY1Q_kYQ",
     "https://towardsai.net/towards-ai-resource-library",
     "https://towardsai.net/book",
     "https://www.amazon.com/s?k=Towards+AI+Building+LLMs+for+Production"
   ]
-}
+}

scripts/build_towardsai_com_catalog.py

@@ -0,0 +1,1342 @@
+from __future__ import annotations
+
+import argparse
+import hashlib
+import json
+import re
+from collections.abc import Iterable
+from datetime import UTC, datetime
+from html.parser import HTMLParser
+from pathlib import Path
+from typing import Any
+from urllib.parse import urljoin, urlparse, urlunparse
+from xml.etree import ElementTree
+
+import requests
+
+ROOT = Path(__file__).resolve().parents[1]
+DEFAULT_COM_OUTPUT = ROOT / "data" / "towardsai_com_pages.json"
+DEFAULT_ACADEMY_OUTPUT = ROOT / "data" / "pages.json"
+# Kept for callers of the original, .com-only builder.
+DEFAULT_OUTPUT = DEFAULT_COM_OUTPUT
+
+COM_SITEMAP_URL = "https://towardsai.com/pages-sitemap.xml"
+ACADEMY_SITEMAP_URL = "https://academy.towardsai.net/sitemap.xml"
+
+COM_HOSTS = frozenset({"towardsai.com", "www.towardsai.com"})
+ACADEMY_HOSTS = frozenset({"academy.towardsai.net"})
+
+MIN_CHUNK_CHARS = 1_200
+TARGET_CHUNK_CHARS = 1_550
+MAX_CHUNK_CHARS = 1_800
+MAX_EVIDENCE_SPAN_CHARS = 320
+SENTENCE_SPLIT_RE = re.compile(r"(?<=[.!?])\s+(?=[\"'(\[]*[A-Z0-9])")
+SPAN_TOKEN_RE = re.compile(r"[A-Za-z0-9]+")
+
+# Collection pages are still scanned and hashed, but are not factual evidence:
+# their compact cards have contradicted the corresponding canonical detail pages.
+COM_EXCLUSIONS = {
+    "/": "homepage offer summaries can conflict with canonical detail pages",
+    "/academy": "catalog summary can conflict with canonical offer detail pages",
+    "/academy/bundles": (
+        "catalog summary can conflict with canonical bundle detail pages"
+    ),
+}
+
+# These Thinkific pages are published in the sitemap, but are unfinished template,
+# collection, or legacy copies of authoritative product pages. They are fetched so
+# that a refresh records their existence and hash, then withheld from retrieval.
+ACADEMY_EXCLUSIONS = {
+    "/collections": ("catalog summary can lag behind canonical offer detail pages"),
+    "/collections/products": (
+        "catalog summary can lag behind canonical offer detail pages"
+    ),
+    "/collections/developers": (
+        "catalog summary can lag behind canonical offer detail pages"
+    ),
+    "/collections/professionals": (
+        "catalog summary can lag behind canonical offer detail pages"
+    ),
+    "/pages/free-resources": (
+        "legacy resource summary conflicts with the current LLM Fundamentals offer"
+    ),
+    "/pages/agent-course-landing-page": (
+        "legacy duplicate of the Agent Engineering course page"
+    ),
+    "/pages/agent-course-new-page-cro": (
+        "alternate sales copy duplicates the Agent Engineering course page"
+    ),
+    "/pages/towards-ai-insider": "unfinished template content",
+    "/pages/webinar": "unfinished template content",
+    "/pages/new-home-page": "legacy duplicate of the Academy home page",
+    "/pages/landing-page-free-email-course": "unfinished template content",
+    "/pages/choose-your-course": "legacy duplicate of the courses collection",
+}
+
+# Cross-site identifiers let retrieval prefer the canonical .com description of
+# an offer while retaining the Thinkific purchase/enrolment page as provenance.
+OFFER_PATHS = {
+    "/academy/full-stack-ai-engineering": "full-stack-ai-engineering",
+    "/courses/beginner-to-advanced-llm-dev": "full-stack-ai-engineering",
+    "/academy/agent-engineering": "agent-engineering",
+    "/academy/agentic-ai-engineering": "agent-engineering",
+    "/courses/agent-engineering": "agent-engineering",
+    "/pages/agent-course-landing-page": "agent-engineering",
+    "/pages/agent-course-new-page-cro": "agent-engineering",
+    "/academy/llm-primer": "llm-primer",
+    "/courses/llm-primer": "llm-primer",
+    "/academy/python-for-ai-engineering": "python-for-ai-engineering",
+    "/courses/python-for-genai": "python-for-ai-engineering",
+    "/academy/ai-for-work": "ai-for-work",
+    "/courses/ai-business-professionals": "ai-for-work",
+    "/academy/building-llms-for-production": "building-llms-for-production",
+    "/courses/buildingllmsforproduction": "building-llms-for-production",
+    "/academy/mentorship": "mentorship",
+    "/bundles/tai-mentorship": "mentorship",
+    "/academy/bundles/get-it-all": "get-it-all",
+    "/bundles/get-it-all": "get-it-all",
+    "/academy/bundles/from-coding-novice-to-advanced-llm-developer": (
+        "from-coding-novice-to-advanced-llm-developer"
+    ),
+    "/bundles/from-coding-novice-to-advanced-llm-developer": (
+        "from-coding-novice-to-advanced-llm-developer"
+    ),
+    "/academy/bundles/10-hour-crash-course-into-llm-developer-expert": (
+        "10-hour-crash-course-into-llm-developer-expert"
+    ),
+    "/bundles/10-hour-crash-course-into-llm-developer-expert": (
+        "10-hour-crash-course-into-llm-developer-expert"
+    ),
+    "/academy/agent-engineering-free-preview": ("agent-engineering-free-preview"),
+    "/academy/full-stack-ai-engineering-free-preview": (
+        "full-stack-ai-engineering-free-preview"
+    ),
+    "/products/digital_downloads/agents-cheatsheet": "agents-cheatsheet",
+    "/products/digital_downloads/anti-slop-framework": "anti-slop-framework",
+}
+
+# These two resources do not belong to either first-party sitemap. Keeping the
+# small, explicit allowlist avoids turning arbitrary outbound links into evidence.
+MANUAL_SAFE_RESOURCES: tuple[dict[str, Any], ...] = (
+    {
+        "url": ("https://www.amazon.com/s?k=Towards+AI+Building+LLMs+for+Production"),
+        "kind": "book_external",
+        "title": "Towards AI book on Amazon",
+        "text": (
+            "Building LLMs for Production by Towards AI — Amazon search resource."
+        ),
+        "links": [
+            {
+                "text": "Amazon book search",
+                "url": (
+                    "https://www.amazon.com/s?k=Towards+AI+Building+LLMs+for+Production"
+                ),
+            }
+        ],
+    },
+    {
+        "url": "https://www.youtube.com/channel/UCQNjFuhOJM1YqFTPY1Q_kYQ",
+        "kind": "free_content_external",
+        "title": "What's AI YouTube channel",
+        "text": ("What's AI by Louis-François Bouchard — YouTube channel."),
+        "links": [
+            {
+                "text": "What's AI on YouTube",
+                "url": ("https://www.youtube.com/channel/UCQNjFuhOJM1YqFTPY1Q_kYQ"),
+            }
+        ],
+    },
+)
+
+# Human-reviewed routing metadata is preserved across sitemap refreshes, but the
+# summary is metadata only: factual answers must cite scraped page chunks.
+CURATED_COM_METADATA: tuple[dict[str, str], ...] = (
+    {
+        "url": "https://towardsai.com/",
+        "review_title": "Homepage",
+        "kind": "page",
+        "reviewed_summary": "Towards AI deployment, education, and company overview.",
+    },
+    {
+        "url": "https://towardsai.com/academy/",
+        "review_title": "Academy hub",
+        "kind": "collection",
+        "reviewed_summary": "Central comparison page for Towards AI learning paths.",
+    },
+    {
+        "url": "https://towardsai.com/academy/full-stack-ai-engineering/",
+        "review_title": "Full Stack AI Engineering",
+        "kind": "course",
+        "reviewed_summary": "Production LLM product engineering course.",
+    },
+    {
+        "url": "https://towardsai.com/academy/agent-engineering/",
+        "review_title": "Agent Engineering",
+        "kind": "course",
+        "reviewed_summary": "Production-focused agent engineering course.",
+    },
+    {
+        "url": "https://towardsai.com/academy/llm-primer/",
+        "review_title": "10-Hour LLM Fundamentals",
+        "kind": "course",
+        "reviewed_summary": "Focused LLM fundamentals video course.",
+    },
+    {
+        "url": "https://towardsai.com/academy/python-for-ai-engineering/",
+        "review_title": "Python for AI Engineering",
+        "kind": "course",
+        "reviewed_summary": "Beginner Python foundations for AI engineering.",
+    },
+    {
+        "url": "https://towardsai.com/academy/ai-for-work/",
+        "review_title": "AI for Work",
+        "kind": "course",
+        "reviewed_summary": "Practical no-code AI training for professionals.",
+    },
+    {
+        "url": "https://towardsai.com/academy/building-llms-for-production/",
+        "review_title": "Building LLMs for Production",
+        "kind": "book",
+        "reviewed_summary": "Towards AI book and companion learning page.",
+    },
+    {
+        "url": "https://towardsai.com/academy/book/",
+        "review_title": "The Book",
+        "kind": "book",
+        "reviewed_summary": "Towards AI book page.",
+    },
+    {
+        "url": "https://towardsai.com/academy/bundles/",
+        "review_title": "Academy bundles",
+        "kind": "collection",
+        "reviewed_summary": "Central collection of current Academy bundles.",
+    },
+    {
+        "url": "https://towardsai.com/academy/mentorship/",
+        "review_title": "Mentorship",
+        "kind": "mentorship",
+        "reviewed_summary": (
+            "Mentorship includes one course, the 10-Hour LLM Fundamentals video "
+            "course from day one; the other currently listed courses are 25% off."
+        ),
+    },
+    {
+        "url": "https://towardsai.com/academy/about/",
+        "review_title": "About",
+        "kind": "page",
+        "reviewed_summary": "Towards AI Academy and team background.",
+    },
+    {
+        "url": "https://towardsai.com/academy/contact/",
+        "review_title": "Contact",
+        "kind": "page",
+        "reviewed_summary": "Towards AI Academy contact options.",
+    },
+    {
+        "url": "https://towardsai.com/academy/affiliate/",
+        "review_title": "Affiliate",
+        "kind": "page",
+        "reviewed_summary": "Towards AI referral and affiliate program.",
+    },
+    {
+        "url": "https://towardsai.com/academy/full-stack-ai-engineering-free-preview/",
+        "review_title": "Full Stack AI Engineering free preview",
+        "kind": "free_resource",
+        "reviewed_summary": "Free preview of Full Stack AI Engineering.",
+    },
+    {
+        "url": "https://towardsai.com/academy/agent-engineering-free-preview/",
+        "review_title": "Agent Engineering free preview",
+        "kind": "free_resource",
+        "reviewed_summary": "Free preview of Agent Engineering.",
+    },
+    {
+        "url": "https://towardsai.com/academy/bundles/get-it-all/",
+        "review_title": "Get It All",
+        "kind": "bundle",
+        "reviewed_summary": "Broadest Academy course bundle.",
+    },
+    {
+        "url": "https://towardsai.com/academy/bundles/from-coding-novice-to-advanced-llm-developer/",
+        "review_title": "From Non-Coder to AI Engineer",
+        "kind": "bundle",
+        "reviewed_summary": "Bundle path from Python foundations to AI engineering.",
+    },
+    {
+        "url": "https://towardsai.com/academy/bundles/10-hour-crash-course-into-llm-developer-expert/",
+        "review_title": "From Developer to Advanced AI Engineer",
+        "kind": "bundle",
+        "reviewed_summary": "Bundle path from LLM fundamentals to advanced engineering.",
+    },
+    {
+        "url": "https://towardsai.com/enterprise/software-developer-to-ai-engineer/",
+        "review_title": "Software Developer to AI Engineer",
+        "kind": "b2b",
+        "reviewed_summary": "Enterprise AI-engineer conversion program.",
+    },
+    {
+        "url": "https://towardsai.com/enterprise/agentic-developer-conversion/",
+        "review_title": "Agentic Developer Conversion",
+        "kind": "b2b",
+        "reviewed_summary": "Enterprise coding-agent adoption program.",
+    },
+    {
+        "url": "https://towardsai.com/enterpriseenablement/",
+        "review_title": "Enterprise Enablement",
+        "kind": "b2b",
+        "reviewed_summary": "Enterprise AI training and enablement.",
+    },
+    {
+        "url": "https://towardsai.com/valuecreation/",
+        "review_title": "Value Creation",
+        "kind": "b2b",
+        "reviewed_summary": "Custom AI development and value-creation consulting.",
+    },
+    {
+        "url": "https://towardsai.com/webinars/agentengineering/",
+        "review_title": "Agent Engineering webinar",
+        "kind": "free_resource",
+        "reviewed_summary": "Free agent engineering webinar.",
+    },
+)
+
+SUPPRESSED_TAGS = frozenset(
+    {"script", "style", "noscript", "svg", "template", "nav", "footer"}
+)
+VOID_TAGS = frozenset(
+    {
+        "area",
+        "base",
+        "br",
+        "col",
+        "embed",
+        "hr",
+        "img",
+        "input",
+        "link",
+        "meta",
+        "param",
+        "source",
+        "track",
+        "wbr",
+    }
+)
+BLOCK_TAGS = frozenset(
+    {
+        "address",
+        "article",
+        "blockquote",
+        "dd",
+        "div",
+        "dl",
+        "dt",
+        "figcaption",
+        "figure",
+        "li",
+        "main",
+        "p",
+        "section",
+        "td",
+        "th",
+    }
+)
+HEADING_TAGS = frozenset({"h1", "h2", "h3", "h4", "h5", "h6"})
+SUPPRESSED_ROLES = frozenset({"navigation", "contentinfo"})
+SUPPRESSED_CLASS_TOKENS = frozenset(
+    {
+        "cookie-banner",
+        "cookie-consent",
+        "footer",
+        "global-footer",
+        "global-nav",
+        "header",
+        "navbar",
+        "navigation",
+        "site-footer",
+        "site-nav",
+        "ta-announce",
+        # Testimonials, endorsements, simulated conversations, and competitor
+        # comparison tables are useful marketing context but are not first-party
+        # offer terms. Keeping them citable lets review copy or competitor facts
+        # conflict with the actual product specification.
+        "ta-ment-thread",
+        "ta-ment-vsnote",
+        "ta-ment-vswrap",
+        "ta-mobile-menu",
+        "ta-quote",
+        "ta-review",
+    }
+)
+
+PREVIEW_SUPPRESSED_SECTION_IDS: dict[str, frozenset[str]] = {
+    # These sections explicitly describe the paid course, not the free preview.
+    "/academy/agent-engineering-free-preview": frozenset({"fullcourse"}),
+    "/academy/full-stack-ai-engineering-free-preview": frozenset(
+        {"outcomes", "cta"}
+    ),
+}
+PAGE_SUPPRESSED_CLASS_TOKENS: dict[str, frozenset[str]] = {
+    # Keep the first-party feature list in the comparison section, but discard
+    # competitor scorecards and bought-separately summaries.
+    "/academy/mentorship": frozenset({"board", "sum"}),
+}
+
+
+def _clean(text: str) -> str:
+    return re.sub(r"\s+", " ", text).strip()
+
+
+def _utc_now() -> str:
+    return datetime.now(UTC).isoformat()
+
+
+def _local_name(tag: str) -> str:
+    return tag.rsplit("}", 1)[-1].lower()
+
+
+def _normalise_url(url: str) -> str:
+    """Return a fragment-free URL with a normalised scheme and hostname."""
+
+    parsed = urlparse(url.strip())
+    scheme = parsed.scheme.lower()
+    hostname = (parsed.hostname or "").lower()
+    if not scheme or not hostname:
+        return url.strip()
+    port = parsed.port
+    if port and not (
+        (scheme == "http" and port == 80) or (scheme == "https" and port == 443)
+    ):
+        netloc = f"{hostname}:{port}"
+    else:
+        netloc = hostname
+    path = parsed.path or "/"
+    return urlunparse((scheme, netloc, path, "", parsed.query, ""))
+
+
+def _path(url: str) -> str:
+    return urlparse(url).path.rstrip("/") or "/"
+
+
+def _sha256(text: str) -> str:
+    return hashlib.sha256(text.encode("utf-8")).hexdigest()
+
+
+def _evidence_hash(page: dict[str, Any]) -> str:
+    canonical = json.dumps(
+        {key: value for key, value in page.items() if key != "evidence_hash"},
+        ensure_ascii=False,
+        sort_keys=True,
+        separators=(",", ":"),
+    )
+    return _sha256(canonical)
+
+
+def _is_suppressed_container(tag: str, attributes: dict[str, str | None]) -> bool:
+    class_tokens = {
+        token
+        for token in re.split(r"[^a-z0-9_-]+", (attributes.get("class") or "").lower())
+        if token
+    }
+    # Offer pages use a semantic <header class="ta-hero"> for product stats,
+    # pricing, and eligibility facts. Preserve that content header while still
+    # suppressing the site's navigation header.
+    content_header = tag == "header" and any(
+        token == "hero" or token.endswith("-hero") for token in class_tokens
+    )
+    if tag in SUPPRESSED_TAGS and not content_header:
+        return True
+    if (attributes.get("role") or "").lower() in SUPPRESSED_ROLES:
+        return True
+    tokens = re.split(
+        r"[^a-z0-9_-]+",
+        " ".join([attributes.get("id") or "", attributes.get("class") or ""]).lower(),
+    )
+    return any(token in SUPPRESSED_CLASS_TOKENS for token in tokens if token)
+
+
+def _is_page_specific_suppressed_container(
+    base_url: str, tag: str, attributes: dict[str, str | None]
+) -> bool:
+    page_path = _path(base_url)
+    if tag == "section":
+        suppressed_ids = PREVIEW_SUPPRESSED_SECTION_IDS.get(page_path, frozenset())
+        if (attributes.get("id") or "").casefold() in suppressed_ids:
+            return True
+    class_tokens = {
+        token
+        for token in re.split(
+            r"[^a-z0-9_-]+", (attributes.get("class") or "").casefold()
+        )
+        if token
+    }
+    return bool(
+        class_tokens & PAGE_SUPPRESSED_CLASS_TOKENS.get(page_path, frozenset())
+    )
+
+
+def _meta_refresh_target(content: str) -> str:
+    for part in content.split(";"):
+        key, separator, value = part.strip().partition("=")
+        if separator and key.strip().lower() == "url":
+            return value.strip().strip("\"'")
+    return ""
+
+
+class PageParser(HTMLParser):
+    """Extract visible, sectioned page content and canonicalisation hints."""
+
+    def __init__(self, base_url: str) -> None:
+        super().__init__(convert_charrefs=True)
+        self.base_url = base_url
+        self.suppressed_depth = 0
+        self.head_depth = 0
+        self.title_parts: list[str] = []
+        self.text_parts: list[str] = []
+        self.headings: list[str] = []
+        self.links: list[dict[str, str]] = []
+        self.sections: list[dict[str, Any]] = []
+        self.meta_description = ""
+        self.canonical_url = ""
+        self.meta_refresh_url = ""
+        self._in_title = False
+        self._heading_parts: list[str] | None = None
+        self._current_heading = ""
+        self._section_parts: list[str] = []
+        self._section_spans: list[str] = []
+        self._block_parts: list[str] = []
+        self._table_row_parts: list[str] | None = None
+        self._table_head_depth = 0
+        self._link_href = ""
+        self._link_parts: list[str] | None = None
+
+    def handle_starttag(self, tag: str, attrs: list[tuple[str, str | None]]) -> None:
+        tag = tag.lower()
+        values = {key.lower(): value for key, value in attrs}
+
+        if self.suppressed_depth:
+            if tag not in VOID_TAGS:
+                self.suppressed_depth += 1
+            return
+        if _is_suppressed_container(tag, values) or _is_page_specific_suppressed_container(
+            self.base_url, tag, values
+        ):
+            self._flush_block()
+            self.suppressed_depth = 1
+            return
+
+        if tag == "head":
+            self.head_depth += 1
+        if tag == "title":
+            self._in_title = True
+        elif tag == "link":
+            rel = {item.lower() for item in (values.get("rel") or "").split()}
+            if "canonical" in rel and not self.canonical_url:
+                self.canonical_url = (values.get("href") or "").strip()
+        elif tag == "meta":
+            name = (values.get("name") or "").lower()
+            http_equiv = (values.get("http-equiv") or "").lower()
+            content = values.get("content") or ""
+            if name == "description":
+                self.meta_description = _clean(content)
+            elif http_equiv == "refresh" and not self.meta_refresh_url:
+                self.meta_refresh_url = _meta_refresh_target(content)
+
+        if self.head_depth:
+            return
+        if tag == "thead":
+            self._table_head_depth += 1
+        elif tag == "tr":
+            self._flush_block()
+            self._table_row_parts = []
+        elif tag in HEADING_TAGS:
+            self._flush_block()
+            self._flush_section()
+            self._heading_parts = []
+        elif tag == "a":
+            self._link_href = values.get("href") or ""
+            self._link_parts = []
+        elif tag == "br":
+            self._flush_block()
+
+    def handle_startendtag(self, tag: str, attrs: list[tuple[str, str | None]]) -> None:
+        # Avoid changing suppression depth for an explicitly self-closing tag.
+        if self.suppressed_depth:
+            return
+        self.handle_starttag(tag, attrs)
+        if tag.lower() not in VOID_TAGS:
+            self.handle_endtag(tag)
+
+    def handle_endtag(self, tag: str) -> None:
+        tag = tag.lower()
+        if self.suppressed_depth:
+            self.suppressed_depth -= 1
+            return
+        if tag == "head":
+            self.head_depth = max(0, self.head_depth - 1)
+            return
+        if tag == "title":
+            self._in_title = False
+            return
+        if self.head_depth:
+            return
+        if tag in HEADING_TAGS and self._heading_parts is not None:
+            heading = _clean(" ".join(self._heading_parts))
+            if heading:
+                self.headings.append(heading)
+                self._current_heading = heading
+            self._heading_parts = None
+        elif tag == "a" and self._link_parts is not None:
+            href = self._link_href.strip()
+            if href and not href.lower().startswith(
+                ("#", "mailto:", "tel:", "javascript:")
+            ):
+                self.links.append(
+                    {
+                        "text": _clean(" ".join(self._link_parts)),
+                        "url": _normalise_url(urljoin(self.base_url, href)),
+                    }
+                )
+            self._link_href = ""
+            self._link_parts = None
+        if tag in BLOCK_TAGS:
+            self._flush_block()
+        if tag == "tr" and self._table_row_parts is not None:
+            self._flush_block()
+            row = _clean(" ".join(self._table_row_parts))
+            if row and self._table_head_depth == 0:
+                self._section_spans.append(row)
+            self._table_row_parts = None
+        if tag == "thead":
+            self._table_head_depth = max(0, self._table_head_depth - 1)
+
+    def handle_data(self, data: str) -> None:
+        value = _clean(data)
+        if not value or self.suppressed_depth:
+            return
+        if self._in_title:
+            self.title_parts.append(value)
+            return
+        if self.head_depth:
+            return
+        if self._heading_parts is not None:
+            self._heading_parts.append(value)
+            return
+        self.text_parts.append(value)
+        self._block_parts.append(value)
+        if self._table_row_parts is not None:
+            self._table_row_parts.append(value)
+        if self._link_parts is not None:
+            self._link_parts.append(value)
+
+    def close(self) -> None:
+        super().close()
+        self._flush_block()
+        self._flush_section()
+
+    def _flush_block(self) -> None:
+        block = _clean(" ".join(self._block_parts))
+        if block:
+            self._section_parts.append(block)
+            if self._table_row_parts is None:
+                self._section_spans.append(block)
+        self._block_parts = []
+
+    def _flush_section(self) -> None:
+        self._flush_block()
+        body = _clean(" ".join(self._section_parts))
+        if body or self._current_heading:
+            self.sections.append(
+                {
+                    "heading": self._current_heading,
+                    "text": body,
+                    "spans": list(dict.fromkeys(self._section_spans)),
+                }
+            )
+        self._section_parts = []
+        self._section_spans = []
+
+
+def _unique_links(links: Iterable[dict[str, str]]) -> list[dict[str, str]]:
+    result: list[dict[str, str]] = []
+    seen: set[tuple[str, str]] = set()
+    for link in links:
+        key = (link["text"], link["url"])
+        if key not in seen:
+            seen.add(key)
+            result.append(link)
+    return result
+
+
+def _split_to_limit(text: str, limit: int) -> list[str]:
+    """Split text without breaking words, preferring paragraph/sentence edges."""
+
+    text = text.strip()
+    pieces: list[str] = []
+    while len(text) > limit:
+        lower_bound = max(1, int(limit * 0.62))
+        candidates = [
+            text.rfind("\n\n", lower_bound, limit + 1),
+            text.rfind(". ", lower_bound, limit + 1),
+            text.rfind("? ", lower_bound, limit + 1),
+            text.rfind("! ", lower_bound, limit + 1),
+            text.rfind(" ", lower_bound, limit + 1),
+        ]
+        cut = max(candidates)
+        if cut < 1:
+            cut = text.rfind(" ", 0, limit + 1)
+        if cut < 1:
+            cut = limit
+        elif text[cut : cut + 2] in {". ", "? ", "! "}:
+            cut += 1
+        pieces.append(text[:cut].strip())
+        text = text[cut:].strip()
+    if text:
+        pieces.append(text)
+    return pieces
+
+
+def _atomic_span_texts(raw_spans: Iterable[str]) -> list[str]:
+    """Return complete, bounded source spans; never emit arbitrary substrings."""
+
+    result: list[str] = []
+    seen: set[str] = set()
+    for raw_span in raw_spans:
+        block = _clean(str(raw_span))
+        if not block:
+            continue
+        # Accordion rows are flattened by the parser as ``Question? + Answer``.
+        # Only the complete answer is evidence: the question may contain a false
+        # premise, comparison price, or visitor-style wording that must never be
+        # treated as a first-party offer fact.
+        faq_parts = re.split(r"(?<=\?)\s*\+\s*", block, maxsplit=1)
+        if len(faq_parts) > 1:
+            candidates = faq_parts[1:]
+        else:
+            sentences = SENTENCE_SPLIT_RE.split(block)
+            candidates = sentences if len(sentences) > 1 else [block]
+        for candidate in candidates:
+            span = _clean(candidate)
+            key = span.casefold()
+            if (
+                len(SPAN_TOKEN_RE.findall(span)) < 2
+                or len(span) > MAX_EVIDENCE_SPAN_CHARS
+                or any(symbol in span for symbol in ("✓", "✗"))
+                or key in seen
+            ):
+                continue
+            seen.add(key)
+            result.append(span)
+    return result
+
+
+def _span_records(
+    canonical_url: str, chunk_id: str, raw_spans: Iterable[str]
+) -> list[dict[str, str]]:
+    records: list[dict[str, str]] = []
+    for text in _atomic_span_texts(raw_spans):
+        identity = f"{canonical_url}\n{chunk_id}\n{text}"
+        records.append(
+            {
+                "span_id": f"span-{_sha256(identity)[:24]}",
+                "text": text,
+            }
+        )
+    return records
+
+
+def build_chunks(
+    sections: Iterable[dict[str, Any]],
+    canonical_url: str,
+    *,
+    min_chars: int = MIN_CHUNK_CHARS,
+    target_chars: int = TARGET_CHUNK_CHARS,
+    max_chars: int = MAX_CHUNK_CHARS,
+) -> list[dict[str, Any]]:
+    """Build bounded deterministic chunks while retaining section headings."""
+
+    if not 0 < min_chars <= target_chars <= max_chars:
+        raise ValueError("chunk sizes must satisfy 0 < min <= target <= max")
+
+    pieces: list[dict[str, Any]] = []
+    for section_index, section in enumerate(sections):
+        heading = _clean(str(section.get("heading", "")))
+        body = _clean(str(section.get("text", "")))
+        raw_section_spans = [str(item) for item in section.get("spans", [])]
+        section_spans = _atomic_span_texts(raw_section_spans)
+        heading_spans = _atomic_span_texts([heading]) if heading else []
+        if not heading and not body:
+            continue
+        prefix = f"{heading}\n" if heading else ""
+        body_limit = max(1, max_chars - len(prefix))
+        body_pieces = _split_to_limit(body, body_limit) if body else [""]
+        for part_index, body_piece in enumerate(body_pieces):
+            text = f"{prefix}{body_piece}".strip()
+            normalized_piece = _clean(body_piece)
+            piece_spans = [span for span in section_spans if span in normalized_piece]
+            if part_index == 0:
+                piece_spans = [
+                    *[span for span in heading_spans if re.search(r"\d|[$€£¥%]", span)],
+                    *piece_spans,
+                ]
+            if not piece_spans:
+                piece_spans.extend(heading_spans)
+            pieces.append(
+                {
+                    "text": text,
+                    "heading": heading or "Overview",
+                    "key": f"section-{section_index}-part-{part_index}",
+                    "spans": piece_spans,
+                }
+            )
+
+    groups: list[list[dict[str, Any]]] = []
+    current: list[dict[str, Any]] = []
+    current_length = 0
+    for piece in pieces:
+        separator_length = 2 if current else 0
+        candidate_length = current_length + separator_length + len(piece["text"])
+        if current and candidate_length > max_chars:
+            groups.append(current)
+            current = []
+            current_length = 0
+            separator_length = 0
+        current.append(piece)
+        current_length += separator_length + len(piece["text"])
+        if current_length >= target_chars:
+            groups.append(current)
+            current = []
+            current_length = 0
+    if current:
+        groups.append(current)
+
+    # Fold a small final group into its predecessor when the hard maximum allows.
+    if len(groups) >= 2:
+        tail_length = sum(len(item["text"]) for item in groups[-1])
+        tail_length += 2 * (len(groups[-1]) - 1)
+        previous_length = sum(len(item["text"]) for item in groups[-2])
+        previous_length += 2 * (len(groups[-2]) - 1)
+        if tail_length < min_chars and previous_length + 2 + tail_length <= max_chars:
+            groups[-2].extend(groups.pop())
+
+    result: list[dict[str, Any]] = []
+    for index, group in enumerate(groups):
+        # The most substantial section is the best label when a chunk spans a run
+        # of short sections. Every original heading remains embedded in the text.
+        anchor = max(group, key=lambda item: len(item["text"]))
+        identity = f"{canonical_url}\n{anchor['key']}"
+        chunk_id = f"tai-{_sha256(identity)[:24]}"
+        result.append(
+            {
+                "chunk_id": chunk_id,
+                "text": "\n\n".join(item["text"] for item in group),
+                "heading": anchor["heading"],
+                "index": index,
+                "evidence_spans": _span_records(
+                    canonical_url,
+                    chunk_id,
+                    (span for item in group for span in item.get("spans", [])),
+                ),
+            }
+        )
+    return result
+
+
+def _document_text(sections: Iterable[dict[str, str]]) -> str:
+    blocks: list[str] = []
+    for section in sections:
+        heading = _clean(str(section.get("heading", "")))
+        body = _clean(str(section.get("text", "")))
+        block = f"{heading}\n{body}".strip()
+        if block:
+            blocks.append(block)
+    return "\n\n".join(blocks)
+
+
+def _parse_sitemap(response_text: str) -> tuple[str, list[dict[str, str]]]:
+    root = ElementTree.fromstring(response_text)
+    root_type = _local_name(root.tag)
+    entries: list[dict[str, str]] = []
+    for child in root:
+        if _local_name(child.tag) not in {"url", "sitemap"}:
+            continue
+        values = {_local_name(item.tag): _clean(item.text or "") for item in child}
+        if values.get("loc"):
+            entries.append(values)
+    return root_type, entries
+
+
+def discover_sitemap_entries(
+    session: requests.Session,
+    sitemap_url: str,
+    *,
+    allowed_hosts: frozenset[str] | None = None,
+    _visited: set[str] | None = None,
+) -> list[dict[str, str]]:
+    """Read a urlset or sitemap index and return de-duplicated page entries."""
+
+    visited = _visited if _visited is not None else set()
+    normalised_sitemap = _normalise_url(sitemap_url)
+    if normalised_sitemap in visited:
+        return []
+    visited.add(normalised_sitemap)
+
+    response = session.get(sitemap_url, timeout=30)
+    response.raise_for_status()
+    root_type, raw_entries = _parse_sitemap(response.text)
+    if root_type == "sitemapindex":
+        nested: list[dict[str, str]] = []
+        for item in raw_entries:
+            nested.extend(
+                discover_sitemap_entries(
+                    session,
+                    item["loc"],
+                    allowed_hosts=allowed_hosts,
+                    _visited=visited,
+                )
+            )
+        return nested
+    if root_type != "urlset":
+        raise ValueError(f"unsupported sitemap root: {root_type}")
+
+    result: list[dict[str, str]] = []
+    seen: set[str] = set()
+    for item in raw_entries:
+        url = _normalise_url(item["loc"])
+        parsed = urlparse(url)
+        if parsed.scheme not in {"http", "https"} or not parsed.hostname:
+            continue
+        if allowed_hosts and parsed.hostname.lower() not in allowed_hosts:
+            continue
+        if url in seen:
+            continue
+        seen.add(url)
+        result.append(
+            {
+                "url": url,
+                "lastmod": item.get("lastmod", ""),
+                "sitemap_url": normalised_sitemap,
+            }
+        )
+    return result
+
+
+def discover_sitemap_urls(
+    session: requests.Session,
+    sitemap_url: str,
+    *,
+    allowed_hosts: frozenset[str] | None = None,
+) -> list[str]:
+    return [
+        item["url"]
+        for item in discover_sitemap_entries(
+            session, sitemap_url, allowed_hosts=allowed_hosts
+        )
+    ]
+
+
+def merge_curated_com_metadata(
+    entries: Iterable[dict[str, str]],
+    curated_pages: Iterable[dict[str, str]] = CURATED_COM_METADATA,
+) -> list[dict[str, str]]:
+    """Attach non-factual labels only to URLs discovered in the live sitemap."""
+
+    curated_by_url = {_normalise_url(spec["url"]): dict(spec) for spec in curated_pages}
+    result: list[dict[str, str]] = []
+    for raw_entry in entries:
+        entry = dict(raw_entry)
+        url = _normalise_url(entry["url"])
+        entry["url"] = url
+        metadata = curated_by_url.get(url)
+        if metadata:
+            entry.update(
+                {key: value for key, value in metadata.items() if key != "url"}
+            )
+        result.append(entry)
+    return result
+
+
+def _fetch_and_resolve(
+    session: requests.Session, url: str, *, max_meta_refreshes: int = 3
+) -> tuple[Any, PageParser, str, list[str]]:
+    """Follow HTTP redirects, HTML meta refreshes, then honour rel=canonical."""
+
+    current_url = url
+    visited: list[str] = []
+    for _hop in range(max_meta_refreshes + 1):
+        response = session.get(current_url, timeout=30)
+        response.raise_for_status()
+        response_url = _normalise_url(response.url)
+        visited.append(response_url)
+        parser = PageParser(response_url)
+        parser.feed(response.text)
+        parser.close()
+
+        refresh = parser.meta_refresh_url
+        if refresh:
+            refresh_url = _normalise_url(urljoin(response_url, refresh))
+            if refresh_url not in visited:
+                current_url = refresh_url
+                continue
+
+        canonical_url = response_url
+        if parser.canonical_url:
+            candidate = _normalise_url(urljoin(response_url, parser.canonical_url))
+            if urlparse(candidate).scheme in {"http", "https"}:
+                canonical_url = candidate
+        return response, parser, canonical_url, visited
+    raise ValueError(f"too many meta refreshes while fetching {url}")
+
+
+def classify_kind(url: str) -> str:
+    path = _path(url).lower()
+    if path in {"/academy", "/academy/bundles"} or path.startswith("/collections"):
+        return "collection"
+    if "/enterprise/" in path or path in {
+        "/enterpriseenablement",
+        "/valuecreation",
+    }:
+        return "b2b"
+    if path.startswith("/valuecreation/"):
+        return "career"
+    if "/mentorship" in path or path.endswith("/tai-mentorship"):
+        return "mentorship"
+    if "free" in path or "/webinars/" in path:
+        return "free_resource"
+    if "/bundles/" in path:
+        return "bundle"
+    if "/courses/" in path or (
+        path.startswith("/academy/")
+        and any(
+            term in path
+            for term in (
+                "full-stack",
+                "agent-engineering",
+                "llm-primer",
+                "python-for-ai",
+                "ai-for-work",
+                "building-llms",
+            )
+        )
+    ):
+        return "course"
+    if "/digital_downloads/" in path:
+        return "digital_download"
+    if path.endswith("/book"):
+        return "book"
+    if path == "/contribute":
+        return "community"
+    return "page"
+
+
+def offer_id_for_url(url: str) -> str | None:
+    return OFFER_PATHS.get(_path(url).lower())
+
+
+def entity_id_for_url(url: str) -> str:
+    offer_id = offer_id_for_url(url)
+    if offer_id:
+        return f"offer:{offer_id}"
+    parsed = urlparse(url)
+    host = (parsed.hostname or "").lower()
+    return f"page:{host}{_path(url).lower()}"
+
+
+def fetch_page(
+    session: requests.Session,
+    spec: dict[str, Any],
+    *,
+    authority: str | None = None,
+    fetched_at: str | None = None,
+    allowed_hosts: frozenset[str] | None = None,
+) -> dict[str, Any]:
+    """Fetch one sitemap entry without adding any non-page summary to its text."""
+
+    discovered_url = _normalise_url(str(spec["url"]))
+    fetched_at = fetched_at or _utc_now()
+    authority = authority or (
+        "official_academy"
+        if urlparse(discovered_url).hostname == "academy.towardsai.net"
+        else "official_site"
+    )
+    response, parser, canonical_url, redirect_chain = _fetch_and_resolve(
+        session, discovered_url
+    )
+    parsed = urlparse(canonical_url)
+    text = _document_text(parser.sections)
+    content_sha256 = _sha256(text)
+    status = "included"
+    excluded_reason = ""
+    if allowed_hosts and (parsed.hostname or "").lower() not in allowed_hosts:
+        status = "excluded"
+        excluded_reason = "canonical URL is outside the sitemap authority"
+
+    discovered_path = _path(discovered_url)
+    if authority == "official_site" and discovered_path in COM_EXCLUSIONS:
+        status = "excluded"
+        excluded_reason = COM_EXCLUSIONS[discovered_path]
+    if authority == "official_academy" and discovered_path in ACADEMY_EXCLUSIONS:
+        status = "excluded"
+        excluded_reason = ACADEMY_EXCLUSIONS[discovered_path]
+    if not text:
+        status = "excluded"
+        excluded_reason = "no usable visible page content"
+
+    title = _clean(" ".join(parser.title_parts))
+    if not title:
+        title = _clean(str(spec.get("review_title", ""))) or canonical_url
+    offer_id = offer_id_for_url(canonical_url) or offer_id_for_url(discovered_url)
+    chunks = build_chunks(parser.sections, canonical_url)
+    page = {
+        "discovered_url": discovered_url,
+        "url": canonical_url,
+        "canonical_url": canonical_url,
+        "host": (parsed.hostname or "").lower(),
+        "path": parsed.path.rstrip("/") or "/",
+        "kind": str(spec.get("kind") or classify_kind(canonical_url)),
+        "offer_id": offer_id,
+        "entity_id": (
+            f"offer:{offer_id}" if offer_id else entity_id_for_url(canonical_url)
+        ),
+        "title": title,
+        "review_title": str(spec.get("review_title", "")),
+        "reviewed_summary": str(spec.get("reviewed_summary", "")),
+        "meta_description": parser.meta_description,
+        "headings": list(dict.fromkeys(parser.headings))[:100],
+        "text": text,
+        "links": _unique_links(parser.links)[:300],
+        "chunks": chunks,
+        "fetched_at": fetched_at,
+        "content_sha256": content_sha256,
+        # content_hash is the generic name consumed by the retrieval layer.
+        "content_hash": content_sha256,
+        "authority": authority,
+        "status": status,
+        "retrieval_eligible": status == "included",
+        "http_status": int(getattr(response, "status_code", 200)),
+        "lastmod": str(spec.get("lastmod", "")),
+        "sitemap_url": str(spec.get("sitemap_url", "")),
+        "redirect_chain": redirect_chain,
+    }
+    if excluded_reason:
+        page["excluded_reason"] = excluded_reason
+    page["evidence_hash"] = _evidence_hash(page)
+    return page
+
+
+def _failed_page(
+    spec: dict[str, Any], authority: str, fetched_at: str, error: Exception
+) -> dict[str, Any]:
+    url = _normalise_url(str(spec["url"]))
+    parsed = urlparse(url)
+    content_sha256 = _sha256("")
+    chunks: list[dict[str, Any]] = []
+    page = {
+        "discovered_url": url,
+        "url": url,
+        "canonical_url": url,
+        "host": (parsed.hostname or "").lower(),
+        "path": parsed.path.rstrip("/") or "/",
+        "kind": classify_kind(url),
+        "offer_id": offer_id_for_url(url),
+        "entity_id": entity_id_for_url(url),
+        "title": url,
+        "review_title": str(spec.get("review_title", "")),
+        "reviewed_summary": str(spec.get("reviewed_summary", "")),
+        "meta_description": "",
+        "headings": [],
+        "text": "",
+        "links": [],
+        "chunks": chunks,
+        "fetched_at": fetched_at,
+        "content_sha256": content_sha256,
+        "content_hash": content_sha256,
+        "authority": authority,
+        "status": "fetch_error",
+        "retrieval_eligible": False,
+        "http_status": None,
+        "lastmod": str(spec.get("lastmod", "")),
+        "sitemap_url": str(spec.get("sitemap_url", "")),
+        "redirect_chain": [],
+        "error": f"{type(error).__name__}: {error}",
+    }
+    page["evidence_hash"] = _evidence_hash(page)
+    return page
+
+
+def _exclude_canonical_duplicates(pages: list[dict[str, Any]]) -> None:
+    seen: dict[str, str] = {}
+    for page in pages:
+        if page["status"] != "included":
+            continue
+        canonical_url = str(page["canonical_url"])
+        if canonical_url in seen:
+            page["status"] = "excluded"
+            page["retrieval_eligible"] = False
+            page["excluded_reason"] = (
+                f"duplicate canonical URL; authoritative scan is {seen[canonical_url]}"
+            )
+        else:
+            seen[canonical_url] = str(page["discovered_url"])
+
+
+def _manual_resource_page(spec: dict[str, Any], fetched_at: str) -> dict[str, Any]:
+    url = _normalise_url(str(spec["url"]))
+    parsed = urlparse(url)
+    text = _clean(str(spec["text"]))
+    content_sha256 = _sha256(text)
+    chunks: list[dict[str, Any]] = []
+    page = {
+        "discovered_url": url,
+        "url": url,
+        "canonical_url": url,
+        "host": (parsed.hostname or "").lower(),
+        "path": parsed.path.rstrip("/") or "/",
+        "kind": str(spec["kind"]),
+        "offer_id": None,
+        "entity_id": entity_id_for_url(url),
+        "title": str(spec["title"]),
+        "meta_description": "",
+        "headings": [str(spec["title"])],
+        "text": text,
+        "links": list(spec.get("links", [])),
+        "chunks": chunks,
+        "fetched_at": fetched_at,
+        "content_sha256": content_sha256,
+        "content_hash": content_sha256,
+        "authority": "curated_external",
+        "status": "excluded",
+        "retrieval_eligible": False,
+        "excluded_reason": (
+            "manual routing-only resource; descriptive text was not fetched"
+        ),
+        "http_status": None,
+        "lastmod": "",
+        "sitemap_url": "manual_allowlist",
+        "redirect_chain": [],
+    }
+    page["evidence_hash"] = _evidence_hash(page)
+    return page
+
+
+def build_catalog(
+    *,
+    session: requests.Session | None = None,
+    sitemap_url: str = COM_SITEMAP_URL,
+    authority: str = "official_site",
+    allowed_hosts: frozenset[str] = COM_HOSTS,
+    fetched_at: str | None = None,
+    include_manual_resources: bool = False,
+) -> dict[str, Any]:
+    """Build one catalog from its sitemap while retaining failed scan records."""
+
+    fetched_at = fetched_at or _utc_now()
+    own_session = session or requests.Session()
+    own_session.headers.setdefault("User-Agent", "TowardsAIHelperCatalog/2.0")
+    entries = discover_sitemap_entries(
+        own_session, sitemap_url, allowed_hosts=allowed_hosts
+    )
+    if authority == "official_site":
+        entries = merge_curated_com_metadata(entries)
+    pages: list[dict[str, Any]] = []
+    for spec in entries:
+        try:
+            pages.append(
+                fetch_page(
+                    own_session,
+                    spec,
+                    authority=authority,
+                    fetched_at=fetched_at,
+                    allowed_hosts=allowed_hosts,
+                )
+            )
+        except (requests.RequestException, ValueError) as error:
+            pages.append(_failed_page(spec, authority, fetched_at, error))
+    _exclude_canonical_duplicates(pages)
+    if include_manual_resources:
+        pages.extend(
+            _manual_resource_page(spec, fetched_at) for spec in MANUAL_SAFE_RESOURCES
+        )
+    for page in pages:
+        page["evidence_hash"] = _evidence_hash(page)
+
+    statuses: dict[str, int] = {}
+    for page in pages:
+        status = str(page["status"])
+        statuses[status] = statuses.get(status, 0) + 1
+    return {
+        "schema_version": 2,
+        "source": _normalise_url(sitemap_url),
+        "generated_at": fetched_at,
+        "authority": authority,
+        "status_counts": statuses,
+        "pages": pages,
+    }
+
+
+def build_catalogs(
+    *,
+    session: requests.Session | None = None,
+    com_sitemap_url: str = COM_SITEMAP_URL,
+    academy_sitemap_url: str = ACADEMY_SITEMAP_URL,
+    fetched_at: str | None = None,
+) -> tuple[dict[str, Any], dict[str, Any]]:
+    """Build the public-site and Academy catalogs in one reproducible scan."""
+
+    fetched_at = fetched_at or _utc_now()
+    own_session = session or requests.Session()
+    own_session.headers.setdefault("User-Agent", "TowardsAIHelperCatalog/2.0")
+    com_catalog = build_catalog(
+        session=own_session,
+        sitemap_url=com_sitemap_url,
+        authority="official_site",
+        allowed_hosts=COM_HOSTS,
+        fetched_at=fetched_at,
+    )
+    academy_catalog = build_catalog(
+        session=own_session,
+        sitemap_url=academy_sitemap_url,
+        authority="official_academy",
+        allowed_hosts=ACADEMY_HOSTS,
+        fetched_at=fetched_at,
+        include_manual_resources=True,
+    )
+    return com_catalog, academy_catalog
+
+
+def _write_catalog(output: Path, payload: dict[str, Any]) -> None:
+    output.parent.mkdir(parents=True, exist_ok=True)
+    output.write_text(json.dumps(payload, indent=2, ensure_ascii=False) + "\n")
+
+
+def main() -> None:
+    parser = argparse.ArgumentParser(
+        description="Refresh the towardsai.com and Academy helper catalogs"
+    )
+    parser.add_argument(
+        "--com-output",
+        "--output",
+        dest="com_output",
+        type=Path,
+        default=DEFAULT_COM_OUTPUT,
+        help="output for towardsai.com pages (legacy alias: --output)",
+    )
+    parser.add_argument("--academy-output", type=Path, default=DEFAULT_ACADEMY_OUTPUT)
+    parser.add_argument("--com-sitemap", default=COM_SITEMAP_URL)
+    parser.add_argument("--academy-sitemap", default=ACADEMY_SITEMAP_URL)
+    args = parser.parse_args()
+
+    com_catalog, academy_catalog = build_catalogs(
+        com_sitemap_url=args.com_sitemap,
+        academy_sitemap_url=args.academy_sitemap,
+    )
+    _write_catalog(args.com_output, com_catalog)
+    _write_catalog(args.academy_output, academy_catalog)
+    print(f"Wrote {len(com_catalog['pages'])} .com pages to {args.com_output}")
+    print(
+        f"Wrote {len(academy_catalog['pages'])} Academy/manual pages "
+        f"to {args.academy_output}"
+    )
+
+
+if __name__ == "__main__":
+    main()

static/widget.js

@@ -16,6 +16,9 @@
   }
   if (!apiBase) return;
 
+  var contactFormUrl = "https://towardsai.com/academy/contact/#contact";
+  var contactLinkMarkdown = "[contact the Towards AI team](" + contactFormUrl + ")";
+
   var state = {
     config: null,
     visible: false,
@@ -268,15 +271,28 @@
 
   function isBlockedPath() {
     var path = normalizePath(window.location.pathname);
-    return /^(courses\/take|enroll|order|checkout|cart|users|account|admin)/.test(
-      path.replace(/^\//, ""),
-    );
+    var privatePath = [
+      "courses/take", "enroll", "order", "checkout", "cart", "users", "account",
+      "admin", "wp-admin", "wp-login", "wp-json", "wp-content", "wp-includes",
+      "xmlrpc.php",
+    ].some(function (prefix) {
+      return path.replace(/^\//, "").indexOf(prefix) === 0;
+    });
+    return privatePath || new URLSearchParams(window.location.search).has("preview");
   }
 
   function pageAllowed(config) {
     if (!config || isSignedIn() || isBlockedPath()) return false;
     var host = window.location.hostname.toLowerCase();
     var path = normalizePath(window.location.pathname);
+    if (
+      Array.isArray(config.siteWideHosts) &&
+      config.siteWideHosts
+        .map(function (item) { return item.toLowerCase(); })
+        .indexOf(host) !== -1
+    ) {
+      return true;
+    }
     var paths = config.allowedPathsByHost && config.allowedPathsByHost[host];
     if (!paths && host.indexOf("www.") === 0) {
       paths = config.allowedPathsByHost && config.allowedPathsByHost[host.slice(4)];
@@ -370,9 +386,17 @@
       .then(function (response) {
         if (!response.ok) {
           if (response.status === 429) {
-            throw new Error("The helper is rate limited right now. Please try again later.");
+            throw new Error(
+              "The helper is rate limited right now. Please try again later or " +
+                contactLinkMarkdown +
+                "."
+            );
           }
-          throw new Error("The helper is unavailable on this page.");
+          throw new Error(
+            "The helper is unavailable on this page. Please " +
+              contactLinkMarkdown +
+              "."
+          );
         }
         return response.json();
       })
@@ -401,7 +425,10 @@
       })
       .catch(function (error) {
         loading.classList.remove("empty");
-        loading.innerHTML = renderMarkdown(error.message || "Something went wrong.");
+        loading.innerHTML = renderMarkdown(
+          error.message ||
+            "Something went wrong. Please " + contactLinkMarkdown + "."
+        );
       })
       .finally(function () {
         setBusy(false);

tai_helper/api.py

@@ -2,6 +2,7 @@ from __future__ import annotations
 
 import asyncio
 import logging
+import re
 from typing import Any
 from urllib.parse import urlparse
 from uuid import uuid4
@@ -16,6 +17,7 @@ from .catalog import (
     allowed_paths_by_host,
     coupon_followup,
     coupon_intent,
+    evidence_offer_ids_for_query,
     forced_prompts,
     in_scope,
     page_is_allowed,
@@ -29,6 +31,8 @@ from .settings import repo_root, settings
 
 logger = logging.getLogger(__name__)
 
+CONTACT_FORM_URL = "https://towardsai.com/academy/contact/#contact"
+
 app = FastAPI(title="Towards AI Helper API", version="0.1.0")
 app.add_middleware(
     CORSMiddleware,
@@ -140,11 +144,64 @@ def _history_text(payload: HelperChatRequest) -> list[str]:
     return [turn.content for turn in payload.history[-settings.max_history_turns :]]
 
 
+def _user_history_text(payload: HelperChatRequest) -> list[str]:
+    return [
+        turn.content
+        for turn in payload.history[-settings.max_history_turns :]
+        if turn.role == "user"
+    ]
+
+
+def _retrieval_query(payload: HelperChatRequest) -> str:
+    """Use current intent, adding only a real user referent when it is needed."""
+
+    query = payload.query.strip()
+    if not payload.history:
+        return query
+
+    words = set(re.findall(r"[a-z0-9'-]+", query.casefold()))
+    contextual_terms = {
+        "it",
+        "that",
+        "this",
+        "they",
+        "them",
+        "those",
+        "price",
+        "cost",
+        "discount",
+        "included",
+        "access",
+        "refund",
+        "duration",
+        "hours",
+        "lessons",
+        "prerequisites",
+    }
+    needs_context = len(words) <= 12 and bool(words & contextual_terms)
+    if not needs_context:
+        return query
+
+    starters = set(forced_prompts())
+    prior_user_turns = [
+        turn.strip()
+        for turn in _user_history_text(payload)
+        if turn.strip() and turn.strip() not in starters and turn.strip() != query
+    ]
+    if not prior_user_turns:
+        return query
+    return f"{prior_user_turns[-1]}\n{query}"
+
+
 def _validate_payload(payload: HelperChatRequest) -> None:
     if payload.context.signedIn:
-        raise HTTPException(status_code=403, detail="Helper is only for signed-out visitors.")
+        raise HTTPException(
+            status_code=403, detail="Helper is only for signed-out visitors."
+        )
     if not page_is_allowed(payload.context.url):
-        raise HTTPException(status_code=403, detail="Helper is only available on public pages.")
+        raise HTTPException(
+            status_code=403, detail="Helper is only available on public pages."
+        )
     if len(payload.query.strip()) > settings.max_query_chars:
         raise HTTPException(status_code=400, detail="Question is too long.")
     if not payload.history and payload.query.strip() not in forced_prompts():
@@ -159,14 +216,38 @@ def _fixed_coupon_answer(payload: HelperChatRequest) -> str:
     if coupon_followup(payload.query, history):
         return (
             "I can't provide a coupon code here. If you have specific context, "
-            "email louis@towardsai.net with what you need and we will do the best possible."
+            f"[contact the Towards AI team]({CONTACT_FORM_URL}) with what you need."
         )
     return (
-        "I can't provide a coupon code here. The best value option is usually the "
-        "Get it all bundle: https://academy.towardsai.net/bundles/get-it-all"
+        "I can't provide a coupon code here. For pricing help based on your "
+        f"circumstances, [contact the Towards AI team]({CONTACT_FORM_URL})."
+    )
+
+
+def _contact_intent(text: str) -> bool:
+    lowered = text.casefold()
+    return any(
+        phrase in lowered
+        for phrase in (
+            "contact form",
+            "contact the team",
+            "contact a person",
+            "contact someone",
+            "customer support",
+            "talk to a person",
+            "talk to someone",
+            "speak to a person",
+            "speak to someone",
+            "reach the team",
+            "email the team",
+        )
     )
 
 
+def _contact_answer() -> str:
+    return f"Please [contact the Towards AI team]({CONTACT_FORM_URL})."
+
+
 def _out_of_scope_answer() -> str:
     return (
         "I can only help with choosing Towards AI courses, bundles, mentorship, "
@@ -175,10 +256,11 @@ def _out_of_scope_answer() -> str:
     )
 
 
-def _fallback_answer() -> str:
+def _insufficient_evidence_answer() -> str:
     return (
-        "I can help, but I need one detail: are you learning for your own career, "
-        "building AI products, or looking for company training?"
+        "I couldn't verify that from the current Towards AI pages, so I don't want "
+        f"to guess. Please [contact the Towards AI team]({CONTACT_FORM_URL}) "
+        "for confirmation."
     )
 
 
@@ -205,6 +287,7 @@ def healthcheck() -> dict[str, str]:
     return {"status": "ok"}
 
 
+@app.get("/helper-widget.js")
 @app.get("/widget.js")
 def widget() -> FileResponse:
     path = STATIC_DIR / "widget.js"
@@ -219,6 +302,7 @@ def public_config() -> dict[str, Any]:
     return {
         "name": "Towards AI Helper",
         "allowedHosts": list(settings.allowed_hosts),
+        "siteWideHosts": list(settings.site_wide_hosts),
         "allowedPathsByHost": allowed_paths_by_host(),
         "forcedPrompts": forced_prompts(),
         "rateLimits": {
@@ -240,37 +324,84 @@ async def chat(request: Request, payload: HelperChatRequest) -> HelperChatRespon
 
     query = payload.query.strip()
     thread_id = payload.threadId.strip() or uuid4().hex
-    selected_pages = retrieve(query, current_url=payload.context.url)
-    sources = sources_from_pages(selected_pages)
+    selected_pages: list[dict[str, Any]] = []
+    sources: list[dict[str, str]] = []
     usage: dict[str, Any] = {}
     latency_ms = 0
     error_message = ""
+    response_status = "insufficient_evidence"
 
     try:
-        if coupon_intent(query):
+        if _contact_intent(query):
+            answer = _contact_answer()
+            response_status = "policy"
+        elif coupon_intent(query):
             answer = _fixed_coupon_answer(payload)
+            response_status = "policy"
         elif not in_scope(query, _history_text(payload)):
             answer = _out_of_scope_answer()
+            response_status = "out_of_scope"
         else:
-            prompt = llm.build_prompt(
-                query=query,
-                selected_prompt=payload.selectedPrompt or query,
-                current_url=payload.context.url,
-                page_title=payload.context.pageTitle,
-                history=[
-                    (turn.role, turn.content)
-                    for turn in payload.history[-settings.max_history_turns :]
-                ],
-                selected_pages=selected_pages,
+            selected_pages = retrieve(
+                _retrieval_query(payload), current_url=payload.context.url
             )
-            result = await asyncio.to_thread(llm.generate_answer, prompt)
-            answer = result.answer or _fallback_answer()
-            usage = result.usage
-            latency_ms = result.latency_ms
+            if not selected_pages:
+                answer = _insufficient_evidence_answer()
+            else:
+                prompt = llm.build_prompt(
+                    query=query,
+                    selected_prompt=payload.selectedPrompt or query,
+                    current_url=payload.context.url,
+                    page_title=payload.context.pageTitle,
+                    history=[
+                        (turn.role, turn.content)
+                        for turn in payload.history[-settings.max_history_turns :]
+                    ],
+                    selected_pages=selected_pages,
+                )
+                retrieval_query = _retrieval_query(payload)
+                target_offer_ids = evidence_offer_ids_for_query(retrieval_query)
+                grounded = llm.extract_mentorship_course_access(
+                    query,
+                    selected_pages,
+                    target_offer_ids=target_offer_ids,
+                )
+                if grounded is None:
+                    grounded = await asyncio.to_thread(
+                        llm.generate_grounded_answer,
+                        prompt,
+                        selected_pages,
+                        query=query,
+                        target_offer_ids=target_offer_ids,
+                    )
+                usage = grounded.usage
+                latency_ms = grounded.latency_ms
+                if grounded.is_answered:
+                    answer = grounded.answer
+                    response_status = "answered"
+                    sources = sources_from_pages(
+                        [
+                            {
+                                "title": chunk.title,
+                                "url": chunk.url,
+                                "kind": chunk.kind,
+                            }
+                            for chunk in grounded.cited_chunks
+                        ]
+                    )
+                else:
+                    answer = _insufficient_evidence_answer()
+                    if grounded.status == "validation_failure":
+                        error_message = (
+                            "grounding validation failed: "
+                            f"{grounded.validation_error[:400]}"
+                        )
     except Exception:
         error_message = "helper generation failed"
         logger.exception("helper generation failed")
-        answer = _fallback_answer()
+        answer = _insufficient_evidence_answer()
+        sources = []
+        response_status = "insufficient_evidence"
 
     monitor = HelperMonitor(
         query=query,
@@ -290,5 +421,6 @@ async def chat(request: Request, payload: HelperChatRequest) -> HelperChatRespon
         answer=answer,
         threadId=thread_id,
         sources=[SourceOut(**source) for source in sources],
+        status=response_status,
         usage=usage,
     )

tai_helper/catalog.py

@@ -1,19 +1,271 @@
 from __future__ import annotations
 
+import hashlib
 import json
+import math
 import re
+from collections import Counter, defaultdict
+from datetime import UTC, datetime
 from functools import lru_cache
 from typing import Any
-from urllib.parse import urlparse
+from urllib.parse import parse_qs, urlparse
 
-from .settings import repo_root
+from .settings import repo_root, settings
 
-WORD_RE = re.compile(r"[a-z0-9][a-z0-9+#.-]*")
+WORD_RE = re.compile(r"[a-z0-9]+(?:\+\+|#)?")
+SPACE_RE = re.compile(r"\s+")
+CONTENT_HASH_RE = re.compile(r"^[0-9a-f]{64}$")
+STOP_WORDS = {
+    "a",
+    "about",
+    "am",
+    "an",
+    "and",
+    "are",
+    "as",
+    "at",
+    "be",
+    "before",
+    "buying",
+    "by",
+    "can",
+    "committing",
+    "complete",
+    "decide",
+    "deciding",
+    "do",
+    "does",
+    "find",
+    "for",
+    "from",
+    "get",
+    "help",
+    "how",
+    "i",
+    "if",
+    "in",
+    "inside",
+    "into",
+    "is",
+    "it",
+    "looking",
+    "me",
+    "more",
+    "my",
+    "need",
+    "not",
+    "of",
+    "on",
+    "or",
+    "our",
+    "please",
+    "right",
+    "that",
+    "take",
+    "the",
+    "their",
+    "this",
+    "to",
+    "us",
+    "want",
+    "we",
+    "what",
+    "which",
+    "with",
+    "you",
+    "your",
+}
+EVIDENCE_STATUS = "included"
+EVIDENCE_AUTHORITIES = {
+    "academy",
+    "academy_detail",
+    "canonical",
+    "canonical_offer",
+    "official_academy",
+    "official_site",
+    "primary",
+    "product_detail",
+}
+MAX_FUTURE_SKEW_SECONDS = 5 * 60
+MAX_EVIDENCE_SPAN_CHARS = 320
+GENERIC_RETRIEVAL_TERMS = {
+    "access",
+    "academy",
+    "ai",
+    "available",
+    "best",
+    "bot",
+    "bundle",
+    "chat",
+    "choose",
+    "choice",
+    "course",
+    "detail",
+    "fact",
+    "having",
+    "include",
+    "included",
+    "learn",
+    "learning",
+    "many",
+    "mentioned",
+    "offer",
+    "option",
+    "part",
+    "price",
+    "pricing",
+    "provide",
+    "program",
+    "resource",
+    "towards",
+    "training",
+    "true",
+    "zero",
+    "one",
+    "two",
+    "three",
+    "four",
+    "five",
+    "six",
+    "seven",
+    "eight",
+    "nine",
+    "ten",
+}
+
+# Explicit offer names are a hard retrieval boundary, not merely a ranking hint.
+# A truthful sentence from another course is still an unsafe answer for the
+# course the visitor named. Preview aliases intentionally resolve to a distinct
+# offer so paid-course entitlements cannot leak into preview answers.
+OFFER_QUERY_ALIASES: dict[str, tuple[str, ...]] = {
+    "full-stack-ai-engineering": (
+        "full stack ai engineering",
+        "full stack engineering",
+        "full stack",
+    ),
+    "agent-engineering": (
+        "agentic ai engineering",
+        "agent engineering",
+    ),
+    "llm-primer": (
+        "10-hour llm fundamentals",
+        "10 hour llm fundamentals",
+        "llm fundamentals",
+        "llm primer",
+    ),
+    "python-for-ai-engineering": (
+        "beginner python for ai engineering",
+        "python for ai engineering",
+        "beginner python",
+        "python course",
+    ),
+    "ai-for-work": (
+        "master ai for work",
+        "ai for work",
+    ),
+    "building-llms-for-production": (
+        "building llms for production",
+        "building llm for production",
+        "the ebook",
+        "e-book",
+        "ebook",
+    ),
+    "get-it-all": (
+        "get it all bundle",
+        "get-it-all bundle",
+        "get it all",
+    ),
+    "from-coding-novice-to-advanced-llm-developer": (
+        "from non-coder to ai engineer",
+        "from non coder to ai engineer",
+        "non-coder to ai engineer bundle",
+    ),
+    "10-hour-crash-course-into-llm-developer-expert": (
+        "from developer to advanced ai engineer",
+        "developer to advanced ai engineer bundle",
+    ),
+    "mentorship": (
+        "towards ai mentorship",
+        "mentorship",
+        "membership",
+    ),
+}
+
+PREVIEW_OFFER_IDS = {
+    "full-stack-ai-engineering": "full-stack-ai-engineering-free-preview",
+    "agent-engineering": "agent-engineering-free-preview",
+}
+
+# The helper was historically embedded on these exact towardsai.net pages. Keep
+# them available for widget routing, but never add them to the evidence corpus.
+LEGACY_PUBLIC_PATHS_BY_HOST: dict[str, frozenset[str]] = {
+    "towardsai.net": frozenset(
+        {
+            "/",
+            "/academy",
+            "/b2b",
+            "/book",
+            "/community",
+            "/let-us-transform-your-team-into-ai-first-employees-to-stay-ahead-of-competitors",
+            "/towards-ai-resource-library",
+        }
+    ),
+}
+
+
+def _catalog_signature(path: str) -> tuple[str, int, int]:
+    catalog_path = repo_root() / "data" / path
+    try:
+        stat = catalog_path.stat()
+    except OSError:
+        return path, 0, 0
+    return path, stat.st_mtime_ns, stat.st_size
+
+
+def _load_catalog(
+    path: str, _signature: tuple[str, int, int]
+) -> tuple[list[dict[str, Any]], str]:
+    catalog_path = repo_root() / "data" / path
+    if not catalog_path.is_file():
+        return [], ""
+    try:
+        payload = json.loads(catalog_path.read_text())
+    except (OSError, UnicodeError, json.JSONDecodeError):
+        return [], ""
+    if not isinstance(payload, dict) or not isinstance(payload.get("pages"), list):
+        return [], ""
+    generated_at = str(payload.get("generated_at", ""))
+    result = []
+    for raw_page in payload.get("pages", []):
+        page = dict(raw_page)
+        page.setdefault("catalog_generated_at", generated_at)
+        result.append(page)
+    return result, generated_at
+
+
+@lru_cache(maxsize=8)
+def _pages_payload_cached(
+    academy_signature: tuple[str, int, int],
+    website_signature: tuple[str, int, int],
+) -> dict[str, Any]:
+    academy_pages, academy_generated_at = _load_catalog("pages.json", academy_signature)
+    website_pages, website_generated_at = _load_catalog(
+        "towardsai_com_pages.json", website_signature
+    )
+    return {
+        "pages": [*academy_pages, *website_pages],
+        "generated_at": {
+            "academy": academy_generated_at,
+            "website": website_generated_at,
+        },
+    }
 
 
-@lru_cache(maxsize=1)
 def pages_payload() -> dict[str, Any]:
-    return json.loads((repo_root() / "data" / "pages.json").read_text())
+    return _pages_payload_cached(
+        _catalog_signature("pages.json"),
+        _catalog_signature("towardsai_com_pages.json"),
+    )
 
 
 @lru_cache(maxsize=1)
@@ -25,12 +277,735 @@ def forced_prompts() -> list[str]:
     return list(assistant_notes()["forced_prompts"])
 
 
+def _canonical_key(url: str) -> tuple[str, str]:
+    host, path = normalized_path(url)
+    host = host.removeprefix("www.")
+    return host, path
+
+
+def _authority(page: dict[str, Any]) -> float:
+    raw = page.get("authority", "")
+    if isinstance(raw, (int, float)):
+        return float(raw)
+    labels = {
+        "canonical": 5.0,
+        "canonical_offer": 5.0,
+        "official_site": 5.0,
+        "primary": 5.0,
+        "academy_detail": 4.0,
+        "official_academy": 4.0,
+        "product_detail": 4.0,
+        "academy": 3.5,
+        "catalog": 3.0,
+        "external": 2.0,
+        "curated_external": 2.0,
+        "legacy": 1.0,
+    }
+    if str(raw).lower() in labels:
+        return labels[str(raw).lower()]
+    host = str(page.get("host") or urlparse(str(page.get("url", ""))).hostname or "")
+    if host.removeprefix("www.") == "towardsai.com":
+        return 5.0
+    if host == "academy.towardsai.net":
+        return 3.5
+    return 2.0
+
+
+def _parse_timestamp(raw: Any) -> datetime | None:
+    if not raw:
+        return None
+    try:
+        value = str(raw).replace("Z", "+00:00")
+        parsed = datetime.fromisoformat(value)
+        if parsed.tzinfo is None:
+            parsed = parsed.replace(tzinfo=UTC)
+        return parsed.astimezone(UTC)
+    except (TypeError, ValueError):
+        return None
+
+
+def _is_fresh(page: dict[str, Any]) -> bool:
+    # A successful per-page fetch time is required. The catalog generation time
+    # cannot make reused or failed content appear current.
+    fetched_at = _parse_timestamp(page.get("fetched_at"))
+    if fetched_at is None:
+        return False
+    age_seconds = (datetime.now(UTC) - fetched_at).total_seconds()
+    max_age_seconds = max(settings.catalog_max_age_days, 0) * 24 * 60 * 60
+    return -MAX_FUTURE_SKEW_SECONDS <= age_seconds <= max_age_seconds
+
+
+def _is_included(page: dict[str, Any]) -> bool:
+    return str(page.get("status", "")).lower() == EVIDENCE_STATUS and not bool(
+        page.get("excluded")
+    )
+
+
+def _is_evidence_page(page: dict[str, Any]) -> bool:
+    raw_chunks = page.get("chunks")
+    text = str(page.get("text", ""))
+    normalized_page_text = SPACE_RE.sub(" ", text).strip()
+    page_url_text = str(page.get("url", ""))
+    canonical_url = str(page.get("canonical_url") or page.get("url") or "")
+    page_url = urlparse(page_url_text)
+    canonical = urlparse(canonical_url)
+    valid_chunks = (
+        isinstance(raw_chunks, list)
+        and bool(raw_chunks)
+        and all(
+            isinstance(chunk, dict)
+            and bool(str(chunk.get("chunk_id", "")).strip())
+            and bool(str(chunk.get("text", "")).strip())
+            and isinstance(chunk.get("evidence_spans"), list)
+            and bool(chunk["evidence_spans"])
+            and all(
+                isinstance(span, dict)
+                and bool(str(span.get("span_id", "")).strip())
+                and bool(str(span.get("text", "")).strip())
+                and len(SPACE_RE.sub(" ", str(span["text"])).strip())
+                <= MAX_EVIDENCE_SPAN_CHARS
+                and SPACE_RE.sub(" ", str(span["text"])).strip()
+                in SPACE_RE.sub(" ", str(chunk["text"])).strip()
+                and SPACE_RE.sub(" ", str(span["text"])).strip() in normalized_page_text
+                for span in chunk["evidence_spans"]
+            )
+            and len({str(span["span_id"]).strip() for span in chunk["evidence_spans"]})
+            == len(chunk["evidence_spans"])
+            for chunk in raw_chunks
+        )
+        and len({str(chunk["chunk_id"]).strip() for chunk in raw_chunks})
+        == len(raw_chunks)
+    )
+    content_hash = str(page.get("content_hash", ""))
+    evidence_hash = str(page.get("evidence_hash", ""))
+    canonical_evidence = json.dumps(
+        {
+            key: value
+            for key, value in page.items()
+            if key not in {"catalog_generated_at", "evidence_hash"}
+        },
+        ensure_ascii=False,
+        sort_keys=True,
+        separators=(",", ":"),
+    )
+    return (
+        str(page.get("status", "")).lower() == EVIDENCE_STATUS
+        and page.get("retrieval_eligible") is True
+        and str(page.get("authority", "")).lower() in EVIDENCE_AUTHORITIES
+        and page.get("http_status") == 200
+        and page_url.scheme == "https"
+        and page_url.hostname in {"towardsai.com", "academy.towardsai.net"}
+        and canonical.scheme == "https"
+        and canonical.hostname in {"towardsai.com", "academy.towardsai.net"}
+        and _canonical_key(page_url_text) == _canonical_key(canonical_url)
+        and str(page.get("host", "")).lower() == (page_url.hostname or "").lower()
+        and (str(page.get("path", "")).rstrip("/") or "/")
+        == (page_url.path.rstrip("/") or "/")
+        and CONTENT_HASH_RE.fullmatch(content_hash) is not None
+        and hashlib.sha256(text.encode("utf-8")).hexdigest() == content_hash
+        and CONTENT_HASH_RE.fullmatch(evidence_hash) is not None
+        and hashlib.sha256(canonical_evidence.encode("utf-8")).hexdigest()
+        == evidence_hash
+        and valid_chunks
+        and _is_fresh(page)
+    )
+
+
+def all_pages() -> list[dict[str, Any]]:
+    """Return known safe pages, including stale pages used only for widget routing."""
+    return [page for page in pages_payload()["pages"] if _is_included(page)]
+
+
+def _page_preference(page: dict[str, Any]) -> tuple[float, float, int]:
+    fetched_at = _parse_timestamp(page.get("fetched_at"))
+    fetched_value = fetched_at.timestamp() if fetched_at else 0.0
+    return _authority(page), fetched_value, len(str(page.get("text", "")))
+
+
+def _fresh_pages() -> tuple[dict[str, Any], ...]:
+    best_by_url: dict[tuple[str, str], dict[str, Any]] = {}
+    for page in all_pages():
+        if not _is_evidence_page(page):
+            continue
+        key = _canonical_key(str(page.get("url", "")))
+        if not all(key):
+            continue
+        previous = best_by_url.get(key)
+        if previous is None or _page_preference(page) > _page_preference(previous):
+            best_by_url[key] = page
+
+    # A canonical offer page wins over lower-authority mirrors for the same
+    # product. This prevents conflicting catalog/Thinkific figures from both
+    # entering one evidence set.
+    best_by_offer: dict[str, dict[str, Any]] = {}
+    pages_without_offer: list[dict[str, Any]] = []
+    for page in best_by_url.values():
+        offer_id = str(page.get("offer_id") or "").strip()
+        if not offer_id:
+            pages_without_offer.append(page)
+            continue
+        previous = best_by_offer.get(offer_id)
+        if previous is None or _page_preference(page) > _page_preference(previous):
+            best_by_offer[offer_id] = page
+    candidates = (*pages_without_offer, *best_by_offer.values())
+    chunk_id_counts = Counter(
+        str(chunk["chunk_id"]).strip()
+        for page in candidates
+        for chunk in page["chunks"]
+    )
+    return tuple(
+        page
+        for page in candidates
+        if all(
+            chunk_id_counts[str(chunk["chunk_id"]).strip()] == 1
+            for chunk in page["chunks"]
+        )
+    )
+
+
 def pages() -> list[dict[str, Any]]:
-    return list(pages_payload()["pages"])
+    """Return only fresh, reviewed pages that may be used as factual evidence."""
+    return list(_fresh_pages())
+
+
+def _token_list(text: str) -> list[str]:
+    tokens = []
+    for raw_token in WORD_RE.findall(text.lower()):
+        token = raw_token.lower()
+        if len(token) <= 1 or token in STOP_WORDS:
+            continue
+        if token == "classes":
+            token = "class"
+        elif token.endswith("ies") and len(token) > 4:
+            token = f"{token[:-3]}y"
+        elif token.endswith("s") and len(token) > 3 and not token.endswith("ss"):
+            token = token[:-1]
+        tokens.append(token)
+    return tokens
 
 
 def tokenize(text: str) -> set[str]:
-    return {token.lower() for token in WORD_RE.findall(text.lower()) if len(token) > 2}
+    return set(_token_list(text))
+
+
+def offer_ids_for_query(query: str) -> frozenset[str]:
+    """Resolve explicitly named offers into a fail-closed retrieval boundary."""
+
+    lowered = SPACE_RE.sub(" ", query.casefold().replace("–", "-")).strip()
+    if "webinar" in lowered:
+        return frozenset()
+    matches = {
+        offer_id
+        for offer_id, aliases in OFFER_QUERY_ALIASES.items()
+        if any(alias in lowered for alias in aliases)
+    }
+    preview_intent = bool(
+        re.search(r"\bpreview\b|\bfree\s+lessons?\b", lowered)
+    )
+    if preview_intent:
+        for paid_offer_id, preview_offer_id in PREVIEW_OFFER_IDS.items():
+            if paid_offer_id in matches:
+                matches.remove(paid_offer_id)
+                matches.add(preview_offer_id)
+
+    if "mentorship" in matches and len(matches) > 1:
+        compares_offers = bool(
+            re.search(r"\b(?:compare|versus|vs\.?|between)\b", lowered)
+        )
+        course_subject_includes_mentorship = bool(
+            re.search(
+                r"(?:full stack|agent(?:ic)? engineering|llm fundamentals|llm primer|"
+                r"python for ai|ai for work|building llms|get it all).{0,80}"
+                r"\b(?:include|come with|provide|offer|have)\b.{0,50}\bmentor",
+                lowered,
+            )
+        )
+        if course_subject_includes_mentorship:
+            matches.remove("mentorship")
+        elif not compares_offers:
+            # When a course is mentioned inside a mentorship access/discount/
+            # cancellation question, the mentorship page is the authority.
+            matches = {"mentorship"}
+    return frozenset(matches)
+
+
+def evidence_offer_ids_for_query(query: str) -> frozenset[str]:
+    """Return offer IDs allowed to supply evidence for this exact question."""
+
+    targets = set(offer_ids_for_query(query))
+    fields = requested_fact_fields(query)
+    if not fields:
+        return frozenset(targets)
+    evidence_targets: set[str] = set()
+    for field in fields:
+        evidence_targets.update(evidence_offer_ids_for_field(query, field))
+    return frozenset(evidence_targets)
+
+
+def evidence_offer_ids_for_field(query: str, field: str) -> frozenset[str]:
+    """Return the offers allowed to prove one requested fact field.
+
+    Evidence exceptions are deliberately field-specific. A paid course page can
+    state the size of its free preview, but that exception must never expose the
+    paid course's access entitlement to a preview access question in the same
+    visitor message.
+    """
+
+    targets = set(offer_ids_for_query(query))
+    # A paid offer page may explicitly state how many lessons its free preview
+    # contains. This is the sole preview↔parent exception; paid entitlements,
+    # certificates, prices, and refund terms remain outside the preview boundary.
+    if field == "lesson_count":
+        reverse_preview_ids = {
+            preview_id: paid_id for paid_id, preview_id in PREVIEW_OFFER_IDS.items()
+        }
+        targets.update(
+            reverse_preview_ids[offer_id]
+            for offer_id in tuple(targets)
+            if offer_id in reverse_preview_ids
+        )
+    return frozenset(targets)
+
+
+def offer_alias_tokens(offer_ids: frozenset[str] | set[str]) -> frozenset[str]:
+    """Return tokens that name the selected offers, for query relevance checks."""
+
+    result: set[str] = set()
+    reverse_preview_ids = {
+        preview_id: paid_id for paid_id, preview_id in PREVIEW_OFFER_IDS.items()
+    }
+    for offer_id in offer_ids:
+        paid_offer_id = reverse_preview_ids.get(offer_id, offer_id)
+        for alias in OFFER_QUERY_ALIASES.get(paid_offer_id, ()):
+            result.update(_token_list(alias))
+            # Query validation tokenizes without stemming, so retain the exact
+            # alias words as well as their retrieval-normalized forms.
+            result.update(WORD_RE.findall(alias.casefold()))
+        if offer_id in reverse_preview_ids:
+            result.update({"free", "preview", "lesson", "lessons"})
+    return frozenset(result)
+
+
+BUNDLE_OFFER_IDS = frozenset(
+    {
+        "get-it-all",
+        "from-coding-novice-to-advanced-llm-developer",
+        "10-hour-crash-course-into-llm-developer-expert",
+    }
+)
+
+TOTAL_LESSON_PATTERNS: dict[str, re.Pattern[str]] = {
+    "full-stack-ai-engineering": re.compile(r"\b92\s+lessons\b", re.IGNORECASE),
+    "agent-engineering": re.compile(r"\b35\s+lessons\b", re.IGNORECASE),
+    "python-for-ai-engineering": re.compile(r"\b38\s+lessons\b", re.IGNORECASE),
+    "ai-for-work": re.compile(r"\b98\s+lessons\b", re.IGNORECASE),
+    "building-llms-for-production": re.compile(
+        r"\b84\s+lessons\b", re.IGNORECASE
+    ),
+    "agent-engineering-free-preview": re.compile(
+        r"\b7\s+(?:free\s+|full\s+)?lessons\b", re.IGNORECASE
+    ),
+    "full-stack-ai-engineering-free-preview": re.compile(
+        r"\b(?:first\s+)?6\s+(?:free\s+|preview\s+)?lessons\b", re.IGNORECASE
+    ),
+}
+
+DURATION_PATTERNS: dict[str, re.Pattern[str]] = {
+    "full-stack-ai-engineering": re.compile(
+        r"\b60\+\s*hours\b", re.IGNORECASE
+    ),
+    "llm-primer": re.compile(
+        r"\b10\s*hours?\b|\bfive\s+(?:in-depth\s+)?2-hour\s+(?:video\s+)?sessions\b",
+        re.IGNORECASE,
+    ),
+    "ai-for-work": re.compile(r"\baround\s+20\s+hours\b", re.IGNORECASE),
+}
+
+MONTHLY_PLAN_PRICE_RE = re.compile(
+    r"\$\s*99(?:\.00)?\s*(?:/\s*month|monthly)\b",
+    re.IGNORECASE,
+)
+YEARLY_PLAN_PRICE_RE = re.compile(
+    r"\$\s*899(?:\.00)?.{0,20}(?:/\s*year|per year|once a year)\b",
+    re.IGNORECASE,
+)
+PERMANENT_ACCESS_RE = re.compile(
+    r"\b(?:lifetime|forever|keep|retain|retained)\b",
+    re.IGNORECASE,
+)
+
+
+def monthly_plan_intent(query: str) -> bool:
+    return bool(
+        re.search(
+            r"\bmonthly\b|\bmonth[- ](?:to|by)[- ]month\b|"
+            r"\b(?:per|each)\s+month\b|"
+            r"\bmonth\s+(?:price|cost|plan|rate|subscription|membership|mentorship)\b",
+            query,
+        )
+    )
+
+
+def yearly_plan_intent(query: str) -> bool:
+    return bool(
+        re.search(
+            r"\b(?:yearly|annual|annually)\b|\bper\s+year\b|"
+            r"\byear\s+(?:price|cost|plan|rate|subscription|membership|mentorship)\b",
+            query,
+        )
+    )
+
+
+def _supports_monthly_plan_price(text: str) -> bool:
+    for match in MONTHLY_PLAN_PRICE_RE.finditer(text):
+        prefix = text[max(0, match.start() - 16) : match.start()]
+        suffix = text[match.end() : match.end() + 32]
+        if re.search(r"\b(?:from|about)\s*$", prefix):
+            continue
+        if re.search(r"\bbilled\s+(?:yearly|annually)\b", suffix):
+            continue
+        return True
+    return False
+
+
+def _supports_yearly_plan_price(text: str) -> bool:
+    for match in YEARLY_PLAN_PRICE_RE.finditer(text):
+        prefix = text[max(0, match.start() - 16) : match.start()]
+        if re.search(r"\b(?:from|about)\s*$", prefix):
+            continue
+        if "a month" in match.group().casefold():
+            continue
+        return True
+    return False
+
+
+def requested_fact_fields(query: str) -> frozenset[str]:
+    """Classify high-risk facts that require typed, target-bound evidence."""
+
+    lowered = SPACE_RE.sub(" ", query.casefold().replace("–", "-")).strip()
+    fields: set[str] = set()
+    if re.search(r"\b(?:price|cost|how much)\b|[$€£¥]\s*\d", lowered):
+        fields.add("price")
+    if "lesson" in lowered and re.search(
+        r"\b(?:how many|number of|total|overall)\b|\b\d+\s+lessons?\b|\blessons?\s*[:=]?\s*\d+\b",
+        lowered,
+    ):
+        fields.add("lesson_count")
+    if "product" in lowered and re.search(
+        r"\b(?:how many|number of|total)\b", lowered
+    ):
+        fields.add("product_count")
+    if "page" in lowered and re.search(
+        r"\b(?:how many|number of|total|pages?)\b", lowered
+    ):
+        fields.add("page_count")
+    if re.search(
+        r"\bhow long\b|\bhow many hours?\b|\bduration\b|\btime to (?:finish|complete)\b|\b\d+\+?\s*hours?\b|\bhours?\s+(?:long|total)\b|\bhours?.{0,24}\b(?:take|finish|complete)\b",
+        lowered,
+    ):
+        fields.add("duration")
+    if re.search(
+        r"\bprerequisites?\b|\brequirements?\b|\bprior experience\b|\bneed to know\b|\bneed (?:python|coding|code)\b",
+        lowered,
+    ):
+        fields.add("prerequisite")
+    if re.search(r"\bcertificat(?:e|ion|ed)\b", lowered):
+        fields.add("certificate")
+    if re.search(r"\brefund\b|\bmoney[- ]back\b", lowered):
+        fields.add("refund")
+    if re.search(r"\bguarantee(?:d)?\b", lowered):
+        fields.add("guarantee")
+    if re.search(
+        r"\bdiscount\b|\bcoupon\b|\bpromo\b|\bsavings?\b|"
+        r"\b\d+(?:\.\d+)?%\s+(?:off|less)\b|"
+        r"\bsaves?\s+\d+(?:\.\d+)?%",
+        lowered,
+    ):
+        fields.add("discount")
+    if re.search(r"\baccess\b|\blifetime\b|\bkeep\b|\bforever\b|\bretain\b", lowered):
+        fields.add("access")
+    if (
+        "mentor" in lowered
+        and not re.search(r"\bcancel|\bcancell|\bafter\b|\blifetime\b|\bkeep\b", lowered)
+        and re.search(
+            r"\b(?:course|courses|llm fundamentals|full stack|agent engineering|master ai for work|choice|choose)\b",
+            lowered,
+        )
+        and re.search(r"\b(?:included|include|access|choice|choose)\b", lowered)
+    ):
+        fields.add("inclusion")
+    return frozenset(fields)
+
+
+def mixed_preview_fact_request(query: str) -> bool:
+    """Return true when one message mixes multiple preview/paid fact scopes.
+
+    A single offer ID cannot safely assign separate clauses to the free preview
+    and paid course. These compound questions therefore fail closed and ask the
+    visitor to use the contact form (or ask the facts separately).
+    """
+
+    lowered = SPACE_RE.sub(" ", query.casefold()).strip()
+    preview_intent = bool(re.search(r"\bpreview\b|\bfree\s+lessons?\b", lowered))
+    return preview_intent and len(requested_fact_fields(query)) > 1
+
+
+def _span_supports_fact(
+    span_text: str,
+    *,
+    field: str,
+    offer_id: str,
+    query: str,
+) -> bool:
+    text = SPACE_RE.sub(" ", span_text.casefold().replace("–", "-")).strip()
+    lowered_query = query.casefold().replace("–", "-")
+    is_preview = offer_id.endswith("-free-preview")
+
+    if field == "price":
+        if offer_id == "get-it-all":
+            asks_where_price_is_displayed = bool(
+                "checkout" in lowered_query
+                and re.search(
+                    r"\b(?:where|shown|displayed|find|see)\b", lowered_query
+                )
+            )
+            return (
+                asks_where_price_is_displayed
+                and "bundle price shown at checkout" in text
+            )
+        if offer_id in BUNDLE_OFFER_IDS:
+            return "bundle price" in text or (
+                "one-time" in text and bool(_CURRENCY_TOKEN_RE.search(text))
+            )
+        if offer_id == "mentorship":
+            if yearly_plan_intent(lowered_query):
+                return _supports_yearly_plan_price(text)
+            if monthly_plan_intent(lowered_query):
+                return _supports_monthly_plan_price(text)
+            return _supports_monthly_plan_price(
+                text
+            ) or _supports_yearly_plan_price(text)
+        if is_preview:
+            return "free" in text
+        return bool(_CURRENCY_TOKEN_RE.search(text)) and any(
+            term in text
+            for term in ("one-time", "/month", "in total", "lifetime access")
+        )
+
+    if field == "lesson_count":
+        if re.search(r"\bpreview\b|\bfree\s+lessons?\b", lowered_query):
+            preview_count_patterns = {
+                "full-stack-ai-engineering": re.compile(
+                    r"\b(?:first|try|explore)\s+6\s+lessons\b|\bfirst\s+six\s+lessons\b",
+                    re.IGNORECASE,
+                ),
+                "agent-engineering": re.compile(
+                    r"\b(?:first|try|explore)\s+7\s+lessons\b|\bfirst\s+seven\s+lessons\b",
+                    re.IGNORECASE,
+                ),
+            }
+            parent_pattern = preview_count_patterns.get(offer_id)
+            if parent_pattern is not None:
+                # The parent-page exception is count-only. Some CTA DOM blocks
+                # combine "Try 7 Lessons Free" with paid lifetime access and a
+                # guarantee; never admit that mixed block as preview evidence.
+                mixed_paid_entitlement = bool(
+                    re.search(
+                        r"\b(?:lifetime|guarantee|refund|certificat(?:e|ion)|"
+                        r"money[- ]back)\b|[$€£¥]\s*\d",
+                        text,
+                    )
+                )
+                return bool(parent_pattern.search(span_text)) and not (
+                    mixed_paid_entitlement
+                )
+        pattern = TOTAL_LESSON_PATTERNS.get(offer_id)
+        return bool(pattern and pattern.search(span_text))
+
+    if field == "product_count":
+        return bool(re.search(r"\b\d+\s+products\b", text))
+
+    if field == "page_count":
+        return bool(re.search(r"\b\d[\d,]*[- ]page\b", text))
+
+    if field == "duration":
+        pattern = DURATION_PATTERNS.get(offer_id)
+        return bool(pattern and pattern.search(span_text))
+
+    if field == "prerequisite":
+        return any(
+            term in text
+            for term in (
+                "prerequisite",
+                "prior experience",
+                "no prior",
+                "no code",
+                "no coding",
+                "basic python",
+                "intermediate python",
+                "complete beginner",
+                "zero prior",
+                "experience with",
+            )
+        )
+
+    if field == "certificate":
+        if not re.search(r"\bcertificat(?:e|ion|ed)\b", text):
+            return False
+        return not is_preview or any(term in text for term in ("free", "preview"))
+
+    if field == "refund":
+        if offer_id == "building-llms-for-production":
+            return False
+        if offer_id == "mentorship":
+            if monthly_plan_intent(lowered_query):
+                return (
+                    "monthly mentorship can be cancelled" in text
+                    or "yearly plan carries a 30-day money-back guarantee" in text
+                    or "money-back on yearly" in text
+                )
+            if yearly_plan_intent(lowered_query):
+                return "yearly" in text and "money-back" in text
+            return "yearly" in text and "money-back" in text
+        return "refund" in text
+
+    if field == "guarantee":
+        if re.search(r"\b(?:job|role|internship|placement|career)\b", lowered_query):
+            return bool(
+                re.search(r"\b(?:job|role|internship|placement|career|pathway)\b", text)
+                and re.search(r"\b(?:guarantee|promised|earned)\b", text)
+            )
+        return _span_supports_fact(
+            span_text, field="refund", offer_id=offer_id, query=query
+        )
+
+    if field == "discount":
+        if offer_id == "mentorship":
+            alpha_intent = bool(re.search(r"\b(?:new courses?|alpha)\b", lowered_query))
+            plan_intent = bool(
+                monthly_plan_intent(lowered_query)
+                or yearly_plan_intent(lowered_query)
+                or re.search(r"\b(?:billing|plan|save|savings?)\b|\b24%", lowered_query)
+            )
+            course_intent = bool(
+                re.search(
+                    r"\b(?:courses?|full stack|agent(?:ic)? engineering|"
+                    r"master ai for work|ai for work|existing)\b",
+                    lowered_query,
+                )
+            )
+            if sum((alpha_intent, plan_intent, course_intent)) > 1:
+                return False
+            if alpha_intent:
+                return "alpha access" in text and "% off" in text
+            if plan_intent:
+                if monthly_plan_intent(lowered_query) and not yearly_plan_intent(
+                    lowered_query
+                ):
+                    return False
+                return "save 24%" in text or (
+                    "24% less" in text and "month to month" in text
+                )
+            if course_intent:
+                return "25% off" in text
+            return False
+        if offer_id in BUNDLE_OFFER_IDS:
+            if re.search(r"\bstudent\b|\bprevious\b|\bbuyer\b", lowered_query):
+                return False
+            if re.search(r"\bgroup\b|\bteam\b|\btwo or more\b", lowered_query):
+                return "group" in text and "bundle pricing" in text
+            return "bundle price" in text and "was" in text
+        return "discount" in text or (
+            "students get 50% off" in text
+            and any(term in text for term in ("previous", "groups of two"))
+        )
+
+    if field == "access":
+        permanent_access_requested = bool(PERMANENT_ACCESS_RE.search(lowered_query))
+        if offer_id == "mentorship":
+            if re.search(r"\bcancel|\bcancell|\bafter\b", lowered_query):
+                return (
+                    "course access is active while" in text
+                    or "if you cancel" in text
+                )
+            if permanent_access_requested:
+                return bool(PERMANENT_ACCESS_RE.search(text))
+            return any(
+                term in text
+                for term in ("included from day one", "course access", "alpha access")
+            )
+        if is_preview:
+            relation = any(
+                term in text for term in ("access", "lifetime", "keep", "forever")
+            )
+            preview_qualified = any(
+                term in text for term in ("free", "preview", "lessons", "no card")
+            )
+            permanent_qualified = not permanent_access_requested or bool(
+                PERMANENT_ACCESS_RE.search(text)
+            )
+            return relation and preview_qualified and permanent_qualified
+        if permanent_access_requested:
+            return bool(PERMANENT_ACCESS_RE.search(text))
+        return any(term in text for term in ("access", "lifetime", "included"))
+
+    if field == "inclusion":
+        if offer_id == "mentorship":
+            return any(
+                term in text
+                for term in ("included from day one", "25% off", "alpha access")
+            )
+        return "included" in text or "comes with" in text
+
+    return False
+
+
+def evidence_span_supports_field(
+    span_text: str, *, field: str, offer_id: str, query: str
+) -> bool:
+    """Public validator counterpart to the typed retrieval evidence gate."""
+
+    return _span_supports_fact(
+        span_text,
+        field=field,
+        offer_id=offer_id,
+        query=query,
+    )
+
+
+_CURRENCY_TOKEN_RE = re.compile(r"[$€£¥]\s*\d")
+
+
+def _restrict_chunk_evidence(
+    chunk: dict[str, Any], query: str, fields: frozenset[str]
+) -> dict[str, Any] | None:
+    offer_id = str(chunk.get("offer_id", ""))
+    allowed_fields = {
+        field
+        for field in fields
+        if offer_id in evidence_offer_ids_for_field(query, field)
+    }
+    if not allowed_fields:
+        return None
+    spans = [
+        span
+        for span in chunk.get("evidence_spans", [])
+        if isinstance(span, dict)
+        and any(
+            _span_supports_fact(
+                str(span.get("text", "")),
+                field=field,
+                offer_id=offer_id,
+                query=query,
+            )
+            for field in allowed_fields
+        )
+    ]
+    if not spans:
+        return None
+    restricted = dict(chunk)
+    restricted["evidence_spans"] = spans
+    return restricted
 
 
 def normalized_path(url: str) -> tuple[str, str]:
@@ -42,100 +1017,483 @@ def normalized_path(url: str) -> tuple[str, str]:
 
 def allowed_paths_by_host() -> dict[str, list[str]]:
     result: dict[str, set[str]] = {}
-    for page in pages():
-        host = str(page.get("host", "")).lower()
-        path = str(page.get("path", "/")).rstrip("/") or "/"
+    for page in all_pages():
+        host = str(
+            page.get("host") or urlparse(str(page.get("url", ""))).hostname or ""
+        ).lower()
+        path = str(page.get("path") or urlparse(str(page.get("url", ""))).path or "/")
+        path = path.rstrip("/") or "/"
         if host:
             result.setdefault(host, set()).add(path)
-            if host == "towardsai.net":
-                result.setdefault("www.towardsai.net", set()).add(path)
+            if host in {"towardsai.com", "towardsai.net"}:
+                result.setdefault(f"www.{host}", set()).add(path)
+    for host, paths in LEGACY_PUBLIC_PATHS_BY_HOST.items():
+        result.setdefault(host, set()).update(paths)
+        result.setdefault(f"www.{host}", set()).update(paths)
     return {host: sorted(paths) for host, paths in result.items()}
 
 
 def page_is_allowed(url: str) -> bool:
     host, path = normalized_path(url)
+    parsed = urlparse(url)
     if not host:
         return False
     if path.startswith(("/courses/take", "/enroll", "/order", "/checkout", "/cart")):
         return False
-    if path.startswith(("/users", "/account", "/admin")):
+    if path.startswith(
+        (
+            "/users",
+            "/account",
+            "/admin",
+            "/wp-admin",
+            "/wp-login",
+            "/wp-json",
+            "/wp-content",
+            "/wp-includes",
+            "/xmlrpc.php",
+        )
+    ):
         return False
+    if "preview" in parse_qs(parsed.query):
+        return False
+    if host in {item.lower() for item in settings.site_wide_hosts}:
+        return True
     allowed = allowed_paths_by_host()
     return path in allowed.get(host, [])
 
 
 def source_for_url(url: str) -> dict[str, Any] | None:
-    host, path = normalized_path(url)
+    key = _canonical_key(url)
     for page in pages():
-        if str(page.get("host", "")).lower() == host and (
-            str(page.get("path", "/")).rstrip("/") or "/"
-        ) == path:
+        if _canonical_key(str(page.get("url", ""))) == key:
             return page
     return None
 
 
-def retrieve(query: str, *, current_url: str = "", limit: int = 7) -> list[dict[str, Any]]:
-    query_tokens = tokenize(query)
-    scored: list[tuple[float, dict[str, Any]]] = []
-    current = source_for_url(current_url)
-    notes_text = json.dumps(assistant_notes(), ensure_ascii=False)
+def chunks() -> tuple[dict[str, Any], ...]:
+    result: list[dict[str, Any]] = []
+    for page in pages():
+        raw_chunks = page["chunks"]
+        for raw_chunk in raw_chunks:
+            text = SPACE_RE.sub(" ", str(raw_chunk.get("text", ""))).strip()
+            url = str(page.get("url", ""))
+            heading = str(raw_chunk.get("heading", "")).strip()
+            chunk_id = str(raw_chunk.get("chunk_id", "")).strip()
+            result.append(
+                {
+                    "chunk_id": chunk_id,
+                    "title": str(page.get("title", "")),
+                    "url": url,
+                    "host": str(page.get("host", "")),
+                    "path": str(page.get("path", "")),
+                    "kind": str(page.get("kind", "page")),
+                    "offer_id": str(page.get("offer_id", "")),
+                    "entity_id": str(page.get("entity_id", "")),
+                    "heading": heading,
+                    "headings": [heading] if heading else [],
+                    "text": text,
+                    "chunk_index": raw_chunk.get("index"),
+                    "evidence_spans": list(raw_chunk.get("evidence_spans", [])),
+                    "authority": page.get("authority", _authority(page)),
+                    "fetched_at": page.get("fetched_at")
+                    or page.get("catalog_generated_at", ""),
+                    "status": page.get("status", "active"),
+                }
+            )
+    return tuple(result)
+
+
+def _expanded_query_tokens(query: str) -> list[str]:
+    tokens = _token_list(query)
+    expansions = {
+        "mentor": ("mentorship",),
+        "mentorship": ("mentor",),
+        "cost": ("price", "pricing"),
+        "price": ("cost", "pricing"),
+        "classes": ("course", "courses"),
+        "class": ("course",),
+        "company": ("enterprise", "team"),
+        "business": ("enterprise", "company"),
+        "certificate": ("certification",),
+    }
+    for token in tuple(tokens):
+        tokens.extend(expansions.get(token, ()))
+    return tokens
+
+
+def _routing_boost(query: str, chunk: dict[str, Any]) -> float:
+    lowered = query.lower()
+    url = str(chunk.get("url", "")).lower()
+    chunk_path = normalized_path(url)[1]
+    chunk_index = chunk.get("chunk_index")
+    kind = str(chunk.get("kind", ""))
+    evidence_text = str(chunk.get("text", ""))
+    evidence_lower = evidence_text.lower()
+    score = 0.0
+
+    course_starter = "help deciding which course to take" in lowered
+    canonical_learning_paths = {
+        "/academy/full-stack-ai-engineering",
+        "/academy/agent-engineering",
+        "/academy/llm-primer",
+        "/academy/python-for-ai-engineering",
+        "/academy/ai-for-work",
+        "/academy/building-llms-for-production",
+    }
+    if course_starter and chunk_path in canonical_learning_paths:
+        score += 36.0 if chunk_index == 0 else 6.0
+
+    free_starter = "free resources to learn before committing" in lowered
+    if free_starter:
+        if kind in {"free_resource", "digital_download"}:
+            score += 42.0 + (8.0 if chunk_index == 0 else 0.0)
+        elif chunk_path == "/academy/book":
+            score += 26.0
+
+    if "integrate ai into my company" in lowered:
+        if chunk_path == "/valuecreation":
+            score += 52.0 + (12.0 if chunk_index == 0 else 0.0)
+        elif chunk_path == "/enterpriseenablement":
+            score += 44.0 + (12.0 if chunk_index == 0 else 0.0)
+        elif chunk_path.startswith("/enterprise/"):
+            score += 28.0 + (8.0 if chunk_index == 0 else 0.0)
+
+    if "training inside my company" in lowered:
+        if chunk_path == "/enterpriseenablement":
+            score += 44.0
+        elif chunk_path.startswith("/enterprise/"):
+            score += 28.0
+
+    if chunk_path == "/academy/bundles/get-it-all" and any(
+        phrase in lowered for phrase in ("best value", "get it all", "every course")
+    ):
+        score += 12.0
+    if chunk_path == "/academy/python-for-ai-engineering" and any(
+        phrase in lowered
+        for phrase in (
+            "beginner",
+            "non-coder",
+            "non coder",
+            "don't code",
+            "do not code",
+        )
+    ):
+        score += 10.0
+    if (
+        chunk_path == "/enterpriseenablement"
+        and "training" in lowered
+        and any(phrase in lowered for phrase in ("company", "business", "team"))
+    ):
+        score += 10.0
+    if "help deciding which course" in lowered and chunk_path == "/academy":
+        score += 5.0
+    if (
+        chunk_path == "/academy/mentorship"
+        and any(term in lowered for term in ("mentor", "mentorship"))
+        and any(term in lowered for term in ("course", "included", "include", "access"))
+        and "llm fundamentals" in evidence_lower
+        and "included from day one" in evidence_lower
+    ):
+        score += 30.0
+
+    wants_free_preview = "free" in lowered and any(
+        term in lowered for term in ("preview", "lesson")
+    )
+    named_offer_routes = (
+        ("full stack", "/academy/full-stack-ai-engineering", 16.0),
+        ("agent engineering", "/academy/agent-engineering", 16.0),
+        ("llm fundamentals", "/academy/llm-primer", 16.0),
+        ("llm primer", "/academy/llm-primer", 16.0),
+        ("python", "/academy/python-for-ai-engineering", 16.0),
+        ("master ai for work", "/academy/ai-for-work", 16.0),
+        ("ai for work", "/academy/ai-for-work", 16.0),
+        (
+            "from non-coder to ai engineer",
+            "/academy/bundles/from-coding-novice-to-advanced-llm-developer",
+            18.0,
+        ),
+        (
+            "from developer to advanced ai engineer",
+            "/academy/bundles/10-hour-crash-course-into-llm-developer-expert",
+            18.0,
+        ),
+    )
+    for phrase, target_path, boost in named_offer_routes:
+        if target_path in {
+            "/academy/full-stack-ai-engineering",
+            "/academy/agent-engineering",
+        } and (wants_free_preview or "webinar" in lowered):
+            continue
+        if phrase in lowered and chunk_path == target_path:
+            score += boost
+    if (
+        "full stack" in lowered
+        and wants_free_preview
+        and chunk_path == "/academy/full-stack-ai-engineering-free-preview"
+    ):
+        score += 24.0
+    if (
+        "agent engineering" in lowered
+        and wants_free_preview
+        and chunk_path == "/academy/agent-engineering-free-preview"
+    ):
+        score += 24.0
+    if "building llms" in lowered:
+        wants_resources = any(term in lowered for term in ("resource", "companion"))
+        target_path = (
+            "/academy/book"
+            if wants_resources
+            else "/academy/building-llms-for-production"
+        )
+        if chunk_path == target_path:
+            score += 18.0
+    if "webinar" in lowered and chunk_path == "/webinars/agentengineering":
+        score += 40.0
+    if (
+        any(term in lowered for term in ("price", "cost", "how much"))
+        and "$" in evidence_text
+    ):
+        score += 10.0
+    if (
+        "lesson" in lowered
+        and any(phrase in lowered for phrase in ("how many", "number of"))
+        and re.search(r"\b\d[\d,+]*\s+lessons\b", evidence_lower)
+    ):
+        score += 16.0
+    if (
+        "page" in lowered
+        and any(phrase in lowered for phrase in ("how many", "number of"))
+        and re.search(r"\b\d[\d,]*[- ]page\b", evidence_lower)
+    ):
+        score += 16.0
+    if (
+        "product" in lowered
+        and any(phrase in lowered for phrase in ("how many", "number of"))
+        and re.search(r"\b\d+\s+products\b", evidence_lower)
+    ):
+        score += 16.0
+
+    rules = (
+        (("mentor", "mentorship"), "/academy/mentorship/", 7.0),
+        (
+            ("bundle", "best value", "every course", "get it all"),
+            "/academy/bundles/",
+            4.0,
+        ),
+        (
+            ("codex", "claude", "coding agent"),
+            "/enterprise/agentic-developer-conversion/",
+            24.0,
+        ),
+        (
+            ("software developer", "developers into ai engineers"),
+            "/enterprise/software-developer-to-ai-engineer/",
+            24.0,
+        ),
+        (
+            ("consulting", "deployment", "value creation", "private equity"),
+            "/valuecreation/",
+            20.0,
+        ),
+        (("enablement", "enterprise academy"), "/enterpriseenablement/", 20.0),
+    )
+    for phrases, path, boost in rules:
+        target_path = path.rstrip("/") or "/"
+        path_matches = (
+            chunk_path.startswith(target_path + "/")
+            if target_path == "/academy/bundles"
+            else chunk_path == target_path
+        )
+        if any(phrase in lowered for phrase in phrases) and path_matches:
+            score += boost
+    return score
+
+
+def retrieve(
+    query: str, *, current_url: str = "", limit: int = 7
+) -> list[dict[str, Any]]:
+    """Retrieve fresh evidence chunks. An empty result means the helper must abstain."""
+    query_terms = _expanded_query_tokens(query)
+    if not query_terms or limit <= 0:
+        return []
+    corpus = list(chunks())
+    if not corpus:
+        return []
+
+    normalized_query = SPACE_RE.sub(" ", query.strip().lower()).rstrip(".")
+    course_paths = {
+        "/academy/full-stack-ai-engineering",
+        "/academy/agent-engineering",
+        "/academy/llm-primer",
+        "/academy/python-for-ai-engineering",
+        "/academy/ai-for-work",
+        "/academy/building-llms-for-production",
+    }
+    if normalized_query == "i want help deciding which course to take":
+        corpus = [chunk for chunk in corpus if chunk.get("path") in course_paths]
+    elif normalized_query == "i want help to integrate ai into my company":
+        corpus = [
+            chunk
+            for chunk in corpus
+            if chunk.get("path") in {"/valuecreation", "/enterpriseenablement"}
+            or str(chunk.get("path", "")).startswith("/enterprise/")
+        ]
+    elif normalized_query == "i want a training inside my company":
+        corpus = [
+            chunk
+            for chunk in corpus
+            if chunk.get("path") == "/enterpriseenablement"
+            or str(chunk.get("path", "")).startswith("/enterprise/")
+        ]
+    elif normalized_query == (
+        "i'm looking for more free resources to learn before committing to buying "
+        "a course"
+    ):
+        corpus = [
+            chunk
+            for chunk in corpus
+            if chunk.get("kind") in {"free_resource", "digital_download"}
+            or chunk.get("path") == "/academy/book"
+        ]
+    elif normalized_query == "i want to find mentors":
+        corpus = [
+            chunk for chunk in corpus if chunk.get("path") == "/academy/mentorship"
+        ]
+
+    target_offer_ids = offer_ids_for_query(query)
+    evidence_offer_ids = evidence_offer_ids_for_query(query)
+    if evidence_offer_ids:
+        corpus = [
+            chunk
+            for chunk in corpus
+            if str(chunk.get("offer_id", "")) in evidence_offer_ids
+        ]
+    fact_fields = requested_fact_fields(query)
+    if fact_fields:
+        # High-risk facts without an explicit offer are ambiguous by definition.
+        # Ask the visitor to contact the team instead of mixing offer policies.
+        if not target_offer_ids:
+            return []
+        if mixed_preview_fact_request(query):
+            return []
+        restricted_corpus: list[dict[str, Any]] = []
+        for chunk in corpus:
+            restricted = _restrict_chunk_evidence(chunk, query, fact_fields)
+            if restricted is not None:
+                restricted_corpus.append(restricted)
+        corpus = restricted_corpus
+    if not corpus:
+        return []
 
-    synthetic_notes = {
-        "title": "Towards AI Helper Routing Notes",
-        "url": "internal://assistant-notes",
-        "kind": "routing_notes",
-        "text": notes_text,
-        "headings": [],
-        "links": [],
+    document_terms = [_token_list(str(chunk.get("text", ""))) for chunk in corpus]
+    document_frequency: Counter[str] = Counter()
+    for terms in document_terms:
+        document_frequency.update(set(terms))
+    average_length = sum(map(len, document_terms)) / max(len(document_terms), 1)
+    query_counts = Counter(query_terms)
+    query_term_set = set(query_counts)
+    informative_query_terms = {
+        term
+        for term in query_term_set - GENERIC_RETRIEVAL_TERMS
+        if not term.replace(".", "", 1).isdigit()
     }
-    all_pages = [synthetic_notes, *pages()]
+    current_key = _canonical_key(current_url)
+    scored: list[tuple[float, int, dict[str, Any]]] = []
 
-    for page in all_pages:
-        text = " ".join(
+    for index, (chunk, terms) in enumerate(zip(corpus, document_terms, strict=True)):
+        counts = Counter(terms)
+        matched = set(query_counts) & set(counts)
+        searchable = " ".join(
             [
-                str(page.get("title", "")),
-                str(page.get("kind", "")),
-                " ".join(str(item) for item in page.get("headings", [])[:30]),
-                str(page.get("meta_description", "")),
-                str(page.get("text", ""))[:8000],
+                str(chunk.get("title", "")),
+                str(chunk.get("heading", "")),
+                str(chunk.get("path", "")),
             ]
         )
-        tokens = tokenize(text)
-        overlap = len(query_tokens & tokens)
-        score = float(overlap)
-        kind = page.get("kind", "")
-        lowered = query.lower()
-        if page is current:
-            score += 7.0
-        if kind == "routing_notes":
-            score += 5.0
-        if "company" in lowered or "b2b" in lowered or "training" in lowered:
-            if kind == "b2b":
-                score += 8.0
-        if "mentor" in lowered or "mentorship" in lowered:
-            if "mentorship" in str(page.get("url", "")).lower():
-                score += 10.0
-        if "free" in lowered or "youtube" in lowered or "resource" in lowered:
-            if kind in {"free_resource", "free_content_external", "community"}:
-                score += 7.0
-        if "book" in lowered or "amazon" in lowered:
-            if kind in {"book", "book_external"}:
-                score += 8.0
-        if "bundle" in lowered or "all" in lowered or "value" in lowered:
-            if "get-it-all" in str(page.get("url", "")):
-                score += 8.0
-        if score > 0:
-            scored.append((score, page))
-
-    scored.sort(key=lambda item: item[0], reverse=True)
-    return [page for _score, page in scored[:limit]]
-
-
-def sources_from_pages(selected: list[dict[str, Any]], limit: int = 4) -> list[dict[str, str]]:
+        metadata_tokens = tokenize(searchable)
+        metadata_matches = set(query_counts) & metadata_tokens
+        all_matches = matched | metadata_matches
+        route_boost = _routing_boost(query, chunk)
+        if fact_fields and target_offer_ids:
+            # Typed evidence has already passed the strict offer/field/qualifier
+            # gate. It should not be discarded merely because the visitor used
+            # a synonym such as "cancelling" while the page says "cancel".
+            route_boost += 20.0
+        if not all_matches and route_boost < 10.0:
+            continue
+        informative_matches = all_matches & informative_query_terms
+        trusted_route = route_boost >= 10.0
+        if informative_query_terms and not informative_matches and not trusted_route:
+            continue
+        informative_coverage = len(informative_matches) / max(
+            len(informative_query_terms), 1
+        )
+        if (
+            informative_query_terms
+            and informative_coverage < 0.67
+            and not trusted_route
+        ):
+            continue
+
+        score = 0.0
+        length = max(len(terms), 1)
+        for term, query_frequency in query_counts.items():
+            frequency = counts.get(term, 0)
+            if not frequency:
+                continue
+            frequency_docs = document_frequency.get(term, 0)
+            inverse_frequency = math.log(
+                1 + (len(corpus) - frequency_docs + 0.5) / (frequency_docs + 0.5)
+            )
+            denominator = frequency + 1.2 * (
+                0.25 + 0.75 * length / max(average_length, 1)
+            )
+            score += (
+                inverse_frequency
+                * (frequency * 2.2 / denominator)
+                * min(query_frequency, 2)
+            )
+        score += 1.4 * len(metadata_matches)
+        score += 2.0 * len(all_matches)
+        # In a tiny corpus BM25's inverse-frequency value is small even for an
+        # exact match. Reward multiple distinct, non-generic matches while a
+        # lone generic word such as "course" still cannot retrieve a page.
+        if len(informative_matches) >= 2:
+            score += 0.75 * len(informative_matches)
+        score += route_boost
+        if current_key == _canonical_key(str(chunk.get("url", ""))):
+            score += 1.0
+        score *= 0.85 + min(_authority(chunk), 5.0) * 0.05
+        if score >= 1.0:
+            scored.append((score, index, chunk))
+
+    scored.sort(key=lambda item: (-item[0], item[1]))
+    result: list[dict[str, Any]] = []
+    per_url: defaultdict[str, int] = defaultdict(int)
+    diversify = normalized_query in {
+        "i want help deciding which course to take",
+        "i want help to integrate ai into my company",
+        "i'm looking for more free resources to learn before committing to buying a course",
+    }
+    per_url_limit = 1 if diversify else 2
+    for _score, _index, chunk in scored:
+        url = str(chunk.get("url", ""))
+        if per_url[url] >= per_url_limit:
+            continue
+        result.append(chunk)
+        per_url[url] += 1
+        if len(result) >= limit:
+            break
+    return result
+
+
+def sources_from_pages(
+    selected: list[dict[str, Any]], limit: int = 4
+) -> list[dict[str, str]]:
     result = []
     for page in selected:
         url = str(page.get("url", ""))
-        if url.startswith("internal://"):
+        if not url or url.startswith("internal://"):
             continue
         result.append(
             {
@@ -154,7 +1512,7 @@ def sources_from_pages(selected: list[dict[str, Any]], limit: int = 4) -> list[d
 
 
 def in_scope(text: str, history: list[str] | None = None) -> bool:
-    haystack = " ".join([text, *(history or [])]).lower()
+    lowered = text.lower()
     allowed_terms = {
         "course",
         "courses",
@@ -166,8 +1524,13 @@ def in_scope(text: str, history: list[str] | None = None) -> bool:
         "training",
         "company",
         "business",
+        "beginner",
+        "code",
+        "coder",
+        "coding",
         "team",
         "consulting",
+        "developer",
         "resource",
         "resources",
         "youtube",
@@ -181,14 +1544,60 @@ def in_scope(text: str, history: list[str] | None = None) -> bool:
         "agents",
         "genai",
         "certificate",
+        "certification",
+        "preview",
+        "lesson",
+        "lessons",
+        "free",
+        "full stack",
+        "ebook",
+        "e-book",
+        "lifetime",
+        "access",
+        "feature",
+        "features",
+        "duration",
+        "hour",
+        "hours",
+        "guarantee",
+        "support",
         "refund",
         "price",
+        "pricing",
+        "cost",
         "coupon",
         "discount",
+        "engineer",
+        "experience",
+        "founder",
         "promo",
+        "professional",
         "career",
     }
-    return any(term in haystack for term in allowed_terms)
+    if any(term in lowered for term in allowed_terms):
+        return True
+
+    # Conversation history is useful only for genuinely referential follow-ups.
+    # It must never make a new, unrelated question appear in scope.
+    followup_terms = {
+        "it",
+        "that",
+        "this",
+        "they",
+        "them",
+        "those",
+        "one",
+        "ones",
+        "included",
+        "access",
+        "prerequisites",
+    }
+    raw_words = {token.lower() for token in WORD_RE.findall(lowered)}
+    is_followup = len(raw_words) <= 10 and bool(raw_words & followup_terms)
+    if not is_followup:
+        return False
+    prior = " ".join(history or []).lower()
+    return any(term in prior for term in allowed_terms)
 
 
 def coupon_intent(text: str) -> bool:
@@ -202,5 +1611,12 @@ def coupon_followup(text: str, history: list[str]) -> bool:
         lowered.count(term) for term in ("coupon", "promo code", "discount code")
     )
     return prior_coupon_mentions >= 2 or any(
-        term in text.lower() for term in ("please", "really", "need", "student", "can't")
+        term in text.lower()
+        for term in ("please", "really", "need", "student", "can't")
     )
+
+
+def clear_catalog_caches() -> None:
+    """Clear catalog caches after an on-disk refresh or in tests."""
+    _pages_payload_cached.cache_clear()
+    assistant_notes.cache_clear()

tai_helper/llm.py

@@ -1,61 +1,530 @@
 from __future__ import annotations
 
+import hashlib
+import json
+import logging
+import re
 import time
-from dataclasses import dataclass, field
-from typing import Any
+import unicodedata
+from dataclasses import dataclass, field, replace
+from typing import Any, Literal
 
+import requests
 from google import genai
 
-from .catalog import assistant_notes
+from .catalog import (
+    evidence_offer_ids_for_field,
+    evidence_span_supports_field,
+    mixed_preview_fact_request,
+    monthly_plan_intent,
+    offer_alias_tokens,
+    offer_ids_for_query,
+    requested_fact_fields,
+)
 from .settings import settings
 
+DEEPSEEK_PROVIDER = "deepseek"
+GEMINI_PROVIDER = "google_genai"
+DEFAULT_TEMPERATURE = 0.0
+MAX_CONTEXT_CHARS = 18000
+MAX_CHUNK_TEXT_CHARS = 4500
+MAX_EVIDENCE_QUOTE_CHARS = 320
+logger = logging.getLogger(__name__)
+
+
+GROUNDING_RESPONSE_SCHEMA: dict[str, Any] = {
+    "type": "object",
+    "additionalProperties": False,
+    "required": ["status", "claims"],
+    "properties": {
+        "status": {"type": "string", "enum": ["answered", "not_found"]},
+        "claims": {
+            "type": "array",
+            "maxItems": 6,
+            "items": {
+                "type": "object",
+                "additionalProperties": False,
+                "required": ["text", "chunk_id", "quote"],
+                "properties": {
+                    "text": {"type": "string", "maxLength": 321},
+                    "chunk_id": {"type": "string"},
+                    "quote": {"type": "string", "maxLength": 320},
+                },
+            },
+        },
+    },
+}
+
 
 SYSTEM_INSTRUCTION = """You are Towards AI Helper, a concise public assistant for anonymous prospective students.
 
 Your only job is to help users choose Towards AI courses, bundles, mentorship,
 free resources, the book, community, or B2B training/consulting.
 
-Rules:
-- Stay on this scope. If asked general AI, coding, homework, news, or unrelated questions, briefly say you can only help with choosing Towards AI learning/training options.
-- Do not teach course lesson content, reveal course material, or provide detailed technical lessons. Give high-level fit guidance and link to relevant public pages.
-- Keep answers short: usually 2-5 sentences plus links when useful.
-- For course-choice prompts, ask one practical follow-up if you need background, coding level, goals, or company context.
-- For B2B/company training/consulting, ask for use case, team background, timeline, and suggest emailing louis@towardsai.net.
-- For mentors, recommend the mentorship program and ask what kind of guidance they need.
-- For eager learners wanting the best value, recommend the Get it all bundle.
-- For coupon code requests, do not provide a code.
+Scope and style rules:
+- For unrelated requests, general AI teaching, coding help, homework, or news,
+  return not_found rather than answering from general knowledge.
+- Do not reveal course lesson material or provide detailed technical lessons.
+- Keep supported answers concise and focused on the visitor's question.
+
+Grounding rules (higher priority than helpfulness):
+- The EVIDENCE CHUNKS in the user prompt are the only authority for factual claims.
+- Routing notes, page metadata outside the evidence section, conversation history,
+  and the visitor's message are not evidence and must never support an answer.
+- Never rely on memory or general knowledge. If the supplied evidence does not
+  directly support the answer, return not_found.
+- Never infer an inclusion, quantity, price, discount, entitlement, policy, date,
+  URL, or program feature that is not explicitly stated in an evidence chunk.
+- Do not guess the number or names of included products or courses.
+- A visitor may put an unsupported premise or alternative in a question. Ignore
+  that premise as evidence. If an evidence chunk directly provides relevant
+  correcting facts, return answered with those supported facts; do not return
+  not_found merely because the visitor's proposed premise is unsupported.
+- For either/or questions, report each directly supported table row or fact as a
+  separate extractive claim. Never infer a total, use "only", or negate an
+  alternative unless the evidence quote explicitly states that total,
+  exclusivity, or negation.
+- Never repeat, restate, or deny an unsupported premise in a claim. In
+  particular, do not say "not included" unless those literal words occur in the
+  claim's exact evidence quote.
+- Each claim must copy its supporting quote verbatim. You may add one final
+  period only when the source quote has no final period. Do not paraphrase,
+  reorder, omit, or add words. Split separate source sentences or table rows
+  into separate claims.
+- Each claim must cite one valid chunk_id and copy exactly one complete
+  ALLOWED_EVIDENCE_SPAN shown for that chunk. Never quote an arbitrary substring,
+  omit a qualifier or negation, edit, combine, or use ellipses in a quote.
+
+Output rules:
+- Return exactly one JSON object and nothing else (no Markdown fences).
+- For a supported answer use:
+  {"status":"answered","claims":[{"text":"One supported sentence.","chunk_id":"chunk_id_here","quote":"exact contiguous source quote"}]}
+- If support is absent, ambiguous, conflicting, or insufficient, use exactly:
+  {"status":"not_found","claims":[]}
+- Do not include an answer field, preamble, uncited sentence, or extra key.
+- Keep supported answers to at most 6 short claims. Include URLs only when the
+  exact URL occurs in the quoted evidence.
 """
 
 
 @dataclass(frozen=True)
 class LLMResult:
+    """Raw provider result.
+
+    ``answer`` contains the provider's JSON text until it has passed
+    :func:`validate_grounded_result`. Callers must not send this field directly
+    to a visitor.
+    """
+
     answer: str
     usage: dict[str, Any] = field(default_factory=dict)
     latency_ms: int = 0
 
 
-def _context_block(selected_pages: list[dict[str, Any]], max_chars: int = 18000) -> str:
-    chunks = []
-    notes = assistant_notes()
-    chunks.append("ROUTING NOTES:\n" + str(notes))
-    for index, page in enumerate(selected_pages, start=1):
+@dataclass(frozen=True)
+class EvidenceSpan:
+    span_id: str
+    text: str
+
+
+@dataclass(frozen=True)
+class EvidenceChunk:
+    chunk_id: str
+    title: str
+    url: str
+    kind: str
+    headings: tuple[str, ...]
+    text: str
+    offer_id: str = ""
+    entity_id: str = ""
+    evidence_spans: tuple[EvidenceSpan, ...] = ()
+
+    @property
+    def evidence_text(self) -> str:
+        spans = "\n".join(
+            f'<SPAN span_id="{span.span_id}">{span.text}</SPAN>'
+            for span in self.evidence_spans
+        )
+        return "\n".join(
+            [
+                f"Title: {self.title}",
+                f"Kind: {self.kind}",
+                f"Offer ID: {self.offer_id or '(none)'}",
+                f"Entity ID: {self.entity_id or '(none)'}",
+                f"URL: {self.url}",
+                f"Headings: {', '.join(self.headings)}",
+                "Allowed evidence spans (quote exactly one complete span):",
+                spans or "(none)",
+            ]
+        )
+
+
+@dataclass(frozen=True)
+class GroundedClaim:
+    text: str
+    chunk_id: str
+    quote: str
+
+
+GroundingStatus = Literal["answered", "not_found", "validation_failure"]
+
+
+@dataclass(frozen=True)
+class GroundingResult:
+    """Safe result returned by the deterministic grounding boundary."""
+
+    valid: bool
+    status: GroundingStatus
+    answer: str = ""
+    claims: tuple[GroundedClaim, ...] = ()
+    cited_chunks: tuple[EvidenceChunk, ...] = ()
+    usage: dict[str, Any] = field(default_factory=dict)
+    latency_ms: int = 0
+    validation_error: str = ""
+
+    @property
+    def is_answered(self) -> bool:
+        return self.valid and self.status == "answered"
+
+    @property
+    def cited_chunk_ids(self) -> tuple[str, ...]:
+        return tuple(chunk.chunk_id for chunk in self.cited_chunks)
+
+
+_SAFE_CHUNK_ID_RE = re.compile(r"^[A-Za-z0-9][A-Za-z0-9_.:-]{0,127}$")
+_WHITESPACE_RE = re.compile(r"\s+")
+_TOKEN_RE = re.compile(r"[a-z0-9]+(?:[-'][a-z0-9]+)*", re.IGNORECASE)
+_SENTENCE_BOUNDARY_RE = re.compile(r"[.!?][\"')\]]*\s+(?=[A-Z0-9])")
+_ATOMIC_SENTENCE_SPLIT_RE = re.compile(r"(?<=[.!?])\s+(?=[\"'(\[]*[A-Z0-9])")
+_URL_RE = re.compile(r"https?://[^\s<>\"']+", re.IGNORECASE)
+_PERCENT_RE = re.compile(r"(?<![\w.])\d+(?:[.,]\d+)*\s*%")
+_CURRENCY_RE = re.compile(
+    r"(?<!\w)(?:(?:US|CA|AU)?[$€£¥]\s*\d+(?:[.,]\d+)*"
+    r"|\d+(?:[.,]\d+)*\s*(?:USD|CAD|AUD|EUR|GBP|JPY))(?!\w)",
+    re.IGNORECASE,
+)
+_NUMBER_RE = re.compile(r"(?<![\w.])\d+(?:[.,]\d+)*(?![\w.])")
+_POLARITY_RE = re.compile(
+    r"\b(?:no|not|never|none|without|cannot|can't|couldn't|doesn't|don't|"
+    r"isn't|aren't|won't)\b",
+    re.IGNORECASE,
+)
+_UNSAFE_COMPARISON_EVIDENCE_RE = re.compile(
+    r"\b(?:a single mentor|senior ai consultant|chatgpt\s*&\s*claude|"
+    r"discord\s*&\s*reddit)\b",
+    re.IGNORECASE,
+)
+_NUMBER_WORDS = frozenset(
+    [
+        "zero",
+        "one",
+        "two",
+        "three",
+        "four",
+        "five",
+        "six",
+        "seven",
+        "eight",
+        "nine",
+        "ten",
+        "eleven",
+        "twelve",
+        "thirteen",
+        "fourteen",
+        "fifteen",
+        "sixteen",
+        "seventeen",
+        "eighteen",
+        "nineteen",
+        "twenty",
+        "thirty",
+        "forty",
+        "fifty",
+        "sixty",
+        "seventy",
+        "eighty",
+        "ninety",
+        "hundred",
+        "thousand",
+        "million",
+        "billion",
+    ]
+)
+
+_QUERY_RELEVANCE_STOP = frozenset(
+    {
+        "a",
+        "about",
+        "all",
+        "am",
+        "an",
+        "and",
+        "any",
+        "are",
+        "as",
+        "at",
+        "be",
+        "can",
+        "come",
+        "comes",
+        "course",
+        "courses",
+        "do",
+        "does",
+        "every",
+        "each",
+        "feature",
+        "features",
+        "for",
+        "from",
+        "get",
+        "give",
+        "has",
+        "have",
+        "having",
+        "how",
+        "i",
+        "if",
+        "in",
+        "include",
+        "included",
+        "includes",
+        "is",
+        "it",
+        "me",
+        "many",
+        "my",
+        "of",
+        "on",
+        "or",
+        "our",
+        "part",
+        "per",
+        "plan",
+        "please",
+        "program",
+        "provide",
+        "really",
+        "that",
+        "the",
+        "their",
+        "there",
+        "this",
+        "to",
+        "towards",
+        "true",
+        "us",
+        "we",
+        "what",
+        "which",
+        "with",
+        "long",
+        "you",
+        "your",
+    }
+)
+
+_FACT_QUERY_TERMS = frozenset(
+    {
+        "access",
+        "annual",
+        "annually",
+        "cancel",
+        "cancelled",
+        "cancelling",
+        "certificate",
+        "certification",
+        "cost",
+        "coupon",
+        "discount",
+        "duration",
+        "forever",
+        "guarantee",
+        "guaranteed",
+        "hour",
+        "hours",
+        "keep",
+        "lesson",
+        "lessons",
+        "lifetime",
+        "money-back",
+        "month",
+        "month-by-month",
+        "month-to-month",
+        "monthly",
+        "page",
+        "pages",
+        "price",
+        "product",
+        "products",
+        "promo",
+        "prerequisite",
+        "prerequisites",
+        "refund",
+        "retain",
+        "year",
+        "yearly",
+    }
+)
+
+
+def _normalize_text(value: str) -> str:
+    normalized = unicodedata.normalize("NFKC", value).replace("’", "'")
+    return _WHITESPACE_RE.sub(" ", normalized).strip()
+
+
+def chunk_id_for_page(page: dict[str, Any]) -> str:
+    """Return a deterministic, retrieval-order-independent evidence ID."""
+
+    explicit = str(page.get("chunk_id", "")).strip()
+    if explicit and _SAFE_CHUNK_ID_RE.fullmatch(explicit):
+        return explicit
+
+    identity = json.dumps(
+        {
+            "url": str(page.get("url", "")),
+            "title": str(page.get("title", "")),
+            "kind": str(page.get("kind", "")),
+            "chunk_index": page.get("chunk_index"),
+            "text": _normalize_text(str(page.get("text", ""))),
+        },
+        ensure_ascii=False,
+        sort_keys=True,
+        separators=(",", ":"),
+    )
+    digest = hashlib.sha256(identity.encode("utf-8")).hexdigest()[:16]
+    return f"chunk_{digest}"
+
+
+def _fallback_span_texts(text: str) -> list[str]:
+    """Derive conservative atomic spans for tests and legacy in-memory inputs."""
+
+    normalized = _normalize_text(text)
+    if not normalized:
+        return []
+    candidates = _ATOMIC_SENTENCE_SPLIT_RE.split(normalized)
+    result: list[str] = []
+    for candidate in candidates:
+        span = _normalize_text(candidate)
+        if 2 <= len(_TOKEN_RE.findall(span)) and len(span) <= MAX_EVIDENCE_QUOTE_CHARS:
+            result.append(span)
+    return list(dict.fromkeys(result))
+
+
+def _evidence_spans_from_page(
+    page: dict[str, Any], chunk_id: str, text: str
+) -> tuple[EvidenceSpan, ...]:
+    normalized_text = _normalize_text(text)
+    raw_spans = page.get("evidence_spans")
+    candidates: list[tuple[str, str]] = []
+    if isinstance(raw_spans, list):
+        for index, raw_span in enumerate(raw_spans):
+            if not isinstance(raw_span, dict):
+                continue
+            span_text = _normalize_text(str(raw_span.get("text", "")))
+            span_id = str(raw_span.get("span_id", "")).strip()
+            if not span_id:
+                span_id = f"{chunk_id}:span-{index}"
+            candidates.append((span_id, span_text))
+    else:
+        candidates = [
+            (f"{chunk_id}:span-{index}", span_text)
+            for index, span_text in enumerate(_fallback_span_texts(text))
+        ]
+
+    result: list[EvidenceSpan] = []
+    seen: set[tuple[str, str]] = set()
+    for span_id, span_text in candidates:
+        key = (span_id, span_text)
+        if (
+            key in seen
+            or not _SAFE_CHUNK_ID_RE.fullmatch(span_id)
+            or not span_text
+            or len(span_text) > MAX_EVIDENCE_QUOTE_CHARS
+            or span_text not in normalized_text
+        ):
+            continue
+        seen.add(key)
+        result.append(EvidenceSpan(span_id=span_id, text=span_text))
+    return tuple(result)
+
+
+def _chunk_from_page(page: dict[str, Any]) -> EvidenceChunk:
+    chunk_id = chunk_id_for_page(page)
+    text = str(page.get("text", ""))[:MAX_CHUNK_TEXT_CHARS]
+    return EvidenceChunk(
+        chunk_id=chunk_id,
+        title=str(page.get("title", "")),
+        url=str(page.get("url", "")),
+        kind=str(page.get("kind", "")),
+        headings=tuple(str(item) for item in page.get("headings", [])[:12]),
+        text=text,
+        offer_id=str(page.get("offer_id", "")),
+        entity_id=str(page.get("entity_id", "")),
+        evidence_spans=_evidence_spans_from_page(page, chunk_id, text),
+    )
+
+
+def _render_chunk(chunk: EvidenceChunk) -> str:
+    return "\n".join(
+        [
+            f'<EVIDENCE_CHUNK chunk_id="{chunk.chunk_id}">',
+            chunk.evidence_text,
+            "</EVIDENCE_CHUNK>",
+        ]
+    )
+
+
+def _routing_notes_section() -> str:
+    return (
+        "<NON_EVIDENCE_ROUTING_NOTES>\n"
+        "Routing was completed deterministically before generation. This section\n"
+        "contains no factual evidence, must not support a claim, and must not be cited.\n"
+        "</NON_EVIDENCE_ROUTING_NOTES>"
+    )
+
+
+def evidence_chunks(
+    selected_pages: list[dict[str, Any]], *, max_chars: int = MAX_CONTEXT_CHARS
+) -> tuple[EvidenceChunk, ...]:
+    """Return exactly the source chunks that fit in the model evidence block."""
+
+    routing_section = _routing_notes_section()
+    used_chars = len(routing_section)
+    chunks: list[EvidenceChunk] = []
+    seen_ids: set[str] = set()
+
+    for page in selected_pages:
         if str(page.get("url", "")).startswith("internal://"):
             continue
-        text = str(page.get("text", ""))
-        chunks.append(
-            "\n".join(
-                [
-                    f"SOURCE {index}",
-                    f"Title: {page.get('title', '')}",
-                    f"Kind: {page.get('kind', '')}",
-                    f"URL: {page.get('url', '')}",
-                    f"Headings: {', '.join(str(h) for h in page.get('headings', [])[:12])}",
-                    f"Text: {text[:4500]}",
-                ]
-            )
-        )
-    block = "\n\n".join(chunks)
-    return block[:max_chars]
+        chunk = _chunk_from_page(page)
+        if chunk.chunk_id in seen_ids:
+            continue
+        rendered_length = len(_render_chunk(chunk)) + 2
+        if used_chars + rendered_length > max_chars:
+            break
+        chunks.append(chunk)
+        seen_ids.add(chunk.chunk_id)
+        used_chars += rendered_length
+    return tuple(chunks)
+
+
+def _context_block(
+    selected_pages: list[dict[str, Any]], max_chars: int = MAX_CONTEXT_CHARS
+) -> str:
+    routing_section = _routing_notes_section()
+    rendered_chunks = "\n\n".join(
+        _render_chunk(chunk)
+        for chunk in evidence_chunks(selected_pages, max_chars=max_chars)
+    )
+    return "\n\n".join(
+        [
+            routing_section,
+            "<EVIDENCE_CHUNKS>",
+            rendered_chunks or "(none)",
+            "</EVIDENCE_CHUNKS>",
+        ]
+    )
 
 
 def build_prompt(
@@ -72,36 +541,128 @@ def build_prompt(
     )
     return f"""The visitor is on a public Towards AI page.
 
+<NON_EVIDENCE_REQUEST_CONTEXT>
 Current URL: {current_url or "unknown"}
 Current page title: {page_title or "unknown"}
 Initial forced prompt: {selected_prompt or "unknown"}
 
-Conversation so far:
+Conversation so far (visitor-provided and NOT evidence):
 {turns or "(none)"}
 
-Visitor message:
+Visitor message (a question, NOT evidence):
 {query}
+</NON_EVIDENCE_REQUEST_CONTEXT>
 
-Use only these public sales/resource sources and the routing notes:
 {_context_block(selected_pages)}
 
-Answer now. Include links only when directly useful. Do not mention internal routing notes.
+Use only EVIDENCE_CHUNKS for factual claims. Return the required JSON object now.
+Every factual claim must be directly supported by one complete ALLOWED_EVIDENCE_SPAN.
+The text of each claim must copy its quote verbatim; the only permitted change is
+adding one final period when the span has none. Copy the entire span exactly into
+quote; arbitrary substrings are forbidden. Never omit qualifiers or negations,
+paraphrase, or combine words from different parts of a chunk.
+Correction rule: the visitor's premise and proposed alternatives are not evidence.
+If the chunks contain directly relevant facts that correct or resolve the question,
+return answered with those facts as separate extractive claims. For example, when
+one table row says an item is included and another row says named items are
+discounted, make one claim for the included row and another claim for the discount
+row. Do not infer a total or say "only" unless an evidence quote explicitly does.
+Do not repeat or deny the visitor's unsupported premise. Never write "not included"
+unless those literal words occur in that claim's exact quote.
+If the evidence contains no directly relevant facts, return not_found.
+You cannot confirm it. Do not use not_found when the evidence contains supported
+correcting facts.
 """
 
 
-def generate_answer(prompt: str) -> LLMResult:
+def _clean_usage(usage: dict[str, Any]) -> dict[str, Any]:
+    return {key: value for key, value in usage.items() if value is not None}
+
+
+def _text_from_chat_content(content: Any) -> str:
+    if isinstance(content, str):
+        return content
+    if isinstance(content, list):
+        parts: list[str] = []
+        for item in content:
+            if isinstance(item, dict):
+                parts.append(str(item.get("text", "")))
+            else:
+                parts.append(str(item))
+        return "".join(parts)
+    return str(content or "")
+
+
+def _deepseek_headers() -> dict[str, str]:
+    return {
+        "Authorization": f"Bearer {settings.deepseek_api_key}",
+        "Content-Type": "application/json",
+    }
+
+
+def _generate_deepseek_answer(prompt: str) -> LLMResult:
+    if not settings.deepseek_api_key:
+        raise RuntimeError("DEEPSEEK_API_KEY is not configured")
+
+    response = requests.post(
+        f"{settings.deepseek_base_url}/chat/completions",
+        headers=_deepseek_headers(),
+        json={
+            "model": settings.primary_model_name,
+            "messages": [
+                {"role": "system", "content": SYSTEM_INSTRUCTION},
+                {"role": "user", "content": prompt},
+            ],
+            "thinking": {"type": settings.deepseek_thinking_type},
+            "response_format": {"type": "json_object"},
+            "temperature": DEFAULT_TEMPERATURE,
+            "max_tokens": settings.max_output_tokens,
+            "stream": False,
+        },
+        timeout=settings.llm_request_timeout_seconds,
+    )
+    if response.status_code >= 400:
+        details = response.text.strip()[:500]
+        raise RuntimeError(
+            f"DeepSeek request failed with HTTP {response.status_code}: {details}"
+        )
+
+    payload = response.json()
+    choices = payload.get("choices") or []
+    if not choices:
+        raise RuntimeError("DeepSeek response did not include any choices")
+
+    message = choices[0].get("message") or {}
+    answer = _text_from_chat_content(message.get("content")).strip()
+    raw_usage = payload.get("usage") or {}
+    usage = _clean_usage(
+        {
+            "input_tokens": raw_usage.get("prompt_tokens")
+            or raw_usage.get("input_tokens"),
+            "output_tokens": raw_usage.get("completion_tokens")
+            or raw_usage.get("output_tokens"),
+            "total_tokens": raw_usage.get("total_tokens"),
+            "provider": DEEPSEEK_PROVIDER,
+            "model": settings.primary_model_name,
+        }
+    )
+    return LLMResult(answer=answer, usage=usage)
+
+
+def _generate_gemini_answer(prompt: str, *, fallback_from: str = "") -> LLMResult:
     if not settings.gemini_api_key:
         raise RuntimeError("GEMINI_API_KEY is not configured")
 
-    started = time.monotonic()
     client = genai.Client(api_key=settings.gemini_api_key)
     response = client.models.generate_content(
-        model=settings.model_name,
+        model=settings.fallback_model_name,
         contents=prompt,
         config={
             "system_instruction": SYSTEM_INSTRUCTION,
-            "temperature": 0.2,
+            "temperature": DEFAULT_TEMPERATURE,
             "max_output_tokens": settings.max_output_tokens,
+            "response_mime_type": "application/json",
+            "response_json_schema": GROUNDING_RESPONSE_SCHEMA,
         },
     )
     usage_metadata = getattr(response, "usage_metadata", None)
@@ -112,8 +673,573 @@ def generate_answer(prompt: str) -> LLMResult:
             "output_tokens": getattr(usage_metadata, "candidates_token_count", None),
             "total_tokens": getattr(usage_metadata, "total_token_count", None),
         }
+    usage.update(
+        {
+            "provider": GEMINI_PROVIDER,
+            "model": settings.fallback_model_name,
+        }
+    )
+    if fallback_from:
+        usage["fallback_from"] = fallback_from
     return LLMResult(
         answer=(getattr(response, "text", "") or "").strip(),
-        usage={key: value for key, value in usage.items() if value is not None},
-        latency_ms=int((time.monotonic() - started) * 1000),
+        usage=_clean_usage(usage),
+    )
+
+
+def _with_latency(result: LLMResult, started: float) -> LLMResult:
+    return replace(result, latency_ms=int((time.monotonic() - started) * 1000))
+
+
+def generate_answer(prompt: str) -> LLMResult:
+    """Generate the raw structured provider response, retaining provider fallback."""
+
+    started = time.monotonic()
+    primary_error: Exception | None = None
+
+    if settings.deepseek_api_key:
+        try:
+            result = _generate_deepseek_answer(prompt)
+            return _with_latency(result, started)
+        except Exception as exc:
+            primary_error = exc
+            logger.warning(
+                "DeepSeek generation failed; falling back to Gemini.",
+                exc_info=True,
+            )
+
+    try:
+        result = _generate_gemini_answer(
+            prompt,
+            fallback_from=DEEPSEEK_PROVIDER if primary_error else "",
+        )
+        return _with_latency(result, started)
+    except Exception as fallback_error:
+        if primary_error is not None:
+            raise RuntimeError(
+                "DeepSeek primary and Gemini fallback generation both failed"
+            ) from fallback_error
+        raise
+
+
+class _DuplicateKeyError(ValueError):
+    pass
+
+
+def _reject_duplicate_keys(pairs: list[tuple[str, Any]]) -> dict[str, Any]:
+    result: dict[str, Any] = {}
+    for key, value in pairs:
+        if key in result:
+            raise _DuplicateKeyError(f"duplicate JSON key: {key}")
+        result[key] = value
+    return result
+
+
+def _reject_json_constant(value: str) -> None:
+    raise ValueError(f"non-standard JSON constant: {value}")
+
+
+def _validation_failure(raw: LLMResult, message: str) -> GroundingResult:
+    return GroundingResult(
+        valid=False,
+        status="validation_failure",
+        usage=raw.usage,
+        latency_ms=raw.latency_ms,
+        validation_error=message,
+    )
+
+
+def _trim_url(value: str) -> str:
+    return value.rstrip(".,;:!?)]}").casefold()
+
+
+def _critical_facts(value: str) -> dict[str, set[str]]:
+    normalized = _normalize_text(value)
+    word_tokens = {token.casefold() for token in _TOKEN_RE.findall(normalized)}
+    return {
+        "URLs": {_trim_url(item) for item in _URL_RE.findall(normalized)},
+        "percentages": {
+            _normalize_text(item).casefold() for item in _PERCENT_RE.findall(normalized)
+        },
+        "currency amounts": {
+            _normalize_text(item).casefold()
+            for item in _CURRENCY_RE.findall(normalized)
+        },
+        "numbers": {
+            _normalize_text(item).casefold() for item in _NUMBER_RE.findall(normalized)
+        },
+        "number words": word_tokens & _NUMBER_WORDS,
+    }
+
+
+def _claim_is_one_sentence(text: str) -> bool:
+    if not text or re.search(r"[.!?][\"')\]]*$", text) is None:
+        return False
+    return _SENTENCE_BOUNDARY_RE.search(text) is None
+
+
+def _validate_claim_reference(
+    claim: GroundedClaim, chunk_by_id: dict[str, EvidenceChunk]
+) -> str:
+    chunk = chunk_by_id.get(claim.chunk_id)
+    if chunk is None:
+        return f"unknown chunk_id: {claim.chunk_id}"
+
+    normalized_quote = _normalize_text(claim.quote)
+    if not normalized_quote:
+        return "evidence quote must not be empty"
+    if len(normalized_quote) > MAX_EVIDENCE_QUOTE_CHARS:
+        return f"evidence quote exceeds {MAX_EVIDENCE_QUOTE_CHARS} characters"
+    normalized_chunk = _normalize_text(chunk.text)
+    if normalized_quote not in normalized_chunk:
+        return "evidence span is not present in its chunk text"
+    allowed_spans = {
+        _normalize_text(span.text) for span in chunk.evidence_spans if span.text
+    }
+    if normalized_quote not in allowed_spans:
+        return "evidence quote must equal one complete server-defined evidence span"
+    return ""
+
+
+def _validate_claim_text(claim: GroundedClaim) -> str:
+    normalized_claim = _normalize_text(claim.text)
+    normalized_quote = _normalize_text(claim.quote)
+    claim_facts = _critical_facts(claim.text)
+    evidence_facts = _critical_facts(claim.quote)
+    for label, facts in claim_facts.items():
+        unsupported = facts - evidence_facts[label]
+        if unsupported:
+            values = ", ".join(sorted(unsupported))
+            return f"claim contains unsupported {label}: {values}"
+
+    claim_polarity = {
+        item.casefold() for item in _POLARITY_RE.findall(_normalize_text(claim.text))
+    }
+    quote_polarity = {
+        item.casefold() for item in _POLARITY_RE.findall(_normalize_text(claim.quote))
+    }
+    if claim_polarity != quote_polarity:
+        return "claim changes or omits evidence negation"
+
+    allowed_claim_texts = {normalized_quote}
+    if not re.search(r"[.!?][\"')\]]*$", normalized_quote):
+        allowed_claim_texts.add(f"{normalized_quote}.")
+    if normalized_claim not in allowed_claim_texts:
+        return "claim must copy its evidence quote verbatim"
+
+    canonical_text = normalized_quote
+    if not re.search(r"[.!?][\"')\]]*$", canonical_text):
+        canonical_text = f"{canonical_text}."
+    if not _claim_is_one_sentence(canonical_text):
+        return "each claim must contain exactly one complete sentence"
+
+    return ""
+
+
+def _specific_query_terms(
+    query: str, target_offer_ids: frozenset[str]
+) -> frozenset[str]:
+    alias_terms = offer_alias_tokens(target_offer_ids)
+    result: set[str] = set()
+    for token in (item.casefold() for item in _TOKEN_RE.findall(query)):
+        if (
+            token in _QUERY_RELEVANCE_STOP
+            or token in _FACT_QUERY_TERMS
+            or token in _NUMBER_WORDS
+            or token in alias_terms
+            or any(character.isdigit() for character in token)
+        ):
+            continue
+        result.add(token)
+    return frozenset(result)
+
+
+def _term_is_present(term: str, evidence_terms: set[str]) -> bool:
+    if term in evidence_terms:
+        return True
+    if term.endswith("s") and term[:-1] in evidence_terms:
+        return True
+    return f"{term}s" in evidence_terms
+
+
+def _validate_query_grounding(
+    query: str,
+    claims: list[GroundedClaim],
+    chunk_by_id: dict[str, EvidenceChunk],
+    target_offer_ids: frozenset[str],
+) -> str:
+    if not query:
+        return ""
+
+    resolved_targets = target_offer_ids or offer_ids_for_query(query)
+    if resolved_targets:
+        for claim in claims:
+            chunk = chunk_by_id[claim.chunk_id]
+            if chunk.offer_id not in resolved_targets:
+                return (
+                    "claim cites offer "
+                    f"{chunk.offer_id or '(none)'} outside the requested offer boundary"
+                )
+            if _UNSAFE_COMPARISON_EVIDENCE_RE.search(claim.quote):
+                return "claim cites competitor comparison rather than offer evidence"
+
+    fields = requested_fact_fields(query)
+    if mixed_preview_fact_request(query):
+        return (
+            "mixed preview and paid-course facts lack separate target-qualified "
+            "evidence scopes"
+        )
+    for claim in claims:
+        chunk = chunk_by_id[claim.chunk_id]
+        if fields and not any(
+            chunk.offer_id in evidence_offer_ids_for_field(query, fact_field)
+            and evidence_span_supports_field(
+                claim.quote,
+                field=fact_field,
+                offer_id=chunk.offer_id,
+                query=query,
+            )
+            for fact_field in fields
+        ):
+            return "claim lacks target-qualified evidence for the requested facts"
+    for fact_field in fields:
+        if not any(
+            chunk_by_id[claim.chunk_id].offer_id
+            in evidence_offer_ids_for_field(query, fact_field)
+            and evidence_span_supports_field(
+                claim.quote,
+                field=fact_field,
+                offer_id=chunk_by_id[claim.chunk_id].offer_id,
+                query=query,
+            )
+            for claim in claims
+        ):
+            return f"answer lacks target-qualified {fact_field} evidence"
+
+    normalized_query = _normalize_text(query).casefold()
+    combined_quotes = " ".join(_normalize_text(claim.quote) for claim in claims)
+    normalized_quotes = combined_quotes.casefold()
+    if (
+        monthly_plan_intent(normalized_query)
+        and fields & {"refund", "guarantee"}
+        and not (
+            "monthly" in normalized_quotes
+            and "yearly" in normalized_quotes
+            and "money-back" in normalized_quotes
+        )
+    ):
+        return "monthly guarantee answer must include the qualified yearly policy"
+
+    specific_terms = _specific_query_terms(query, resolved_targets)
+    if specific_terms:
+        evidence_terms = {
+            item.casefold() for item in _TOKEN_RE.findall(combined_quotes)
+        }
+        matched = sum(
+            _term_is_present(term, evidence_terms) for term in specific_terms
+        )
+        # Require at least two thirds of the visitor's non-generic qualifiers.
+        # Corrections involving a false number remain possible because proposed
+        # numbers and number words are deliberately excluded above.
+        if matched * 3 < len(specific_terms) * 2:
+            return "answer does not address enough of the requested qualifiers"
+    return ""
+
+
+def validate_grounded_result(
+    raw_result: LLMResult | str,
+    selected_pages: list[dict[str, Any]],
+    *,
+    query: str = "",
+    target_offer_ids: frozenset[str] | None = None,
+) -> GroundingResult:
+    """Strictly validate provider JSON and return only evidence-backed text.
+
+    Any malformed schema, bad citation, unsafe evidence span, or non-verbatim
+    claim produces ``validation_failure`` with an empty ``answer``. Raw model
+    text is never copied into the safe result.
+    """
+
+    raw = raw_result if isinstance(raw_result, LLMResult) else LLMResult(raw_result)
+    try:
+        payload = json.loads(
+            raw.answer,
+            object_pairs_hook=_reject_duplicate_keys,
+            parse_constant=_reject_json_constant,
+        )
+    except (json.JSONDecodeError, TypeError, ValueError) as exc:
+        return _validation_failure(raw, f"malformed JSON: {exc}")
+
+    if not isinstance(payload, dict) or set(payload) != {"status", "claims"}:
+        return _validation_failure(raw, "root must contain exactly status and claims")
+    status = payload["status"]
+    claims_payload = payload["claims"]
+    if status not in {"answered", "not_found"}:
+        return _validation_failure(raw, "status must be answered or not_found")
+    if not isinstance(claims_payload, list):
+        return _validation_failure(raw, "claims must be an array")
+    if len(claims_payload) > 6:
+        return _validation_failure(raw, "claims must contain at most 6 items")
+
+    if status == "not_found":
+        if claims_payload:
+            return _validation_failure(raw, "not_found must contain no claims")
+        return GroundingResult(
+            valid=True,
+            status="not_found",
+            usage=raw.usage,
+            latency_ms=raw.latency_ms,
+        )
+    if not claims_payload:
+        return _validation_failure(raw, "answered must contain at least one claim")
+
+    chunks = evidence_chunks(selected_pages)
+    chunk_by_id = {chunk.chunk_id: chunk for chunk in chunks}
+    claims: list[GroundedClaim] = []
+    for index, item in enumerate(claims_payload):
+        if not isinstance(item, dict) or set(item) != {"text", "chunk_id", "quote"}:
+            return _validation_failure(
+                raw,
+                f"claim {index + 1} must contain exactly text, chunk_id, and quote",
+            )
+        if not all(isinstance(item[key], str) for key in item):
+            return _validation_failure(raw, f"claim {index + 1} fields must be strings")
+        claim = GroundedClaim(
+            text=item["text"].strip(),
+            chunk_id=item["chunk_id"].strip(),
+            quote=item["quote"].strip(),
+        )
+        reference_error = _validate_claim_reference(claim, chunk_by_id)
+        if reference_error:
+            return _validation_failure(raw, f"claim {index + 1}: {reference_error}")
+        text_error = _validate_claim_text(claim)
+        if text_error:
+            return _validation_failure(raw, f"claim {index + 1}: {text_error}")
+        safe_text = _normalize_text(claim.quote)
+        if not re.search(r"[.!?][\"')\]]*$", safe_text):
+            safe_text = f"{safe_text}."
+        claim = replace(claim, text=safe_text)
+        claims.append(claim)
+
+    query_error = _validate_query_grounding(
+        query,
+        claims,
+        chunk_by_id,
+        target_offer_ids or frozenset(),
+    )
+    if query_error:
+        return _validation_failure(raw, query_error)
+
+    cited_chunks: list[EvidenceChunk] = []
+    cited_ids: set[str] = set()
+    for claim in claims:
+        if claim.chunk_id not in cited_ids:
+            cited_chunks.append(chunk_by_id[claim.chunk_id])
+            cited_ids.add(claim.chunk_id)
+
+    return GroundingResult(
+        valid=True,
+        status="answered",
+        answer=" ".join(claim.text for claim in claims),
+        claims=tuple(claims),
+        cited_chunks=tuple(cited_chunks),
+        usage=raw.usage,
+        latency_ms=raw.latency_ms,
+    )
+
+
+def extract_mentorship_course_access(
+    query: str,
+    selected_pages: list[dict[str, Any]],
+    *,
+    target_offer_ids: frozenset[str] | None = None,
+) -> GroundingResult | None:
+    """Extract high-risk mentorship entitlements and policies without generation."""
+
+    lowered = query.casefold()
+    if "mentor" not in lowered:
+        return None
+
+    chunks = evidence_chunks(selected_pages)
+
+    def find_span(*phrases: str) -> tuple[EvidenceChunk, EvidenceSpan] | None:
+        for chunk in chunks:
+            if chunk.offer_id != "mentorship":
+                continue
+            for span in chunk.evidence_spans:
+                span_lower = span.text.casefold()
+                if all(phrase in span_lower for phrase in phrases):
+                    return chunk, span
+        return None
+
+    def validated_extraction(
+        evidence: list[tuple[EvidenceChunk, EvidenceSpan]],
+    ) -> GroundingResult:
+        if not evidence:
+            return GroundingResult(valid=True, status="not_found")
+        raw = json.dumps(
+            {
+                "status": "answered",
+                "claims": [
+                    {
+                        "text": span.text,
+                        "chunk_id": chunk.chunk_id,
+                        "quote": span.text,
+                    }
+                    for chunk, span in evidence
+                ],
+            },
+            ensure_ascii=False,
+        )
+        result = validate_grounded_result(
+            LLMResult(raw, usage={"provider": "deterministic_extraction"}),
+            selected_pages,
+            target_offer_ids=target_offer_ids,
+        )
+        return (
+            result
+            if result.valid
+            else GroundingResult(valid=True, status="not_found")
+        )
+
+    course_access_intent = any(
+        term in lowered
+        for term in (
+            "course",
+            "courses",
+            "llm fundamentals",
+            "full stack",
+            "agent engineering",
+            "master ai for work",
+        )
+    ) and any(
+        term in lowered
+        for term in (
+            "include",
+            "included",
+            "access",
+            "choice",
+            "choose",
+            "discount",
+            "% off",
+            "free",
+            "2",
+            "two",
+        )
+    )
+    cancellation_intent = bool(
+        re.search(r"\bcancel|\bcancell|\bafter\b", lowered)
+        and any(term in lowered for term in ("course", "access", "lifetime", "keep"))
+    )
+    if cancellation_intent:
+        active = find_span("course access is active while")
+        upgrade = find_span("if you cancel", "lifetime access")
+        if active is None or upgrade is None:
+            return GroundingResult(valid=True, status="not_found")
+        return validated_extraction([active, upgrade])
+
+    guarantee_intent = any(
+        term in lowered for term in ("refund", "money-back", "money back", "guarantee")
+    )
+    if guarantee_intent:
+        yearly = find_span("yearly plan", "money-back guarantee")
+        if yearly is None:
+            yearly = find_span("money-back on yearly")
+        if monthly_plan_intent(lowered):
+            monthly = find_span("monthly mentorship", "cancelled")
+            if monthly is None or yearly is None:
+                return GroundingResult(valid=True, status="not_found")
+            return validated_extraction([monthly, yearly])
+        if yearly is None:
+            return GroundingResult(valid=True, status="not_found")
+        return validated_extraction([yearly])
+
+    if not course_access_intent:
+        return None
+
+    included: tuple[EvidenceChunk, EvidenceSpan] | None = None
+    discounted: tuple[EvidenceChunk, EvidenceSpan] | None = None
+    alpha: tuple[EvidenceChunk, EvidenceSpan] | None = None
+    for chunk in chunks:
+        if "/academy/mentorship" not in chunk.url:
+            continue
+        for span in chunk.evidence_spans:
+            span_lower = span.text.casefold()
+            if (
+                included is None
+                and "llm fundamentals" in span_lower
+                and re.search(r"\bincluded\b", span_lower)
+            ):
+                included = (chunk, span)
+            if (
+                discounted is None
+                and any(
+                    name in span_lower
+                    for name in (
+                        "full stack ai engineering",
+                        "agent engineering",
+                        "master ai for work",
+                    )
+                )
+                and re.search(r"\b\d+(?:[–-]\d+)?%\s+off\b", span_lower)
+            ):
+                discounted = (chunk, span)
+            if (
+                alpha is None
+                and "new courses" in span_lower
+                and "alpha access" in span_lower
+                and "% off" in span_lower
+            ):
+                alpha = (chunk, span)
+
+    if any(term in lowered for term in ("new course", "new courses", "alpha")):
+        return (
+            validated_extraction([alpha])
+            if alpha is not None
+            else GroundingResult(valid=True, status="not_found")
+        )
+
+    wants_discount = any(
+        term in lowered for term in ("discount", "% off", "off full", "off agent")
+    ) or bool(re.search(r"\b\d+(?:[.-]\d+)?%", lowered))
+    wants_comparison = any(
+        term in lowered
+        for term in (
+            "courses",
+            "choice",
+            "choose",
+            "2",
+            "two",
+            "full stack",
+            "agent engineering",
+            "master ai for work",
+        )
+    )
+    if wants_discount and discounted is None:
+        return GroundingResult(valid=True, status="not_found")
+    if not wants_discount and included is None:
+        return GroundingResult(valid=True, status="not_found")
+
+    evidence: list[tuple[EvidenceChunk, EvidenceSpan]] = []
+    if included is not None and (not wants_discount or wants_comparison):
+        evidence.append(included)
+    if discounted is not None and (wants_discount or wants_comparison):
+        evidence.append(discounted)
+    return validated_extraction(evidence)
+
+
+def generate_grounded_answer(
+    prompt: str,
+    selected_pages: list[dict[str, Any]],
+    *,
+    query: str = "",
+    target_offer_ids: frozenset[str] | None = None,
+) -> GroundingResult:
+    """Generate and cross the fail-closed grounding boundary in one call."""
+
+    return validate_grounded_result(
+        generate_answer(prompt),
+        selected_pages,
+        query=query,
+        target_offer_ids=target_offer_ids,
     )

tai_helper/monitoring.py

@@ -25,9 +25,11 @@ def _import_opik() -> Any | None:
     global _OPIK_IMPORT_WARNING_EMITTED
     try:
         import opik  # type: ignore[import-not-found]
-    except Exception:
+    except ImportError:
         if not _OPIK_IMPORT_WARNING_EMITTED:
-            logger.warning("Opik monitoring is enabled but the opik package is unavailable.")
+            logger.warning(
+                "Opik monitoring is enabled but the opik package is unavailable."
+            )
             _OPIK_IMPORT_WARNING_EMITTED = True
         return None
     return opik
@@ -120,8 +122,8 @@ class HelperMonitor:
             },
             "tags": tags,
             "project_name": self.configured.opik_project_name,
-            "model": self.configured.model_name,
-            "provider": "google_genai",
+            "model": self.usage.get("model") or self.configured.model_name,
+            "provider": self.usage.get("provider") or "unknown",
             "flush": True,
         }
         with opik.start_as_current_span(**span_kwargs) as span:

tai_helper/schemas.py

@@ -36,4 +36,7 @@ class HelperChatResponse(BaseModel):
     answer: str
     threadId: str
     sources: list[SourceOut]
+    status: Literal["answered", "insufficient_evidence", "out_of_scope", "policy"] = (
+        "answered"
+    )
     usage: dict[str, Any] = Field(default_factory=dict)

tai_helper/settings.py

@@ -28,6 +28,16 @@ def _int_env(name: str, default: int) -> int:
         return default
 
 
+def _float_env(name: str, default: float) -> float:
+    raw = os.getenv(name)
+    if raw is None:
+        return default
+    try:
+        return float(raw)
+    except ValueError:
+        return default
+
+
 def _bool_env(name: str, default: bool = False) -> bool:
     raw = os.getenv(name)
     if raw is None:
@@ -40,25 +50,78 @@ class Settings:
     allowed_origins: tuple[str, ...] = field(
         default_factory=lambda: _csv_env(
             "HELPER_ALLOWED_ORIGINS",
-            "https://academy.towardsai.net,https://towardsai.net,https://www.towardsai.net",
+            (
+                "https://towardsai.com,https://www.towardsai.com,"
+                "https://academy.towardsai.net,https://towardsai.net,"
+                "https://www.towardsai.net"
+            ),
         )
     )
     allowed_hosts: tuple[str, ...] = field(
         default_factory=lambda: _csv_env(
             "HELPER_ALLOWED_HOSTS",
-            "academy.towardsai.net,towardsai.net,www.towardsai.net",
+            (
+                "towardsai.com,www.towardsai.com,academy.towardsai.net,"
+                "towardsai.net,www.towardsai.net"
+            ),
+        )
+    )
+    site_wide_hosts: tuple[str, ...] = field(
+        default_factory=lambda: _csv_env(
+            "HELPER_SITE_WIDE_HOSTS",
+            "towardsai.com,www.towardsai.com",
         )
     )
-    model_name: str = field(
-        default_factory=lambda: os.getenv("HELPER_MODEL", "gemini-2.5-flash").strip()
-        or "gemini-2.5-flash"
+    deepseek_api_key: str = field(
+        default_factory=lambda: os.getenv("DEEPSEEK_API_KEY", "").strip()
+    )
+    deepseek_base_url: str = field(
+        default_factory=lambda: (
+            os.getenv(
+                "DEEPSEEK_BASE_URL",
+                "https://api.deepseek.com",
+            )
+            .strip()
+            .rstrip("/")
+            or "https://api.deepseek.com"
+        )
+    )
+    deepseek_thinking_type: str = field(
+        default_factory=lambda: (
+            os.getenv(
+                "HELPER_DEEPSEEK_THINKING",
+                "disabled",
+            ).strip()
+            or "disabled"
+        )
+    )
+    primary_model_name: str = field(
+        default_factory=lambda: (
+            os.getenv(
+                "HELPER_PRIMARY_MODEL",
+                os.getenv("HELPER_MODEL", "deepseek-v4-flash"),
+            ).strip()
+            or "deepseek-v4-flash"
+        )
     )
     gemini_api_key: str = field(
         default_factory=lambda: os.getenv("GEMINI_API_KEY", "").strip()
     )
+    fallback_model_name: str = field(
+        default_factory=lambda: (
+            os.getenv(
+                "HELPER_FALLBACK_MODEL",
+                "gemini-2.5-flash",
+            ).strip()
+            or "gemini-2.5-flash"
+        )
+    )
     max_output_tokens: int = field(
         default_factory=lambda: _int_env("HELPER_MAX_OUTPUT_TOKENS", 420)
     )
+    llm_request_timeout_seconds: float = field(
+        default_factory=lambda: _float_env("HELPER_LLM_REQUEST_TIMEOUT_SECONDS", 20.0)
+    )
     max_body_bytes: int = field(
         default_factory=lambda: _int_env("HELPER_MAX_BODY_BYTES", 64 * 1024)
     )
@@ -68,6 +131,9 @@ class Settings:
     max_history_turns: int = field(
         default_factory=lambda: _int_env("HELPER_MAX_HISTORY_TURNS", 8)
     )
+    catalog_max_age_days: int = field(
+        default_factory=lambda: _int_env("HELPER_CATALOG_MAX_AGE_DAYS", 14)
+    )
     rate_limit_per_minute: int = field(
         default_factory=lambda: _int_env("HELPER_RATE_LIMIT_PER_MINUTE", 3)
     )
@@ -85,11 +151,13 @@ class Settings:
         default_factory=lambda: os.getenv("OPIK_WORKSPACE", "").strip()
     )
     opik_project_name: str = field(
-        default_factory=lambda: os.getenv(
-            "OPIK_PROJECT_NAME",
-            "towards-ai-helper",
-        ).strip()
-        or "towards-ai-helper"
+        default_factory=lambda: (
+            os.getenv(
+                "OPIK_PROJECT_NAME",
+                "towards-ai-helper",
+            ).strip()
+            or "towards-ai-helper"
+        )
     )
     opik_max_text_chars: int = field(
         default_factory=lambda: _int_env("OPIK_MAX_TEXT_CHARS", 4000)
@@ -98,5 +166,9 @@ class Settings:
     def cors_origins(self) -> list[str]:
         return list(self.allowed_origins)
 
+    @property
+    def model_name(self) -> str:
+        return self.primary_model_name
+
 
 settings = Settings()

tests/conftest.py

@@ -0,0 +1,14 @@
+from __future__ import annotations
+
+import os
+
+os.environ["HELPER_ALLOWED_ORIGINS"] = (
+    "https://towardsai.com,https://www.towardsai.com,"
+    "https://academy.towardsai.net,https://towardsai.net,"
+    "https://www.towardsai.net"
+)
+os.environ["HELPER_ALLOWED_HOSTS"] = (
+    "towardsai.com,www.towardsai.com,academy.towardsai.net,"
+    "towardsai.net,www.towardsai.net"
+)
+os.environ["HELPER_SITE_WIDE_HOSTS"] = "towardsai.com,www.towardsai.com"

tests/test_api.py

@@ -1,14 +1,17 @@
 from __future__ import annotations
 
+import json
+
+import pytest
 from fastapi.testclient import TestClient
 
 from tai_helper import api
-from tai_helper.llm import LLMResult
+from tai_helper.llm import EvidenceChunk, GroundingResult
 from tai_helper.rate_limiter import FixedWindowRateLimiter, RateLimit
 
 client = TestClient(api.app)
-HEADERS = {"Origin": "https://academy.towardsai.net"}
-PUBLIC_URL = "https://academy.towardsai.net/courses/agent-engineering"
+HEADERS = {"Origin": "https://towardsai.com"}
+PUBLIC_URL = "https://towardsai.com/academy/agent-engineering/"
 FIRST_PROMPT = "I want help deciding which course to take."
 
 
@@ -30,7 +33,9 @@ def reset_limiters() -> None:
     )
 
 
-def payload(query: str = FIRST_PROMPT, *, url: str = PUBLIC_URL, signed_in: bool = False):
+def payload(
+    query: str = FIRST_PROMPT, *, url: str = PUBLIC_URL, signed_in: bool = False
+):
     return {
         "query": query,
         "selectedPrompt": query if query == FIRST_PROMPT else FIRST_PROMPT,
@@ -45,6 +50,13 @@ def payload(query: str = FIRST_PROMPT, *, url: str = PUBLIC_URL, signed_in: bool
     }
 
 
+def assert_contact_handoff(body: dict, *, status: str = "insufficient_evidence") -> None:
+    assert body["status"] == status
+    assert body["sources"] == []
+    assert body["answer"].count(api.CONTACT_FORM_URL) == 1
+    assert "louis@towardsai.net" not in body["answer"]
+
+
 def test_config_exposes_public_widget_contract() -> None:
     response = client.get("/api/helper/config")
 
@@ -52,7 +64,33 @@ def test_config_exposes_public_widget_contract() -> None:
     data = response.json()
     assert data["name"] == "Towards AI Helper"
     assert FIRST_PROMPT in data["forcedPrompts"]
-    assert "/courses/agent-engineering" in data["allowedPathsByHost"]["academy.towardsai.net"]
+    assert "towardsai.com" in data["allowedHosts"]
+    assert "towardsai.com" in data["siteWideHosts"]
+    assert "/academy/agent-engineering" in data["allowedPathsByHost"]["towardsai.com"]
+    assert (
+        "/courses/agent-engineering"
+        in data["allowedPathsByHost"]["academy.towardsai.net"]
+    )
+
+
+def test_footer_compatible_widget_path_is_served() -> None:
+    response = client.get("/helper-widget.js")
+
+    assert response.status_code == 200
+    assert "Towards AI Helper" in response.text
+
+
+def test_cors_preflight_accepts_towardsai_com() -> None:
+    response = client.options(
+        "/api/helper/chat",
+        headers={
+            "Origin": "https://towardsai.com",
+            "Access-Control-Request-Method": "POST",
+        },
+    )
+
+    assert response.status_code == 200
+    assert response.headers["access-control-allow-origin"] == "https://towardsai.com"
 
 
 def test_chat_requires_allowed_origin_public_page_signed_out_and_first_prompt() -> None:
@@ -62,7 +100,7 @@ def test_chat_requires_allowed_origin_public_page_signed_out_and_first_prompt()
     assert (
         client.post(
             "/api/helper/chat",
-            json=payload(url="https://academy.towardsai.net/courses/take/x"),
+            json=payload(url="https://towardsai.com/wp-admin/edit.php"),
             headers=HEADERS,
         ).status_code
         == 403
@@ -85,38 +123,506 @@ def test_chat_requires_allowed_origin_public_page_signed_out_and_first_prompt()
     )
 
 
-def test_chat_calls_gemini_with_retrieved_sources(monkeypatch) -> None:
+def test_chat_returns_only_grounded_answer_and_cited_sources(monkeypatch) -> None:
     reset_limiters()
     prompts = []
 
-    def fake_generate_answer(prompt: str) -> LLMResult:
+    def fake_generate_grounded_answer(
+        prompt: str, selected_pages: list[dict], **_kwargs
+    ) -> GroundingResult:
         prompts.append(prompt)
-        return LLMResult(
+        assert selected_pages
+        cited = EvidenceChunk(
+            chunk_id=str(selected_pages[0]["chunk_id"]),
+            title=str(selected_pages[0]["title"]),
+            url=str(selected_pages[0]["url"]),
+            kind=str(selected_pages[0]["kind"]),
+            headings=tuple(selected_pages[0].get("headings", [])),
+            text=str(selected_pages[0]["text"]),
+        )
+        return GroundingResult(
+            valid=True,
+            status="answered",
             answer="Tell me your coding background and goal.",
+            cited_chunks=(cited,),
             usage={"input_tokens": 10, "output_tokens": 8, "total_tokens": 18},
             latency_ms=123,
         )
 
-    monkeypatch.setattr(api.llm, "generate_answer", fake_generate_answer)
+    monkeypatch.setattr(
+        api.llm, "generate_grounded_answer", fake_generate_grounded_answer
+    )
 
     response = client.post("/api/helper/chat", json=payload(), headers=HEADERS)
 
     assert response.status_code == 200
     data = response.json()
     assert data["answer"] == "Tell me your coding background and goal."
+    assert data["status"] == "answered"
     assert data["threadId"]
     assert data["sources"]
     assert data["usage"]["total_tokens"] == 18
     assert "Agent Engineering" in prompts[0]
 
 
+def test_chat_still_accepts_legacy_net_origin(monkeypatch) -> None:
+    reset_limiters()
+    monkeypatch.setattr(
+        api.llm,
+        "generate_grounded_answer",
+        lambda _prompt, selected, **_kwargs: GroundingResult(
+            valid=True,
+            status="answered",
+            answer="Supported answer.",
+            cited_chunks=(
+                EvidenceChunk(
+                    chunk_id=str(selected[0]["chunk_id"]),
+                    title=str(selected[0]["title"]),
+                    url=str(selected[0]["url"]),
+                    kind=str(selected[0]["kind"]),
+                    headings=(),
+                    text=str(selected[0]["text"]),
+                ),
+            ),
+        ),
+    )
+
+    response = client.post(
+        "/api/helper/chat",
+        json=payload(url="https://towardsai.net/b2b"),
+        headers={"Origin": "https://towardsai.net"},
+    )
+
+    assert response.status_code == 200
+
+
+def test_chat_abstains_without_retrieved_evidence(monkeypatch) -> None:
+    reset_limiters()
+    monkeypatch.setattr(api, "retrieve", lambda *_args, **_kwargs: [])
+
+    def unexpected_model(*_args, **_kwargs):
+        raise AssertionError("model must not run without retrieved evidence")
+
+    monkeypatch.setattr(api.llm, "generate_grounded_answer", unexpected_model)
+
+    response = client.post("/api/helper/chat", json=payload(), headers=HEADERS)
+
+    assert response.status_code == 200
+    assert_contact_handoff(response.json())
+    assert "don't want to guess" in response.json()["answer"]
+
+
+def test_chat_abstains_on_model_not_found_or_validation_failure(monkeypatch) -> None:
+    reset_limiters()
+    evidence = {
+        "chunk_id": "mentorship-inclusions",
+        "title": "Towards AI Mentorship",
+        "url": "https://towardsai.com/academy/mentorship/",
+        "kind": "mentorship",
+        "headings": ["Course included"],
+        "text": "10-Hour LLM Fundamentals video course — $199 — Included from day one.",
+    }
+    monkeypatch.setattr(api, "retrieve", lambda *_args, **_kwargs: [evidence])
+
+    for grounded in (
+        GroundingResult(valid=True, status="not_found"),
+        GroundingResult(
+            valid=False,
+            status="validation_failure",
+            validation_error="invented quote",
+        ),
+    ):
+        monkeypatch.setattr(
+            api.llm,
+            "generate_grounded_answer",
+            lambda *_args, result=grounded: result,
+        )
+        response = client.post("/api/helper/chat", json=payload(), headers=HEADERS)
+
+        assert response.status_code == 200
+        assert_contact_handoff(response.json())
+        assert "don't want to guess" in response.json()["answer"]
+
+
+def test_chat_abstains_when_model_omits_negation_from_an_evidence_span(
+    monkeypatch,
+) -> None:
+    reset_limiters()
+    selected = api.retrieve(FIRST_PROMPT)
+    ai_work = next(page for page in selected if "No code required" in page["text"])
+    raw = json.dumps(
+        {
+            "status": "answered",
+            "claims": [
+                {
+                    "text": "code required.",
+                    "chunk_id": ai_work["chunk_id"],
+                    "quote": "code required",
+                }
+            ],
+        }
+    )
+    monkeypatch.setattr(
+        api.llm,
+        "generate_answer",
+        lambda _prompt: api.llm.LLMResult(answer=raw),
+    )
+
+    response = client.post("/api/helper/chat", json=payload(), headers=HEADERS)
+
+    assert response.status_code == 200
+    assert_contact_handoff(response.json())
+    assert "don't want to guess" in response.json()["answer"]
+
+
+@pytest.mark.parametrize(
+    "query",
+    [
+        (
+            "Do the 7 lessons in the Agent Engineering preview include "
+            "lifetime access?"
+        ),
+        (
+            "Do the 7 lessons in the Agent Engineering preview have a "
+            "30-day refund guarantee?"
+        ),
+        "Do the 7 lessons in the Agent Engineering preview give a certificate?",
+        "Do the 7 lessons in the Agent Engineering preview cost $499?",
+        "Do the 7 lessons in the Agent Engineering preview get a 50% discount?",
+    ],
+)
+def test_preview_count_cannot_borrow_other_paid_course_facts(
+    monkeypatch, query: str
+) -> None:
+    reset_limiters()
+    assert api.retrieve(query) == []
+
+    def unexpected(*_args, **_kwargs):
+        raise AssertionError("mixed preview facts must not invoke the model")
+
+    monkeypatch.setattr(api.llm, "generate_grounded_answer", unexpected)
+    request_payload = payload(query=query)
+    request_payload["history"] = [{"role": "user", "content": FIRST_PROMPT}]
+
+    response = client.post(
+        "/api/helper/chat", json=request_payload, headers=HEADERS
+    )
+
+    assert response.status_code == 200
+    assert_contact_handoff(response.json())
+
+
+def test_preview_count_cannot_mask_an_unsupported_feature(monkeypatch) -> None:
+    reset_limiters()
+    query = (
+        "Do the 7 lessons in the Agent Engineering preview include weekly "
+        "code reviews?"
+    )
+    selected = api.retrieve(query)
+    count_chunk = next(
+        chunk
+        for chunk in selected
+        if any("7" in span["text"] for span in chunk["evidence_spans"])
+    )
+    count_span = next(
+        span["text"]
+        for span in count_chunk["evidence_spans"]
+        if "7" in span["text"]
+    )
+    raw = json.dumps(
+        {
+            "status": "answered",
+            "claims": [
+                {
+                    "text": count_span,
+                    "chunk_id": count_chunk["chunk_id"],
+                    "quote": count_span,
+                }
+            ],
+        }
+    )
+    monkeypatch.setattr(
+        api.llm,
+        "generate_answer",
+        lambda _prompt: api.llm.LLMResult(answer=raw),
+    )
+    request_payload = payload(query=query)
+    request_payload["history"] = [{"role": "user", "content": FIRST_PROMPT}]
+
+    response = client.post(
+        "/api/helper/chat", json=request_payload, headers=HEADERS
+    )
+
+    assert response.status_code == 200
+    assert_contact_handoff(response.json())
+
+
+def test_mentorship_course_access_incident_uses_exact_retrieved_rows(
+    monkeypatch,
+) -> None:
+    reset_limiters()
+
+    def unexpected_model(*_args, **_kwargs):
+        raise AssertionError("high-risk mentorship access answer must be extractive")
+
+    monkeypatch.setattr(api.llm, "generate_grounded_answer", unexpected_model)
+    request_payload = payload(
+        query=(
+            "The chatbot mentioned access to 2 courses of our choice as part of "
+            "mentorship. Is that true?"
+        ),
+        url="https://towardsai.com/academy/mentorship/",
+    )
+    request_payload["history"] = [{"role": "user", "content": FIRST_PROMPT}]
+
+    response = client.post("/api/helper/chat", json=request_payload, headers=HEADERS)
+
+    assert response.status_code == 200
+    body = response.json()
+    assert body["status"] == "answered"
+    assert "10-Hour LLM Fundamentals" in body["answer"]
+    assert "Included from day one" in body["answer"]
+    assert "25% off, always" in body["answer"]
+    assert "two courses" not in body["answer"].lower()
+    assert len(body["sources"]) == 1
+    assert body["sources"][0]["url"] == ("https://towardsai.com/academy/mentorship/")
+    assert body["sources"][0]["kind"] == "mentorship"
+
+
+def test_mentorship_cancellation_uses_exact_current_access_policy(
+    monkeypatch,
+) -> None:
+    reset_limiters()
+
+    def unexpected_model(*_args, **_kwargs):
+        raise AssertionError("mentorship cancellation answer must be extractive")
+
+    monkeypatch.setattr(api.llm, "generate_grounded_answer", unexpected_model)
+    request_payload = payload(
+        query=(
+            "Do I keep LLM Fundamentals lifetime access after cancelling mentorship?"
+        ),
+        url="https://towardsai.com/academy/mentorship/",
+    )
+    request_payload["history"] = [{"role": "user", "content": FIRST_PROMPT}]
+
+    response = client.post("/api/helper/chat", json=request_payload, headers=HEADERS)
+
+    assert response.status_code == 200
+    body = response.json()
+    assert body["status"] == "answered"
+    assert "Course access is active while your mentorship is active" in body["answer"]
+    assert "If you cancel" in body["answer"]
+    assert "lifetime access at a reduced price" in body["answer"]
+    assert "Included from day one" not in body["answer"]
+    assert {source["url"] for source in body["sources"]} == {
+        "https://towardsai.com/academy/mentorship/"
+    }
+
+
+@pytest.mark.parametrize(
+    "query",
+    [
+        "Does the monthly mentorship have a 30-day money-back guarantee?",
+        "Does the month-to-month mentorship have a 30-day money-back guarantee?",
+    ],
+)
+def test_monthly_mentorship_guarantee_names_the_yearly_qualification(
+    monkeypatch, query: str
+) -> None:
+    reset_limiters()
+
+    def unexpected_model(*_args, **_kwargs):
+        raise AssertionError("mentorship guarantee answer must be extractive")
+
+    monkeypatch.setattr(api.llm, "generate_grounded_answer", unexpected_model)
+    request_payload = payload(
+        query=query,
+        url="https://towardsai.com/academy/mentorship/",
+    )
+    request_payload["history"] = [{"role": "user", "content": FIRST_PROMPT}]
+
+    response = client.post("/api/helper/chat", json=request_payload, headers=HEADERS)
+
+    assert response.status_code == 200
+    body = response.json()
+    assert body["status"] == "answered"
+    assert "monthly mentorship can be cancelled at any time" in body["answer"]
+    assert "yearly plan carries a 30-day money-back guarantee" in body["answer"]
+    assert "Join the Mentorship 30-day" not in body["answer"]
+    assert len(body["sources"]) == 1
+
+
+def test_late_source_error_resets_answered_status_to_insufficient_evidence(
+    monkeypatch,
+) -> None:
+    reset_limiters()
+    selected = api.retrieve(FIRST_PROMPT)
+    cited = EvidenceChunk(
+        chunk_id=str(selected[0]["chunk_id"]),
+        title=str(selected[0]["title"]),
+        url=str(selected[0]["url"]),
+        kind=str(selected[0]["kind"]),
+        headings=tuple(selected[0].get("headings", [])),
+        text=str(selected[0]["text"]),
+    )
+    monkeypatch.setattr(
+        api.llm,
+        "generate_grounded_answer",
+        lambda *_args: GroundingResult(
+            valid=True,
+            status="answered",
+            answer="Verified source span.",
+            cited_chunks=(cited,),
+        ),
+    )
+    monkeypatch.setattr(
+        api,
+        "sources_from_pages",
+        lambda _pages: (_ for _ in ()).throw(RuntimeError("late source failure")),
+    )
+
+    response = client.post("/api/helper/chat", json=payload(), headers=HEADERS)
+
+    assert response.status_code == 200
+    assert_contact_handoff(response.json())
+    assert "don't want to guess" in response.json()["answer"]
+
+
+def test_prior_assistant_claim_is_not_added_to_retrieval_query(monkeypatch) -> None:
+    reset_limiters()
+    captured = []
+
+    def fake_retrieve(query: str, **_kwargs):
+        captured.append(query)
+        return []
+
+    monkeypatch.setattr(api, "retrieve", fake_retrieve)
+    request_payload = payload(query="Is that true?")
+    request_payload["history"] = [
+        {"role": "user", "content": FIRST_PROMPT},
+        {
+            "role": "assistant",
+            "content": "The mentorship includes two courses of your choice.",
+        },
+    ]
+
+    response = client.post("/api/helper/chat", json=request_payload, headers=HEADERS)
+
+    assert response.status_code == 200
+    assert captured
+    assert "two courses" not in captured[0].lower()
+    assert response.json()["status"] == "insufficient_evidence"
+
+
+def test_substantive_followup_drops_starter_prompt_from_retrieval(monkeypatch) -> None:
+    reset_limiters()
+    captured: list[str] = []
+    query = "Does LLM Fundamentals include community support and an AI tutor?"
+
+    def fake_retrieve(retrieval_query: str, **_kwargs):
+        captured.append(retrieval_query)
+        return []
+
+    monkeypatch.setattr(api, "retrieve", fake_retrieve)
+    request_payload = payload(query=query)
+    request_payload["history"] = [{"role": "user", "content": FIRST_PROMPT}]
+
+    response = client.post("/api/helper/chat", json=request_payload, headers=HEADERS)
+
+    assert response.status_code == 200
+    assert captured == [query]
+    assert_contact_handoff(response.json())
+
+
+def test_short_referential_followup_uses_only_last_substantive_user_turn(
+    monkeypatch,
+) -> None:
+    reset_limiters()
+    captured: list[str] = []
+
+    def fake_retrieve(retrieval_query: str, **_kwargs):
+        captured.append(retrieval_query)
+        return []
+
+    monkeypatch.setattr(api, "retrieve", fake_retrieve)
+    request_payload = payload(query="What is its price?")
+    request_payload["history"] = [
+        {"role": "user", "content": FIRST_PROMPT},
+        {"role": "user", "content": "Tell me about Agent Engineering."},
+        {
+            "role": "assistant",
+            "content": "Unverified claim: it costs $1 and guarantees a job.",
+        },
+    ]
+
+    response = client.post("/api/helper/chat", json=request_payload, headers=HEADERS)
+
+    assert response.status_code == 200
+    assert captured == ["Tell me about Agent Engineering.\nWhat is its price?"]
+    assert "$1" not in captured[0]
+    assert_contact_handoff(response.json())
+
+
+def test_direct_contact_requests_bypass_retrieval_and_model(monkeypatch) -> None:
+    reset_limiters()
+
+    def unexpected(*_args, **_kwargs):
+        raise AssertionError("direct contact intent must not use retrieval or a model")
+
+    monkeypatch.setattr(api, "retrieve", unexpected)
+    monkeypatch.setattr(api.llm, "generate_grounded_answer", unexpected)
+    request_payload = payload(query="Where is your contact form?")
+    request_payload["history"] = [{"role": "user", "content": FIRST_PROMPT}]
+
+    response = client.post("/api/helper/chat", json=request_payload, headers=HEADERS)
+
+    assert response.status_code == 200
+    assert_contact_handoff(response.json(), status="policy")
+
+
+@pytest.mark.parametrize(
+    "query",
+    [
+        "Does Agent Engineering free preview include lifetime access and require no card?",
+        "Does the Full Stack free preview give me a certificate?",
+        "Does Building LLMs for Production have a refund guarantee?",
+        "How long is Agent Engineering?",
+        "How many hours does Python for AI Engineering take?",
+        "How many lessons are in LLM Fundamentals?",
+        "Does mentorship include a certificate?",
+        "Does the Get It All bundle get 50% off?",
+        "What is the Get It All bundle price?",
+        "Does monthly mentorship have a discount?",
+    ],
+)
+def test_absent_offer_facts_go_directly_to_contact_form(
+    monkeypatch, query: str
+) -> None:
+    reset_limiters()
+
+    def unexpected(*_args, **_kwargs):
+        raise AssertionError("model must not run without target-qualified evidence")
+
+    monkeypatch.setattr(api.llm, "generate_grounded_answer", unexpected)
+    request_payload = payload(query=query)
+    request_payload["history"] = [{"role": "user", "content": FIRST_PROMPT}]
+
+    response = client.post("/api/helper/chat", json=request_payload, headers=HEADERS)
+
+    assert response.status_code == 200
+    assert_contact_handoff(response.json())
+
+
 def test_coupon_answer_is_deterministic_and_does_not_call_model(monkeypatch) -> None:
     reset_limiters()
 
-    def unexpected_generate_answer(_prompt: str) -> LLMResult:
+    def unexpected_generate_answer(
+        _prompt: str, _selected: list[dict], **_kwargs
+    ) -> GroundingResult:
         raise AssertionError("model should not be called for coupon intent")
 
-    monkeypatch.setattr(api.llm, "generate_answer", unexpected_generate_answer)
+    monkeypatch.setattr(api.llm, "generate_grounded_answer", unexpected_generate_answer)
     first = payload(query="Do you have a coupon code?")
     first["selectedPrompt"] = FIRST_PROMPT
     first["history"] = [{"role": "user", "content": FIRST_PROMPT}]
@@ -131,9 +637,10 @@ def test_coupon_answer_is_deterministic_and_does_not_call_model(monkeypatch) ->
     second_response = client.post("/api/helper/chat", json=second, headers=HEADERS)
 
     assert first_response.status_code == 200
-    assert "Get it all bundle" in first_response.json()["answer"]
+    assert api.CONTACT_FORM_URL in first_response.json()["answer"]
+    assert first_response.json()["status"] == "policy"
     assert second_response.status_code == 200
-    assert "louis@towardsai.net" in second_response.json()["answer"]
+    assert api.CONTACT_FORM_URL in second_response.json()["answer"]
 
 
 def test_rate_limit_is_hard(monkeypatch) -> None:
@@ -145,8 +652,22 @@ def test_rate_limit_is_hard(monkeypatch) -> None:
 
     monkeypatch.setattr(
         api.llm,
-        "generate_answer",
-        lambda _prompt: LLMResult(answer="ok"),
+        "generate_grounded_answer",
+        lambda _prompt, selected, **_kwargs: GroundingResult(
+            valid=True,
+            status="answered",
+            answer="Supported answer.",
+            cited_chunks=(
+                EvidenceChunk(
+                    chunk_id=str(selected[0]["chunk_id"]),
+                    title=str(selected[0]["title"]),
+                    url=str(selected[0]["url"]),
+                    kind=str(selected[0]["kind"]),
+                    headings=(),
+                    text=str(selected[0]["text"]),
+                ),
+            ),
+        ),
     )
 
     first = client.post("/api/helper/chat", json=payload(), headers=HEADERS)

tests/test_catalog.py

@@ -1,5 +1,7 @@
 from __future__ import annotations
 
+import pytest
+
 from tai_helper.catalog import (
     allowed_paths_by_host,
     coupon_followup,
@@ -7,10 +9,102 @@ from tai_helper.catalog import (
     forced_prompts,
     in_scope,
     page_is_allowed,
+    pages,
     retrieve,
     sources_from_pages,
 )
 
+CANONICAL_OFFER_RETRIEVAL_CASES = (
+    (
+        "Full Stack AI Engineering production LLM product course",
+        "https://towardsai.com/academy/full-stack-ai-engineering/",
+    ),
+    (
+        "Agent Engineering production AI agents course",
+        "https://towardsai.com/academy/agent-engineering/",
+    ),
+    (
+        "10-Hour LLM Fundamentals video crash course",
+        "https://towardsai.com/academy/llm-primer/",
+    ),
+    (
+        "Beginner Python for AI Engineering LLM-native course",
+        "https://towardsai.com/academy/python-for-ai-engineering/",
+    ),
+    (
+        "Master AI for Work no-code professionals course",
+        "https://towardsai.com/academy/ai-for-work/",
+    ),
+    (
+        "Building LLMs for Production ebook taught as a course",
+        "https://towardsai.com/academy/building-llms-for-production/",
+    ),
+    (
+        "Get It All every course one bundle",
+        "https://towardsai.com/academy/bundles/get-it-all/",
+    ),
+    (
+        "From Non-Coder to AI Engineer course bundle",
+        "https://towardsai.com/academy/bundles/from-coding-novice-to-advanced-llm-developer/",
+    ),
+    (
+        "From Developer to Advanced AI Engineer course bundle",
+        "https://towardsai.com/academy/bundles/10-hour-crash-course-into-llm-developer-expert/",
+    ),
+    (
+        "Towards AI Mentorship senior engineers on call",
+        "https://towardsai.com/academy/mentorship/",
+    ),
+    (
+        "Agent Engineering course preview 7 free lessons",
+        "https://towardsai.com/academy/agent-engineering-free-preview/",
+    ),
+    (
+        "Full Stack AI Engineering free preview lessons",
+        "https://towardsai.com/academy/full-stack-ai-engineering-free-preview/",
+    ),
+    (
+        "Agentic AI engineering webinar",
+        "https://towardsai.com/webinars/agentengineering/",
+    ),
+    (
+        "Towards AI free Building LLMs book resources and learning toolkit",
+        "https://towardsai.com/academy/book/",
+    ),
+    (
+        "Towards AI ultimate agents cheatsheet digital download",
+        "https://academy.towardsai.net/products/digital_downloads/agents-cheatsheet",
+    ),
+    (
+        "Anti-slop framework digital download",
+        "https://academy.towardsai.net/products/digital_downloads/anti-slop-framework",
+    ),
+    (
+        "Building AI for Production book resources and links",
+        "https://towardsai.com/academy/book/",
+    ),
+    (
+        "The AI Taste Gap",
+        "https://towardsai.com/theaitastegap/",
+    ),
+    (
+        "Claude Code Codex one-day agentic developer bootcamp team training",
+        "https://towardsai.com/enterprise/agentic-developer-conversion/",
+    ),
+    (
+        "Convert software developers into AI engineers enterprise bootcamp",
+        "https://towardsai.com/enterprise/software-developer-to-ai-engineer/",
+    ),
+    (
+        "enterprise enablement academy",
+        "https://towardsai.com/enterpriseenablement/",
+    ),
+    (
+        "Custom AI development deployment private equity value creation consulting",
+        "https://towardsai.com/valuecreation/",
+    ),
+)
+
 
 def test_forced_prompts_match_expected_options() -> None:
     assert forced_prompts() == [
@@ -26,11 +120,17 @@ def test_allowed_public_pages_include_sitemap_urls_and_exclude_private_paths() -
     assert page_is_allowed("https://academy.towardsai.net/courses/agent-engineering")
     assert page_is_allowed("https://towardsai.net/b2b")
     assert page_is_allowed("https://www.towardsai.net/b2b")
+    assert page_is_allowed("https://towardsai.com/academy/full-stack-ai-engineering/")
+    assert page_is_allowed("https://towardsai.com/academy/mentorship/")
+    assert page_is_allowed("https://www.towardsai.com/a-future-public-page/")
     assert not page_is_allowed(
         "https://academy.towardsai.net/courses/take/agent-engineering/lessons/x"
     )
     assert not page_is_allowed("https://academy.towardsai.net/enroll/123")
+    assert not page_is_allowed("https://towardsai.com/wp-admin/edit.php")
+    assert not page_is_allowed("https://towardsai.com/academy/?preview=true")
     assert "/b2b" in allowed_paths_by_host()["www.towardsai.net"]
+    assert "/academy/mentorship" in allowed_paths_by_host()["www.towardsai.com"]
 
 
 def test_retrieval_routes_mentorship_b2b_and_bundle_queries() -> None:
@@ -38,9 +138,93 @@ def test_retrieval_routes_mentorship_b2b_and_bundle_queries() -> None:
     b2b = sources_from_pages(retrieve("I want training inside my company"))
     bundle = sources_from_pages(retrieve("What is the best value bundle?"))
 
-    assert any("tai-mentorship" in source["url"] for source in mentorship)
-    assert any("towardsai.net/b2b" in source["url"] for source in b2b)
-    assert any("get-it-all" in source["url"] for source in bundle)
+    assert any(
+        "towardsai.com/academy/mentorship" in source["url"] for source in mentorship
+    )
+    assert any(source["url"].startswith("https://towardsai.com/") for source in b2b)
+    assert any(
+        source["url"] == "https://towardsai.com/academy/bundles/get-it-all/"
+        for source in bundle
+    )
+
+
+def test_retrieval_routes_specialized_enterprise_queries() -> None:
+    coding_agents = sources_from_pages(
+        retrieve("We need Claude Code and Codex training for our engineering team")
+    )
+    developer_conversion = sources_from_pages(
+        retrieve("Convert our software developers into AI engineers")
+    )
+    consulting = sources_from_pages(
+        retrieve("We need AI deployment and value creation consulting")
+    )
+
+    assert coding_agents[0]["url"].endswith("/agentic-developer-conversion/")
+    assert developer_conversion[0]["url"].endswith(
+        "/software-developer-to-ai-engineer/"
+    )
+    assert consulting[0]["url"] == "https://towardsai.com/valuecreation/"
+
+
+@pytest.mark.parametrize(
+    ("query", "expected_url"),
+    CANONICAL_OFFER_RETRIEVAL_CASES,
+    ids=[
+        expected_url.rstrip("/").rsplit("/", 1)[-1]
+        for _, expected_url in CANONICAL_OFFER_RETRIEVAL_CASES
+    ],
+)
+def test_every_current_offer_routes_to_its_canonical_page(
+    query: str, expected_url: str
+) -> None:
+    selected = retrieve(query, limit=1)
+
+    assert selected
+    assert selected[0]["url"] == expected_url
+
+
+def test_current_mentorship_offer_replaces_stale_membership_claim() -> None:
+    mentorship_pages = [
+        page
+        for page in pages()
+        if page.get("url") == "https://towardsai.com/academy/mentorship/"
+    ]
+
+    assert len(mentorship_pages) == 1
+    mentorship = mentorship_pages[0]
+    searchable_text = " ".join(
+        [
+            str(mentorship.get("meta_description", "")),
+            " ".join(str(item) for item in mentorship.get("headings", [])),
+            str(mentorship.get("text", "")),
+        ]
+    )
+    assert "10-Hour LLM Fundamentals" in searchable_text
+    assert "Two courses included" not in searchable_text
+    chunks = mentorship.get("chunks", [])
+    assert chunks
+    assert all(len(str(chunk.get("text", ""))) <= 2400 for chunk in chunks)
+    offer_chunks = [
+        str(chunk.get("text", ""))
+        for chunk in chunks
+        if chunk.get("heading") == "The curriculum comes with the team"
+    ]
+    assert len(offer_chunks) == 1
+    offer_table = offer_chunks[0]
+    assert offer_table.count("Included from day one") == 1
+    assert (
+        "10-Hour LLM Fundamentals video course Five in-depth 2-hour video "
+        "sessions, from a basic prompt to a full production rollout $199 "
+        "Included from day one"
+    ) in offer_table
+    assert (
+        "Full Stack AI Engineering · Agent Engineering · Master AI for Work "
+        "$349–499 each 25% off, always"
+    ) in offer_table
+    assert not any(
+        page.get("url") == "https://towardsai.com/academy/membership/"
+        for page in pages()
+    )
 
 
 def test_scope_and_coupon_detection() -> None:

tests/test_catalog_builder.py

@@ -0,0 +1,487 @@
+from __future__ import annotations
+
+import requests
+
+from scripts.build_towardsai_com_catalog import (
+    ACADEMY_EXCLUSIONS,
+    ACADEMY_HOSTS,
+    ACADEMY_SITEMAP_URL,
+    COM_HOSTS,
+    PageParser,
+    build_catalog,
+    build_catalogs,
+    build_chunks,
+    discover_sitemap_entries,
+    fetch_page,
+)
+
+
+class FakeResponse:
+    def __init__(
+        self,
+        url: str,
+        text: str,
+        *,
+        status_code: int = 200,
+    ) -> None:
+        self.url = url
+        self.text = text
+        self.status_code = status_code
+
+    def raise_for_status(self) -> None:
+        if self.status_code >= 400:
+            raise requests.HTTPError(f"HTTP {self.status_code} for {self.url}")
+
+
+class FakeSession:
+    def __init__(self, routes: dict[str, FakeResponse]) -> None:
+        self.routes = routes
+        self.headers: dict[str, str] = {}
+        self.calls: list[str] = []
+
+    def get(self, url: str, *, timeout: int) -> FakeResponse:
+        assert timeout == 30
+        self.calls.append(url)
+        if url not in self.routes:
+            raise AssertionError(f"unexpected network request: {url}")
+        return self.routes[url]
+
+
+def sitemap(*urls: str) -> str:
+    rows = "".join(f"<url><loc>{url}</loc></url>" for url in urls)
+    return (
+        '<?xml version="1.0"?>'
+        '<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">'
+        f"{rows}</urlset>"
+    )
+
+
+def page_html(title: str, body: str, *, canonical: str = "") -> str:
+    canonical_tag = (
+        f'<link href="{canonical}" rel="alternate canonical">' if canonical else ""
+    )
+    return f"""
+        <html>
+          <head>
+            <title>{title}</title>
+            <meta name="description" content="A fetched description">
+            {canonical_tag}
+          </head>
+          <body>
+            <nav>Navigation misinformation</nav>
+            <main><h1>{title}</h1><p>{body}</p></main>
+            <footer>Footer misinformation</footer>
+            <script>Script misinformation</script>
+            <style>.misinformation {{ display: block }}</style>
+          </body>
+        </html>
+    """
+
+
+def test_parser_suppresses_chrome_and_extracts_canonical_hints() -> None:
+    parser = PageParser("https://towardsai.com/original/")
+    parser.feed(
+        """
+        <html><head>
+          <title>Course title</title>
+          <link rel="canonical" href="/academy/course/">
+          <meta http-equiv="Refresh" content="0; URL='/moved/'">
+        </head><body>
+          <header class="header"><a>Skip to main content</a><button>Toggle menu</button></header>
+          <header class="ta-hero"><h1>Current offer</h1><p>60+ Hrs</p></header>
+          <div class="ta-announce">New: Towards AI Mentorship</div>
+          <div class="navbar"><p>Bad navigation claim</p></div>
+          <div class="ta-mobile-menu">Duplicated mobile course links</div>
+          <h1>Course facts</h1><p>Only retrieved facts survive.</p>
+          <div role="contentinfo">Bad footer claim</div>
+          <script>Bad script claim</script><style>Bad style claim</style>
+        </body></html>
+        """
+    )
+    parser.close()
+
+    extracted = " ".join(section["text"] for section in parser.sections)
+    assert parser.canonical_url == "/academy/course/"
+    assert parser.meta_refresh_url == "/moved/"
+    assert parser.headings == ["Current offer", "Course facts"]
+    assert extracted == "60+ Hrs Only retrieved facts survive."
+    assert "Bad" not in extracted
+    assert "Mentorship" not in extracted
+    assert "Toggle menu" not in extracted
+
+
+def test_sitemap_discovery_uses_fake_http_and_filters_foreign_hosts() -> None:
+    sitemap_url = "https://towardsai.com/pages-sitemap.xml"
+    session = FakeSession(
+        {
+            sitemap_url: FakeResponse(
+                sitemap_url,
+                sitemap(
+                    "https://towardsai.com/new-page/",
+                    "https://towardsai.com/new-page/",
+                    "https://untrusted.example/poison",
+                ),
+            )
+        }
+    )
+
+    entries = discover_sitemap_entries(session, sitemap_url, allowed_hosts=COM_HOSTS)
+
+    assert [entry["url"] for entry in entries] == ["https://towardsai.com/new-page/"]
+    assert session.calls == [sitemap_url]
+
+
+def test_fetch_resolves_http_canonical_and_meta_refresh_without_summary() -> None:
+    old_url = "https://towardsai.com/old/"
+    http_final = "https://towardsai.com/http-final/"
+    canonical = "https://towardsai.com/canonical/"
+    refresh_start = "https://academy.towardsai.net/refresh"
+    refresh_target = "https://academy.towardsai.net/final"
+    session = FakeSession(
+        {
+            old_url: FakeResponse(
+                http_final,
+                page_html(
+                    "Canonical offer", "Retrieved offer facts.", canonical=canonical
+                ),
+            ),
+            refresh_start: FakeResponse(
+                refresh_start,
+                '<meta http-equiv="refresh" content="0; url=/final">',
+            ),
+            refresh_target: FakeResponse(
+                refresh_target,
+                page_html("Final Academy page", "Academy evidence."),
+            ),
+        }
+    )
+
+    page = fetch_page(
+        session,
+        {
+            "url": old_url,
+            "summary": "HANDWRITTEN FALSE CLAIM",
+            "kind": "course",
+        },
+        allowed_hosts=COM_HOSTS,
+        fetched_at="2026-08-08T00:00:00+00:00",
+    )
+    refreshed = fetch_page(
+        session,
+        {"url": refresh_start},
+        authority="official_academy",
+        allowed_hosts=ACADEMY_HOSTS,
+        fetched_at="2026-08-08T00:00:00+00:00",
+    )
+
+    assert page["discovered_url"] == old_url
+    assert page["url"] == canonical
+    assert page["canonical_url"] == canonical
+    assert page["redirect_chain"] == [http_final]
+    assert "Retrieved offer facts." in page["text"]
+    assert "HANDWRITTEN FALSE CLAIM" not in page["text"]
+    assert "Navigation misinformation" not in page["text"]
+    assert refreshed["canonical_url"] == refresh_target
+    assert refreshed["redirect_chain"] == [refresh_start, refresh_target]
+
+
+def test_chunks_are_heading_aware_bounded_and_stable() -> None:
+    sections = [
+        {
+            "heading": "Mentorship access",
+            "text": "evidence " * 350,
+        }
+    ]
+    changed_sections = [
+        {
+            "heading": "Mentorship access",
+            "text": ("evidence " * 349) + "updated",
+        }
+    ]
+
+    first = build_chunks(sections, "https://towardsai.com/academy/mentorship/")
+    second = build_chunks(sections, "https://towardsai.com/academy/mentorship/")
+    changed = build_chunks(
+        changed_sections, "https://towardsai.com/academy/mentorship/"
+    )
+
+    assert len(first) == 2
+    assert [chunk["chunk_id"] for chunk in first] == [
+        chunk["chunk_id"] for chunk in second
+    ]
+    assert [chunk["chunk_id"] for chunk in first] == [
+        chunk["chunk_id"] for chunk in changed
+    ]
+    assert all(chunk["heading"] == "Mentorship access" for chunk in first)
+    assert all(1_200 <= len(chunk["text"]) <= 1_800 for chunk in first)
+
+
+def test_table_rows_become_atomic_evidence_spans() -> None:
+    parser = PageParser("https://towardsai.com/academy/mentorship/")
+    parser.feed(
+        """
+        <main>
+          <h2>The curriculum comes with the team</h2>
+          <table><tbody>
+            <tr><th>10-Hour LLM Fundamentals course</th><td>$199</td>
+                <td>Included from day one</td></tr>
+            <tr><th>Full Stack AI Engineering</th><td>$349</td>
+                <td>25% off, always</td></tr>
+          </tbody></table>
+        </main>
+        """
+    )
+    parser.close()
+
+    built = build_chunks(parser.sections, parser.base_url)
+    spans = {span["text"] for chunk in built for span in chunk["evidence_spans"]}
+
+    assert "10-Hour LLM Fundamentals course $199 Included from day one" in spans
+    assert "Full Stack AI Engineering $349 25% off, always" in spans
+    assert "Included from day one" not in spans
+    assert "25% off, always" not in spans
+
+
+def test_accordion_uses_only_the_answer_as_an_atomic_evidence_span() -> None:
+    built = build_chunks(
+        [
+            {
+                "heading": "Frequently asked questions",
+                "text": (
+                    "How long does this course take to complete? + "
+                    "It is self-paced; the average completion time is 10 hours "
+                    "across five 2-hour sessions."
+                ),
+                "spans": [
+                    (
+                        "How long does this course take to complete? + "
+                        "It is self-paced; the average completion time is 10 hours "
+                        "across five 2-hour sessions."
+                    )
+                ],
+            }
+        ],
+        "https://towardsai.com/academy/llm-primer/",
+    )
+    spans = {span["text"] for chunk in built for span in chunk["evidence_spans"]}
+
+    assert "How long does this course take to complete?" not in spans
+    assert (
+        "It is self-paced; the average completion time is 10 hours across five "
+        "2-hour sessions."
+    ) in spans
+    assert not any("? +" in span for span in spans)
+
+
+def test_build_both_catalogs_records_exclusions_manual_entries_and_identity() -> None:
+    fetched_at = "2026-08-08T12:00:00+00:00"
+    com_sitemap = "https://towardsai.com/pages-sitemap.xml"
+    academy_sitemap = "https://academy.towardsai.net/sitemap.xml"
+    mentorship = "https://towardsai.com/academy/mentorship/"
+    academy_mentorship = "https://academy.towardsai.net/bundles/tai-mentorship"
+    placeholder = "https://academy.towardsai.net/pages/webinar"
+    session = FakeSession(
+        {
+            com_sitemap: FakeResponse(com_sitemap, sitemap(mentorship)),
+            academy_sitemap: FakeResponse(
+                academy_sitemap, sitemap(academy_mentorship, placeholder)
+            ),
+            mentorship: FakeResponse(
+                mentorship,
+                page_html(
+                    "Mentorship",
+                    "The retrieved page defines the included course access.",
+                ),
+            ),
+            academy_mentorship: FakeResponse(
+                academy_mentorship,
+                page_html("Mentorship checkout", "Thinkific purchase facts."),
+            ),
+            placeholder: FakeResponse(
+                placeholder,
+                page_html("Webinar", "Add a concise subheading about your product."),
+            ),
+        }
+    )
+
+    com, academy = build_catalogs(session=session, fetched_at=fetched_at)
+
+    assert [page["discovered_url"] for page in com["pages"]] == [mentorship]
+    assert len(academy["pages"]) == 4
+    com_offer = com["pages"][0]
+    academy_offer = next(
+        page
+        for page in academy["pages"]
+        if page["discovered_url"] == academy_mentorship
+    )
+    excluded = next(
+        page for page in academy["pages"] if page["discovered_url"] == placeholder
+    )
+    manual = [
+        page for page in academy["pages"] if page["authority"] == "curated_external"
+    ]
+
+    assert com_offer["offer_id"] == academy_offer["offer_id"] == "mentorship"
+    assert com_offer["entity_id"] == academy_offer["entity_id"]
+    assert excluded["status"] == "excluded"
+    assert excluded["retrieval_eligible"] is False
+    assert excluded["excluded_reason"] == "unfinished template content"
+    assert placeholder in session.calls  # excluded URLs are still fetched and hashed
+    assert len(manual) == 2
+    assert all(page["status"] == "excluded" for page in manual)
+    assert all(page["retrieval_eligible"] is False for page in manual)
+    assert all(page["chunks"] == [] for page in manual)
+
+    required = {
+        "canonical_url",
+        "fetched_at",
+        "content_sha256",
+        "content_hash",
+        "evidence_hash",
+        "authority",
+        "status",
+        "retrieval_eligible",
+        "chunks",
+        "entity_id",
+        "offer_id",
+    }
+    assert all(required <= page.keys() for page in [*com["pages"], *academy["pages"]])
+    assert all(page["fetched_at"] == fetched_at for page in com["pages"])
+
+
+def test_failed_page_is_recorded_but_never_retrieval_eligible() -> None:
+    sitemap_url = "https://towardsai.com/pages-sitemap.xml"
+    broken_url = "https://towardsai.com/broken/"
+    session = FakeSession(
+        {
+            sitemap_url: FakeResponse(sitemap_url, sitemap(broken_url)),
+            broken_url: FakeResponse(broken_url, "upstream failed", status_code=503),
+        }
+    )
+
+    catalog = build_catalog(
+        session=session,
+        sitemap_url=sitemap_url,
+        fetched_at="2026-08-08T00:00:00+00:00",
+    )
+
+    assert len(catalog["pages"]) == 1
+    page = catalog["pages"][0]
+    assert page["status"] == "fetch_error"
+    assert page["retrieval_eligible"] is False
+    assert page["chunks"] == []
+    assert page["content_sha256"] == page["content_hash"]
+    assert len(page["evidence_hash"]) == 64
+    assert catalog["status_counts"] == {"fetch_error": 1}
+
+
+def test_stale_academy_summaries_are_fetched_but_excluded_from_retrieval() -> None:
+    stale_paths = (
+        "/collections",
+        "/collections/products",
+        "/collections/developers",
+        "/collections/professionals",
+        "/pages/free-resources",
+    )
+    urls = tuple(f"https://academy.towardsai.net{path}" for path in stale_paths)
+    routes = {
+        ACADEMY_SITEMAP_URL: FakeResponse(
+            ACADEMY_SITEMAP_URL,
+            sitemap(*urls),
+        )
+    }
+    routes.update(
+        {
+            url: FakeResponse(
+                url,
+                page_html("Legacy Academy summary", f"Stale content for {path}."),
+            )
+            for url, path in zip(urls, stale_paths, strict=True)
+        }
+    )
+    session = FakeSession(routes)
+
+    catalog = build_catalog(
+        session=session,
+        sitemap_url=ACADEMY_SITEMAP_URL,
+        authority="official_academy",
+        allowed_hosts=ACADEMY_HOSTS,
+        fetched_at="2026-08-08T00:00:00+00:00",
+    )
+
+    pages_by_path = {page["path"]: page for page in catalog["pages"]}
+    assert set(pages_by_path) == set(stale_paths)
+    for path in stale_paths:
+        page = pages_by_path[path]
+        assert page["status"] == "excluded"
+        assert page["retrieval_eligible"] is False
+        assert page["excluded_reason"] == ACADEMY_EXCLUSIONS[path]
+        assert page["text"]
+        assert page["content_hash"] != ""
+        assert f"https://academy.towardsai.net{path}" in session.calls
+
+
+def test_parser_excludes_reviews_comparisons_and_simulated_conversations() -> None:
+    parser = PageParser("https://towardsai.com/academy/llm-primer/")
+    parser.feed(
+        """
+        <main>
+          <h2>Official duration</h2><p>Average completion time is 10 hours.</p>
+          <article class="ta-review"><p>The course actually has 12 hours.</p></article>
+          <div class="ta-ment-vswrap"><p>A competitor costs $500.</p></div>
+          <p class="ta-ment-vsnote">A single mentor costs $150 monthly.</p>
+          <div class="ta-ment-thread"><p>An example user says guaranteed.</p></div>
+        </main>
+        """
+    )
+    parser.close()
+    text = " ".join(section["text"] for section in parser.sections)
+
+    assert "Average completion time is 10 hours" in text
+    assert "12 hours" not in text
+    assert "competitor costs" not in text
+    assert "single mentor costs" not in text
+    assert "example user" not in text
+
+
+def test_parser_excludes_paid_course_outcomes_from_preview_evidence() -> None:
+    parser = PageParser(
+        "https://towardsai.com/academy/full-stack-ai-engineering-free-preview/"
+    )
+    parser.feed(
+        """
+        <main>
+          <section id="preview"><h2>Free preview</h2><p>No card required.</p></section>
+          <section id="outcomes">
+            <h2>The full course outcomes</h2>
+            <p>A certification that unlocks six-figure roles.</p>
+          </section>
+          <section id="cta"><p>Buy the paid course.</p></section>
+        </main>
+        """
+    )
+    parser.close()
+    text = " ".join(section["text"] for section in parser.sections)
+
+    assert "No card required" in text
+    assert "certification" not in text
+    assert "paid course" not in text
+
+
+def test_mentorship_comparison_keeps_own_features_but_drops_competitors() -> None:
+    parser = PageParser("https://towardsai.com/academy/mentorship/")
+    parser.feed(
+        """
+        <main><section id="compare">
+          <div class="inc"><p>Resume &amp; project reviews in 48–72h</p></div>
+          <div class="board"><p>A single mentor costs $150/month.</p></div>
+          <p class="sum">Bought separately: $2,000+ a month.</p>
+        </section></main>
+        """
+    )
+    parser.close()
+    text = " ".join(section["text"] for section in parser.sections)
+
+    assert "Resume & project reviews in 48–72h" in text
+    assert "single mentor" not in text
+    assert "Bought separately" not in text

tests/test_grounding.py

@@ -0,0 +1,866 @@
+from __future__ import annotations
+
+import json
+
+import pytest
+
+from tai_helper import llm
+
+MENTORSHIP_PAGE = {
+    "title": "Towards AI Mentorship",
+    "kind": "mentorship",
+    "url": "https://towardsai.com/academy/mentorship/",
+    "headings": ["Everything Inside the Mentorship"],
+    "text": (
+        "The Mentorship program includes one course: the 10-Hour LLM "
+        "Fundamentals course. Members also receive two live sessions every week."
+    ),
+}
+
+
+def _model_json(*, text: str, quote: str, chunk_id: str | None = None) -> str:
+    return json.dumps(
+        {
+            "status": "answered",
+            "claims": [
+                {
+                    "text": text,
+                    "chunk_id": chunk_id or llm.chunk_id_for_page(MENTORSHIP_PAGE),
+                    "quote": quote,
+                }
+            ],
+        }
+    )
+
+
+def _with_spans(page: dict, *spans: str) -> dict:
+    return {
+        **page,
+        "evidence_spans": [
+            {"span_id": f"{page['chunk_id']}:span-{index}", "text": span}
+            for index, span in enumerate(spans)
+        ],
+    }
+
+
+def test_prompt_uses_stable_chunk_ids_and_marks_notes_as_non_evidence() -> None:
+    prompt = llm.build_prompt(
+        query="Does mentorship include a course?",
+        selected_prompt="I want to find mentors",
+        current_url="https://towardsai.com/academy/mentorship/",
+        page_title="Towards AI Mentorship",
+        history=[("assistant", "Earlier unverified answer: two courses")],
+        selected_pages=[MENTORSHIP_PAGE],
+    )
+    chunk_id = llm.chunk_id_for_page(MENTORSHIP_PAGE)
+
+    assert llm.chunk_id_for_page(dict(MENTORSHIP_PAGE)) == chunk_id
+    assert f'<EVIDENCE_CHUNK chunk_id="{chunk_id}">' in prompt
+    assert "<NON_EVIDENCE_ROUTING_NOTES>" in prompt
+    assert "must not be cited" in prompt
+    assert "visitor-provided and NOT evidence" in prompt
+    assert "return not_found" in prompt
+
+
+def test_valid_mentorship_claim_returns_only_validated_text_and_cited_chunk() -> None:
+    sentence = (
+        "The Mentorship program includes one course: the 10-Hour LLM "
+        "Fundamentals course."
+    )
+    raw = llm.LLMResult(
+        answer=_model_json(text=sentence, quote=sentence),
+        usage={"provider": "deepseek"},
+        latency_ms=42,
+    )
+
+    result = llm.validate_grounded_result(raw, [MENTORSHIP_PAGE])
+
+    assert result.valid
+    assert result.is_answered
+    assert result.status == "answered"
+    assert result.answer == sentence
+    assert result.cited_chunk_ids == (llm.chunk_id_for_page(MENTORSHIP_PAGE),)
+    assert result.cited_chunks[0].url == MENTORSHIP_PAGE["url"]
+    assert result.usage == {"provider": "deepseek"}
+    assert result.latency_ms == 42
+
+
+def test_valid_unpunctuated_atomic_span_gets_only_a_server_added_period() -> None:
+    quote = "10-Hour LLM Fundamentals course $199 Included from day one"
+    page = _with_spans(
+        {
+            **MENTORSHIP_PAGE,
+            "chunk_id": "mentorship-row",
+            "text": quote,
+        },
+        quote,
+    )
+    result = llm.validate_grounded_result(
+        _model_json(text=quote, quote=quote, chunk_id="mentorship-row"),
+        [page],
+    )
+
+    assert result.valid
+    assert result.answer == f"{quote}."
+    assert result.claims[0].quote == quote
+
+
+def test_mentorship_table_supports_two_separate_correcting_claims() -> None:
+    table_text = (
+        "What Full price As a member 10-Hour LLM Fundamentals video course "
+        "Five in-depth 2-hour video sessions $199 Included from day one "
+        "Full Stack AI Engineering · Agent Engineering · Master AI for Work "
+        "$349–499 each 25% off, always"
+    )
+    included_quote = (
+        "10-Hour LLM Fundamentals video course Five in-depth 2-hour video "
+        "sessions $199 Included from day one"
+    )
+    discount_quote = (
+        "Full Stack AI Engineering · Agent Engineering · Master AI for Work "
+        "$349–499 each 25% off, always"
+    )
+    table_page = _with_spans(
+        {
+            "chunk_id": "mentorship-curriculum-table",
+            "title": "Towards AI Mentorship",
+            "kind": "mentorship",
+            "url": "https://towardsai.com/academy/mentorship/",
+            "headings": ["The curriculum comes with the team"],
+            "text": table_text,
+        },
+        included_quote,
+        discount_quote,
+    )
+    raw = json.dumps(
+        {
+            "status": "answered",
+            "claims": [
+                {
+                    "text": f"{included_quote}.",
+                    "chunk_id": "mentorship-curriculum-table",
+                    "quote": included_quote,
+                },
+                {
+                    "text": f"{discount_quote}.",
+                    "chunk_id": "mentorship-curriculum-table",
+                    "quote": discount_quote,
+                },
+            ],
+        }
+    )
+
+    result = llm.validate_grounded_result(raw, [table_page])
+
+    assert result.valid
+    assert result.is_answered
+    assert len(result.claims) == 2
+    assert "included from day one" in result.answer.lower()
+    assert "25% off, always" in result.answer
+    assert "only" not in result.answer.lower()
+    assert result.cited_chunk_ids == ("mentorship-curriculum-table",)
+
+
+def test_invalid_claim_text_fails_closed_even_with_valid_table_row_quotes() -> None:
+    included_quote = "10-Hour LLM Fundamentals video course $199 Included from day one"
+    discount_quote = (
+        "Full Stack AI Engineering · Agent Engineering · Master AI for Work "
+        "$349–499 each 25% off, always"
+    )
+    table_page = _with_spans(
+        {
+            "chunk_id": "mentorship-table",
+            "title": "Towards AI Mentorship",
+            "kind": "mentorship",
+            "url": "https://towardsai.com/academy/mentorship/",
+            "headings": ["The curriculum comes with the team"],
+            "text": f"{included_quote} {discount_quote}",
+        },
+        included_quote,
+        discount_quote,
+    )
+    raw = json.dumps(
+        {
+            "status": "answered",
+            "claims": [
+                {
+                    "text": (
+                        "The mentorship includes the LLM Fundamentals course, "
+                        "not two courses of your choice."
+                    ),
+                    "chunk_id": "mentorship-table",
+                    "quote": included_quote,
+                },
+                {
+                    "text": (
+                        "Full Stack AI Engineering, Agent Engineering, and Master "
+                        "AI for Work are not included; they are discounted."
+                    ),
+                    "chunk_id": "mentorship-table",
+                    "quote": discount_quote,
+                },
+            ],
+        }
+    )
+
+    result = llm.validate_grounded_result(raw, [table_page])
+
+    assert not result.valid
+    assert result.status == "validation_failure"
+    assert result.answer == ""
+    assert "unsupported number words" in result.validation_error
+
+
+def test_claim_cannot_add_framing_to_an_exact_span() -> None:
+    quote = "10-Hour LLM Fundamentals video course $199 Included from day one"
+    table_page = _with_spans(
+        {
+            "chunk_id": "mentorship-table",
+            "title": "Towards AI Mentorship",
+            "kind": "mentorship",
+            "url": "https://towardsai.com/academy/mentorship/",
+            "headings": ["The curriculum comes with the team"],
+            "text": quote,
+        },
+        quote,
+    )
+    raw = _model_json(
+        text=(
+            "The mentorship includes the 10-Hour LLM Fundamentals video course "
+            "from day one."
+        ),
+        quote=quote,
+        chunk_id="mentorship-table",
+    )
+
+    result = llm.validate_grounded_result(raw, [table_page])
+
+    assert not result.valid
+    assert result.status == "validation_failure"
+    assert result.answer == ""
+    assert "verbatim" in result.validation_error
+
+
+def test_claim_cannot_borrow_framing_from_an_unrelated_chunk() -> None:
+    quote = "10-Hour LLM Fundamentals video course $199 Included from day one"
+    table_page = _with_spans(
+        {
+            "chunk_id": "mentorship-table",
+            "title": "Towards AI Mentorship",
+            "kind": "mentorship",
+            "url": "https://towardsai.com/academy/mentorship/",
+            "headings": ["The curriculum comes with the team"],
+            "text": quote,
+        },
+        quote,
+    )
+    unrelated_page = {
+        "chunk_id": "unrelated-premium-page",
+        "title": "Premium Executive Subscription",
+        "kind": "course",
+        "url": "https://towardsai.com/academy/unrelated/",
+        "headings": ["Guaranteed career accelerator"],
+        "text": "Unrelated material.",
+    }
+    raw = _model_json(
+        text=(
+            "The premium executive subscription includes the 10-Hour LLM "
+            "Fundamentals video course from day one."
+        ),
+        quote=quote,
+        chunk_id="mentorship-table",
+    )
+
+    result = llm.validate_grounded_result(raw, [table_page, unrelated_page])
+
+    assert not result.valid
+    assert result.status == "validation_failure"
+    assert result.answer == ""
+    assert "verbatim" in result.validation_error
+
+
+def test_not_found_is_valid_but_has_no_user_answer_or_citations() -> None:
+    result = llm.validate_grounded_result(
+        '{"status":"not_found","claims":[]}', [MENTORSHIP_PAGE]
+    )
+
+    assert result.valid
+    assert not result.is_answered
+    assert result.status == "not_found"
+    assert result.answer == ""
+    assert result.cited_chunks == ()
+
+
+@pytest.mark.parametrize(
+    "raw",
+    [
+        "The mentorship includes two courses.",
+        '```json\n{"status":"not_found","claims":[]}\n```',
+        '{"status":"answered","claims":[}',
+        '{"status":"not_found","claims":[],"answer":"invented"}',
+        '{"status":"answered","status":"not_found","claims":[]}',
+        '{"status":"answered","claims":[],"confidence":1}',
+    ],
+)
+def test_malformed_or_non_schema_output_fails_closed(raw: str) -> None:
+    result = llm.validate_grounded_result(raw, [MENTORSHIP_PAGE])
+
+    assert not result.valid
+    assert result.status == "validation_failure"
+    assert result.answer == ""
+    assert result.cited_chunks == ()
+    assert result.validation_error
+
+
+def test_invented_quote_fails_closed() -> None:
+    result = llm.validate_grounded_result(
+        _model_json(
+            text="The Mentorship program includes two courses.",
+            quote="The Mentorship program includes two courses.",
+        ),
+        [MENTORSHIP_PAGE],
+    )
+
+    assert result.status == "validation_failure"
+    assert result.answer == ""
+    assert "evidence span" in result.validation_error
+
+
+def test_bad_chunk_id_fails_closed_even_when_quote_exists_elsewhere() -> None:
+    sentence = (
+        "The Mentorship program includes one course: the 10-Hour LLM "
+        "Fundamentals course."
+    )
+    result = llm.validate_grounded_result(
+        _model_json(text=sentence, quote=sentence, chunk_id="chunk_not_retrieved"),
+        [MENTORSHIP_PAGE],
+    )
+
+    assert result.status == "validation_failure"
+    assert result.answer == ""
+    assert "unknown chunk_id" in result.validation_error
+
+
+@pytest.mark.parametrize(
+    ("claim", "quote", "error_fragment"),
+    [
+        (
+            "The Mentorship program includes 2 courses.",
+            "The Mentorship program includes 1 course",
+            "numbers",
+        ),
+        (
+            "The Mentorship program includes two courses.",
+            "The Mentorship program includes one course",
+            "number words",
+        ),
+        (
+            "Members receive 3 live sessions every week.",
+            "Members also receive two live sessions every week.",
+            "numbers",
+        ),
+        (
+            "The yearly plan saves 25%.",
+            "The yearly plan saves 24%",
+            "percentages",
+        ),
+        (
+            "The plan costs $199.",
+            "The plan costs $99",
+            "currency amounts",
+        ),
+        (
+            "Details are at https://towardsai.com/invented.",
+            "Details are at https://towardsai.com/academy/mentorship/",
+            "URLs",
+        ),
+    ],
+)
+def test_unsupported_critical_facts_fail_closed(
+    claim: str, quote: str, error_fragment: str
+) -> None:
+    page = {**MENTORSHIP_PAGE, "text": f"{MENTORSHIP_PAGE['text']} {quote}"}
+    result = llm.validate_grounded_result(
+        _model_json(
+            text=claim,
+            quote=quote,
+            chunk_id=llm.chunk_id_for_page(page),
+        ),
+        [page],
+    )
+
+    assert not result.valid
+    assert result.status == "validation_failure"
+    assert result.answer == ""
+    assert error_fragment in result.validation_error
+
+
+def test_loose_paraphrase_fails_closed() -> None:
+    quote = "Members also receive two live sessions every week."
+    claim = "Experts provide unlimited private coaching and personal hiring referrals."
+    result = llm.validate_grounded_result(
+        _model_json(text=claim, quote=quote), [MENTORSHIP_PAGE]
+    )
+
+    assert not result.valid
+    assert result.status == "validation_failure"
+    assert result.answer == ""
+    assert "verbatim" in result.validation_error
+
+
+def test_changed_negation_fails_closed() -> None:
+    quote = "The Mentorship program does not include two courses."
+    claim = "The Mentorship program does include two courses."
+    page = {**MENTORSHIP_PAGE, "text": quote}
+    result = llm.validate_grounded_result(
+        _model_json(
+            text=claim,
+            quote=quote,
+            chunk_id=llm.chunk_id_for_page(page),
+        ),
+        [page],
+    )
+
+    assert not result.valid
+    assert result.status == "validation_failure"
+    assert result.answer == ""
+    assert "negation" in result.validation_error
+
+
+def test_oversized_exact_quote_fails_closed() -> None:
+    quote = "x" * (llm.MAX_EVIDENCE_QUOTE_CHARS + 1)
+    page = {**MENTORSHIP_PAGE, "text": quote}
+    result = llm.validate_grounded_result(
+        _model_json(
+            text="Unsupported framing.",
+            quote=quote,
+            chunk_id=llm.chunk_id_for_page(page),
+        ),
+        [page],
+    )
+
+    assert not result.valid
+    assert result.status == "validation_failure"
+    assert result.answer == ""
+    assert "exceeds" in result.validation_error
+
+
+@pytest.mark.parametrize(
+    "claim",
+    [
+        "Two courses are included.",
+        "Full Stack AI Engineering is included from day one.",
+        "Agent Engineering is included from day one.",
+    ],
+)
+def test_claim_cannot_recombine_words_from_a_table_chunk(claim: str) -> None:
+    full_table = (
+        "Two live Q&A calls every week Course Full price As a member "
+        "10-Hour LLM Fundamentals video course $199 Included from day one "
+        "Full Stack AI Engineering · Agent Engineering · Master AI for Work "
+        "$349–499 each 25% off, always"
+    )
+    included_row = "10-Hour LLM Fundamentals video course $199 Included from day one"
+    discount_row = (
+        "Full Stack AI Engineering · Agent Engineering · Master AI for Work "
+        "$349–499 each 25% off, always"
+    )
+    page = _with_spans(
+        {**MENTORSHIP_PAGE, "chunk_id": "mentorship-table", "text": full_table},
+        included_row,
+        discount_row,
+    )
+    quote = claim.removesuffix(".")
+    result = llm.validate_grounded_result(
+        _model_json(text=claim, quote=quote, chunk_id="mentorship-table"),
+        [page],
+    )
+
+    assert result.status == "validation_failure"
+    assert result.answer == ""
+    assert "evidence span" in result.validation_error
+
+
+def test_evidence_quote_length_is_bounded() -> None:
+    quote = "A" * (llm.MAX_EVIDENCE_QUOTE_CHARS + 1)
+    page = {**MENTORSHIP_PAGE, "chunk_id": "oversized", "text": quote}
+    result = llm.validate_grounded_result(
+        _model_json(text=f"{quote}.", quote=quote, chunk_id="oversized"),
+        [page],
+    )
+
+    assert result.status == "validation_failure"
+    assert result.answer == ""
+    assert "exceeds" in result.validation_error
+
+
+def test_claim_must_be_a_single_complete_sentence() -> None:
+    quote = (
+        "The Mentorship program includes one course: the 10-Hour LLM "
+        "Fundamentals course. Members also receive two live sessions every week."
+    )
+    result = llm.validate_grounded_result(
+        _model_json(text=quote, quote=quote), [MENTORSHIP_PAGE]
+    )
+
+    assert result.status == "validation_failure"
+    assert "complete server-defined evidence span" in result.validation_error
+
+
+@pytest.mark.parametrize(
+    ("source_span", "unsafe_substring"),
+    [
+        ("No code required: learn by doing, not watching.", "code required"),
+        ("Never guaranteed, always earned.", "guaranteed"),
+        ("Earned, never promised.", "promised"),
+        (
+            "If you cancel the monthly plan, you can upgrade before renewal.",
+            "you can upgrade before renewal",
+        ),
+        ("Pay once $399 $349 one-time Save 12%", "$399"),
+    ],
+)
+def test_arbitrary_substrings_cannot_omit_negation_or_conditions(
+    source_span: str, unsafe_substring: str
+) -> None:
+    page = _with_spans(
+        {
+            **MENTORSHIP_PAGE,
+            "chunk_id": "atomic-source",
+            "text": source_span,
+        },
+        source_span,
+    )
+    result = llm.validate_grounded_result(
+        _model_json(
+            text=f"{unsafe_substring}.",
+            quote=unsafe_substring,
+            chunk_id="atomic-source",
+        ),
+        [page],
+    )
+
+    assert result.status == "validation_failure"
+    assert result.answer == ""
+    assert "complete server-defined evidence span" in result.validation_error
+
+
+def test_generate_grounded_answer_never_returns_raw_unvalidated_text(
+    monkeypatch: pytest.MonkeyPatch,
+) -> None:
+    monkeypatch.setattr(
+        llm,
+        "generate_answer",
+        lambda _prompt: llm.LLMResult("An unsupported provider sentence."),
+    )
+
+    result = llm.generate_grounded_answer("prompt", [MENTORSHIP_PAGE])
+
+    assert result.status == "validation_failure"
+    assert result.answer == ""
+
+
+def _query_bound_result(page: dict, query: str, target_offer_id: str):
+    span = page["evidence_spans"][0]["text"]
+    raw = _model_json(
+        text=span if span.endswith((".", "!", "?")) else f"{span}.",
+        quote=span,
+        chunk_id=page["chunk_id"],
+    )
+    return llm.validate_grounded_result(
+        raw,
+        [page],
+        query=query,
+        target_offer_ids=frozenset({target_offer_id}),
+    )
+
+
+def test_query_binding_rejects_an_exact_span_from_the_wrong_offer() -> None:
+    page = _with_spans(
+        {
+            "chunk_id": "book-community",
+            "title": "Building LLMs for Production",
+            "kind": "book",
+            "offer_id": "building-llms-for-production",
+            "entity_id": "offer:building-llms-for-production",
+            "url": "https://towardsai.com/academy/building-llms-for-production/",
+            "headings": ["Community"],
+            "text": "Community access and our own AI Tutor",
+        },
+        "Community access and our own AI Tutor",
+    )
+
+    result = _query_bound_result(
+        page,
+        "Does LLM Fundamentals include community support and an AI tutor?",
+        "llm-primer",
+    )
+
+    assert result.status == "validation_failure"
+    assert "outside the requested offer boundary" in result.validation_error
+
+
+@pytest.mark.parametrize(
+    ("offer_id", "query", "span"),
+    [
+        (
+            "full-stack-ai-engineering-free-preview",
+            "Does the Full Stack free preview give me a certificate?",
+            "03 A certification that unlocks six-figure roles",
+        ),
+        (
+            "mentorship",
+            "Does the monthly mentorship plan have a 30-day money-back guarantee?",
+            "Join the Mentorship 30-day money-back guarantee",
+        ),
+        (
+            "get-it-all",
+            "What is the Get It All bundle price?",
+            "$1,625 combined price bought separately",
+        ),
+        (
+            "agent-engineering",
+            "Does Agent Engineering include lifetime access?",
+            "Get instant access today",
+        ),
+        (
+            "mentorship",
+            "What is the monthly mentorship price?",
+            "From $75/month",
+        ),
+        (
+            "mentorship",
+            "What is the month-to-month mentorship price?",
+            "From $75/month",
+        ),
+        (
+            "mentorship",
+            "What does mentorship cost each month?",
+            "How is this different from a $150/month mentor?",
+        ),
+        (
+            "mentorship",
+            "What is the monthly mentorship price?",
+            "1× Guest workshop monthly, recorded",
+        ),
+        (
+            "mentorship",
+            "What is the yearly mentorship price?",
+            "Yearly Save 24% · $289 off",
+        ),
+        (
+            "mentorship",
+            "Is the yearly mentorship plan 24% off?",
+            (
+                "Full Stack AI Engineering · Agent Engineering · Master AI for "
+                "Work $349–499 each 25% off, always"
+            ),
+        ),
+        (
+            "mentorship",
+            "Does monthly mentorship have a discount?",
+            (
+                "Full Stack AI Engineering · Agent Engineering · Master AI for "
+                "Work $349–499 each 25% off, always"
+            ),
+        ),
+    ],
+)
+def test_query_binding_rejects_wrong_qualifier_within_the_right_offer(
+    offer_id: str, query: str, span: str
+) -> None:
+    page = _with_spans(
+        {
+            "chunk_id": f"{offer_id}-unsafe-field",
+            "title": offer_id,
+            "kind": "course",
+            "offer_id": offer_id,
+            "entity_id": f"offer:{offer_id}",
+            "url": f"https://towardsai.com/academy/{offer_id}/",
+            "headings": ["Offer"],
+            "text": span,
+        },
+        span,
+    )
+
+    result = _query_bound_result(page, query, offer_id)
+
+    assert result.status == "validation_failure"
+    assert "target-qualified" in result.validation_error
+
+
+@pytest.mark.parametrize(
+    ("offer_id", "query", "span", "targets"),
+    [
+        (
+            "full-stack-ai-engineering",
+            "How many Full Stack AI Engineering lessons can I preview for free?",
+            "Explore the first 6 lessons free.",
+            frozenset(
+                {
+                    "full-stack-ai-engineering",
+                    "full-stack-ai-engineering-free-preview",
+                }
+            ),
+        ),
+        (
+            "llm-primer",
+            "Is LLM Fundamentals 10 or 12 hours long?",
+            (
+                "It is self-paced; the average completion time is 10 hours "
+                "across five 2-hour sessions."
+            ),
+            frozenset({"llm-primer"}),
+        ),
+        (
+            "mentorship",
+            "What is the month-to-month mentorship price?",
+            "$99 /month",
+            frozenset({"mentorship"}),
+        ),
+    ],
+)
+def test_query_binding_accepts_exact_target_qualified_facts(
+    offer_id: str,
+    query: str,
+    span: str,
+    targets: frozenset[str],
+) -> None:
+    page = _with_spans(
+        {
+            "chunk_id": f"{offer_id}-valid-field",
+            "title": offer_id,
+            "kind": "course",
+            "offer_id": offer_id,
+            "entity_id": f"offer:{offer_id}",
+            "url": f"https://towardsai.com/academy/{offer_id}/",
+            "headings": ["Offer"],
+            "text": span,
+        },
+        span,
+    )
+    raw = _model_json(
+        text=span,
+        quote=span,
+        chunk_id=page["chunk_id"],
+    )
+
+    result = llm.validate_grounded_result(
+        raw,
+        [page],
+        query=query,
+        target_offer_ids=targets,
+    )
+
+    assert result.is_answered
+    assert result.answer == (
+        span if span.endswith((".", "!", "?")) else f"{span}."
+    )
+
+
+def test_preview_count_exception_rejects_paid_course_access_claim() -> None:
+    preview_count = "Free preview · 7 full lessons"
+    paid_access = "Unlock Lifetime Access"
+    preview_page = _with_spans(
+        {
+            "chunk_id": "agent-preview-count",
+            "title": "Agent Engineering free preview",
+            "kind": "course_preview",
+            "offer_id": "agent-engineering-free-preview",
+            "entity_id": "offer:agent-engineering-free-preview",
+            "url": (
+                "https://towardsai.com/academy/"
+                "agent-engineering-free-preview/"
+            ),
+            "headings": ["Free preview"],
+            "text": preview_count,
+        },
+        preview_count,
+    )
+    paid_page = _with_spans(
+        {
+            "chunk_id": "agent-paid-access",
+            "title": "Agent Engineering",
+            "kind": "course",
+            "offer_id": "agent-engineering",
+            "entity_id": "offer:agent-engineering",
+            "url": "https://towardsai.com/academy/agent-engineering/",
+            "headings": ["Purchase"],
+            "text": paid_access,
+        },
+        paid_access,
+    )
+    raw = json.dumps(
+        {
+            "status": "answered",
+            "claims": [
+                {
+                    "text": preview_count,
+                    "chunk_id": preview_page["chunk_id"],
+                    "quote": preview_count,
+                },
+                {
+                    "text": paid_access,
+                    "chunk_id": paid_page["chunk_id"],
+                    "quote": paid_access,
+                },
+            ],
+        }
+    )
+    query = (
+        "Do the 7 lessons in the Agent Engineering preview include lifetime access?"
+    )
+
+    result = llm.validate_grounded_result(
+        raw,
+        [preview_page, paid_page],
+        query=query,
+        target_offer_ids=frozenset(
+            {"agent-engineering-free-preview", "agent-engineering"}
+        ),
+    )
+
+    assert result.status == "validation_failure"
+    assert "target-qualified" in result.validation_error
+
+
+@pytest.mark.parametrize(
+    ("query", "span"),
+    [
+        (
+            "Does mentorship include lifetime course access?",
+            (
+                "A senior AI consultant runs $200–500 an hour, with retainers "
+                "from $3,000 a month."
+            ),
+        ),
+        (
+            "Does mentorship include resume and project reviews?",
+            (
+                "A single mentor runs $120–450 a month, with resume reviews "
+                "billed $150–300 each."
+            ),
+        ),
+    ],
+)
+def test_competitor_comparison_spans_are_never_offer_evidence(
+    query: str, span: str
+) -> None:
+    page = _with_spans(
+        {
+            "chunk_id": "mentorship-comparison",
+            "title": "Towards AI Mentorship",
+            "kind": "mentorship",
+            "offer_id": "mentorship",
+            "entity_id": "offer:mentorship",
+            "url": "https://towardsai.com/academy/mentorship/",
+            "headings": ["Comparison"],
+            "text": span,
+        },
+        span,
+    )
+
+    result = _query_bound_result(page, query, "mentorship")
+
+    assert result.status == "validation_failure"
+    assert "competitor comparison" in result.validation_error

tests/test_live_smoke.py

@@ -7,13 +7,16 @@ import pytest
 import requests
 
 LIVE_BASE_URL = os.getenv("LIVE_SPACE_BASE_URL", "").rstrip("/")
+LIVE_HELPER_WIDGET_PATH = "/" + os.getenv(
+    "LIVE_HELPER_WIDGET_PATH", "/helper-widget.js"
+).strip("/")
 RUN_LIVE_CHAT = os.getenv("RUN_LIVE_CHAT_SMOKE", "").lower() in {
     "1",
     "true",
     "yes",
     "on",
 }
-HEADERS = {"Origin": "https://academy.towardsai.net"}
+HEADERS = {"Origin": "https://towardsai.com"}
 FIRST_PROMPT = "I want help deciding which course to take."
 
 
@@ -42,7 +45,9 @@ def test_live_health_widget_and_config() -> None:
     base_url = require_live_base_url()
 
     health, health_seconds = timed_request("GET", f"{base_url}/healthz", timeout=120)
-    widget, widget_seconds = timed_request("GET", f"{base_url}/widget.js", timeout=120)
+    widget, widget_seconds = timed_request(
+        "GET", f"{base_url}{LIVE_HELPER_WIDGET_PATH}", timeout=120
+    )
     config, config_seconds = timed_request(
         "GET", f"{base_url}/api/helper/config", timeout=120
     )
@@ -58,7 +63,14 @@ def test_live_health_widget_and_config() -> None:
     assert config.status_code == 200
     payload = config.json()
     assert FIRST_PROMPT in payload["forcedPrompts"]
-    assert "/courses/agent-engineering" in payload["allowedPathsByHost"]["academy.towardsai.net"]
+    assert "towardsai.com" in payload["siteWideHosts"]
+    assert (
+        "/academy/agent-engineering" in payload["allowedPathsByHost"]["towardsai.com"]
+    )
+    assert (
+        "/courses/agent-engineering"
+        in payload["allowedPathsByHost"]["academy.towardsai.net"]
+    )
     assert config_seconds <= max_seconds("LIVE_SMOKE_MAX_CONFIG_SECONDS", 30)
 
 
@@ -77,7 +89,7 @@ def test_live_chat_returns_concise_answer() -> None:
             "selectedPrompt": FIRST_PROMPT,
             "visitorId": "github-action-smoke",
             "context": {
-                "url": "https://academy.towardsai.net/courses/agent-engineering",
+                "url": "https://towardsai.com/academy/agent-engineering/",
                 "pageTitle": "Agent course",
                 "signedIn": False,
             },
@@ -92,3 +104,53 @@ def test_live_chat_returns_concise_answer() -> None:
     assert len(payload["answer"]) < 1400
     assert payload["sources"]
     assert elapsed <= max_seconds("LIVE_SMOKE_MAX_CHAT_SECONDS", 90)
+
+
+@pytest.mark.live
+@pytest.mark.skipif(
+    not RUN_LIVE_CHAT,
+    reason="RUN_LIVE_CHAT_SMOKE is not enabled",
+)
+def test_live_chat_fails_closed_for_offer_access_and_guarantees() -> None:
+    base_url = require_live_base_url()
+
+    def ask(query: str, visitor_id: str) -> dict:
+        response = requests.post(
+            f"{base_url}/api/helper/chat",
+            json={
+                "query": query,
+                "selectedPrompt": FIRST_PROMPT,
+                "visitorId": visitor_id,
+                "threadId": "",
+                "history": [{"role": "user", "content": FIRST_PROMPT}],
+                "context": {
+                    "url": "https://towardsai.com/academy/mentorship/",
+                    "pageTitle": "Mentorship",
+                    "signedIn": False,
+                },
+            },
+            headers=HEADERS,
+            timeout=120,
+        )
+        assert response.status_code == 200
+        return response.json()
+
+    incident = ask(
+        "Does mentorship include access to 2 courses of my choice?",
+        "github-action-mentorship-grounding",
+    )
+    assert incident["status"] == "answered"
+    assert "Included from day one" in incident["answer"]
+    assert "25% off, always" in incident["answer"]
+    assert "two courses" not in incident["answer"].casefold()
+    assert {source["url"] for source in incident["sources"]} == {
+        "https://towardsai.com/academy/mentorship/"
+    }
+
+    unsupported = ask(
+        "Does Agent Engineering guarantee me a job and include a $500 discount?",
+        "github-action-unsupported-offer-fact",
+    )
+    assert unsupported["status"] == "insufficient_evidence"
+    assert unsupported["sources"] == []
+    assert "https://towardsai.com/academy/contact/#contact" in unsupported["answer"]

tests/test_llm.py

@@ -0,0 +1,189 @@
+from __future__ import annotations
+
+from typing import Any
+
+from tai_helper import llm
+from tai_helper.settings import Settings
+
+
+class FakeDeepSeekResponse:
+    def __init__(self, status_code: int, payload: dict[str, Any], text: str = ""):
+        self.status_code = status_code
+        self._payload = payload
+        self.text = text
+
+    def json(self) -> dict[str, Any]:
+        return self._payload
+
+
+class FakeUsageMetadata:
+    prompt_token_count = 5
+    candidates_token_count = 7
+    total_token_count = 12
+
+
+class FakeGeminiResponse:
+    text = "Gemini fallback answer"
+    usage_metadata = FakeUsageMetadata()
+
+
+class FakeGeminiModels:
+    def __init__(self) -> None:
+        self.calls: list[dict[str, Any]] = []
+
+    def generate_content(self, **kwargs) -> FakeGeminiResponse:
+        self.calls.append(kwargs)
+        return FakeGeminiResponse()
+
+
+class FakeGeminiClient:
+    models = FakeGeminiModels()
+
+    def __init__(self, api_key: str) -> None:
+        self.api_key = api_key
+
+
+def test_prompt_requires_direct_support_for_exact_offer_claims() -> None:
+    prompt = llm.build_prompt(
+        query="How many courses are included in mentorship?",
+        selected_prompt="I want to find mentors",
+        current_url="https://towardsai.com/academy/mentorship/",
+        page_title="Towards AI Mentorship",
+        history=[],
+        selected_pages=[
+            {
+                "title": "Towards AI Mentorship",
+                "kind": "mentorship",
+                "url": "https://towardsai.com/academy/mentorship/",
+                "headings": ["The curriculum comes with the team"],
+                "text": (
+                    "Towards AI Mentorship includes one course: the 10-Hour LLM "
+                    "Fundamentals video course from day one."
+                ),
+            }
+        ],
+    )
+
+    instruction = llm.SYSTEM_INSTRUCTION.lower()
+    assert "only authority for factual claims" in instruction
+    assert "number or names of included products or courses" in instruction
+    assert "do not guess" in instruction
+    assert "every factual claim must be directly supported" in prompt.lower()
+    assert "cannot confirm it" in prompt.lower()
+    assert "includes one course" in prompt
+
+
+def test_prompt_corrects_unsupported_alternatives_with_supported_table_facts() -> None:
+    prompt = llm.build_prompt(
+        query=(
+            "Does the mentorship include two courses of our choice, or only the "
+            "LLM Fundamentals course?"
+        ),
+        selected_prompt="I want to find mentors",
+        current_url="https://towardsai.com/academy/mentorship/",
+        page_title="Towards AI Mentorship",
+        history=[],
+        selected_pages=[
+            {
+                "chunk_id": "mentorship-curriculum-table",
+                "title": "Towards AI Mentorship",
+                "kind": "mentorship",
+                "url": "https://towardsai.com/academy/mentorship/",
+                "headings": ["The curriculum comes with the team"],
+                "text": (
+                    "10-Hour LLM Fundamentals video course $199 Included from "
+                    "day one Full Stack AI Engineering · Agent Engineering · "
+                    "Master AI for Work $349–499 each 25% off, always"
+                ),
+            }
+        ],
+    )
+
+    instruction = llm.SYSTEM_INSTRUCTION.lower()
+    assert "unsupported premise or alternative" in instruction
+    assert "do not return" in instruction
+    assert "merely because the visitor's proposed premise is unsupported" in instruction
+    assert 'never infer a total, use "only"' in instruction
+    prompt_lower = prompt.lower()
+    assert "correction rule" in prompt_lower
+    assert (
+        "return answered with those facts as separate extractive claims" in prompt_lower
+    )
+    assert "one table row says an item is included" in prompt_lower
+    assert 'do not infer a total or say "only"' in prompt_lower
+    assert "do not repeat or deny the visitor's unsupported premise" in prompt_lower
+    assert 'never write "not included"' in prompt_lower
+    assert (
+        "unless those literal words occur in that claim's exact quote" in prompt_lower
+    )
+
+
+def test_generate_answer_uses_deepseek_primary(monkeypatch) -> None:
+    request_calls = []
+    monkeypatch.setattr(
+        llm,
+        "settings",
+        Settings(deepseek_api_key="deepseek-key", gemini_api_key="gemini-key"),
+    )
+
+    def fake_post(*args, **kwargs):
+        request_calls.append({"args": args, **kwargs})
+        return FakeDeepSeekResponse(
+            200,
+            {
+                "choices": [{"message": {"content": "DeepSeek answer"}}],
+                "usage": {
+                    "prompt_tokens": 10,
+                    "completion_tokens": 8,
+                    "total_tokens": 18,
+                },
+            },
+        )
+
+    monkeypatch.setattr(llm.requests, "post", fake_post)
+
+    result = llm.generate_answer("Visitor prompt")
+
+    assert result.answer == "DeepSeek answer"
+    assert result.usage["provider"] == "deepseek"
+    assert result.usage["model"] == "deepseek-v4-flash"
+    assert result.usage["total_tokens"] == 18
+    assert request_calls[0]["args"] == ("https://api.deepseek.com/chat/completions",)
+    assert request_calls[0]["headers"] == {
+        "Authorization": "Bearer deepseek-key",
+        "Content-Type": "application/json",
+    }
+    assert request_calls[0]["json"]["model"] == "deepseek-v4-flash"
+    assert request_calls[0]["json"]["thinking"] == {"type": "disabled"}
+    assert request_calls[0]["json"]["messages"][0]["role"] == "system"
+    assert request_calls[0]["json"]["messages"][1] == {
+        "role": "user",
+        "content": "Visitor prompt",
+    }
+
+
+def test_generate_answer_falls_back_to_gemini_when_deepseek_fails(
+    monkeypatch,
+) -> None:
+    FakeGeminiClient.models = FakeGeminiModels()
+    monkeypatch.setattr(
+        llm,
+        "settings",
+        Settings(deepseek_api_key="deepseek-key", gemini_api_key="gemini-key"),
+    )
+    monkeypatch.setattr(
+        llm.requests,
+        "post",
+        lambda *_args, **_kwargs: FakeDeepSeekResponse(
+            503, {}, "temporarily unavailable"
+        ),
+    )
+    monkeypatch.setattr(llm.genai, "Client", FakeGeminiClient)
+
+    result = llm.generate_answer("Visitor prompt")
+
+    assert result.answer == "Gemini fallback answer"
+    assert result.usage["provider"] == "google_genai"
+    assert result.usage["model"] == "gemini-2.5-flash"
+    assert result.usage["fallback_from"] == "deepseek"
+    assert FakeGeminiClient.models.calls[0]["model"] == "gemini-2.5-flash"

tests/test_offers.py

@@ -0,0 +1,753 @@
+from __future__ import annotations
+
+import json
+from urllib.parse import urlparse
+
+import pytest
+
+from tai_helper import llm
+from tai_helper.catalog import pages, retrieve
+from tai_helper.settings import repo_root
+
+CURRENT_COM_URLS = {
+    "https://towardsai.com/",
+    "https://towardsai.com/academy/",
+    "https://towardsai.com/academy/full-stack-ai-engineering/",
+    "https://towardsai.com/academy/agent-engineering/",
+    "https://towardsai.com/academy/llm-primer/",
+    "https://towardsai.com/academy/python-for-ai-engineering/",
+    "https://towardsai.com/academy/ai-for-work/",
+    "https://towardsai.com/academy/building-llms-for-production/",
+    "https://towardsai.com/academy/bundles/",
+    "https://towardsai.com/academy/mentorship/",
+    "https://towardsai.com/academy/about/",
+    "https://towardsai.com/academy/contact/",
+    "https://towardsai.com/contribute/",
+    "https://towardsai.com/enterprise/software-developer-to-ai-engineer/",
+    "https://towardsai.com/enterprise/agentic-developer-conversion/",
+    "https://towardsai.com/academy/affiliate/",
+    "https://towardsai.com/academy/agent-engineering-free-preview/",
+    "https://towardsai.com/academy/full-stack-ai-engineering-free-preview/",
+    "https://towardsai.com/academy/book/",
+    "https://towardsai.com/academy/bundles/get-it-all/",
+    (
+        "https://towardsai.com/academy/bundles/"
+        "from-coding-novice-to-advanced-llm-developer/"
+    ),
+    (
+        "https://towardsai.com/academy/bundles/"
+        "10-hour-crash-course-into-llm-developer-expert/"
+    ),
+    "https://towardsai.com/enterpriseenablement/",
+    "https://towardsai.com/valuecreation/",
+    "https://towardsai.com/valuecreation/careers/",
+    "https://towardsai.com/valuecreation/deployment-strategist/",
+    "https://towardsai.com/valuecreation/junior-ai-engineer/",
+    "https://towardsai.com/valuecreation/senior-ai-engineer/",
+    "https://towardsai.com/theaitastegap/",
+    "https://towardsai.com/webinars/agentengineering/",
+}
+
+
+def _catalog(name: str) -> dict:
+    return json.loads((repo_root() / "data" / name).read_text())
+
+
+def _page_text(url: str) -> str:
+    matching = [page for page in pages() if page["url"] == url]
+    assert len(matching) == 1, f"expected one active canonical page for {url}"
+    return matching[0]["text"].lower()
+
+
+def test_live_refresh_scans_every_current_towardsai_com_page() -> None:
+    catalog = _catalog("towardsai_com_pages.json")
+
+    assert {page["discovered_url"] for page in catalog["pages"]} == CURRENT_COM_URLS
+    assert {page["url"] for page in catalog["pages"]} == CURRENT_COM_URLS
+    assert catalog["status_counts"] == {"excluded": 3, "included": 27}
+    excluded_urls = {
+        page["url"] for page in catalog["pages"] if not page["retrieval_eligible"]
+    }
+    assert excluded_urls == {
+        "https://towardsai.com/",
+        "https://towardsai.com/academy/",
+        "https://towardsai.com/academy/bundles/",
+    }
+
+
+def test_academy_refresh_scans_sitemap_and_excludes_unsafe_pages() -> None:
+    catalog = _catalog("pages.json")
+    academy_pages = [
+        page
+        for page in catalog["pages"]
+        if page["discovered_url"].startswith("https://academy.towardsai.net")
+    ]
+    excluded_paths = {
+        urlparse(page["discovered_url"]).path.rstrip("/") or "/"
+        for page in academy_pages
+        if not page["retrieval_eligible"]
+    }
+
+    assert len(academy_pages) == 28
+    assert {
+        "/",
+        "/pages/agent-course-landing-page",
+        "/pages/towards-ai-insider",
+        "/pages/agent-course-new-page-cro",
+        "/pages/webinar",
+        "/pages/new-home-page",
+        "/pages/landing-page-free-email-course",
+        "/pages/choose-your-course",
+        "/collections",
+        "/collections/products",
+        "/collections/developers",
+        "/collections/professionals",
+        "/pages/free-resources",
+    } <= excluded_paths
+    assert not any(
+        page["retrieval_eligible"]
+        for page in catalog["pages"]
+        if page["authority"] == "curated_external"
+    )
+
+
+def test_every_evidence_page_has_fresh_provenance_and_real_chunks() -> None:
+    for filename in ("towardsai_com_pages.json", "pages.json"):
+        for page in _catalog(filename)["pages"]:
+            if not page["retrieval_eligible"]:
+                continue
+            assert page["status"] == "included"
+            assert page["fetched_at"]
+            assert len(page["content_hash"]) == 64
+            assert page["http_status"] == 200
+            assert page["chunks"]
+            assert all(chunk["chunk_id"] and chunk["text"] for chunk in page["chunks"])
+            assert all(len(chunk["text"]) <= 1800 for chunk in page["chunks"])
+            assert all(chunk["evidence_spans"] for chunk in page["chunks"])
+            assert all(
+                span["text"] in " ".join(chunk["text"].split())
+                for chunk in page["chunks"]
+                for span in chunk["evidence_spans"]
+            )
+
+
+@pytest.mark.parametrize(
+    ("url", "facts"),
+    [
+        (
+            "https://towardsai.com/academy/full-stack-ai-engineering/",
+            ("$349 one-time", "92 lessons", "60+ hrs"),
+        ),
+        (
+            "https://towardsai.com/academy/agent-engineering/",
+            ("$499 one-time", "35 lessons", "two production agents"),
+        ),
+        (
+            "https://towardsai.com/academy/llm-primer/",
+            ("$199 one-time", "5 in-depth 2-hour video sessions"),
+        ),
+        (
+            "https://towardsai.com/academy/python-for-ai-engineering/",
+            ("$149 one-time", "38 lessons", "complete beginners welcome"),
+        ),
+        (
+            "https://towardsai.com/academy/ai-for-work/",
+            ("$399 one-time", "98 lessons", "no code required"),
+        ),
+        (
+            "https://towardsai.com/academy/building-llms-for-production/",
+            ("$29.99", "470-page", "84 lessons"),
+        ),
+        (
+            "https://towardsai.com/academy/mentorship/",
+            (
+                "10-hour llm fundamentals video course",
+                "included from day one",
+                "25% off, always",
+            ),
+        ),
+        (
+            "https://towardsai.com/academy/bundles/get-it-all/",
+            ("6 products in the bundle", "$1,625 combined price"),
+        ),
+        (
+            (
+                "https://towardsai.com/academy/bundles/"
+                "from-coding-novice-to-advanced-llm-developer/"
+            ),
+            ("$599 bundle price, was $727", "4 products"),
+        ),
+        (
+            (
+                "https://towardsai.com/academy/bundles/"
+                "10-hour-crash-course-into-llm-developer-expert/"
+            ),
+            ("$948 bundle price, was $1,078", "4 products"),
+        ),
+        (
+            "https://towardsai.com/academy/agent-engineering-free-preview/",
+            ("free preview · 7 full lessons",),
+        ),
+        (
+            "https://towardsai.com/academy/full-stack-ai-engineering-free-preview/",
+            ("free preview lessons · no card required", "instant access"),
+        ),
+    ],
+)
+def test_current_course_and_bundle_facts_are_present(
+    url: str, facts: tuple[str, ...]
+) -> None:
+    text = _page_text(url)
+
+    for fact in facts:
+        assert fact in text
+
+
+@pytest.mark.parametrize(
+    ("query", "expected_url"),
+    [
+        (
+            "What is included in the Full Stack AI Engineering course and what does it cost?",
+            "https://towardsai.com/academy/full-stack-ai-engineering/",
+        ),
+        (
+            "How many lessons and what price is the Agent Engineering course?",
+            "https://towardsai.com/academy/agent-engineering/",
+        ),
+        (
+            "What is the 10-Hour LLM Fundamentals course price?",
+            "https://towardsai.com/academy/llm-primer/",
+        ),
+        (
+            "Is Beginner Python for AI Engineering for non-coders?",
+            "https://towardsai.com/academy/python-for-ai-engineering/",
+        ),
+        (
+            "What is Master AI for Work for professionals?",
+            "https://towardsai.com/academy/ai-for-work/",
+        ),
+        (
+            "What is the Building LLMs for Production course and ebook?",
+            "https://towardsai.com/academy/building-llms-for-production/",
+        ),
+        (
+            "Where are the Building LLMs book companion resources?",
+            "https://towardsai.com/academy/book/",
+        ),
+        (
+            "What course is included in mentorship?",
+            "https://towardsai.com/academy/mentorship/",
+        ),
+        (
+            "What does the Get It All bundle include?",
+            "https://towardsai.com/academy/bundles/get-it-all/",
+        ),
+        (
+            "What is in the From Non-Coder to AI Engineer bundle?",
+            (
+                "https://towardsai.com/academy/bundles/"
+                "from-coding-novice-to-advanced-llm-developer/"
+            ),
+        ),
+        (
+            "What is in the From Developer to Advanced AI Engineer bundle?",
+            (
+                "https://towardsai.com/academy/bundles/"
+                "10-hour-crash-course-into-llm-developer-expert/"
+            ),
+        ),
+        (
+            "Where is the Full Stack free course preview?",
+            "https://towardsai.com/academy/full-stack-ai-engineering-free-preview/",
+        ),
+        (
+            "How many Agent Engineering free preview lessons are there?",
+            "https://towardsai.com/academy/agent-engineering-free-preview/",
+        ),
+        (
+            "Where is the free Agent Engineering webinar?",
+            "https://towardsai.com/webinars/agentengineering/",
+        ),
+        (
+            "Where can I get the free agents cheatsheet?",
+            (
+                "https://academy.towardsai.net/products/"
+                "digital_downloads/agents-cheatsheet"
+            ),
+        ),
+        (
+            "Where can I get the Anti-Slop Framework?",
+            (
+                "https://academy.towardsai.net/products/"
+                "digital_downloads/anti-slop-framework"
+            ),
+        ),
+        (
+            "What enterprise AI enablement and training do you offer?",
+            "https://towardsai.com/enterpriseenablement/",
+        ),
+        (
+            "Can you convert software developers into AI engineers?",
+            ("https://towardsai.com/enterprise/software-developer-to-ai-engineer/"),
+        ),
+        (
+            "Do you offer Claude Code and Codex training?",
+            ("https://towardsai.com/enterprise/agentic-developer-conversion/"),
+        ),
+        (
+            "Do you do AI deployment value creation consulting?",
+            "https://towardsai.com/valuecreation/",
+        ),
+    ],
+)
+def test_every_current_offer_routes_to_its_canonical_page(
+    query: str, expected_url: str
+) -> None:
+    selected = retrieve(query)
+
+    assert selected, query
+    assert selected[0]["url"] == expected_url
+
+
+def test_course_decider_starter_returns_one_canonical_hero_per_course() -> None:
+    selected = retrieve("I want help deciding which course to take.")
+
+    assert len(selected) == 6
+    assert len({page["url"] for page in selected}) == 6
+    assert all(page["chunk_index"] == 0 for page in selected)
+    assert {page["path"] for page in selected} == {
+        "/academy/full-stack-ai-engineering",
+        "/academy/agent-engineering",
+        "/academy/llm-primer",
+        "/academy/python-for-ai-engineering",
+        "/academy/ai-for-work",
+        "/academy/building-llms-for-production",
+    }
+    assert len(llm.evidence_chunks(selected)) == len(selected)
+
+
+def test_company_starters_return_only_canonical_b2b_evidence() -> None:
+    integration = retrieve("I want help to integrate AI into my company")
+    training = retrieve("I want a training inside my company")
+
+    assert integration[0]["path"] == "/valuecreation"
+    assert integration[1]["path"] == "/enterpriseenablement"
+    assert all(page["kind"] == "b2b" for page in integration)
+    assert training[0]["path"] == "/enterpriseenablement"
+    assert all(page["kind"] == "b2b" for page in training)
+
+
+def test_free_resource_starter_does_not_retrieve_paid_offer_evidence() -> None:
+    selected = retrieve(
+        "I'm looking for more free resources to learn before committing to buying "
+        "a course"
+    )
+
+    assert selected
+    assert selected[0]["kind"] == "free_resource"
+    assert all(
+        page["kind"] in {"free_resource", "digital_download"}
+        or page["path"] == "/academy/book"
+        for page in selected
+    )
+    assert len(llm.evidence_chunks(selected)) == len(selected)
+
+
+def test_mentor_starter_returns_only_mentorship_evidence() -> None:
+    selected = retrieve("I want to find mentors")
+
+    assert selected
+    assert all(page["path"] == "/academy/mentorship" for page in selected)
+
+
+def test_canonical_offer_pages_suppress_lower_authority_mirrors() -> None:
+    com_offer_ids = {
+        page["offer_id"]
+        for page in _catalog("towardsai_com_pages.json")["pages"]
+        if page.get("offer_id")
+    }
+
+    for page in pages():
+        if page.get("offer_id") in com_offer_ids:
+            assert page["authority"] == "official_site"
+
+
+def test_mentorship_incident_is_answered_only_from_exact_current_evidence() -> None:
+    selected = retrieve(
+        "The chatbot mentioned access to 2 courses of our choice as part of "
+        "mentorship. Is that true?",
+        current_url="https://towardsai.com/academy/mentorship/",
+    )
+
+    assert selected[0]["url"] == "https://towardsai.com/academy/mentorship/"
+    assert "Included from day one" in selected[0]["text"]
+    quote = (
+        "10-Hour LLM Fundamentals video course Five in-depth 2-hour video "
+        "sessions, from a basic prompt to a full production rollout $199 "
+        "Included from day one"
+    )
+    claim = f"{quote}."
+    raw = json.dumps(
+        {
+            "status": "answered",
+            "claims": [
+                {
+                    "text": claim,
+                    "chunk_id": selected[0]["chunk_id"],
+                    "quote": quote,
+                }
+            ],
+        }
+    )
+
+    result = llm.validate_grounded_result(raw, selected)
+
+    assert result.is_answered
+    assert result.answer == claim
+    assert "two courses" not in result.answer.lower()
+
+
+def test_no_active_chunk_contains_the_retired_two_course_offer() -> None:
+    retired_phrases = (
+        "two courses included",
+        "pick one from these options",
+        "ai coding: a fixed course",
+    )
+    active_text = "\n".join(
+        chunk["text"] for page in pages() for chunk in page["chunks"]
+    ).lower()
+
+    assert not any(phrase in active_text for phrase in retired_phrases)
+    assert not any(
+        page["url"] == "https://towardsai.com/academy/membership/" for page in pages()
+    )
+
+
+def test_stale_academy_summaries_are_not_retrieval_evidence() -> None:
+    blocked_urls = {
+        "https://towardsai.com/",
+        "https://towardsai.com/academy/",
+        "https://towardsai.com/academy/bundles/",
+        "https://academy.towardsai.net/collections",
+        "https://academy.towardsai.net/collections/products",
+        "https://academy.towardsai.net/collections/developers",
+        "https://academy.towardsai.net/collections/professionals",
+        "https://academy.towardsai.net/pages/free-resources",
+    }
+    active_pages = pages()
+    active_urls = {page["url"] for page in active_pages}
+    active_text = "\n".join(page["text"] for page in active_pages).lower()
+
+    assert blocked_urls.isdisjoint(active_urls)
+    assert "124 lessons" not in active_text
+    assert "free resources from 8-hour llm primer" not in active_text
+    assert "465-page reference" not in active_text
+
+
+def test_navigation_and_announcement_boilerplate_is_not_evidence() -> None:
+    active_text = "\n".join(page["text"] for page in pages()).lower()
+
+    assert "new: towards ai mentorship" not in active_text
+    assert "skip to main content" not in active_text
+    assert "toggle menu" not in active_text
+
+
+@pytest.mark.parametrize(
+    ("query", "expected_url", "retired_fact"),
+    [
+        (
+            "How many lessons are in Full Stack AI Engineering?",
+            "https://towardsai.com/academy/full-stack-ai-engineering/",
+            "124 lessons",
+        ),
+        (
+            "How many lessons are in Agent Engineering?",
+            "https://towardsai.com/academy/agent-engineering/",
+            "50 lessons",
+        ),
+        (
+            "Do you have an 8-hour LLM Primer?",
+            "https://towardsai.com/academy/llm-primer/",
+            "8-hour llm primer",
+        ),
+    ],
+)
+def test_named_offer_queries_use_current_canonical_evidence(
+    query: str, expected_url: str, retired_fact: str
+) -> None:
+    selected = retrieve(query)
+
+    assert selected
+    assert selected[0]["url"] == expected_url
+    assert retired_fact not in "\n".join(page["text"] for page in selected).lower()
+
+
+def test_book_page_count_is_an_answerable_atomic_span() -> None:
+    selected = retrieve("How many pages is Building LLMs for Production?")
+
+    assert selected
+    assert selected[0]["url"] == (
+        "https://towardsai.com/academy/building-llms-for-production/"
+    )
+    assert any("470-page" in span["text"] for span in selected[0]["evidence_spans"])
+
+
+@pytest.mark.parametrize(
+    "query",
+    [
+        "Does Agent Engineering free preview include lifetime access and require no card?",
+        "Does the Full Stack free preview give me a certificate?",
+        "Does Building LLMs for Production have a refund guarantee?",
+        "How long is Agent Engineering?",
+        "How many hours does Python for AI Engineering take?",
+        "How many lessons are in LLM Fundamentals?",
+        "Does mentorship include a certificate?",
+        "Does the Get It All bundle get 50% off?",
+    ],
+)
+def test_unpublished_high_risk_offer_facts_have_no_citable_evidence(
+    query: str,
+) -> None:
+    assert retrieve(query) == []
+
+
+def test_named_llm_fundamentals_feature_query_cannot_retrieve_another_offer() -> None:
+    selected = retrieve(
+        "Does LLM Fundamentals include community support and an AI tutor?"
+    )
+
+    assert selected
+    assert {chunk["offer_id"] for chunk in selected} == {"llm-primer"}
+    assert any(
+        "community" in span["text"].casefold()
+        and "ai tutor" in span["text"].casefold()
+        for chunk in selected
+        for span in chunk["evidence_spans"]
+    )
+
+
+def test_get_it_all_exact_price_abstains_when_only_checkout_policy_is_published() -> None:
+    assert retrieve("What is the Get It All bundle price?") == []
+
+    location = retrieve("Where is the Get It All bundle price shown at checkout?")
+    spans = [
+        span["text"]
+        for chunk in location
+        for span in chunk.get("evidence_spans", [])
+    ]
+    assert location
+    assert {chunk["offer_id"] for chunk in location} == {"get-it-all"}
+    assert spans
+    assert all("Bundle price shown at checkout" in span for span in spans)
+    assert all("$1,625" not in span and "$948" not in span for span in spans)
+
+
+def test_full_stack_preview_count_uses_only_explicit_preview_count_evidence() -> None:
+    selected = retrieve("How many Full Stack free preview lessons are there?")
+    spans = [
+        span["text"]
+        for chunk in selected
+        for span in chunk.get("evidence_spans", [])
+    ]
+
+    assert selected
+    assert {chunk["offer_id"] for chunk in selected} <= {
+        "full-stack-ai-engineering-free-preview",
+        "full-stack-ai-engineering",
+    }
+    assert spans
+    assert all("6" in span or "six" in span.casefold() for span in spans)
+    assert all("92 lessons" not in span for span in spans)
+
+
+def test_preview_count_exception_never_exposes_paid_access_evidence() -> None:
+    selected = retrieve("How many Agent Engineering preview lessons are there?")
+    evidence = [
+        span["text"].casefold()
+        for chunk in selected
+        for span in chunk["evidence_spans"]
+    ]
+
+    assert selected
+    assert any("7" in span and "lesson" in span for span in evidence)
+    assert all("lifetime access" not in span for span in evidence)
+
+
+@pytest.mark.parametrize(
+    "query",
+    [
+        (
+            "How many Agent Engineering preview lessons are there, and what does "
+            "the full course cost?"
+        ),
+        (
+            "How many Full Stack preview lessons are there, and does the full "
+            "course include lifetime access?"
+        ),
+    ],
+)
+def test_mixed_preview_and_paid_fact_questions_fail_closed(query: str) -> None:
+    assert retrieve(query) == []
+
+
+def test_lifetime_access_requires_an_explicit_permanent_access_qualifier() -> None:
+    selected = retrieve("Does the Full Stack free preview include lifetime access?")
+    evidence = [
+        span["text"].casefold()
+        for chunk in selected
+        for span in chunk["evidence_spans"]
+    ]
+
+    assert selected
+    assert evidence
+    assert all(
+        any(term in span for term in ("lifetime", "forever", "keep", "retain"))
+        for span in evidence
+    )
+
+
+def test_mentorship_cancellation_retrieves_only_the_exact_access_policy() -> None:
+    selected = retrieve(
+        "Do I keep LLM Fundamentals lifetime access after cancelling mentorship?"
+    )
+    spans = {
+        span["text"]
+        for chunk in selected
+        for span in chunk.get("evidence_spans", [])
+    }
+
+    assert selected
+    assert {chunk["offer_id"] for chunk in selected} == {"mentorship"}
+    assert any("Course access is active while" in span for span in spans)
+    assert any("If you cancel" in span and "lifetime access" in span for span in spans)
+    assert all("Included from day one" not in span for span in spans)
+
+
+@pytest.mark.parametrize(
+    "query",
+    [
+        "Does the monthly mentorship plan have a 30-day money-back guarantee?",
+        "Does the month-to-month mentorship have a 30-day money-back guarantee?",
+    ],
+)
+def test_monthly_mentorship_guarantee_retrieves_both_plan_qualifiers(
+    query: str,
+) -> None:
+    selected = retrieve(query)
+    spans = {
+        span["text"]
+        for chunk in selected
+        for span in chunk.get("evidence_spans", [])
+    }
+
+    assert selected
+    assert any("monthly mentorship" in span.casefold() for span in spans)
+    assert any(
+        "yearly" in span.casefold() and "money-back" in span.casefold()
+        for span in spans
+    )
+    assert all("Join the Mentorship 30-day" not in span for span in spans)
+
+
+def test_mentorship_plan_prices_require_exact_plan_denominations() -> None:
+    monthly_queries = (
+        "What is the monthly mentorship price?",
+        "Mentorship month price?",
+        "What is the month-to-month mentorship price?",
+        "What is the mentorship price per month?",
+        "What does mentorship cost each month?",
+    )
+    monthly_results = [
+        [
+            span["text"]
+            for chunk in retrieve(query)
+            for span in chunk["evidence_spans"]
+        ]
+        for query in monthly_queries
+    ]
+    yearly_queries = (
+        "What is the yearly mentorship price?",
+        "Mentorship year price?",
+        "What is the annual mentorship price?",
+        "What is the mentorship price per year?",
+    )
+    yearly_results = [
+        [
+            span["text"]
+            for chunk in retrieve(query)
+            for span in chunk["evidence_spans"]
+        ]
+        for query in yearly_queries
+    ]
+
+    assert all(monthly_results)
+    for monthly in monthly_results:
+        assert all("Guest workshop monthly" not in span for span in monthly)
+        assert all("Production blueprint monthly" not in span for span in monthly)
+        assert all("From $75/month" not in span for span in monthly)
+        assert any("$99" in span for span in monthly)
+    assert all(yearly_results)
+    for yearly in yearly_results:
+        assert all("$289 off" not in span for span in yearly)
+        assert all("$75 a month billed yearly" not in span for span in yearly)
+        assert any("$899" in span and "/year" in span for span in yearly)
+
+
+def test_mentorship_discounts_are_bound_to_the_requested_plan_or_courses() -> None:
+    yearly = [
+        span["text"]
+        for chunk in retrieve("Is the yearly mentorship plan 24% off?")
+        for span in chunk["evidence_spans"]
+    ]
+    saved = [
+        span["text"]
+        for chunk in retrieve("Does mentorship save 24%?")
+        for span in chunk["evidence_spans"]
+    ]
+    courses = [
+        span["text"]
+        for chunk in retrieve(
+            "What discount does mentorship give on Full Stack AI Engineering?"
+        )
+        for span in chunk["evidence_spans"]
+    ]
+
+    assert yearly
+    assert all(
+        "save 24%" in span.casefold() or "24% less" in span.casefold()
+        for span in yearly
+    )
+    assert all("25% off" not in span for span in yearly)
+    assert saved
+    assert all(
+        "save 24%" in span.casefold() or "24% less" in span.casefold()
+        for span in saved
+    )
+    assert courses
+    assert all("25% off" in span for span in courses)
+    assert retrieve("Does monthly mentorship have a discount?") == []
+
+
+def test_mentorship_access_and_features_exclude_comparison_entities() -> None:
+    access = [
+        span["text"]
+        for chunk in retrieve("Does mentorship include lifetime course access?")
+        for span in chunk["evidence_spans"]
+    ]
+    reviews = [
+        span["text"]
+        for chunk in retrieve("Does mentorship include resume and project reviews?")
+        for span in chunk["evidence_spans"]
+    ]
+
+    assert access
+    assert all("retainers" not in span.casefold() for span in access)
+    assert any("lifetime access" in span.casefold() for span in access)
+    assert reviews
+    assert any(
+        "resume" in span.casefold()
+        and "project" in span.casefold()
+        and "reviews" in span.casefold()
+        for span in reviews
+    )
+    assert all("single mentor" not in span.casefold() for span in reviews)

tests/test_retrieval_safety.py

@@ -0,0 +1,414 @@
+from __future__ import annotations
+
+import hashlib
+import json
+from datetime import UTC, datetime, timedelta
+from typing import Any
+
+import pytest
+from fastapi.testclient import TestClient
+
+from tai_helper import api, catalog, llm
+from tai_helper.schemas import HelperChatRequest
+
+
+def _evidence_hash(page: dict[str, Any]) -> str:
+    payload = json.dumps(
+        {key: value for key, value in page.items() if key != "evidence_hash"},
+        ensure_ascii=False,
+        sort_keys=True,
+        separators=(",", ":"),
+    )
+    return hashlib.sha256(payload.encode()).hexdigest()
+
+
+def _page(
+    *,
+    slug: str = "agent-engineering",
+    text: str = "Build production-ready AI agents with practical engineering lessons.",
+    fetched_at: str | None = None,
+    **overrides: Any,
+) -> dict[str, Any]:
+    fetched_at = fetched_at or datetime.now(UTC).isoformat()
+    page: dict[str, Any] = {
+        "url": f"https://towardsai.com/academy/{slug}/",
+        "canonical_url": f"https://towardsai.com/academy/{slug}/",
+        "host": "towardsai.com",
+        "path": f"/academy/{slug}",
+        "title": slug.replace("-", " ").title(),
+        "kind": "course",
+        "offer_id": slug,
+        "entity_id": f"offer:{slug}",
+        "status": "included",
+        "retrieval_eligible": True,
+        "authority": "canonical_offer",
+        "fetched_at": fetched_at,
+        "content_hash": hashlib.sha256(text.encode()).hexdigest(),
+        "http_status": 200,
+        "text": text,
+        "chunks": [
+            {
+                "chunk_id": f"{slug}:overview",
+                "heading": "Overview",
+                "text": text,
+                "evidence_spans": [
+                    {"span_id": f"{slug}:overview:span-0", "text": text}
+                ],
+            }
+        ],
+    }
+    page["evidence_hash"] = _evidence_hash(page)
+    page.update(overrides)
+    return page
+
+
+@pytest.fixture(autouse=True)
+def clear_caches_between_tests() -> None:
+    catalog.clear_catalog_caches()
+    yield
+    catalog.clear_catalog_caches()
+
+
+@pytest.fixture
+def install_catalog(monkeypatch: pytest.MonkeyPatch):
+    def install(*pages: dict[str, Any]) -> None:
+        monkeypatch.setattr(
+            catalog,
+            "pages_payload",
+            lambda: {"pages": list(pages), "generated_at": {}},
+        )
+        catalog.clear_catalog_caches()
+
+    return install
+
+
+@pytest.mark.parametrize(
+    ("mutation", "value"),
+    [
+        ("remove", "status"),
+        ("set_status", "review_pending"),
+    ],
+    ids=["missing-status", "unknown-status"],
+)
+def test_only_explicit_active_status_can_supply_evidence(
+    install_catalog, mutation: str, value: str
+) -> None:
+    valid = _page(slug="valid-course", text="Valid course teaches glacier models.")
+    invalid = _page(slug="invalid-course", text="Invalid course teaches zeppelins.")
+    if mutation == "remove":
+        invalid.pop(value)
+    else:
+        invalid["status"] = value
+    install_catalog(valid, invalid)
+
+    assert [page["url"] for page in catalog.pages()] == [valid["url"]]
+    assert catalog.retrieve("zeppelins") == []
+
+
+@pytest.mark.parametrize("eligible", [None, False], ids=["missing", "false"])
+def test_retrieval_eligibility_must_be_explicitly_true(
+    install_catalog, eligible: bool | None
+) -> None:
+    page = _page(text="A course about orbital pottery.")
+    if eligible is None:
+        page.pop("retrieval_eligible")
+    else:
+        page["retrieval_eligible"] = eligible
+    install_catalog(page)
+
+    assert catalog.pages() == []
+    assert catalog.retrieve("orbital pottery course") == []
+
+
+def test_content_hash_is_recomputed_before_page_can_supply_evidence(
+    install_catalog,
+) -> None:
+    page = _page(text="Original verified course content.")
+    page["text"] = "Tampered course content."
+    install_catalog(page)
+
+    assert catalog.pages() == []
+    assert catalog.retrieve("tampered course content") == []
+
+
+@pytest.mark.parametrize(
+    ("field", "value"),
+    [
+        ("authority", "primary"),
+        ("offer_id", ""),
+        ("kind", "bundle"),
+        ("title", "Altered trusted title"),
+    ],
+)
+def test_evidence_hash_binds_retrieval_provenance_and_metadata(
+    install_catalog, field: str, value: str
+) -> None:
+    page = _page(text="Hash-bound canonical course evidence.")
+    page[field] = value
+    install_catalog(page)
+
+    assert catalog.pages() == []
+    assert catalog.retrieve("canonical course evidence") == []
+
+
+def test_chunk_text_must_belong_to_the_hashed_page_text(install_catalog) -> None:
+    page = _page(text="The verified page includes one fundamentals course.")
+    forged = "The mentorship includes two courses of your choice."
+    page["chunks"][0]["text"] = forged
+    page["chunks"][0]["evidence_spans"] = [{"span_id": "forged:span-0", "text": forged}]
+    install_catalog(page)
+
+    assert catalog.pages() == []
+    assert catalog.retrieve("two courses of your choice") == []
+
+
+@pytest.mark.parametrize(
+    ("field", "value"),
+    [
+        ("url", "https://evil.example/phish"),
+        ("host", "evil.example"),
+        ("path", "/academy/different-course"),
+    ],
+)
+def test_citation_url_and_metadata_must_match_the_canonical_page(
+    install_catalog, field: str, value: str
+) -> None:
+    page = _page(text="Verified canonical course evidence.")
+    page[field] = value
+    install_catalog(page)
+
+    assert catalog.pages() == []
+    assert catalog.retrieve("canonical course evidence") == []
+
+
+def test_duplicate_chunk_ids_across_pages_fail_closed(install_catalog) -> None:
+    first = _page(slug="first", text="First unique course evidence.")
+    second = _page(slug="second", text="Second unique course evidence.")
+    second["chunks"][0]["chunk_id"] = first["chunks"][0]["chunk_id"]
+    second["evidence_hash"] = _evidence_hash(second)
+    install_catalog(first, second)
+
+    assert catalog.pages() == []
+    assert catalog.retrieve("unique course evidence") == []
+
+
+@pytest.mark.parametrize("mutation", ["missing", "not_in_chunk"])
+def test_atomic_evidence_spans_are_required_and_verified(
+    install_catalog, mutation: str
+) -> None:
+    page = _page(text="No code required for this course.")
+    if mutation == "missing":
+        page["chunks"][0].pop("evidence_spans")
+    else:
+        page["chunks"][0]["evidence_spans"][0]["text"] = "Invented span."
+    install_catalog(page)
+
+    assert catalog.pages() == []
+    assert catalog.retrieve("code required") == []
+
+
+@pytest.mark.parametrize(
+    "fetched_at",
+    [
+        None,
+        (
+            datetime.now(UTC)
+            - timedelta(days=max(catalog.settings.catalog_max_age_days, 0) + 1)
+        ).isoformat(),
+        (datetime.now(UTC) + timedelta(minutes=10)).isoformat(),
+    ],
+    ids=["missing", "stale", "future"],
+)
+def test_missing_stale_or_future_fetch_time_cannot_supply_evidence(
+    install_catalog, fetched_at: str | None
+) -> None:
+    page = _page(text="A course about lunar basket weaving.")
+    if fetched_at is None:
+        page.pop("fetched_at")
+    else:
+        page["fetched_at"] = fetched_at
+    install_catalog(page)
+
+    assert catalog.pages() == []
+    assert catalog.retrieve("lunar basket weaving course") == []
+
+
+def test_corrupt_catalog_reload_discards_previously_cached_evidence(
+    monkeypatch: pytest.MonkeyPatch, tmp_path
+) -> None:
+    data_dir = tmp_path / "data"
+    data_dir.mkdir()
+    academy_path = data_dir / "pages.json"
+    website_path = data_dir / "towardsai_com_pages.json"
+    academy_path.write_text(json.dumps({"pages": []}))
+    website_path.write_text(
+        json.dumps({"pages": [_page(text="Quantum origami course.")]})
+    )
+    monkeypatch.setattr(catalog, "repo_root", lambda: tmp_path)
+    catalog.clear_catalog_caches()
+
+    assert [page["url"] for page in catalog.pages()] == [
+        "https://towardsai.com/academy/agent-engineering/"
+    ]
+
+    website_path.write_text("{ definitely not valid JSON")
+
+    assert catalog.pages() == []
+    assert catalog.retrieve("quantum origami course") == []
+
+
+@pytest.mark.parametrize(
+    "query",
+    [
+        "Does your course include scuba-diving lessons?",
+        "Can company training certify helicopter pilots?",
+        "Does the mentorship provide veterinary surgery?",
+    ],
+)
+def test_unsupported_but_in_scope_subjects_return_no_chunks(
+    install_catalog, query: str
+) -> None:
+    install_catalog(
+        _page(text="This AI engineering course covers agents and deployment."),
+    )
+
+    assert catalog.in_scope(query)
+    assert catalog.retrieve(query) == []
+
+
+def test_punctuation_and_hyphens_do_not_break_relevant_retrieval(
+    install_catalog,
+) -> None:
+    page = _page(
+        text="The course teaches production-ready agent-engineering workflows in C++ and C#.",
+    )
+    install_catalog(page)
+
+    assert {
+        "production",
+        "ready",
+        "agent",
+        "engineering",
+        "c++",
+        "c#",
+    } <= catalog.tokenize("production-ready agent-engineering C++/C#")
+    selected = catalog.retrieve("Is it production-ready for agent engineering?")
+
+    assert selected
+    assert selected[0]["url"] == page["url"]
+
+
+def test_scope_uses_history_only_for_referential_followups() -> None:
+    course_history = ["Tell me about the Towards AI mentorship and its courses."]
+
+    assert catalog.in_scope("Is it included?", course_history)
+    assert not catalog.in_scope("Who won the World Cup?", course_history)
+    assert not catalog.in_scope("Is it included?", ["Who won the World Cup?"])
+
+
+def test_retrieval_query_never_uses_prior_assistant_claims_as_evidence() -> None:
+    payload = HelperChatRequest.model_validate(
+        {
+            "query": "Is that accurate?",
+            "selectedPrompt": "I want to find mentors",
+            "history": [
+                {
+                    "role": "assistant",
+                    "content": "The mentorship includes two courses of your choice.",
+                },
+                {"role": "user", "content": "Tell me about mentorship."},
+            ],
+            "context": {"url": "https://towardsai.com/academy/mentorship/"},
+        }
+    )
+
+    retrieval_query = api._retrieval_query(payload)
+
+    assert "two courses" not in retrieval_query
+    assert "Tell me about mentorship" in retrieval_query
+    assert "Is that accurate" in retrieval_query
+
+
+def test_api_exposes_only_sources_for_chunks_cited_by_validated_answer(
+    monkeypatch: pytest.MonkeyPatch,
+) -> None:
+    selected = [
+        {
+            "chunk_id": "course-a:overview",
+            "title": "Course A",
+            "url": "https://towardsai.com/academy/course-a/",
+            "kind": "course",
+            "headings": ["Overview"],
+            "text": "Course A is available.",
+        },
+        {
+            "chunk_id": "course-b:pricing",
+            "title": "Course B",
+            "url": "https://towardsai.com/academy/course-b/",
+            "kind": "course",
+            "headings": ["Pricing"],
+            "text": "Course B costs $20.",
+        },
+    ]
+    cited = llm.EvidenceChunk(
+        chunk_id="course-b:pricing",
+        title="Course B",
+        url="https://towardsai.com/academy/course-b/",
+        kind="course",
+        headings=("Pricing",),
+        text="Course B costs $20.",
+    )
+    grounded = llm.GroundingResult(
+        valid=True,
+        status="answered",
+        answer="Course B costs $20.",
+        claims=(
+            llm.GroundedClaim(
+                text="Course B costs $20.",
+                chunk_id="course-b:pricing",
+                quote="Course B costs $20.",
+            ),
+        ),
+        cited_chunks=(cited,),
+    )
+    monkeypatch.setattr(api, "page_is_allowed", lambda _url: True)
+    monkeypatch.setattr(api, "in_scope", lambda _query, _history: True)
+    monkeypatch.setattr(api, "_check_rate_limits", lambda _request, _payload: None)
+    monkeypatch.setattr(api, "_schedule_monitor", lambda _monitor: None)
+    monkeypatch.setattr(
+        api,
+        "retrieve",
+        lambda _query, *, current_url="", limit=7: selected,
+    )
+    monkeypatch.setattr(api.llm, "build_prompt", lambda **_kwargs: "prompt")
+    monkeypatch.setattr(
+        api.llm,
+        "generate_grounded_answer",
+        lambda _prompt, _selected, **_kwargs: grounded,
+    )
+
+    response = TestClient(api.app).post(
+        "/api/helper/chat",
+        headers={"Origin": "https://towardsai.com"},
+        json={
+            "query": "I want help deciding which course to take.",
+            "selectedPrompt": "I want help deciding which course to take.",
+            "visitorId": "retrieval-safety-test",
+            "history": [],
+            "context": {
+                "url": "https://towardsai.com/academy/course-b/",
+                "pageTitle": "Course B",
+                "signedIn": False,
+            },
+        },
+    )
+
+    assert response.status_code == 200
+    assert response.json()["status"] == "answered"
+    assert response.json()["sources"] == [
+        {
+            "title": "Course B",
+            "url": "https://towardsai.com/academy/course-b/",
+            "kind": "course",
+        }
+    ]

tests/test_widget.py

@@ -0,0 +1,14 @@
+from __future__ import annotations
+
+from tai_helper.api import CONTACT_FORM_URL
+from tai_helper.settings import repo_root
+
+
+def test_widget_network_failures_keep_a_clickable_contact_handoff() -> None:
+    source = (repo_root() / "static" / "widget.js").read_text()
+
+    assert f'var contactFormUrl = "{CONTACT_FORM_URL}";' in source
+    assert "contactLinkMarkdown" in source
+    assert "The helper is unavailable on this page. Please " in source
+    assert "Something went wrong. Please " in source
+    assert "renderMarkdown(" in source