Fix broad starter grounding and monitoring

tai_helper/llm.py

@@ -265,10 +265,13 @@ _QUERY_RELEVANCE_STOP = frozenset(
         "at",
         "be",
         "can",
+        "choose",
         "come",
         "comes",
         "course",
         "courses",
+        "decide",
+        "deciding",
         "do",
         "does",
         "every",
@@ -282,6 +285,7 @@ _QUERY_RELEVANCE_STOP = frozenset(
         "has",
         "have",
         "having",
+        "help",
         "how",
         "i",
         "if",
@@ -304,12 +308,17 @@ _QUERY_RELEVANCE_STOP = frozenset(
         "please",
         "program",
         "provide",
+        "recommend",
+        "recommendation",
+        "recommendations",
+        "recommended",
         "really",
         "that",
         "the",
         "their",
         "there",
         "this",
+        "take",
         "to",
         "towards",
         "true",
@@ -317,6 +326,7 @@ _QUERY_RELEVANCE_STOP = frozenset(
         "we",
         "what",
         "which",
+        "want",
         "with",
         "long",
         "you",

tai_helper/monitoring.py

@@ -135,6 +135,7 @@ class HelperMonitor:
                 }
             if self.error_message:
                 span.error_info = {
-                    "type": "RuntimeError",
+                    "exception_type": "RuntimeError",
                     "message": _truncate(self.error_message, 1000),
+                    "traceback": "",
                 }

tests/test_grounding.py

@@ -575,6 +575,38 @@ def _query_bound_result(page: dict, query: str, target_offer_id: str):
     )
 
 
+def test_broad_course_decision_starter_accepts_retrieved_offer_evidence() -> None:
+    span = "Build and evaluate production-ready agentic AI systems."
+    page = _with_spans(
+        {
+            "chunk_id": "agent-engineering-overview",
+            "title": "Agent Engineering",
+            "kind": "course",
+            "offer_id": "agent-engineering",
+            "entity_id": "offer:agent-engineering",
+            "url": "https://towardsai.com/academy/agent-engineering/",
+            "headings": ["Agent Engineering"],
+            "text": span,
+        },
+        span,
+    )
+    raw = _model_json(
+        text=span,
+        quote=span,
+        chunk_id=page["chunk_id"],
+    )
+
+    result = llm.validate_grounded_result(
+        raw,
+        [page],
+        query="I want help deciding which course to take.",
+        target_offer_ids=frozenset(),
+    )
+
+    assert result.is_answered
+    assert result.answer == span
+
+
 def test_query_binding_rejects_an_exact_span_from_the_wrong_offer() -> None:
     page = _with_spans(
         {

tests/test_monitoring.py

@@ -0,0 +1,55 @@
+from __future__ import annotations
+
+from typing import Any
+
+from tai_helper.monitoring import HelperMonitor
+
+
+class _FakeSpan:
+    usage: dict[str, Any] | None = None
+    error_info: dict[str, str] | None = None
+
+
+class _FakeSpanContext:
+    def __init__(self, span: _FakeSpan) -> None:
+        self.span = span
+
+    def __enter__(self) -> _FakeSpan:
+        return self.span
+
+    def __exit__(self, *_args: object) -> None:
+        return None
+
+
+class _FakeOpik:
+    def __init__(self) -> None:
+        self.span = _FakeSpan()
+        self.kwargs: dict[str, Any] = {}
+
+    def start_as_current_span(self, **kwargs: Any) -> _FakeSpanContext:
+        self.kwargs = kwargs
+        return _FakeSpanContext(self.span)
+
+
+def test_error_span_uses_current_opik_error_info_schema() -> None:
+    opik = _FakeOpik()
+    monitor = HelperMonitor(
+        query="Unsupported claim",
+        answer="Contact the team.",
+        current_url="https://towardsai.com/academy/mentorship/",
+        selected_prompt="I want to find mentors",
+        visitor_key="visitor",
+        thread_id="thread",
+        sources=[],
+        usage={},
+        latency_ms=1,
+        error_message="grounding validation failed",
+    )
+
+    monitor._write_span(opik)
+
+    assert opik.span.error_info == {
+        "exception_type": "RuntimeError",
+        "message": "grounding validation failed",
+        "traceback": "",
+    }