fix(evals): never judge faithfulness on silently truncated evidence

run_battery captures tool outputs at 40k chars and evidence_is_complete
gates faithfulness rows on nothing being truncated at capture time, but
faithfulness_evidence then sliced the joined evidence to 12k (behind a
stale "~6000 chars per output" comment predating the 40k raise): the
judge could see under a third of the grounding it was grading, yielding
false fabrication verdicts, the exact blind-judge failure class the
completeness gate exists to prevent.

Evidence now reaches the judge untruncated or the row is not emitted:
a new evidence_fits gate (cap raised 12k -> 200k, covering what capture
guarantees) excludes over-cap turns instead of slicing them, matching
the harness's exclude-don't-judge-blind philosophy. Stale comments
fixed. (The test file also carries the CLI tests for the check_triggers
fix in the next commit.)

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>

evals/grade.py

@@ -50,10 +50,16 @@ RETRIEVAL_TOOLS = {"retrieve_tutor_context", "run_kb_command"}
 # in the command or its output reveal which corpus source the agent touched.
 KB_RAW_SOURCE_RE = re.compile(r"raw/(?:docs|courses)/([A-Za-z0-9_.\-]+)/")
 
-# Per-bundle evidence cap for faithfulness rows. Tool outputs in a bundle are each
-# capped at ~6000 chars; a few calls per turn, so ~12k keeps the gist of every
-# call without bloating the grading prompt.
-FAITHFULNESS_EVIDENCE_MAX = 12_000
+# Judge-prompt budget for a faithfulness row's evidence. Bundles capture each
+# tool output at run_battery.TOOL_OUTPUT_MAX_CHARS = 40_000 chars (the KB
+# shell's own per-call output cap), so one turn's complete evidence can span
+# several 40k outputs; 200k covers ~5 max-size calls. This is a GATE, not a
+# slice: quality_sheet_rows emits a faithfulness row only when the FULL
+# assembled evidence fits (see evidence_fits), so the judge never grades
+# silently truncated grounding (the evals.md F23/F24 blind-signal class).
+# Worst case per row is ~200k chars (~50k tokens) of judge prompt; typical
+# turns are far smaller, and anything larger is excluded, not sliced.
+FAITHFULNESS_EVIDENCE_MAX = 200_000
 
 # Holistic = the "would staff approve sending this?" gate (evals/background.md
 # Layer 4). Emitted for every single-turn answer; the full rubric lives in
@@ -74,16 +80,17 @@ FAITHFULNESS_CRITERION = (
 )
 
 
-def faithfulness_evidence(
-    bundle: dict[str, Any], max_chars: int = FAITHFULNESS_EVIDENCE_MAX
-) -> str:
+def faithfulness_evidence(bundle: dict[str, Any]) -> str:
     """Concatenate the retrieval/KB evidence the tutor saw, for grounding grades.
 
     Joins each retrieval/KB tool call's command + output (and, for
     `retrieve_tutor_context`, the ranked source titles/URLs) so the judge can
-    check the answer's claims against exactly what the tools returned. Empty when
-    the turn used no retrieval/KB tool (e.g. answer_general), which is the signal
-    to skip the faithfulness row for that case.
+    check the answer's claims against exactly what the tools returned. Returns
+    the evidence UNTRUNCATED: whether it fits the judge prompt is a separate
+    gate (evidence_fits), so an oversized turn is excluded from faithfulness
+    grading rather than judged against a silent slice. Empty when the turn used
+    no retrieval/KB tool (e.g. answer_general), which is the signal to skip the
+    faithfulness row for that case.
     """
     parts: list[str] = []
     for call in bundle.get("tool_calls") or []:
@@ -100,7 +107,7 @@ def faithfulness_evidence(
             body = f"{body}\nSOURCES: {cites}".strip()
         if header or body:
             parts.append(f"{header}\n{body}".strip())
-    return "\n\n".join(parts)[:max_chars]
+    return "\n\n".join(parts)
 
 
 def evidence_is_complete(bundle: dict[str, Any]) -> bool:
@@ -125,6 +132,17 @@ def evidence_is_complete(bundle: dict[str, Any]) -> bool:
     return True
 
 
+def evidence_fits(evidence: str, max_chars: int = FAITHFULNESS_EVIDENCE_MAX) -> bool:
+    """True if the assembled evidence fits the faithfulness judge-prompt budget.
+
+    Companion gate to ``evidence_is_complete``: that one guarantees nothing was
+    lost at CAPTURE time, this one guarantees nothing would be lost at JUDGE
+    time. A turn whose complete evidence still exceeds the cap is excluded from
+    faithfulness grading instead of being judged on truncated grounding.
+    """
+    return len(evidence) <= max_chars
+
+
 def index_battery(battery: list[dict[str, Any]]) -> dict[str, dict[str, Any]]:
     index: dict[str, dict[str, Any]] = {}
     for record in battery:
@@ -143,8 +161,9 @@ def kb_browsed_sources(bundle: dict[str, Any]) -> set[str]:
 
     Lets recall credit KB grounding, which recall@shown cannot see (KB output is
     raw text, not structured matches). Caveats: bundle output is capped at
-    6000 chars, so a path past the cap is missed; a broad ``rg`` can surface many
-    sources at once (an over-count toward "shown", not "used").
+    ``run_battery.TOOL_OUTPUT_MAX_CHARS`` (40k) per call, so a path past the cap
+    is missed; a broad ``rg`` can surface many sources at once (an over-count
+    toward "shown", not "used").
     """
     seen: set[str] = set()
     for call in bundle.get("tool_calls") or []:
@@ -321,12 +340,15 @@ def quality_sheet_rows(
             reference=reference,
         )
     ]
-    # Faithfulness only when the agent's grounding was captured in full -- on
-    # bundles recorded under the old 6k tool-output cap, KB-browse evidence is
-    # truncated, so a faithfulness grade would score capture-completeness, not
-    # grounding (evals.md F23/F24 class). Re-record with the raised cap to enable.
+    # Faithfulness only when the judge can see 100% of what the agent grounded
+    # on: (1) every retrieval/KB output was captured in full at record time
+    # (bundles recorded under the old 6k tool-output cap fail this; re-record
+    # at the 40k cap to re-enable), and (2) the full assembled evidence fits
+    # the judge-prompt budget. A turn failing either is excluded rather than
+    # judged on silently truncated evidence, which would score
+    # capture-completeness, not grounding (evals.md F23/F24 class).
     evidence = faithfulness_evidence(bundle)
-    if evidence and evidence_is_complete(bundle):
+    if evidence and evidence_is_complete(bundle) and evidence_fits(evidence):
         rows.append(
             sheet_row(
                 bundle=bundle,

tests/test_evals_grade.py

@@ -1,13 +1,18 @@
 from __future__ import annotations
 
 import csv
+import sys
 import tempfile
 import unittest
 from pathlib import Path
+from unittest import mock
 
+from evals import check_triggers
 from evals.common import detect_battery_type, normalize_url, percentile
 from evals.grade import (
+    FAITHFULNESS_EVIDENCE_MAX,
     behavior_heuristic,
+    evidence_fits,
     evidence_is_complete,
     faithfulness_evidence,
     grade_persona_question,
@@ -182,11 +187,16 @@ class FaithfulnessEvidenceTests(unittest.TestCase):
         self.assertEqual(faithfulness_evidence(bundle), "")
         self.assertEqual(faithfulness_evidence({}), "")
 
-    def test_respects_max_chars(self) -> None:
+    def test_returns_full_evidence_untruncated(self) -> None:
+        # One full-size KB capture (run_battery caps each output at 40k). The
+        # old 12k slice here silently cut the judge's evidence to <1/3 of the
+        # grounding; evidence must now come back whole -- fitting the judge
+        # prompt is a separate gate (evidence_fits), not a slice.
         bundle = {
-            "tool_calls": [{"tool_name": "run_kb_command", "output_text": "z" * 5000}]
+            "tool_calls": [{"tool_name": "run_kb_command", "output_text": "z" * 40_000}]
         }
-        self.assertEqual(len(faithfulness_evidence(bundle, max_chars=100)), 100)
+        evidence = faithfulness_evidence(bundle)
+        self.assertGreaterEqual(len(evidence), 40_000)
 
 
 class EvidenceCompletenessTests(unittest.TestCase):
@@ -252,6 +262,97 @@ class EvidenceCompletenessTests(unittest.TestCase):
         self.assertIn("faithfulness", types_full)  # full evidence -> emitted
 
 
+class EvidenceFitsTests(unittest.TestCase):
+    """The judge-prompt side of the completeness guarantee: a faithfulness row
+    carries the FULL evidence or is not emitted at all -- never a silent slice
+    (the F23/F24 blind-judge class the capture gate exists to prevent)."""
+
+    @staticmethod
+    def _bundle(output_text: str) -> dict:
+        return {
+            "run_id": "r",
+            "battery_type": "singleturn",
+            "preset": "prod",
+            "query": "q",
+            "answer": "a",
+            "tool_calls": [
+                {
+                    "tool_name": "run_kb_command",
+                    "args_text": "cat raw/docs/x/big.md",
+                    # Fully captured: output_chars == len(output_text), so the
+                    # capture-completeness gate passes.
+                    "output_text": output_text,
+                    "output_chars": len(output_text),
+                }
+            ],
+        }
+
+    def test_evidence_fits_predicate(self) -> None:
+        self.assertTrue(evidence_fits("x" * FAITHFULNESS_EVIDENCE_MAX))
+        self.assertFalse(evidence_fits("x" * (FAITHFULNESS_EVIDENCE_MAX + 1)))
+        self.assertTrue(evidence_fits("abc", max_chars=3))
+        self.assertFalse(evidence_fits("abcd", max_chars=3))
+
+    def test_cap_covers_a_realistic_multi_call_kb_turn(self) -> None:
+        # The gate must not exclude what capture guarantees: a few full 40k
+        # KB outputs (run_battery.TOOL_OUTPUT_MAX_CHARS) per turn must fit.
+        self.assertGreaterEqual(FAITHFULNESS_EVIDENCE_MAX, 4 * 40_000)
+
+    def test_faithfulness_row_carries_full_evidence(self) -> None:
+        # One complete 40k KB output: under the old 12k slice the judge saw
+        # ~30% of the grounding. The row must now carry all of it.
+        text = ("y" * 39_990) + "END-MARKER"
+        rows = [
+            r
+            for r in quality_sheet_rows(self._bundle(text))
+            if r["item_type"] == "faithfulness"
+        ]
+        self.assertEqual(len(rows), 1)
+        self.assertGreater(len(rows[0]["reference"]), 12_000)  # old cap is gone
+        self.assertTrue(rows[0]["reference"].endswith("END-MARKER"))
+
+    def test_faithfulness_row_excluded_when_evidence_exceeds_cap(self) -> None:
+        # Fully captured but too big for the judge prompt: excluded, not
+        # silently truncated. Holistic (which never uses evidence) still emits.
+        bundle = self._bundle("z" * (FAITHFULNESS_EVIDENCE_MAX + 1))
+        types = {r["item_type"] for r in quality_sheet_rows(bundle)}
+        self.assertIn("holistic", types)
+        self.assertNotIn("faithfulness", types)
+
+
+class CheckTriggersMainTests(unittest.TestCase):
+    """check_triggers must evaluate EVERY run: all() over a bare generator
+    short-circuited on the first failing run, hiding later runs' diagnostics."""
+
+    def _run_main(self, runs: list[str], results: list[bool]) -> tuple[list[Path], int]:
+        calls: list[Path] = []
+
+        def fake_check_run(run_dir: Path, expect_none: bool) -> bool:
+            calls.append(run_dir)
+            return results[len(calls) - 1]
+
+        argv = ["check_triggers", "--runs", *runs]
+        with (
+            mock.patch.object(check_triggers, "check_run", fake_check_run),
+            mock.patch.object(sys, "argv", argv),
+            self.assertRaises(SystemExit) as ctx,
+        ):
+            check_triggers.main()
+        return calls, ctx.exception.code
+
+    def test_later_runs_still_checked_after_a_failure(self) -> None:
+        calls, code = self._run_main(
+            ["runs/a", "runs/b", "runs/c"], [False, True, True]
+        )
+        self.assertEqual(calls, [Path("runs/a"), Path("runs/b"), Path("runs/c")])
+        self.assertEqual(code, 1)  # exit semantics preserved
+
+    def test_exit_zero_when_all_runs_pass(self) -> None:
+        calls, code = self._run_main(["runs/a", "runs/b"], [True, True])
+        self.assertEqual(len(calls), 2)
+        self.assertEqual(code, 0)
+
+
 class JudgeRubricTests(unittest.TestCase):
     def test_holistic_and_faithfulness_rubrics_exist(self) -> None:
         self.assertIs(rubric_for("holistic"), RUBRICS["holistic"])