File size: 4,910 Bytes
5dcfc5c | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 | import json
from pathlib import Path
from figment.config import FigmentConfig
from scripts import run_eval
INITIAL_CASES = Path("data/eval/initial_handwritten_cases.jsonl")
def _jsonl(path: Path) -> list[dict]:
return [json.loads(line) for line in path.read_text(encoding="utf-8").splitlines() if line]
def test_canned_eval_runner_keeps_fallback_out_of_model_competence(tmp_path: Path) -> None:
output_path = tmp_path / "eval-results.jsonl"
summary = run_eval.run_eval(
case_paths=[INITIAL_CASES],
output_path=output_path,
config=FigmentConfig(model_backend="canned"),
)
records = _jsonl(output_path)
assert summary["total_cases"] == 10
assert len(records) == 10
assert summary["raw_configured_model_successes"] == 0
assert summary["repair_successes"] == 0
assert summary["canned_fallback_successes"] == 10
assert summary["competence_successes"] == 0
assert summary["final_validation_successes"] == 10
assert "expected_label_successes" in summary
assert "expected_label_check_successes" in summary
first = records[0]
assert first["case_id"] == "initial-ams-confusion-001"
assert first["model_backend"] == "canned"
assert first["model_stack"] == "omni_native"
assert first["active_model_id"]
assert first["fallback_tier"] == "canned"
assert first["fallback_reason"] == "canned_backend"
assert first["raw_configured_model_attempted"] is False
assert first["raw_configured_model_success"] is False
assert first["repair_attempted"] is False
assert first["repair_success"] is False
assert first["canned_fallback_used"] is True
assert first["canned_fallback_success"] is True
assert first["competence_success"] is False
assert first["final_validation"]["passed"] is True
assert first["expected_source_card_ids"] == [
"AMS-RED-FLAGS-v1",
"SAFETY-BOUNDARIES-v1",
"REFERRAL-SBAR-v1",
]
assert first["expected_missing_observations"]
assert first["forbidden_behavior"]
assert first["actual_protocol_urgency"] == first["final_output"]["protocol_urgency"]
assert first["actual_source_card_ids"] == first["final_output"]["source_cards"]
assert "expected_label_score" in first
assert first["expected_label_score"]["red_flags_match"] is True
assert first["expected_label_score"]["min_urgency_met"] is True
assert first["field_provenance"]["protocol_urgency"] == "deterministic_fallback"
assert summary["records_with_field_provenance"] == 10
assert summary["model_field_pass_rate"] == 0.0
assert summary["model_visible_fields_retained"] == 0.0
assert summary["deterministic_patch_count"] == len(first["field_provenance"]) * 10
assert first["latency_ms"] >= 0
assert isinstance(first["trace_hash"], str)
assert len(first["trace_hash"]) >= 12
assert first["raw_model_output"] is None
assert first["repaired_output"] is None
assert isinstance(first["fallback_output"], dict)
def test_eval_cli_runs_initial_cases_against_canned_without_network(tmp_path: Path) -> None:
output_path = tmp_path / "cli-results.jsonl"
exit_code = run_eval.main(
[
"--backend",
"canned",
"--cases",
str(INITIAL_CASES),
"--output",
str(output_path),
]
)
records = _jsonl(output_path)
assert exit_code == 0
assert len(records) == 10
assert {record["raw_configured_model_success"] for record in records} == {False}
assert {record["canned_fallback_used"] for record in records} == {True}
assert {record["final_validation"]["passed"] for record in records} == {True}
assert {record["field_provenance"]["source_cards"] for record in records} == {"deterministic_fallback"}
assert all("expected_label_score" in record for record in records)
def test_llama_eval_summary_describes_real_eval_evidence_scope(tmp_path: Path) -> None:
summary = run_eval._summarize(
[
{
"raw_configured_model_success": True,
"repair_success": False,
"canned_fallback_used": False,
"canned_fallback_success": False,
"competence_success": True,
"final_validation": {"passed": True},
}
],
FigmentConfig(model_backend="llama_cpp", model_stack="local_4b_parakeet"),
[INITIAL_CASES],
tmp_path / "local-eval.jsonl",
)
assert summary["local_llm_evidence"]["proof_status"] == "eval_records_summarized"
assert summary["local_llm_evidence"]["model_backend"] == "llama_cpp"
assert summary["local_llm_evidence"]["counts_as_50_case_local_llm_competence"] is False
assert summary["local_llm_evidence"]["competence_successes"] == 1
assert "MODEL_BACKEND=llama_cpp" in summary["local_llm_evidence"]["real_eval_command"]
|