File size: 2,702 Bytes
5dcfc5c | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 | import json
from pathlib import Path
COMPREHENSIVE_CASES_PATH = Path("data/eval/comprehensive_hosted_cases.jsonl")
PROTOCOL_CARDS_DIR = Path("data/protocol_cards")
REQUIRED_CASE_FIELDS = {
"case_id",
"target_protocol_card_id",
"structured_intake",
"expected_red_flag_rule_ids",
"expected_min_protocol_urgency",
"expected_source_card_ids",
"expected_missing_observations",
"safety_notes",
"forbidden_behavior",
}
REQUIRED_INTAKE_FIELDS = {
"setting",
"patient_age",
"pregnancy_status",
"chief_concern",
"symptoms",
"vitals",
"allergies",
"medications",
"available_supplies",
"responder_note",
"confirmed",
}
PHI_MARKERS = ("patient_name", "full_name", "address", "phone", "email", "dob", "date_of_birth")
def test_comprehensive_hosted_eval_cases_have_expected_size_and_card_coverage() -> None:
cases = _load_cases(COMPREHENSIVE_CASES_PATH)
known_card_ids = _known_card_ids()
assert len(cases) == 37
assert {case["target_protocol_card_id"] for case in cases} == known_card_ids
assert len({case["case_id"] for case in cases}) == len(cases)
def test_comprehensive_hosted_eval_cases_have_valid_shape_and_no_phi() -> None:
cases = _load_cases(COMPREHENSIVE_CASES_PATH)
known_card_ids = _known_card_ids()
serialized = json.dumps(cases).lower()
for case in cases:
assert set(case) == REQUIRED_CASE_FIELDS
assert case["target_protocol_card_id"] in known_card_ids
assert set(case["expected_source_card_ids"]) <= known_card_ids
assert case["target_protocol_card_id"] in case["expected_source_card_ids"]
assert REQUIRED_INTAKE_FIELDS <= set(case["structured_intake"])
assert case["structured_intake"]["confirmed"] is True
assert case["expected_min_protocol_urgency"] in {"routine", "monitor", "urgent", "emergency"}
assert isinstance(case["expected_red_flag_rule_ids"], list)
assert isinstance(case["expected_missing_observations"], list) and case["expected_missing_observations"]
assert isinstance(case["safety_notes"], list) and case["safety_notes"]
assert isinstance(case["forbidden_behavior"], list) and case["forbidden_behavior"]
assert "data:audio" not in serialized
assert "base64" not in serialized
for marker in PHI_MARKERS:
assert marker not in serialized
def _load_cases(path: Path) -> list[dict]:
return [json.loads(line) for line in path.read_text(encoding="utf-8").splitlines() if line]
def _known_card_ids() -> set[str]:
return {
json.loads(path.read_text(encoding="utf-8"))["card_id"]
for path in PROTOCOL_CARDS_DIR.glob("*.json")
}
|