"""Validate the published dataset hashes, counts and response structure."""
import hashlib,json
from pathlib import Path
root=Path(__file__).resolve().parent
spec={
 "train":(1138,"33a9949ef2e43f97581ba6e371007298027c35c4266924c40a5dc8c3dc25ada0"),
 "validation":(126,"dc1d73be7d960fa952303951aac2ed45b1068e3aeda17c2a28b8a0407fd6a47f"),
}
groups={}
for split,(count,digest) in spec.items():
    blob=(root/"data"/(split+".jsonl")).read_bytes()
    assert hashlib.sha256(blob).hexdigest()==digest,split
    rows=[json.loads(x) for x in blob.decode("utf-8").splitlines() if x.strip()]
    assert len(rows)==count
    groups[split]={r["group_id"] for r in rows}
    for r in rows:
        assert [m["role"] for m in r["messages"]]==["system","user","assistant"]
        assert json.loads(r["messages"][-1]["content"])==r["response"]
        assert set(r["response"])=={"answer","practice_question","practice_answer","new_words","encouragement"}
    print(split,count,digest)
assert not groups["train"] & groups["validation"],"Concept-group leakage"
print("Hashes, counts, messages and concept separation verified.")