| """Compare the extracted model to frozen decisions from the original v5 server.""" |
| from pathlib import Path |
| import argparse,json,urllib.request |
| ROOT=Path(__file__).resolve().parents[1] |
| def main(): |
| p=argparse.ArgumentParser();p.add_argument('--endpoint',default='http://127.0.0.1:8774');p.add_argument('--out',type=Path,default=ROOT/'.local/parity.json');a=p.parse_args() |
| reference=json.loads((ROOT/'tests/reference-decisions.json').read_text());results=[] |
| for case in reference['cases']: |
| r=json.load(urllib.request.urlopen(urllib.request.Request(a.endpoint+'/api/evaluate',json.dumps(case['request']).encode(),{'Content-Type':'application/json'}),timeout=60)) |
| diffs=[];choices=[] |
| for expected,actual in zip(case['expected'],r['states'],strict=True): |
| assert expected['id']==actual['id'] |
| for qid,e in expected['answers'].items(): |
| found=actual['answers'][qid];assert set(e['probabilities'])==set(found['probabilities']) |
| diffs.extend(abs(v-found['probabilities'][k]) for k,v in e['probabilities'].items());choices.append(e['choice']==found['choice']) |
| delta=max(diffs);passed=delta<=1e-4 and all(choices) |
| results.append({'case':case['case'],'maxProbabilityDifference':delta,'sameChoices':all(choices),'passed':passed}) |
| print(json.dumps(results[-1]),flush=True) |
| out={'passed':all(x['passed'] for x in results),'cases':results,'tolerance':1e-4} |
| a.out.parent.mkdir(parents=True,exist_ok=True);a.out.write_text(json.dumps(out,indent=2)+'\n') |
| if not out['passed']:raise SystemExit(1) |
| if __name__=='__main__':main() |
|
|
