File size: 1,633 Bytes
a0a9254 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 | """Compare the extracted model to frozen decisions from the original v5 server."""
from pathlib import Path
import argparse,json,urllib.request
ROOT=Path(__file__).resolve().parents[1]
def main():
p=argparse.ArgumentParser();p.add_argument('--endpoint',default='http://127.0.0.1:8774');p.add_argument('--out',type=Path,default=ROOT/'.local/parity.json');a=p.parse_args()
reference=json.loads((ROOT/'tests/reference-decisions.json').read_text());results=[]
for case in reference['cases']:
r=json.load(urllib.request.urlopen(urllib.request.Request(a.endpoint+'/api/evaluate',json.dumps(case['request']).encode(),{'Content-Type':'application/json'}),timeout=60))
diffs=[];choices=[]
for expected,actual in zip(case['expected'],r['states'],strict=True):
assert expected['id']==actual['id']
for qid,e in expected['answers'].items():
found=actual['answers'][qid];assert set(e['probabilities'])==set(found['probabilities'])
diffs.extend(abs(v-found['probabilities'][k]) for k,v in e['probabilities'].items());choices.append(e['choice']==found['choice'])
delta=max(diffs);passed=delta<=1e-4 and all(choices)
results.append({'case':case['case'],'maxProbabilityDifference':delta,'sameChoices':all(choices),'passed':passed})
print(json.dumps(results[-1]),flush=True)
out={'passed':all(x['passed'] for x in results),'cases':results,'tolerance':1e-4}
a.out.parent.mkdir(parents=True,exist_ok=True);a.out.write_text(json.dumps(out,indent=2)+'\n')
if not out['passed']:raise SystemExit(1)
if __name__=='__main__':main()
|