| """Current release acceptance; no test count is a production maturity score.""" |
| from pathlib import Path |
| import argparse,csv,json,subprocess,sys,re,hashlib,ast |
| ROOT=Path(__file__).resolve().parents[1];sys.path.insert(0,str(ROOT)) |
| import numpy as np |
|
|
|
|
| def load(name):return json.loads((ROOT/name).read_text()) |
|
|
|
|
| def run(output,require_replay=False): |
| gates=[] |
| def gate(name,passed,detail):gates.append({'name':name,'passed':bool(passed),'details':detail}) |
| required=['README.md','MANUSCRIPT.md','AUREOLE_R_v3.0.0_Certified_Innovation_Rendering.pdf','STATUS.json','CLAIMS.json','AI_AGENT_INDEX.json', |
| 'THEOREM_INDEX.md','docs/INNOVATION_THEORY.md','docs/CERTIFICATE_API.md','models/visibility_prior.npz', |
| 'results_v3/innovation_raw.csv','results_v3/queries_raw.csv','RUN_INNOVATION.bat','PUBLISH_HF.bat','CITATION.cff','LICENSE'] |
| missing=[p for p in required if not (ROOT/p).is_file()] |
| gate('required_files',not missing,{'missing':missing}) |
| total=0 |
| for start,count,name in [('tests',44,'current'),('legacy/code',14,'foundational')]: |
| p=subprocess.run([sys.executable,'-m','unittest','discover','-s',start,'-p','test_*.py','-v'],cwd=ROOT,capture_output=True,text=True) |
| match=re.search(r'Ran (\d+) tests?',p.stderr);actual=int(match.group(1)) if match else 0;total+=actual |
| if output.startswith('results_v3/'):(ROOT/f'results_v3/{name}_tests.log').write_text(p.stdout+p.stderr) |
| gate(name+'_tests',p.returncode==0 and actual==count,{'passed_count':actual,'expected':count}) |
| r=load('results_v3/innovation_report.json');q=load('results_v3/queries_report.json');status=load('STATUS.json') |
| old=set(load('results/training.json')['training_scene_ids'])|set(load('results/training.json')['validation_scene_ids'])|set(load('results/training.json')['test_scene_ids'])|set(load('results/followup_report.json')['protocol']['scene_ids']) |
| a=set(r['protocol']['scene_ids']);b=set(q['protocol']['scene_ids']) |
| gate('held_out_splits',not(a&b or a&old or b&old),{'new_motion_scenes':len(a),'new_query_scenes':len(b)}) |
| raw=list(csv.DictReader((ROOT/'results_v3/innovation_raw.csv').open())) |
| errs=[] |
| for s in r['summary']: |
| vals=[float(x['expected_mse']) for x in raw if x['phase']==s['phase'] and x['method']==s['method']] |
| errs.append(abs(np.mean(vals)-s['expected_mse'])) |
| rays=sum(int(x['rays']) for x in raw) |
| gate('motion_raw_consistency',len(raw)==r['records'] and rays==r['online_segment_queries'] and max(errs)<1e-12, |
| {'records':len(raw),'queries':rays,'max_summary_difference':float(max(errs))}) |
| supported=[x for x in raw if x['method']!='unsafe_eliminate'] |
| false=sum(int(x['false_certificates']) for x in supported);width=max(float(x['enclosure_violation']) for x in supported) |
| gate('supported_certificate_audit',false==0 and width<1e-12,{'false_acceptances':false,'max_interval_violation':width}) |
| unsafe=sum(int(x['false_certificates']) for x in raw if x['method']=='unsafe_eliminate') |
| gate('unsafe_negative_control_retained',unsafe>0,{'false_acceptances_in_deliberately_unsafe_ablation':unsafe}) |
| qr=list(csv.DictReader((ROOT/'results_v3/queries_raw.csv').open())) |
| fresh=sum(int(x['fresh_queries']) for x in qr);base=sum(int(x['shared_fresh_baseline_queries']) for x in qr) |
| gate('query_family_accounting',fresh==q['residual_queries'] and base==q['shared_fresh_baseline_queries'] and q['total_queries_including_initialization']==fresh+q['initialization_queries'], |
| {'initialization':q['initialization_queries'],'residual_queries':fresh,'shared_baseline':base}) |
| gate('independent_readout_audit',q['false_certificates']==0 and q['max_linear_rgb_error']<1e-12, |
| {'false_certificates':q['false_certificates'],'max_linear_error':q['max_linear_rgb_error']}) |
| hashes=[] |
| for file,report in [('experiments_innovation.json',r),('experiments_queries.json',q)]: |
| hashes.append(hashlib.sha256((ROOT/file).read_bytes()).hexdigest()==report['protocol_sha256']) |
| gate('frozen_protocol_hashes',all(hashes),{'both_match':all(hashes)}) |
| v2=load('results/followup_report.json');retained=load('results/rendering_report.json') |
| gate('prior_evidence_retained',v2['record_count']==5280 and retained['record_count']==7392, |
| {'v2_frame_method_records':v2['record_count']+retained['record_count']}) |
| errors=[] |
| if require_replay: |
| rr=load('innovation_reproduced/innovation_report.json');qq=load('queries_reproduced/queries_report.json') |
| for original,again in zip(r['summary'],rr['summary'],strict=True): |
| for key in ('expected_mse','observed_mse','rays','certified_fraction_before','false_certificates','max_enclosure_violation'): |
| errors.append(abs(original[key]-again[key])) |
| for key in ('initialization_queries','residual_queries','false_certificates','max_linear_rgb_error'): |
| errors.append(abs(q[key]-qq[key])) |
| gate('full_v3_replay',max(errors)<1e-12,{'metrics':len(errors),'max_difference':float(max(errors)),'timings_excluded':True}) |
| else: |
| recorded=ROOT/'results_v3/replay_validation.json' |
| gate('recorded_v3_replay',recorded.exists() and load('results_v3/replay_validation.json')['passed'],{'live_replay':False}) |
| demo=load('results_v3/demo/demo.json') |
| gate('finite_completion_demo',demo['final_max_error']<1e-12 and demo['physical_queries']==demo['full_domain_terms'] and demo['zero_query_final_frames']>0,demo|{'records':'see demo JSON'}) |
| sources=list((ROOT/'aureole').glob('*.py'))+list((ROOT/'scripts').glob('*.py')) |
| for p in sources:ast.parse(p.read_text()) |
| gate('source_syntax',True,{'files':len(sources)}) |
| gate('honest_scope',status['verified']['gpu_benchmarks']==0 and status['verified']['full_sr_rr_fg_experiments']==0 and status['overall_maturity_estimate_percent']<100, |
| {'maturity_is_subjective':True,'no_gpu_or_full_pipeline_claim':True,'executable_tests':total}) |
| report={'version':'3.0.0','passed':all(x['passed'] for x in gates),'gates_passed':sum(x['passed'] for x in gates),'gates_total':len(gates),'tests':total,'gates':gates, |
| 'scope':'Acceptance of the finite CPU reference; not full scientific maturity or production certification.'} |
| path=ROOT/output;path.parent.mkdir(parents=True,exist_ok=True);path.write_text(json.dumps(report,indent=2)+'\n') |
| if require_replay and output.startswith('results_v3/'): |
| (ROOT/'results_v3/replay_validation.json').write_text(json.dumps({'passed':max(errors)<1e-12,'metrics':len(errors),'max_nontiming_difference':float(max(errors))},indent=2)+'\n') |
| print(json.dumps({'passed':report['passed'],'gates_passed':report['gates_passed'],'gates_total':report['gates_total'],'tests':total,'failures':[x for x in gates if not x['passed']]},indent=2)) |
| return 0 if report['passed'] else 1 |
|
|
|
|
| if __name__=='__main__': |
| p=argparse.ArgumentParser();p.add_argument('--output',default='validation_reproduced.json');p.add_argument('--require-replay',action='store_true');args=p.parse_args();raise SystemExit(run(args.output,args.require_replay)) |
|
|
