| """Credential-free native validation; source and quant run identical cases.""" |
| import datetime |
| import json |
| import sys |
| from pathlib import Path |
| from huggingface_hub import HfApi, CommitOperationAdd, Volume, hf_hub_download |
|
|
| ROOT=Path(__file__).resolve().parent |
| repo='ProCreations/Agnes-3.0-Flash-NVFP4' |
| mode=sys.argv[1] |
| assert mode in ['bf16','nvfp4'] |
| api=HfApi() |
| rows=[json.loads(s) for s in (ROOT/'jobs.jsonl').read_text().splitlines()] |
| committed=0 |
| for row in rows: |
| job=api.inspect_job(job_id=row['job_id']) |
| assert not (row['phase']=='native-'+mode and job.status.stage in ['RUNNING','SCHEDULING']) |
| if job.status.stage in ['RUNNING','SCHEDULING']: |
| committed+=row['maximum_cost_usd'] |
| else: |
| elapsed=max(0,((job.finished_at or datetime.datetime.now(datetime.timezone.utc))-job.created_at).total_seconds()) |
| if job.status.stage=='CANCELED': elapsed=min(elapsed,row.get('canceled_elapsed_seconds_upper_bound',elapsed)) |
| committed+=min(elapsed/3600,row['timeout_hours'])*row['rate_usd_hour'] |
| rate=next(h.unit_cost_usd*60 for h in api.list_jobs_hardware() if h.name=='rtx-pro-6000') |
| hours=3 |
| assert committed+hours*rate<=40 |
| names=['native_eval.py','agent_eval.py','long_agent_eval.py','evalplus_compat.py', |
| 'stage_checkpoint.py','select_longbench.py','longbench_selection.json', |
| 'native_plan.json','bootstrap_native.sh','dispatch_native.py'] |
| if mode=='nvfp4': |
| info=api.model_info(repo,files_metadata=True) |
| report=json.loads(Path(hf_hub_download(repo,'quality_report.json',revision=info.sha)).read_text()) |
| assert report['status']=='packed_export_verified_pending_native_evaluation' |
| files=[] |
| for file in info.siblings: |
| name=file.rfilename |
| if name.startswith('reproducibility/') or name in ['README.md','.gitattributes','quality_report.json']: continue |
| if (name.endswith(('.safetensors','.json','.py','.jinja','.txt','.sh')) or name.startswith('sglang_patch/')): |
| files.append(dict(name=name,size=file.size,hash_algorithm='sha256' if file.lfs else 'git-sha1',hash=file.lfs.sha256 if file.lfs else file.blob_id)) |
| assert sum(f['name'].endswith('.safetensors') for f in files)>=2 |
| (ROOT/'native_quant_files.json').write_text(json.dumps(dict(source_files_revision=info.sha,files=sorted(files,key=lambda x:x['name'])),indent=2)) |
| names+=['native_quant_files.json'] |
| commit=api.create_commit(repo_id=repo,operations=[CommitOperationAdd(path_in_repo='reproducibility/'+name,path_or_fileobj=str(ROOT/name)) for name in names],commit_message='Pin matched Agnes native validation suite') |
| image_info=json.loads((ROOT/'runtime_image.json').read_text()) |
| image=image_info['image']+'@'+image_info['digest'] |
| job=api.run_job(image=image,flavor='rtx-pro-6000',timeout=int(hours*3600), |
| command=['bash','-c',(ROOT/'bootstrap_native.sh').read_text()], |
| env={'AGNES_EVAL_MODE':mode,'AGNES_QUANT_REVISION':commit.oid,'AGNES_CODE_REVISION':commit.oid, |
| 'HF_HOME':'/workspace/native-eval/hf-cache','PYTHONUNBUFFERED':'1', |
| 'TOKENIZERS_PARALLELISM':'false','OMP_NUM_THREADS':'8', |
| 'HF_HUB_DOWNLOAD_TIMEOUT':'180','TQDM_MININTERVAL':'30'}, |
| volumes=[Volume(type='model',source=repo,mount_path='/models/quant',revision=commit.oid,read_only=True)], |
| labels={'project':'agnes-flash-nvfp4','phase':'native-'+mode}) |
| row=dict(job_id=job.id,url=job.url,phase='native-'+mode,mode=mode,flavor='rtx-pro-6000', |
| image=image,rate_usd_hour=rate,timeout_hours=hours,maximum_cost_usd=hours*rate, |
| code_revision=commit.oid,credentials_passed=False) |
| with (ROOT/'jobs.jsonl').open('a') as f:f.write(json.dumps(row)+'\n') |
| print(json.dumps(row)) |
|
|
