ProCreations commited on
Commit
fd26a7a
·
verified ·
1 Parent(s): 63f6cfd

Pin plain NVFP4 build, source, and bounded validation plan

Browse files
reproducibility/bootstrap_build.sh CHANGED
@@ -5,7 +5,7 @@ python -m pip install --break-system-packages --no-cache-dir uv
5
  uv venv --system-site-packages /workspace/agnes/venv
6
  export PATH="/workspace/agnes/venv/bin:$PATH"
7
  uv pip install --python /workspace/agnes/venv/bin/python \
8
- 'torch==2.13.0' 'transformers==5.14.0' 'accelerate==1.12.0' \
9
  'huggingface_hub==1.18.0' 'pillow>=12' 'sentencepiece>=0.2' 'einops' \
10
  'nvidia-modelopt @ https://github.com/NVIDIA/Model-Optimizer/archive/5cae3940402f1ced98069a666b0bec72ec8b33b5.tar.gz'
11
  python - <<'PY'
 
5
  uv venv --system-site-packages /workspace/agnes/venv
6
  export PATH="/workspace/agnes/venv/bin:$PATH"
7
  uv pip install --python /workspace/agnes/venv/bin/python \
8
+ 'torch==2.13.0' 'transformers==5.12.1' 'accelerate==1.12.0' \
9
  'huggingface_hub==1.18.0' 'pillow>=12' 'sentencepiece>=0.2' 'einops' \
10
  'nvidia-modelopt @ https://github.com/NVIDIA/Model-Optimizer/archive/5cae3940402f1ced98069a666b0bec72ec8b33b5.tar.gz'
11
  python - <<'PY'
reproducibility/build.py CHANGED
@@ -55,6 +55,11 @@ def main():
55
  REPORT['versions'] = {p:importlib.metadata.version(p) for p in ['torch','transformers','nvidia-modelopt','accelerate']}
56
  REPORT['hardware'] = [torch.cuda.get_device_name(0)]
57
  record()
 
 
 
 
 
58
  source = Path(snapshot_download(SOURCE, revision=REV, max_workers=8))
59
  src_index = json.loads((source/'model.safetensors.index.json').read_text())
60
  src_map = src_index['weight_map']
 
55
  REPORT['versions'] = {p:importlib.metadata.version(p) for p in ['torch','transformers','nvidia-modelopt','accelerate']}
56
  REPORT['hardware'] = [torch.cuda.get_device_name(0)]
57
  record()
58
+ from transformers.dynamic_module_utils import get_class_from_dynamic_module
59
+ source = Path(snapshot_download(SOURCE, revision=REV, allow_patterns=['*.py','config.json'], max_workers=8))
60
+ cls = get_class_from_dynamic_module('modeling_agnes.AgnesForConditionalGeneration',source)
61
+ assert cls.__name__ == 'AgnesForConditionalGeneration'
62
+ print('AGNES_REMOTE_IMPLEMENTATION_IMPORT_PASS',flush=True)
63
  source = Path(snapshot_download(SOURCE, revision=REV, max_workers=8))
64
  src_index = json.loads((source/'model.safetensors.index.json').read_text())
65
  src_map = src_index['weight_map']
reproducibility/dispatch_build.py CHANGED
@@ -1,5 +1,6 @@
1
  """One data-free build; hard timeout and a separate bounded Agnes ledger."""
2
  import json
 
3
  from pathlib import Path
4
  from huggingface_hub import HfApi, CommitOperationAdd, get_token
5
 
@@ -8,7 +9,13 @@ api=HfApi()
8
  repo='ProCreations/Agnes-3.0-Flash-NVFP4'
9
  assert api.whoami()['name']=='ProCreations'
10
  ledger=ROOT/'jobs.jsonl'
11
- assert not ledger.exists(),'Inspect previous attempts before dispatching again'
 
 
 
 
 
 
12
  api.create_repo(repo_id=repo,private=True,exist_ok=True)
13
  assert api.model_info(repo).private
14
  assert not any(n.endswith('.safetensors') for n in api.list_repo_files(repo))
@@ -48,5 +55,5 @@ job=api.run_job(image='pytorch/pytorch:2.13.0-cuda13.0-cudnn9-devel',flavor='h20
48
  secrets={'HF_TOKEN':get_token()},labels={'project':'agnes-flash-nvfp4','phase':'plain-build'})
49
  row=dict(job_id=job.id,url=job.url,phase='plain-build',flavor='h200',rate_usd_hour=rate,
50
  timeout_hours=1.5,maximum_cost_usd=rate*1.5,code_revision=commit.oid)
51
- ledger.write_text(json.dumps(row)+'\n')
52
  print(json.dumps(row))
 
1
  """One data-free build; hard timeout and a separate bounded Agnes ledger."""
2
  import json
3
+ import sys
4
  from pathlib import Path
5
  from huggingface_hub import HfApi, CommitOperationAdd, get_token
6
 
 
9
  repo='ProCreations/Agnes-3.0-Flash-NVFP4'
10
  assert api.whoami()['name']=='ProCreations'
11
  ledger=ROOT/'jobs.jsonl'
12
+ previous=[json.loads(s) for s in ledger.read_text().splitlines()] if ledger.exists() else []
13
+ if previous:
14
+ assert sys.argv[1:]==['--retry-setup']
15
+ assert all(api.inspect_job(job_id=r['job_id']).status.stage=='ERROR' for r in previous if r['phase']=='plain-build')
16
+ assert sum(r['maximum_cost_usd'] for r in previous)+7.5<=40
17
+ else:
18
+ assert not sys.argv[1:]
19
  api.create_repo(repo_id=repo,private=True,exist_ok=True)
20
  assert api.model_info(repo).private
21
  assert not any(n.endswith('.safetensors') for n in api.list_repo_files(repo))
 
55
  secrets={'HF_TOKEN':get_token()},labels={'project':'agnes-flash-nvfp4','phase':'plain-build'})
56
  row=dict(job_id=job.id,url=job.url,phase='plain-build',flavor='h200',rate_usd_hour=rate,
57
  timeout_hours=1.5,maximum_cost_usd=rate*1.5,code_revision=commit.oid)
58
+ with ledger.open('a') as f:f.write(json.dumps(row)+'\n')
59
  print(json.dumps(row))