| """Commit run artifacts atomically and verify every changed file by SHA256.""" |
| import hashlib, json, os, time |
| from pathlib import Path |
| from huggingface_hub import HfApi, CommitOperationAdd, hf_hub_download |
|
|
| REPO = 'User-2468/mini-unet-colorizer' |
| PREFIX = 'experiments/round6-20260928' |
|
|
| class DurableRun: |
| def __init__(self, root): |
| if not os.environ.get('HF_TOKEN'): |
| raise RuntimeError('A write token is required before starting') |
| self.api = HfApi(token=os.environ['HF_TOKEN']) |
| if self.api.whoami()['name'] != 'User-2468': |
| raise RuntimeError('Unexpected account') |
| self.root = Path(root) |
| self.saved = {} |
|
|
| def sync(self, reason): |
| files = [p for p in sorted(self.root.rglob('*')) if p.is_file()] |
| hashes = {p.relative_to(self.root).as_posix(): hashlib.sha256(p.read_bytes()).hexdigest() for p in files} |
| changed = [p for p in files if self.saved.get(p.relative_to(self.root).as_posix()) != hashes[p.relative_to(self.root).as_posix()]] |
| if not changed: |
| return |
| for attempt in range(3): |
| try: |
| head = self.api.model_info(REPO, revision='main').sha |
| operations = [CommitOperationAdd(path_in_repo=PREFIX+'/'+p.relative_to(self.root).as_posix(), path_or_fileobj=str(p)) for p in changed] |
| operations.append(CommitOperationAdd(path_in_repo=PREFIX+'/SHA256SUMS.json', path_or_fileobj=json.dumps(hashes,sort_keys=True,indent=2).encode())) |
| commit = self.api.create_commit(repo_id=REPO, revision='main', parent_commit=head, operations=operations, commit_message='Colorizer round6: '+reason) |
| for p in changed: |
| name = p.relative_to(self.root).as_posix() |
| downloaded = hf_hub_download(REPO, PREFIX+'/'+name, revision=commit.oid, force_download=True, token=os.environ['HF_TOKEN']) |
| if hashlib.sha256(Path(downloaded).read_bytes()).hexdigest() != hashes[name]: |
| raise RuntimeError('Remote checksum mismatch: '+name) |
| self.saved = hashes |
| print('PERSISTED',reason,commit.oid,'verified_files',len(changed),flush=True) |
| return |
| except Exception: |
| if attempt == 2: |
| raise |
| time.sleep(2**attempt) |
|
|
