"""Publish an explicit directory-cleanup plan using two guarded HF commits. Usage: --plan PLAN.json --output NEW_STATUS_DIRECTORY The first commit contains the payload and explicit obsolete source-file deletes. Only after integrity/preservation checks does the second commit replace README.md and PUBLICATION.json. Historical archives, existing pointers and privacy stay fixed. """ import argparse import hashlib import json import logging import os from pathlib import Path, PurePosixPath import re import signal import sys import tarfile ROOT = Path(__file__).resolve().parents[1] sys.path.insert(0, str(ROOT)) from scripts.publish_hf_final import digest, read_json, write_json, utc, deadline_alarm, stop_requested MODEL_SHA256 = 'e270e3da905604d97bf5a8f380ea308133403d1c4790a5c012cb1c12e9b6f348' PROTECTED_PREFIXES = ('snapshots/', 'final/', 'profiles/', 'sources/', 'publications/') PROTECTED_POINTERS = {'CURRENT_SNAPSHOT.json', 'FINAL_MODEL.json', 'PROFILE_RESULTS.json'} ROOT_FILES = {'README.md', 'publication-manifest.json'} DESTINATION_ROOTS = {'source', 'docs', 'results', 'model', 'archive'} ALLOWED_ROOTS = (ROOT, Path.home() / 'ai/opensysone') SAFE_SUFFIXES = {'.py', '.md', '.json', '.jsonl', '.csv', '.txt', '.log', '.png', '.svg', '.sh', '.cjs', '.js', '.html', '.css', '.sha256', '.yaml', '.yml', '.toml', '.ini'} SECRET_NAMES = {'.env', '.netrc', 'token', 'access_token', 'credentials', 'credentials.json', 'auth.json', 'id_rsa', 'id_ed25519'} SECRET_DIRS = {'.git', '.ssh', '.aws', '.gnupg', '.cache', 'secrets', '__pycache__'} SECRET_NAME = re.compile(r'(^|[_.-])(credentials?|secrets?|access[_-]?tokens?|api[_-]?keys?|private[_-]?keys?)([_.-]|$)', re.I) SECRET_CONTENT = re.compile(rb'hf_[A-Za-z0-9]{30,}|-----BEGIN (?:RSA |EC |OPENSSH )?PRIVATE KEY-----') def relative_path(name): if not isinstance(name, str) or '\\' in name or any(ord(c) < 32 for c in name): raise ValueError('Invalid publication path') path = PurePosixPath(name) if path.is_absolute() or not path.parts or '..' in path.parts or str(path) != name: raise ValueError('Invalid publication path') return path def check_name(name, allow_model=False, allow_archive=False): path = relative_path(name) if (any(part.lower() in SECRET_DIRS for part in path.parts) or path.name.lower() in SECRET_NAMES or path.name.lower().startswith('.env.') or SECRET_NAME.search(path.name)): raise ValueError('Secret file or directory prohibited') if allow_model and name == 'model/model.pt': return if allow_archive and name.startswith('archive/') and name.endswith('.tar.gz'): return if path.suffix.lower() not in SAFE_SUFFIXES and path.name not in {'.gitignore', 'exit_code', 'LICENSE', 'SHA256SUMS'}: raise ValueError('Unsupported or base-weight file prohibited') def safe_source(value, roots): path = Path(value) if (not path.is_absolute() or '..' in path.parts or path.resolve() != path or not path.is_file() or not any(path.is_relative_to(Path(root).resolve()) for root in roots)): raise ValueError('Source must be an absolute regular nonsymlink file in approved roots') if 'models' in path.parts or any(part.lower() in SECRET_DIRS for part in path.parts): raise ValueError('Base-model or secret source directory prohibited') return path def source_content_guard(path, destination): if destination == 'model/model.pt': return # The one exact, separately pinned calibrated adapter/head artifact. if destination.endswith('.tar.gz'): with tarfile.open(path, 'r:gz') as archive: seen = set() for member in archive: if member.isdir(): relative_path(member.name.rstrip('/')) continue check_name(member.name) if not member.isfile() or member.name in seen: raise ValueError('Archive has nonregular or duplicate source entry') seen.add(member.name) with archive.extractfile(member) as handle: if SECRET_CONTENT.search(handle.read()): raise ValueError('Credential content prohibited') elif SECRET_CONTENT.search(path.read_bytes()): raise ValueError('Credential content prohibited') def inventory_map(records): result = {} for record in records: name = str(relative_path(record['path'])) if name in result or type(record['size']) is not int or record['size'] < 0: raise ValueError('Invalid or duplicate inventory record') if not re.fullmatch('[0-9a-f]{40}', record.get('blob_id', '')): raise ValueError('Missing Git object checksum') if record.get('lfs_sha256') is not None and not re.fullmatch('[0-9a-f]{64}', record['lfs_sha256']): raise ValueError('Invalid LFS checksum') result[name] = {key: record.get(key) for key in ('size', 'blob_id', 'lfs_sha256')} return result def remote_inventory(api, repo_id, revision): records = [] for item in api.list_repo_tree(repo_id, recursive=True, revision=revision, repo_type='model'): if not hasattr(item, 'blob_id'): continue # RepoFolder is not a file. lfs = getattr(item, 'lfs', None) records.append({'path': item.path, 'size': item.size, 'blob_id': item.blob_id, 'lfs_sha256': (lfs.get('sha256') if isinstance(lfs, dict) else lfs.sha256) if lfs else None}) return inventory_map(records) def validate_plan(plan, roots=ALLOWED_ROOTS): if not re.fullmatch(r'[A-Za-z0-9_.-]+/[A-Za-z0-9_.-]+', plan.get('repo_id', '')): raise ValueError('Invalid repository identifier') for key in ('expected_revision', 'source_commit'): if not re.fullmatch('[0-9a-f]{40}', plan.get(key, '')): raise ValueError('Exact commit required') if type(plan.get('expected_private')) is not bool: raise ValueError('Explicit expected privacy required') inventory = read_json(safe_source(plan['inventory_path'], roots)) if (inventory['repo_id'] != plan['repo_id'] or inventory['revision'] != plan['expected_revision'] or inventory['private'] is not plan['expected_private']): raise ValueError('Preflight inventory does not match plan') original = inventory_map(inventory['files']) if not PROTECTED_POINTERS <= original.keys() or 'README.md' not in original: raise ValueError('Required original pointers/card missing') files = plan.get('files', {}) if not (ROOT_FILES | {'model/model.pt'}) <= files.keys(): raise ValueError('Publication card, manifest and exact model are required') expected = {} for name, record in files.items(): path = relative_path(name) if (name.startswith(PROTECTED_PREFIXES) or name in PROTECTED_POINTERS or (name not in ROOT_FILES and (len(path.parts) < 2 or path.parts[0] not in DESTINATION_ROOTS))): raise ValueError('Protected or unapproved destination') check_name(name, allow_model=True, allow_archive=True) source = safe_source(record['source'], roots) # Renaming cannot conceal an unsafe source basename or file type. source_name = 'model/model.pt' if name == 'model/model.pt' and source.name == 'model.pt' else ( 'archive/' + source.name if name.endswith('.tar.gz') else source.name) check_name(source_name, allow_model=name == 'model/model.pt', allow_archive=name.endswith('.tar.gz')) if source.suffix != path.suffix: raise ValueError('Source file type cannot be disguised') if not re.fullmatch('[0-9a-f]{64}', record.get('sha256', '')) or digest(source) != record['sha256']: raise ValueError('Source checksum mismatch') if name == 'model/model.pt' and record['sha256'] != MODEL_SHA256: raise ValueError('Only the exact calibrated adapter/head artifact is permitted') source_content_guard(source, name) expected[name] = {'size': source.stat().st_size, 'sha256': record['sha256'], 'blob_id': digest(source, 'sha1', git_blob=True)} deletes = plan.get('delete_source_files', []) if not isinstance(deletes, list) or len(set(deletes)) != len(deletes): raise ValueError('Explicit unique file deletions required') for name in deletes: relative_path(name) if not name.startswith('source/') or name not in original or name in files: raise ValueError('Only existing obsolete source files may be deleted') return original, expected def check_visibility_revision(api, plan, revision): current = api.repo_info(plan['repo_id'], repo_type='model', timeout=30) if current.sha != revision or current.private is not plan['expected_private']: raise ValueError('Repository revision or visibility changed') def verify_remote(original, actual, expected, deleted): if set(actual) != (set(original) | set(expected)) - set(deleted): raise ValueError('Remote file inventory changed unexpectedly') for name, old in original.items(): if name not in expected and name not in deleted and actual.get(name) != old: raise ValueError('Preserved remote file changed') for name, record in expected.items(): remote = actual.get(name) if not remote or remote['size'] != record['size']: raise ValueError('Remote payload size mismatch') if name == 'model/model.pt' and not remote['lfs_sha256']: raise ValueError('Calibrated model needs LFS checksum metadata') checksum = record['sha256'] if remote['lfs_sha256'] else record['blob_id'] if (remote['lfs_sha256'] or remote['blob_id']) != checksum: raise ValueError('Remote payload checksum mismatch') def verify_lfs_append(before, after, expected, actual): """Accept HF's exact per-file LFS additions, never edits or pattern rules.""" if not after.startswith(before) or (before and not before.endswith(b'\n')): raise ValueError('Existing .gitattributes bytes changed') appended = after[len(before):] if not appended or not appended.endswith(b'\n'): raise ValueError('Invalid appended .gitattributes rules') suffix = b' filter=lfs diff=lfs merge=lfs -text\n' added = [] for line in appended.splitlines(keepends=True): if not line.endswith(suffix): raise ValueError('Only exact per-file LFS rules may be appended') name = line[:-len(suffix)].decode('utf-8') relative_path(name) if (any(c.isspace() or c in '*?[]\\' for c in name) or name not in expected or name in added): raise ValueError('Wildcard, duplicate or unplanned LFS rule') remote = actual.get(name) if (remote is None or remote['size'] != expected[name]['size'] or remote['lfs_sha256'] != expected[name]['sha256']): raise ValueError('Appended LFS rule lacks matching payload checksum') added.append(name) return {'path': '.gitattributes', 'before_sha256': hashlib.sha256(before).hexdigest(), 'sha256': hashlib.sha256(after).hexdigest(), 'size': len(after), 'blob_id': hashlib.sha1(f'blob {len(after)}\0'.encode() + after).hexdigest(), 'added_paths': added, 'added_lfs_sha256': {name: expected[name]['sha256'] for name in added}} def accepted_lfs_attributes(api, repo_id, before_revision, after_revision, original, actual, expected): name = '.gitattributes' if original.get(name) == actual.get(name): return original, None if name not in original or name not in actual: raise ValueError('Original .gitattributes file must be preserved') contents = [] for revision, inventory in ((before_revision, original), (after_revision, actual)): downloaded = api.hf_hub_download(repo_id, name, revision=revision, repo_type='model', etag_timeout=30) content = Path(downloaded).read_bytes() checksum = hashlib.sha256(content).hexdigest() if inventory[name]['lfs_sha256'] else ( hashlib.sha1(f'blob {len(content)}\0'.encode() + content).hexdigest()) if len(content) != inventory[name]['size'] or checksum != (inventory[name]['lfs_sha256'] or inventory[name]['blob_id']): raise ValueError('Downloaded .gitattributes checksum mismatch') contents.append(content) proof = verify_lfs_append(*contents, expected, actual) # This exact post-payload inventory becomes the preserved baseline for the # final commit; no future .gitattributes difference is excused. return {**original, name: actual[name]}, proof def card_license(content): import yaml text = content.decode('utf-8') if not text.startswith('---\n') or '\n---\n' not in text[4:]: raise ValueError('Model card metadata required') metadata = yaml.safe_load(text[4:].split('\n---\n', 1)[0]) if not isinstance(metadata, dict): raise ValueError('Invalid model card metadata') return {key: value for key, value in metadata.items() if str(key).startswith('license')} def publish(api, plan, progress, roots=ALLOWED_ROOTS, add_factory=None, delete_factory=None): original, expected = validate_plan(plan, roots) repo_id = plan['repo_id'] check_visibility_revision(api, plan, plan['expected_revision']) if remote_inventory(api, repo_id, plan['expected_revision']) != original: raise ValueError('Fresh inventory differs from preflight') card = Path(plan['files']['README.md']['source']).read_bytes() if hashlib.sha256(card).hexdigest() != expected['README.md']['sha256']: raise ValueError('Card source changed after validation') old_card = Path(api.hf_hub_download(repo_id, 'README.md', revision=plan['expected_revision'], repo_type='model', etag_timeout=30)).read_bytes() if card_license(card) != card_license(old_card): raise ValueError('Card license metadata must remain unchanged') if add_factory is None or delete_factory is None: from huggingface_hub import CommitOperationAdd, CommitOperationDelete add_factory = add_factory or CommitOperationAdd delete_factory = delete_factory or CommitOperationDelete payload = {name: value for name, value in expected.items() if name != 'README.md'} operations = [add_factory(path_in_repo=name, path_or_fileobj=plan['files'][name]['source']) for name in sorted(payload)] operations += [delete_factory(path_in_repo=name, is_folder=False) for name in plan.get('delete_source_files', [])] progress('uploading_payload', source_commit=plan['source_commit'], file_count=len(payload)) uploaded = api.create_commit(repo_id, repo_type='model', operations=operations, parent_commit=plan['expected_revision'], commit_message='Organize verified OpenSysOne publication payload') progress('verifying_payload', payload_commit=uploaded.oid) payload_inventory = remote_inventory(api, repo_id, uploaded.oid) preserved, attributes_proof = accepted_lfs_attributes(api, repo_id, plan['expected_revision'], uploaded.oid, original, payload_inventory, payload) verify_remote(preserved, payload_inventory, payload, plan.get('delete_source_files', [])) if attributes_proof is not None: progress('verified_payload', payload_commit=uploaded.oid, accepted_lfs_attributes=attributes_proof) check_visibility_revision(api, plan, uploaded.oid) manifest_name = 'publication-manifest.json' manifest = Path(api.hf_hub_download(repo_id, manifest_name, revision=uploaded.oid, repo_type='model', etag_timeout=30)).read_bytes() if hashlib.sha256(manifest).hexdigest() != expected[manifest_name]['sha256']: raise ValueError('Publication manifest bytes differ') pointer = {'format': 'opensysone-publication-pointer-v1', 'repo_id': repo_id, 'manifest_path': manifest_name, 'manifest_sha256': expected[manifest_name]['sha256'], 'payload_commit': uploaded.oid, 'source_commit': plan['source_commit'], 'verified_utc': utc(), 'accepted_lfs_attributes': attributes_proof} pointer_bytes = (json.dumps(pointer, indent=2, sort_keys=True) + '\n').encode() front = {'README.md': card, 'PUBLICATION.json': pointer_bytes} # Recheck every input, including the card, immediately before publishing links. for name, record in plan['files'].items(): if digest(record['source']) != expected[name]['sha256']: raise ValueError('Local source changed during publication') check_visibility_revision(api, plan, uploaded.oid) progress('publishing_front', payload_commit=uploaded.oid, accepted_lfs_attributes=attributes_proof) committed = api.create_commit(repo_id, repo_type='model', parent_commit=uploaded.oid, operations=[add_factory(path_in_repo=name, path_or_fileobj=content) for name, content in front.items()], commit_message='Publish verified OpenSysOne directory index and manifest pointer') for name, content in front.items(): downloaded = api.hf_hub_download(repo_id, name, revision=committed.oid, repo_type='model', etag_timeout=30) if Path(downloaded).read_bytes() != content: raise ValueError('Published front file bytes differ') pointer_expected = {'size': len(pointer_bytes), 'sha256': hashlib.sha256(pointer_bytes).hexdigest(), 'blob_id': hashlib.sha1(f'blob {len(pointer_bytes)}\0'.encode() + pointer_bytes).hexdigest()} verify_remote(preserved, remote_inventory(api, repo_id, committed.oid), {**expected, 'PUBLICATION.json': pointer_expected}, plan.get('delete_source_files', [])) check_visibility_revision(api, plan, committed.oid) return {**pointer, 'publication_commit': committed.oid, 'repository_private': plan['expected_private'], 'preserved_original_files': len(set(original) - set(expected) - set(plan.get('delete_source_files', [])) - ({'.gitattributes'} if attributes_proof else set()))} def main(): parser = argparse.ArgumentParser(description=__doc__) parser.add_argument('--plan', required=True) parser.add_argument('--output', required=True) args = parser.parse_args() output = Path(args.output).resolve() try: output.mkdir(parents=True, exist_ok=False) except Exception as error: print(json.dumps({'status': 'failed', 'error_type': type(error).__name__}), file=sys.stderr) return 1 state = {'status': 'running', 'pid': os.getpid(), 'started_utc': utc(), 'plan_path': str(Path(args.plan).resolve())} def progress(stage, **values): state.update(stage=stage, heartbeat_utc=utc(), **values) write_json(output / 'state.json', state) code = 1 signal.signal(signal.SIGALRM, deadline_alarm) signal.signal(signal.SIGTERM, stop_requested) signal.alarm(1800) try: progress('validating_plan') Path('/proc/self/oom_score_adj').write_text('0') os.environ.update(HF_HUB_DISABLE_PROGRESS_BARS='1', HF_HUB_DISABLE_XET='1', HF_HUB_DOWNLOAD_TIMEOUT='60', HF_HUB_ETAG_TIMEOUT='30') from huggingface_hub import HfApi, set_client_factory import httpx for name in ('huggingface_hub', 'httpx', 'httpcore'): logging.getLogger(name).setLevel(logging.CRITICAL) set_client_factory(lambda: httpx.Client(timeout=httpx.Timeout(60, connect=15), follow_redirects=True)) plan = read_json(args.plan) progress('validating_plan', plan_sha256=digest(args.plan), repo_id=plan['repo_id']) result = publish(HfApi(), plan, progress) write_json(output / 'publication.json', result) progress('complete', status='complete', finished_utc=utc(), **result) code = 0 except BaseException as error: progress('failed', status='failed', error_type=type(error).__name__, finished_utc=utc()) finally: signal.alarm(0) (output / 'exit_code').write_text(str(code) + '\n') return code if __name__ == '__main__': raise SystemExit(main())