compatibility/run-conversion.py

"""Run the official fixed MLX conversion and record resource usage."""
import json
import os
import shutil
import subprocess
import time
from datetime import datetime, timezone
from pathlib import Path

root = Path(__file__).resolve().parents[1]
os.chdir(root)
source = Path(os.environ['SWIFT_SOURCE_DIR']).resolve(strict=True)
output = Path(os.environ.get('SWIFT_MLX_OUTPUT', root / 'Swift-1.5-4bit-MLX')).resolve()
logs = Path(os.environ.get('SWIFT_VALIDATION_DIR', root / 'validation-output')).resolve()
logs.mkdir(parents=True, exist_ok=True)
assert not output.exists(), 'Never overwrite an existing artifact'
command = [str(root / '.venv/bin/mlx_lm.convert'), '--hf-path', str(source), '--mlx-path', str(output), '--quantize', '--q-mode', 'affine', '--q-bits', '4', '--q-group-size', '64']
record = {'command': command, 'started_at': datetime.now(timezone.utc).isoformat(), 'source_repo': 'ukisai/Swift-1.5-Qwen3.8-27b', 'source_revision': '00ccd14e006897d28cb0ed5bf26390e60d274251', 'source_manifest_sha256': '0a00065b88ab003281853a7fb9bd5ce0086bc3781b36136d8c39da19933923ae', 'quantization': {'mode': 'affine', 'bits': 4, 'group_size': 64}}
(logs / 'conversion-command.json').write_text(json.dumps(record, indent=2)+'\n')
env = dict(os.environ, HF_HUB_OFFLINE='1', TRANSFORMERS_OFFLINE='1', PYTHONUNBUFFERED='1', OMP_NUM_THREADS='8', OPENBLAS_NUM_THREADS='8')
start = time.monotonic()
with (logs / 'conversion.log').open('x') as log, (logs / 'conversion-resources.jsonl').open('x') as monitor:
    process = subprocess.Popen(command, stdout=log, stderr=subprocess.STDOUT, env=env)
    record['pid'] = process.pid
    (logs / 'conversion-pid').write_text(str(process.pid)+'\n')
    while process.poll() is None:
        memory = dict(line.split(':', 1) for line in Path('/proc/meminfo').read_text().splitlines())
        status = Path(f'/proc/{process.pid}/status')
        stats = dict(line.split(':', 1) for line in status.read_text().splitlines()) if status.exists() else {}
        disk = shutil.disk_usage(root)
        sample = {'elapsed_seconds': time.monotonic()-start, 'rss': stats.get('VmRSS', '').strip(), 'peak_rss': stats.get('VmHWM', '').strip(), 'process_swap': stats.get('VmSwap', '').strip(), 'memory_available': memory['MemAvailable'].strip(), 'swap_free': memory['SwapFree'].strip(), 'disk_free_bytes': disk.free}
        monitor.write(json.dumps(sample)+'\n'); monitor.flush()
        if disk.free < 1024**3:
            record['critical_stop_reason'] = 'Less than 1 GiB free disk space'
            process.terminate()
        time.sleep(5)
    record.update(returncode=process.returncode, elapsed_seconds=time.monotonic()-start, finished_at=datetime.now(timezone.utc).isoformat())
(logs / 'conversion-result.json').write_text(json.dumps(record,indent=2)+'\n')
print(json.dumps(record,indent=2))
raise SystemExit(process.returncode)