| { |
| "checked_utc": "2026-09-17T02:15:20.561079+00:00", |
| "campaign": "/home/andy/ai/opensysone/runs/20260916T193741Z-24h", |
| "state": { |
| "supervisor_pid": 1096483, |
| "campaign": "/home/andy/ai/opensysone/runs/20260916T193741Z-24h", |
| "stage": "training", |
| "status": "running", |
| "source_commit": "4a60423c39d70f8d50472ce4f4f7fa4a4bd9fce1", |
| "child_pid": 1096505, |
| "child_command": [ |
| "/home/andy/ai/envs/opensysone/bin/python", |
| "/home/andy/projects/opensysone/experiment.py", |
| "train", |
| "--output", |
| "/home/andy/ai/opensysone/runs/20260916T193741Z-24h/training", |
| "--resume", |
| "/home/andy/ai/opensysone/runs/20260916T193721Z-selection-parent/checkpoint.pt", |
| "--deadline", |
| "2026-09-17T16:00:00+00:00", |
| "--model", |
| "/home/andy/ai/models/opensysone/Qwen3-4B-Instruct-2507-cdbee75f", |
| "--dataset", |
| "/home/andy/ai/opensysone/data/public-decisions-v1-20260916", |
| "--epochs", |
| "3", |
| "--rank", |
| "8", |
| "--alpha", |
| "16.0", |
| "--lr", |
| "0.0001", |
| "--head-lr", |
| "0.0001", |
| "--seed", |
| "431", |
| "--effective-batch", |
| "4", |
| "--branch-batch-size", |
| "1", |
| "--max-tokens", |
| "512", |
| "--save-steps", |
| "250", |
| "--save-seconds", |
| "900", |
| "--eval-steps", |
| "500", |
| "--validation-per-family", |
| "128", |
| "--patience", |
| "8", |
| "--selection-metric", |
| "crossfit_temperature_nll_v1", |
| "--two-pass" |
| ], |
| "stage_started_utc": "2026-09-16T19:37:42.516405+00:00", |
| "heartbeat_utc": "2026-09-17T02:15:17.134956+00:00" |
| }, |
| "latest_step": 2570, |
| "new_steps": 2392, |
| "all_finite": true, |
| "peak_cuda_allocated_bytes": 16776178176, |
| "selected": { |
| "metric": "crossfit_temperature_nll_v1", |
| "score": 0.18864030037724333, |
| "raw_macro_nll": 0.30729043330681044, |
| "accuracy": 0.935546875, |
| "per_family": { |
| "arc": { |
| "count": 128, |
| "score": 0.14837152412562352, |
| "raw_macro_nll": 0.10747530600586384, |
| "accuracy": 0.953125 |
| }, |
| "banking": { |
| "count": 128, |
| "score": 0.06337688481028368, |
| "raw_macro_nll": 0.09334830709159572, |
| "accuracy": 0.9765625 |
| }, |
| "boolq": { |
| "count": 128, |
| "score": 0.3119620718470344, |
| "raw_macro_nll": 0.6294207016878283, |
| "accuracy": 0.890625 |
| }, |
| "snli": { |
| "count": 128, |
| "score": 0.23085072072603174, |
| "raw_macro_nll": 0.3989174184419539, |
| "accuracy": 0.921875 |
| } |
| } |
| }, |
| "validation_history": [ |
| { |
| "step": 500, |
| "selection_score": 0.24236371302131984, |
| "raw_nll": 0.4442408630852608, |
| "accuracy": 0.923828125, |
| "improved": true |
| }, |
| { |
| "step": 1000, |
| "selection_score": 0.2668680418742837, |
| "raw_nll": 0.32059778917582193, |
| "accuracy": 0.90234375, |
| "improved": false |
| }, |
| { |
| "step": 1500, |
| "selection_score": 0.27272764961594875, |
| "raw_nll": 0.46897214887359445, |
| "accuracy": 0.912109375, |
| "improved": false |
| }, |
| { |
| "step": 2000, |
| "selection_score": 0.21467134961969922, |
| "raw_nll": 0.29243061315785635, |
| "accuracy": 0.939453125, |
| "improved": true |
| }, |
| { |
| "step": 2500, |
| "selection_score": 0.18864030037724333, |
| "raw_nll": 0.30729043451049853, |
| "accuracy": 0.935546875, |
| "improved": true |
| } |
| ], |
| "checkpoint_mtime": "2026-09-17T02:04:29.125144+00:00", |
| "best_mtime": "2026-09-17T02:04:28.801145+00:00", |
| "best_sha256": "8956eb6c0cfbb02124aeefd99c3b418c55f55fdb9a64260350622d98dbba1aec", |
| "oom_score_adj": { |
| "supervisor_pid": "0", |
| "child_pid": "0" |
| }, |
| "free_b": " total used free shared buff/cache available\nMem: 130594160640 24156987392 75718725632 113389568 31422525440 106437173248\nSwap: 0 0 0\n", |
| "gpu_processes": "pid, process_name, used_gpu_memory [MiB]\n4674, /home/andy/ai/apps/llama.cpp/build/bin/llama-server, 170 MiB\n1096505, /home/andy/ai/envs/opensysone/bin/python, 16313 MiB\n", |
| "source_commit": "4a60423c39d70f8d50472ce4f4f7fa4a4bd9fce1", |
| "exit_code": null |
| } |
|
|
