| { |
| "checked_utc": "2026-09-16T19:50:01.290313+00:00", |
| "candidates": { |
| "gx10-4b": { |
| "campaign": "/home/andy/ai/opensysone/runs/20260916T193741Z-24h", |
| "state": { |
| "supervisor_pid": 1096483, |
| "campaign": "/home/andy/ai/opensysone/runs/20260916T193741Z-24h", |
| "stage": "training", |
| "status": "running", |
| "source_commit": "4a60423c39d70f8d50472ce4f4f7fa4a4bd9fce1", |
| "child_pid": 1096505, |
| "child_command": [ |
| "/home/andy/ai/envs/opensysone/bin/python", |
| "/home/andy/projects/opensysone/experiment.py", |
| "train", |
| "--output", |
| "/home/andy/ai/opensysone/runs/20260916T193741Z-24h/training", |
| "--resume", |
| "/home/andy/ai/opensysone/runs/20260916T193721Z-selection-parent/checkpoint.pt", |
| "--deadline", |
| "2026-09-17T16:00:00+00:00", |
| "--model", |
| "/home/andy/ai/models/opensysone/Qwen3-4B-Instruct-2507-cdbee75f", |
| "--dataset", |
| "/home/andy/ai/opensysone/data/public-decisions-v1-20260916", |
| "--epochs", |
| "3", |
| "--rank", |
| "8", |
| "--alpha", |
| "16.0", |
| "--lr", |
| "0.0001", |
| "--head-lr", |
| "0.0001", |
| "--seed", |
| "431", |
| "--effective-batch", |
| "4", |
| "--branch-batch-size", |
| "1", |
| "--max-tokens", |
| "512", |
| "--save-steps", |
| "250", |
| "--save-seconds", |
| "900", |
| "--eval-steps", |
| "500", |
| "--validation-per-family", |
| "128", |
| "--patience", |
| "8", |
| "--selection-metric", |
| "crossfit_temperature_nll_v1", |
| "--two-pass" |
| ], |
| "stage_started_utc": "2026-09-16T19:37:42.516405+00:00", |
| "heartbeat_utc": "2026-09-16T19:49:57.778208+00:00" |
| }, |
| "plan": { |
| "resume_checkpoint": "/home/andy/ai/opensysone/runs/20260916T193721Z-selection-parent/checkpoint.pt", |
| "resume_sha256": "1516441922c7a5a77e3fe3fe2fb5e511c6bd6a531e180d182f43d3e8cc460b3c", |
| "final_deadline": "2026-09-17T18:16:10+00:00", |
| "eval_reserve_seconds": 7200, |
| "evaluation_estimate": { |
| "pilot_validation_decisions": 512, |
| "pilot_validation_seconds": 329.73491883277893, |
| "evaluation_decisions": 6640, |
| "estimated_prediction_seconds": 4276.249728612602 |
| }, |
| "port": 18081, |
| "training_deadline": "2026-09-17T16:00:00+00:00", |
| "train_only": true, |
| "selection_metric": "crossfit_temperature_nll_v1", |
| "inference_max_tokens": 1024, |
| "created_utc": "2026-09-16T19:37:41.824450+00:00", |
| "source_commit": "4a60423c39d70f8d50472ce4f4f7fa4a4bd9fce1" |
| }, |
| "last_update": { |
| "step": 217, |
| "loss": 0.8743088433766388, |
| "gradient_norm": 11.0603666305542, |
| "seconds": 9.080218113012961, |
| "decisions": 4, |
| "branches": 11, |
| "actual_branch_tokens": 1616, |
| "padded_branch_tokens": 1616, |
| "peak_cuda_allocated_bytes": 16635998208, |
| "lr_factor": 0.9999638978983244 |
| }, |
| "all_logged_losses_gradients_finite": true, |
| "max_observed_allocated_bytes": 16635998208, |
| "oom_score_adj": { |
| "supervisor_pid": "0", |
| "child_pid": "0" |
| }, |
| "validation": { |
| "metric": "crossfit_temperature_nll_v1", |
| "score": 0.30382450138162254, |
| "raw_macro_nll": 0.40419831451885363, |
| "accuracy": 0.904296875 |
| }, |
| "exit_code": null |
| }, |
| "spark-a-4b-low-lr": { |
| "campaign": "/home/andy/ai/opensysone/runs/20260916T194258Z-24h", |
| "state": { |
| "supervisor_pid": 327084, |
| "campaign": "/home/andy/ai/opensysone/runs/20260916T194258Z-24h", |
| "stage": "training", |
| "status": "running", |
| "source_commit": "4a60423c39d70f8d50472ce4f4f7fa4a4bd9fce1", |
| "child_pid": 327116, |
| "child_command": [ |
| "/home/andy/ai/envs/opensysone/bin/python", |
| "/home/andy/projects/opensysone/experiment.py", |
| "train", |
| "--output", |
| "/home/andy/ai/opensysone/runs/20260916T194258Z-24h/training", |
| "--resume", |
| "/home/andy/ai/opensysone/runs/20260916T192910Z-train/artifacts/checkpoint.pt", |
| "--deadline", |
| "2026-09-17T16:00:00+00:00", |
| "--model", |
| "/home/andy/ai/models/opensysone/Qwen3-4B-Instruct-2507-cdbee75f", |
| "--dataset", |
| "/home/andy/ai/opensysone/data/public-decisions-v1-20260916", |
| "--epochs", |
| "3", |
| "--rank", |
| "8", |
| "--alpha", |
| "16.0", |
| "--lr", |
| "3e-05", |
| "--head-lr", |
| "3e-05", |
| "--seed", |
| "431", |
| "--effective-batch", |
| "4", |
| "--branch-batch-size", |
| "1", |
| "--max-tokens", |
| "512", |
| "--save-steps", |
| "250", |
| "--save-seconds", |
| "900", |
| "--eval-steps", |
| "500", |
| "--validation-per-family", |
| "128", |
| "--patience", |
| "8", |
| "--schedule-steps", |
| "7500", |
| "--selection-metric", |
| "crossfit_temperature_nll_v1", |
| "--two-pass" |
| ], |
| "stage_started_utc": "2026-09-16T19:42:59.139378+00:00", |
| "heartbeat_utc": "2026-09-16T19:49:59.242335+00:00" |
| }, |
| "plan": { |
| "resume_checkpoint": "/home/andy/ai/opensysone/runs/20260916T192910Z-train/artifacts/checkpoint.pt", |
| "resume_sha256": "cada37aff491907a0d5c8d3dde0aa5bc5a826219bbf062e6931d892d5ab6950f", |
| "final_deadline": "2026-09-17T18:16:10+00:00", |
| "eval_reserve_seconds": 7200, |
| "evaluation_estimate": { |
| "pilot_validation_decisions": 512, |
| "pilot_validation_seconds": 305.2858760356903, |
| "evaluation_decisions": 6640, |
| "estimated_prediction_seconds": 3959.1762048378587 |
| }, |
| "port": 18081, |
| "training_deadline": "2026-09-17T16:00:00+00:00", |
| "train_only": true, |
| "selection_metric": "crossfit_temperature_nll_v1", |
| "inference_max_tokens": 1024, |
| "created_utc": "2026-09-16T19:42:58.450848+00:00", |
| "source_commit": "4a60423c39d70f8d50472ce4f4f7fa4a4bd9fce1" |
| }, |
| "last_update": { |
| "step": 17, |
| "loss": 0.13068320939783007, |
| "gradient_norm": 4.887901782989502, |
| "seconds": 7.85438717299985, |
| "decisions": 4, |
| "branches": 12, |
| "actual_branch_tokens": 1330, |
| "padded_branch_tokens": 1330, |
| "peak_cuda_allocated_bytes": 16538889728, |
| "lr_factor": 0.17 |
| }, |
| "all_logged_losses_gradients_finite": true, |
| "max_observed_allocated_bytes": 16538889728, |
| "oom_score_adj": { |
| "supervisor_pid": "0", |
| "child_pid": "0" |
| }, |
| "validation": { |
| "metric": "crossfit_temperature_nll_v1", |
| "score": 0.35823514092465186, |
| "raw_macro_nll": 0.40539722293900615, |
| "accuracy": 0.869140625 |
| }, |
| "exit_code": null |
| }, |
| "spark-b-2b": { |
| "campaign": "/home/andy/ai/opensysone/runs/20260916T193803Z-24h", |
| "state": { |
| "supervisor_pid": 286092, |
| "campaign": "/home/andy/ai/opensysone/runs/20260916T193803Z-24h", |
| "stage": "training", |
| "status": "running", |
| "source_commit": "4a60423c39d70f8d50472ce4f4f7fa4a4bd9fce1", |
| "child_pid": 286131, |
| "child_command": [ |
| "/home/andy/ai/envs/opensysone/bin/python", |
| "/home/andy/projects/opensysone/experiment.py", |
| "train", |
| "--output", |
| "/home/andy/ai/opensysone/runs/20260916T193803Z-24h/training", |
| "--resume", |
| "/home/andy/ai/opensysone/runs/20260916T182352Z-train/artifacts/checkpoint.pt", |
| "--deadline", |
| "2026-09-17T16:00:00+00:00", |
| "--model", |
| "/home/andy/ai/models/opensysone/Qwen3.5-2B-15852e8c", |
| "--dataset", |
| "/home/andy/ai/opensysone/data/public-decisions-v1-20260916", |
| "--epochs", |
| "3", |
| "--rank", |
| "16", |
| "--alpha", |
| "32", |
| "--lr", |
| "0.0001", |
| "--head-lr", |
| "0.0001", |
| "--seed", |
| "431", |
| "--effective-batch", |
| "4", |
| "--branch-batch-size", |
| "2", |
| "--max-tokens", |
| "768", |
| "--save-steps", |
| "250", |
| "--save-seconds", |
| "900", |
| "--eval-steps", |
| "500", |
| "--validation-per-family", |
| "128", |
| "--patience", |
| "8", |
| "--selection-metric", |
| "crossfit_temperature_nll_v1" |
| ], |
| "stage_started_utc": "2026-09-16T19:38:04.584070+00:00", |
| "heartbeat_utc": "2026-09-16T19:49:59.736176+00:00" |
| }, |
| "plan": { |
| "resume_checkpoint": "/home/andy/ai/opensysone/runs/20260916T182352Z-train/artifacts/checkpoint.pt", |
| "resume_sha256": "af5790ae2f2b56477ebbdf6ab9c418d895e48d2bf5a6416e11b6e9863ad1db55", |
| "final_deadline": "2026-09-17T18:16:10+00:00", |
| "eval_reserve_seconds": 7200, |
| "evaluation_estimate": { |
| "pilot_validation_decisions": 512, |
| "pilot_validation_seconds": 130.97493624687195, |
| "evaluation_decisions": 6654, |
| "estimated_prediction_seconds": 1702.162550364621 |
| }, |
| "port": 18081, |
| "training_deadline": "2026-09-17T16:00:00+00:00", |
| "train_only": true, |
| "selection_metric": "crossfit_temperature_nll_v1", |
| "inference_max_tokens": 1024, |
| "created_utc": "2026-09-16T19:38:03.883204+00:00", |
| "source_commit": "4a60423c39d70f8d50472ce4f4f7fa4a4bd9fce1" |
| }, |
| "last_update": { |
| "step": 200, |
| "loss": 0.0026399965569510186, |
| "gradient_norm": 0.3787257671356201, |
| "seconds": 3.6704137939959764, |
| "decisions": 4, |
| "branches": 14, |
| "actual_branch_tokens": 1471, |
| "padded_branch_tokens": 1478, |
| "peak_cuda_allocated_bytes": 8306663936, |
| "lr_factor": 0.9999736544095965 |
| }, |
| "all_logged_losses_gradients_finite": true, |
| "max_observed_allocated_bytes": 8306663936, |
| "oom_score_adj": { |
| "supervisor_pid": "0", |
| "child_pid": "0" |
| }, |
| "validation": { |
| "metric": "crossfit_temperature_nll_v1", |
| "score": 0.47607192779394353, |
| "raw_macro_nll": 0.49815331890580655, |
| "accuracy": 0.828125 |
| }, |
| "exit_code": null |
| } |
| }, |
| "coordinator": { |
| "campaign": "/home/andy/ai/opensysone/runs/20260916T194403396250Z-fleet", |
| "supervisor_pid": 1098466, |
| "supervisor_command": [ |
| "/home/andy/ai/envs/opensysone/bin/python", |
| "/home/andy/projects/opensysone/scripts/fleet_campaign.py", |
| "--campaign", |
| "/home/andy/ai/opensysone/runs/20260916T194403396250Z-fleet", |
| "--run" |
| ], |
| "status": "running", |
| "stage": "waiting_for_selection", |
| "started_utc": "2026-09-16T19:44:03.429624+00:00", |
| "source_commit": "6a7b0ed69c1413d8cb6ff7c790dc38f84de2b113", |
| "source_sha256": "6f78b3df9ee3854330792d8218f91a4e907657e47bcca7f5d5e908044cbc5e78", |
| "heartbeat_utc": "2026-09-16T19:49:58.506550+00:00" |
| } |
| } |
|
|
