opensysone / source /results /20260916-fleet-setup /spark-b-current-status.json
andyshu's picture
Back up verified OpenSysOne training snapshot and pinned source
2d5c26a verified
Raw History Blame
2.86 kB
{
"checked_utc": "2026-09-16T19:43:59.853093+00:00",
"plan": {
"resume_checkpoint": "/home/andy/ai/opensysone/runs/20260916T182352Z-train/artifacts/checkpoint.pt",
"resume_sha256": "af5790ae2f2b56477ebbdf6ab9c418d895e48d2bf5a6416e11b6e9863ad1db55",
"final_deadline": "2026-09-17T18:16:10+00:00",
"eval_reserve_seconds": 7200,
"evaluation_estimate": {
"pilot_validation_decisions": 512,
"pilot_validation_seconds": 130.97493624687195,
"evaluation_decisions": 6654,
"estimated_prediction_seconds": 1702.162550364621
},
"port": 18081,
"training_deadline": "2026-09-17T16:00:00+00:00",
"train_only": true,
"selection_metric": "crossfit_temperature_nll_v1",
"inference_max_tokens": 1024,
"created_utc": "2026-09-16T19:38:03.883204+00:00",
"source_commit": "4a60423c39d70f8d50472ce4f4f7fa4a4bd9fce1"
},
"state": {
"supervisor_pid": 286092,
"campaign": "/home/andy/ai/opensysone/runs/20260916T193803Z-24h",
"stage": "training",
"status": "running",
"source_commit": "4a60423c39d70f8d50472ce4f4f7fa4a4bd9fce1",
"child_pid": 286131,
"child_command": [
"/home/andy/ai/envs/opensysone/bin/python",
"/home/andy/projects/opensysone/experiment.py",
"train",
"--output",
"/home/andy/ai/opensysone/runs/20260916T193803Z-24h/training",
"--resume",
"/home/andy/ai/opensysone/runs/20260916T182352Z-train/artifacts/checkpoint.pt",
"--deadline",
"2026-09-17T16:00:00+00:00",
"--model",
"/home/andy/ai/models/opensysone/Qwen3.5-2B-15852e8c",
"--dataset",
"/home/andy/ai/opensysone/data/public-decisions-v1-20260916",
"--epochs",
"3",
"--rank",
"16",
"--alpha",
"32",
"--lr",
"0.0001",
"--head-lr",
"0.0001",
"--seed",
"431",
"--effective-batch",
"4",
"--branch-batch-size",
"2",
"--max-tokens",
"768",
"--save-steps",
"250",
"--save-seconds",
"900",
"--eval-steps",
"500",
"--validation-per-family",
"128",
"--patience",
"8",
"--selection-metric",
"crossfit_temperature_nll_v1"
],
"stage_started_utc": "2026-09-16T19:38:04.584070+00:00",
"heartbeat_utc": "2026-09-16T19:43:59.656334+00:00"
},
"setup_exit_code": 0,
"campaign_exit_code": null,
"training_exit_code": null,
"latest_training_update": {
"step": 98,
"loss": 0.49353107113165606,
"gradient_norm": 18.723804473876953,
"seconds": 3.8619721549985115,
"decisions": 4,
"branches": 13,
"actual_branch_tokens": 1750,
"padded_branch_tokens": 1756,
"peak_cuda_allocated_bytes": 8286311936,
"lr_factor": 0.98
},
"source_commit": "4a60423c39d70f8d50472ce4f4f7fa4a4bd9fce1",
"source_status": ""
}