{
  "checked_utc": "2026-09-16T19:50:01.290313+00:00",
  "candidates": {
    "gx10-4b": {
      "campaign": "/home/andy/ai/opensysone/runs/20260916T193741Z-24h",
      "state": {
        "supervisor_pid": 1096483,
        "campaign": "/home/andy/ai/opensysone/runs/20260916T193741Z-24h",
        "stage": "training",
        "status": "running",
        "source_commit": "4a60423c39d70f8d50472ce4f4f7fa4a4bd9fce1",
        "child_pid": 1096505,
        "child_command": [
          "/home/andy/ai/envs/opensysone/bin/python",
          "/home/andy/projects/opensysone/experiment.py",
          "train",
          "--output",
          "/home/andy/ai/opensysone/runs/20260916T193741Z-24h/training",
          "--resume",
          "/home/andy/ai/opensysone/runs/20260916T193721Z-selection-parent/checkpoint.pt",
          "--deadline",
          "2026-09-17T16:00:00+00:00",
          "--model",
          "/home/andy/ai/models/opensysone/Qwen3-4B-Instruct-2507-cdbee75f",
          "--dataset",
          "/home/andy/ai/opensysone/data/public-decisions-v1-20260916",
          "--epochs",
          "3",
          "--rank",
          "8",
          "--alpha",
          "16.0",
          "--lr",
          "0.0001",
          "--head-lr",
          "0.0001",
          "--seed",
          "431",
          "--effective-batch",
          "4",
          "--branch-batch-size",
          "1",
          "--max-tokens",
          "512",
          "--save-steps",
          "250",
          "--save-seconds",
          "900",
          "--eval-steps",
          "500",
          "--validation-per-family",
          "128",
          "--patience",
          "8",
          "--selection-metric",
          "crossfit_temperature_nll_v1",
          "--two-pass"
        ],
        "stage_started_utc": "2026-09-16T19:37:42.516405+00:00",
        "heartbeat_utc": "2026-09-16T19:49:57.778208+00:00"
      },
      "plan": {
        "resume_checkpoint": "/home/andy/ai/opensysone/runs/20260916T193721Z-selection-parent/checkpoint.pt",
        "resume_sha256": "1516441922c7a5a77e3fe3fe2fb5e511c6bd6a531e180d182f43d3e8cc460b3c",
        "final_deadline": "2026-09-17T18:16:10+00:00",
        "eval_reserve_seconds": 7200,
        "evaluation_estimate": {
          "pilot_validation_decisions": 512,
          "pilot_validation_seconds": 329.73491883277893,
          "evaluation_decisions": 6640,
          "estimated_prediction_seconds": 4276.249728612602
        },
        "port": 18081,
        "training_deadline": "2026-09-17T16:00:00+00:00",
        "train_only": true,
        "selection_metric": "crossfit_temperature_nll_v1",
        "inference_max_tokens": 1024,
        "created_utc": "2026-09-16T19:37:41.824450+00:00",
        "source_commit": "4a60423c39d70f8d50472ce4f4f7fa4a4bd9fce1"
      },
      "last_update": {
        "step": 217,
        "loss": 0.8743088433766388,
        "gradient_norm": 11.0603666305542,
        "seconds": 9.080218113012961,
        "decisions": 4,
        "branches": 11,
        "actual_branch_tokens": 1616,
        "padded_branch_tokens": 1616,
        "peak_cuda_allocated_bytes": 16635998208,
        "lr_factor": 0.9999638978983244
      },
      "all_logged_losses_gradients_finite": true,
      "max_observed_allocated_bytes": 16635998208,
      "oom_score_adj": {
        "supervisor_pid": "0",
        "child_pid": "0"
      },
      "validation": {
        "metric": "crossfit_temperature_nll_v1",
        "score": 0.30382450138162254,
        "raw_macro_nll": 0.40419831451885363,
        "accuracy": 0.904296875
      },
      "exit_code": null
    },
    "spark-a-4b-low-lr": {
      "campaign": "/home/andy/ai/opensysone/runs/20260916T194258Z-24h",
      "state": {
        "supervisor_pid": 327084,
        "campaign": "/home/andy/ai/opensysone/runs/20260916T194258Z-24h",
        "stage": "training",
        "status": "running",
        "source_commit": "4a60423c39d70f8d50472ce4f4f7fa4a4bd9fce1",
        "child_pid": 327116,
        "child_command": [
          "/home/andy/ai/envs/opensysone/bin/python",
          "/home/andy/projects/opensysone/experiment.py",
          "train",
          "--output",
          "/home/andy/ai/opensysone/runs/20260916T194258Z-24h/training",
          "--resume",
          "/home/andy/ai/opensysone/runs/20260916T192910Z-train/artifacts/checkpoint.pt",
          "--deadline",
          "2026-09-17T16:00:00+00:00",
          "--model",
          "/home/andy/ai/models/opensysone/Qwen3-4B-Instruct-2507-cdbee75f",
          "--dataset",
          "/home/andy/ai/opensysone/data/public-decisions-v1-20260916",
          "--epochs",
          "3",
          "--rank",
          "8",
          "--alpha",
          "16.0",
          "--lr",
          "3e-05",
          "--head-lr",
          "3e-05",
          "--seed",
          "431",
          "--effective-batch",
          "4",
          "--branch-batch-size",
          "1",
          "--max-tokens",
          "512",
          "--save-steps",
          "250",
          "--save-seconds",
          "900",
          "--eval-steps",
          "500",
          "--validation-per-family",
          "128",
          "--patience",
          "8",
          "--schedule-steps",
          "7500",
          "--selection-metric",
          "crossfit_temperature_nll_v1",
          "--two-pass"
        ],
        "stage_started_utc": "2026-09-16T19:42:59.139378+00:00",
        "heartbeat_utc": "2026-09-16T19:49:59.242335+00:00"
      },
      "plan": {
        "resume_checkpoint": "/home/andy/ai/opensysone/runs/20260916T192910Z-train/artifacts/checkpoint.pt",
        "resume_sha256": "cada37aff491907a0d5c8d3dde0aa5bc5a826219bbf062e6931d892d5ab6950f",
        "final_deadline": "2026-09-17T18:16:10+00:00",
        "eval_reserve_seconds": 7200,
        "evaluation_estimate": {
          "pilot_validation_decisions": 512,
          "pilot_validation_seconds": 305.2858760356903,
          "evaluation_decisions": 6640,
          "estimated_prediction_seconds": 3959.1762048378587
        },
        "port": 18081,
        "training_deadline": "2026-09-17T16:00:00+00:00",
        "train_only": true,
        "selection_metric": "crossfit_temperature_nll_v1",
        "inference_max_tokens": 1024,
        "created_utc": "2026-09-16T19:42:58.450848+00:00",
        "source_commit": "4a60423c39d70f8d50472ce4f4f7fa4a4bd9fce1"
      },
      "last_update": {
        "step": 17,
        "loss": 0.13068320939783007,
        "gradient_norm": 4.887901782989502,
        "seconds": 7.85438717299985,
        "decisions": 4,
        "branches": 12,
        "actual_branch_tokens": 1330,
        "padded_branch_tokens": 1330,
        "peak_cuda_allocated_bytes": 16538889728,
        "lr_factor": 0.17
      },
      "all_logged_losses_gradients_finite": true,
      "max_observed_allocated_bytes": 16538889728,
      "oom_score_adj": {
        "supervisor_pid": "0",
        "child_pid": "0"
      },
      "validation": {
        "metric": "crossfit_temperature_nll_v1",
        "score": 0.35823514092465186,
        "raw_macro_nll": 0.40539722293900615,
        "accuracy": 0.869140625
      },
      "exit_code": null
    },
    "spark-b-2b": {
      "campaign": "/home/andy/ai/opensysone/runs/20260916T193803Z-24h",
      "state": {
        "supervisor_pid": 286092,
        "campaign": "/home/andy/ai/opensysone/runs/20260916T193803Z-24h",
        "stage": "training",
        "status": "running",
        "source_commit": "4a60423c39d70f8d50472ce4f4f7fa4a4bd9fce1",
        "child_pid": 286131,
        "child_command": [
          "/home/andy/ai/envs/opensysone/bin/python",
          "/home/andy/projects/opensysone/experiment.py",
          "train",
          "--output",
          "/home/andy/ai/opensysone/runs/20260916T193803Z-24h/training",
          "--resume",
          "/home/andy/ai/opensysone/runs/20260916T182352Z-train/artifacts/checkpoint.pt",
          "--deadline",
          "2026-09-17T16:00:00+00:00",
          "--model",
          "/home/andy/ai/models/opensysone/Qwen3.5-2B-15852e8c",
          "--dataset",
          "/home/andy/ai/opensysone/data/public-decisions-v1-20260916",
          "--epochs",
          "3",
          "--rank",
          "16",
          "--alpha",
          "32",
          "--lr",
          "0.0001",
          "--head-lr",
          "0.0001",
          "--seed",
          "431",
          "--effective-batch",
          "4",
          "--branch-batch-size",
          "2",
          "--max-tokens",
          "768",
          "--save-steps",
          "250",
          "--save-seconds",
          "900",
          "--eval-steps",
          "500",
          "--validation-per-family",
          "128",
          "--patience",
          "8",
          "--selection-metric",
          "crossfit_temperature_nll_v1"
        ],
        "stage_started_utc": "2026-09-16T19:38:04.584070+00:00",
        "heartbeat_utc": "2026-09-16T19:49:59.736176+00:00"
      },
      "plan": {
        "resume_checkpoint": "/home/andy/ai/opensysone/runs/20260916T182352Z-train/artifacts/checkpoint.pt",
        "resume_sha256": "af5790ae2f2b56477ebbdf6ab9c418d895e48d2bf5a6416e11b6e9863ad1db55",
        "final_deadline": "2026-09-17T18:16:10+00:00",
        "eval_reserve_seconds": 7200,
        "evaluation_estimate": {
          "pilot_validation_decisions": 512,
          "pilot_validation_seconds": 130.97493624687195,
          "evaluation_decisions": 6654,
          "estimated_prediction_seconds": 1702.162550364621
        },
        "port": 18081,
        "training_deadline": "2026-09-17T16:00:00+00:00",
        "train_only": true,
        "selection_metric": "crossfit_temperature_nll_v1",
        "inference_max_tokens": 1024,
        "created_utc": "2026-09-16T19:38:03.883204+00:00",
        "source_commit": "4a60423c39d70f8d50472ce4f4f7fa4a4bd9fce1"
      },
      "last_update": {
        "step": 200,
        "loss": 0.0026399965569510186,
        "gradient_norm": 0.3787257671356201,
        "seconds": 3.6704137939959764,
        "decisions": 4,
        "branches": 14,
        "actual_branch_tokens": 1471,
        "padded_branch_tokens": 1478,
        "peak_cuda_allocated_bytes": 8306663936,
        "lr_factor": 0.9999736544095965
      },
      "all_logged_losses_gradients_finite": true,
      "max_observed_allocated_bytes": 8306663936,
      "oom_score_adj": {
        "supervisor_pid": "0",
        "child_pid": "0"
      },
      "validation": {
        "metric": "crossfit_temperature_nll_v1",
        "score": 0.47607192779394353,
        "raw_macro_nll": 0.49815331890580655,
        "accuracy": 0.828125
      },
      "exit_code": null
    }
  },
  "coordinator": {
    "campaign": "/home/andy/ai/opensysone/runs/20260916T194403396250Z-fleet",
    "supervisor_pid": 1098466,
    "supervisor_command": [
      "/home/andy/ai/envs/opensysone/bin/python",
      "/home/andy/projects/opensysone/scripts/fleet_campaign.py",
      "--campaign",
      "/home/andy/ai/opensysone/runs/20260916T194403396250Z-fleet",
      "--run"
    ],
    "status": "running",
    "stage": "waiting_for_selection",
    "started_utc": "2026-09-16T19:44:03.429624+00:00",
    "source_commit": "6a7b0ed69c1413d8cb6ff7c790dc38f84de2b113",
    "source_sha256": "6f78b3df9ee3854330792d8218f91a4e907657e47bcca7f5d5e908044cbc5e78",
    "heartbeat_utc": "2026-09-16T19:49:58.506550+00:00"
  }
}