{
  "scope": "Validation-only diagnostic motivating an explicitly revised checkpoint selection policy; no reserved calibration/test/holdout predictions accessed",
  "selection_policy": {
    "id": "crossfit_temperature_nll_v1",
    "folds": 4,
    "seed": 431,
    "grouping": "source group, nested within task family; groups shuffled within sorted families and assigned round-robin",
    "temperature_grid": {
      "count": 101,
      "log10_min": -1.0,
      "log10_max": 1.3,
      "arithmetic": "Python float64"
    },
    "temperature_fit": "minimum macro-family NLL on the other three folds; smallest temperature wins ties",
    "score": "macro-family mean of all held-out decision NLLs",
    "deployment_temperature": "fit afresh on reserved calibration only after model selection"
  },
  "selection_source_sha256": "be0a7a8496b5b830aa572ceba93606320f442063fd38180503fd6980dc1c578f",
  "model_results": [
    {
      "step": 40,
      "predictions_path": "/home/andy/ai/opensysone/runs/20260916T183823Z-train/artifacts/validation_step_000040_predictions.json",
      "predictions_sha256": "e671e1508185765552b0f933ba03f356be62143c531d8ef534457d34b1645c9b",
      "raw_macro_nll": 0.39566109237724656,
      "crossfit_macro_nll": 0.3595222692275388,
      "accuracy": 0.875,
      "fold_temperatures": [
        1.5703628043335522,
        1.6557699634695275,
        1.5703628043335522,
        1.7458221529205038
      ],
      "per_family": {
        "arc": {
          "count": 128,
          "score": 0.3246282279964344,
          "raw_macro_nll": 0.3743996537632195,
          "accuracy": 0.90625
        },
        "banking": {
          "count": 128,
          "score": 0.23241229297629554,
          "raw_macro_nll": 0.22994083210335414,
          "accuracy": 0.9140625
        },
        "boolq": {
          "count": 128,
          "score": 0.4348036430263749,
          "raw_macro_nll": 0.58309304281893,
          "accuracy": 0.84375
        },
        "snli": {
          "count": 128,
          "score": 0.4462449129110504,
          "raw_macro_nll": 0.39521084082348246,
          "accuracy": 0.8359375
        }
      }
    },
    {
      "step": 128,
      "predictions_path": "/home/andy/ai/opensysone/runs/20260916T192239Z-24h/training/validation_step_000128_predictions.json",
      "predictions_sha256": "e686d63b8f93d466dc085e4e32ab904b218839246dee8211e8b71cff512db985",
      "raw_macro_nll": 0.44268335003700615,
      "crossfit_macro_nll": 0.3185177281077473,
      "accuracy": 0.890625,
      "fold_temperatures": [
        2.2750974307720706,
        2.2750974307720706,
        2.39883291901949,
        2.39883291901949
      ],
      "per_family": {
        "arc": {
          "count": 128,
          "score": 0.29531450555418093,
          "raw_macro_nll": 0.34207193492789617,
          "accuracy": 0.90625
        },
        "banking": {
          "count": 128,
          "score": 0.18676987812687565,
          "raw_macro_nll": 0.17689879145408868,
          "accuracy": 0.9375
        },
        "boolq": {
          "count": 128,
          "score": 0.4084573236595828,
          "raw_macro_nll": 0.7099410978597552,
          "accuracy": 0.859375
        },
        "snli": {
          "count": 128,
          "score": 0.38352920509034977,
          "raw_macro_nll": 0.5418215759062844,
          "accuracy": 0.859375
        }
      }
    }
  ],
  "bootstrap": {
    "replicates": 5000,
    "seed": 431,
    "method": "Paired source-group resampling stratified by family, preserving fixed group folds; refit each fold temperature within every replicate using macro-family training NLL",
    "step128_minus40_crossfit_nll": -0.041004541119791516,
    "crossfit_nll_ci95": [
      -0.07982194525314908,
      -0.0011518414800151043
    ],
    "step128_minus40_accuracy": 0.015625,
    "accuracy_ci95": [
      -0.01171875,
      0.04296875
    ],
    "limitations": "Conditional on these validation decisions and fixed folds; not adjusted for prior model/checkpoint selection or this criterion revision; independent test/holdout remain decisive"
  },
  "discordant_accuracy": {
    "gains": 29,
    "losses": 21,
    "mcnemar_exact_two_sided_p": 0.3222363203575469
  },
  "selection_cpu_seconds_for_two_512_row_candidates": 0.06691360900003929,
  "decision": "Adopt fixed crossfit_temperature_nll_v1 for new fleet checkpoints; retain raw NLL separately; fit final serving temperature afresh only on reserved calibration after selection"
}