{ "checked_utc": "2026-09-17T02:11:24.308357+00:00", "host": "spark-3e2a", "campaign": "/home/andy/ai/opensysone/runs/20260916T193803Z-24h", "plan": { "resume_checkpoint": "/home/andy/ai/opensysone/runs/20260916T182352Z-train/artifacts/checkpoint.pt", "resume_sha256": "af5790ae2f2b56477ebbdf6ab9c418d895e48d2bf5a6416e11b6e9863ad1db55", "final_deadline": "2026-09-17T18:16:10+00:00", "eval_reserve_seconds": 7200, "evaluation_estimate": { "pilot_validation_decisions": 512, "pilot_validation_seconds": 130.97493624687195, "evaluation_decisions": 6654, "estimated_prediction_seconds": 1702.162550364621 }, "port": 18081, "training_deadline": "2026-09-17T16:00:00+00:00", "train_only": true, "selection_metric": "crossfit_temperature_nll_v1", "inference_max_tokens": 1024, "created_utc": "2026-09-16T19:38:03.883204+00:00", "source_commit": "4a60423c39d70f8d50472ce4f4f7fa4a4bd9fce1" }, "state": { "supervisor_pid": 286092, "campaign": "/home/andy/ai/opensysone/runs/20260916T193803Z-24h", "stage": "training_complete", "status": "training_complete", "source_commit": "4a60423c39d70f8d50472ce4f4f7fa4a4bd9fce1", "child_pid": null, "child_command": [ "/home/andy/ai/envs/opensysone/bin/python", "/home/andy/projects/opensysone/experiment.py", "train", "--output", "/home/andy/ai/opensysone/runs/20260916T193803Z-24h/training", "--resume", "/home/andy/ai/opensysone/runs/20260916T182352Z-train/artifacts/checkpoint.pt", "--deadline", "2026-09-17T16:00:00+00:00", "--model", "/home/andy/ai/models/opensysone/Qwen3.5-2B-15852e8c", "--dataset", "/home/andy/ai/opensysone/data/public-decisions-v1-20260916", "--epochs", "3", "--rank", "16", "--alpha", "32", "--lr", "0.0001", "--head-lr", "0.0001", "--seed", "431", "--effective-batch", "4", "--branch-batch-size", "2", "--max-tokens", "768", "--save-steps", "250", "--save-seconds", "900", "--eval-steps", "500", "--validation-per-family", "128", "--patience", "8", "--selection-metric", "crossfit_temperature_nll_v1" ], "stage_started_utc": "2026-09-16T19:38:04.584070+00:00", "heartbeat_utc": "2026-09-17T01:59:24.923675+00:00", "training_exit_code": 0, "training_output": "/home/andy/ai/opensysone/runs/20260916T193803Z-24h/training", "resume_checkpoint": "/home/andy/ai/opensysone/runs/20260916T193803Z-24h/training/checkpoint.pt", "training_summary": { "completed_steps": 6000, "best_validation_macro_nll": 0.25676974726429014, "status": "validation_early_stop", "peak_cuda_allocated_bytes": 8786432512, "selection_metric": "crossfit_temperature_nll_v1", "best_validation_selection_score": 0.2552943737691716 }, "train_only": true, "best_checkpoint": "/home/andy/ai/opensysone/runs/20260916T193803Z-24h/training/best.pt", "best_sha256": "be08efdc9121a64547b5c9331d5908eeb0978c4fbc962885fd34f9758ec89d11", "finished_utc": "2026-09-17T01:59:29.971800+00:00", "selection": "Await cross-candidate validation selection before calibration/test/holdout", "best_validation_predictions": "/home/andy/ai/opensysone/runs/20260916T193803Z-24h/training/best_validation_predictions.json", "best_validation_predictions_sha256": "980c9df18461f5de79e54701bfee1b5fbff19dd3b85125e64f619a88c60f80e4" }, "summary": { "status": "validation_early_stop", "completed_steps": 6000, "target_steps": 30703, "initial_validation": { "overall": { "n": 512, "accuracy": 0.828125, "nll": 0.4981533203378369, "brier_multiclass_sum": 0.2607127369212848, "ece_top_label_10_equal_width_bins": 0.04818323429208249, "reliability_bins": [ { "count": 0, "confidence_sum": 0.0, "correct_sum": 0.0 }, { "count": 0, "confidence_sum": 0.0, "correct_sum": 0.0 }, { "count": 2, "confidence_sum": 0.5327933728694916, "correct_sum": 1.0 }, { "count": 4, "confidence_sum": 1.4359374046325684, "correct_sum": 3.0 }, { "count": 17, "confidence_sum": 7.9667291939258575, "correct_sum": 7.0 }, { "count": 55, "confidence_sum": 30.154281735420227, "correct_sum": 35.0 }, { "count": 53, "confidence_sum": 34.67930006980896, "correct_sum": 34.0 }, { "count": 46, "confidence_sum": 34.177616238594055, "correct_sum": 36.0 }, { "count": 54, "confidence_sum": 46.13812476396561, "correct_sum": 44.0 }, { "count": 281, "confidence_sum": 276.18629068136215, "correct_sum": 264.0 } ], "accuracy_vs_coverage": { "0.25": { "n": 128, "accuracy": 0.96875, "min_confidence": 0.9978359341621399 }, "0.5": { "n": 256, "accuracy": 0.94140625, "min_confidence": 0.9372246861457825 }, "0.75": { "n": 384, "accuracy": 0.9010416666666666, "min_confidence": 0.6965566873550415 }, "1.0": { "n": 512, "accuracy": 0.828125, "min_confidence": 0.2647492587566376 } } }, "per_family": { "arc": { "n": 128, "accuracy": 0.8125, "nll": 0.4990136543534618, "brier_multiclass_sum": 0.26976813507500774, "ece_top_label_10_equal_width_bins": 0.048822794342413545, "reliability_bins": [ { "count": 0, "confidence_sum": 0.0, "correct_sum": 0.0 }, { "count": 0, "confidence_sum": 0.0, "correct_sum": 0.0 }, { "count": 2, "confidence_sum": 0.5327933728694916, "correct_sum": 1.0 }, { "count": 4, "confidence_sum": 1.4359374046325684, "correct_sum": 3.0 }, { "count": 8, "confidence_sum": 3.675512135028839, "correct_sum": 2.0 }, { "count": 9, "confidence_sum": 4.856119275093079, "correct_sum": 6.0 }, { "count": 13, "confidence_sum": 8.502536118030548, "correct_sum": 8.0 }, { "count": 13, "confidence_sum": 9.665025651454926, "correct_sum": 10.0 }, { "count": 28, "confidence_sum": 24.128599762916565, "correct_sum": 24.0 }, { "count": 51, "confidence_sum": 49.567454636096954, "correct_sum": 50.0 } ], "accuracy_vs_coverage": { "0.25": { "n": 32, "accuracy": 1.0, "min_confidence": 0.974073588848114 }, "0.5": { "n": 64, "accuracy": 0.953125, "min_confidence": 0.8664795756340027 }, "0.75": { "n": 96, "accuracy": 0.8958333333333334, "min_confidence": 0.6717178225517273 }, "1.0": { "n": 128, "accuracy": 0.8125, "min_confidence": 0.2647492587566376 } } }, "banking": { "n": 128, "accuracy": 0.8828125, "nll": 0.2938805352743866, "brier_multiclass_sum": 0.15680863450549035, "ece_top_label_10_equal_width_bins": 0.060145952040329576, "reliability_bins": [ { "count": 0, "confidence_sum": 0.0, "correct_sum": 0.0 }, { "count": 0, "confidence_sum": 0.0, "correct_sum": 0.0 }, { "count": 0, "confidence_sum": 0.0, "correct_sum": 0.0 }, { "count": 0, "confidence_sum": 0.0, "correct_sum": 0.0 }, { "count": 1, "confidence_sum": 0.47516098618507385, "correct_sum": 0.0 }, { "count": 6, "confidence_sum": 3.3597115874290466, "correct_sum": 1.0 }, { "count": 5, "confidence_sum": 3.223210096359253, "correct_sum": 3.0 }, { "count": 3, "confidence_sum": 2.2593541145324707, "correct_sum": 1.0 }, { "count": 7, "confidence_sum": 5.898111343383789, "correct_sum": 4.0 }, { "count": 106, "confidence_sum": 105.48313373327255, "correct_sum": 104.0 } ], "accuracy_vs_coverage": { "0.25": { "n": 32, "accuracy": 1.0, "min_confidence": 0.9999927282333374 }, "0.5": { "n": 64, "accuracy": 1.0, "min_confidence": 0.9998255372047424 }, "0.75": { "n": 96, "accuracy": 0.9895833333333334, "min_confidence": 0.9847974181175232 }, "1.0": { "n": 128, "accuracy": 0.8828125, "min_confidence": 0.47516098618507385 } } }, "boolq": { "n": 128, "accuracy": 0.796875, "nll": 0.730819056238805, "brier_multiclass_sum": 0.3320798331012733, "ece_top_label_10_equal_width_bins": 0.1302144955843687, "reliability_bins": [ { "count": 0, "confidence_sum": 0.0, "correct_sum": 0.0 }, { "count": 0, "confidence_sum": 0.0, "correct_sum": 0.0 }, { "count": 0, "confidence_sum": 0.0, "correct_sum": 0.0 }, { "count": 0, "confidence_sum": 0.0, "correct_sum": 0.0 }, { "count": 0, "confidence_sum": 0.0, "correct_sum": 0.0 }, { "count": 9, "confidence_sum": 5.026383757591248, "correct_sum": 6.0 }, { "count": 7, "confidence_sum": 4.494362056255341, "correct_sum": 4.0 }, { "count": 6, "confidence_sum": 4.508205056190491, "correct_sum": 3.0 }, { "count": 10, "confidence_sum": 8.55346018075943, "correct_sum": 7.0 }, { "count": 96, "confidence_sum": 94.13781189918518, "correct_sum": 82.0 } ], "accuracy_vs_coverage": { "0.25": { "n": 32, "accuracy": 0.875, "min_confidence": 0.9985519051551819 }, "0.5": { "n": 64, "accuracy": 0.890625, "min_confidence": 0.9844649434089661 }, "0.75": { "n": 96, "accuracy": 0.8541666666666666, "min_confidence": 0.9054988026618958 }, "1.0": { "n": 128, "accuracy": 0.796875, "min_confidence": 0.5037087798118591 } } }, "snli": { "n": 128, "accuracy": 0.8203125, "nll": 0.46890003548469394, "brier_multiclass_sum": 0.284194345003368, "ece_top_label_10_equal_width_bins": 0.10556101333349943, "reliability_bins": [ { "count": 0, "confidence_sum": 0.0, "correct_sum": 0.0 }, { "count": 0, "confidence_sum": 0.0, "correct_sum": 0.0 }, { "count": 0, "confidence_sum": 0.0, "correct_sum": 0.0 }, { "count": 0, "confidence_sum": 0.0, "correct_sum": 0.0 }, { "count": 8, "confidence_sum": 3.8160560727119446, "correct_sum": 5.0 }, { "count": 31, "confidence_sum": 16.912067115306854, "correct_sum": 22.0 }, { "count": 28, "confidence_sum": 18.45919179916382, "correct_sum": 19.0 }, { "count": 24, "confidence_sum": 17.745031416416168, "correct_sum": 22.0 }, { "count": 9, "confidence_sum": 7.557953476905823, "correct_sum": 9.0 }, { "count": 28, "confidence_sum": 26.997890412807465, "correct_sum": 28.0 } ], "accuracy_vs_coverage": { "0.25": { "n": 32, "accuracy": 1.0, "min_confidence": 0.845478892326355 }, "0.5": { "n": 64, "accuracy": 0.96875, "min_confidence": 0.695654571056366 }, "0.75": { "n": 96, "accuracy": 0.8645833333333334, "min_confidence": 0.5729489326477051 }, "1.0": { "n": 128, "accuracy": 0.8203125, "min_confidence": 0.4349120557308197 } } } } }, "best_validation_macro_nll": 0.25676974726429014, "selection_metric": "crossfit_temperature_nll_v1", "best_validation_selection_score": 0.2552943737691716, "training_seconds": 22739.080890835998, "median_step_seconds": 3.5843077870013076, "processed_decisions": 23840, "actual_branch_tokens": 9008732, "padded_branch_tokens": 9051318, "peak_cuda_allocated_bytes": 8786432512, "peak_cuda_reserved_bytes": 8896118784, "final_mem_available_bytes": 114020495360, "correctness": { "branch_chunks_1_probability_max_abs": 4.76837158203125e-07, "branch_chunks_2_probability_max_abs": 0.0, "branch_chunks_4_probability_max_abs": 6.556510925292969e-07, "question_isolation_probability_max_abs": 2.0489096641540527e-08, "candidate_permutation_probability_max_abs": 1.1920928955078125e-07, "repeat_probability_max_abs": 0.0, "tolerance_probability_abs": 0.0001 }, "final_correctness_status": "passed", "checkpoint_sha256": "ab70dcb62432039d93ec71e200157250dc59c563404237e4dfad3662fd30f44f", "best_sha256": "be08efdc9121a64547b5c9331d5908eeb0978c4fbc962885fd34f9758ec89d11" }, "manifest": { "config": { "command": "train", "model": "/home/andy/ai/models/opensysone/Qwen3.5-2B-15852e8c", "dataset": "/home/andy/ai/opensysone/data/public-decisions-v1-20260916", "output": "/home/andy/ai/opensysone/runs/20260916T193803Z-24h/training", "resume": "/home/andy/ai/opensysone/runs/20260916T182352Z-train/artifacts/checkpoint.pt", "warm_start": null, "steps": null, "epochs": 3, "rank": 16, "alpha": 32.0, "head_only": false, "two_pass": false, "lr": 0.0001, "head_lr": 0.0001, "seed": 431, "effective_batch": 4, "branch_batch_size": 2, "max_tokens": 768, "save_steps": 250, "save_seconds": 900, "eval_steps": 500, "validation_per_family": 128, "patience": 8, "deadline": "2026-09-17T16:00:00+00:00", "schedule_steps": null, "selection_metric": "crossfit_temperature_nll_v1", "adapters": true }, "pid": 286131, "hostname": "spark-3e2a", "started_utc": "2026-09-16T19:38:06.219880+00:00", "git_commit": "4a60423c39d70f8d50472ce4f4f7fa4a4bd9fce1", "git_status": "", "source_sha256": { "jev_harness.py": "4d4e979cb7ae352bcdacaaa6d64045e6b5e550b1a9721d4bad545045bee6c67f", "smoke_data.py": "06b3cbac1c8c4a86b8aecbee4459073cc3e46d4ddcd576392f3cb4805924f815", "experiment.py": "c036cbe9fbf6f71c1bb141dcfbecfb06a2f4bd697b070d2274b1931480e92031", "training_model.py": "d5b0aefeeb5290816bc0b669aa0a8cbbe27f6a12b9cb23c141ac9b9ae9ee4e65", "smoke_train.py": "8cdeb2b397177fc9c26638aaa871501ddab8e3871aa1573ecd98f66590f5c228", "decision_model.py": "a3d8aeb02a1ac765c6cc30ff175acad0664560f01ab5403e22cade924d17371e", "selection.py": "be0a7a8496b5b830aa572ceba93606320f442063fd38180503fd6980dc1c578f", "scripts/download_model.py": "72ad9a5de44d09e2ee4ed8afb7c3c0ff6fb48987410a3f7f0368353572bf1f4d", "scripts/download_candidate.py": "d06a2c01be0cf6577f927fb37e3bc1eab014949fd934e4f4d9825674adba608e", "scripts/run_precision.sh": "766d82b30cf3e83951f662685b4472ee053c7fbc171c4ff140823bf4d1c2782f", "scripts/campaign_status.py": "1500b5e24f06231c7aafd7277aefd840582e35997e265f79db93614828d34411", "scripts/verify_artifact.py": "9833350e9d72c0065b15206bb71c5a8b5a6b3185219db90074e369ede563a985", "scripts/start_spark_candidate.sh": "c2ca18b008a144de7cb264c9fcca634d68c4a8a317db3638e8a70b3dbcff064b", "scripts/run_experiment.sh": "661a6309fc54a2a8aff918f14a553c72dcb21730bd6a3cfc55d6ccd4700d11d3", "scripts/fleet_campaign.py": "9e8d0511a579ace145b7ad079c11ce9b7465950569e86b22099caec2831d5f21", "scripts/diagnose_parity.py": "08b5d66d316ebda98a2226251a4f952701f86a1d5726ce7a4d7e8fb22755da5a", "scripts/prepare_public_data.py": "32ea84aa719818e1141b958b6ef27a85f7ddb86bfcd7c1585fc487b25253d977", "scripts/run_smoke.sh": "39d59f2120f362729d1c2e384391b82be1e580dcc1aca0eaa7ab231115225574", "scripts/launch_24h.py": "39c26dc10535d3adac09209b0743edf2ed384563e732512ad62f8c6f37161a83", "scripts/investigate_precision.py": "609b744a926d8a45b87ba8d225e5312ee0c71a7096b21bd3589c5846e1dc847c" }, "packages": { "torch": "2.11.0+cu130", "transformers": "5.15.0", "pyarrow": "25.0.1", "numpy": "2.5.2" }, "cuda": "13.0", "gpu": "NVIDIA GB10", "capability": [ 12, 1 ], "cuda_cap_bytes": 17179869184, "initial_mem_available_bytes": 125287321600, "oom_score_adj": "0", "parent_checkpoint_sha256": "af5790ae2f2b56477ebbdf6ab9c418d895e48d2bf5a6416e11b6e9863ad1db55", "initialization": { "kind": "resume", "parent_checkpoint": "/home/andy/ai/opensysone/runs/20260916T182352Z-train/artifacts/checkpoint.pt", "parent_checkpoint_sha256": "af5790ae2f2b56477ebbdf6ab9c418d895e48d2bf5a6416e11b6e9863ad1db55", "restores_optimizer": true, "restores_rng": true, "parent_step": 40, "parent_source_commit": "f1c9322c5313021cf9f90984c8a451bea165f633", "parent_initialization": null, "selection_policy_change": { "from": "raw_nll", "to": "crossfit_temperature_nll_v1", "optimizer_and_rng_unchanged": true } }, "model_provenance": { "model_id": "Qwen/Qwen3.5-2B", "revision": "15852e8c16360a2fea060d615a32b45270f8a8fc", "license": "apache-2.0" }, "data_signature": "dedef2b6406f252c165ee317925935079282f8a8ac96f034e89b07048d79f9e1", "total_parameters": 1898646337, "trainable_parameters": 16821249, "adapter_modules": [ "layers.0.linear_attn.out_proj", "layers.0.linear_attn.in_proj_qkv", "layers.0.linear_attn.in_proj_z", "layers.0.linear_attn.in_proj_b", "layers.0.linear_attn.in_proj_a", "layers.0.mlp.gate_proj", "layers.0.mlp.up_proj", "layers.0.mlp.down_proj", "layers.1.linear_attn.out_proj", "layers.1.linear_attn.in_proj_qkv", "layers.1.linear_attn.in_proj_z", "layers.1.linear_attn.in_proj_b", "layers.1.linear_attn.in_proj_a", "layers.1.mlp.gate_proj", "layers.1.mlp.up_proj", "layers.1.mlp.down_proj", "layers.2.linear_attn.out_proj", "layers.2.linear_attn.in_proj_qkv", "layers.2.linear_attn.in_proj_z", "layers.2.linear_attn.in_proj_b", "layers.2.linear_attn.in_proj_a", "layers.2.mlp.gate_proj", "layers.2.mlp.up_proj", "layers.2.mlp.down_proj", "layers.3.self_attn.q_proj", "layers.3.self_attn.k_proj", "layers.3.self_attn.v_proj", "layers.3.self_attn.o_proj", "layers.3.mlp.gate_proj", "layers.3.mlp.up_proj", "layers.3.mlp.down_proj", "layers.4.linear_attn.out_proj", "layers.4.linear_attn.in_proj_qkv", "layers.4.linear_attn.in_proj_z", "layers.4.linear_attn.in_proj_b", "layers.4.linear_attn.in_proj_a", "layers.4.mlp.gate_proj", "layers.4.mlp.up_proj", "layers.4.mlp.down_proj", "layers.5.linear_attn.out_proj", "layers.5.linear_attn.in_proj_qkv", "layers.5.linear_attn.in_proj_z", "layers.5.linear_attn.in_proj_b", "layers.5.linear_attn.in_proj_a", "layers.5.mlp.gate_proj", "layers.5.mlp.up_proj", "layers.5.mlp.down_proj", "layers.6.linear_attn.out_proj", "layers.6.linear_attn.in_proj_qkv", "layers.6.linear_attn.in_proj_z", "layers.6.linear_attn.in_proj_b", "layers.6.linear_attn.in_proj_a", "layers.6.mlp.gate_proj", "layers.6.mlp.up_proj", "layers.6.mlp.down_proj", "layers.7.self_attn.q_proj", "layers.7.self_attn.k_proj", "layers.7.self_attn.v_proj", "layers.7.self_attn.o_proj", "layers.7.mlp.gate_proj", "layers.7.mlp.up_proj", "layers.7.mlp.down_proj", "layers.8.linear_attn.out_proj", "layers.8.linear_attn.in_proj_qkv", "layers.8.linear_attn.in_proj_z", "layers.8.linear_attn.in_proj_b", "layers.8.linear_attn.in_proj_a", "layers.8.mlp.gate_proj", "layers.8.mlp.up_proj", "layers.8.mlp.down_proj", "layers.9.linear_attn.out_proj", "layers.9.linear_attn.in_proj_qkv", "layers.9.linear_attn.in_proj_z", "layers.9.linear_attn.in_proj_b", "layers.9.linear_attn.in_proj_a", "layers.9.mlp.gate_proj", "layers.9.mlp.up_proj", "layers.9.mlp.down_proj", "layers.10.linear_attn.out_proj", "layers.10.linear_attn.in_proj_qkv", "layers.10.linear_attn.in_proj_z", "layers.10.linear_attn.in_proj_b", "layers.10.linear_attn.in_proj_a", "layers.10.mlp.gate_proj", "layers.10.mlp.up_proj", "layers.10.mlp.down_proj", "layers.11.self_attn.q_proj", "layers.11.self_attn.k_proj", "layers.11.self_attn.v_proj", "layers.11.self_attn.o_proj", "layers.11.mlp.gate_proj", "layers.11.mlp.up_proj", "layers.11.mlp.down_proj", "layers.12.linear_attn.out_proj", "layers.12.linear_attn.in_proj_qkv", "layers.12.linear_attn.in_proj_z", "layers.12.linear_attn.in_proj_b", "layers.12.linear_attn.in_proj_a", "layers.12.mlp.gate_proj", "layers.12.mlp.up_proj", "layers.12.mlp.down_proj", "layers.13.linear_attn.out_proj", "layers.13.linear_attn.in_proj_qkv", "layers.13.linear_attn.in_proj_z", "layers.13.linear_attn.in_proj_b", "layers.13.linear_attn.in_proj_a", "layers.13.mlp.gate_proj", "layers.13.mlp.up_proj", "layers.13.mlp.down_proj", "layers.14.linear_attn.out_proj", "layers.14.linear_attn.in_proj_qkv", "layers.14.linear_attn.in_proj_z", "layers.14.linear_attn.in_proj_b", "layers.14.linear_attn.in_proj_a", "layers.14.mlp.gate_proj", "layers.14.mlp.up_proj", "layers.14.mlp.down_proj", "layers.15.self_attn.q_proj", "layers.15.self_attn.k_proj", "layers.15.self_attn.v_proj", "layers.15.self_attn.o_proj", "layers.15.mlp.gate_proj", "layers.15.mlp.up_proj", "layers.15.mlp.down_proj", "layers.16.linear_attn.out_proj", "layers.16.linear_attn.in_proj_qkv", "layers.16.linear_attn.in_proj_z", "layers.16.linear_attn.in_proj_b", "layers.16.linear_attn.in_proj_a", "layers.16.mlp.gate_proj", "layers.16.mlp.up_proj", "layers.16.mlp.down_proj", "layers.17.linear_attn.out_proj", "layers.17.linear_attn.in_proj_qkv", "layers.17.linear_attn.in_proj_z", "layers.17.linear_attn.in_proj_b", "layers.17.linear_attn.in_proj_a", "layers.17.mlp.gate_proj", "layers.17.mlp.up_proj", "layers.17.mlp.down_proj", "layers.18.linear_attn.out_proj", "layers.18.linear_attn.in_proj_qkv", "layers.18.linear_attn.in_proj_z", "layers.18.linear_attn.in_proj_b", "layers.18.linear_attn.in_proj_a", "layers.18.mlp.gate_proj", "layers.18.mlp.up_proj", "layers.18.mlp.down_proj", "layers.19.self_attn.q_proj", "layers.19.self_attn.k_proj", "layers.19.self_attn.v_proj", "layers.19.self_attn.o_proj", "layers.19.mlp.gate_proj", "layers.19.mlp.up_proj", "layers.19.mlp.down_proj", "layers.20.linear_attn.out_proj", "layers.20.linear_attn.in_proj_qkv", "layers.20.linear_attn.in_proj_z", "layers.20.linear_attn.in_proj_b", "layers.20.linear_attn.in_proj_a", "layers.20.mlp.gate_proj", "layers.20.mlp.up_proj", "layers.20.mlp.down_proj", "layers.21.linear_attn.out_proj", "layers.21.linear_attn.in_proj_qkv", "layers.21.linear_attn.in_proj_z", "layers.21.linear_attn.in_proj_b", "layers.21.linear_attn.in_proj_a", "layers.21.mlp.gate_proj", "layers.21.mlp.up_proj", "layers.21.mlp.down_proj", "layers.22.linear_attn.out_proj", "layers.22.linear_attn.in_proj_qkv", "layers.22.linear_attn.in_proj_z", "layers.22.linear_attn.in_proj_b", "layers.22.linear_attn.in_proj_a", "layers.22.mlp.gate_proj", "layers.22.mlp.up_proj", "layers.22.mlp.down_proj", "layers.23.self_attn.q_proj", "layers.23.self_attn.k_proj", "layers.23.self_attn.v_proj", "layers.23.self_attn.o_proj", "layers.23.mlp.gate_proj", "layers.23.mlp.up_proj", "layers.23.mlp.down_proj" ], "load_and_tokenize_seconds": 2.767755454995495 }, "exit_codes": { "exit_code": "0" }, "processes": " PID PPID STAT ELAPSED COMMAND\n", "gpu_processes": "", "memory": " total used free shared buff/cache available\nMem: 130661138432 4221775872 89653522432 4759552 37928640512 126439362560\nSwap: 0 0 0\n", "source_commit": "4a60423c39d70f8d50472ce4f4f7fa4a4bd9fce1", "source_status": "", "training_audit": { "rows": 5960, "first_step": 41, "last_step": 6000, "steps_contiguous": true, "finite_losses": true, "finite_gradients": true, "max_gradient_norm": 267.5868225097656, "max_loss": 7.096554353829106, "max_cuda_allocated_bytes": 8786432512, "median_step_seconds": 3.5843077870013076, "last_100_median_step_seconds": 3.610054773500451, "last_update": { "step": 6000, "loss": 0.0019449404207989573, "gradient_norm": 0.06868523359298706, "seconds": 3.64171387499664, "decisions": 4, "branches": 14, "actual_branch_tokens": 1412, "padded_branch_tokens": 1423, "peak_cuda_allocated_bytes": 8786432512, "lr_factor": 0.911059698995469 } }, "validation_history": [ { "step": 500, "macro_nll": 0.45433305375755806, "best_macro_nll": 0.45433305375755806, "selection": { "metric": "crossfit_temperature_nll_v1", "score": 0.36986992229694793, "raw_macro_nll": 0.45433305295880055, "accuracy": 0.876953125, "folds": [ { "fold": 0, "temperature_index": 58, "temperature": 2.1577444091526656, "training_macro_nll": 0.3830001160949712, "heldout_ids": [ "arc:validation:ARC-Challenge:122", "arc:validation:ARC-Challenge:126", "arc:validation:ARC-Challenge:175", "arc:validation:ARC-Challenge:187", "arc:validation:ARC-Challenge:189", "arc:validation:ARC-Challenge:192", "arc:validation:ARC-Challenge:208", "arc:validation:ARC-Challenge:238", "arc:validation:ARC-Challenge:290", "arc:validation:ARC-Challenge:293", "arc:validation:ARC-Challenge:35", "arc:validation:ARC-Challenge:43", "arc:validation:ARC-Challenge:58", "arc:validation:ARC-Challenge:7", "arc:validation:ARC-Easy:160", "arc:validation:ARC-Easy:201", "arc:validation:ARC-Easy:229", "arc:validation:ARC-Easy:238", "arc:validation:ARC-Easy:253", "arc:validation:ARC-Easy:259", "arc:validation:ARC-Easy:272", "arc:validation:ARC-Easy:298", "arc:validation:ARC-Easy:329", "arc:validation:ARC-Easy:331", "arc:validation:ARC-Easy:337", "arc:validation:ARC-Easy:362", "arc:validation:ARC-Easy:443", "arc:validation:ARC-Easy:500", "arc:validation:ARC-Easy:545", "arc:validation:ARC-Easy:68", "arc:validation:ARC-Easy:7", "arc:validation:ARC-Easy:74", "banking:train:1053", "banking:train:1594", "banking:train:1747", "banking:train:202", "banking:train:2377", "banking:train:2528", "banking:train:2546", "banking:train:2715", "banking:train:3220", "banking:train:3499", "banking:train:3806", "banking:train:3838", "banking:train:3993", "banking:train:4220", "banking:train:5", "banking:train:5192", "banking:train:5394", "banking:train:5846", "banking:train:5863", "banking:train:6032", "banking:train:6508", "banking:train:7579", "banking:train:7600", "banking:train:7987", "banking:train:8155", "banking:train:850", "banking:train:8626", "banking:train:8929", "banking:train:9355", "banking:train:9812", "banking:train:987", "banking:train:9972", "boolq:validation:103", "boolq:validation:1042", "boolq:validation:1059", "boolq:validation:1541", "boolq:validation:1592", "boolq:validation:1927", "boolq:validation:2014", "boolq:validation:2023", "boolq:validation:2098", "boolq:validation:2139", "boolq:validation:2439", "boolq:validation:2485", "boolq:validation:2607", "boolq:validation:2611", "boolq:validation:2720", "boolq:validation:2751", "boolq:validation:282", "boolq:validation:2843", "boolq:validation:2953", "boolq:validation:301", "boolq:validation:3089", "boolq:validation:313", "boolq:validation:3201", "boolq:validation:322", "boolq:validation:3231", "boolq:validation:39", "boolq:validation:461", "boolq:validation:558", "boolq:validation:589", "boolq:validation:731", "boolq:validation:883", "boolq:validation:915", "snli:validation:1156", "snli:validation:1330", "snli:validation:1559", "snli:validation:1735", "snli:validation:1876", "snli:validation:2251", "snli:validation:2257", "snli:validation:2377", "snli:validation:2881", "snli:validation:291", "snli:validation:300", "snli:validation:3049", "snli:validation:3496", "snli:validation:3871", "snli:validation:4414", "snli:validation:5140", "snli:validation:5440", "snli:validation:5806", "snli:validation:6442", "snli:validation:6772", "snli:validation:6949", "snli:validation:7228", "snli:validation:7465", "snli:validation:7510", "snli:validation:7726", "snli:validation:8095", "snli:validation:8143", "snli:validation:8623", "snli:validation:8962", "snli:validation:9451", "snli:validation:9574", "snli:validation:9655" ], "heldout_group_count": 128, "training_group_count": 384 }, { "fold": 1, "temperature_index": 57, "temperature": 2.046444636724674, "training_macro_nll": 0.36597041087667326, "heldout_ids": [ "arc:validation:ARC-Challenge:101", "arc:validation:ARC-Challenge:129", "arc:validation:ARC-Challenge:148", "arc:validation:ARC-Challenge:156", "arc:validation:ARC-Challenge:167", "arc:validation:ARC-Challenge:249", "arc:validation:ARC-Challenge:263", "arc:validation:ARC-Challenge:266", "arc:validation:ARC-Challenge:267", "arc:validation:ARC-Challenge:28", "arc:validation:ARC-Challenge:281", "arc:validation:ARC-Challenge:57", "arc:validation:ARC-Challenge:63", "arc:validation:ARC-Challenge:64", "arc:validation:ARC-Challenge:86", "arc:validation:ARC-Challenge:99", "arc:validation:ARC-Easy:106", "arc:validation:ARC-Easy:151", "arc:validation:ARC-Easy:157", "arc:validation:ARC-Easy:166", "arc:validation:ARC-Easy:202", "arc:validation:ARC-Easy:219", "arc:validation:ARC-Easy:224", "arc:validation:ARC-Easy:322", "arc:validation:ARC-Easy:344", "arc:validation:ARC-Easy:382", "arc:validation:ARC-Easy:409", "arc:validation:ARC-Easy:455", "arc:validation:ARC-Easy:50", "arc:validation:ARC-Easy:538", "arc:validation:ARC-Easy:544", "arc:validation:ARC-Easy:552", "banking:train:1050", "banking:train:1131", "banking:train:1190", "banking:train:124", "banking:train:137", "banking:train:1493", "banking:train:1498", "banking:train:1624", "banking:train:2614", "banking:train:2739", "banking:train:3055", "banking:train:306", "banking:train:3312", "banking:train:3724", "banking:train:4070", "banking:train:476", "banking:train:4882", "banking:train:5075", "banking:train:5332", "banking:train:5445", "banking:train:6096", "banking:train:746", "banking:train:7828", "banking:train:8176", "banking:train:8315", "banking:train:8653", "banking:train:9019", "banking:train:9370", "banking:train:9454", "banking:train:9595", "banking:train:9862", "banking:train:9963", "boolq:validation:1189", "boolq:validation:1320", "boolq:validation:1326", "boolq:validation:1381", "boolq:validation:1410", "boolq:validation:149", "boolq:validation:1513", "boolq:validation:1732", "boolq:validation:1739", "boolq:validation:1741", "boolq:validation:1814", "boolq:validation:1848", "boolq:validation:1940", "boolq:validation:2077", "boolq:validation:2091", "boolq:validation:2148", "boolq:validation:2215", "boolq:validation:2251", "boolq:validation:2283", "boolq:validation:2321", "boolq:validation:2398", "boolq:validation:253", "boolq:validation:2575", "boolq:validation:2907", "boolq:validation:2909", "boolq:validation:2978", "boolq:validation:359", "boolq:validation:394", "boolq:validation:436", "boolq:validation:579", "boolq:validation:807", "boolq:validation:949", "snli:validation:2389", "snli:validation:2485", "snli:validation:255", "snli:validation:3169", "snli:validation:333", "snli:validation:3700", "snli:validation:3709", "snli:validation:4093", "snli:validation:420", "snli:validation:4492", "snli:validation:4516", "snli:validation:4648", "snli:validation:4888", "snli:validation:5185", "snli:validation:5332", "snli:validation:5947", "snli:validation:6124", "snli:validation:660", "snli:validation:6619", "snli:validation:6898", "snli:validation:7387", "snli:validation:7492", "snli:validation:7544", "snli:validation:7996", "snli:validation:8074", "snli:validation:8554", "snli:validation:8971", "snli:validation:9163", "snli:validation:9277", "snli:validation:9490", "snli:validation:9595", "snli:validation:9688" ], "heldout_group_count": 128, "training_group_count": 384 }, { "fold": 2, "temperature_index": 57, "temperature": 2.046444636724674, "training_macro_nll": 0.3553716993757123, "heldout_ids": [ "arc:validation:ARC-Challenge:120", "arc:validation:ARC-Challenge:130", "arc:validation:ARC-Challenge:194", "arc:validation:ARC-Challenge:218", "arc:validation:ARC-Challenge:235", "arc:validation:ARC-Challenge:272", "arc:validation:ARC-Challenge:277", "arc:validation:ARC-Challenge:48", "arc:validation:ARC-Challenge:88", "arc:validation:ARC-Easy:115", "arc:validation:ARC-Easy:12", "arc:validation:ARC-Easy:131", "arc:validation:ARC-Easy:145", "arc:validation:ARC-Easy:179", "arc:validation:ARC-Easy:18", "arc:validation:ARC-Easy:186", "arc:validation:ARC-Easy:191", "arc:validation:ARC-Easy:215", "arc:validation:ARC-Easy:283", "arc:validation:ARC-Easy:314", "arc:validation:ARC-Easy:315", "arc:validation:ARC-Easy:378", "arc:validation:ARC-Easy:384", "arc:validation:ARC-Easy:403", "arc:validation:ARC-Easy:432", "arc:validation:ARC-Easy:439", "arc:validation:ARC-Easy:448", "arc:validation:ARC-Easy:461", "arc:validation:ARC-Easy:469", "arc:validation:ARC-Easy:470", "arc:validation:ARC-Easy:504", "arc:validation:ARC-Easy:561", "banking:train:1142", "banking:train:148", "banking:train:1633", "banking:train:1927", "banking:train:2186", "banking:train:2251", "banking:train:23", "banking:train:3048", "banking:train:3665", "banking:train:3952", "banking:train:3999", "banking:train:4364", "banking:train:4619", "banking:train:5049", "banking:train:5335", "banking:train:549", "banking:train:5957", "banking:train:5985", "banking:train:6132", "banking:train:6240", "banking:train:6488", "banking:train:7065", "banking:train:7749", "banking:train:8470", "banking:train:8761", "banking:train:895", "banking:train:9256", "banking:train:9406", "banking:train:9450", "banking:train:9467", "banking:train:9588", "banking:train:9716", "boolq:validation:1130", "boolq:validation:1167", "boolq:validation:124", "boolq:validation:1255", "boolq:validation:1256", "boolq:validation:1373", "boolq:validation:15", "boolq:validation:155", "boolq:validation:1568", "boolq:validation:16", "boolq:validation:1654", "boolq:validation:1702", "boolq:validation:1792", "boolq:validation:1818", "boolq:validation:210", "boolq:validation:2247", "boolq:validation:2259", "boolq:validation:2362", "boolq:validation:2443", "boolq:validation:2524", "boolq:validation:2553", "boolq:validation:2623", "boolq:validation:2660", "boolq:validation:2669", "boolq:validation:2679", "boolq:validation:2832", "boolq:validation:357", "boolq:validation:453", "boolq:validation:49", "boolq:validation:808", "boolq:validation:90", "boolq:validation:910", "snli:validation:1612", "snli:validation:168", "snli:validation:1693", "snli:validation:2134", "snli:validation:2248", "snli:validation:2422", "snli:validation:2587", "snli:validation:3007", "snli:validation:3097", "snli:validation:3772", "snli:validation:4012", "snli:validation:4168", "snli:validation:4537", "snli:validation:4681", "snli:validation:5050", "snli:validation:5632", "snli:validation:5761", "snli:validation:6055", "snli:validation:6283", "snli:validation:6715", "snli:validation:7117", "snli:validation:7192", "snli:validation:7507", "snli:validation:7594", "snli:validation:7924", "snli:validation:7954", "snli:validation:8035", "snli:validation:8110", "snli:validation:8995", "snli:validation:949", "snli:validation:9523", "snli:validation:9589" ], "heldout_group_count": 128, "training_group_count": 384 }, { "fold": 3, "temperature_index": 57, "temperature": 2.046444636724674, "training_macro_nll": 0.3736932084232408, "heldout_ids": [ "arc:validation:ARC-Challenge:117", "arc:validation:ARC-Challenge:135", "arc:validation:ARC-Challenge:139", "arc:validation:ARC-Challenge:154", "arc:validation:ARC-Challenge:174", "arc:validation:ARC-Challenge:199", "arc:validation:ARC-Challenge:287", "arc:validation:ARC-Challenge:295", "arc:validation:ARC-Challenge:4", "arc:validation:ARC-Challenge:50", "arc:validation:ARC-Challenge:51", "arc:validation:ARC-Challenge:61", "arc:validation:ARC-Easy:107", "arc:validation:ARC-Easy:121", "arc:validation:ARC-Easy:124", "arc:validation:ARC-Easy:171", "arc:validation:ARC-Easy:235", "arc:validation:ARC-Easy:241", "arc:validation:ARC-Easy:279", "arc:validation:ARC-Easy:289", "arc:validation:ARC-Easy:37", "arc:validation:ARC-Easy:389", "arc:validation:ARC-Easy:394", "arc:validation:ARC-Easy:399", "arc:validation:ARC-Easy:423", "arc:validation:ARC-Easy:492", "arc:validation:ARC-Easy:499", "arc:validation:ARC-Easy:523", "arc:validation:ARC-Easy:533", "arc:validation:ARC-Easy:569", "arc:validation:ARC-Easy:69", "arc:validation:ARC-Easy:93", "banking:train:2049", "banking:train:2350", "banking:train:2497", "banking:train:2506", "banking:train:2720", "banking:train:2788", "banking:train:2967", "banking:train:3502", "banking:train:3726", "banking:train:4014", "banking:train:4735", "banking:train:4803", "banking:train:4917", "banking:train:5477", "banking:train:5851", "banking:train:6551", "banking:train:6839", "banking:train:6926", "banking:train:7809", "banking:train:8092", "banking:train:8186", "banking:train:8215", "banking:train:8667", "banking:train:8752", "banking:train:8763", "banking:train:8851", "banking:train:8917", "banking:train:9145", "banking:train:9279", "banking:train:9676", "banking:train:9837", "banking:train:9967", "boolq:validation:106", "boolq:validation:1065", "boolq:validation:1097", "boolq:validation:1336", "boolq:validation:1350", "boolq:validation:1383", "boolq:validation:1458", "boolq:validation:1641", "boolq:validation:1873", "boolq:validation:1930", "boolq:validation:1989", "boolq:validation:2103", "boolq:validation:234", "boolq:validation:2355", "boolq:validation:2384", "boolq:validation:2440", "boolq:validation:2444", "boolq:validation:2478", "boolq:validation:2872", "boolq:validation:2879", "boolq:validation:2924", "boolq:validation:2971", "boolq:validation:3164", "boolq:validation:4", "boolq:validation:415", "boolq:validation:419", "boolq:validation:428", "boolq:validation:512", "boolq:validation:563", "boolq:validation:643", "boolq:validation:696", "boolq:validation:874", "snli:validation:1102", "snli:validation:1240", "snli:validation:1261", "snli:validation:1738", "snli:validation:1978", "snli:validation:2563", "snli:validation:2659", "snli:validation:2767", "snli:validation:2836", "snli:validation:3232", "snli:validation:3655", "snli:validation:3913", "snli:validation:405", "snli:validation:4051", "snli:validation:4300", "snli:validation:4846", "snli:validation:498", "snli:validation:5224", "snli:validation:5575", "snli:validation:6598", "snli:validation:666", "snli:validation:6925", "snli:validation:6964", "snli:validation:7045", "snli:validation:7351", "snli:validation:7588", "snli:validation:7723", "snli:validation:7750", "snli:validation:7909", "snli:validation:8974", "snli:validation:9385", "snli:validation:9646" ], "heldout_group_count": 128, "training_group_count": 384 } ], "per_family": { "arc": { "count": 128, "score": 0.4232126018397948, "raw_macro_nll": 0.5933205790802253, "accuracy": 0.859375 }, "banking": { "count": 128, "score": 0.15421812640309446, "raw_macro_nll": 0.2594607499213546, "accuracy": 0.9453125 }, "boolq": { "count": 128, "score": 0.4010937069730148, "raw_macro_nll": 0.43871330066590836, "accuracy": 0.8515625 }, "snli": { "count": 128, "score": 0.5009552539718878, "raw_macro_nll": 0.5258375821677139, "accuracy": 0.8515625 } }, "policy": { "id": "crossfit_temperature_nll_v1", "folds": 4, "seed": 431, "grouping": "source group, nested within task family; groups shuffled within sorted families and assigned round-robin", "temperature_grid": { "count": 101, "log10_min": -1.0, "log10_max": 1.3, "arithmetic": "Python float64" }, "temperature_fit": "minimum macro-family NLL on the other three folds; smallest temperature wins ties", "score": "macro-family mean of all held-out decision NLLs", "deployment_temperature": "fit afresh on reserved calibration only after model selection" } }, "best_selection_score": 0.36986992229694793, "improved": true, "metrics": { "overall": { "n": 512, "accuracy": 0.876953125, "nll": 0.45433305375755806, "brier_multiclass_sum": 0.1898162259719647, "ece_top_label_10_equal_width_bins": 0.05299526819726452, "reliability_bins": [ { "count": 0, "confidence_sum": 0.0, "correct_sum": 0.0 }, { "count": 0, "confidence_sum": 0.0, "correct_sum": 0.0 }, { "count": 0, "confidence_sum": 0.0, "correct_sum": 0.0 }, { "count": 0, "confidence_sum": 0.0, "correct_sum": 0.0 }, { "count": 7, "confidence_sum": 3.3479487597942352, "correct_sum": 0.0 }, { "count": 14, "confidence_sum": 7.681548655033112, "correct_sum": 9.0 }, { "count": 27, "confidence_sum": 17.5001118183136, "correct_sum": 15.0 }, { "count": 25, "confidence_sum": 18.57009321451187, "correct_sum": 17.0 }, { "count": 43, "confidence_sum": 37.1960312128067, "correct_sum": 36.0 }, { "count": 396, "confidence_sum": 389.20094096660614, "correct_sum": 372.0 } ], "accuracy_vs_coverage": { "0.25": { "n": 128, "accuracy": 1.0, "min_confidence": 0.9999995231628418 }, "0.5": { "n": 256, "accuracy": 0.97265625, "min_confidence": 0.993850588798523 }, "0.75": { "n": 384, "accuracy": 0.9427083333333334, "min_confidence": 0.9114243984222412 }, "1.0": { "n": 512, "accuracy": 0.876953125, "min_confidence": 0.4447111487388611 } } }, "per_family": { "arc": { "n": 128, "accuracy": 0.859375, "nll": 0.5933205873676035, "brier_multiclass_sum": 0.20391137341560844, "ece_top_label_10_equal_width_bins": 0.09224436432123184, "reliability_bins": [ { "count": 0, "confidence_sum": 0.0, "correct_sum": 0.0 }, { "count": 0, "confidence_sum": 0.0, "correct_sum": 0.0 }, { "count": 0, "confidence_sum": 0.0, "correct_sum": 0.0 }, { "count": 0, "confidence_sum": 0.0, "correct_sum": 0.0 }, { "count": 3, "confidence_sum": 1.3755568861961365, "correct_sum": 0.0 }, { "count": 5, "confidence_sum": 2.768744707107544, "correct_sum": 1.0 }, { "count": 3, "confidence_sum": 1.9373115301132202, "correct_sum": 3.0 }, { "count": 5, "confidence_sum": 3.6977739930152893, "correct_sum": 3.0 }, { "count": 9, "confidence_sum": 7.801501452922821, "correct_sum": 6.0 }, { "count": 103, "confidence_sum": 102.1010131239891, "correct_sum": 97.0 } ], "accuracy_vs_coverage": { "0.25": { "n": 32, "accuracy": 1.0, "min_confidence": 0.9999940395355225 }, "0.5": { "n": 64, "accuracy": 1.0, "min_confidence": 0.9995550513267517 }, "0.75": { "n": 96, "accuracy": 0.9583333333333334, "min_confidence": 0.9565119743347168 }, "1.0": { "n": 128, "accuracy": 0.859375, "min_confidence": 0.4447111487388611 } } }, "banking": { "n": 128, "accuracy": 0.9453125, "nll": 0.25946074807588604, "brier_multiclass_sum": 0.08189167536775005, "ece_top_label_10_equal_width_bins": 0.04227270046249032, "reliability_bins": [ { "count": 0, "confidence_sum": 0.0, "correct_sum": 0.0 }, { "count": 0, "confidence_sum": 0.0, "correct_sum": 0.0 }, { "count": 0, "confidence_sum": 0.0, "correct_sum": 0.0 }, { "count": 0, "confidence_sum": 0.0, "correct_sum": 0.0 }, { "count": 1, "confidence_sum": 0.481073260307312, "correct_sum": 0.0 }, { "count": 0, "confidence_sum": 0.0, "correct_sum": 0.0 }, { "count": 2, "confidence_sum": 1.2654950618743896, "correct_sum": 0.0 }, { "count": 0, "confidence_sum": 0.0, "correct_sum": 0.0 }, { "count": 0, "confidence_sum": 0.0, "correct_sum": 0.0 }, { "count": 125, "confidence_sum": 124.66433733701706, "correct_sum": 121.0 } ], "accuracy_vs_coverage": { "0.25": { "n": 32, "accuracy": 1.0, "min_confidence": 1.0 }, "0.5": { "n": 64, "accuracy": 1.0, "min_confidence": 1.0 }, "0.75": { "n": 96, "accuracy": 1.0, "min_confidence": 0.9999990463256836 }, "1.0": { "n": 128, "accuracy": 0.9453125, "min_confidence": 0.481073260307312 } } }, "boolq": { "n": 128, "accuracy": 0.8515625, "nll": 0.43871330363498373, "brier_multiclass_sum": 0.23252863315887956, "ece_top_label_10_equal_width_bins": 0.08019732777029276, "reliability_bins": [ { "count": 0, "confidence_sum": 0.0, "correct_sum": 0.0 }, { "count": 0, "confidence_sum": 0.0, "correct_sum": 0.0 }, { "count": 0, "confidence_sum": 0.0, "correct_sum": 0.0 }, { "count": 0, "confidence_sum": 0.0, "correct_sum": 0.0 }, { "count": 0, "confidence_sum": 0.0, "correct_sum": 0.0 }, { "count": 4, "confidence_sum": 2.1604248881340027, "correct_sum": 4.0 }, { "count": 11, "confidence_sum": 7.190720975399017, "correct_sum": 5.0 }, { "count": 8, "confidence_sum": 5.9833953976631165, "correct_sum": 4.0 }, { "count": 17, "confidence_sum": 14.755406379699707, "correct_sum": 15.0 }, { "count": 88, "confidence_sum": 85.00697284936905, "correct_sum": 81.0 } ], "accuracy_vs_coverage": { "0.25": { "n": 32, "accuracy": 0.9375, "min_confidence": 0.9843237996101379 }, "0.5": { "n": 64, "accuracy": 0.921875, "min_confidence": 0.9393304586410522 }, "0.75": { "n": 96, "accuracy": 0.9166666666666666, "min_confidence": 0.8829596042633057 }, "1.0": { "n": 128, "accuracy": 0.8515625, "min_confidence": 0.5294721722602844 } } }, "snli": { "n": 128, "accuracy": 0.8515625, "nll": 0.5258375759517588, "brier_multiclass_sum": 0.2409332219456208, "ece_top_label_10_equal_width_bins": 0.06832886137999594, "reliability_bins": [ { "count": 0, "confidence_sum": 0.0, "correct_sum": 0.0 }, { "count": 0, "confidence_sum": 0.0, "correct_sum": 0.0 }, { "count": 0, "confidence_sum": 0.0, "correct_sum": 0.0 }, { "count": 0, "confidence_sum": 0.0, "correct_sum": 0.0 }, { "count": 3, "confidence_sum": 1.4913186132907867, "correct_sum": 0.0 }, { "count": 5, "confidence_sum": 2.752379059791565, "correct_sum": 4.0 }, { "count": 11, "confidence_sum": 7.106584250926971, "correct_sum": 7.0 }, { "count": 12, "confidence_sum": 8.888923823833466, "correct_sum": 10.0 }, { "count": 17, "confidence_sum": 14.639123380184174, "correct_sum": 15.0 }, { "count": 80, "confidence_sum": 77.42861765623093, "correct_sum": 73.0 } ], "accuracy_vs_coverage": { "0.25": { "n": 32, "accuracy": 0.9375, "min_confidence": 0.9971461892127991 }, "0.5": { "n": 64, "accuracy": 0.921875, "min_confidence": 0.9333295226097107 }, "0.75": { "n": 96, "accuracy": 0.90625, "min_confidence": 0.8133017420768738 }, "1.0": { "n": 128, "accuracy": 0.8515625, "min_confidence": 0.49619075655937195 } } } } } }, { "step": 1000, "macro_nll": 0.4657101983683963, "best_macro_nll": 0.4657101983683963, "selection": { "metric": "crossfit_temperature_nll_v1", "score": 0.3281878376433132, "raw_macro_nll": 0.4657101993268551, "accuracy": 0.880859375, "folds": [ { "fold": 0, "temperature_index": 60, "temperature": 2.39883291901949, "training_macro_nll": 0.346048313200281, "heldout_ids": [ "arc:validation:ARC-Challenge:122", "arc:validation:ARC-Challenge:126", "arc:validation:ARC-Challenge:175", "arc:validation:ARC-Challenge:187", "arc:validation:ARC-Challenge:189", "arc:validation:ARC-Challenge:192", "arc:validation:ARC-Challenge:208", "arc:validation:ARC-Challenge:238", "arc:validation:ARC-Challenge:290", "arc:validation:ARC-Challenge:293", "arc:validation:ARC-Challenge:35", "arc:validation:ARC-Challenge:43", "arc:validation:ARC-Challenge:58", "arc:validation:ARC-Challenge:7", "arc:validation:ARC-Easy:160", "arc:validation:ARC-Easy:201", "arc:validation:ARC-Easy:229", "arc:validation:ARC-Easy:238", "arc:validation:ARC-Easy:253", "arc:validation:ARC-Easy:259", "arc:validation:ARC-Easy:272", "arc:validation:ARC-Easy:298", "arc:validation:ARC-Easy:329", "arc:validation:ARC-Easy:331", "arc:validation:ARC-Easy:337", "arc:validation:ARC-Easy:362", "arc:validation:ARC-Easy:443", "arc:validation:ARC-Easy:500", "arc:validation:ARC-Easy:545", "arc:validation:ARC-Easy:68", "arc:validation:ARC-Easy:7", "arc:validation:ARC-Easy:74", "banking:train:1053", "banking:train:1594", "banking:train:1747", "banking:train:202", "banking:train:2377", "banking:train:2528", "banking:train:2546", "banking:train:2715", "banking:train:3220", "banking:train:3499", "banking:train:3806", "banking:train:3838", "banking:train:3993", "banking:train:4220", "banking:train:5", "banking:train:5192", "banking:train:5394", "banking:train:5846", "banking:train:5863", "banking:train:6032", "banking:train:6508", "banking:train:7579", "banking:train:7600", "banking:train:7987", "banking:train:8155", "banking:train:850", "banking:train:8626", "banking:train:8929", "banking:train:9355", "banking:train:9812", "banking:train:987", "banking:train:9972", "boolq:validation:103", "boolq:validation:1042", "boolq:validation:1059", "boolq:validation:1541", "boolq:validation:1592", "boolq:validation:1927", "boolq:validation:2014", "boolq:validation:2023", "boolq:validation:2098", "boolq:validation:2139", "boolq:validation:2439", "boolq:validation:2485", "boolq:validation:2607", "boolq:validation:2611", "boolq:validation:2720", "boolq:validation:2751", "boolq:validation:282", "boolq:validation:2843", "boolq:validation:2953", "boolq:validation:301", "boolq:validation:3089", "boolq:validation:313", "boolq:validation:3201", "boolq:validation:322", "boolq:validation:3231", "boolq:validation:39", "boolq:validation:461", "boolq:validation:558", "boolq:validation:589", "boolq:validation:731", "boolq:validation:883", "boolq:validation:915", "snli:validation:1156", "snli:validation:1330", "snli:validation:1559", "snli:validation:1735", "snli:validation:1876", "snli:validation:2251", "snli:validation:2257", "snli:validation:2377", "snli:validation:2881", "snli:validation:291", "snli:validation:300", "snli:validation:3049", "snli:validation:3496", "snli:validation:3871", "snli:validation:4414", "snli:validation:5140", "snli:validation:5440", "snli:validation:5806", "snli:validation:6442", "snli:validation:6772", "snli:validation:6949", "snli:validation:7228", "snli:validation:7465", "snli:validation:7510", "snli:validation:7726", "snli:validation:8095", "snli:validation:8143", "snli:validation:8623", "snli:validation:8962", "snli:validation:9451", "snli:validation:9574", "snli:validation:9655" ], "heldout_group_count": 128, "training_group_count": 384 }, { "fold": 1, "temperature_index": 60, "temperature": 2.39883291901949, "training_macro_nll": 0.33008355176738974, "heldout_ids": [ "arc:validation:ARC-Challenge:101", "arc:validation:ARC-Challenge:129", "arc:validation:ARC-Challenge:148", "arc:validation:ARC-Challenge:156", "arc:validation:ARC-Challenge:167", "arc:validation:ARC-Challenge:249", "arc:validation:ARC-Challenge:263", "arc:validation:ARC-Challenge:266", "arc:validation:ARC-Challenge:267", "arc:validation:ARC-Challenge:28", "arc:validation:ARC-Challenge:281", "arc:validation:ARC-Challenge:57", "arc:validation:ARC-Challenge:63", "arc:validation:ARC-Challenge:64", "arc:validation:ARC-Challenge:86", "arc:validation:ARC-Challenge:99", "arc:validation:ARC-Easy:106", "arc:validation:ARC-Easy:151", "arc:validation:ARC-Easy:157", "arc:validation:ARC-Easy:166", "arc:validation:ARC-Easy:202", "arc:validation:ARC-Easy:219", "arc:validation:ARC-Easy:224", "arc:validation:ARC-Easy:322", "arc:validation:ARC-Easy:344", "arc:validation:ARC-Easy:382", "arc:validation:ARC-Easy:409", "arc:validation:ARC-Easy:455", "arc:validation:ARC-Easy:50", "arc:validation:ARC-Easy:538", "arc:validation:ARC-Easy:544", "arc:validation:ARC-Easy:552", "banking:train:1050", "banking:train:1131", "banking:train:1190", "banking:train:124", "banking:train:137", "banking:train:1493", "banking:train:1498", "banking:train:1624", "banking:train:2614", "banking:train:2739", "banking:train:3055", "banking:train:306", "banking:train:3312", "banking:train:3724", "banking:train:4070", "banking:train:476", "banking:train:4882", "banking:train:5075", "banking:train:5332", "banking:train:5445", "banking:train:6096", "banking:train:746", "banking:train:7828", "banking:train:8176", "banking:train:8315", "banking:train:8653", "banking:train:9019", "banking:train:9370", "banking:train:9454", "banking:train:9595", "banking:train:9862", "banking:train:9963", "boolq:validation:1189", "boolq:validation:1320", "boolq:validation:1326", "boolq:validation:1381", "boolq:validation:1410", "boolq:validation:149", "boolq:validation:1513", "boolq:validation:1732", "boolq:validation:1739", "boolq:validation:1741", "boolq:validation:1814", "boolq:validation:1848", "boolq:validation:1940", "boolq:validation:2077", "boolq:validation:2091", "boolq:validation:2148", "boolq:validation:2215", "boolq:validation:2251", "boolq:validation:2283", "boolq:validation:2321", "boolq:validation:2398", "boolq:validation:253", "boolq:validation:2575", "boolq:validation:2907", "boolq:validation:2909", "boolq:validation:2978", "boolq:validation:359", "boolq:validation:394", "boolq:validation:436", "boolq:validation:579", "boolq:validation:807", "boolq:validation:949", "snli:validation:2389", "snli:validation:2485", "snli:validation:255", "snli:validation:3169", "snli:validation:333", "snli:validation:3700", "snli:validation:3709", "snli:validation:4093", "snli:validation:420", "snli:validation:4492", "snli:validation:4516", "snli:validation:4648", "snli:validation:4888", "snli:validation:5185", "snli:validation:5332", "snli:validation:5947", "snli:validation:6124", "snli:validation:660", "snli:validation:6619", "snli:validation:6898", "snli:validation:7387", "snli:validation:7492", "snli:validation:7544", "snli:validation:7996", "snli:validation:8074", "snli:validation:8554", "snli:validation:8971", "snli:validation:9163", "snli:validation:9277", "snli:validation:9490", "snli:validation:9595", "snli:validation:9688" ], "heldout_group_count": 128, "training_group_count": 384 }, { "fold": 2, "temperature_index": 59, "temperature": 2.2750974307720706, "training_macro_nll": 0.30676367425506046, "heldout_ids": [ "arc:validation:ARC-Challenge:120", "arc:validation:ARC-Challenge:130", "arc:validation:ARC-Challenge:194", "arc:validation:ARC-Challenge:218", "arc:validation:ARC-Challenge:235", "arc:validation:ARC-Challenge:272", "arc:validation:ARC-Challenge:277", "arc:validation:ARC-Challenge:48", "arc:validation:ARC-Challenge:88", "arc:validation:ARC-Easy:115", "arc:validation:ARC-Easy:12", "arc:validation:ARC-Easy:131", "arc:validation:ARC-Easy:145", "arc:validation:ARC-Easy:179", "arc:validation:ARC-Easy:18", "arc:validation:ARC-Easy:186", "arc:validation:ARC-Easy:191", "arc:validation:ARC-Easy:215", "arc:validation:ARC-Easy:283", "arc:validation:ARC-Easy:314", "arc:validation:ARC-Easy:315", "arc:validation:ARC-Easy:378", "arc:validation:ARC-Easy:384", "arc:validation:ARC-Easy:403", "arc:validation:ARC-Easy:432", "arc:validation:ARC-Easy:439", "arc:validation:ARC-Easy:448", "arc:validation:ARC-Easy:461", "arc:validation:ARC-Easy:469", "arc:validation:ARC-Easy:470", "arc:validation:ARC-Easy:504", "arc:validation:ARC-Easy:561", "banking:train:1142", "banking:train:148", "banking:train:1633", "banking:train:1927", "banking:train:2186", "banking:train:2251", "banking:train:23", "banking:train:3048", "banking:train:3665", "banking:train:3952", "banking:train:3999", "banking:train:4364", "banking:train:4619", "banking:train:5049", "banking:train:5335", "banking:train:549", "banking:train:5957", "banking:train:5985", "banking:train:6132", "banking:train:6240", "banking:train:6488", "banking:train:7065", "banking:train:7749", "banking:train:8470", "banking:train:8761", "banking:train:895", "banking:train:9256", "banking:train:9406", "banking:train:9450", "banking:train:9467", "banking:train:9588", "banking:train:9716", "boolq:validation:1130", "boolq:validation:1167", "boolq:validation:124", "boolq:validation:1255", "boolq:validation:1256", "boolq:validation:1373", "boolq:validation:15", "boolq:validation:155", "boolq:validation:1568", "boolq:validation:16", "boolq:validation:1654", "boolq:validation:1702", "boolq:validation:1792", "boolq:validation:1818", "boolq:validation:210", "boolq:validation:2247", "boolq:validation:2259", "boolq:validation:2362", "boolq:validation:2443", "boolq:validation:2524", "boolq:validation:2553", "boolq:validation:2623", "boolq:validation:2660", "boolq:validation:2669", "boolq:validation:2679", "boolq:validation:2832", "boolq:validation:357", "boolq:validation:453", "boolq:validation:49", "boolq:validation:808", "boolq:validation:90", "boolq:validation:910", "snli:validation:1612", "snli:validation:168", "snli:validation:1693", "snli:validation:2134", "snli:validation:2248", "snli:validation:2422", "snli:validation:2587", "snli:validation:3007", "snli:validation:3097", "snli:validation:3772", "snli:validation:4012", "snli:validation:4168", "snli:validation:4537", "snli:validation:4681", "snli:validation:5050", "snli:validation:5632", "snli:validation:5761", "snli:validation:6055", "snli:validation:6283", "snli:validation:6715", "snli:validation:7117", "snli:validation:7192", "snli:validation:7507", "snli:validation:7594", "snli:validation:7924", "snli:validation:7954", "snli:validation:8035", "snli:validation:8110", "snli:validation:8995", "snli:validation:949", "snli:validation:9523", "snli:validation:9589" ], "heldout_group_count": 128, "training_group_count": 384 }, { "fold": 3, "temperature_index": 59, "temperature": 2.2750974307720706, "training_macro_nll": 0.32565293380222415, "heldout_ids": [ "arc:validation:ARC-Challenge:117", "arc:validation:ARC-Challenge:135", "arc:validation:ARC-Challenge:139", "arc:validation:ARC-Challenge:154", "arc:validation:ARC-Challenge:174", "arc:validation:ARC-Challenge:199", "arc:validation:ARC-Challenge:287", "arc:validation:ARC-Challenge:295", "arc:validation:ARC-Challenge:4", "arc:validation:ARC-Challenge:50", "arc:validation:ARC-Challenge:51", "arc:validation:ARC-Challenge:61", "arc:validation:ARC-Easy:107", "arc:validation:ARC-Easy:121", "arc:validation:ARC-Easy:124", "arc:validation:ARC-Easy:171", "arc:validation:ARC-Easy:235", "arc:validation:ARC-Easy:241", "arc:validation:ARC-Easy:279", "arc:validation:ARC-Easy:289", "arc:validation:ARC-Easy:37", "arc:validation:ARC-Easy:389", "arc:validation:ARC-Easy:394", "arc:validation:ARC-Easy:399", "arc:validation:ARC-Easy:423", "arc:validation:ARC-Easy:492", "arc:validation:ARC-Easy:499", "arc:validation:ARC-Easy:523", "arc:validation:ARC-Easy:533", "arc:validation:ARC-Easy:569", "arc:validation:ARC-Easy:69", "arc:validation:ARC-Easy:93", "banking:train:2049", "banking:train:2350", "banking:train:2497", "banking:train:2506", "banking:train:2720", "banking:train:2788", "banking:train:2967", "banking:train:3502", "banking:train:3726", "banking:train:4014", "banking:train:4735", "banking:train:4803", "banking:train:4917", "banking:train:5477", "banking:train:5851", "banking:train:6551", "banking:train:6839", "banking:train:6926", "banking:train:7809", "banking:train:8092", "banking:train:8186", "banking:train:8215", "banking:train:8667", "banking:train:8752", "banking:train:8763", "banking:train:8851", "banking:train:8917", "banking:train:9145", "banking:train:9279", "banking:train:9676", "banking:train:9837", "banking:train:9967", "boolq:validation:106", "boolq:validation:1065", "boolq:validation:1097", "boolq:validation:1336", "boolq:validation:1350", "boolq:validation:1383", "boolq:validation:1458", "boolq:validation:1641", "boolq:validation:1873", "boolq:validation:1930", "boolq:validation:1989", "boolq:validation:2103", "boolq:validation:234", "boolq:validation:2355", "boolq:validation:2384", "boolq:validation:2440", "boolq:validation:2444", "boolq:validation:2478", "boolq:validation:2872", "boolq:validation:2879", "boolq:validation:2924", "boolq:validation:2971", "boolq:validation:3164", "boolq:validation:4", "boolq:validation:415", "boolq:validation:419", "boolq:validation:428", "boolq:validation:512", "boolq:validation:563", "boolq:validation:643", "boolq:validation:696", "boolq:validation:874", "snli:validation:1102", "snli:validation:1240", "snli:validation:1261", "snli:validation:1738", "snli:validation:1978", "snli:validation:2563", "snli:validation:2659", "snli:validation:2767", "snli:validation:2836", "snli:validation:3232", "snli:validation:3655", "snli:validation:3913", "snli:validation:405", "snli:validation:4051", "snli:validation:4300", "snli:validation:4846", "snli:validation:498", "snli:validation:5224", "snli:validation:5575", "snli:validation:6598", "snli:validation:666", "snli:validation:6925", "snli:validation:6964", "snli:validation:7045", "snli:validation:7351", "snli:validation:7588", "snli:validation:7723", "snli:validation:7750", "snli:validation:7909", "snli:validation:8974", "snli:validation:9385", "snli:validation:9646" ], "heldout_group_count": 128, "training_group_count": 384 } ], "per_family": { "arc": { "count": 128, "score": 0.38706141779183195, "raw_macro_nll": 0.5262972364291403, "accuracy": 0.890625 }, "banking": { "count": 128, "score": 0.132631760684535, "raw_macro_nll": 0.22976245986744437, "accuracy": 0.953125 }, "boolq": { "count": 128, "score": 0.35232187333420917, "raw_macro_nll": 0.5460250472998999, "accuracy": 0.8359375 }, "snli": { "count": 128, "score": 0.4407362987626767, "raw_macro_nll": 0.560756053710936, "accuracy": 0.84375 } }, "policy": { "id": "crossfit_temperature_nll_v1", "folds": 4, "seed": 431, "grouping": "source group, nested within task family; groups shuffled within sorted families and assigned round-robin", "temperature_grid": { "count": 101, "log10_min": -1.0, "log10_max": 1.3, "arithmetic": "Python float64" }, "temperature_fit": "minimum macro-family NLL on the other three folds; smallest temperature wins ties", "score": "macro-family mean of all held-out decision NLLs", "deployment_temperature": "fit afresh on reserved calibration only after model selection" } }, "best_selection_score": 0.3281878376433132, "improved": true, "metrics": { "overall": { "n": 512, "accuracy": 0.880859375, "nll": 0.4657101983683963, "brier_multiclass_sum": 0.1966596386060834, "ece_top_label_10_equal_width_bins": 0.08195119712036103, "reliability_bins": [ { "count": 0, "confidence_sum": 0.0, "correct_sum": 0.0 }, { "count": 0, "confidence_sum": 0.0, "correct_sum": 0.0 }, { "count": 0, "confidence_sum": 0.0, "correct_sum": 0.0 }, { "count": 0, "confidence_sum": 0.0, "correct_sum": 0.0 }, { "count": 4, "confidence_sum": 1.8117924332618713, "correct_sum": 4.0 }, { "count": 16, "confidence_sum": 8.777001142501831, "correct_sum": 6.0 }, { "count": 6, "confidence_sum": 3.8906716108322144, "correct_sum": 2.0 }, { "count": 17, "confidence_sum": 12.783255934715271, "correct_sum": 9.0 }, { "count": 20, "confidence_sum": 17.166792809963226, "correct_sum": 17.0 }, { "count": 449, "confidence_sum": 444.1530838608742, "correct_sum": 413.0 } ], "accuracy_vs_coverage": { "0.25": { "n": 128, "accuracy": 1.0, "min_confidence": 0.999997615814209 }, "0.5": { "n": 256, "accuracy": 0.9921875, "min_confidence": 0.999165415763855 }, "0.75": { "n": 384, "accuracy": 0.9453125, "min_confidence": 0.9738333225250244 }, "1.0": { "n": 512, "accuracy": 0.880859375, "min_confidence": 0.40963199734687805 } } }, "per_family": { "arc": { "n": 128, "accuracy": 0.890625, "nll": 0.5262972319795522, "brier_multiclass_sum": 0.19759357566681648, "ece_top_label_10_equal_width_bins": 0.08619366958737373, "reliability_bins": [ { "count": 0, "confidence_sum": 0.0, "correct_sum": 0.0 }, { "count": 0, "confidence_sum": 0.0, "correct_sum": 0.0 }, { "count": 0, "confidence_sum": 0.0, "correct_sum": 0.0 }, { "count": 0, "confidence_sum": 0.0, "correct_sum": 0.0 }, { "count": 4, "confidence_sum": 1.8117924332618713, "correct_sum": 4.0 }, { "count": 5, "confidence_sum": 2.6929340958595276, "correct_sum": 3.0 }, { "count": 1, "confidence_sum": 0.6008858680725098, "correct_sum": 0.0 }, { "count": 6, "confidence_sum": 4.4938578605651855, "correct_sum": 4.0 }, { "count": 4, "confidence_sum": 3.384382665157318, "correct_sum": 3.0 }, { "count": 108, "confidence_sum": 107.05838984251022, "correct_sum": 100.0 } ], "accuracy_vs_coverage": { "0.25": { "n": 32, "accuracy": 1.0, "min_confidence": 0.999995231628418 }, "0.5": { "n": 64, "accuracy": 1.0, "min_confidence": 0.9993054866790771 }, "0.75": { "n": 96, "accuracy": 0.9479166666666666, "min_confidence": 0.9695820808410645 }, "1.0": { "n": 128, "accuracy": 0.890625, "min_confidence": 0.40963199734687805 } } }, "banking": { "n": 128, "accuracy": 0.953125, "nll": 0.22976245635574838, "brier_multiclass_sum": 0.08413616641877218, "ece_top_label_10_equal_width_bins": 0.04192498791962862, "reliability_bins": [ { "count": 0, "confidence_sum": 0.0, "correct_sum": 0.0 }, { "count": 0, "confidence_sum": 0.0, "correct_sum": 0.0 }, { "count": 0, "confidence_sum": 0.0, "correct_sum": 0.0 }, { "count": 0, "confidence_sum": 0.0, "correct_sum": 0.0 }, { "count": 0, "confidence_sum": 0.0, "correct_sum": 0.0 }, { "count": 0, "confidence_sum": 0.0, "correct_sum": 0.0 }, { "count": 0, "confidence_sum": 0.0, "correct_sum": 0.0 }, { "count": 2, "confidence_sum": 1.5179685354232788, "correct_sum": 1.0 }, { "count": 1, "confidence_sum": 0.8676369190216064, "correct_sum": 1.0 }, { "count": 125, "confidence_sum": 124.71606683731079, "correct_sum": 120.0 } ], "accuracy_vs_coverage": { "0.25": { "n": 32, "accuracy": 1.0, "min_confidence": 1.0 }, "0.5": { "n": 64, "accuracy": 1.0, "min_confidence": 1.0 }, "0.75": { "n": 96, "accuracy": 1.0, "min_confidence": 0.9999890327453613 }, "1.0": { "n": 128, "accuracy": 0.953125, "min_confidence": 0.7520317435264587 } } }, "boolq": { "n": 128, "accuracy": 0.8359375, "nll": 0.5460250501834345, "brier_multiclass_sum": 0.2505062007079596, "ece_top_label_10_equal_width_bins": 0.11891159228980541, "reliability_bins": [ { "count": 0, "confidence_sum": 0.0, "correct_sum": 0.0 }, { "count": 0, "confidence_sum": 0.0, "correct_sum": 0.0 }, { "count": 0, "confidence_sum": 0.0, "correct_sum": 0.0 }, { "count": 0, "confidence_sum": 0.0, "correct_sum": 0.0 }, { "count": 0, "confidence_sum": 0.0, "correct_sum": 0.0 }, { "count": 4, "confidence_sum": 2.2954535484313965, "correct_sum": 0.0 }, { "count": 1, "confidence_sum": 0.6820338368415833, "correct_sum": 0.0 }, { "count": 6, "confidence_sum": 4.4924063086509705, "correct_sum": 3.0 }, { "count": 6, "confidence_sum": 5.07128119468689, "correct_sum": 5.0 }, { "count": 111, "confidence_sum": 109.67950892448425, "correct_sum": 99.0 } ], "accuracy_vs_coverage": { "0.25": { "n": 32, "accuracy": 1.0, "min_confidence": 0.9998440742492676 }, "0.5": { "n": 64, "accuracy": 0.96875, "min_confidence": 0.9980106949806213 }, "0.75": { "n": 96, "accuracy": 0.8958333333333334, "min_confidence": 0.9715626835823059 }, "1.0": { "n": 128, "accuracy": 0.8359375, "min_confidence": 0.5350096821784973 } } }, "snli": { "n": 128, "accuracy": 0.84375, "nll": 0.56075605495485, "brier_multiclass_sum": 0.2544026116307853, "ece_top_label_10_equal_width_bins": 0.09008605359122157, "reliability_bins": [ { "count": 0, "confidence_sum": 0.0, "correct_sum": 0.0 }, { "count": 0, "confidence_sum": 0.0, "correct_sum": 0.0 }, { "count": 0, "confidence_sum": 0.0, "correct_sum": 0.0 }, { "count": 0, "confidence_sum": 0.0, "correct_sum": 0.0 }, { "count": 0, "confidence_sum": 0.0, "correct_sum": 0.0 }, { "count": 7, "confidence_sum": 3.788613498210907, "correct_sum": 3.0 }, { "count": 4, "confidence_sum": 2.6077519059181213, "correct_sum": 2.0 }, { "count": 3, "confidence_sum": 2.279023230075836, "correct_sum": 1.0 }, { "count": 9, "confidence_sum": 7.843492031097412, "correct_sum": 8.0 }, { "count": 105, "confidence_sum": 102.69911825656891, "correct_sum": 94.0 } ], "accuracy_vs_coverage": { "0.25": { "n": 32, "accuracy": 0.96875, "min_confidence": 0.995185911655426 }, "0.5": { "n": 64, "accuracy": 0.96875, "min_confidence": 0.980144739151001 }, "0.75": { "n": 96, "accuracy": 0.90625, "min_confidence": 0.9447706341743469 }, "1.0": { "n": 128, "accuracy": 0.84375, "min_confidence": 0.5119929313659668 } } } } } }, { "step": 1500, "macro_nll": 0.3541424148692931, "best_macro_nll": 0.4657101983683963, "selection": { "metric": "crossfit_temperature_nll_v1", "score": 0.3300765088337009, "raw_macro_nll": 0.35414241381908923, "accuracy": 0.8828125, "folds": [ { "fold": 0, "temperature_index": 52, "temperature": 1.5703628043335522, "training_macro_nll": 0.3473758787149227, "heldout_ids": [ "arc:validation:ARC-Challenge:122", "arc:validation:ARC-Challenge:126", "arc:validation:ARC-Challenge:175", "arc:validation:ARC-Challenge:187", "arc:validation:ARC-Challenge:189", "arc:validation:ARC-Challenge:192", "arc:validation:ARC-Challenge:208", "arc:validation:ARC-Challenge:238", "arc:validation:ARC-Challenge:290", "arc:validation:ARC-Challenge:293", "arc:validation:ARC-Challenge:35", "arc:validation:ARC-Challenge:43", "arc:validation:ARC-Challenge:58", "arc:validation:ARC-Challenge:7", "arc:validation:ARC-Easy:160", "arc:validation:ARC-Easy:201", "arc:validation:ARC-Easy:229", "arc:validation:ARC-Easy:238", "arc:validation:ARC-Easy:253", "arc:validation:ARC-Easy:259", "arc:validation:ARC-Easy:272", "arc:validation:ARC-Easy:298", "arc:validation:ARC-Easy:329", "arc:validation:ARC-Easy:331", "arc:validation:ARC-Easy:337", "arc:validation:ARC-Easy:362", "arc:validation:ARC-Easy:443", "arc:validation:ARC-Easy:500", "arc:validation:ARC-Easy:545", "arc:validation:ARC-Easy:68", "arc:validation:ARC-Easy:7", "arc:validation:ARC-Easy:74", "banking:train:1053", "banking:train:1594", "banking:train:1747", "banking:train:202", "banking:train:2377", "banking:train:2528", "banking:train:2546", "banking:train:2715", "banking:train:3220", "banking:train:3499", "banking:train:3806", "banking:train:3838", "banking:train:3993", "banking:train:4220", "banking:train:5", "banking:train:5192", "banking:train:5394", "banking:train:5846", "banking:train:5863", "banking:train:6032", "banking:train:6508", "banking:train:7579", "banking:train:7600", "banking:train:7987", "banking:train:8155", "banking:train:850", "banking:train:8626", "banking:train:8929", "banking:train:9355", "banking:train:9812", "banking:train:987", "banking:train:9972", "boolq:validation:103", "boolq:validation:1042", "boolq:validation:1059", "boolq:validation:1541", "boolq:validation:1592", "boolq:validation:1927", "boolq:validation:2014", "boolq:validation:2023", "boolq:validation:2098", "boolq:validation:2139", "boolq:validation:2439", "boolq:validation:2485", "boolq:validation:2607", "boolq:validation:2611", "boolq:validation:2720", "boolq:validation:2751", "boolq:validation:282", "boolq:validation:2843", "boolq:validation:2953", "boolq:validation:301", "boolq:validation:3089", "boolq:validation:313", "boolq:validation:3201", "boolq:validation:322", "boolq:validation:3231", "boolq:validation:39", "boolq:validation:461", "boolq:validation:558", "boolq:validation:589", "boolq:validation:731", "boolq:validation:883", "boolq:validation:915", "snli:validation:1156", "snli:validation:1330", "snli:validation:1559", "snli:validation:1735", "snli:validation:1876", "snli:validation:2251", "snli:validation:2257", "snli:validation:2377", "snli:validation:2881", "snli:validation:291", "snli:validation:300", "snli:validation:3049", "snli:validation:3496", "snli:validation:3871", "snli:validation:4414", "snli:validation:5140", "snli:validation:5440", "snli:validation:5806", "snli:validation:6442", "snli:validation:6772", "snli:validation:6949", "snli:validation:7228", "snli:validation:7465", "snli:validation:7510", "snli:validation:7726", "snli:validation:8095", "snli:validation:8143", "snli:validation:8623", "snli:validation:8962", "snli:validation:9451", "snli:validation:9574", "snli:validation:9655" ], "heldout_group_count": 128, "training_group_count": 384 }, { "fold": 1, "temperature_index": 51, "temperature": 1.4893610777109154, "training_macro_nll": 0.33312313335211086, "heldout_ids": [ "arc:validation:ARC-Challenge:101", "arc:validation:ARC-Challenge:129", "arc:validation:ARC-Challenge:148", "arc:validation:ARC-Challenge:156", "arc:validation:ARC-Challenge:167", "arc:validation:ARC-Challenge:249", "arc:validation:ARC-Challenge:263", "arc:validation:ARC-Challenge:266", "arc:validation:ARC-Challenge:267", "arc:validation:ARC-Challenge:28", "arc:validation:ARC-Challenge:281", "arc:validation:ARC-Challenge:57", "arc:validation:ARC-Challenge:63", "arc:validation:ARC-Challenge:64", "arc:validation:ARC-Challenge:86", "arc:validation:ARC-Challenge:99", "arc:validation:ARC-Easy:106", "arc:validation:ARC-Easy:151", "arc:validation:ARC-Easy:157", "arc:validation:ARC-Easy:166", "arc:validation:ARC-Easy:202", "arc:validation:ARC-Easy:219", "arc:validation:ARC-Easy:224", "arc:validation:ARC-Easy:322", "arc:validation:ARC-Easy:344", "arc:validation:ARC-Easy:382", "arc:validation:ARC-Easy:409", "arc:validation:ARC-Easy:455", "arc:validation:ARC-Easy:50", "arc:validation:ARC-Easy:538", "arc:validation:ARC-Easy:544", "arc:validation:ARC-Easy:552", "banking:train:1050", "banking:train:1131", "banking:train:1190", "banking:train:124", "banking:train:137", "banking:train:1493", "banking:train:1498", "banking:train:1624", "banking:train:2614", "banking:train:2739", "banking:train:3055", "banking:train:306", "banking:train:3312", "banking:train:3724", "banking:train:4070", "banking:train:476", "banking:train:4882", "banking:train:5075", "banking:train:5332", "banking:train:5445", "banking:train:6096", "banking:train:746", "banking:train:7828", "banking:train:8176", "banking:train:8315", "banking:train:8653", "banking:train:9019", "banking:train:9370", "banking:train:9454", "banking:train:9595", "banking:train:9862", "banking:train:9963", "boolq:validation:1189", "boolq:validation:1320", "boolq:validation:1326", "boolq:validation:1381", "boolq:validation:1410", "boolq:validation:149", "boolq:validation:1513", "boolq:validation:1732", "boolq:validation:1739", "boolq:validation:1741", "boolq:validation:1814", "boolq:validation:1848", "boolq:validation:1940", "boolq:validation:2077", "boolq:validation:2091", "boolq:validation:2148", "boolq:validation:2215", "boolq:validation:2251", "boolq:validation:2283", "boolq:validation:2321", "boolq:validation:2398", "boolq:validation:253", "boolq:validation:2575", "boolq:validation:2907", "boolq:validation:2909", "boolq:validation:2978", "boolq:validation:359", "boolq:validation:394", "boolq:validation:436", "boolq:validation:579", "boolq:validation:807", "boolq:validation:949", "snli:validation:2389", "snli:validation:2485", "snli:validation:255", "snli:validation:3169", "snli:validation:333", "snli:validation:3700", "snli:validation:3709", "snli:validation:4093", "snli:validation:420", "snli:validation:4492", "snli:validation:4516", "snli:validation:4648", "snli:validation:4888", "snli:validation:5185", "snli:validation:5332", "snli:validation:5947", "snli:validation:6124", "snli:validation:660", "snli:validation:6619", "snli:validation:6898", "snli:validation:7387", "snli:validation:7492", "snli:validation:7544", "snli:validation:7996", "snli:validation:8074", "snli:validation:8554", "snli:validation:8971", "snli:validation:9163", "snli:validation:9277", "snli:validation:9490", "snli:validation:9595", "snli:validation:9688" ], "heldout_group_count": 128, "training_group_count": 384 }, { "fold": 2, "temperature_index": 51, "temperature": 1.4893610777109154, "training_macro_nll": 0.32299109700120576, "heldout_ids": [ "arc:validation:ARC-Challenge:120", "arc:validation:ARC-Challenge:130", "arc:validation:ARC-Challenge:194", "arc:validation:ARC-Challenge:218", "arc:validation:ARC-Challenge:235", "arc:validation:ARC-Challenge:272", "arc:validation:ARC-Challenge:277", "arc:validation:ARC-Challenge:48", "arc:validation:ARC-Challenge:88", "arc:validation:ARC-Easy:115", "arc:validation:ARC-Easy:12", "arc:validation:ARC-Easy:131", "arc:validation:ARC-Easy:145", "arc:validation:ARC-Easy:179", "arc:validation:ARC-Easy:18", "arc:validation:ARC-Easy:186", "arc:validation:ARC-Easy:191", "arc:validation:ARC-Easy:215", "arc:validation:ARC-Easy:283", "arc:validation:ARC-Easy:314", "arc:validation:ARC-Easy:315", "arc:validation:ARC-Easy:378", "arc:validation:ARC-Easy:384", "arc:validation:ARC-Easy:403", "arc:validation:ARC-Easy:432", "arc:validation:ARC-Easy:439", "arc:validation:ARC-Easy:448", "arc:validation:ARC-Easy:461", "arc:validation:ARC-Easy:469", "arc:validation:ARC-Easy:470", "arc:validation:ARC-Easy:504", "arc:validation:ARC-Easy:561", "banking:train:1142", "banking:train:148", "banking:train:1633", "banking:train:1927", "banking:train:2186", "banking:train:2251", "banking:train:23", "banking:train:3048", "banking:train:3665", "banking:train:3952", "banking:train:3999", "banking:train:4364", "banking:train:4619", "banking:train:5049", "banking:train:5335", "banking:train:549", "banking:train:5957", "banking:train:5985", "banking:train:6132", "banking:train:6240", "banking:train:6488", "banking:train:7065", "banking:train:7749", "banking:train:8470", "banking:train:8761", "banking:train:895", "banking:train:9256", "banking:train:9406", "banking:train:9450", "banking:train:9467", "banking:train:9588", "banking:train:9716", "boolq:validation:1130", "boolq:validation:1167", "boolq:validation:124", "boolq:validation:1255", "boolq:validation:1256", "boolq:validation:1373", "boolq:validation:15", "boolq:validation:155", "boolq:validation:1568", "boolq:validation:16", "boolq:validation:1654", "boolq:validation:1702", "boolq:validation:1792", "boolq:validation:1818", "boolq:validation:210", "boolq:validation:2247", "boolq:validation:2259", "boolq:validation:2362", "boolq:validation:2443", "boolq:validation:2524", "boolq:validation:2553", "boolq:validation:2623", "boolq:validation:2660", "boolq:validation:2669", "boolq:validation:2679", "boolq:validation:2832", "boolq:validation:357", "boolq:validation:453", "boolq:validation:49", "boolq:validation:808", "boolq:validation:90", "boolq:validation:910", "snli:validation:1612", "snli:validation:168", "snli:validation:1693", "snli:validation:2134", "snli:validation:2248", "snli:validation:2422", "snli:validation:2587", "snli:validation:3007", "snli:validation:3097", "snli:validation:3772", "snli:validation:4012", "snli:validation:4168", "snli:validation:4537", "snli:validation:4681", "snli:validation:5050", "snli:validation:5632", "snli:validation:5761", "snli:validation:6055", "snli:validation:6283", "snli:validation:6715", "snli:validation:7117", "snli:validation:7192", "snli:validation:7507", "snli:validation:7594", "snli:validation:7924", "snli:validation:7954", "snli:validation:8035", "snli:validation:8110", "snli:validation:8995", "snli:validation:949", "snli:validation:9523", "snli:validation:9589" ], "heldout_group_count": 128, "training_group_count": 384 }, { "fold": 3, "temperature_index": 49, "temperature": 1.3396766874259345, "training_macro_nll": 0.30270275653699436, "heldout_ids": [ "arc:validation:ARC-Challenge:117", "arc:validation:ARC-Challenge:135", "arc:validation:ARC-Challenge:139", "arc:validation:ARC-Challenge:154", "arc:validation:ARC-Challenge:174", "arc:validation:ARC-Challenge:199", "arc:validation:ARC-Challenge:287", "arc:validation:ARC-Challenge:295", "arc:validation:ARC-Challenge:4", "arc:validation:ARC-Challenge:50", "arc:validation:ARC-Challenge:51", "arc:validation:ARC-Challenge:61", "arc:validation:ARC-Easy:107", "arc:validation:ARC-Easy:121", "arc:validation:ARC-Easy:124", "arc:validation:ARC-Easy:171", "arc:validation:ARC-Easy:235", "arc:validation:ARC-Easy:241", "arc:validation:ARC-Easy:279", "arc:validation:ARC-Easy:289", "arc:validation:ARC-Easy:37", "arc:validation:ARC-Easy:389", "arc:validation:ARC-Easy:394", "arc:validation:ARC-Easy:399", "arc:validation:ARC-Easy:423", "arc:validation:ARC-Easy:492", "arc:validation:ARC-Easy:499", "arc:validation:ARC-Easy:523", "arc:validation:ARC-Easy:533", "arc:validation:ARC-Easy:569", "arc:validation:ARC-Easy:69", "arc:validation:ARC-Easy:93", "banking:train:2049", "banking:train:2350", "banking:train:2497", "banking:train:2506", "banking:train:2720", "banking:train:2788", "banking:train:2967", "banking:train:3502", "banking:train:3726", "banking:train:4014", "banking:train:4735", "banking:train:4803", "banking:train:4917", "banking:train:5477", "banking:train:5851", "banking:train:6551", "banking:train:6839", "banking:train:6926", "banking:train:7809", "banking:train:8092", "banking:train:8186", "banking:train:8215", "banking:train:8667", "banking:train:8752", "banking:train:8763", "banking:train:8851", "banking:train:8917", "banking:train:9145", "banking:train:9279", "banking:train:9676", "banking:train:9837", "banking:train:9967", "boolq:validation:106", "boolq:validation:1065", "boolq:validation:1097", "boolq:validation:1336", "boolq:validation:1350", "boolq:validation:1383", "boolq:validation:1458", "boolq:validation:1641", "boolq:validation:1873", "boolq:validation:1930", "boolq:validation:1989", "boolq:validation:2103", "boolq:validation:234", "boolq:validation:2355", "boolq:validation:2384", "boolq:validation:2440", "boolq:validation:2444", "boolq:validation:2478", "boolq:validation:2872", "boolq:validation:2879", "boolq:validation:2924", "boolq:validation:2971", "boolq:validation:3164", "boolq:validation:4", "boolq:validation:415", "boolq:validation:419", "boolq:validation:428", "boolq:validation:512", "boolq:validation:563", "boolq:validation:643", "boolq:validation:696", "boolq:validation:874", "snli:validation:1102", "snli:validation:1240", "snli:validation:1261", "snli:validation:1738", "snli:validation:1978", "snli:validation:2563", "snli:validation:2659", "snli:validation:2767", "snli:validation:2836", "snli:validation:3232", "snli:validation:3655", "snli:validation:3913", "snli:validation:405", "snli:validation:4051", "snli:validation:4300", "snli:validation:4846", "snli:validation:498", "snli:validation:5224", "snli:validation:5575", "snli:validation:6598", "snli:validation:666", "snli:validation:6925", "snli:validation:6964", "snli:validation:7045", "snli:validation:7351", "snli:validation:7588", "snli:validation:7723", "snli:validation:7750", "snli:validation:7909", "snli:validation:8974", "snli:validation:9385", "snli:validation:9646" ], "heldout_group_count": 128, "training_group_count": 384 } ], "per_family": { "arc": { "count": 128, "score": 0.39971936917627643, "raw_macro_nll": 0.44148110277449976, "accuracy": 0.8828125 }, "banking": { "count": 128, "score": 0.13308767041911446, "raw_macro_nll": 0.1467244923439624, "accuracy": 0.9375 }, "boolq": { "count": 128, "score": 0.41748072217353377, "raw_macro_nll": 0.4907469603329647, "accuracy": 0.8203125 }, "snli": { "count": 128, "score": 0.3700182735658788, "raw_macro_nll": 0.3376170998249301, "accuracy": 0.890625 } }, "policy": { "id": "crossfit_temperature_nll_v1", "folds": 4, "seed": 431, "grouping": "source group, nested within task family; groups shuffled within sorted families and assigned round-robin", "temperature_grid": { "count": 101, "log10_min": -1.0, "log10_max": 1.3, "arithmetic": "Python float64" }, "temperature_fit": "minimum macro-family NLL on the other three folds; smallest temperature wins ties", "score": "macro-family mean of all held-out decision NLLs", "deployment_temperature": "fit afresh on reserved calibration only after model selection" } }, "best_selection_score": 0.3281878376433132, "improved": false, "metrics": { "overall": { "n": 512, "accuracy": 0.8828125, "nll": 0.3541424148692931, "brier_multiclass_sum": 0.1805443581641838, "ece_top_label_10_equal_width_bins": 0.04613619588781148, "reliability_bins": [ { "count": 0, "confidence_sum": 0.0, "correct_sum": 0.0 }, { "count": 0, "confidence_sum": 0.0, "correct_sum": 0.0 }, { "count": 0, "confidence_sum": 0.0, "correct_sum": 0.0 }, { "count": 3, "confidence_sum": 1.1111035645008087, "correct_sum": 1.0 }, { "count": 7, "confidence_sum": 3.200128525495529, "correct_sum": 4.0 }, { "count": 13, "confidence_sum": 7.04876035451889, "correct_sum": 5.0 }, { "count": 17, "confidence_sum": 11.040592074394226, "correct_sum": 11.0 }, { "count": 22, "confidence_sum": 16.702092111110687, "correct_sum": 14.0 }, { "count": 57, "confidence_sum": 48.59177905321121, "correct_sum": 45.0 }, { "count": 393, "confidence_sum": 386.3275336623192, "correct_sum": 372.0 } ], "accuracy_vs_coverage": { "0.25": { "n": 128, "accuracy": 1.0, "min_confidence": 0.9998112320899963 }, "0.5": { "n": 256, "accuracy": 0.984375, "min_confidence": 0.9876569509506226 }, "0.75": { "n": 384, "accuracy": 0.9479166666666666, "min_confidence": 0.9104589223861694 }, "1.0": { "n": 512, "accuracy": 0.8828125, "min_confidence": 0.3282764256000519 } } }, "per_family": { "arc": { "n": 128, "accuracy": 0.8828125, "nll": 0.4414811164252181, "brier_multiclass_sum": 0.18197369388678808, "ece_top_label_10_equal_width_bins": 0.0416633328422904, "reliability_bins": [ { "count": 0, "confidence_sum": 0.0, "correct_sum": 0.0 }, { "count": 0, "confidence_sum": 0.0, "correct_sum": 0.0 }, { "count": 0, "confidence_sum": 0.0, "correct_sum": 0.0 }, { "count": 3, "confidence_sum": 1.1111035645008087, "correct_sum": 1.0 }, { "count": 5, "confidence_sum": 2.229231983423233, "correct_sum": 3.0 }, { "count": 6, "confidence_sum": 3.2244449257850647, "correct_sum": 3.0 }, { "count": 0, "confidence_sum": 0.0, "correct_sum": 0.0 }, { "count": 3, "confidence_sum": 2.3234559297561646, "correct_sum": 2.0 }, { "count": 12, "confidence_sum": 10.301778793334961, "correct_sum": 10.0 }, { "count": 99, "confidence_sum": 97.6013553738594, "correct_sum": 94.0 } ], "accuracy_vs_coverage": { "0.25": { "n": 32, "accuracy": 1.0, "min_confidence": 0.9998188614845276 }, "0.5": { "n": 64, "accuracy": 0.984375, "min_confidence": 0.9946053624153137 }, "0.75": { "n": 96, "accuracy": 0.9479166666666666, "min_confidence": 0.91451096534729 }, "1.0": { "n": 128, "accuracy": 0.8828125, "min_confidence": 0.3282764256000519 } } }, "banking": { "n": 128, "accuracy": 0.9375, "nll": 0.14672449013245747, "brier_multiclass_sum": 0.08483707779111284, "ece_top_label_10_equal_width_bins": 0.028193810489028692, "reliability_bins": [ { "count": 0, "confidence_sum": 0.0, "correct_sum": 0.0 }, { "count": 0, "confidence_sum": 0.0, "correct_sum": 0.0 }, { "count": 0, "confidence_sum": 0.0, "correct_sum": 0.0 }, { "count": 0, "confidence_sum": 0.0, "correct_sum": 0.0 }, { "count": 2, "confidence_sum": 0.9708965420722961, "correct_sum": 1.0 }, { "count": 2, "confidence_sum": 1.1783032417297363, "correct_sum": 0.0 }, { "count": 3, "confidence_sum": 1.9266068935394287, "correct_sum": 2.0 }, { "count": 3, "confidence_sum": 2.2358115315437317, "correct_sum": 2.0 }, { "count": 3, "confidence_sum": 2.4956213235855103, "correct_sum": 2.0 }, { "count": 115, "confidence_sum": 114.59657508134842, "correct_sum": 113.0 } ], "accuracy_vs_coverage": { "0.25": { "n": 32, "accuracy": 1.0, "min_confidence": 1.0 }, "0.5": { "n": 64, "accuracy": 1.0, "min_confidence": 0.999998927116394 }, "0.75": { "n": 96, "accuracy": 1.0, "min_confidence": 0.9996840953826904 }, "1.0": { "n": 128, "accuracy": 0.9375, "min_confidence": 0.47107839584350586 } } }, "boolq": { "n": 128, "accuracy": 0.8203125, "nll": 0.49074695723096795, "brier_multiclass_sum": 0.2747688886485723, "ece_top_label_10_equal_width_bins": 0.1055659051053226, "reliability_bins": [ { "count": 0, "confidence_sum": 0.0, "correct_sum": 0.0 }, { "count": 0, "confidence_sum": 0.0, "correct_sum": 0.0 }, { "count": 0, "confidence_sum": 0.0, "correct_sum": 0.0 }, { "count": 0, "confidence_sum": 0.0, "correct_sum": 0.0 }, { "count": 0, "confidence_sum": 0.0, "correct_sum": 0.0 }, { "count": 3, "confidence_sum": 1.5206995010375977, "correct_sum": 1.0 }, { "count": 6, "confidence_sum": 3.793248772621155, "correct_sum": 3.0 }, { "count": 9, "confidence_sum": 6.839642345905304, "correct_sum": 5.0 }, { "count": 9, "confidence_sum": 7.7483291029930115, "correct_sum": 6.0 }, { "count": 101, "confidence_sum": 98.61051613092422, "correct_sum": 90.0 } ], "accuracy_vs_coverage": { "0.25": { "n": 32, "accuracy": 0.9375, "min_confidence": 0.9944236874580383 }, "0.5": { "n": 64, "accuracy": 0.9375, "min_confidence": 0.9793309569358826 }, "0.75": { "n": 96, "accuracy": 0.8958333333333334, "min_confidence": 0.9120692014694214 }, "1.0": { "n": 128, "accuracy": 0.8203125, "min_confidence": 0.5004621744155884 } } }, "snli": { "n": 128, "accuracy": 0.890625, "nll": 0.33761709568852893, "brier_multiclass_sum": 0.18059777233026203, "ece_top_label_10_equal_width_bins": 0.020881996024399996, "reliability_bins": [ { "count": 0, "confidence_sum": 0.0, "correct_sum": 0.0 }, { "count": 0, "confidence_sum": 0.0, "correct_sum": 0.0 }, { "count": 0, "confidence_sum": 0.0, "correct_sum": 0.0 }, { "count": 0, "confidence_sum": 0.0, "correct_sum": 0.0 }, { "count": 0, "confidence_sum": 0.0, "correct_sum": 0.0 }, { "count": 2, "confidence_sum": 1.1253126859664917, "correct_sum": 1.0 }, { "count": 8, "confidence_sum": 5.320736408233643, "correct_sum": 6.0 }, { "count": 7, "confidence_sum": 5.303182303905487, "correct_sum": 5.0 }, { "count": 33, "confidence_sum": 28.04604983329773, "correct_sum": 27.0 }, { "count": 78, "confidence_sum": 75.51908707618713, "correct_sum": 75.0 } ], "accuracy_vs_coverage": { "0.25": { "n": 32, "accuracy": 0.96875, "min_confidence": 0.9792487621307373 }, "0.5": { "n": 64, "accuracy": 0.96875, "min_confidence": 0.9421285390853882 }, "0.75": { "n": 96, "accuracy": 0.9479166666666666, "min_confidence": 0.847649335861206 }, "1.0": { "n": 128, "accuracy": 0.890625, "min_confidence": 0.5262982249259949 } } } } } }, { "step": 2000, "macro_nll": 0.25676974726429014, "best_macro_nll": 0.25676974726429014, "selection": { "metric": "crossfit_temperature_nll_v1", "score": 0.2552943737691716, "raw_macro_nll": 0.25676974955497256, "accuracy": 0.8984375, "folds": [ { "fold": 0, "temperature_index": 48, "temperature": 1.2705741052085413, "training_macro_nll": 0.27533710550694335, "heldout_ids": [ "arc:validation:ARC-Challenge:122", "arc:validation:ARC-Challenge:126", "arc:validation:ARC-Challenge:175", "arc:validation:ARC-Challenge:187", "arc:validation:ARC-Challenge:189", "arc:validation:ARC-Challenge:192", "arc:validation:ARC-Challenge:208", "arc:validation:ARC-Challenge:238", "arc:validation:ARC-Challenge:290", "arc:validation:ARC-Challenge:293", "arc:validation:ARC-Challenge:35", "arc:validation:ARC-Challenge:43", "arc:validation:ARC-Challenge:58", "arc:validation:ARC-Challenge:7", "arc:validation:ARC-Easy:160", "arc:validation:ARC-Easy:201", "arc:validation:ARC-Easy:229", "arc:validation:ARC-Easy:238", "arc:validation:ARC-Easy:253", "arc:validation:ARC-Easy:259", "arc:validation:ARC-Easy:272", "arc:validation:ARC-Easy:298", "arc:validation:ARC-Easy:329", "arc:validation:ARC-Easy:331", "arc:validation:ARC-Easy:337", "arc:validation:ARC-Easy:362", "arc:validation:ARC-Easy:443", "arc:validation:ARC-Easy:500", "arc:validation:ARC-Easy:545", "arc:validation:ARC-Easy:68", "arc:validation:ARC-Easy:7", "arc:validation:ARC-Easy:74", "banking:train:1053", "banking:train:1594", "banking:train:1747", "banking:train:202", "banking:train:2377", "banking:train:2528", "banking:train:2546", "banking:train:2715", "banking:train:3220", "banking:train:3499", "banking:train:3806", "banking:train:3838", "banking:train:3993", "banking:train:4220", "banking:train:5", "banking:train:5192", "banking:train:5394", "banking:train:5846", "banking:train:5863", "banking:train:6032", "banking:train:6508", "banking:train:7579", "banking:train:7600", "banking:train:7987", "banking:train:8155", "banking:train:850", "banking:train:8626", "banking:train:8929", "banking:train:9355", "banking:train:9812", "banking:train:987", "banking:train:9972", "boolq:validation:103", "boolq:validation:1042", "boolq:validation:1059", "boolq:validation:1541", "boolq:validation:1592", "boolq:validation:1927", "boolq:validation:2014", "boolq:validation:2023", "boolq:validation:2098", "boolq:validation:2139", "boolq:validation:2439", "boolq:validation:2485", "boolq:validation:2607", "boolq:validation:2611", "boolq:validation:2720", "boolq:validation:2751", "boolq:validation:282", "boolq:validation:2843", "boolq:validation:2953", "boolq:validation:301", "boolq:validation:3089", "boolq:validation:313", "boolq:validation:3201", "boolq:validation:322", "boolq:validation:3231", "boolq:validation:39", "boolq:validation:461", "boolq:validation:558", "boolq:validation:589", "boolq:validation:731", "boolq:validation:883", "boolq:validation:915", "snli:validation:1156", "snli:validation:1330", "snli:validation:1559", "snli:validation:1735", "snli:validation:1876", "snli:validation:2251", "snli:validation:2257", "snli:validation:2377", "snli:validation:2881", "snli:validation:291", "snli:validation:300", "snli:validation:3049", "snli:validation:3496", "snli:validation:3871", "snli:validation:4414", "snli:validation:5140", "snli:validation:5440", "snli:validation:5806", "snli:validation:6442", "snli:validation:6772", "snli:validation:6949", "snli:validation:7228", "snli:validation:7465", "snli:validation:7510", "snli:validation:7726", "snli:validation:8095", "snli:validation:8143", "snli:validation:8623", "snli:validation:8962", "snli:validation:9451", "snli:validation:9574", "snli:validation:9655" ], "heldout_group_count": 128, "training_group_count": 384 }, { "fold": 1, "temperature_index": 47, "temperature": 1.2050359403717974, "training_macro_nll": 0.25585027399066257, "heldout_ids": [ "arc:validation:ARC-Challenge:101", "arc:validation:ARC-Challenge:129", "arc:validation:ARC-Challenge:148", "arc:validation:ARC-Challenge:156", "arc:validation:ARC-Challenge:167", "arc:validation:ARC-Challenge:249", "arc:validation:ARC-Challenge:263", "arc:validation:ARC-Challenge:266", "arc:validation:ARC-Challenge:267", "arc:validation:ARC-Challenge:28", "arc:validation:ARC-Challenge:281", "arc:validation:ARC-Challenge:57", "arc:validation:ARC-Challenge:63", "arc:validation:ARC-Challenge:64", "arc:validation:ARC-Challenge:86", "arc:validation:ARC-Challenge:99", "arc:validation:ARC-Easy:106", "arc:validation:ARC-Easy:151", "arc:validation:ARC-Easy:157", "arc:validation:ARC-Easy:166", "arc:validation:ARC-Easy:202", "arc:validation:ARC-Easy:219", "arc:validation:ARC-Easy:224", "arc:validation:ARC-Easy:322", "arc:validation:ARC-Easy:344", "arc:validation:ARC-Easy:382", "arc:validation:ARC-Easy:409", "arc:validation:ARC-Easy:455", "arc:validation:ARC-Easy:50", "arc:validation:ARC-Easy:538", "arc:validation:ARC-Easy:544", "arc:validation:ARC-Easy:552", "banking:train:1050", "banking:train:1131", "banking:train:1190", "banking:train:124", "banking:train:137", "banking:train:1493", "banking:train:1498", "banking:train:1624", "banking:train:2614", "banking:train:2739", "banking:train:3055", "banking:train:306", "banking:train:3312", "banking:train:3724", "banking:train:4070", "banking:train:476", "banking:train:4882", "banking:train:5075", "banking:train:5332", "banking:train:5445", "banking:train:6096", "banking:train:746", "banking:train:7828", "banking:train:8176", "banking:train:8315", "banking:train:8653", "banking:train:9019", "banking:train:9370", "banking:train:9454", "banking:train:9595", "banking:train:9862", "banking:train:9963", "boolq:validation:1189", "boolq:validation:1320", "boolq:validation:1326", "boolq:validation:1381", "boolq:validation:1410", "boolq:validation:149", "boolq:validation:1513", "boolq:validation:1732", "boolq:validation:1739", "boolq:validation:1741", "boolq:validation:1814", "boolq:validation:1848", "boolq:validation:1940", "boolq:validation:2077", "boolq:validation:2091", "boolq:validation:2148", "boolq:validation:2215", "boolq:validation:2251", "boolq:validation:2283", "boolq:validation:2321", "boolq:validation:2398", "boolq:validation:253", "boolq:validation:2575", "boolq:validation:2907", "boolq:validation:2909", "boolq:validation:2978", "boolq:validation:359", "boolq:validation:394", "boolq:validation:436", "boolq:validation:579", "boolq:validation:807", "boolq:validation:949", "snli:validation:2389", "snli:validation:2485", "snli:validation:255", "snli:validation:3169", "snli:validation:333", "snli:validation:3700", "snli:validation:3709", "snli:validation:4093", "snli:validation:420", "snli:validation:4492", "snli:validation:4516", "snli:validation:4648", "snli:validation:4888", "snli:validation:5185", "snli:validation:5332", "snli:validation:5947", "snli:validation:6124", "snli:validation:660", "snli:validation:6619", "snli:validation:6898", "snli:validation:7387", "snli:validation:7492", "snli:validation:7544", "snli:validation:7996", "snli:validation:8074", "snli:validation:8554", "snli:validation:8971", "snli:validation:9163", "snli:validation:9277", "snli:validation:9490", "snli:validation:9595", "snli:validation:9688" ], "heldout_group_count": 128, "training_group_count": 384 }, { "fold": 2, "temperature_index": 45, "temperature": 1.0839269140212033, "training_macro_nll": 0.22881493804363362, "heldout_ids": [ "arc:validation:ARC-Challenge:120", "arc:validation:ARC-Challenge:130", "arc:validation:ARC-Challenge:194", "arc:validation:ARC-Challenge:218", "arc:validation:ARC-Challenge:235", "arc:validation:ARC-Challenge:272", "arc:validation:ARC-Challenge:277", "arc:validation:ARC-Challenge:48", "arc:validation:ARC-Challenge:88", "arc:validation:ARC-Easy:115", "arc:validation:ARC-Easy:12", "arc:validation:ARC-Easy:131", "arc:validation:ARC-Easy:145", "arc:validation:ARC-Easy:179", "arc:validation:ARC-Easy:18", "arc:validation:ARC-Easy:186", "arc:validation:ARC-Easy:191", "arc:validation:ARC-Easy:215", "arc:validation:ARC-Easy:283", "arc:validation:ARC-Easy:314", "arc:validation:ARC-Easy:315", "arc:validation:ARC-Easy:378", "arc:validation:ARC-Easy:384", "arc:validation:ARC-Easy:403", "arc:validation:ARC-Easy:432", "arc:validation:ARC-Easy:439", "arc:validation:ARC-Easy:448", "arc:validation:ARC-Easy:461", "arc:validation:ARC-Easy:469", "arc:validation:ARC-Easy:470", "arc:validation:ARC-Easy:504", "arc:validation:ARC-Easy:561", "banking:train:1142", "banking:train:148", "banking:train:1633", "banking:train:1927", "banking:train:2186", "banking:train:2251", "banking:train:23", "banking:train:3048", "banking:train:3665", "banking:train:3952", "banking:train:3999", "banking:train:4364", "banking:train:4619", "banking:train:5049", "banking:train:5335", "banking:train:549", "banking:train:5957", "banking:train:5985", "banking:train:6132", "banking:train:6240", "banking:train:6488", "banking:train:7065", "banking:train:7749", "banking:train:8470", "banking:train:8761", "banking:train:895", "banking:train:9256", "banking:train:9406", "banking:train:9450", "banking:train:9467", "banking:train:9588", "banking:train:9716", "boolq:validation:1130", "boolq:validation:1167", "boolq:validation:124", "boolq:validation:1255", "boolq:validation:1256", "boolq:validation:1373", "boolq:validation:15", "boolq:validation:155", "boolq:validation:1568", "boolq:validation:16", "boolq:validation:1654", "boolq:validation:1702", "boolq:validation:1792", "boolq:validation:1818", "boolq:validation:210", "boolq:validation:2247", "boolq:validation:2259", "boolq:validation:2362", "boolq:validation:2443", "boolq:validation:2524", "boolq:validation:2553", "boolq:validation:2623", "boolq:validation:2660", "boolq:validation:2669", "boolq:validation:2679", "boolq:validation:2832", "boolq:validation:357", "boolq:validation:453", "boolq:validation:49", "boolq:validation:808", "boolq:validation:90", "boolq:validation:910", "snli:validation:1612", "snli:validation:168", "snli:validation:1693", "snli:validation:2134", "snli:validation:2248", "snli:validation:2422", "snli:validation:2587", "snli:validation:3007", "snli:validation:3097", "snli:validation:3772", "snli:validation:4012", "snli:validation:4168", "snli:validation:4537", "snli:validation:4681", "snli:validation:5050", "snli:validation:5632", "snli:validation:5761", "snli:validation:6055", "snli:validation:6283", "snli:validation:6715", "snli:validation:7117", "snli:validation:7192", "snli:validation:7507", "snli:validation:7594", "snli:validation:7924", "snli:validation:7954", "snli:validation:8035", "snli:validation:8110", "snli:validation:8995", "snli:validation:949", "snli:validation:9523", "snli:validation:9589" ], "heldout_group_count": 128, "training_group_count": 384 }, { "fold": 3, "temperature_index": 47, "temperature": 1.2050359403717974, "training_macro_nll": 0.24681503020373022, "heldout_ids": [ "arc:validation:ARC-Challenge:117", "arc:validation:ARC-Challenge:135", "arc:validation:ARC-Challenge:139", "arc:validation:ARC-Challenge:154", "arc:validation:ARC-Challenge:174", "arc:validation:ARC-Challenge:199", "arc:validation:ARC-Challenge:287", "arc:validation:ARC-Challenge:295", "arc:validation:ARC-Challenge:4", "arc:validation:ARC-Challenge:50", "arc:validation:ARC-Challenge:51", "arc:validation:ARC-Challenge:61", "arc:validation:ARC-Easy:107", "arc:validation:ARC-Easy:121", "arc:validation:ARC-Easy:124", "arc:validation:ARC-Easy:171", "arc:validation:ARC-Easy:235", "arc:validation:ARC-Easy:241", "arc:validation:ARC-Easy:279", "arc:validation:ARC-Easy:289", "arc:validation:ARC-Easy:37", "arc:validation:ARC-Easy:389", "arc:validation:ARC-Easy:394", "arc:validation:ARC-Easy:399", "arc:validation:ARC-Easy:423", "arc:validation:ARC-Easy:492", "arc:validation:ARC-Easy:499", "arc:validation:ARC-Easy:523", "arc:validation:ARC-Easy:533", "arc:validation:ARC-Easy:569", "arc:validation:ARC-Easy:69", "arc:validation:ARC-Easy:93", "banking:train:2049", "banking:train:2350", "banking:train:2497", "banking:train:2506", "banking:train:2720", "banking:train:2788", "banking:train:2967", "banking:train:3502", "banking:train:3726", "banking:train:4014", "banking:train:4735", "banking:train:4803", "banking:train:4917", "banking:train:5477", "banking:train:5851", "banking:train:6551", "banking:train:6839", "banking:train:6926", "banking:train:7809", "banking:train:8092", "banking:train:8186", "banking:train:8215", "banking:train:8667", "banking:train:8752", "banking:train:8763", "banking:train:8851", "banking:train:8917", "banking:train:9145", "banking:train:9279", "banking:train:9676", "banking:train:9837", "banking:train:9967", "boolq:validation:106", "boolq:validation:1065", "boolq:validation:1097", "boolq:validation:1336", "boolq:validation:1350", "boolq:validation:1383", "boolq:validation:1458", "boolq:validation:1641", "boolq:validation:1873", "boolq:validation:1930", "boolq:validation:1989", "boolq:validation:2103", "boolq:validation:234", "boolq:validation:2355", "boolq:validation:2384", "boolq:validation:2440", "boolq:validation:2444", "boolq:validation:2478", "boolq:validation:2872", "boolq:validation:2879", "boolq:validation:2924", "boolq:validation:2971", "boolq:validation:3164", "boolq:validation:4", "boolq:validation:415", "boolq:validation:419", "boolq:validation:428", "boolq:validation:512", "boolq:validation:563", "boolq:validation:643", "boolq:validation:696", "boolq:validation:874", "snli:validation:1102", "snli:validation:1240", "snli:validation:1261", "snli:validation:1738", "snli:validation:1978", "snli:validation:2563", "snli:validation:2659", "snli:validation:2767", "snli:validation:2836", "snli:validation:3232", "snli:validation:3655", "snli:validation:3913", "snli:validation:405", "snli:validation:4051", "snli:validation:4300", "snli:validation:4846", "snli:validation:498", "snli:validation:5224", "snli:validation:5575", "snli:validation:6598", "snli:validation:666", "snli:validation:6925", "snli:validation:6964", "snli:validation:7045", "snli:validation:7351", "snli:validation:7588", "snli:validation:7723", "snli:validation:7750", "snli:validation:7909", "snli:validation:8974", "snli:validation:9385", "snli:validation:9646" ], "heldout_group_count": 128, "training_group_count": 384 } ], "per_family": { "arc": { "count": 128, "score": 0.3379051798328397, "raw_macro_nll": 0.33955257651665527, "accuracy": 0.890625 }, "banking": { "count": 128, "score": 0.06793271948328157, "raw_macro_nll": 0.06537141992655102, "accuracy": 0.9609375 }, "boolq": { "count": 128, "score": 0.33084980282335186, "raw_macro_nll": 0.34265219288051024, "accuracy": 0.84375 }, "snli": { "count": 128, "score": 0.28448979293721316, "raw_macro_nll": 0.2795028088961736, "accuracy": 0.8984375 } }, "policy": { "id": "crossfit_temperature_nll_v1", "folds": 4, "seed": 431, "grouping": "source group, nested within task family; groups shuffled within sorted families and assigned round-robin", "temperature_grid": { "count": 101, "log10_min": -1.0, "log10_max": 1.3, "arithmetic": "Python float64" }, "temperature_fit": "minimum macro-family NLL on the other three folds; smallest temperature wins ties", "score": "macro-family mean of all held-out decision NLLs", "deployment_temperature": "fit afresh on reserved calibration only after model selection" } }, "best_selection_score": 0.2552943737691716, "improved": true, "metrics": { "overall": { "n": 512, "accuracy": 0.8984375, "nll": 0.25676974726429014, "brier_multiclass_sum": 0.14451170508324393, "ece_top_label_10_equal_width_bins": 0.030249507224652916, "reliability_bins": [ { "count": 0, "confidence_sum": 0.0, "correct_sum": 0.0 }, { "count": 0, "confidence_sum": 0.0, "correct_sum": 0.0 }, { "count": 0, "confidence_sum": 0.0, "correct_sum": 0.0 }, { "count": 3, "confidence_sum": 1.0243161022663116, "correct_sum": 0.0 }, { "count": 5, "confidence_sum": 2.3768064975738525, "correct_sum": 4.0 }, { "count": 20, "confidence_sum": 10.845634400844574, "correct_sum": 10.0 }, { "count": 23, "confidence_sum": 15.07276839017868, "correct_sum": 13.0 }, { "count": 17, "confidence_sum": 12.791166067123413, "correct_sum": 12.0 }, { "count": 40, "confidence_sum": 34.14968603849411, "correct_sum": 30.0 }, { "count": 404, "confidence_sum": 395.98098319768906, "correct_sum": 391.0 } ], "accuracy_vs_coverage": { "0.25": { "n": 128, "accuracy": 1.0, "min_confidence": 0.9993056058883667 }, "0.5": { "n": 256, "accuracy": 1.0, "min_confidence": 0.9854145050048828 }, "0.75": { "n": 384, "accuracy": 0.9739583333333334, "min_confidence": 0.9169918894767761 }, "1.0": { "n": 512, "accuracy": 0.8984375, "min_confidence": 0.3054785430431366 } } }, "per_family": { "arc": { "n": 128, "accuracy": 0.890625, "nll": 0.33955256911365517, "brier_multiclass_sum": 0.16622784887383055, "ece_top_label_10_equal_width_bins": 0.059738497948274016, "reliability_bins": [ { "count": 0, "confidence_sum": 0.0, "correct_sum": 0.0 }, { "count": 0, "confidence_sum": 0.0, "correct_sum": 0.0 }, { "count": 0, "confidence_sum": 0.0, "correct_sum": 0.0 }, { "count": 3, "confidence_sum": 1.0243161022663116, "correct_sum": 0.0 }, { "count": 2, "confidence_sum": 0.9794403314590454, "correct_sum": 2.0 }, { "count": 6, "confidence_sum": 3.2076597809791565, "correct_sum": 4.0 }, { "count": 6, "confidence_sum": 3.9651999473571777, "correct_sum": 5.0 }, { "count": 5, "confidence_sum": 3.7508853673934937, "correct_sum": 3.0 }, { "count": 8, "confidence_sum": 6.78725129365921, "correct_sum": 5.0 }, { "count": 98, "confidence_sum": 96.23637503385544, "correct_sum": 95.0 } ], "accuracy_vs_coverage": { "0.25": { "n": 32, "accuracy": 1.0, "min_confidence": 0.9996716976165771 }, "0.5": { "n": 64, "accuracy": 1.0, "min_confidence": 0.9919483661651611 }, "0.75": { "n": 96, "accuracy": 0.96875, "min_confidence": 0.9052539467811584 }, "1.0": { "n": 128, "accuracy": 0.890625, "min_confidence": 0.3054785430431366 } } }, "banking": { "n": 128, "accuracy": 0.9609375, "nll": 0.06537142041187444, "brier_multiclass_sum": 0.04192916462498076, "ece_top_label_10_equal_width_bins": 0.02170603978447616, "reliability_bins": [ { "count": 0, "confidence_sum": 0.0, "correct_sum": 0.0 }, { "count": 0, "confidence_sum": 0.0, "correct_sum": 0.0 }, { "count": 0, "confidence_sum": 0.0, "correct_sum": 0.0 }, { "count": 0, "confidence_sum": 0.0, "correct_sum": 0.0 }, { "count": 1, "confidence_sum": 0.48401013016700745, "correct_sum": 0.0 }, { "count": 2, "confidence_sum": 1.0103490948677063, "correct_sum": 1.0 }, { "count": 1, "confidence_sum": 0.6197635531425476, "correct_sum": 0.0 }, { "count": 4, "confidence_sum": 3.016280710697174, "correct_sum": 2.0 }, { "count": 1, "confidence_sum": 0.8994449973106384, "correct_sum": 1.0 }, { "count": 119, "confidence_sum": 118.45258539915085, "correct_sum": 119.0 } ], "accuracy_vs_coverage": { "0.25": { "n": 32, "accuracy": 1.0, "min_confidence": 0.9999996423721313 }, "0.5": { "n": 64, "accuracy": 1.0, "min_confidence": 0.9999728202819824 }, "0.75": { "n": 96, "accuracy": 1.0, "min_confidence": 0.9991067051887512 }, "1.0": { "n": 128, "accuracy": 0.9609375, "min_confidence": 0.48401013016700745 } } }, "boolq": { "n": 128, "accuracy": 0.84375, "nll": 0.342652197339703, "brier_multiclass_sum": 0.21037350762899634, "ece_top_label_10_equal_width_bins": 0.06018189946189523, "reliability_bins": [ { "count": 0, "confidence_sum": 0.0, "correct_sum": 0.0 }, { "count": 0, "confidence_sum": 0.0, "correct_sum": 0.0 }, { "count": 0, "confidence_sum": 0.0, "correct_sum": 0.0 }, { "count": 0, "confidence_sum": 0.0, "correct_sum": 0.0 }, { "count": 0, "confidence_sum": 0.0, "correct_sum": 0.0 }, { "count": 8, "confidence_sum": 4.464165925979614, "correct_sum": 3.0 }, { "count": 13, "confidence_sum": 8.543370068073273, "correct_sum": 7.0 }, { "count": 3, "confidence_sum": 2.237887918949127, "correct_sum": 3.0 }, { "count": 18, "confidence_sum": 15.334045886993408, "correct_sum": 14.0 }, { "count": 86, "confidence_sum": 83.59958916902542, "correct_sum": 81.0 } ], "accuracy_vs_coverage": { "0.25": { "n": 32, "accuracy": 1.0, "min_confidence": 0.9892238974571228 }, "0.5": { "n": 64, "accuracy": 0.953125, "min_confidence": 0.9593857526779175 }, "0.75": { "n": 96, "accuracy": 0.9166666666666666, "min_confidence": 0.8476907014846802 }, "1.0": { "n": 128, "accuracy": 0.84375, "min_confidence": 0.5223926901817322 } } }, "snli": { "n": 128, "accuracy": 0.8984375, "nll": 0.2795028021919279, "brier_multiclass_sum": 0.1595162992051681, "ece_top_label_10_equal_width_bins": 0.04085784195922315, "reliability_bins": [ { "count": 0, "confidence_sum": 0.0, "correct_sum": 0.0 }, { "count": 0, "confidence_sum": 0.0, "correct_sum": 0.0 }, { "count": 0, "confidence_sum": 0.0, "correct_sum": 0.0 }, { "count": 0, "confidence_sum": 0.0, "correct_sum": 0.0 }, { "count": 2, "confidence_sum": 0.9133560359477997, "correct_sum": 2.0 }, { "count": 4, "confidence_sum": 2.163459599018097, "correct_sum": 2.0 }, { "count": 3, "confidence_sum": 1.9444348216056824, "correct_sum": 1.0 }, { "count": 5, "confidence_sum": 3.786112070083618, "correct_sum": 4.0 }, { "count": 13, "confidence_sum": 11.128943860530853, "correct_sum": 10.0 }, { "count": 101, "confidence_sum": 97.69243359565735, "correct_sum": 96.0 } ], "accuracy_vs_coverage": { "0.25": { "n": 32, "accuracy": 1.0, "min_confidence": 0.9896841049194336 }, "0.5": { "n": 64, "accuracy": 0.984375, "min_confidence": 0.9620543718338013 }, "0.75": { "n": 96, "accuracy": 0.9479166666666666, "min_confidence": 0.9089819192886353 }, "1.0": { "n": 128, "accuracy": 0.8984375, "min_confidence": 0.43042799830436707 } } } } } }, { "step": 2500, "macro_nll": 0.34164803258559817, "best_macro_nll": 0.25676974726429014, "selection": { "metric": "crossfit_temperature_nll_v1", "score": 0.2873950432295314, "raw_macro_nll": 0.3416480337510342, "accuracy": 0.908203125, "folds": [ { "fold": 0, "temperature_index": 54, "temperature": 1.7458221529205038, "training_macro_nll": 0.2923693492865519, "heldout_ids": [ "arc:validation:ARC-Challenge:122", "arc:validation:ARC-Challenge:126", "arc:validation:ARC-Challenge:175", "arc:validation:ARC-Challenge:187", "arc:validation:ARC-Challenge:189", "arc:validation:ARC-Challenge:192", "arc:validation:ARC-Challenge:208", "arc:validation:ARC-Challenge:238", "arc:validation:ARC-Challenge:290", "arc:validation:ARC-Challenge:293", "arc:validation:ARC-Challenge:35", "arc:validation:ARC-Challenge:43", "arc:validation:ARC-Challenge:58", "arc:validation:ARC-Challenge:7", "arc:validation:ARC-Easy:160", "arc:validation:ARC-Easy:201", "arc:validation:ARC-Easy:229", "arc:validation:ARC-Easy:238", "arc:validation:ARC-Easy:253", "arc:validation:ARC-Easy:259", "arc:validation:ARC-Easy:272", "arc:validation:ARC-Easy:298", "arc:validation:ARC-Easy:329", "arc:validation:ARC-Easy:331", "arc:validation:ARC-Easy:337", "arc:validation:ARC-Easy:362", "arc:validation:ARC-Easy:443", "arc:validation:ARC-Easy:500", "arc:validation:ARC-Easy:545", "arc:validation:ARC-Easy:68", "arc:validation:ARC-Easy:7", "arc:validation:ARC-Easy:74", "banking:train:1053", "banking:train:1594", "banking:train:1747", "banking:train:202", "banking:train:2377", "banking:train:2528", "banking:train:2546", "banking:train:2715", "banking:train:3220", "banking:train:3499", "banking:train:3806", "banking:train:3838", "banking:train:3993", "banking:train:4220", "banking:train:5", "banking:train:5192", "banking:train:5394", "banking:train:5846", "banking:train:5863", "banking:train:6032", "banking:train:6508", "banking:train:7579", "banking:train:7600", "banking:train:7987", "banking:train:8155", "banking:train:850", "banking:train:8626", "banking:train:8929", "banking:train:9355", "banking:train:9812", "banking:train:987", "banking:train:9972", "boolq:validation:103", "boolq:validation:1042", "boolq:validation:1059", "boolq:validation:1541", "boolq:validation:1592", "boolq:validation:1927", "boolq:validation:2014", "boolq:validation:2023", "boolq:validation:2098", "boolq:validation:2139", "boolq:validation:2439", "boolq:validation:2485", "boolq:validation:2607", "boolq:validation:2611", "boolq:validation:2720", "boolq:validation:2751", "boolq:validation:282", "boolq:validation:2843", "boolq:validation:2953", "boolq:validation:301", "boolq:validation:3089", "boolq:validation:313", "boolq:validation:3201", "boolq:validation:322", "boolq:validation:3231", "boolq:validation:39", "boolq:validation:461", "boolq:validation:558", "boolq:validation:589", "boolq:validation:731", "boolq:validation:883", "boolq:validation:915", "snli:validation:1156", "snli:validation:1330", "snli:validation:1559", "snli:validation:1735", "snli:validation:1876", "snli:validation:2251", "snli:validation:2257", "snli:validation:2377", "snli:validation:2881", "snli:validation:291", "snli:validation:300", "snli:validation:3049", "snli:validation:3496", "snli:validation:3871", "snli:validation:4414", "snli:validation:5140", "snli:validation:5440", "snli:validation:5806", "snli:validation:6442", "snli:validation:6772", "snli:validation:6949", "snli:validation:7228", "snli:validation:7465", "snli:validation:7510", "snli:validation:7726", "snli:validation:8095", "snli:validation:8143", "snli:validation:8623", "snli:validation:8962", "snli:validation:9451", "snli:validation:9574", "snli:validation:9655" ], "heldout_group_count": 128, "training_group_count": 384 }, { "fold": 1, "temperature_index": 54, "temperature": 1.7458221529205038, "training_macro_nll": 0.29119915448817607, "heldout_ids": [ "arc:validation:ARC-Challenge:101", "arc:validation:ARC-Challenge:129", "arc:validation:ARC-Challenge:148", "arc:validation:ARC-Challenge:156", "arc:validation:ARC-Challenge:167", "arc:validation:ARC-Challenge:249", "arc:validation:ARC-Challenge:263", "arc:validation:ARC-Challenge:266", "arc:validation:ARC-Challenge:267", "arc:validation:ARC-Challenge:28", "arc:validation:ARC-Challenge:281", "arc:validation:ARC-Challenge:57", "arc:validation:ARC-Challenge:63", "arc:validation:ARC-Challenge:64", "arc:validation:ARC-Challenge:86", "arc:validation:ARC-Challenge:99", "arc:validation:ARC-Easy:106", "arc:validation:ARC-Easy:151", "arc:validation:ARC-Easy:157", "arc:validation:ARC-Easy:166", "arc:validation:ARC-Easy:202", "arc:validation:ARC-Easy:219", "arc:validation:ARC-Easy:224", "arc:validation:ARC-Easy:322", "arc:validation:ARC-Easy:344", "arc:validation:ARC-Easy:382", "arc:validation:ARC-Easy:409", "arc:validation:ARC-Easy:455", "arc:validation:ARC-Easy:50", "arc:validation:ARC-Easy:538", "arc:validation:ARC-Easy:544", "arc:validation:ARC-Easy:552", "banking:train:1050", "banking:train:1131", "banking:train:1190", "banking:train:124", "banking:train:137", "banking:train:1493", "banking:train:1498", "banking:train:1624", "banking:train:2614", "banking:train:2739", "banking:train:3055", "banking:train:306", "banking:train:3312", "banking:train:3724", "banking:train:4070", "banking:train:476", "banking:train:4882", "banking:train:5075", "banking:train:5332", "banking:train:5445", "banking:train:6096", "banking:train:746", "banking:train:7828", "banking:train:8176", "banking:train:8315", "banking:train:8653", "banking:train:9019", "banking:train:9370", "banking:train:9454", "banking:train:9595", "banking:train:9862", "banking:train:9963", "boolq:validation:1189", "boolq:validation:1320", "boolq:validation:1326", "boolq:validation:1381", "boolq:validation:1410", "boolq:validation:149", "boolq:validation:1513", "boolq:validation:1732", "boolq:validation:1739", "boolq:validation:1741", "boolq:validation:1814", "boolq:validation:1848", "boolq:validation:1940", "boolq:validation:2077", "boolq:validation:2091", "boolq:validation:2148", "boolq:validation:2215", "boolq:validation:2251", "boolq:validation:2283", "boolq:validation:2321", "boolq:validation:2398", "boolq:validation:253", "boolq:validation:2575", "boolq:validation:2907", "boolq:validation:2909", "boolq:validation:2978", "boolq:validation:359", "boolq:validation:394", "boolq:validation:436", "boolq:validation:579", "boolq:validation:807", "boolq:validation:949", "snli:validation:2389", "snli:validation:2485", "snli:validation:255", "snli:validation:3169", "snli:validation:333", "snli:validation:3700", "snli:validation:3709", "snli:validation:4093", "snli:validation:420", "snli:validation:4492", "snli:validation:4516", "snli:validation:4648", "snli:validation:4888", "snli:validation:5185", "snli:validation:5332", "snli:validation:5947", "snli:validation:6124", "snli:validation:660", "snli:validation:6619", "snli:validation:6898", "snli:validation:7387", "snli:validation:7492", "snli:validation:7544", "snli:validation:7996", "snli:validation:8074", "snli:validation:8554", "snli:validation:8971", "snli:validation:9163", "snli:validation:9277", "snli:validation:9490", "snli:validation:9595", "snli:validation:9688" ], "heldout_group_count": 128, "training_group_count": 384 }, { "fold": 2, "temperature_index": 54, "temperature": 1.7458221529205038, "training_macro_nll": 0.27996013831109845, "heldout_ids": [ "arc:validation:ARC-Challenge:120", "arc:validation:ARC-Challenge:130", "arc:validation:ARC-Challenge:194", "arc:validation:ARC-Challenge:218", "arc:validation:ARC-Challenge:235", "arc:validation:ARC-Challenge:272", "arc:validation:ARC-Challenge:277", "arc:validation:ARC-Challenge:48", "arc:validation:ARC-Challenge:88", "arc:validation:ARC-Easy:115", "arc:validation:ARC-Easy:12", "arc:validation:ARC-Easy:131", "arc:validation:ARC-Easy:145", "arc:validation:ARC-Easy:179", "arc:validation:ARC-Easy:18", "arc:validation:ARC-Easy:186", "arc:validation:ARC-Easy:191", "arc:validation:ARC-Easy:215", "arc:validation:ARC-Easy:283", "arc:validation:ARC-Easy:314", "arc:validation:ARC-Easy:315", "arc:validation:ARC-Easy:378", "arc:validation:ARC-Easy:384", "arc:validation:ARC-Easy:403", "arc:validation:ARC-Easy:432", "arc:validation:ARC-Easy:439", "arc:validation:ARC-Easy:448", "arc:validation:ARC-Easy:461", "arc:validation:ARC-Easy:469", "arc:validation:ARC-Easy:470", "arc:validation:ARC-Easy:504", "arc:validation:ARC-Easy:561", "banking:train:1142", "banking:train:148", "banking:train:1633", "banking:train:1927", "banking:train:2186", "banking:train:2251", "banking:train:23", "banking:train:3048", "banking:train:3665", "banking:train:3952", "banking:train:3999", "banking:train:4364", "banking:train:4619", "banking:train:5049", "banking:train:5335", "banking:train:549", "banking:train:5957", "banking:train:5985", "banking:train:6132", "banking:train:6240", "banking:train:6488", "banking:train:7065", "banking:train:7749", "banking:train:8470", "banking:train:8761", "banking:train:895", "banking:train:9256", "banking:train:9406", "banking:train:9450", "banking:train:9467", "banking:train:9588", "banking:train:9716", "boolq:validation:1130", "boolq:validation:1167", "boolq:validation:124", "boolq:validation:1255", "boolq:validation:1256", "boolq:validation:1373", "boolq:validation:15", "boolq:validation:155", "boolq:validation:1568", "boolq:validation:16", "boolq:validation:1654", "boolq:validation:1702", "boolq:validation:1792", "boolq:validation:1818", "boolq:validation:210", "boolq:validation:2247", "boolq:validation:2259", "boolq:validation:2362", "boolq:validation:2443", "boolq:validation:2524", "boolq:validation:2553", "boolq:validation:2623", "boolq:validation:2660", "boolq:validation:2669", "boolq:validation:2679", "boolq:validation:2832", "boolq:validation:357", "boolq:validation:453", "boolq:validation:49", "boolq:validation:808", "boolq:validation:90", "boolq:validation:910", "snli:validation:1612", "snli:validation:168", "snli:validation:1693", "snli:validation:2134", "snli:validation:2248", "snli:validation:2422", "snli:validation:2587", "snli:validation:3007", "snli:validation:3097", "snli:validation:3772", "snli:validation:4012", "snli:validation:4168", "snli:validation:4537", "snli:validation:4681", "snli:validation:5050", "snli:validation:5632", "snli:validation:5761", "snli:validation:6055", "snli:validation:6283", "snli:validation:6715", "snli:validation:7117", "snli:validation:7192", "snli:validation:7507", "snli:validation:7594", "snli:validation:7924", "snli:validation:7954", "snli:validation:8035", "snli:validation:8110", "snli:validation:8995", "snli:validation:949", "snli:validation:9523", "snli:validation:9589" ], "heldout_group_count": 128, "training_group_count": 384 }, { "fold": 3, "temperature_index": 53, "temperature": 1.6557699634695275, "training_macro_nll": 0.2844243480235682, "heldout_ids": [ "arc:validation:ARC-Challenge:117", "arc:validation:ARC-Challenge:135", "arc:validation:ARC-Challenge:139", "arc:validation:ARC-Challenge:154", "arc:validation:ARC-Challenge:174", "arc:validation:ARC-Challenge:199", "arc:validation:ARC-Challenge:287", "arc:validation:ARC-Challenge:295", "arc:validation:ARC-Challenge:4", "arc:validation:ARC-Challenge:50", "arc:validation:ARC-Challenge:51", "arc:validation:ARC-Challenge:61", "arc:validation:ARC-Easy:107", "arc:validation:ARC-Easy:121", "arc:validation:ARC-Easy:124", "arc:validation:ARC-Easy:171", "arc:validation:ARC-Easy:235", "arc:validation:ARC-Easy:241", "arc:validation:ARC-Easy:279", "arc:validation:ARC-Easy:289", "arc:validation:ARC-Easy:37", "arc:validation:ARC-Easy:389", "arc:validation:ARC-Easy:394", "arc:validation:ARC-Easy:399", "arc:validation:ARC-Easy:423", "arc:validation:ARC-Easy:492", "arc:validation:ARC-Easy:499", "arc:validation:ARC-Easy:523", "arc:validation:ARC-Easy:533", "arc:validation:ARC-Easy:569", "arc:validation:ARC-Easy:69", "arc:validation:ARC-Easy:93", "banking:train:2049", "banking:train:2350", "banking:train:2497", "banking:train:2506", "banking:train:2720", "banking:train:2788", "banking:train:2967", "banking:train:3502", "banking:train:3726", "banking:train:4014", "banking:train:4735", "banking:train:4803", "banking:train:4917", "banking:train:5477", "banking:train:5851", "banking:train:6551", "banking:train:6839", "banking:train:6926", "banking:train:7809", "banking:train:8092", "banking:train:8186", "banking:train:8215", "banking:train:8667", "banking:train:8752", "banking:train:8763", "banking:train:8851", "banking:train:8917", "banking:train:9145", "banking:train:9279", "banking:train:9676", "banking:train:9837", "banking:train:9967", "boolq:validation:106", "boolq:validation:1065", "boolq:validation:1097", "boolq:validation:1336", "boolq:validation:1350", "boolq:validation:1383", "boolq:validation:1458", "boolq:validation:1641", "boolq:validation:1873", "boolq:validation:1930", "boolq:validation:1989", "boolq:validation:2103", "boolq:validation:234", "boolq:validation:2355", "boolq:validation:2384", "boolq:validation:2440", "boolq:validation:2444", "boolq:validation:2478", "boolq:validation:2872", "boolq:validation:2879", "boolq:validation:2924", "boolq:validation:2971", "boolq:validation:3164", "boolq:validation:4", "boolq:validation:415", "boolq:validation:419", "boolq:validation:428", "boolq:validation:512", "boolq:validation:563", "boolq:validation:643", "boolq:validation:696", "boolq:validation:874", "snli:validation:1102", "snli:validation:1240", "snli:validation:1261", "snli:validation:1738", "snli:validation:1978", "snli:validation:2563", "snli:validation:2659", "snli:validation:2767", "snli:validation:2836", "snli:validation:3232", "snli:validation:3655", "snli:validation:3913", "snli:validation:405", "snli:validation:4051", "snli:validation:4300", "snli:validation:4846", "snli:validation:498", "snli:validation:5224", "snli:validation:5575", "snli:validation:6598", "snli:validation:666", "snli:validation:6925", "snli:validation:6964", "snli:validation:7045", "snli:validation:7351", "snli:validation:7588", "snli:validation:7723", "snli:validation:7750", "snli:validation:7909", "snli:validation:8974", "snli:validation:9385", "snli:validation:9646" ], "heldout_group_count": 128, "training_group_count": 384 } ], "per_family": { "arc": { "count": 128, "score": 0.37032308473279407, "raw_macro_nll": 0.4008364961392736, "accuracy": 0.890625 }, "banking": { "count": 128, "score": 0.08469598977961221, "raw_macro_nll": 0.08458895137039663, "accuracy": 0.9765625 }, "boolq": { "count": 128, "score": 0.4290453198465801, "raw_macro_nll": 0.6039625253530503, "accuracy": 0.8359375 }, "snli": { "count": 128, "score": 0.2655157785591392, "raw_macro_nll": 0.27720416214141624, "accuracy": 0.9296875 } }, "policy": { "id": "crossfit_temperature_nll_v1", "folds": 4, "seed": 431, "grouping": "source group, nested within task family; groups shuffled within sorted families and assigned round-robin", "temperature_grid": { "count": 101, "log10_min": -1.0, "log10_max": 1.3, "arithmetic": "Python float64" }, "temperature_fit": "minimum macro-family NLL on the other three folds; smallest temperature wins ties", "score": "macro-family mean of all held-out decision NLLs", "deployment_temperature": "fit afresh on reserved calibration only after model selection" } }, "best_selection_score": 0.2552943737691716, "improved": false, "metrics": { "overall": { "n": 512, "accuracy": 0.908203125, "nll": 0.34164803258559817, "brier_multiclass_sum": 0.15958235995790504, "ece_top_label_10_equal_width_bins": 0.06509242358151823, "reliability_bins": [ { "count": 0, "confidence_sum": 0.0, "correct_sum": 0.0 }, { "count": 0, "confidence_sum": 0.0, "correct_sum": 0.0 }, { "count": 0, "confidence_sum": 0.0, "correct_sum": 0.0 }, { "count": 4, "confidence_sum": 1.4218776524066925, "correct_sum": 1.0 }, { "count": 7, "confidence_sum": 3.1050805151462555, "correct_sum": 5.0 }, { "count": 8, "confidence_sum": 4.550839066505432, "correct_sum": 8.0 }, { "count": 13, "confidence_sum": 8.378304839134216, "correct_sum": 7.0 }, { "count": 15, "confidence_sum": 11.172361552715302, "correct_sum": 12.0 }, { "count": 18, "confidence_sum": 15.334031105041504, "correct_sum": 11.0 }, { "count": 447, "confidence_sum": 442.0213884115219, "correct_sum": 421.0 } ], "accuracy_vs_coverage": { "0.25": { "n": 128, "accuracy": 1.0, "min_confidence": 0.9997068047523499 }, "0.5": { "n": 256, "accuracy": 0.984375, "min_confidence": 0.9974879026412964 }, "0.75": { "n": 384, "accuracy": 0.96875, "min_confidence": 0.9728125333786011 }, "1.0": { "n": 512, "accuracy": 0.908203125, "min_confidence": 0.32846468687057495 } } }, "per_family": { "arc": { "n": 128, "accuracy": 0.890625, "nll": 0.4008364924734096, "brier_multiclass_sum": 0.1836061792924404, "ece_top_label_10_equal_width_bins": 0.07639543549157679, "reliability_bins": [ { "count": 0, "confidence_sum": 0.0, "correct_sum": 0.0 }, { "count": 0, "confidence_sum": 0.0, "correct_sum": 0.0 }, { "count": 0, "confidence_sum": 0.0, "correct_sum": 0.0 }, { "count": 3, "confidence_sum": 1.0558361113071442, "correct_sum": 1.0 }, { "count": 6, "confidence_sum": 2.6552467942237854, "correct_sum": 4.0 }, { "count": 3, "confidence_sum": 1.7100982666015625, "correct_sum": 3.0 }, { "count": 4, "confidence_sum": 2.608660101890564, "correct_sum": 2.0 }, { "count": 3, "confidence_sum": 2.2052063941955566, "correct_sum": 3.0 }, { "count": 6, "confidence_sum": 5.102949798107147, "correct_sum": 4.0 }, { "count": 103, "confidence_sum": 101.58172118663788, "correct_sum": 97.0 } ], "accuracy_vs_coverage": { "0.25": { "n": 32, "accuracy": 1.0, "min_confidence": 0.9998447895050049 }, "0.5": { "n": 64, "accuracy": 1.0, "min_confidence": 0.9945265650749207 }, "0.75": { "n": 96, "accuracy": 0.96875, "min_confidence": 0.9348756074905396 }, "1.0": { "n": 128, "accuracy": 0.890625, "min_confidence": 0.32846468687057495 } } }, "banking": { "n": 128, "accuracy": 0.9765625, "nll": 0.08458894538692885, "brier_multiclass_sum": 0.03869071261468688, "ece_top_label_10_equal_width_bins": 0.01209510350599885, "reliability_bins": [ { "count": 0, "confidence_sum": 0.0, "correct_sum": 0.0 }, { "count": 0, "confidence_sum": 0.0, "correct_sum": 0.0 }, { "count": 0, "confidence_sum": 0.0, "correct_sum": 0.0 }, { "count": 0, "confidence_sum": 0.0, "correct_sum": 0.0 }, { "count": 0, "confidence_sum": 0.0, "correct_sum": 0.0 }, { "count": 0, "confidence_sum": 0.0, "correct_sum": 0.0 }, { "count": 5, "confidence_sum": 3.2286646962165833, "correct_sum": 3.0 }, { "count": 3, "confidence_sum": 2.218995213508606, "correct_sum": 3.0 }, { "count": 0, "confidence_sum": 0.0, "correct_sum": 0.0 }, { "count": 120, "confidence_sum": 119.53850376605988, "correct_sum": 119.0 } ], "accuracy_vs_coverage": { "0.25": { "n": 32, "accuracy": 1.0, "min_confidence": 0.9999996423721313 }, "0.5": { "n": 64, "accuracy": 1.0, "min_confidence": 0.9999823570251465 }, "0.75": { "n": 96, "accuracy": 1.0, "min_confidence": 0.9979609251022339 }, "1.0": { "n": 128, "accuracy": 0.9765625, "min_confidence": 0.6143197417259216 } } }, "boolq": { "n": 128, "accuracy": 0.8359375, "nll": 0.6039625278771723, "brier_multiclass_sum": 0.2873108505217812, "ece_top_label_10_equal_width_bins": 0.1287803715094924, "reliability_bins": [ { "count": 0, "confidence_sum": 0.0, "correct_sum": 0.0 }, { "count": 0, "confidence_sum": 0.0, "correct_sum": 0.0 }, { "count": 0, "confidence_sum": 0.0, "correct_sum": 0.0 }, { "count": 0, "confidence_sum": 0.0, "correct_sum": 0.0 }, { "count": 0, "confidence_sum": 0.0, "correct_sum": 0.0 }, { "count": 2, "confidence_sum": 1.103356122970581, "correct_sum": 2.0 }, { "count": 4, "confidence_sum": 2.540980041027069, "correct_sum": 2.0 }, { "count": 7, "confidence_sum": 5.245690405368805, "correct_sum": 5.0 }, { "count": 6, "confidence_sum": 5.077808380126953, "correct_sum": 2.0 }, { "count": 109, "confidence_sum": 107.72276484966278, "correct_sum": 96.0 } ], "accuracy_vs_coverage": { "0.25": { "n": 32, "accuracy": 1.0, "min_confidence": 0.9992809891700745 }, "0.5": { "n": 64, "accuracy": 0.9375, "min_confidence": 0.9971458315849304 }, "0.75": { "n": 96, "accuracy": 0.90625, "min_confidence": 0.9584174156188965 }, "1.0": { "n": 128, "accuracy": 0.8359375, "min_confidence": 0.5258947014808655 } } }, "snli": { "n": 128, "accuracy": 0.9296875, "nll": 0.27720416460488195, "brier_multiclass_sum": 0.12872169740271175, "ece_top_label_10_equal_width_bins": 0.05478878295980394, "reliability_bins": [ { "count": 0, "confidence_sum": 0.0, "correct_sum": 0.0 }, { "count": 0, "confidence_sum": 0.0, "correct_sum": 0.0 }, { "count": 0, "confidence_sum": 0.0, "correct_sum": 0.0 }, { "count": 1, "confidence_sum": 0.36604154109954834, "correct_sum": 0.0 }, { "count": 1, "confidence_sum": 0.4498337209224701, "correct_sum": 1.0 }, { "count": 3, "confidence_sum": 1.7373846769332886, "correct_sum": 3.0 }, { "count": 0, "confidence_sum": 0.0, "correct_sum": 0.0 }, { "count": 2, "confidence_sum": 1.502469539642334, "correct_sum": 1.0 }, { "count": 6, "confidence_sum": 5.153272926807404, "correct_sum": 5.0 }, { "count": 115, "confidence_sum": 113.17839860916138, "correct_sum": 109.0 } ], "accuracy_vs_coverage": { "0.25": { "n": 32, "accuracy": 1.0, "min_confidence": 0.9986096620559692 }, "0.5": { "n": 64, "accuracy": 0.96875, "min_confidence": 0.9895986318588257 }, "0.75": { "n": 96, "accuracy": 0.96875, "min_confidence": 0.9699889421463013 }, "1.0": { "n": 128, "accuracy": 0.9296875, "min_confidence": 0.36604154109954834 } } } } } }, { "step": 3000, "macro_nll": 0.343850982809214, "best_macro_nll": 0.25676974726429014, "selection": { "metric": "crossfit_temperature_nll_v1", "score": 0.26419258441127697, "raw_macro_nll": 0.3438509819060138, "accuracy": 0.91015625, "folds": [ { "fold": 0, "temperature_index": 57, "temperature": 2.046444636724674, "training_macro_nll": 0.27731212139717715, "heldout_ids": [ "arc:validation:ARC-Challenge:122", "arc:validation:ARC-Challenge:126", "arc:validation:ARC-Challenge:175", "arc:validation:ARC-Challenge:187", "arc:validation:ARC-Challenge:189", "arc:validation:ARC-Challenge:192", "arc:validation:ARC-Challenge:208", "arc:validation:ARC-Challenge:238", "arc:validation:ARC-Challenge:290", "arc:validation:ARC-Challenge:293", "arc:validation:ARC-Challenge:35", "arc:validation:ARC-Challenge:43", "arc:validation:ARC-Challenge:58", "arc:validation:ARC-Challenge:7", "arc:validation:ARC-Easy:160", "arc:validation:ARC-Easy:201", "arc:validation:ARC-Easy:229", "arc:validation:ARC-Easy:238", "arc:validation:ARC-Easy:253", "arc:validation:ARC-Easy:259", "arc:validation:ARC-Easy:272", "arc:validation:ARC-Easy:298", "arc:validation:ARC-Easy:329", "arc:validation:ARC-Easy:331", "arc:validation:ARC-Easy:337", "arc:validation:ARC-Easy:362", "arc:validation:ARC-Easy:443", "arc:validation:ARC-Easy:500", "arc:validation:ARC-Easy:545", "arc:validation:ARC-Easy:68", "arc:validation:ARC-Easy:7", "arc:validation:ARC-Easy:74", "banking:train:1053", "banking:train:1594", "banking:train:1747", "banking:train:202", "banking:train:2377", "banking:train:2528", "banking:train:2546", "banking:train:2715", "banking:train:3220", "banking:train:3499", "banking:train:3806", "banking:train:3838", "banking:train:3993", "banking:train:4220", "banking:train:5", "banking:train:5192", "banking:train:5394", "banking:train:5846", "banking:train:5863", "banking:train:6032", "banking:train:6508", "banking:train:7579", "banking:train:7600", "banking:train:7987", "banking:train:8155", "banking:train:850", "banking:train:8626", "banking:train:8929", "banking:train:9355", "banking:train:9812", "banking:train:987", "banking:train:9972", "boolq:validation:103", "boolq:validation:1042", "boolq:validation:1059", "boolq:validation:1541", "boolq:validation:1592", "boolq:validation:1927", "boolq:validation:2014", "boolq:validation:2023", "boolq:validation:2098", "boolq:validation:2139", "boolq:validation:2439", "boolq:validation:2485", "boolq:validation:2607", "boolq:validation:2611", "boolq:validation:2720", "boolq:validation:2751", "boolq:validation:282", "boolq:validation:2843", "boolq:validation:2953", "boolq:validation:301", "boolq:validation:3089", "boolq:validation:313", "boolq:validation:3201", "boolq:validation:322", "boolq:validation:3231", "boolq:validation:39", "boolq:validation:461", "boolq:validation:558", "boolq:validation:589", "boolq:validation:731", "boolq:validation:883", "boolq:validation:915", "snli:validation:1156", "snli:validation:1330", "snli:validation:1559", "snli:validation:1735", "snli:validation:1876", "snli:validation:2251", "snli:validation:2257", "snli:validation:2377", "snli:validation:2881", "snli:validation:291", "snli:validation:300", "snli:validation:3049", "snli:validation:3496", "snli:validation:3871", "snli:validation:4414", "snli:validation:5140", "snli:validation:5440", "snli:validation:5806", "snli:validation:6442", "snli:validation:6772", "snli:validation:6949", "snli:validation:7228", "snli:validation:7465", "snli:validation:7510", "snli:validation:7726", "snli:validation:8095", "snli:validation:8143", "snli:validation:8623", "snli:validation:8962", "snli:validation:9451", "snli:validation:9574", "snli:validation:9655" ], "heldout_group_count": 128, "training_group_count": 384 }, { "fold": 1, "temperature_index": 56, "temperature": 1.9408858775927773, "training_macro_nll": 0.25991780083545074, "heldout_ids": [ "arc:validation:ARC-Challenge:101", "arc:validation:ARC-Challenge:129", "arc:validation:ARC-Challenge:148", "arc:validation:ARC-Challenge:156", "arc:validation:ARC-Challenge:167", "arc:validation:ARC-Challenge:249", "arc:validation:ARC-Challenge:263", "arc:validation:ARC-Challenge:266", "arc:validation:ARC-Challenge:267", "arc:validation:ARC-Challenge:28", "arc:validation:ARC-Challenge:281", "arc:validation:ARC-Challenge:57", "arc:validation:ARC-Challenge:63", "arc:validation:ARC-Challenge:64", "arc:validation:ARC-Challenge:86", "arc:validation:ARC-Challenge:99", "arc:validation:ARC-Easy:106", "arc:validation:ARC-Easy:151", "arc:validation:ARC-Easy:157", "arc:validation:ARC-Easy:166", "arc:validation:ARC-Easy:202", "arc:validation:ARC-Easy:219", "arc:validation:ARC-Easy:224", "arc:validation:ARC-Easy:322", "arc:validation:ARC-Easy:344", "arc:validation:ARC-Easy:382", "arc:validation:ARC-Easy:409", "arc:validation:ARC-Easy:455", "arc:validation:ARC-Easy:50", "arc:validation:ARC-Easy:538", "arc:validation:ARC-Easy:544", "arc:validation:ARC-Easy:552", "banking:train:1050", "banking:train:1131", "banking:train:1190", "banking:train:124", "banking:train:137", "banking:train:1493", "banking:train:1498", "banking:train:1624", "banking:train:2614", "banking:train:2739", "banking:train:3055", "banking:train:306", "banking:train:3312", "banking:train:3724", "banking:train:4070", "banking:train:476", "banking:train:4882", "banking:train:5075", "banking:train:5332", "banking:train:5445", "banking:train:6096", "banking:train:746", "banking:train:7828", "banking:train:8176", "banking:train:8315", "banking:train:8653", "banking:train:9019", "banking:train:9370", "banking:train:9454", "banking:train:9595", "banking:train:9862", "banking:train:9963", "boolq:validation:1189", "boolq:validation:1320", "boolq:validation:1326", "boolq:validation:1381", "boolq:validation:1410", "boolq:validation:149", "boolq:validation:1513", "boolq:validation:1732", "boolq:validation:1739", "boolq:validation:1741", "boolq:validation:1814", "boolq:validation:1848", "boolq:validation:1940", "boolq:validation:2077", "boolq:validation:2091", "boolq:validation:2148", "boolq:validation:2215", "boolq:validation:2251", "boolq:validation:2283", "boolq:validation:2321", "boolq:validation:2398", "boolq:validation:253", "boolq:validation:2575", "boolq:validation:2907", "boolq:validation:2909", "boolq:validation:2978", "boolq:validation:359", "boolq:validation:394", "boolq:validation:436", "boolq:validation:579", "boolq:validation:807", "boolq:validation:949", "snli:validation:2389", "snli:validation:2485", "snli:validation:255", "snli:validation:3169", "snli:validation:333", "snli:validation:3700", "snli:validation:3709", "snli:validation:4093", "snli:validation:420", "snli:validation:4492", "snli:validation:4516", "snli:validation:4648", "snli:validation:4888", "snli:validation:5185", "snli:validation:5332", "snli:validation:5947", "snli:validation:6124", "snli:validation:660", "snli:validation:6619", "snli:validation:6898", "snli:validation:7387", "snli:validation:7492", "snli:validation:7544", "snli:validation:7996", "snli:validation:8074", "snli:validation:8554", "snli:validation:8971", "snli:validation:9163", "snli:validation:9277", "snli:validation:9490", "snli:validation:9595", "snli:validation:9688" ], "heldout_group_count": 128, "training_group_count": 384 }, { "fold": 2, "temperature_index": 57, "temperature": 2.046444636724674, "training_macro_nll": 0.2569765609800613, "heldout_ids": [ "arc:validation:ARC-Challenge:120", "arc:validation:ARC-Challenge:130", "arc:validation:ARC-Challenge:194", "arc:validation:ARC-Challenge:218", "arc:validation:ARC-Challenge:235", "arc:validation:ARC-Challenge:272", "arc:validation:ARC-Challenge:277", "arc:validation:ARC-Challenge:48", "arc:validation:ARC-Challenge:88", "arc:validation:ARC-Easy:115", "arc:validation:ARC-Easy:12", "arc:validation:ARC-Easy:131", "arc:validation:ARC-Easy:145", "arc:validation:ARC-Easy:179", "arc:validation:ARC-Easy:18", "arc:validation:ARC-Easy:186", "arc:validation:ARC-Easy:191", "arc:validation:ARC-Easy:215", "arc:validation:ARC-Easy:283", "arc:validation:ARC-Easy:314", "arc:validation:ARC-Easy:315", "arc:validation:ARC-Easy:378", "arc:validation:ARC-Easy:384", "arc:validation:ARC-Easy:403", "arc:validation:ARC-Easy:432", "arc:validation:ARC-Easy:439", "arc:validation:ARC-Easy:448", "arc:validation:ARC-Easy:461", "arc:validation:ARC-Easy:469", "arc:validation:ARC-Easy:470", "arc:validation:ARC-Easy:504", "arc:validation:ARC-Easy:561", "banking:train:1142", "banking:train:148", "banking:train:1633", "banking:train:1927", "banking:train:2186", "banking:train:2251", "banking:train:23", "banking:train:3048", "banking:train:3665", "banking:train:3952", "banking:train:3999", "banking:train:4364", "banking:train:4619", "banking:train:5049", "banking:train:5335", "banking:train:549", "banking:train:5957", "banking:train:5985", "banking:train:6132", "banking:train:6240", "banking:train:6488", "banking:train:7065", "banking:train:7749", "banking:train:8470", "banking:train:8761", "banking:train:895", "banking:train:9256", "banking:train:9406", "banking:train:9450", "banking:train:9467", "banking:train:9588", "banking:train:9716", "boolq:validation:1130", "boolq:validation:1167", "boolq:validation:124", "boolq:validation:1255", "boolq:validation:1256", "boolq:validation:1373", "boolq:validation:15", "boolq:validation:155", "boolq:validation:1568", "boolq:validation:16", "boolq:validation:1654", "boolq:validation:1702", "boolq:validation:1792", "boolq:validation:1818", "boolq:validation:210", "boolq:validation:2247", "boolq:validation:2259", "boolq:validation:2362", "boolq:validation:2443", "boolq:validation:2524", "boolq:validation:2553", "boolq:validation:2623", "boolq:validation:2660", "boolq:validation:2669", "boolq:validation:2679", "boolq:validation:2832", "boolq:validation:357", "boolq:validation:453", "boolq:validation:49", "boolq:validation:808", "boolq:validation:90", "boolq:validation:910", "snli:validation:1612", "snli:validation:168", "snli:validation:1693", "snli:validation:2134", "snli:validation:2248", "snli:validation:2422", "snli:validation:2587", "snli:validation:3007", "snli:validation:3097", "snli:validation:3772", "snli:validation:4012", "snli:validation:4168", "snli:validation:4537", "snli:validation:4681", "snli:validation:5050", "snli:validation:5632", "snli:validation:5761", "snli:validation:6055", "snli:validation:6283", "snli:validation:6715", "snli:validation:7117", "snli:validation:7192", "snli:validation:7507", "snli:validation:7594", "snli:validation:7924", "snli:validation:7954", "snli:validation:8035", "snli:validation:8110", "snli:validation:8995", "snli:validation:949", "snli:validation:9523", "snli:validation:9589" ], "heldout_group_count": 128, "training_group_count": 384 }, { "fold": 3, "temperature_index": 56, "temperature": 1.9408858775927773, "training_macro_nll": 0.2592448225500369, "heldout_ids": [ "arc:validation:ARC-Challenge:117", "arc:validation:ARC-Challenge:135", "arc:validation:ARC-Challenge:139", "arc:validation:ARC-Challenge:154", "arc:validation:ARC-Challenge:174", "arc:validation:ARC-Challenge:199", "arc:validation:ARC-Challenge:287", "arc:validation:ARC-Challenge:295", "arc:validation:ARC-Challenge:4", "arc:validation:ARC-Challenge:50", "arc:validation:ARC-Challenge:51", "arc:validation:ARC-Challenge:61", "arc:validation:ARC-Easy:107", "arc:validation:ARC-Easy:121", "arc:validation:ARC-Easy:124", "arc:validation:ARC-Easy:171", "arc:validation:ARC-Easy:235", "arc:validation:ARC-Easy:241", "arc:validation:ARC-Easy:279", "arc:validation:ARC-Easy:289", "arc:validation:ARC-Easy:37", "arc:validation:ARC-Easy:389", "arc:validation:ARC-Easy:394", "arc:validation:ARC-Easy:399", "arc:validation:ARC-Easy:423", "arc:validation:ARC-Easy:492", "arc:validation:ARC-Easy:499", "arc:validation:ARC-Easy:523", "arc:validation:ARC-Easy:533", "arc:validation:ARC-Easy:569", "arc:validation:ARC-Easy:69", "arc:validation:ARC-Easy:93", "banking:train:2049", "banking:train:2350", "banking:train:2497", "banking:train:2506", "banking:train:2720", "banking:train:2788", "banking:train:2967", "banking:train:3502", "banking:train:3726", "banking:train:4014", "banking:train:4735", "banking:train:4803", "banking:train:4917", "banking:train:5477", "banking:train:5851", "banking:train:6551", "banking:train:6839", "banking:train:6926", "banking:train:7809", "banking:train:8092", "banking:train:8186", "banking:train:8215", "banking:train:8667", "banking:train:8752", "banking:train:8763", "banking:train:8851", "banking:train:8917", "banking:train:9145", "banking:train:9279", "banking:train:9676", "banking:train:9837", "banking:train:9967", "boolq:validation:106", "boolq:validation:1065", "boolq:validation:1097", "boolq:validation:1336", "boolq:validation:1350", "boolq:validation:1383", "boolq:validation:1458", "boolq:validation:1641", "boolq:validation:1873", "boolq:validation:1930", "boolq:validation:1989", "boolq:validation:2103", "boolq:validation:234", "boolq:validation:2355", "boolq:validation:2384", "boolq:validation:2440", "boolq:validation:2444", "boolq:validation:2478", "boolq:validation:2872", "boolq:validation:2879", "boolq:validation:2924", "boolq:validation:2971", "boolq:validation:3164", "boolq:validation:4", "boolq:validation:415", "boolq:validation:419", "boolq:validation:428", "boolq:validation:512", "boolq:validation:563", "boolq:validation:643", "boolq:validation:696", "boolq:validation:874", "snli:validation:1102", "snli:validation:1240", "snli:validation:1261", "snli:validation:1738", "snli:validation:1978", "snli:validation:2563", "snli:validation:2659", "snli:validation:2767", "snli:validation:2836", "snli:validation:3232", "snli:validation:3655", "snli:validation:3913", "snli:validation:405", "snli:validation:4051", "snli:validation:4300", "snli:validation:4846", "snli:validation:498", "snli:validation:5224", "snli:validation:5575", "snli:validation:6598", "snli:validation:666", "snli:validation:6925", "snli:validation:6964", "snli:validation:7045", "snli:validation:7351", "snli:validation:7588", "snli:validation:7723", "snli:validation:7750", "snli:validation:7909", "snli:validation:8974", "snli:validation:9385", "snli:validation:9646" ], "heldout_group_count": 128, "training_group_count": 384 } ], "per_family": { "arc": { "count": 128, "score": 0.35546071765134163, "raw_macro_nll": 0.5363843868768633, "accuracy": 0.9140625 }, "banking": { "count": 128, "score": 0.022742246686383003, "raw_macro_nll": 0.030751403774512704, "accuracy": 0.9921875 }, "boolq": { "count": 128, "score": 0.33967224778120886, "raw_macro_nll": 0.4241840175504803, "accuracy": 0.8515625 }, "snli": { "count": 128, "score": 0.3388951255261744, "raw_macro_nll": 0.38408411942219894, "accuracy": 0.8828125 } }, "policy": { "id": "crossfit_temperature_nll_v1", "folds": 4, "seed": 431, "grouping": "source group, nested within task family; groups shuffled within sorted families and assigned round-robin", "temperature_grid": { "count": 101, "log10_min": -1.0, "log10_max": 1.3, "arithmetic": "Python float64" }, "temperature_fit": "minimum macro-family NLL on the other three folds; smallest temperature wins ties", "score": "macro-family mean of all held-out decision NLLs", "deployment_temperature": "fit afresh on reserved calibration only after model selection" } }, "best_selection_score": 0.2552943737691716, "improved": false, "metrics": { "overall": { "n": 512, "accuracy": 0.91015625, "nll": 0.343850982809214, "brier_multiclass_sum": 0.1506205599699681, "ece_top_label_10_equal_width_bins": 0.05876007775077596, "reliability_bins": [ { "count": 0, "confidence_sum": 0.0, "correct_sum": 0.0 }, { "count": 0, "confidence_sum": 0.0, "correct_sum": 0.0 }, { "count": 0, "confidence_sum": 0.0, "correct_sum": 0.0 }, { "count": 0, "confidence_sum": 0.0, "correct_sum": 0.0 }, { "count": 2, "confidence_sum": 0.8661302030086517, "correct_sum": 1.0 }, { "count": 7, "confidence_sum": 3.8665531277656555, "correct_sum": 5.0 }, { "count": 9, "confidence_sum": 5.823009371757507, "correct_sum": 4.0 }, { "count": 12, "confidence_sum": 8.999069094657898, "correct_sum": 8.0 }, { "count": 27, "confidence_sum": 23.208462595939636, "correct_sum": 17.0 }, { "count": 455, "confidence_sum": 450.78730207681656, "correct_sum": 431.0 } ], "accuracy_vs_coverage": { "0.25": { "n": 128, "accuracy": 1.0, "min_confidence": 0.9999998807907104 }, "0.5": { "n": 256, "accuracy": 0.99609375, "min_confidence": 0.9987045526504517 }, "0.75": { "n": 384, "accuracy": 0.9713541666666666, "min_confidence": 0.9814466238021851 }, "1.0": { "n": 512, "accuracy": 0.91015625, "min_confidence": 0.42066824436187744 } } }, "per_family": { "arc": { "n": 128, "accuracy": 0.9140625, "nll": 0.5363843899576184, "brier_multiclass_sum": 0.14942062283772675, "ece_top_label_10_equal_width_bins": 0.060488176764920354, "reliability_bins": [ { "count": 0, "confidence_sum": 0.0, "correct_sum": 0.0 }, { "count": 0, "confidence_sum": 0.0, "correct_sum": 0.0 }, { "count": 0, "confidence_sum": 0.0, "correct_sum": 0.0 }, { "count": 0, "confidence_sum": 0.0, "correct_sum": 0.0 }, { "count": 2, "confidence_sum": 0.8661302030086517, "correct_sum": 1.0 }, { "count": 4, "confidence_sum": 2.261072337627411, "correct_sum": 2.0 }, { "count": 1, "confidence_sum": 0.6171885132789612, "correct_sum": 1.0 }, { "count": 4, "confidence_sum": 3.027863085269928, "correct_sum": 3.0 }, { "count": 4, "confidence_sum": 3.4061849117279053, "correct_sum": 4.0 }, { "count": 113, "confidence_sum": 112.34305483102798, "correct_sum": 106.0 } ], "accuracy_vs_coverage": { "0.25": { "n": 32, "accuracy": 1.0, "min_confidence": 0.9999995231628418 }, "0.5": { "n": 64, "accuracy": 0.984375, "min_confidence": 0.9999145269393921 }, "0.75": { "n": 96, "accuracy": 0.9479166666666666, "min_confidence": 0.993623673915863 }, "1.0": { "n": 128, "accuracy": 0.9140625, "min_confidence": 0.42066824436187744 } } }, "banking": { "n": 128, "accuracy": 0.9921875, "nll": 0.030751401536174172, "brier_multiclass_sum": 0.015997644546320234, "ece_top_label_10_equal_width_bins": 0.009750935714691877, "reliability_bins": [ { "count": 0, "confidence_sum": 0.0, "correct_sum": 0.0 }, { "count": 0, "confidence_sum": 0.0, "correct_sum": 0.0 }, { "count": 0, "confidence_sum": 0.0, "correct_sum": 0.0 }, { "count": 0, "confidence_sum": 0.0, "correct_sum": 0.0 }, { "count": 0, "confidence_sum": 0.0, "correct_sum": 0.0 }, { "count": 0, "confidence_sum": 0.0, "correct_sum": 0.0 }, { "count": 1, "confidence_sum": 0.6965740919113159, "correct_sum": 1.0 }, { "count": 0, "confidence_sum": 0.0, "correct_sum": 0.0 }, { "count": 0, "confidence_sum": 0.0, "correct_sum": 0.0 }, { "count": 127, "confidence_sum": 126.94469386339188, "correct_sum": 126.0 } ], "accuracy_vs_coverage": { "0.25": { "n": 32, "accuracy": 1.0, "min_confidence": 1.0 }, "0.5": { "n": 64, "accuracy": 1.0, "min_confidence": 1.0 }, "0.75": { "n": 96, "accuracy": 1.0, "min_confidence": 1.0 }, "1.0": { "n": 128, "accuracy": 0.9921875, "min_confidence": 0.6965740919113159 } } }, "boolq": { "n": 128, "accuracy": 0.8515625, "nll": 0.42418402199575667, "brier_multiclass_sum": 0.24188416272422544, "ece_top_label_10_equal_width_bins": 0.10255556367337704, "reliability_bins": [ { "count": 0, "confidence_sum": 0.0, "correct_sum": 0.0 }, { "count": 0, "confidence_sum": 0.0, "correct_sum": 0.0 }, { "count": 0, "confidence_sum": 0.0, "correct_sum": 0.0 }, { "count": 0, "confidence_sum": 0.0, "correct_sum": 0.0 }, { "count": 0, "confidence_sum": 0.0, "correct_sum": 0.0 }, { "count": 1, "confidence_sum": 0.5557354688644409, "correct_sum": 1.0 }, { "count": 3, "confidence_sum": 1.84639310836792, "correct_sum": 1.0 }, { "count": 6, "confidence_sum": 4.499466836452484, "correct_sum": 4.0 }, { "count": 14, "confidence_sum": 11.978712320327759, "correct_sum": 8.0 }, { "count": 104, "confidence_sum": 102.35827535390854, "correct_sum": 95.0 } ], "accuracy_vs_coverage": { "0.25": { "n": 32, "accuracy": 1.0, "min_confidence": 0.9973174929618835 }, "0.5": { "n": 64, "accuracy": 0.96875, "min_confidence": 0.9890179634094238 }, "0.75": { "n": 96, "accuracy": 0.9270833333333334, "min_confidence": 0.9507815837860107 }, "1.0": { "n": 128, "accuracy": 0.8515625, "min_confidence": 0.5557354688644409 } } }, "snli": { "n": 128, "accuracy": 0.8828125, "nll": 0.38408411774730666, "brier_multiclass_sum": 0.19517980977159996, "ece_top_label_10_equal_width_bins": 0.08632571017369628, "reliability_bins": [ { "count": 0, "confidence_sum": 0.0, "correct_sum": 0.0 }, { "count": 0, "confidence_sum": 0.0, "correct_sum": 0.0 }, { "count": 0, "confidence_sum": 0.0, "correct_sum": 0.0 }, { "count": 0, "confidence_sum": 0.0, "correct_sum": 0.0 }, { "count": 0, "confidence_sum": 0.0, "correct_sum": 0.0 }, { "count": 2, "confidence_sum": 1.0497453212738037, "correct_sum": 2.0 }, { "count": 4, "confidence_sum": 2.6628536581993103, "correct_sum": 1.0 }, { "count": 2, "confidence_sum": 1.4717391729354858, "correct_sum": 1.0 }, { "count": 9, "confidence_sum": 7.823565363883972, "correct_sum": 5.0 }, { "count": 111, "confidence_sum": 109.14127802848816, "correct_sum": 104.0 } ], "accuracy_vs_coverage": { "0.25": { "n": 32, "accuracy": 1.0, "min_confidence": 0.9989041090011597 }, "0.5": { "n": 64, "accuracy": 0.96875, "min_confidence": 0.9890118837356567 }, "0.75": { "n": 96, "accuracy": 0.9375, "min_confidence": 0.9636436700820923 }, "1.0": { "n": 128, "accuracy": 0.8828125, "min_confidence": 0.5159494876861572 } } } } } }, { "step": 3500, "macro_nll": 0.32187738830487217, "best_macro_nll": 0.25676974726429014, "selection": { "metric": "crossfit_temperature_nll_v1", "score": 0.2805558053510232, "raw_macro_nll": 0.32187738794463494, "accuracy": 0.9140625, "folds": [ { "fold": 0, "temperature_index": 54, "temperature": 1.7458221529205038, "training_macro_nll": 0.3014060109823255, "heldout_ids": [ "arc:validation:ARC-Challenge:122", "arc:validation:ARC-Challenge:126", "arc:validation:ARC-Challenge:175", "arc:validation:ARC-Challenge:187", "arc:validation:ARC-Challenge:189", "arc:validation:ARC-Challenge:192", "arc:validation:ARC-Challenge:208", "arc:validation:ARC-Challenge:238", "arc:validation:ARC-Challenge:290", "arc:validation:ARC-Challenge:293", "arc:validation:ARC-Challenge:35", "arc:validation:ARC-Challenge:43", "arc:validation:ARC-Challenge:58", "arc:validation:ARC-Challenge:7", "arc:validation:ARC-Easy:160", "arc:validation:ARC-Easy:201", "arc:validation:ARC-Easy:229", "arc:validation:ARC-Easy:238", "arc:validation:ARC-Easy:253", "arc:validation:ARC-Easy:259", "arc:validation:ARC-Easy:272", "arc:validation:ARC-Easy:298", "arc:validation:ARC-Easy:329", "arc:validation:ARC-Easy:331", "arc:validation:ARC-Easy:337", "arc:validation:ARC-Easy:362", "arc:validation:ARC-Easy:443", "arc:validation:ARC-Easy:500", "arc:validation:ARC-Easy:545", "arc:validation:ARC-Easy:68", "arc:validation:ARC-Easy:7", "arc:validation:ARC-Easy:74", "banking:train:1053", "banking:train:1594", "banking:train:1747", "banking:train:202", "banking:train:2377", "banking:train:2528", "banking:train:2546", "banking:train:2715", "banking:train:3220", "banking:train:3499", "banking:train:3806", "banking:train:3838", "banking:train:3993", "banking:train:4220", "banking:train:5", "banking:train:5192", "banking:train:5394", "banking:train:5846", "banking:train:5863", "banking:train:6032", "banking:train:6508", "banking:train:7579", "banking:train:7600", "banking:train:7987", "banking:train:8155", "banking:train:850", "banking:train:8626", "banking:train:8929", "banking:train:9355", "banking:train:9812", "banking:train:987", "banking:train:9972", "boolq:validation:103", "boolq:validation:1042", "boolq:validation:1059", "boolq:validation:1541", "boolq:validation:1592", "boolq:validation:1927", "boolq:validation:2014", "boolq:validation:2023", "boolq:validation:2098", "boolq:validation:2139", "boolq:validation:2439", "boolq:validation:2485", "boolq:validation:2607", "boolq:validation:2611", "boolq:validation:2720", "boolq:validation:2751", "boolq:validation:282", "boolq:validation:2843", "boolq:validation:2953", "boolq:validation:301", "boolq:validation:3089", "boolq:validation:313", "boolq:validation:3201", "boolq:validation:322", "boolq:validation:3231", "boolq:validation:39", "boolq:validation:461", "boolq:validation:558", "boolq:validation:589", "boolq:validation:731", "boolq:validation:883", "boolq:validation:915", "snli:validation:1156", "snli:validation:1330", "snli:validation:1559", "snli:validation:1735", "snli:validation:1876", "snli:validation:2251", "snli:validation:2257", "snli:validation:2377", "snli:validation:2881", "snli:validation:291", "snli:validation:300", "snli:validation:3049", "snli:validation:3496", "snli:validation:3871", "snli:validation:4414", "snli:validation:5140", "snli:validation:5440", "snli:validation:5806", "snli:validation:6442", "snli:validation:6772", "snli:validation:6949", "snli:validation:7228", "snli:validation:7465", "snli:validation:7510", "snli:validation:7726", "snli:validation:8095", "snli:validation:8143", "snli:validation:8623", "snli:validation:8962", "snli:validation:9451", "snli:validation:9574", "snli:validation:9655" ], "heldout_group_count": 128, "training_group_count": 384 }, { "fold": 1, "temperature_index": 52, "temperature": 1.5703628043335522, "training_macro_nll": 0.2684132604949462, "heldout_ids": [ "arc:validation:ARC-Challenge:101", "arc:validation:ARC-Challenge:129", "arc:validation:ARC-Challenge:148", "arc:validation:ARC-Challenge:156", "arc:validation:ARC-Challenge:167", "arc:validation:ARC-Challenge:249", "arc:validation:ARC-Challenge:263", "arc:validation:ARC-Challenge:266", "arc:validation:ARC-Challenge:267", "arc:validation:ARC-Challenge:28", "arc:validation:ARC-Challenge:281", "arc:validation:ARC-Challenge:57", "arc:validation:ARC-Challenge:63", "arc:validation:ARC-Challenge:64", "arc:validation:ARC-Challenge:86", "arc:validation:ARC-Challenge:99", "arc:validation:ARC-Easy:106", "arc:validation:ARC-Easy:151", "arc:validation:ARC-Easy:157", "arc:validation:ARC-Easy:166", "arc:validation:ARC-Easy:202", "arc:validation:ARC-Easy:219", "arc:validation:ARC-Easy:224", "arc:validation:ARC-Easy:322", "arc:validation:ARC-Easy:344", "arc:validation:ARC-Easy:382", "arc:validation:ARC-Easy:409", "arc:validation:ARC-Easy:455", "arc:validation:ARC-Easy:50", "arc:validation:ARC-Easy:538", "arc:validation:ARC-Easy:544", "arc:validation:ARC-Easy:552", "banking:train:1050", "banking:train:1131", "banking:train:1190", "banking:train:124", "banking:train:137", "banking:train:1493", "banking:train:1498", "banking:train:1624", "banking:train:2614", "banking:train:2739", "banking:train:3055", "banking:train:306", "banking:train:3312", "banking:train:3724", "banking:train:4070", "banking:train:476", "banking:train:4882", "banking:train:5075", "banking:train:5332", "banking:train:5445", "banking:train:6096", "banking:train:746", "banking:train:7828", "banking:train:8176", "banking:train:8315", "banking:train:8653", "banking:train:9019", "banking:train:9370", "banking:train:9454", "banking:train:9595", "banking:train:9862", "banking:train:9963", "boolq:validation:1189", "boolq:validation:1320", "boolq:validation:1326", "boolq:validation:1381", "boolq:validation:1410", "boolq:validation:149", "boolq:validation:1513", "boolq:validation:1732", "boolq:validation:1739", "boolq:validation:1741", "boolq:validation:1814", "boolq:validation:1848", "boolq:validation:1940", "boolq:validation:2077", "boolq:validation:2091", "boolq:validation:2148", "boolq:validation:2215", "boolq:validation:2251", "boolq:validation:2283", "boolq:validation:2321", "boolq:validation:2398", "boolq:validation:253", "boolq:validation:2575", "boolq:validation:2907", "boolq:validation:2909", "boolq:validation:2978", "boolq:validation:359", "boolq:validation:394", "boolq:validation:436", "boolq:validation:579", "boolq:validation:807", "boolq:validation:949", "snli:validation:2389", "snli:validation:2485", "snli:validation:255", "snli:validation:3169", "snli:validation:333", "snli:validation:3700", "snli:validation:3709", "snli:validation:4093", "snli:validation:420", "snli:validation:4492", "snli:validation:4516", "snli:validation:4648", "snli:validation:4888", "snli:validation:5185", "snli:validation:5332", "snli:validation:5947", "snli:validation:6124", "snli:validation:660", "snli:validation:6619", "snli:validation:6898", "snli:validation:7387", "snli:validation:7492", "snli:validation:7544", "snli:validation:7996", "snli:validation:8074", "snli:validation:8554", "snli:validation:8971", "snli:validation:9163", "snli:validation:9277", "snli:validation:9490", "snli:validation:9595", "snli:validation:9688" ], "heldout_group_count": 128, "training_group_count": 384 }, { "fold": 2, "temperature_index": 53, "temperature": 1.6557699634695275, "training_macro_nll": 0.2710780825791906, "heldout_ids": [ "arc:validation:ARC-Challenge:120", "arc:validation:ARC-Challenge:130", "arc:validation:ARC-Challenge:194", "arc:validation:ARC-Challenge:218", "arc:validation:ARC-Challenge:235", "arc:validation:ARC-Challenge:272", "arc:validation:ARC-Challenge:277", "arc:validation:ARC-Challenge:48", "arc:validation:ARC-Challenge:88", "arc:validation:ARC-Easy:115", "arc:validation:ARC-Easy:12", "arc:validation:ARC-Easy:131", "arc:validation:ARC-Easy:145", "arc:validation:ARC-Easy:179", "arc:validation:ARC-Easy:18", "arc:validation:ARC-Easy:186", "arc:validation:ARC-Easy:191", "arc:validation:ARC-Easy:215", "arc:validation:ARC-Easy:283", "arc:validation:ARC-Easy:314", "arc:validation:ARC-Easy:315", "arc:validation:ARC-Easy:378", "arc:validation:ARC-Easy:384", "arc:validation:ARC-Easy:403", "arc:validation:ARC-Easy:432", "arc:validation:ARC-Easy:439", "arc:validation:ARC-Easy:448", "arc:validation:ARC-Easy:461", "arc:validation:ARC-Easy:469", "arc:validation:ARC-Easy:470", "arc:validation:ARC-Easy:504", "arc:validation:ARC-Easy:561", "banking:train:1142", "banking:train:148", "banking:train:1633", "banking:train:1927", "banking:train:2186", "banking:train:2251", "banking:train:23", "banking:train:3048", "banking:train:3665", "banking:train:3952", "banking:train:3999", "banking:train:4364", "banking:train:4619", "banking:train:5049", "banking:train:5335", "banking:train:549", "banking:train:5957", "banking:train:5985", "banking:train:6132", "banking:train:6240", "banking:train:6488", "banking:train:7065", "banking:train:7749", "banking:train:8470", "banking:train:8761", "banking:train:895", "banking:train:9256", "banking:train:9406", "banking:train:9450", "banking:train:9467", "banking:train:9588", "banking:train:9716", "boolq:validation:1130", "boolq:validation:1167", "boolq:validation:124", "boolq:validation:1255", "boolq:validation:1256", "boolq:validation:1373", "boolq:validation:15", "boolq:validation:155", "boolq:validation:1568", "boolq:validation:16", "boolq:validation:1654", "boolq:validation:1702", "boolq:validation:1792", "boolq:validation:1818", "boolq:validation:210", "boolq:validation:2247", "boolq:validation:2259", "boolq:validation:2362", "boolq:validation:2443", "boolq:validation:2524", "boolq:validation:2553", "boolq:validation:2623", "boolq:validation:2660", "boolq:validation:2669", "boolq:validation:2679", "boolq:validation:2832", "boolq:validation:357", "boolq:validation:453", "boolq:validation:49", "boolq:validation:808", "boolq:validation:90", "boolq:validation:910", "snli:validation:1612", "snli:validation:168", "snli:validation:1693", "snli:validation:2134", "snli:validation:2248", "snli:validation:2422", "snli:validation:2587", "snli:validation:3007", "snli:validation:3097", "snli:validation:3772", "snli:validation:4012", "snli:validation:4168", "snli:validation:4537", "snli:validation:4681", "snli:validation:5050", "snli:validation:5632", "snli:validation:5761", "snli:validation:6055", "snli:validation:6283", "snli:validation:6715", "snli:validation:7117", "snli:validation:7192", "snli:validation:7507", "snli:validation:7594", "snli:validation:7924", "snli:validation:7954", "snli:validation:8035", "snli:validation:8110", "snli:validation:8995", "snli:validation:949", "snli:validation:9523", "snli:validation:9589" ], "heldout_group_count": 128, "training_group_count": 384 }, { "fold": 3, "temperature_index": 52, "temperature": 1.5703628043335522, "training_macro_nll": 0.27198244895421647, "heldout_ids": [ "arc:validation:ARC-Challenge:117", "arc:validation:ARC-Challenge:135", "arc:validation:ARC-Challenge:139", "arc:validation:ARC-Challenge:154", "arc:validation:ARC-Challenge:174", "arc:validation:ARC-Challenge:199", "arc:validation:ARC-Challenge:287", "arc:validation:ARC-Challenge:295", "arc:validation:ARC-Challenge:4", "arc:validation:ARC-Challenge:50", "arc:validation:ARC-Challenge:51", "arc:validation:ARC-Challenge:61", "arc:validation:ARC-Easy:107", "arc:validation:ARC-Easy:121", "arc:validation:ARC-Easy:124", "arc:validation:ARC-Easy:171", "arc:validation:ARC-Easy:235", "arc:validation:ARC-Easy:241", "arc:validation:ARC-Easy:279", "arc:validation:ARC-Easy:289", "arc:validation:ARC-Easy:37", "arc:validation:ARC-Easy:389", "arc:validation:ARC-Easy:394", "arc:validation:ARC-Easy:399", "arc:validation:ARC-Easy:423", "arc:validation:ARC-Easy:492", "arc:validation:ARC-Easy:499", "arc:validation:ARC-Easy:523", "arc:validation:ARC-Easy:533", "arc:validation:ARC-Easy:569", "arc:validation:ARC-Easy:69", "arc:validation:ARC-Easy:93", "banking:train:2049", "banking:train:2350", "banking:train:2497", "banking:train:2506", "banking:train:2720", "banking:train:2788", "banking:train:2967", "banking:train:3502", "banking:train:3726", "banking:train:4014", "banking:train:4735", "banking:train:4803", "banking:train:4917", "banking:train:5477", "banking:train:5851", "banking:train:6551", "banking:train:6839", "banking:train:6926", "banking:train:7809", "banking:train:8092", "banking:train:8186", "banking:train:8215", "banking:train:8667", "banking:train:8752", "banking:train:8763", "banking:train:8851", "banking:train:8917", "banking:train:9145", "banking:train:9279", "banking:train:9676", "banking:train:9837", "banking:train:9967", "boolq:validation:106", "boolq:validation:1065", "boolq:validation:1097", "boolq:validation:1336", "boolq:validation:1350", "boolq:validation:1383", "boolq:validation:1458", "boolq:validation:1641", "boolq:validation:1873", "boolq:validation:1930", "boolq:validation:1989", "boolq:validation:2103", "boolq:validation:234", "boolq:validation:2355", "boolq:validation:2384", "boolq:validation:2440", "boolq:validation:2444", "boolq:validation:2478", "boolq:validation:2872", "boolq:validation:2879", "boolq:validation:2924", "boolq:validation:2971", "boolq:validation:3164", "boolq:validation:4", "boolq:validation:415", "boolq:validation:419", "boolq:validation:428", "boolq:validation:512", "boolq:validation:563", "boolq:validation:643", "boolq:validation:696", "boolq:validation:874", "snli:validation:1102", "snli:validation:1240", "snli:validation:1261", "snli:validation:1738", "snli:validation:1978", "snli:validation:2563", "snli:validation:2659", "snli:validation:2767", "snli:validation:2836", "snli:validation:3232", "snli:validation:3655", "snli:validation:3913", "snli:validation:405", "snli:validation:4051", "snli:validation:4300", "snli:validation:4846", "snli:validation:498", "snli:validation:5224", "snli:validation:5575", "snli:validation:6598", "snli:validation:666", "snli:validation:6925", "snli:validation:6964", "snli:validation:7045", "snli:validation:7351", "snli:validation:7588", "snli:validation:7723", "snli:validation:7750", "snli:validation:7909", "snli:validation:8974", "snli:validation:9385", "snli:validation:9646" ], "heldout_group_count": 128, "training_group_count": 384 } ], "per_family": { "arc": { "count": 128, "score": 0.3012568093094588, "raw_macro_nll": 0.33594514952792776, "accuracy": 0.90625 }, "banking": { "count": 128, "score": 0.04181559751379856, "raw_macro_nll": 0.03417229952375239, "accuracy": 0.9765625 }, "boolq": { "count": 128, "score": 0.43627154539585905, "raw_macro_nll": 0.5853439575679401, "accuracy": 0.859375 }, "snli": { "count": 128, "score": 0.3428792691849764, "raw_macro_nll": 0.33204814515891956, "accuracy": 0.9140625 } }, "policy": { "id": "crossfit_temperature_nll_v1", "folds": 4, "seed": 431, "grouping": "source group, nested within task family; groups shuffled within sorted families and assigned round-robin", "temperature_grid": { "count": 101, "log10_min": -1.0, "log10_max": 1.3, "arithmetic": "Python float64" }, "temperature_fit": "minimum macro-family NLL on the other three folds; smallest temperature wins ties", "score": "macro-family mean of all held-out decision NLLs", "deployment_temperature": "fit afresh on reserved calibration only after model selection" } }, "best_selection_score": 0.2552943737691716, "improved": false, "metrics": { "overall": { "n": 512, "accuracy": 0.9140625, "nll": 0.32187738830487217, "brier_multiclass_sum": 0.14736465059578374, "ece_top_label_10_equal_width_bins": 0.04893872793763876, "reliability_bins": [ { "count": 0, "confidence_sum": 0.0, "correct_sum": 0.0 }, { "count": 0, "confidence_sum": 0.0, "correct_sum": 0.0 }, { "count": 0, "confidence_sum": 0.0, "correct_sum": 0.0 }, { "count": 2, "confidence_sum": 0.7409681677818298, "correct_sum": 0.0 }, { "count": 2, "confidence_sum": 0.8548331260681152, "correct_sum": 1.0 }, { "count": 15, "confidence_sum": 8.04920494556427, "correct_sum": 10.0 }, { "count": 7, "confidence_sum": 4.55422168970108, "correct_sum": 6.0 }, { "count": 18, "confidence_sum": 13.7101429104805, "correct_sum": 14.0 }, { "count": 28, "confidence_sum": 24.01894336938858, "correct_sum": 19.0 }, { "count": 440, "confidence_sum": 433.4651198387146, "correct_sum": 418.0 } ], "accuracy_vs_coverage": { "0.25": { "n": 128, "accuracy": 1.0, "min_confidence": 0.9999927282333374 }, "0.5": { "n": 256, "accuracy": 0.98828125, "min_confidence": 0.9974467754364014 }, "0.75": { "n": 384, "accuracy": 0.9557291666666666, "min_confidence": 0.9531290531158447 }, "1.0": { "n": 512, "accuracy": 0.9140625, "min_confidence": 0.3422239124774933 } } }, "per_family": { "arc": { "n": 128, "accuracy": 0.90625, "nll": 0.33594515031489874, "brier_multiclass_sum": 0.15290327723897154, "ece_top_label_10_equal_width_bins": 0.049574420088902116, "reliability_bins": [ { "count": 0, "confidence_sum": 0.0, "correct_sum": 0.0 }, { "count": 0, "confidence_sum": 0.0, "correct_sum": 0.0 }, { "count": 0, "confidence_sum": 0.0, "correct_sum": 0.0 }, { "count": 1, "confidence_sum": 0.3422239124774933, "correct_sum": 0.0 }, { "count": 2, "confidence_sum": 0.8548331260681152, "correct_sum": 1.0 }, { "count": 7, "confidence_sum": 3.7372488379478455, "correct_sum": 5.0 }, { "count": 4, "confidence_sum": 2.650745928287506, "correct_sum": 3.0 }, { "count": 4, "confidence_sum": 3.025138795375824, "correct_sum": 3.0 }, { "count": 6, "confidence_sum": 5.112928211688995, "correct_sum": 4.0 }, { "count": 104, "confidence_sum": 103.10806274414062, "correct_sum": 100.0 } ], "accuracy_vs_coverage": { "0.25": { "n": 32, "accuracy": 1.0, "min_confidence": 0.9999889135360718 }, "0.5": { "n": 64, "accuracy": 1.0, "min_confidence": 0.9987491369247437 }, "0.75": { "n": 96, "accuracy": 0.96875, "min_confidence": 0.959827721118927 }, "1.0": { "n": 128, "accuracy": 0.90625, "min_confidence": 0.3422239124774933 } } }, "banking": { "n": 128, "accuracy": 0.9765625, "nll": 0.034172295577444056, "brier_multiclass_sum": 0.02131978426881908, "ece_top_label_10_equal_width_bins": 0.008815546752884984, "reliability_bins": [ { "count": 0, "confidence_sum": 0.0, "correct_sum": 0.0 }, { "count": 0, "confidence_sum": 0.0, "correct_sum": 0.0 }, { "count": 0, "confidence_sum": 0.0, "correct_sum": 0.0 }, { "count": 1, "confidence_sum": 0.39874425530433655, "correct_sum": 0.0 }, { "count": 0, "confidence_sum": 0.0, "correct_sum": 0.0 }, { "count": 1, "confidence_sum": 0.5265122652053833, "correct_sum": 0.0 }, { "count": 0, "confidence_sum": 0.0, "correct_sum": 0.0 }, { "count": 4, "confidence_sum": 3.018770754337311, "correct_sum": 3.0 }, { "count": 0, "confidence_sum": 0.0, "correct_sum": 0.0 }, { "count": 122, "confidence_sum": 121.81563729047775, "correct_sum": 122.0 } ], "accuracy_vs_coverage": { "0.25": { "n": 32, "accuracy": 1.0, "min_confidence": 1.0 }, "0.5": { "n": 64, "accuracy": 1.0, "min_confidence": 1.0 }, "0.75": { "n": 96, "accuracy": 1.0, "min_confidence": 0.9999971389770508 }, "1.0": { "n": 128, "accuracy": 0.9765625, "min_confidence": 0.39874425530433655 } } }, "boolq": { "n": 128, "accuracy": 0.859375, "nll": 0.5853439643498177, "brier_multiclass_sum": 0.2555628278708776, "ece_top_label_10_equal_width_bins": 0.10254310397431254, "reliability_bins": [ { "count": 0, "confidence_sum": 0.0, "correct_sum": 0.0 }, { "count": 0, "confidence_sum": 0.0, "correct_sum": 0.0 }, { "count": 0, "confidence_sum": 0.0, "correct_sum": 0.0 }, { "count": 0, "confidence_sum": 0.0, "correct_sum": 0.0 }, { "count": 0, "confidence_sum": 0.0, "correct_sum": 0.0 }, { "count": 6, "confidence_sum": 3.190328001976013, "correct_sum": 4.0 }, { "count": 2, "confidence_sum": 1.281350314617157, "correct_sum": 2.0 }, { "count": 8, "confidence_sum": 6.086918532848358, "correct_sum": 6.0 }, { "count": 11, "confidence_sum": 9.494810223579407, "correct_sum": 8.0 }, { "count": 101, "confidence_sum": 100.01546686887741, "correct_sum": 90.0 } ], "accuracy_vs_coverage": { "0.25": { "n": 32, "accuracy": 0.96875, "min_confidence": 0.9992445707321167 }, "0.5": { "n": 64, "accuracy": 0.921875, "min_confidence": 0.9960445165634155 }, "0.75": { "n": 96, "accuracy": 0.8854166666666666, "min_confidence": 0.9512931704521179 }, "1.0": { "n": 128, "accuracy": 0.859375, "min_confidence": 0.5006566047668457 } } }, "snli": { "n": 128, "accuracy": 0.9140625, "nll": 0.3320481429773281, "brier_multiclass_sum": 0.15967271300446675, "ece_top_label_10_equal_width_bins": 0.04797345120459795, "reliability_bins": [ { "count": 0, "confidence_sum": 0.0, "correct_sum": 0.0 }, { "count": 0, "confidence_sum": 0.0, "correct_sum": 0.0 }, { "count": 0, "confidence_sum": 0.0, "correct_sum": 0.0 }, { "count": 0, "confidence_sum": 0.0, "correct_sum": 0.0 }, { "count": 0, "confidence_sum": 0.0, "correct_sum": 0.0 }, { "count": 1, "confidence_sum": 0.5951158404350281, "correct_sum": 1.0 }, { "count": 1, "confidence_sum": 0.6221254467964172, "correct_sum": 1.0 }, { "count": 2, "confidence_sum": 1.5793148279190063, "correct_sum": 2.0 }, { "count": 11, "confidence_sum": 9.411204934120178, "correct_sum": 7.0 }, { "count": 113, "confidence_sum": 108.52595293521881, "correct_sum": 106.0 } ], "accuracy_vs_coverage": { "0.25": { "n": 32, "accuracy": 0.9375, "min_confidence": 0.9775771498680115 }, "0.5": { "n": 64, "accuracy": 0.953125, "min_confidence": 0.9560697078704834 }, "0.75": { "n": 96, "accuracy": 0.9479166666666666, "min_confidence": 0.9285557866096497 }, "1.0": { "n": 128, "accuracy": 0.9140625, "min_confidence": 0.5951158404350281 } } } } } }, { "step": 4000, "macro_nll": 0.6221038123596042, "best_macro_nll": 0.25676974726429014, "selection": { "metric": "crossfit_temperature_nll_v1", "score": 0.31337868570193705, "raw_macro_nll": 0.6221038143217102, "accuracy": 0.896484375, "folds": [ { "fold": 0, "temperature_index": 67, "temperature": 3.4753616144320576, "training_macro_nll": 0.32442736125181126, "heldout_ids": [ "arc:validation:ARC-Challenge:122", "arc:validation:ARC-Challenge:126", "arc:validation:ARC-Challenge:175", "arc:validation:ARC-Challenge:187", "arc:validation:ARC-Challenge:189", "arc:validation:ARC-Challenge:192", "arc:validation:ARC-Challenge:208", "arc:validation:ARC-Challenge:238", "arc:validation:ARC-Challenge:290", "arc:validation:ARC-Challenge:293", "arc:validation:ARC-Challenge:35", "arc:validation:ARC-Challenge:43", "arc:validation:ARC-Challenge:58", "arc:validation:ARC-Challenge:7", "arc:validation:ARC-Easy:160", "arc:validation:ARC-Easy:201", "arc:validation:ARC-Easy:229", "arc:validation:ARC-Easy:238", "arc:validation:ARC-Easy:253", "arc:validation:ARC-Easy:259", "arc:validation:ARC-Easy:272", "arc:validation:ARC-Easy:298", "arc:validation:ARC-Easy:329", "arc:validation:ARC-Easy:331", "arc:validation:ARC-Easy:337", "arc:validation:ARC-Easy:362", "arc:validation:ARC-Easy:443", "arc:validation:ARC-Easy:500", "arc:validation:ARC-Easy:545", "arc:validation:ARC-Easy:68", "arc:validation:ARC-Easy:7", "arc:validation:ARC-Easy:74", "banking:train:1053", "banking:train:1594", "banking:train:1747", "banking:train:202", "banking:train:2377", "banking:train:2528", "banking:train:2546", "banking:train:2715", "banking:train:3220", "banking:train:3499", "banking:train:3806", "banking:train:3838", "banking:train:3993", "banking:train:4220", "banking:train:5", "banking:train:5192", "banking:train:5394", "banking:train:5846", "banking:train:5863", "banking:train:6032", "banking:train:6508", "banking:train:7579", "banking:train:7600", "banking:train:7987", "banking:train:8155", "banking:train:850", "banking:train:8626", "banking:train:8929", "banking:train:9355", "banking:train:9812", "banking:train:987", "banking:train:9972", "boolq:validation:103", "boolq:validation:1042", "boolq:validation:1059", "boolq:validation:1541", "boolq:validation:1592", "boolq:validation:1927", "boolq:validation:2014", "boolq:validation:2023", "boolq:validation:2098", "boolq:validation:2139", "boolq:validation:2439", "boolq:validation:2485", "boolq:validation:2607", "boolq:validation:2611", "boolq:validation:2720", "boolq:validation:2751", "boolq:validation:282", "boolq:validation:2843", "boolq:validation:2953", "boolq:validation:301", "boolq:validation:3089", "boolq:validation:313", "boolq:validation:3201", "boolq:validation:322", "boolq:validation:3231", "boolq:validation:39", "boolq:validation:461", "boolq:validation:558", "boolq:validation:589", "boolq:validation:731", "boolq:validation:883", "boolq:validation:915", "snli:validation:1156", "snli:validation:1330", "snli:validation:1559", "snli:validation:1735", "snli:validation:1876", "snli:validation:2251", "snli:validation:2257", "snli:validation:2377", "snli:validation:2881", "snli:validation:291", "snli:validation:300", "snli:validation:3049", "snli:validation:3496", "snli:validation:3871", "snli:validation:4414", "snli:validation:5140", "snli:validation:5440", "snli:validation:5806", "snli:validation:6442", "snli:validation:6772", "snli:validation:6949", "snli:validation:7228", "snli:validation:7465", "snli:validation:7510", "snli:validation:7726", "snli:validation:8095", "snli:validation:8143", "snli:validation:8623", "snli:validation:8962", "snli:validation:9451", "snli:validation:9574", "snli:validation:9655" ], "heldout_group_count": 128, "training_group_count": 384 }, { "fold": 1, "temperature_index": 67, "temperature": 3.4753616144320576, "training_macro_nll": 0.3142750882453091, "heldout_ids": [ "arc:validation:ARC-Challenge:101", "arc:validation:ARC-Challenge:129", "arc:validation:ARC-Challenge:148", "arc:validation:ARC-Challenge:156", "arc:validation:ARC-Challenge:167", "arc:validation:ARC-Challenge:249", "arc:validation:ARC-Challenge:263", "arc:validation:ARC-Challenge:266", "arc:validation:ARC-Challenge:267", "arc:validation:ARC-Challenge:28", "arc:validation:ARC-Challenge:281", "arc:validation:ARC-Challenge:57", "arc:validation:ARC-Challenge:63", "arc:validation:ARC-Challenge:64", "arc:validation:ARC-Challenge:86", "arc:validation:ARC-Challenge:99", "arc:validation:ARC-Easy:106", "arc:validation:ARC-Easy:151", "arc:validation:ARC-Easy:157", "arc:validation:ARC-Easy:166", "arc:validation:ARC-Easy:202", "arc:validation:ARC-Easy:219", "arc:validation:ARC-Easy:224", "arc:validation:ARC-Easy:322", "arc:validation:ARC-Easy:344", "arc:validation:ARC-Easy:382", "arc:validation:ARC-Easy:409", "arc:validation:ARC-Easy:455", "arc:validation:ARC-Easy:50", "arc:validation:ARC-Easy:538", "arc:validation:ARC-Easy:544", "arc:validation:ARC-Easy:552", "banking:train:1050", "banking:train:1131", "banking:train:1190", "banking:train:124", "banking:train:137", "banking:train:1493", "banking:train:1498", "banking:train:1624", "banking:train:2614", "banking:train:2739", "banking:train:3055", "banking:train:306", "banking:train:3312", "banking:train:3724", "banking:train:4070", "banking:train:476", "banking:train:4882", "banking:train:5075", "banking:train:5332", "banking:train:5445", "banking:train:6096", "banking:train:746", "banking:train:7828", "banking:train:8176", "banking:train:8315", "banking:train:8653", "banking:train:9019", "banking:train:9370", "banking:train:9454", "banking:train:9595", "banking:train:9862", "banking:train:9963", "boolq:validation:1189", "boolq:validation:1320", "boolq:validation:1326", "boolq:validation:1381", "boolq:validation:1410", "boolq:validation:149", "boolq:validation:1513", "boolq:validation:1732", "boolq:validation:1739", "boolq:validation:1741", "boolq:validation:1814", "boolq:validation:1848", "boolq:validation:1940", "boolq:validation:2077", "boolq:validation:2091", "boolq:validation:2148", "boolq:validation:2215", "boolq:validation:2251", "boolq:validation:2283", "boolq:validation:2321", "boolq:validation:2398", "boolq:validation:253", "boolq:validation:2575", "boolq:validation:2907", "boolq:validation:2909", "boolq:validation:2978", "boolq:validation:359", "boolq:validation:394", "boolq:validation:436", "boolq:validation:579", "boolq:validation:807", "boolq:validation:949", "snli:validation:2389", "snli:validation:2485", "snli:validation:255", "snli:validation:3169", "snli:validation:333", "snli:validation:3700", "snli:validation:3709", "snli:validation:4093", "snli:validation:420", "snli:validation:4492", "snli:validation:4516", "snli:validation:4648", "snli:validation:4888", "snli:validation:5185", "snli:validation:5332", "snli:validation:5947", "snli:validation:6124", "snli:validation:660", "snli:validation:6619", "snli:validation:6898", "snli:validation:7387", "snli:validation:7492", "snli:validation:7544", "snli:validation:7996", "snli:validation:8074", "snli:validation:8554", "snli:validation:8971", "snli:validation:9163", "snli:validation:9277", "snli:validation:9490", "snli:validation:9595", "snli:validation:9688" ], "heldout_group_count": 128, "training_group_count": 384 }, { "fold": 2, "temperature_index": 65, "temperature": 3.126079367123956, "training_macro_nll": 0.2825683057725843, "heldout_ids": [ "arc:validation:ARC-Challenge:120", "arc:validation:ARC-Challenge:130", "arc:validation:ARC-Challenge:194", "arc:validation:ARC-Challenge:218", "arc:validation:ARC-Challenge:235", "arc:validation:ARC-Challenge:272", "arc:validation:ARC-Challenge:277", "arc:validation:ARC-Challenge:48", "arc:validation:ARC-Challenge:88", "arc:validation:ARC-Easy:115", "arc:validation:ARC-Easy:12", "arc:validation:ARC-Easy:131", "arc:validation:ARC-Easy:145", "arc:validation:ARC-Easy:179", "arc:validation:ARC-Easy:18", "arc:validation:ARC-Easy:186", "arc:validation:ARC-Easy:191", "arc:validation:ARC-Easy:215", "arc:validation:ARC-Easy:283", "arc:validation:ARC-Easy:314", "arc:validation:ARC-Easy:315", "arc:validation:ARC-Easy:378", "arc:validation:ARC-Easy:384", "arc:validation:ARC-Easy:403", "arc:validation:ARC-Easy:432", "arc:validation:ARC-Easy:439", "arc:validation:ARC-Easy:448", "arc:validation:ARC-Easy:461", "arc:validation:ARC-Easy:469", "arc:validation:ARC-Easy:470", "arc:validation:ARC-Easy:504", "arc:validation:ARC-Easy:561", "banking:train:1142", "banking:train:148", "banking:train:1633", "banking:train:1927", "banking:train:2186", "banking:train:2251", "banking:train:23", "banking:train:3048", "banking:train:3665", "banking:train:3952", "banking:train:3999", "banking:train:4364", "banking:train:4619", "banking:train:5049", "banking:train:5335", "banking:train:549", "banking:train:5957", "banking:train:5985", "banking:train:6132", "banking:train:6240", "banking:train:6488", "banking:train:7065", "banking:train:7749", "banking:train:8470", "banking:train:8761", "banking:train:895", "banking:train:9256", "banking:train:9406", "banking:train:9450", "banking:train:9467", "banking:train:9588", "banking:train:9716", "boolq:validation:1130", "boolq:validation:1167", "boolq:validation:124", "boolq:validation:1255", "boolq:validation:1256", "boolq:validation:1373", "boolq:validation:15", "boolq:validation:155", "boolq:validation:1568", "boolq:validation:16", "boolq:validation:1654", "boolq:validation:1702", "boolq:validation:1792", "boolq:validation:1818", "boolq:validation:210", "boolq:validation:2247", "boolq:validation:2259", "boolq:validation:2362", "boolq:validation:2443", "boolq:validation:2524", "boolq:validation:2553", "boolq:validation:2623", "boolq:validation:2660", "boolq:validation:2669", "boolq:validation:2679", "boolq:validation:2832", "boolq:validation:357", "boolq:validation:453", "boolq:validation:49", "boolq:validation:808", "boolq:validation:90", "boolq:validation:910", "snli:validation:1612", "snli:validation:168", "snli:validation:1693", "snli:validation:2134", "snli:validation:2248", "snli:validation:2422", "snli:validation:2587", "snli:validation:3007", "snli:validation:3097", "snli:validation:3772", "snli:validation:4012", "snli:validation:4168", "snli:validation:4537", "snli:validation:4681", "snli:validation:5050", "snli:validation:5632", "snli:validation:5761", "snli:validation:6055", "snli:validation:6283", "snli:validation:6715", "snli:validation:7117", "snli:validation:7192", "snli:validation:7507", "snli:validation:7594", "snli:validation:7924", "snli:validation:7954", "snli:validation:8035", "snli:validation:8110", "snli:validation:8995", "snli:validation:949", "snli:validation:9523", "snli:validation:9589" ], "heldout_group_count": 128, "training_group_count": 384 }, { "fold": 3, "temperature_index": 67, "temperature": 3.4753616144320576, "training_macro_nll": 0.3223639562301427, "heldout_ids": [ "arc:validation:ARC-Challenge:117", "arc:validation:ARC-Challenge:135", "arc:validation:ARC-Challenge:139", "arc:validation:ARC-Challenge:154", "arc:validation:ARC-Challenge:174", "arc:validation:ARC-Challenge:199", "arc:validation:ARC-Challenge:287", "arc:validation:ARC-Challenge:295", "arc:validation:ARC-Challenge:4", "arc:validation:ARC-Challenge:50", "arc:validation:ARC-Challenge:51", "arc:validation:ARC-Challenge:61", "arc:validation:ARC-Easy:107", "arc:validation:ARC-Easy:121", "arc:validation:ARC-Easy:124", "arc:validation:ARC-Easy:171", "arc:validation:ARC-Easy:235", "arc:validation:ARC-Easy:241", "arc:validation:ARC-Easy:279", "arc:validation:ARC-Easy:289", "arc:validation:ARC-Easy:37", "arc:validation:ARC-Easy:389", "arc:validation:ARC-Easy:394", "arc:validation:ARC-Easy:399", "arc:validation:ARC-Easy:423", "arc:validation:ARC-Easy:492", "arc:validation:ARC-Easy:499", "arc:validation:ARC-Easy:523", "arc:validation:ARC-Easy:533", "arc:validation:ARC-Easy:569", "arc:validation:ARC-Easy:69", "arc:validation:ARC-Easy:93", "banking:train:2049", "banking:train:2350", "banking:train:2497", "banking:train:2506", "banking:train:2720", "banking:train:2788", "banking:train:2967", "banking:train:3502", "banking:train:3726", "banking:train:4014", "banking:train:4735", "banking:train:4803", "banking:train:4917", "banking:train:5477", "banking:train:5851", "banking:train:6551", "banking:train:6839", "banking:train:6926", "banking:train:7809", "banking:train:8092", "banking:train:8186", "banking:train:8215", "banking:train:8667", "banking:train:8752", "banking:train:8763", "banking:train:8851", "banking:train:8917", "banking:train:9145", "banking:train:9279", "banking:train:9676", "banking:train:9837", "banking:train:9967", "boolq:validation:106", "boolq:validation:1065", "boolq:validation:1097", "boolq:validation:1336", "boolq:validation:1350", "boolq:validation:1383", "boolq:validation:1458", "boolq:validation:1641", "boolq:validation:1873", "boolq:validation:1930", "boolq:validation:1989", "boolq:validation:2103", "boolq:validation:234", "boolq:validation:2355", "boolq:validation:2384", "boolq:validation:2440", "boolq:validation:2444", "boolq:validation:2478", "boolq:validation:2872", "boolq:validation:2879", "boolq:validation:2924", "boolq:validation:2971", "boolq:validation:3164", "boolq:validation:4", "boolq:validation:415", "boolq:validation:419", "boolq:validation:428", "boolq:validation:512", "boolq:validation:563", "boolq:validation:643", "boolq:validation:696", "boolq:validation:874", "snli:validation:1102", "snli:validation:1240", "snli:validation:1261", "snli:validation:1738", "snli:validation:1978", "snli:validation:2563", "snli:validation:2659", "snli:validation:2767", "snli:validation:2836", "snli:validation:3232", "snli:validation:3655", "snli:validation:3913", "snli:validation:405", "snli:validation:4051", "snli:validation:4300", "snli:validation:4846", "snli:validation:498", "snli:validation:5224", "snli:validation:5575", "snli:validation:6598", "snli:validation:666", "snli:validation:6925", "snli:validation:6964", "snli:validation:7045", "snli:validation:7351", "snli:validation:7588", "snli:validation:7723", "snli:validation:7750", "snli:validation:7909", "snli:validation:8974", "snli:validation:9385", "snli:validation:9646" ], "heldout_group_count": 128, "training_group_count": 384 } ], "per_family": { "arc": { "count": 128, "score": 0.4118695814660189, "raw_macro_nll": 0.8736200777910155, "accuracy": 0.875 }, "banking": { "count": 128, "score": 0.06946594441189612, "raw_macro_nll": 0.106002592628879, "accuracy": 0.9765625 }, "boolq": { "count": 128, "score": 0.3332506839528764, "raw_macro_nll": 0.7364644020524043, "accuracy": 0.875 }, "snli": { "count": 128, "score": 0.43892853297695694, "raw_macro_nll": 0.7723281848145425, "accuracy": 0.859375 } }, "policy": { "id": "crossfit_temperature_nll_v1", "folds": 4, "seed": 431, "grouping": "source group, nested within task family; groups shuffled within sorted families and assigned round-robin", "temperature_grid": { "count": 101, "log10_min": -1.0, "log10_max": 1.3, "arithmetic": "Python float64" }, "temperature_fit": "minimum macro-family NLL on the other three folds; smallest temperature wins ties", "score": "macro-family mean of all held-out decision NLLs", "deployment_temperature": "fit afresh on reserved calibration only after model selection" } }, "best_selection_score": 0.2552943737691716, "improved": false, "metrics": { "overall": { "n": 512, "accuracy": 0.896484375, "nll": 0.6221038123596042, "brier_multiclass_sum": 0.19169821909519502, "ece_top_label_10_equal_width_bins": 0.08989131019916385, "reliability_bins": [ { "count": 0, "confidence_sum": 0.0, "correct_sum": 0.0 }, { "count": 0, "confidence_sum": 0.0, "correct_sum": 0.0 }, { "count": 0, "confidence_sum": 0.0, "correct_sum": 0.0 }, { "count": 1, "confidence_sum": 0.39359351992607117, "correct_sum": 1.0 }, { "count": 1, "confidence_sum": 0.42377254366874695, "correct_sum": 0.0 }, { "count": 7, "confidence_sum": 3.8932124972343445, "correct_sum": 5.0 }, { "count": 9, "confidence_sum": 5.776679337024689, "correct_sum": 7.0 }, { "count": 9, "confidence_sum": 6.852575600147247, "correct_sum": 7.0 }, { "count": 12, "confidence_sum": 10.222121059894562, "correct_sum": 6.0 }, { "count": 473, "confidence_sum": 471.29451817274094, "correct_sum": 433.0 } ], "accuracy_vs_coverage": { "0.25": { "n": 128, "accuracy": 1.0, "min_confidence": 1.0 }, "0.5": { "n": 256, "accuracy": 0.98828125, "min_confidence": 0.9999780654907227 }, "0.75": { "n": 384, "accuracy": 0.953125, "min_confidence": 0.9980484247207642 }, "1.0": { "n": 512, "accuracy": 0.896484375, "min_confidence": 0.39359351992607117 } } }, "per_family": { "arc": { "n": 128, "accuracy": 0.875, "nll": 0.8736200741430675, "brier_multiclass_sum": 0.21730879220548308, "ece_top_label_10_equal_width_bins": 0.11237058648839593, "reliability_bins": [ { "count": 0, "confidence_sum": 0.0, "correct_sum": 0.0 }, { "count": 0, "confidence_sum": 0.0, "correct_sum": 0.0 }, { "count": 0, "confidence_sum": 0.0, "correct_sum": 0.0 }, { "count": 1, "confidence_sum": 0.39359351992607117, "correct_sum": 1.0 }, { "count": 1, "confidence_sum": 0.42377254366874695, "correct_sum": 0.0 }, { "count": 2, "confidence_sum": 1.1089537739753723, "correct_sum": 2.0 }, { "count": 6, "confidence_sum": 3.8407031297683716, "correct_sum": 4.0 }, { "count": 2, "confidence_sum": 1.4807611107826233, "correct_sum": 2.0 }, { "count": 5, "confidence_sum": 4.257317125797272, "correct_sum": 1.0 }, { "count": 111, "confidence_sum": 110.5263569355011, "correct_sum": 102.0 } ], "accuracy_vs_coverage": { "0.25": { "n": 32, "accuracy": 1.0, "min_confidence": 1.0 }, "0.5": { "n": 64, "accuracy": 0.984375, "min_confidence": 0.9999995231628418 }, "0.75": { "n": 96, "accuracy": 0.9583333333333334, "min_confidence": 0.9975321292877197 }, "1.0": { "n": 128, "accuracy": 0.875, "min_confidence": 0.39359351992607117 } } }, "banking": { "n": 128, "accuracy": 0.9765625, "nll": 0.10600259243186233, "brier_multiclass_sum": 0.04318536812967019, "ece_top_label_10_equal_width_bins": 0.025164072401821613, "reliability_bins": [ { "count": 0, "confidence_sum": 0.0, "correct_sum": 0.0 }, { "count": 0, "confidence_sum": 0.0, "correct_sum": 0.0 }, { "count": 0, "confidence_sum": 0.0, "correct_sum": 0.0 }, { "count": 0, "confidence_sum": 0.0, "correct_sum": 0.0 }, { "count": 0, "confidence_sum": 0.0, "correct_sum": 0.0 }, { "count": 2, "confidence_sum": 1.099687397480011, "correct_sum": 2.0 }, { "count": 0, "confidence_sum": 0.0, "correct_sum": 0.0 }, { "count": 2, "confidence_sum": 1.5330036878585815, "correct_sum": 1.0 }, { "count": 1, "confidence_sum": 0.8130332827568054, "correct_sum": 0.0 }, { "count": 123, "confidence_sum": 122.97465169429779, "correct_sum": 122.0 } ], "accuracy_vs_coverage": { "0.25": { "n": 32, "accuracy": 1.0, "min_confidence": 1.0 }, "0.5": { "n": 64, "accuracy": 1.0, "min_confidence": 1.0 }, "0.75": { "n": 96, "accuracy": 1.0, "min_confidence": 1.0 }, "1.0": { "n": 128, "accuracy": 0.9765625, "min_confidence": 0.545770525932312 } } }, "boolq": { "n": 128, "accuracy": 0.875, "nll": 0.7364644002551142, "brier_multiclass_sum": 0.24680240271797868, "ece_top_label_10_equal_width_bins": 0.13037992408499122, "reliability_bins": [ { "count": 0, "confidence_sum": 0.0, "correct_sum": 0.0 }, { "count": 0, "confidence_sum": 0.0, "correct_sum": 0.0 }, { "count": 0, "confidence_sum": 0.0, "correct_sum": 0.0 }, { "count": 0, "confidence_sum": 0.0, "correct_sum": 0.0 }, { "count": 0, "confidence_sum": 0.0, "correct_sum": 0.0 }, { "count": 1, "confidence_sum": 0.5227815508842468, "correct_sum": 1.0 }, { "count": 2, "confidence_sum": 1.2790811657905579, "correct_sum": 2.0 }, { "count": 2, "confidence_sum": 1.5391781330108643, "correct_sum": 1.0 }, { "count": 3, "confidence_sum": 2.5299994945526123, "correct_sum": 3.0 }, { "count": 120, "confidence_sum": 119.48131436109543, "correct_sum": 105.0 } ], "accuracy_vs_coverage": { "0.25": { "n": 32, "accuracy": 0.96875, "min_confidence": 0.9999909400939941 }, "0.5": { "n": 64, "accuracy": 0.96875, "min_confidence": 0.9999333620071411 }, "0.75": { "n": 96, "accuracy": 0.9270833333333334, "min_confidence": 0.9974094033241272 }, "1.0": { "n": 128, "accuracy": 0.875, "min_confidence": 0.5227815508842468 } } }, "snli": { "n": 128, "accuracy": 0.859375, "nll": 0.7723281826083728, "brier_multiclass_sum": 0.25949631332764816, "ece_top_label_10_equal_width_bins": 0.1339002219028771, "reliability_bins": [ { "count": 0, "confidence_sum": 0.0, "correct_sum": 0.0 }, { "count": 0, "confidence_sum": 0.0, "correct_sum": 0.0 }, { "count": 0, "confidence_sum": 0.0, "correct_sum": 0.0 }, { "count": 0, "confidence_sum": 0.0, "correct_sum": 0.0 }, { "count": 0, "confidence_sum": 0.0, "correct_sum": 0.0 }, { "count": 2, "confidence_sum": 1.1617897748947144, "correct_sum": 0.0 }, { "count": 1, "confidence_sum": 0.6568950414657593, "correct_sum": 1.0 }, { "count": 3, "confidence_sum": 2.2996326684951782, "correct_sum": 3.0 }, { "count": 3, "confidence_sum": 2.6217711567878723, "correct_sum": 2.0 }, { "count": 119, "confidence_sum": 118.31219518184662, "correct_sum": 104.0 } ], "accuracy_vs_coverage": { "0.25": { "n": 32, "accuracy": 0.9375, "min_confidence": 0.9998804330825806 }, "0.5": { "n": 64, "accuracy": 0.953125, "min_confidence": 0.999064028263092 }, "0.75": { "n": 96, "accuracy": 0.8958333333333334, "min_confidence": 0.9940833449363708 }, "1.0": { "n": 128, "accuracy": 0.859375, "min_confidence": 0.5666037201881409 } } } } } }, { "step": 4500, "macro_nll": 0.4348410999061469, "best_macro_nll": 0.25676974726429014, "selection": { "metric": "crossfit_temperature_nll_v1", "score": 0.2841199300221572, "raw_macro_nll": 0.4348410990047701, "accuracy": 0.8984375, "folds": [ { "fold": 0, "temperature_index": 62, "temperature": 2.666858664521479, "training_macro_nll": 0.30822706909779624, "heldout_ids": [ "arc:validation:ARC-Challenge:122", "arc:validation:ARC-Challenge:126", "arc:validation:ARC-Challenge:175", "arc:validation:ARC-Challenge:187", "arc:validation:ARC-Challenge:189", "arc:validation:ARC-Challenge:192", "arc:validation:ARC-Challenge:208", "arc:validation:ARC-Challenge:238", "arc:validation:ARC-Challenge:290", "arc:validation:ARC-Challenge:293", "arc:validation:ARC-Challenge:35", "arc:validation:ARC-Challenge:43", "arc:validation:ARC-Challenge:58", "arc:validation:ARC-Challenge:7", "arc:validation:ARC-Easy:160", "arc:validation:ARC-Easy:201", "arc:validation:ARC-Easy:229", "arc:validation:ARC-Easy:238", "arc:validation:ARC-Easy:253", "arc:validation:ARC-Easy:259", "arc:validation:ARC-Easy:272", "arc:validation:ARC-Easy:298", "arc:validation:ARC-Easy:329", "arc:validation:ARC-Easy:331", "arc:validation:ARC-Easy:337", "arc:validation:ARC-Easy:362", "arc:validation:ARC-Easy:443", "arc:validation:ARC-Easy:500", "arc:validation:ARC-Easy:545", "arc:validation:ARC-Easy:68", "arc:validation:ARC-Easy:7", "arc:validation:ARC-Easy:74", "banking:train:1053", "banking:train:1594", "banking:train:1747", "banking:train:202", "banking:train:2377", "banking:train:2528", "banking:train:2546", "banking:train:2715", "banking:train:3220", "banking:train:3499", "banking:train:3806", "banking:train:3838", "banking:train:3993", "banking:train:4220", "banking:train:5", "banking:train:5192", "banking:train:5394", "banking:train:5846", "banking:train:5863", "banking:train:6032", "banking:train:6508", "banking:train:7579", "banking:train:7600", "banking:train:7987", "banking:train:8155", "banking:train:850", "banking:train:8626", "banking:train:8929", "banking:train:9355", "banking:train:9812", "banking:train:987", "banking:train:9972", "boolq:validation:103", "boolq:validation:1042", "boolq:validation:1059", "boolq:validation:1541", "boolq:validation:1592", "boolq:validation:1927", "boolq:validation:2014", "boolq:validation:2023", "boolq:validation:2098", "boolq:validation:2139", "boolq:validation:2439", "boolq:validation:2485", "boolq:validation:2607", "boolq:validation:2611", "boolq:validation:2720", "boolq:validation:2751", "boolq:validation:282", "boolq:validation:2843", "boolq:validation:2953", "boolq:validation:301", "boolq:validation:3089", "boolq:validation:313", "boolq:validation:3201", "boolq:validation:322", "boolq:validation:3231", "boolq:validation:39", "boolq:validation:461", "boolq:validation:558", "boolq:validation:589", "boolq:validation:731", "boolq:validation:883", "boolq:validation:915", "snli:validation:1156", "snli:validation:1330", "snli:validation:1559", "snli:validation:1735", "snli:validation:1876", "snli:validation:2251", "snli:validation:2257", "snli:validation:2377", "snli:validation:2881", "snli:validation:291", "snli:validation:300", "snli:validation:3049", "snli:validation:3496", "snli:validation:3871", "snli:validation:4414", "snli:validation:5140", "snli:validation:5440", "snli:validation:5806", "snli:validation:6442", "snli:validation:6772", "snli:validation:6949", "snli:validation:7228", "snli:validation:7465", "snli:validation:7510", "snli:validation:7726", "snli:validation:8095", "snli:validation:8143", "snli:validation:8623", "snli:validation:8962", "snli:validation:9451", "snli:validation:9574", "snli:validation:9655" ], "heldout_group_count": 128, "training_group_count": 384 }, { "fold": 1, "temperature_index": 60, "temperature": 2.39883291901949, "training_macro_nll": 0.27078229308933455, "heldout_ids": [ "arc:validation:ARC-Challenge:101", "arc:validation:ARC-Challenge:129", "arc:validation:ARC-Challenge:148", "arc:validation:ARC-Challenge:156", "arc:validation:ARC-Challenge:167", "arc:validation:ARC-Challenge:249", "arc:validation:ARC-Challenge:263", "arc:validation:ARC-Challenge:266", "arc:validation:ARC-Challenge:267", "arc:validation:ARC-Challenge:28", "arc:validation:ARC-Challenge:281", "arc:validation:ARC-Challenge:57", "arc:validation:ARC-Challenge:63", "arc:validation:ARC-Challenge:64", "arc:validation:ARC-Challenge:86", "arc:validation:ARC-Challenge:99", "arc:validation:ARC-Easy:106", "arc:validation:ARC-Easy:151", "arc:validation:ARC-Easy:157", "arc:validation:ARC-Easy:166", "arc:validation:ARC-Easy:202", "arc:validation:ARC-Easy:219", "arc:validation:ARC-Easy:224", "arc:validation:ARC-Easy:322", "arc:validation:ARC-Easy:344", "arc:validation:ARC-Easy:382", "arc:validation:ARC-Easy:409", "arc:validation:ARC-Easy:455", "arc:validation:ARC-Easy:50", "arc:validation:ARC-Easy:538", "arc:validation:ARC-Easy:544", "arc:validation:ARC-Easy:552", "banking:train:1050", "banking:train:1131", "banking:train:1190", "banking:train:124", "banking:train:137", "banking:train:1493", "banking:train:1498", "banking:train:1624", "banking:train:2614", "banking:train:2739", "banking:train:3055", "banking:train:306", "banking:train:3312", "banking:train:3724", "banking:train:4070", "banking:train:476", "banking:train:4882", "banking:train:5075", "banking:train:5332", "banking:train:5445", "banking:train:6096", "banking:train:746", "banking:train:7828", "banking:train:8176", "banking:train:8315", "banking:train:8653", "banking:train:9019", "banking:train:9370", "banking:train:9454", "banking:train:9595", "banking:train:9862", "banking:train:9963", "boolq:validation:1189", "boolq:validation:1320", "boolq:validation:1326", "boolq:validation:1381", "boolq:validation:1410", "boolq:validation:149", "boolq:validation:1513", "boolq:validation:1732", "boolq:validation:1739", "boolq:validation:1741", "boolq:validation:1814", "boolq:validation:1848", "boolq:validation:1940", "boolq:validation:2077", "boolq:validation:2091", "boolq:validation:2148", "boolq:validation:2215", "boolq:validation:2251", "boolq:validation:2283", "boolq:validation:2321", "boolq:validation:2398", "boolq:validation:253", "boolq:validation:2575", "boolq:validation:2907", "boolq:validation:2909", "boolq:validation:2978", "boolq:validation:359", "boolq:validation:394", "boolq:validation:436", "boolq:validation:579", "boolq:validation:807", "boolq:validation:949", "snli:validation:2389", "snli:validation:2485", "snli:validation:255", "snli:validation:3169", "snli:validation:333", "snli:validation:3700", "snli:validation:3709", "snli:validation:4093", "snli:validation:420", "snli:validation:4492", "snli:validation:4516", "snli:validation:4648", "snli:validation:4888", "snli:validation:5185", "snli:validation:5332", "snli:validation:5947", "snli:validation:6124", "snli:validation:660", "snli:validation:6619", "snli:validation:6898", "snli:validation:7387", "snli:validation:7492", "snli:validation:7544", "snli:validation:7996", "snli:validation:8074", "snli:validation:8554", "snli:validation:8971", "snli:validation:9163", "snli:validation:9277", "snli:validation:9490", "snli:validation:9595", "snli:validation:9688" ], "heldout_group_count": 128, "training_group_count": 384 }, { "fold": 2, "temperature_index": 60, "temperature": 2.39883291901949, "training_macro_nll": 0.25779374717908343, "heldout_ids": [ "arc:validation:ARC-Challenge:120", "arc:validation:ARC-Challenge:130", "arc:validation:ARC-Challenge:194", "arc:validation:ARC-Challenge:218", "arc:validation:ARC-Challenge:235", "arc:validation:ARC-Challenge:272", "arc:validation:ARC-Challenge:277", "arc:validation:ARC-Challenge:48", "arc:validation:ARC-Challenge:88", "arc:validation:ARC-Easy:115", "arc:validation:ARC-Easy:12", "arc:validation:ARC-Easy:131", "arc:validation:ARC-Easy:145", "arc:validation:ARC-Easy:179", "arc:validation:ARC-Easy:18", "arc:validation:ARC-Easy:186", "arc:validation:ARC-Easy:191", "arc:validation:ARC-Easy:215", "arc:validation:ARC-Easy:283", "arc:validation:ARC-Easy:314", "arc:validation:ARC-Easy:315", "arc:validation:ARC-Easy:378", "arc:validation:ARC-Easy:384", "arc:validation:ARC-Easy:403", "arc:validation:ARC-Easy:432", "arc:validation:ARC-Easy:439", "arc:validation:ARC-Easy:448", "arc:validation:ARC-Easy:461", "arc:validation:ARC-Easy:469", "arc:validation:ARC-Easy:470", "arc:validation:ARC-Easy:504", "arc:validation:ARC-Easy:561", "banking:train:1142", "banking:train:148", "banking:train:1633", "banking:train:1927", "banking:train:2186", "banking:train:2251", "banking:train:23", "banking:train:3048", "banking:train:3665", "banking:train:3952", "banking:train:3999", "banking:train:4364", "banking:train:4619", "banking:train:5049", "banking:train:5335", "banking:train:549", "banking:train:5957", "banking:train:5985", "banking:train:6132", "banking:train:6240", "banking:train:6488", "banking:train:7065", "banking:train:7749", "banking:train:8470", "banking:train:8761", "banking:train:895", "banking:train:9256", "banking:train:9406", "banking:train:9450", "banking:train:9467", "banking:train:9588", "banking:train:9716", "boolq:validation:1130", "boolq:validation:1167", "boolq:validation:124", "boolq:validation:1255", "boolq:validation:1256", "boolq:validation:1373", "boolq:validation:15", "boolq:validation:155", "boolq:validation:1568", "boolq:validation:16", "boolq:validation:1654", "boolq:validation:1702", "boolq:validation:1792", "boolq:validation:1818", "boolq:validation:210", "boolq:validation:2247", "boolq:validation:2259", "boolq:validation:2362", "boolq:validation:2443", "boolq:validation:2524", "boolq:validation:2553", "boolq:validation:2623", "boolq:validation:2660", "boolq:validation:2669", "boolq:validation:2679", "boolq:validation:2832", "boolq:validation:357", "boolq:validation:453", "boolq:validation:49", "boolq:validation:808", "boolq:validation:90", "boolq:validation:910", "snli:validation:1612", "snli:validation:168", "snli:validation:1693", "snli:validation:2134", "snli:validation:2248", "snli:validation:2422", "snli:validation:2587", "snli:validation:3007", "snli:validation:3097", "snli:validation:3772", "snli:validation:4012", "snli:validation:4168", "snli:validation:4537", "snli:validation:4681", "snli:validation:5050", "snli:validation:5632", "snli:validation:5761", "snli:validation:6055", "snli:validation:6283", "snli:validation:6715", "snli:validation:7117", "snli:validation:7192", "snli:validation:7507", "snli:validation:7594", "snli:validation:7924", "snli:validation:7954", "snli:validation:8035", "snli:validation:8110", "snli:validation:8995", "snli:validation:949", "snli:validation:9523", "snli:validation:9589" ], "heldout_group_count": 128, "training_group_count": 384 }, { "fold": 3, "temperature_index": 61, "temperature": 2.5292979964461435, "training_macro_nll": 0.2911054608892844, "heldout_ids": [ "arc:validation:ARC-Challenge:117", "arc:validation:ARC-Challenge:135", "arc:validation:ARC-Challenge:139", "arc:validation:ARC-Challenge:154", "arc:validation:ARC-Challenge:174", "arc:validation:ARC-Challenge:199", "arc:validation:ARC-Challenge:287", "arc:validation:ARC-Challenge:295", "arc:validation:ARC-Challenge:4", "arc:validation:ARC-Challenge:50", "arc:validation:ARC-Challenge:51", "arc:validation:ARC-Challenge:61", "arc:validation:ARC-Easy:107", "arc:validation:ARC-Easy:121", "arc:validation:ARC-Easy:124", "arc:validation:ARC-Easy:171", "arc:validation:ARC-Easy:235", "arc:validation:ARC-Easy:241", "arc:validation:ARC-Easy:279", "arc:validation:ARC-Easy:289", "arc:validation:ARC-Easy:37", "arc:validation:ARC-Easy:389", "arc:validation:ARC-Easy:394", "arc:validation:ARC-Easy:399", "arc:validation:ARC-Easy:423", "arc:validation:ARC-Easy:492", "arc:validation:ARC-Easy:499", "arc:validation:ARC-Easy:523", "arc:validation:ARC-Easy:533", "arc:validation:ARC-Easy:569", "arc:validation:ARC-Easy:69", "arc:validation:ARC-Easy:93", "banking:train:2049", "banking:train:2350", "banking:train:2497", "banking:train:2506", "banking:train:2720", "banking:train:2788", "banking:train:2967", "banking:train:3502", "banking:train:3726", "banking:train:4014", "banking:train:4735", "banking:train:4803", "banking:train:4917", "banking:train:5477", "banking:train:5851", "banking:train:6551", "banking:train:6839", "banking:train:6926", "banking:train:7809", "banking:train:8092", "banking:train:8186", "banking:train:8215", "banking:train:8667", "banking:train:8752", "banking:train:8763", "banking:train:8851", "banking:train:8917", "banking:train:9145", "banking:train:9279", "banking:train:9676", "banking:train:9837", "banking:train:9967", "boolq:validation:106", "boolq:validation:1065", "boolq:validation:1097", "boolq:validation:1336", "boolq:validation:1350", "boolq:validation:1383", "boolq:validation:1458", "boolq:validation:1641", "boolq:validation:1873", "boolq:validation:1930", "boolq:validation:1989", "boolq:validation:2103", "boolq:validation:234", "boolq:validation:2355", "boolq:validation:2384", "boolq:validation:2440", "boolq:validation:2444", "boolq:validation:2478", "boolq:validation:2872", "boolq:validation:2879", "boolq:validation:2924", "boolq:validation:2971", "boolq:validation:3164", "boolq:validation:4", "boolq:validation:415", "boolq:validation:419", "boolq:validation:428", "boolq:validation:512", "boolq:validation:563", "boolq:validation:643", "boolq:validation:696", "boolq:validation:874", "snli:validation:1102", "snli:validation:1240", "snli:validation:1261", "snli:validation:1738", "snli:validation:1978", "snli:validation:2563", "snli:validation:2659", "snli:validation:2767", "snli:validation:2836", "snli:validation:3232", "snli:validation:3655", "snli:validation:3913", "snli:validation:405", "snli:validation:4051", "snli:validation:4300", "snli:validation:4846", "snli:validation:498", "snli:validation:5224", "snli:validation:5575", "snli:validation:6598", "snli:validation:666", "snli:validation:6925", "snli:validation:6964", "snli:validation:7045", "snli:validation:7351", "snli:validation:7588", "snli:validation:7723", "snli:validation:7750", "snli:validation:7909", "snli:validation:8974", "snli:validation:9385", "snli:validation:9646" ], "heldout_group_count": 128, "training_group_count": 384 } ], "per_family": { "arc": { "count": 128, "score": 0.45409842583654353, "raw_macro_nll": 0.7961190826926177, "accuracy": 0.8515625 }, "banking": { "count": 128, "score": 0.055047587673440465, "raw_macro_nll": 0.09841930191488779, "accuracy": 0.9765625 }, "boolq": { "count": 128, "score": 0.3088651912550437, "raw_macro_nll": 0.5010539159559688, "accuracy": 0.8515625 }, "snli": { "count": 128, "score": 0.318468515323601, "raw_macro_nll": 0.3437720954556061, "accuracy": 0.9140625 } }, "policy": { "id": "crossfit_temperature_nll_v1", "folds": 4, "seed": 431, "grouping": "source group, nested within task family; groups shuffled within sorted families and assigned round-robin", "temperature_grid": { "count": 101, "log10_min": -1.0, "log10_max": 1.3, "arithmetic": "Python float64" }, "temperature_fit": "minimum macro-family NLL on the other three folds; smallest temperature wins ties", "score": "macro-family mean of all held-out decision NLLs", "deployment_temperature": "fit afresh on reserved calibration only after model selection" } }, "best_selection_score": 0.2552943737691716, "improved": false, "metrics": { "overall": { "n": 512, "accuracy": 0.8984375, "nll": 0.4348410999061469, "brier_multiclass_sum": 0.16255039658970658, "ece_top_label_10_equal_width_bins": 0.0713859559618868, "reliability_bins": [ { "count": 0, "confidence_sum": 0.0, "correct_sum": 0.0 }, { "count": 0, "confidence_sum": 0.0, "correct_sum": 0.0 }, { "count": 0, "confidence_sum": 0.0, "correct_sum": 0.0 }, { "count": 0, "confidence_sum": 0.0, "correct_sum": 0.0 }, { "count": 1, "confidence_sum": 0.4954971969127655, "correct_sum": 1.0 }, { "count": 13, "confidence_sum": 7.076289176940918, "correct_sum": 4.0 }, { "count": 9, "confidence_sum": 5.850665271282196, "correct_sum": 3.0 }, { "count": 4, "confidence_sum": 2.993383467197418, "correct_sum": 2.0 }, { "count": 20, "confidence_sum": 17.163459360599518, "correct_sum": 14.0 }, { "count": 465, "confidence_sum": 461.96130937337875, "correct_sum": 436.0 } ], "accuracy_vs_coverage": { "0.25": { "n": 128, "accuracy": 1.0, "min_confidence": 1.0 }, "0.5": { "n": 256, "accuracy": 0.99609375, "min_confidence": 0.9998538494110107 }, "0.75": { "n": 384, "accuracy": 0.9713541666666666, "min_confidence": 0.989825427532196 }, "1.0": { "n": 512, "accuracy": 0.8984375, "min_confidence": 0.4954971969127655 } } }, "per_family": { "arc": { "n": 128, "accuracy": 0.8515625, "nll": 0.7961190888847145, "brier_multiclass_sum": 0.2236221681839995, "ece_top_label_10_equal_width_bins": 0.11635206383652985, "reliability_bins": [ { "count": 0, "confidence_sum": 0.0, "correct_sum": 0.0 }, { "count": 0, "confidence_sum": 0.0, "correct_sum": 0.0 }, { "count": 0, "confidence_sum": 0.0, "correct_sum": 0.0 }, { "count": 0, "confidence_sum": 0.0, "correct_sum": 0.0 }, { "count": 1, "confidence_sum": 0.4954971969127655, "correct_sum": 1.0 }, { "count": 6, "confidence_sum": 3.2659605145454407, "correct_sum": 0.0 }, { "count": 1, "confidence_sum": 0.6727883815765381, "correct_sum": 0.0 }, { "count": 2, "confidence_sum": 1.5027006268501282, "correct_sum": 2.0 }, { "count": 10, "confidence_sum": 8.412083506584167, "correct_sum": 6.0 }, { "count": 108, "confidence_sum": 107.54042959213257, "correct_sum": 100.0 } ], "accuracy_vs_coverage": { "0.25": { "n": 32, "accuracy": 1.0, "min_confidence": 1.0 }, "0.5": { "n": 64, "accuracy": 0.984375, "min_confidence": 0.9999865293502808 }, "0.75": { "n": 96, "accuracy": 0.9583333333333334, "min_confidence": 0.985666811466217 }, "1.0": { "n": 128, "accuracy": 0.8515625, "min_confidence": 0.4954971969127655 } } }, "banking": { "n": 128, "accuracy": 0.9765625, "nll": 0.09841930049332331, "brier_multiclass_sum": 0.042800634513240954, "ece_top_label_10_equal_width_bins": 0.020601469557732344, "reliability_bins": [ { "count": 0, "confidence_sum": 0.0, "correct_sum": 0.0 }, { "count": 0, "confidence_sum": 0.0, "correct_sum": 0.0 }, { "count": 0, "confidence_sum": 0.0, "correct_sum": 0.0 }, { "count": 0, "confidence_sum": 0.0, "correct_sum": 0.0 }, { "count": 0, "confidence_sum": 0.0, "correct_sum": 0.0 }, { "count": 0, "confidence_sum": 0.0, "correct_sum": 0.0 }, { "count": 0, "confidence_sum": 0.0, "correct_sum": 0.0 }, { "count": 0, "confidence_sum": 0.0, "correct_sum": 0.0 }, { "count": 1, "confidence_sum": 0.8777487277984619, "correct_sum": 0.0 }, { "count": 127, "confidence_sum": 126.75923937559128, "correct_sum": 125.0 } ], "accuracy_vs_coverage": { "0.25": { "n": 32, "accuracy": 1.0, "min_confidence": 1.0 }, "0.5": { "n": 64, "accuracy": 1.0, "min_confidence": 1.0 }, "0.75": { "n": 96, "accuracy": 1.0, "min_confidence": 1.0 }, "1.0": { "n": 128, "accuracy": 0.9765625, "min_confidence": 0.8777487277984619 } } }, "boolq": { "n": 128, "accuracy": 0.8515625, "nll": 0.5010539169356036, "brier_multiclass_sum": 0.22973198097830894, "ece_top_label_10_equal_width_bins": 0.10703934449702501, "reliability_bins": [ { "count": 0, "confidence_sum": 0.0, "correct_sum": 0.0 }, { "count": 0, "confidence_sum": 0.0, "correct_sum": 0.0 }, { "count": 0, "confidence_sum": 0.0, "correct_sum": 0.0 }, { "count": 0, "confidence_sum": 0.0, "correct_sum": 0.0 }, { "count": 0, "confidence_sum": 0.0, "correct_sum": 0.0 }, { "count": 6, "confidence_sum": 3.307970881462097, "correct_sum": 3.0 }, { "count": 5, "confidence_sum": 3.20867782831192, "correct_sum": 2.0 }, { "count": 2, "confidence_sum": 1.49068284034729, "correct_sum": 0.0 }, { "count": 3, "confidence_sum": 2.652133047580719, "correct_sum": 3.0 }, { "count": 112, "confidence_sum": 111.34583759307861, "correct_sum": 101.0 } ], "accuracy_vs_coverage": { "0.25": { "n": 32, "accuracy": 1.0, "min_confidence": 0.9999068975448608 }, "0.5": { "n": 64, "accuracy": 1.0, "min_confidence": 0.9995142221450806 }, "0.75": { "n": 96, "accuracy": 0.9583333333333334, "min_confidence": 0.9929304122924805 }, "1.0": { "n": 128, "accuracy": 0.8515625, "min_confidence": 0.5254567861557007 } } }, "snli": { "n": 128, "accuracy": 0.9140625, "nll": 0.34377209331094605, "brier_multiclass_sum": 0.15404680268327697, "ece_top_label_10_equal_width_bins": 0.06253232946619391, "reliability_bins": [ { "count": 0, "confidence_sum": 0.0, "correct_sum": 0.0 }, { "count": 0, "confidence_sum": 0.0, "correct_sum": 0.0 }, { "count": 0, "confidence_sum": 0.0, "correct_sum": 0.0 }, { "count": 0, "confidence_sum": 0.0, "correct_sum": 0.0 }, { "count": 0, "confidence_sum": 0.0, "correct_sum": 0.0 }, { "count": 1, "confidence_sum": 0.5023577809333801, "correct_sum": 1.0 }, { "count": 3, "confidence_sum": 1.9691990613937378, "correct_sum": 1.0 }, { "count": 0, "confidence_sum": 0.0, "correct_sum": 0.0 }, { "count": 6, "confidence_sum": 5.221494078636169, "correct_sum": 5.0 }, { "count": 118, "confidence_sum": 116.3158028125763, "correct_sum": 110.0 } ], "accuracy_vs_coverage": { "0.25": { "n": 32, "accuracy": 1.0, "min_confidence": 0.9988596439361572 }, "0.5": { "n": 64, "accuracy": 1.0, "min_confidence": 0.9949784874916077 }, "0.75": { "n": 96, "accuracy": 0.9479166666666666, "min_confidence": 0.9736598134040833 }, "1.0": { "n": 128, "accuracy": 0.9140625, "min_confidence": 0.5023577809333801 } } } } } }, { "step": 5000, "macro_nll": 0.2879291938128496, "best_macro_nll": 0.25676974726429014, "selection": { "metric": "crossfit_temperature_nll_v1", "score": 0.279832908363464, "raw_macro_nll": 0.2879291926537506, "accuracy": 0.90625, "folds": [ { "fold": 0, "temperature_index": 48, "temperature": 1.2705741052085413, "training_macro_nll": 0.2949829510321852, "heldout_ids": [ "arc:validation:ARC-Challenge:122", "arc:validation:ARC-Challenge:126", "arc:validation:ARC-Challenge:175", "arc:validation:ARC-Challenge:187", "arc:validation:ARC-Challenge:189", "arc:validation:ARC-Challenge:192", "arc:validation:ARC-Challenge:208", "arc:validation:ARC-Challenge:238", "arc:validation:ARC-Challenge:290", "arc:validation:ARC-Challenge:293", "arc:validation:ARC-Challenge:35", "arc:validation:ARC-Challenge:43", "arc:validation:ARC-Challenge:58", "arc:validation:ARC-Challenge:7", "arc:validation:ARC-Easy:160", "arc:validation:ARC-Easy:201", "arc:validation:ARC-Easy:229", "arc:validation:ARC-Easy:238", "arc:validation:ARC-Easy:253", "arc:validation:ARC-Easy:259", "arc:validation:ARC-Easy:272", "arc:validation:ARC-Easy:298", "arc:validation:ARC-Easy:329", "arc:validation:ARC-Easy:331", "arc:validation:ARC-Easy:337", "arc:validation:ARC-Easy:362", "arc:validation:ARC-Easy:443", "arc:validation:ARC-Easy:500", "arc:validation:ARC-Easy:545", "arc:validation:ARC-Easy:68", "arc:validation:ARC-Easy:7", "arc:validation:ARC-Easy:74", "banking:train:1053", "banking:train:1594", "banking:train:1747", "banking:train:202", "banking:train:2377", "banking:train:2528", "banking:train:2546", "banking:train:2715", "banking:train:3220", "banking:train:3499", "banking:train:3806", "banking:train:3838", "banking:train:3993", "banking:train:4220", "banking:train:5", "banking:train:5192", "banking:train:5394", "banking:train:5846", "banking:train:5863", "banking:train:6032", "banking:train:6508", "banking:train:7579", "banking:train:7600", "banking:train:7987", "banking:train:8155", "banking:train:850", "banking:train:8626", "banking:train:8929", "banking:train:9355", "banking:train:9812", "banking:train:987", "banking:train:9972", "boolq:validation:103", "boolq:validation:1042", "boolq:validation:1059", "boolq:validation:1541", "boolq:validation:1592", "boolq:validation:1927", "boolq:validation:2014", "boolq:validation:2023", "boolq:validation:2098", "boolq:validation:2139", "boolq:validation:2439", "boolq:validation:2485", "boolq:validation:2607", "boolq:validation:2611", "boolq:validation:2720", "boolq:validation:2751", "boolq:validation:282", "boolq:validation:2843", "boolq:validation:2953", "boolq:validation:301", "boolq:validation:3089", "boolq:validation:313", "boolq:validation:3201", "boolq:validation:322", "boolq:validation:3231", "boolq:validation:39", "boolq:validation:461", "boolq:validation:558", "boolq:validation:589", "boolq:validation:731", "boolq:validation:883", "boolq:validation:915", "snli:validation:1156", "snli:validation:1330", "snli:validation:1559", "snli:validation:1735", "snli:validation:1876", "snli:validation:2251", "snli:validation:2257", "snli:validation:2377", "snli:validation:2881", "snli:validation:291", "snli:validation:300", "snli:validation:3049", "snli:validation:3496", "snli:validation:3871", "snli:validation:4414", "snli:validation:5140", "snli:validation:5440", "snli:validation:5806", "snli:validation:6442", "snli:validation:6772", "snli:validation:6949", "snli:validation:7228", "snli:validation:7465", "snli:validation:7510", "snli:validation:7726", "snli:validation:8095", "snli:validation:8143", "snli:validation:8623", "snli:validation:8962", "snli:validation:9451", "snli:validation:9574", "snli:validation:9655" ], "heldout_group_count": 128, "training_group_count": 384 }, { "fold": 1, "temperature_index": 48, "temperature": 1.2705741052085413, "training_macro_nll": 0.26910043519983107, "heldout_ids": [ "arc:validation:ARC-Challenge:101", "arc:validation:ARC-Challenge:129", "arc:validation:ARC-Challenge:148", "arc:validation:ARC-Challenge:156", "arc:validation:ARC-Challenge:167", "arc:validation:ARC-Challenge:249", "arc:validation:ARC-Challenge:263", "arc:validation:ARC-Challenge:266", "arc:validation:ARC-Challenge:267", "arc:validation:ARC-Challenge:28", "arc:validation:ARC-Challenge:281", "arc:validation:ARC-Challenge:57", "arc:validation:ARC-Challenge:63", "arc:validation:ARC-Challenge:64", "arc:validation:ARC-Challenge:86", "arc:validation:ARC-Challenge:99", "arc:validation:ARC-Easy:106", "arc:validation:ARC-Easy:151", "arc:validation:ARC-Easy:157", "arc:validation:ARC-Easy:166", "arc:validation:ARC-Easy:202", "arc:validation:ARC-Easy:219", "arc:validation:ARC-Easy:224", "arc:validation:ARC-Easy:322", "arc:validation:ARC-Easy:344", "arc:validation:ARC-Easy:382", "arc:validation:ARC-Easy:409", "arc:validation:ARC-Easy:455", "arc:validation:ARC-Easy:50", "arc:validation:ARC-Easy:538", "arc:validation:ARC-Easy:544", "arc:validation:ARC-Easy:552", "banking:train:1050", "banking:train:1131", "banking:train:1190", "banking:train:124", "banking:train:137", "banking:train:1493", "banking:train:1498", "banking:train:1624", "banking:train:2614", "banking:train:2739", "banking:train:3055", "banking:train:306", "banking:train:3312", "banking:train:3724", "banking:train:4070", "banking:train:476", "banking:train:4882", "banking:train:5075", "banking:train:5332", "banking:train:5445", "banking:train:6096", "banking:train:746", "banking:train:7828", "banking:train:8176", "banking:train:8315", "banking:train:8653", "banking:train:9019", "banking:train:9370", "banking:train:9454", "banking:train:9595", "banking:train:9862", "banking:train:9963", "boolq:validation:1189", "boolq:validation:1320", "boolq:validation:1326", "boolq:validation:1381", "boolq:validation:1410", "boolq:validation:149", "boolq:validation:1513", "boolq:validation:1732", "boolq:validation:1739", "boolq:validation:1741", "boolq:validation:1814", "boolq:validation:1848", "boolq:validation:1940", "boolq:validation:2077", "boolq:validation:2091", "boolq:validation:2148", "boolq:validation:2215", "boolq:validation:2251", "boolq:validation:2283", "boolq:validation:2321", "boolq:validation:2398", "boolq:validation:253", "boolq:validation:2575", "boolq:validation:2907", "boolq:validation:2909", "boolq:validation:2978", "boolq:validation:359", "boolq:validation:394", "boolq:validation:436", "boolq:validation:579", "boolq:validation:807", "boolq:validation:949", "snli:validation:2389", "snli:validation:2485", "snli:validation:255", "snli:validation:3169", "snli:validation:333", "snli:validation:3700", "snli:validation:3709", "snli:validation:4093", "snli:validation:420", "snli:validation:4492", "snli:validation:4516", "snli:validation:4648", "snli:validation:4888", "snli:validation:5185", "snli:validation:5332", "snli:validation:5947", "snli:validation:6124", "snli:validation:660", "snli:validation:6619", "snli:validation:6898", "snli:validation:7387", "snli:validation:7492", "snli:validation:7544", "snli:validation:7996", "snli:validation:8074", "snli:validation:8554", "snli:validation:8971", "snli:validation:9163", "snli:validation:9277", "snli:validation:9490", "snli:validation:9595", "snli:validation:9688" ], "heldout_group_count": 128, "training_group_count": 384 }, { "fold": 2, "temperature_index": 47, "temperature": 1.2050359403717974, "training_macro_nll": 0.25473528099447806, "heldout_ids": [ "arc:validation:ARC-Challenge:120", "arc:validation:ARC-Challenge:130", "arc:validation:ARC-Challenge:194", "arc:validation:ARC-Challenge:218", "arc:validation:ARC-Challenge:235", "arc:validation:ARC-Challenge:272", "arc:validation:ARC-Challenge:277", "arc:validation:ARC-Challenge:48", "arc:validation:ARC-Challenge:88", "arc:validation:ARC-Easy:115", "arc:validation:ARC-Easy:12", "arc:validation:ARC-Easy:131", "arc:validation:ARC-Easy:145", "arc:validation:ARC-Easy:179", "arc:validation:ARC-Easy:18", "arc:validation:ARC-Easy:186", "arc:validation:ARC-Easy:191", "arc:validation:ARC-Easy:215", "arc:validation:ARC-Easy:283", "arc:validation:ARC-Easy:314", "arc:validation:ARC-Easy:315", "arc:validation:ARC-Easy:378", "arc:validation:ARC-Easy:384", "arc:validation:ARC-Easy:403", "arc:validation:ARC-Easy:432", "arc:validation:ARC-Easy:439", "arc:validation:ARC-Easy:448", "arc:validation:ARC-Easy:461", "arc:validation:ARC-Easy:469", "arc:validation:ARC-Easy:470", "arc:validation:ARC-Easy:504", "arc:validation:ARC-Easy:561", "banking:train:1142", "banking:train:148", "banking:train:1633", "banking:train:1927", "banking:train:2186", "banking:train:2251", "banking:train:23", "banking:train:3048", "banking:train:3665", "banking:train:3952", "banking:train:3999", "banking:train:4364", "banking:train:4619", "banking:train:5049", "banking:train:5335", "banking:train:549", "banking:train:5957", "banking:train:5985", "banking:train:6132", "banking:train:6240", "banking:train:6488", "banking:train:7065", "banking:train:7749", "banking:train:8470", "banking:train:8761", "banking:train:895", "banking:train:9256", "banking:train:9406", "banking:train:9450", "banking:train:9467", "banking:train:9588", "banking:train:9716", "boolq:validation:1130", "boolq:validation:1167", "boolq:validation:124", "boolq:validation:1255", "boolq:validation:1256", "boolq:validation:1373", "boolq:validation:15", "boolq:validation:155", "boolq:validation:1568", "boolq:validation:16", "boolq:validation:1654", "boolq:validation:1702", "boolq:validation:1792", "boolq:validation:1818", "boolq:validation:210", "boolq:validation:2247", "boolq:validation:2259", "boolq:validation:2362", "boolq:validation:2443", "boolq:validation:2524", "boolq:validation:2553", "boolq:validation:2623", "boolq:validation:2660", "boolq:validation:2669", "boolq:validation:2679", "boolq:validation:2832", "boolq:validation:357", "boolq:validation:453", "boolq:validation:49", "boolq:validation:808", "boolq:validation:90", "boolq:validation:910", "snli:validation:1612", "snli:validation:168", "snli:validation:1693", "snli:validation:2134", "snli:validation:2248", "snli:validation:2422", "snli:validation:2587", "snli:validation:3007", "snli:validation:3097", "snli:validation:3772", "snli:validation:4012", "snli:validation:4168", "snli:validation:4537", "snli:validation:4681", "snli:validation:5050", "snli:validation:5632", "snli:validation:5761", "snli:validation:6055", "snli:validation:6283", "snli:validation:6715", "snli:validation:7117", "snli:validation:7192", "snli:validation:7507", "snli:validation:7594", "snli:validation:7924", "snli:validation:7954", "snli:validation:8035", "snli:validation:8110", "snli:validation:8995", "snli:validation:949", "snli:validation:9523", "snli:validation:9589" ], "heldout_group_count": 128, "training_group_count": 384 }, { "fold": 3, "temperature_index": 49, "temperature": 1.3396766874259345, "training_macro_nll": 0.2946301125911753, "heldout_ids": [ "arc:validation:ARC-Challenge:117", "arc:validation:ARC-Challenge:135", "arc:validation:ARC-Challenge:139", "arc:validation:ARC-Challenge:154", "arc:validation:ARC-Challenge:174", "arc:validation:ARC-Challenge:199", "arc:validation:ARC-Challenge:287", "arc:validation:ARC-Challenge:295", "arc:validation:ARC-Challenge:4", "arc:validation:ARC-Challenge:50", "arc:validation:ARC-Challenge:51", "arc:validation:ARC-Challenge:61", "arc:validation:ARC-Easy:107", "arc:validation:ARC-Easy:121", "arc:validation:ARC-Easy:124", "arc:validation:ARC-Easy:171", "arc:validation:ARC-Easy:235", "arc:validation:ARC-Easy:241", "arc:validation:ARC-Easy:279", "arc:validation:ARC-Easy:289", "arc:validation:ARC-Easy:37", "arc:validation:ARC-Easy:389", "arc:validation:ARC-Easy:394", "arc:validation:ARC-Easy:399", "arc:validation:ARC-Easy:423", "arc:validation:ARC-Easy:492", "arc:validation:ARC-Easy:499", "arc:validation:ARC-Easy:523", "arc:validation:ARC-Easy:533", "arc:validation:ARC-Easy:569", "arc:validation:ARC-Easy:69", "arc:validation:ARC-Easy:93", "banking:train:2049", "banking:train:2350", "banking:train:2497", "banking:train:2506", "banking:train:2720", "banking:train:2788", "banking:train:2967", "banking:train:3502", "banking:train:3726", "banking:train:4014", "banking:train:4735", "banking:train:4803", "banking:train:4917", "banking:train:5477", "banking:train:5851", "banking:train:6551", "banking:train:6839", "banking:train:6926", "banking:train:7809", "banking:train:8092", "banking:train:8186", "banking:train:8215", "banking:train:8667", "banking:train:8752", "banking:train:8763", "banking:train:8851", "banking:train:8917", "banking:train:9145", "banking:train:9279", "banking:train:9676", "banking:train:9837", "banking:train:9967", "boolq:validation:106", "boolq:validation:1065", "boolq:validation:1097", "boolq:validation:1336", "boolq:validation:1350", "boolq:validation:1383", "boolq:validation:1458", "boolq:validation:1641", "boolq:validation:1873", "boolq:validation:1930", "boolq:validation:1989", "boolq:validation:2103", "boolq:validation:234", "boolq:validation:2355", "boolq:validation:2384", "boolq:validation:2440", "boolq:validation:2444", "boolq:validation:2478", "boolq:validation:2872", "boolq:validation:2879", "boolq:validation:2924", "boolq:validation:2971", "boolq:validation:3164", "boolq:validation:4", "boolq:validation:415", "boolq:validation:419", "boolq:validation:428", "boolq:validation:512", "boolq:validation:563", "boolq:validation:643", "boolq:validation:696", "boolq:validation:874", "snli:validation:1102", "snli:validation:1240", "snli:validation:1261", "snli:validation:1738", "snli:validation:1978", "snli:validation:2563", "snli:validation:2659", "snli:validation:2767", "snli:validation:2836", "snli:validation:3232", "snli:validation:3655", "snli:validation:3913", "snli:validation:405", "snli:validation:4051", "snli:validation:4300", "snli:validation:4846", "snli:validation:498", "snli:validation:5224", "snli:validation:5575", "snli:validation:6598", "snli:validation:666", "snli:validation:6925", "snli:validation:6964", "snli:validation:7045", "snli:validation:7351", "snli:validation:7588", "snli:validation:7723", "snli:validation:7750", "snli:validation:7909", "snli:validation:8974", "snli:validation:9385", "snli:validation:9646" ], "heldout_group_count": 128, "training_group_count": 384 } ], "per_family": { "arc": { "count": 128, "score": 0.4283105853506469, "raw_macro_nll": 0.4595502587159514, "accuracy": 0.859375 }, "banking": { "count": 128, "score": 0.05047459421857176, "raw_macro_nll": 0.0495573852832617, "accuracy": 0.9921875 }, "boolq": { "count": 128, "score": 0.3196884642718922, "raw_macro_nll": 0.3320905225271662, "accuracy": 0.8828125 }, "snli": { "count": 128, "score": 0.32085798961274503, "raw_macro_nll": 0.3105186040886232, "accuracy": 0.890625 } }, "policy": { "id": "crossfit_temperature_nll_v1", "folds": 4, "seed": 431, "grouping": "source group, nested within task family; groups shuffled within sorted families and assigned round-robin", "temperature_grid": { "count": 101, "log10_min": -1.0, "log10_max": 1.3, "arithmetic": "Python float64" }, "temperature_fit": "minimum macro-family NLL on the other three folds; smallest temperature wins ties", "score": "macro-family mean of all held-out decision NLLs", "deployment_temperature": "fit afresh on reserved calibration only after model selection" } }, "best_selection_score": 0.2552943737691716, "improved": false, "metrics": { "overall": { "n": 512, "accuracy": 0.90625, "nll": 0.2879291938128496, "brier_multiclass_sum": 0.1459957167817959, "ece_top_label_10_equal_width_bins": 0.03560248663416132, "reliability_bins": [ { "count": 0, "confidence_sum": 0.0, "correct_sum": 0.0 }, { "count": 0, "confidence_sum": 0.0, "correct_sum": 0.0 }, { "count": 0, "confidence_sum": 0.0, "correct_sum": 0.0 }, { "count": 2, "confidence_sum": 0.7507493495941162, "correct_sum": 0.0 }, { "count": 6, "confidence_sum": 2.8289709389209747, "correct_sum": 1.0 }, { "count": 12, "confidence_sum": 6.604806303977966, "correct_sum": 8.0 }, { "count": 22, "confidence_sum": 14.299658298492432, "correct_sum": 17.0 }, { "count": 24, "confidence_sum": 18.386729657649994, "correct_sum": 16.0 }, { "count": 46, "confidence_sum": 39.259098529815674, "correct_sum": 33.0 }, { "count": 400, "confidence_sum": 391.90738928318024, "correct_sum": 389.0 } ], "accuracy_vs_coverage": { "0.25": { "n": 128, "accuracy": 1.0, "min_confidence": 0.9997970461845398 }, "0.5": { "n": 256, "accuracy": 0.98828125, "min_confidence": 0.9788828492164612 }, "0.75": { "n": 384, "accuracy": 0.9739583333333334, "min_confidence": 0.9226401448249817 }, "1.0": { "n": 512, "accuracy": 0.90625, "min_confidence": 0.36794614791870117 } } }, "per_family": { "arc": { "n": 128, "accuracy": 0.859375, "nll": 0.4595502640510675, "brier_multiclass_sum": 0.19233675665365213, "ece_top_label_10_equal_width_bins": 0.04693025094456971, "reliability_bins": [ { "count": 0, "confidence_sum": 0.0, "correct_sum": 0.0 }, { "count": 0, "confidence_sum": 0.0, "correct_sum": 0.0 }, { "count": 0, "confidence_sum": 0.0, "correct_sum": 0.0 }, { "count": 2, "confidence_sum": 0.7507493495941162, "correct_sum": 0.0 }, { "count": 5, "confidence_sum": 2.3596443235874176, "correct_sum": 1.0 }, { "count": 2, "confidence_sum": 1.1291241645812988, "correct_sum": 1.0 }, { "count": 9, "confidence_sum": 5.849187076091766, "correct_sum": 5.0 }, { "count": 5, "confidence_sum": 3.8443750739097595, "correct_sum": 4.0 }, { "count": 16, "confidence_sum": 13.591759443283081, "correct_sum": 13.0 }, { "count": 89, "confidence_sum": 88.170982837677, "correct_sum": 86.0 } ], "accuracy_vs_coverage": { "0.25": { "n": 32, "accuracy": 1.0, "min_confidence": 0.9997974038124084 }, "0.5": { "n": 64, "accuracy": 1.0, "min_confidence": 0.9940166473388672 }, "0.75": { "n": 96, "accuracy": 0.9479166666666666, "min_confidence": 0.8691695928573608 }, "1.0": { "n": 128, "accuracy": 0.859375, "min_confidence": 0.36794614791870117 } } }, "banking": { "n": 128, "accuracy": 0.9921875, "nll": 0.04955738214622285, "brier_multiclass_sum": 0.02729644176736861, "ece_top_label_10_equal_width_bins": 0.018931797705590725, "reliability_bins": [ { "count": 0, "confidence_sum": 0.0, "correct_sum": 0.0 }, { "count": 0, "confidence_sum": 0.0, "correct_sum": 0.0 }, { "count": 0, "confidence_sum": 0.0, "correct_sum": 0.0 }, { "count": 0, "confidence_sum": 0.0, "correct_sum": 0.0 }, { "count": 0, "confidence_sum": 0.0, "correct_sum": 0.0 }, { "count": 4, "confidence_sum": 2.1734386682510376, "correct_sum": 4.0 }, { "count": 0, "confidence_sum": 0.0, "correct_sum": 0.0 }, { "count": 0, "confidence_sum": 0.0, "correct_sum": 0.0 }, { "count": 1, "confidence_sum": 0.8818942308425903, "correct_sum": 1.0 }, { "count": 123, "confidence_sum": 122.47860300540924, "correct_sum": 122.0 } ], "accuracy_vs_coverage": { "0.25": { "n": 32, "accuracy": 1.0, "min_confidence": 0.9999998807907104 }, "0.5": { "n": 64, "accuracy": 1.0, "min_confidence": 0.9999978542327881 }, "0.75": { "n": 96, "accuracy": 1.0, "min_confidence": 0.9997970461845398 }, "1.0": { "n": 128, "accuracy": 0.9921875, "min_confidence": 0.5277213454246521 } } }, "boolq": { "n": 128, "accuracy": 0.8828125, "nll": 0.33209052507845627, "brier_multiclass_sum": 0.1964803123360663, "ece_top_label_10_equal_width_bins": 0.06582438806071877, "reliability_bins": [ { "count": 0, "confidence_sum": 0.0, "correct_sum": 0.0 }, { "count": 0, "confidence_sum": 0.0, "correct_sum": 0.0 }, { "count": 0, "confidence_sum": 0.0, "correct_sum": 0.0 }, { "count": 0, "confidence_sum": 0.0, "correct_sum": 0.0 }, { "count": 0, "confidence_sum": 0.0, "correct_sum": 0.0 }, { "count": 6, "confidence_sum": 3.30224347114563, "correct_sum": 3.0 }, { "count": 9, "confidence_sum": 5.869049251079559, "correct_sum": 8.0 }, { "count": 7, "confidence_sum": 5.441415011882782, "correct_sum": 6.0 }, { "count": 11, "confidence_sum": 9.470021307468414, "correct_sum": 7.0 }, { "count": 95, "confidence_sum": 91.9637211561203, "correct_sum": 89.0 } ], "accuracy_vs_coverage": { "0.25": { "n": 32, "accuracy": 0.96875, "min_confidence": 0.9858549237251282 }, "0.5": { "n": 64, "accuracy": 0.984375, "min_confidence": 0.9649767875671387 }, "0.75": { "n": 96, "accuracy": 0.9270833333333334, "min_confidence": 0.8953233957290649 }, "1.0": { "n": 128, "accuracy": 0.8828125, "min_confidence": 0.512544572353363 } } }, "snli": { "n": 128, "accuracy": 0.890625, "nll": 0.3105186039756518, "brier_multiclass_sum": 0.1678693563700965, "ece_top_label_10_equal_width_bins": 0.0860170740634203, "reliability_bins": [ { "count": 0, "confidence_sum": 0.0, "correct_sum": 0.0 }, { "count": 0, "confidence_sum": 0.0, "correct_sum": 0.0 }, { "count": 0, "confidence_sum": 0.0, "correct_sum": 0.0 }, { "count": 0, "confidence_sum": 0.0, "correct_sum": 0.0 }, { "count": 1, "confidence_sum": 0.46932661533355713, "correct_sum": 0.0 }, { "count": 0, "confidence_sum": 0.0, "correct_sum": 0.0 }, { "count": 4, "confidence_sum": 2.581421971321106, "correct_sum": 4.0 }, { "count": 12, "confidence_sum": 9.100939571857452, "correct_sum": 6.0 }, { "count": 18, "confidence_sum": 15.315423548221588, "correct_sum": 12.0 }, { "count": 93, "confidence_sum": 89.2940822839737, "correct_sum": 92.0 } ], "accuracy_vs_coverage": { "0.25": { "n": 32, "accuracy": 1.0, "min_confidence": 0.9734072685241699 }, "0.5": { "n": 64, "accuracy": 1.0, "min_confidence": 0.9560357928276062 }, "0.75": { "n": 96, "accuracy": 0.9791666666666666, "min_confidence": 0.8869620561599731 }, "1.0": { "n": 128, "accuracy": 0.890625, "min_confidence": 0.46932661533355713 } } } } } }, { "step": 5500, "macro_nll": 0.388565277749868, "best_macro_nll": 0.25676974726429014, "selection": { "metric": "crossfit_temperature_nll_v1", "score": 0.30398422164076616, "raw_macro_nll": 0.38856527338080316, "accuracy": 0.896484375, "folds": [ { "fold": 0, "temperature_index": 58, "temperature": 2.1577444091526656, "training_macro_nll": 0.3199929810058706, "heldout_ids": [ "arc:validation:ARC-Challenge:122", "arc:validation:ARC-Challenge:126", "arc:validation:ARC-Challenge:175", "arc:validation:ARC-Challenge:187", "arc:validation:ARC-Challenge:189", "arc:validation:ARC-Challenge:192", "arc:validation:ARC-Challenge:208", "arc:validation:ARC-Challenge:238", "arc:validation:ARC-Challenge:290", "arc:validation:ARC-Challenge:293", "arc:validation:ARC-Challenge:35", "arc:validation:ARC-Challenge:43", "arc:validation:ARC-Challenge:58", "arc:validation:ARC-Challenge:7", "arc:validation:ARC-Easy:160", "arc:validation:ARC-Easy:201", "arc:validation:ARC-Easy:229", "arc:validation:ARC-Easy:238", "arc:validation:ARC-Easy:253", "arc:validation:ARC-Easy:259", "arc:validation:ARC-Easy:272", "arc:validation:ARC-Easy:298", "arc:validation:ARC-Easy:329", "arc:validation:ARC-Easy:331", "arc:validation:ARC-Easy:337", "arc:validation:ARC-Easy:362", "arc:validation:ARC-Easy:443", "arc:validation:ARC-Easy:500", "arc:validation:ARC-Easy:545", "arc:validation:ARC-Easy:68", "arc:validation:ARC-Easy:7", "arc:validation:ARC-Easy:74", "banking:train:1053", "banking:train:1594", "banking:train:1747", "banking:train:202", "banking:train:2377", "banking:train:2528", "banking:train:2546", "banking:train:2715", "banking:train:3220", "banking:train:3499", "banking:train:3806", "banking:train:3838", "banking:train:3993", "banking:train:4220", "banking:train:5", "banking:train:5192", "banking:train:5394", "banking:train:5846", "banking:train:5863", "banking:train:6032", "banking:train:6508", "banking:train:7579", "banking:train:7600", "banking:train:7987", "banking:train:8155", "banking:train:850", "banking:train:8626", "banking:train:8929", "banking:train:9355", "banking:train:9812", "banking:train:987", "banking:train:9972", "boolq:validation:103", "boolq:validation:1042", "boolq:validation:1059", "boolq:validation:1541", "boolq:validation:1592", "boolq:validation:1927", "boolq:validation:2014", "boolq:validation:2023", "boolq:validation:2098", "boolq:validation:2139", "boolq:validation:2439", "boolq:validation:2485", "boolq:validation:2607", "boolq:validation:2611", "boolq:validation:2720", "boolq:validation:2751", "boolq:validation:282", "boolq:validation:2843", "boolq:validation:2953", "boolq:validation:301", "boolq:validation:3089", "boolq:validation:313", "boolq:validation:3201", "boolq:validation:322", "boolq:validation:3231", "boolq:validation:39", "boolq:validation:461", "boolq:validation:558", "boolq:validation:589", "boolq:validation:731", "boolq:validation:883", "boolq:validation:915", "snli:validation:1156", "snli:validation:1330", "snli:validation:1559", "snli:validation:1735", "snli:validation:1876", "snli:validation:2251", "snli:validation:2257", "snli:validation:2377", "snli:validation:2881", "snli:validation:291", "snli:validation:300", "snli:validation:3049", "snli:validation:3496", "snli:validation:3871", "snli:validation:4414", "snli:validation:5140", "snli:validation:5440", "snli:validation:5806", "snli:validation:6442", "snli:validation:6772", "snli:validation:6949", "snli:validation:7228", "snli:validation:7465", "snli:validation:7510", "snli:validation:7726", "snli:validation:8095", "snli:validation:8143", "snli:validation:8623", "snli:validation:8962", "snli:validation:9451", "snli:validation:9574", "snli:validation:9655" ], "heldout_group_count": 128, "training_group_count": 384 }, { "fold": 1, "temperature_index": 57, "temperature": 2.046444636724674, "training_macro_nll": 0.30257257772561696, "heldout_ids": [ "arc:validation:ARC-Challenge:101", "arc:validation:ARC-Challenge:129", "arc:validation:ARC-Challenge:148", "arc:validation:ARC-Challenge:156", "arc:validation:ARC-Challenge:167", "arc:validation:ARC-Challenge:249", "arc:validation:ARC-Challenge:263", "arc:validation:ARC-Challenge:266", "arc:validation:ARC-Challenge:267", "arc:validation:ARC-Challenge:28", "arc:validation:ARC-Challenge:281", "arc:validation:ARC-Challenge:57", "arc:validation:ARC-Challenge:63", "arc:validation:ARC-Challenge:64", "arc:validation:ARC-Challenge:86", "arc:validation:ARC-Challenge:99", "arc:validation:ARC-Easy:106", "arc:validation:ARC-Easy:151", "arc:validation:ARC-Easy:157", "arc:validation:ARC-Easy:166", "arc:validation:ARC-Easy:202", "arc:validation:ARC-Easy:219", "arc:validation:ARC-Easy:224", "arc:validation:ARC-Easy:322", "arc:validation:ARC-Easy:344", "arc:validation:ARC-Easy:382", "arc:validation:ARC-Easy:409", "arc:validation:ARC-Easy:455", "arc:validation:ARC-Easy:50", "arc:validation:ARC-Easy:538", "arc:validation:ARC-Easy:544", "arc:validation:ARC-Easy:552", "banking:train:1050", "banking:train:1131", "banking:train:1190", "banking:train:124", "banking:train:137", "banking:train:1493", "banking:train:1498", "banking:train:1624", "banking:train:2614", "banking:train:2739", "banking:train:3055", "banking:train:306", "banking:train:3312", "banking:train:3724", "banking:train:4070", "banking:train:476", "banking:train:4882", "banking:train:5075", "banking:train:5332", "banking:train:5445", "banking:train:6096", "banking:train:746", "banking:train:7828", "banking:train:8176", "banking:train:8315", "banking:train:8653", "banking:train:9019", "banking:train:9370", "banking:train:9454", "banking:train:9595", "banking:train:9862", "banking:train:9963", "boolq:validation:1189", "boolq:validation:1320", "boolq:validation:1326", "boolq:validation:1381", "boolq:validation:1410", "boolq:validation:149", "boolq:validation:1513", "boolq:validation:1732", "boolq:validation:1739", "boolq:validation:1741", "boolq:validation:1814", "boolq:validation:1848", "boolq:validation:1940", "boolq:validation:2077", "boolq:validation:2091", "boolq:validation:2148", "boolq:validation:2215", "boolq:validation:2251", "boolq:validation:2283", "boolq:validation:2321", "boolq:validation:2398", "boolq:validation:253", "boolq:validation:2575", "boolq:validation:2907", "boolq:validation:2909", "boolq:validation:2978", "boolq:validation:359", "boolq:validation:394", "boolq:validation:436", "boolq:validation:579", "boolq:validation:807", "boolq:validation:949", "snli:validation:2389", "snli:validation:2485", "snli:validation:255", "snli:validation:3169", "snli:validation:333", "snli:validation:3700", "snli:validation:3709", "snli:validation:4093", "snli:validation:420", "snli:validation:4492", "snli:validation:4516", "snli:validation:4648", "snli:validation:4888", "snli:validation:5185", "snli:validation:5332", "snli:validation:5947", "snli:validation:6124", "snli:validation:660", "snli:validation:6619", "snli:validation:6898", "snli:validation:7387", "snli:validation:7492", "snli:validation:7544", "snli:validation:7996", "snli:validation:8074", "snli:validation:8554", "snli:validation:8971", "snli:validation:9163", "snli:validation:9277", "snli:validation:9490", "snli:validation:9595", "snli:validation:9688" ], "heldout_group_count": 128, "training_group_count": 384 }, { "fold": 2, "temperature_index": 57, "temperature": 2.046444636724674, "training_macro_nll": 0.29578256432862426, "heldout_ids": [ "arc:validation:ARC-Challenge:120", "arc:validation:ARC-Challenge:130", "arc:validation:ARC-Challenge:194", "arc:validation:ARC-Challenge:218", "arc:validation:ARC-Challenge:235", "arc:validation:ARC-Challenge:272", "arc:validation:ARC-Challenge:277", "arc:validation:ARC-Challenge:48", "arc:validation:ARC-Challenge:88", "arc:validation:ARC-Easy:115", "arc:validation:ARC-Easy:12", "arc:validation:ARC-Easy:131", "arc:validation:ARC-Easy:145", "arc:validation:ARC-Easy:179", "arc:validation:ARC-Easy:18", "arc:validation:ARC-Easy:186", "arc:validation:ARC-Easy:191", "arc:validation:ARC-Easy:215", "arc:validation:ARC-Easy:283", "arc:validation:ARC-Easy:314", "arc:validation:ARC-Easy:315", "arc:validation:ARC-Easy:378", "arc:validation:ARC-Easy:384", "arc:validation:ARC-Easy:403", "arc:validation:ARC-Easy:432", "arc:validation:ARC-Easy:439", "arc:validation:ARC-Easy:448", "arc:validation:ARC-Easy:461", "arc:validation:ARC-Easy:469", "arc:validation:ARC-Easy:470", "arc:validation:ARC-Easy:504", "arc:validation:ARC-Easy:561", "banking:train:1142", "banking:train:148", "banking:train:1633", "banking:train:1927", "banking:train:2186", "banking:train:2251", "banking:train:23", "banking:train:3048", "banking:train:3665", "banking:train:3952", "banking:train:3999", "banking:train:4364", "banking:train:4619", "banking:train:5049", "banking:train:5335", "banking:train:549", "banking:train:5957", "banking:train:5985", "banking:train:6132", "banking:train:6240", "banking:train:6488", "banking:train:7065", "banking:train:7749", "banking:train:8470", "banking:train:8761", "banking:train:895", "banking:train:9256", "banking:train:9406", "banking:train:9450", "banking:train:9467", "banking:train:9588", "banking:train:9716", "boolq:validation:1130", "boolq:validation:1167", "boolq:validation:124", "boolq:validation:1255", "boolq:validation:1256", "boolq:validation:1373", "boolq:validation:15", "boolq:validation:155", "boolq:validation:1568", "boolq:validation:16", "boolq:validation:1654", "boolq:validation:1702", "boolq:validation:1792", "boolq:validation:1818", "boolq:validation:210", "boolq:validation:2247", "boolq:validation:2259", "boolq:validation:2362", "boolq:validation:2443", "boolq:validation:2524", "boolq:validation:2553", "boolq:validation:2623", "boolq:validation:2660", "boolq:validation:2669", "boolq:validation:2679", "boolq:validation:2832", "boolq:validation:357", "boolq:validation:453", "boolq:validation:49", "boolq:validation:808", "boolq:validation:90", "boolq:validation:910", "snli:validation:1612", "snli:validation:168", "snli:validation:1693", "snli:validation:2134", "snli:validation:2248", "snli:validation:2422", "snli:validation:2587", "snli:validation:3007", "snli:validation:3097", "snli:validation:3772", "snli:validation:4012", "snli:validation:4168", "snli:validation:4537", "snli:validation:4681", "snli:validation:5050", "snli:validation:5632", "snli:validation:5761", "snli:validation:6055", "snli:validation:6283", "snli:validation:6715", "snli:validation:7117", "snli:validation:7192", "snli:validation:7507", "snli:validation:7594", "snli:validation:7924", "snli:validation:7954", "snli:validation:8035", "snli:validation:8110", "snli:validation:8995", "snli:validation:949", "snli:validation:9523", "snli:validation:9589" ], "heldout_group_count": 128, "training_group_count": 384 }, { "fold": 3, "temperature_index": 56, "temperature": 1.9408858775927773, "training_macro_nll": 0.29129510768094824, "heldout_ids": [ "arc:validation:ARC-Challenge:117", "arc:validation:ARC-Challenge:135", "arc:validation:ARC-Challenge:139", "arc:validation:ARC-Challenge:154", "arc:validation:ARC-Challenge:174", "arc:validation:ARC-Challenge:199", "arc:validation:ARC-Challenge:287", "arc:validation:ARC-Challenge:295", "arc:validation:ARC-Challenge:4", "arc:validation:ARC-Challenge:50", "arc:validation:ARC-Challenge:51", "arc:validation:ARC-Challenge:61", "arc:validation:ARC-Easy:107", "arc:validation:ARC-Easy:121", "arc:validation:ARC-Easy:124", "arc:validation:ARC-Easy:171", "arc:validation:ARC-Easy:235", "arc:validation:ARC-Easy:241", "arc:validation:ARC-Easy:279", "arc:validation:ARC-Easy:289", "arc:validation:ARC-Easy:37", "arc:validation:ARC-Easy:389", "arc:validation:ARC-Easy:394", "arc:validation:ARC-Easy:399", "arc:validation:ARC-Easy:423", "arc:validation:ARC-Easy:492", "arc:validation:ARC-Easy:499", "arc:validation:ARC-Easy:523", "arc:validation:ARC-Easy:533", "arc:validation:ARC-Easy:569", "arc:validation:ARC-Easy:69", "arc:validation:ARC-Easy:93", "banking:train:2049", "banking:train:2350", "banking:train:2497", "banking:train:2506", "banking:train:2720", "banking:train:2788", "banking:train:2967", "banking:train:3502", "banking:train:3726", "banking:train:4014", "banking:train:4735", "banking:train:4803", "banking:train:4917", "banking:train:5477", "banking:train:5851", "banking:train:6551", "banking:train:6839", "banking:train:6926", "banking:train:7809", "banking:train:8092", "banking:train:8186", "banking:train:8215", "banking:train:8667", "banking:train:8752", "banking:train:8763", "banking:train:8851", "banking:train:8917", "banking:train:9145", "banking:train:9279", "banking:train:9676", "banking:train:9837", "banking:train:9967", "boolq:validation:106", "boolq:validation:1065", "boolq:validation:1097", "boolq:validation:1336", "boolq:validation:1350", "boolq:validation:1383", "boolq:validation:1458", "boolq:validation:1641", "boolq:validation:1873", "boolq:validation:1930", "boolq:validation:1989", "boolq:validation:2103", "boolq:validation:234", "boolq:validation:2355", "boolq:validation:2384", "boolq:validation:2440", "boolq:validation:2444", "boolq:validation:2478", "boolq:validation:2872", "boolq:validation:2879", "boolq:validation:2924", "boolq:validation:2971", "boolq:validation:3164", "boolq:validation:4", "boolq:validation:415", "boolq:validation:419", "boolq:validation:428", "boolq:validation:512", "boolq:validation:563", "boolq:validation:643", "boolq:validation:696", "boolq:validation:874", "snli:validation:1102", "snli:validation:1240", "snli:validation:1261", "snli:validation:1738", "snli:validation:1978", "snli:validation:2563", "snli:validation:2659", "snli:validation:2767", "snli:validation:2836", "snli:validation:3232", "snli:validation:3655", "snli:validation:3913", "snli:validation:405", "snli:validation:4051", "snli:validation:4300", "snli:validation:4846", "snli:validation:498", "snli:validation:5224", "snli:validation:5575", "snli:validation:6598", "snli:validation:666", "snli:validation:6925", "snli:validation:6964", "snli:validation:7045", "snli:validation:7351", "snli:validation:7588", "snli:validation:7723", "snli:validation:7750", "snli:validation:7909", "snli:validation:8974", "snli:validation:9385", "snli:validation:9646" ], "heldout_group_count": 128, "training_group_count": 384 } ], "per_family": { "arc": { "count": 128, "score": 0.3994756698273555, "raw_macro_nll": 0.5594607852608507, "accuracy": 0.8828125 }, "banking": { "count": 128, "score": 0.029118518016223807, "raw_macro_nll": 0.021053720166049603, "accuracy": 0.9921875 }, "boolq": { "count": 128, "score": 0.36662860933042185, "raw_macro_nll": 0.5107546041960799, "accuracy": 0.84375 }, "snli": { "count": 128, "score": 0.4207140893890635, "raw_macro_nll": 0.4629919839002324, "accuracy": 0.8671875 } }, "policy": { "id": "crossfit_temperature_nll_v1", "folds": 4, "seed": 431, "grouping": "source group, nested within task family; groups shuffled within sorted families and assigned round-robin", "temperature_grid": { "count": 101, "log10_min": -1.0, "log10_max": 1.3, "arithmetic": "Python float64" }, "temperature_fit": "minimum macro-family NLL on the other three folds; smallest temperature wins ties", "score": "macro-family mean of all held-out decision NLLs", "deployment_temperature": "fit afresh on reserved calibration only after model selection" } }, "best_selection_score": 0.2552943737691716, "improved": false, "metrics": { "overall": { "n": 512, "accuracy": 0.896484375, "nll": 0.388565277749868, "brier_multiclass_sum": 0.1666528894976072, "ece_top_label_10_equal_width_bins": 0.05876279535004869, "reliability_bins": [ { "count": 0, "confidence_sum": 0.0, "correct_sum": 0.0 }, { "count": 0, "confidence_sum": 0.0, "correct_sum": 0.0 }, { "count": 0, "confidence_sum": 0.0, "correct_sum": 0.0 }, { "count": 1, "confidence_sum": 0.3927515745162964, "correct_sum": 0.0 }, { "count": 4, "confidence_sum": 1.8952177464962006, "correct_sum": 2.0 }, { "count": 11, "confidence_sum": 6.0297017097473145, "correct_sum": 7.0 }, { "count": 15, "confidence_sum": 9.80898916721344, "correct_sum": 5.0 }, { "count": 10, "confidence_sum": 7.650386393070221, "correct_sum": 6.0 }, { "count": 34, "confidence_sum": 29.33014488220215, "correct_sum": 24.0 }, { "count": 437, "confidence_sum": 431.82919865846634, "correct_sum": 415.0 } ], "accuracy_vs_coverage": { "0.25": { "n": 128, "accuracy": 1.0, "min_confidence": 0.9999988079071045 }, "0.5": { "n": 256, "accuracy": 0.984375, "min_confidence": 0.998248815536499 }, "0.75": { "n": 384, "accuracy": 0.953125, "min_confidence": 0.9573226571083069 }, "1.0": { "n": 512, "accuracy": 0.896484375, "min_confidence": 0.3927515745162964 } } }, "per_family": { "arc": { "n": 128, "accuracy": 0.8828125, "nll": 0.559460803464001, "brier_multiclass_sum": 0.18063573367263372, "ece_top_label_10_equal_width_bins": 0.0859838502947241, "reliability_bins": [ { "count": 0, "confidence_sum": 0.0, "correct_sum": 0.0 }, { "count": 0, "confidence_sum": 0.0, "correct_sum": 0.0 }, { "count": 0, "confidence_sum": 0.0, "correct_sum": 0.0 }, { "count": 1, "confidence_sum": 0.3927515745162964, "correct_sum": 0.0 }, { "count": 3, "confidence_sum": 1.4031347334384918, "correct_sum": 1.0 }, { "count": 5, "confidence_sum": 2.770816743373871, "correct_sum": 4.0 }, { "count": 8, "confidence_sum": 5.263402879238129, "correct_sum": 2.0 }, { "count": 2, "confidence_sum": 1.5604954361915588, "correct_sum": 2.0 }, { "count": 6, "confidence_sum": 5.152871131896973, "correct_sum": 6.0 }, { "count": 103, "confidence_sum": 102.43082696199417, "correct_sum": 98.0 } ], "accuracy_vs_coverage": { "0.25": { "n": 32, "accuracy": 1.0, "min_confidence": 0.9999986886978149 }, "0.5": { "n": 64, "accuracy": 0.96875, "min_confidence": 0.9996984004974365 }, "0.75": { "n": 96, "accuracy": 0.9479166666666666, "min_confidence": 0.9729017615318298 }, "1.0": { "n": 128, "accuracy": 0.8828125, "min_confidence": 0.3927515745162964 } } }, "banking": { "n": 128, "accuracy": 0.9921875, "nll": 0.02105371563040176, "brier_multiclass_sum": 0.013577903502986103, "ece_top_label_10_equal_width_bins": 0.01120622968301177, "reliability_bins": [ { "count": 0, "confidence_sum": 0.0, "correct_sum": 0.0 }, { "count": 0, "confidence_sum": 0.0, "correct_sum": 0.0 }, { "count": 0, "confidence_sum": 0.0, "correct_sum": 0.0 }, { "count": 0, "confidence_sum": 0.0, "correct_sum": 0.0 }, { "count": 0, "confidence_sum": 0.0, "correct_sum": 0.0 }, { "count": 0, "confidence_sum": 0.0, "correct_sum": 0.0 }, { "count": 1, "confidence_sum": 0.685830295085907, "correct_sum": 1.0 }, { "count": 1, "confidence_sum": 0.7017979025840759, "correct_sum": 1.0 }, { "count": 2, "confidence_sum": 1.6746965050697327, "correct_sum": 1.0 }, { "count": 124, "confidence_sum": 123.85267090797424, "correct_sum": 124.0 } ], "accuracy_vs_coverage": { "0.25": { "n": 32, "accuracy": 1.0, "min_confidence": 1.0 }, "0.5": { "n": 64, "accuracy": 1.0, "min_confidence": 1.0 }, "0.75": { "n": 96, "accuracy": 1.0, "min_confidence": 0.9999998807907104 }, "1.0": { "n": 128, "accuracy": 0.9921875, "min_confidence": 0.685830295085907 } } }, "boolq": { "n": 128, "accuracy": 0.84375, "nll": 0.510754603942928, "brier_multiclass_sum": 0.25073990983894767, "ece_top_label_10_equal_width_bins": 0.10163650894537568, "reliability_bins": [ { "count": 0, "confidence_sum": 0.0, "correct_sum": 0.0 }, { "count": 0, "confidence_sum": 0.0, "correct_sum": 0.0 }, { "count": 0, "confidence_sum": 0.0, "correct_sum": 0.0 }, { "count": 0, "confidence_sum": 0.0, "correct_sum": 0.0 }, { "count": 0, "confidence_sum": 0.0, "correct_sum": 0.0 }, { "count": 4, "confidence_sum": 2.1253692507743835, "correct_sum": 2.0 }, { "count": 5, "confidence_sum": 3.2207350730895996, "correct_sum": 2.0 }, { "count": 4, "confidence_sum": 3.1362109780311584, "correct_sum": 2.0 }, { "count": 9, "confidence_sum": 7.752904176712036, "correct_sum": 7.0 }, { "count": 106, "confidence_sum": 104.77425366640091, "correct_sum": 95.0 } ], "accuracy_vs_coverage": { "0.25": { "n": 32, "accuracy": 0.96875, "min_confidence": 0.9983727335929871 }, "0.5": { "n": 64, "accuracy": 0.96875, "min_confidence": 0.9950273633003235 }, "0.75": { "n": 96, "accuracy": 0.8958333333333334, "min_confidence": 0.9621425271034241 }, "1.0": { "n": 128, "accuracy": 0.84375, "min_confidence": 0.5126386284828186 } } }, "snli": { "n": 128, "accuracy": 0.8671875, "nll": 0.46299198796214114, "brier_multiclass_sum": 0.22165801097586132, "ece_top_label_10_equal_width_bins": 0.07854262413457036, "reliability_bins": [ { "count": 0, "confidence_sum": 0.0, "correct_sum": 0.0 }, { "count": 0, "confidence_sum": 0.0, "correct_sum": 0.0 }, { "count": 0, "confidence_sum": 0.0, "correct_sum": 0.0 }, { "count": 0, "confidence_sum": 0.0, "correct_sum": 0.0 }, { "count": 1, "confidence_sum": 0.49208301305770874, "correct_sum": 1.0 }, { "count": 2, "confidence_sum": 1.13351571559906, "correct_sum": 1.0 }, { "count": 1, "confidence_sum": 0.6390209197998047, "correct_sum": 0.0 }, { "count": 3, "confidence_sum": 2.2518820762634277, "correct_sum": 1.0 }, { "count": 17, "confidence_sum": 14.749673068523407, "correct_sum": 10.0 }, { "count": 104, "confidence_sum": 100.77144712209702, "correct_sum": 98.0 } ], "accuracy_vs_coverage": { "0.25": { "n": 32, "accuracy": 0.96875, "min_confidence": 0.9963485598564148 }, "0.5": { "n": 64, "accuracy": 0.953125, "min_confidence": 0.9578976631164551 }, "0.75": { "n": 96, "accuracy": 0.9479166666666666, "min_confidence": 0.9151947498321533 }, "1.0": { "n": 128, "accuracy": 0.8671875, "min_confidence": 0.49208301305770874 } } } } } }, { "step": 6000, "macro_nll": 0.40940693625302227, "best_macro_nll": 0.25676974726429014, "selection": { "metric": "crossfit_temperature_nll_v1", "score": 0.35159254105589344, "raw_macro_nll": 0.40940693985472393, "accuracy": 0.869140625, "folds": [ { "fold": 0, "temperature_index": 54, "temperature": 1.7458221529205038, "training_macro_nll": 0.34777796230347097, "heldout_ids": [ "arc:validation:ARC-Challenge:122", "arc:validation:ARC-Challenge:126", "arc:validation:ARC-Challenge:175", "arc:validation:ARC-Challenge:187", "arc:validation:ARC-Challenge:189", "arc:validation:ARC-Challenge:192", "arc:validation:ARC-Challenge:208", "arc:validation:ARC-Challenge:238", "arc:validation:ARC-Challenge:290", "arc:validation:ARC-Challenge:293", "arc:validation:ARC-Challenge:35", "arc:validation:ARC-Challenge:43", "arc:validation:ARC-Challenge:58", "arc:validation:ARC-Challenge:7", "arc:validation:ARC-Easy:160", "arc:validation:ARC-Easy:201", "arc:validation:ARC-Easy:229", "arc:validation:ARC-Easy:238", "arc:validation:ARC-Easy:253", "arc:validation:ARC-Easy:259", "arc:validation:ARC-Easy:272", "arc:validation:ARC-Easy:298", "arc:validation:ARC-Easy:329", "arc:validation:ARC-Easy:331", "arc:validation:ARC-Easy:337", "arc:validation:ARC-Easy:362", "arc:validation:ARC-Easy:443", "arc:validation:ARC-Easy:500", "arc:validation:ARC-Easy:545", "arc:validation:ARC-Easy:68", "arc:validation:ARC-Easy:7", "arc:validation:ARC-Easy:74", "banking:train:1053", "banking:train:1594", "banking:train:1747", "banking:train:202", "banking:train:2377", "banking:train:2528", "banking:train:2546", "banking:train:2715", "banking:train:3220", "banking:train:3499", "banking:train:3806", "banking:train:3838", "banking:train:3993", "banking:train:4220", "banking:train:5", "banking:train:5192", "banking:train:5394", "banking:train:5846", "banking:train:5863", "banking:train:6032", "banking:train:6508", "banking:train:7579", "banking:train:7600", "banking:train:7987", "banking:train:8155", "banking:train:850", "banking:train:8626", "banking:train:8929", "banking:train:9355", "banking:train:9812", "banking:train:987", "banking:train:9972", "boolq:validation:103", "boolq:validation:1042", "boolq:validation:1059", "boolq:validation:1541", "boolq:validation:1592", "boolq:validation:1927", "boolq:validation:2014", "boolq:validation:2023", "boolq:validation:2098", "boolq:validation:2139", "boolq:validation:2439", "boolq:validation:2485", "boolq:validation:2607", "boolq:validation:2611", "boolq:validation:2720", "boolq:validation:2751", "boolq:validation:282", "boolq:validation:2843", "boolq:validation:2953", "boolq:validation:301", "boolq:validation:3089", "boolq:validation:313", "boolq:validation:3201", "boolq:validation:322", "boolq:validation:3231", "boolq:validation:39", "boolq:validation:461", "boolq:validation:558", "boolq:validation:589", "boolq:validation:731", "boolq:validation:883", "boolq:validation:915", "snli:validation:1156", "snli:validation:1330", "snli:validation:1559", "snli:validation:1735", "snli:validation:1876", "snli:validation:2251", "snli:validation:2257", "snli:validation:2377", "snli:validation:2881", "snli:validation:291", "snli:validation:300", "snli:validation:3049", "snli:validation:3496", "snli:validation:3871", "snli:validation:4414", "snli:validation:5140", "snli:validation:5440", "snli:validation:5806", "snli:validation:6442", "snli:validation:6772", "snli:validation:6949", "snli:validation:7228", "snli:validation:7465", "snli:validation:7510", "snli:validation:7726", "snli:validation:8095", "snli:validation:8143", "snli:validation:8623", "snli:validation:8962", "snli:validation:9451", "snli:validation:9574", "snli:validation:9655" ], "heldout_group_count": 128, "training_group_count": 384 }, { "fold": 1, "temperature_index": 55, "temperature": 1.8407720014689555, "training_macro_nll": 0.3529714060051017, "heldout_ids": [ "arc:validation:ARC-Challenge:101", "arc:validation:ARC-Challenge:129", "arc:validation:ARC-Challenge:148", "arc:validation:ARC-Challenge:156", "arc:validation:ARC-Challenge:167", "arc:validation:ARC-Challenge:249", "arc:validation:ARC-Challenge:263", "arc:validation:ARC-Challenge:266", "arc:validation:ARC-Challenge:267", "arc:validation:ARC-Challenge:28", "arc:validation:ARC-Challenge:281", "arc:validation:ARC-Challenge:57", "arc:validation:ARC-Challenge:63", "arc:validation:ARC-Challenge:64", "arc:validation:ARC-Challenge:86", "arc:validation:ARC-Challenge:99", "arc:validation:ARC-Easy:106", "arc:validation:ARC-Easy:151", "arc:validation:ARC-Easy:157", "arc:validation:ARC-Easy:166", "arc:validation:ARC-Easy:202", "arc:validation:ARC-Easy:219", "arc:validation:ARC-Easy:224", "arc:validation:ARC-Easy:322", "arc:validation:ARC-Easy:344", "arc:validation:ARC-Easy:382", "arc:validation:ARC-Easy:409", "arc:validation:ARC-Easy:455", "arc:validation:ARC-Easy:50", "arc:validation:ARC-Easy:538", "arc:validation:ARC-Easy:544", "arc:validation:ARC-Easy:552", "banking:train:1050", "banking:train:1131", "banking:train:1190", "banking:train:124", "banking:train:137", "banking:train:1493", "banking:train:1498", "banking:train:1624", "banking:train:2614", "banking:train:2739", "banking:train:3055", "banking:train:306", "banking:train:3312", "banking:train:3724", "banking:train:4070", "banking:train:476", "banking:train:4882", "banking:train:5075", "banking:train:5332", "banking:train:5445", "banking:train:6096", "banking:train:746", "banking:train:7828", "banking:train:8176", "banking:train:8315", "banking:train:8653", "banking:train:9019", "banking:train:9370", "banking:train:9454", "banking:train:9595", "banking:train:9862", "banking:train:9963", "boolq:validation:1189", "boolq:validation:1320", "boolq:validation:1326", "boolq:validation:1381", "boolq:validation:1410", "boolq:validation:149", "boolq:validation:1513", "boolq:validation:1732", "boolq:validation:1739", "boolq:validation:1741", "boolq:validation:1814", "boolq:validation:1848", "boolq:validation:1940", "boolq:validation:2077", "boolq:validation:2091", "boolq:validation:2148", "boolq:validation:2215", "boolq:validation:2251", "boolq:validation:2283", "boolq:validation:2321", "boolq:validation:2398", "boolq:validation:253", "boolq:validation:2575", "boolq:validation:2907", "boolq:validation:2909", "boolq:validation:2978", "boolq:validation:359", "boolq:validation:394", "boolq:validation:436", "boolq:validation:579", "boolq:validation:807", "boolq:validation:949", "snli:validation:2389", "snli:validation:2485", "snli:validation:255", "snli:validation:3169", "snli:validation:333", "snli:validation:3700", "snli:validation:3709", "snli:validation:4093", "snli:validation:420", "snli:validation:4492", "snli:validation:4516", "snli:validation:4648", "snli:validation:4888", "snli:validation:5185", "snli:validation:5332", "snli:validation:5947", "snli:validation:6124", "snli:validation:660", "snli:validation:6619", "snli:validation:6898", "snli:validation:7387", "snli:validation:7492", "snli:validation:7544", "snli:validation:7996", "snli:validation:8074", "snli:validation:8554", "snli:validation:8971", "snli:validation:9163", "snli:validation:9277", "snli:validation:9490", "snli:validation:9595", "snli:validation:9688" ], "heldout_group_count": 128, "training_group_count": 384 }, { "fold": 2, "temperature_index": 55, "temperature": 1.8407720014689555, "training_macro_nll": 0.3578657995753054, "heldout_ids": [ "arc:validation:ARC-Challenge:120", "arc:validation:ARC-Challenge:130", "arc:validation:ARC-Challenge:194", "arc:validation:ARC-Challenge:218", "arc:validation:ARC-Challenge:235", "arc:validation:ARC-Challenge:272", "arc:validation:ARC-Challenge:277", "arc:validation:ARC-Challenge:48", "arc:validation:ARC-Challenge:88", "arc:validation:ARC-Easy:115", "arc:validation:ARC-Easy:12", "arc:validation:ARC-Easy:131", "arc:validation:ARC-Easy:145", "arc:validation:ARC-Easy:179", "arc:validation:ARC-Easy:18", "arc:validation:ARC-Easy:186", "arc:validation:ARC-Easy:191", "arc:validation:ARC-Easy:215", "arc:validation:ARC-Easy:283", "arc:validation:ARC-Easy:314", "arc:validation:ARC-Easy:315", "arc:validation:ARC-Easy:378", "arc:validation:ARC-Easy:384", "arc:validation:ARC-Easy:403", "arc:validation:ARC-Easy:432", "arc:validation:ARC-Easy:439", "arc:validation:ARC-Easy:448", "arc:validation:ARC-Easy:461", "arc:validation:ARC-Easy:469", "arc:validation:ARC-Easy:470", "arc:validation:ARC-Easy:504", "arc:validation:ARC-Easy:561", "banking:train:1142", "banking:train:148", "banking:train:1633", "banking:train:1927", "banking:train:2186", "banking:train:2251", "banking:train:23", "banking:train:3048", "banking:train:3665", "banking:train:3952", "banking:train:3999", "banking:train:4364", "banking:train:4619", "banking:train:5049", "banking:train:5335", "banking:train:549", "banking:train:5957", "banking:train:5985", "banking:train:6132", "banking:train:6240", "banking:train:6488", "banking:train:7065", "banking:train:7749", "banking:train:8470", "banking:train:8761", "banking:train:895", "banking:train:9256", "banking:train:9406", "banking:train:9450", "banking:train:9467", "banking:train:9588", "banking:train:9716", "boolq:validation:1130", "boolq:validation:1167", "boolq:validation:124", "boolq:validation:1255", "boolq:validation:1256", "boolq:validation:1373", "boolq:validation:15", "boolq:validation:155", "boolq:validation:1568", "boolq:validation:16", "boolq:validation:1654", "boolq:validation:1702", "boolq:validation:1792", "boolq:validation:1818", "boolq:validation:210", "boolq:validation:2247", "boolq:validation:2259", "boolq:validation:2362", "boolq:validation:2443", "boolq:validation:2524", "boolq:validation:2553", "boolq:validation:2623", "boolq:validation:2660", "boolq:validation:2669", "boolq:validation:2679", "boolq:validation:2832", "boolq:validation:357", "boolq:validation:453", "boolq:validation:49", "boolq:validation:808", "boolq:validation:90", "boolq:validation:910", "snli:validation:1612", "snli:validation:168", "snli:validation:1693", "snli:validation:2134", "snli:validation:2248", "snli:validation:2422", "snli:validation:2587", "snli:validation:3007", "snli:validation:3097", "snli:validation:3772", "snli:validation:4012", "snli:validation:4168", "snli:validation:4537", "snli:validation:4681", "snli:validation:5050", "snli:validation:5632", "snli:validation:5761", "snli:validation:6055", "snli:validation:6283", "snli:validation:6715", "snli:validation:7117", "snli:validation:7192", "snli:validation:7507", "snli:validation:7594", "snli:validation:7924", "snli:validation:7954", "snli:validation:8035", "snli:validation:8110", "snli:validation:8995", "snli:validation:949", "snli:validation:9523", "snli:validation:9589" ], "heldout_group_count": 128, "training_group_count": 384 }, { "fold": 3, "temperature_index": 52, "temperature": 1.5703628043335522, "training_macro_nll": 0.32735544756754187, "heldout_ids": [ "arc:validation:ARC-Challenge:117", "arc:validation:ARC-Challenge:135", "arc:validation:ARC-Challenge:139", "arc:validation:ARC-Challenge:154", "arc:validation:ARC-Challenge:174", "arc:validation:ARC-Challenge:199", "arc:validation:ARC-Challenge:287", "arc:validation:ARC-Challenge:295", "arc:validation:ARC-Challenge:4", "arc:validation:ARC-Challenge:50", "arc:validation:ARC-Challenge:51", "arc:validation:ARC-Challenge:61", "arc:validation:ARC-Easy:107", "arc:validation:ARC-Easy:121", "arc:validation:ARC-Easy:124", "arc:validation:ARC-Easy:171", "arc:validation:ARC-Easy:235", "arc:validation:ARC-Easy:241", "arc:validation:ARC-Easy:279", "arc:validation:ARC-Easy:289", "arc:validation:ARC-Easy:37", "arc:validation:ARC-Easy:389", "arc:validation:ARC-Easy:394", "arc:validation:ARC-Easy:399", "arc:validation:ARC-Easy:423", "arc:validation:ARC-Easy:492", "arc:validation:ARC-Easy:499", "arc:validation:ARC-Easy:523", "arc:validation:ARC-Easy:533", "arc:validation:ARC-Easy:569", "arc:validation:ARC-Easy:69", "arc:validation:ARC-Easy:93", "banking:train:2049", "banking:train:2350", "banking:train:2497", "banking:train:2506", "banking:train:2720", "banking:train:2788", "banking:train:2967", "banking:train:3502", "banking:train:3726", "banking:train:4014", "banking:train:4735", "banking:train:4803", "banking:train:4917", "banking:train:5477", "banking:train:5851", "banking:train:6551", "banking:train:6839", "banking:train:6926", "banking:train:7809", "banking:train:8092", "banking:train:8186", "banking:train:8215", "banking:train:8667", "banking:train:8752", "banking:train:8763", "banking:train:8851", "banking:train:8917", "banking:train:9145", "banking:train:9279", "banking:train:9676", "banking:train:9837", "banking:train:9967", "boolq:validation:106", "boolq:validation:1065", "boolq:validation:1097", "boolq:validation:1336", "boolq:validation:1350", "boolq:validation:1383", "boolq:validation:1458", "boolq:validation:1641", "boolq:validation:1873", "boolq:validation:1930", "boolq:validation:1989", "boolq:validation:2103", "boolq:validation:234", "boolq:validation:2355", "boolq:validation:2384", "boolq:validation:2440", "boolq:validation:2444", "boolq:validation:2478", "boolq:validation:2872", "boolq:validation:2879", "boolq:validation:2924", "boolq:validation:2971", "boolq:validation:3164", "boolq:validation:4", "boolq:validation:415", "boolq:validation:419", "boolq:validation:428", "boolq:validation:512", "boolq:validation:563", "boolq:validation:643", "boolq:validation:696", "boolq:validation:874", "snli:validation:1102", "snli:validation:1240", "snli:validation:1261", "snli:validation:1738", "snli:validation:1978", "snli:validation:2563", "snli:validation:2659", "snli:validation:2767", "snli:validation:2836", "snli:validation:3232", "snli:validation:3655", "snli:validation:3913", "snli:validation:405", "snli:validation:4051", "snli:validation:4300", "snli:validation:4846", "snli:validation:498", "snli:validation:5224", "snli:validation:5575", "snli:validation:6598", "snli:validation:666", "snli:validation:6925", "snli:validation:6964", "snli:validation:7045", "snli:validation:7351", "snli:validation:7588", "snli:validation:7723", "snli:validation:7750", "snli:validation:7909", "snli:validation:8974", "snli:validation:9385", "snli:validation:9646" ], "heldout_group_count": 128, "training_group_count": 384 } ], "per_family": { "arc": { "count": 128, "score": 0.4975187847350728, "raw_macro_nll": 0.4627253663720225, "accuracy": 0.8125 }, "banking": { "count": 128, "score": 0.028150747289219766, "raw_macro_nll": 0.02860187647413226, "accuracy": 0.984375 }, "boolq": { "count": 128, "score": 0.4220618375304737, "raw_macro_nll": 0.5700947268218147, "accuracy": 0.8359375 }, "snli": { "count": 128, "score": 0.45863879466880736, "raw_macro_nll": 0.5762057897509263, "accuracy": 0.84375 } }, "policy": { "id": "crossfit_temperature_nll_v1", "folds": 4, "seed": 431, "grouping": "source group, nested within task family; groups shuffled within sorted families and assigned round-robin", "temperature_grid": { "count": 101, "log10_min": -1.0, "log10_max": 1.3, "arithmetic": "Python float64" }, "temperature_fit": "minimum macro-family NLL on the other three folds; smallest temperature wins ties", "score": "macro-family mean of all held-out decision NLLs", "deployment_temperature": "fit afresh on reserved calibration only after model selection" } }, "best_selection_score": 0.2552943737691716, "improved": false, "metrics": { "overall": { "n": 512, "accuracy": 0.869140625, "nll": 0.40940693625302227, "brier_multiclass_sum": 0.20434636874989778, "ece_top_label_10_equal_width_bins": 0.073907851066906, "reliability_bins": [ { "count": 0, "confidence_sum": 0.0, "correct_sum": 0.0 }, { "count": 0, "confidence_sum": 0.0, "correct_sum": 0.0 }, { "count": 0, "confidence_sum": 0.0, "correct_sum": 0.0 }, { "count": 8, "confidence_sum": 2.964146226644516, "correct_sum": 1.0 }, { "count": 4, "confidence_sum": 1.775023639202118, "correct_sum": 1.0 }, { "count": 12, "confidence_sum": 6.521916568279266, "correct_sum": 4.0 }, { "count": 23, "confidence_sum": 15.011595368385315, "correct_sum": 17.0 }, { "count": 13, "confidence_sum": 9.764744758605957, "correct_sum": 10.0 }, { "count": 31, "confidence_sum": 26.539382338523865, "correct_sum": 21.0 }, { "count": 421, "confidence_sum": 415.8166911005974, "correct_sum": 391.0 } ], "accuracy_vs_coverage": { "0.25": { "n": 128, "accuracy": 1.0, "min_confidence": 0.9999061822891235 }, "0.5": { "n": 256, "accuracy": 0.9765625, "min_confidence": 0.992233395576477 }, "0.75": { "n": 384, "accuracy": 0.9479166666666666, "min_confidence": 0.9575716257095337 }, "1.0": { "n": 512, "accuracy": 0.869140625, "min_confidence": 0.3231365382671356 } } }, "per_family": { "arc": { "n": 128, "accuracy": 0.8125, "nll": 0.46272536673021847, "brier_multiclass_sum": 0.2405542295009345, "ece_top_label_10_equal_width_bins": 0.07007392193190753, "reliability_bins": [ { "count": 0, "confidence_sum": 0.0, "correct_sum": 0.0 }, { "count": 0, "confidence_sum": 0.0, "correct_sum": 0.0 }, { "count": 0, "confidence_sum": 0.0, "correct_sum": 0.0 }, { "count": 8, "confidence_sum": 2.964146226644516, "correct_sum": 1.0 }, { "count": 4, "confidence_sum": 1.775023639202118, "correct_sum": 1.0 }, { "count": 9, "confidence_sum": 4.836179673671722, "correct_sum": 4.0 }, { "count": 14, "confidence_sum": 9.198149383068085, "correct_sum": 10.0 }, { "count": 8, "confidence_sum": 6.019686222076416, "correct_sum": 7.0 }, { "count": 12, "confidence_sum": 10.27308714389801, "correct_sum": 8.0 }, { "count": 73, "confidence_sum": 71.6611390709877, "correct_sum": 73.0 } ], "accuracy_vs_coverage": { "0.25": { "n": 32, "accuracy": 1.0, "min_confidence": 0.9957687854766846 }, "0.5": { "n": 64, "accuracy": 1.0, "min_confidence": 0.9485200643539429 }, "0.75": { "n": 96, "accuracy": 0.9375, "min_confidence": 0.6923663020133972 }, "1.0": { "n": 128, "accuracy": 0.8125, "min_confidence": 0.3231365382671356 } } }, "banking": { "n": 128, "accuracy": 0.984375, "nll": 0.028601872775282866, "brier_multiclass_sum": 0.01876134001342418, "ece_top_label_10_equal_width_bins": 0.011103796772658825, "reliability_bins": [ { "count": 0, "confidence_sum": 0.0, "correct_sum": 0.0 }, { "count": 0, "confidence_sum": 0.0, "correct_sum": 0.0 }, { "count": 0, "confidence_sum": 0.0, "correct_sum": 0.0 }, { "count": 0, "confidence_sum": 0.0, "correct_sum": 0.0 }, { "count": 0, "confidence_sum": 0.0, "correct_sum": 0.0 }, { "count": 1, "confidence_sum": 0.5795621275901794, "correct_sum": 0.0 }, { "count": 0, "confidence_sum": 0.0, "correct_sum": 0.0 }, { "count": 0, "confidence_sum": 0.0, "correct_sum": 0.0 }, { "count": 1, "confidence_sum": 0.8920316100120544, "correct_sum": 1.0 }, { "count": 126, "confidence_sum": 125.7337554693222, "correct_sum": 125.0 } ], "accuracy_vs_coverage": { "0.25": { "n": 32, "accuracy": 1.0, "min_confidence": 1.0 }, "0.5": { "n": 64, "accuracy": 1.0, "min_confidence": 1.0 }, "0.75": { "n": 96, "accuracy": 1.0, "min_confidence": 0.9999983310699463 }, "1.0": { "n": 128, "accuracy": 0.984375, "min_confidence": 0.5795621275901794 } } }, "boolq": { "n": 128, "accuracy": 0.8359375, "nll": 0.5700947275718136, "brier_multiclass_sum": 0.29235693626842657, "ece_top_label_10_equal_width_bins": 0.1428935481235385, "reliability_bins": [ { "count": 0, "confidence_sum": 0.0, "correct_sum": 0.0 }, { "count": 0, "confidence_sum": 0.0, "correct_sum": 0.0 }, { "count": 0, "confidence_sum": 0.0, "correct_sum": 0.0 }, { "count": 0, "confidence_sum": 0.0, "correct_sum": 0.0 }, { "count": 0, "confidence_sum": 0.0, "correct_sum": 0.0 }, { "count": 1, "confidence_sum": 0.527448296546936, "correct_sum": 0.0 }, { "count": 4, "confidence_sum": 2.6020392775535583, "correct_sum": 4.0 }, { "count": 1, "confidence_sum": 0.7223353981971741, "correct_sum": 0.0 }, { "count": 12, "confidence_sum": 10.259824514389038, "correct_sum": 9.0 }, { "count": 110, "confidence_sum": 108.38280522823334, "correct_sum": 94.0 } ], "accuracy_vs_coverage": { "0.25": { "n": 32, "accuracy": 0.96875, "min_confidence": 0.9951808452606201 }, "0.5": { "n": 64, "accuracy": 0.9375, "min_confidence": 0.9884833693504333 }, "0.75": { "n": 96, "accuracy": 0.8958333333333334, "min_confidence": 0.9682738780975342 }, "1.0": { "n": 128, "accuracy": 0.8359375, "min_confidence": 0.527448296546936 } } }, "snli": { "n": 128, "accuracy": 0.84375, "nll": 0.5762057779347742, "brier_multiclass_sum": 0.26571296921680587, "ece_top_label_10_equal_width_bins": 0.10911161499097943, "reliability_bins": [ { "count": 0, "confidence_sum": 0.0, "correct_sum": 0.0 }, { "count": 0, "confidence_sum": 0.0, "correct_sum": 0.0 }, { "count": 0, "confidence_sum": 0.0, "correct_sum": 0.0 }, { "count": 0, "confidence_sum": 0.0, "correct_sum": 0.0 }, { "count": 0, "confidence_sum": 0.0, "correct_sum": 0.0 }, { "count": 1, "confidence_sum": 0.5787264704704285, "correct_sum": 0.0 }, { "count": 5, "confidence_sum": 3.211406707763672, "correct_sum": 3.0 }, { "count": 4, "confidence_sum": 3.022723138332367, "correct_sum": 3.0 }, { "count": 6, "confidence_sum": 5.114439070224762, "correct_sum": 3.0 }, { "count": 112, "confidence_sum": 110.03899133205414, "correct_sum": 99.0 } ], "accuracy_vs_coverage": { "0.25": { "n": 32, "accuracy": 0.9375, "min_confidence": 0.9953944683074951 }, "0.5": { "n": 64, "accuracy": 0.9375, "min_confidence": 0.9888361096382141 }, "0.75": { "n": 96, "accuracy": 0.9166666666666666, "min_confidence": 0.9661396741867065 }, "1.0": { "n": 128, "accuracy": 0.84375, "min_confidence": 0.5787264704704285 } } } } } } ], "checkpoints": { "checkpoint.pt": { "path": "/home/andy/ai/opensysone/runs/20260916T193803Z-24h/training/checkpoint.pt", "bytes": 202309085, "modified_utc": "2026-09-17T01:59:13.373975+00:00", "sha256": "ab70dcb62432039d93ec71e200157250dc59c563404237e4dfad3662fd30f44f", "metadata": { "step": 6000, "best_validation_macro_nll": 0.25676974726429014, "selection_metric": "crossfit_temperature_nll_v1", "best_validation_selection_score": 0.2552943737691716, "stale_evaluations": 8, "source_commit": "4a60423c39d70f8d50472ce4f4f7fa4a4bd9fce1", "config": { "command": "train", "model": "/home/andy/ai/models/opensysone/Qwen3.5-2B-15852e8c", "dataset": "/home/andy/ai/opensysone/data/public-decisions-v1-20260916", "output": "/home/andy/ai/opensysone/runs/20260916T193803Z-24h/training", "resume": "/home/andy/ai/opensysone/runs/20260916T182352Z-train/artifacts/checkpoint.pt", "warm_start": null, "steps": null, "epochs": 3, "rank": 16, "alpha": 32.0, "head_only": false, "two_pass": false, "lr": 0.0001, "head_lr": 0.0001, "seed": 431, "effective_batch": 4, "branch_batch_size": 2, "max_tokens": 768, "save_steps": 250, "save_seconds": 900, "eval_steps": 500, "validation_per_family": 128, "patience": 8, "deadline": "2026-09-17T16:00:00+00:00", "schedule_steps": null, "selection_metric": "crossfit_temperature_nll_v1", "adapters": true }, "initialization": { "kind": "resume", "parent_checkpoint": "/home/andy/ai/opensysone/runs/20260916T182352Z-train/artifacts/checkpoint.pt", "parent_checkpoint_sha256": "af5790ae2f2b56477ebbdf6ab9c418d895e48d2bf5a6416e11b6e9863ad1db55", "restores_optimizer": true, "restores_rng": true, "parent_step": 40, "parent_source_commit": "f1c9322c5313021cf9f90984c8a451bea165f633", "parent_initialization": null, "selection_policy_change": { "from": "raw_nll", "to": "crossfit_temperature_nll_v1", "optimizer_and_rng_unchanged": true } } }, "has_optimizer": true, "has_rng": true }, "best.pt": { "path": "/home/andy/ai/opensysone/runs/20260916T193803Z-24h/training/best.pt", "bytes": 67383903, "modified_utc": "2026-09-16T21:45:31.514410+00:00", "sha256": "be08efdc9121a64547b5c9331d5908eeb0978c4fbc962885fd34f9758ec89d11", "metadata": { "step": 2000, "best_validation_macro_nll": 0.25676974726429014, "selection_metric": "crossfit_temperature_nll_v1", "best_validation_selection_score": 0.2552943737691716, "stale_evaluations": 0, "source_commit": "4a60423c39d70f8d50472ce4f4f7fa4a4bd9fce1", "config": { "command": "train", "model": "/home/andy/ai/models/opensysone/Qwen3.5-2B-15852e8c", "dataset": "/home/andy/ai/opensysone/data/public-decisions-v1-20260916", "output": "/home/andy/ai/opensysone/runs/20260916T193803Z-24h/training", "resume": "/home/andy/ai/opensysone/runs/20260916T182352Z-train/artifacts/checkpoint.pt", "warm_start": null, "steps": null, "epochs": 3, "rank": 16, "alpha": 32.0, "head_only": false, "two_pass": false, "lr": 0.0001, "head_lr": 0.0001, "seed": 431, "effective_batch": 4, "branch_batch_size": 2, "max_tokens": 768, "save_steps": 250, "save_seconds": 900, "eval_steps": 500, "validation_per_family": 128, "patience": 8, "deadline": "2026-09-17T16:00:00+00:00", "schedule_steps": null, "selection_metric": "crossfit_temperature_nll_v1", "adapters": true }, "initialization": { "kind": "resume", "parent_checkpoint": "/home/andy/ai/opensysone/runs/20260916T182352Z-train/artifacts/checkpoint.pt", "parent_checkpoint_sha256": "af5790ae2f2b56477ebbdf6ab9c418d895e48d2bf5a6416e11b6e9863ad1db55", "restores_optimizer": true, "restores_rng": true, "parent_step": 40, "parent_source_commit": "f1c9322c5313021cf9f90984c8a451bea165f633", "parent_initialization": null, "selection_policy_change": { "from": "raw_nll", "to": "crossfit_temperature_nll_v1", "optimizer_and_rng_unchanged": true } } }, "has_optimizer": false, "has_rng": false } }, "final_correctness": { "branch_chunks_1_probability_max_abs": 4.76837158203125e-07, "branch_chunks_2_probability_max_abs": 0.0, "branch_chunks_4_probability_max_abs": 6.556510925292969e-07, "question_isolation_probability_max_abs": 2.0489096641540527e-08, "candidate_permutation_probability_max_abs": 1.1920928955078125e-07, "repeat_probability_max_abs": 0.0, "tolerance_probability_abs": 0.0001 }, "best_selection": { "metric": "crossfit_temperature_nll_v1", "score": 0.2552943737691716, "raw_macro_nll": 0.25676974955497256, "accuracy": 0.8984375, "folds": [ { "fold": 0, "temperature_index": 48, "temperature": 1.2705741052085413, "training_macro_nll": 0.27533710550694335, "heldout_ids": [ "arc:validation:ARC-Challenge:122", "arc:validation:ARC-Challenge:126", "arc:validation:ARC-Challenge:175", "arc:validation:ARC-Challenge:187", "arc:validation:ARC-Challenge:189", "arc:validation:ARC-Challenge:192", "arc:validation:ARC-Challenge:208", "arc:validation:ARC-Challenge:238", "arc:validation:ARC-Challenge:290", "arc:validation:ARC-Challenge:293", "arc:validation:ARC-Challenge:35", "arc:validation:ARC-Challenge:43", "arc:validation:ARC-Challenge:58", "arc:validation:ARC-Challenge:7", "arc:validation:ARC-Easy:160", "arc:validation:ARC-Easy:201", "arc:validation:ARC-Easy:229", "arc:validation:ARC-Easy:238", "arc:validation:ARC-Easy:253", "arc:validation:ARC-Easy:259", "arc:validation:ARC-Easy:272", "arc:validation:ARC-Easy:298", "arc:validation:ARC-Easy:329", "arc:validation:ARC-Easy:331", "arc:validation:ARC-Easy:337", "arc:validation:ARC-Easy:362", "arc:validation:ARC-Easy:443", "arc:validation:ARC-Easy:500", "arc:validation:ARC-Easy:545", "arc:validation:ARC-Easy:68", "arc:validation:ARC-Easy:7", "arc:validation:ARC-Easy:74", "banking:train:1053", "banking:train:1594", "banking:train:1747", "banking:train:202", "banking:train:2377", "banking:train:2528", "banking:train:2546", "banking:train:2715", "banking:train:3220", "banking:train:3499", "banking:train:3806", "banking:train:3838", "banking:train:3993", "banking:train:4220", "banking:train:5", "banking:train:5192", "banking:train:5394", "banking:train:5846", "banking:train:5863", "banking:train:6032", "banking:train:6508", "banking:train:7579", "banking:train:7600", "banking:train:7987", "banking:train:8155", "banking:train:850", "banking:train:8626", "banking:train:8929", "banking:train:9355", "banking:train:9812", "banking:train:987", "banking:train:9972", "boolq:validation:103", "boolq:validation:1042", "boolq:validation:1059", "boolq:validation:1541", "boolq:validation:1592", "boolq:validation:1927", "boolq:validation:2014", "boolq:validation:2023", "boolq:validation:2098", "boolq:validation:2139", "boolq:validation:2439", "boolq:validation:2485", "boolq:validation:2607", "boolq:validation:2611", "boolq:validation:2720", "boolq:validation:2751", "boolq:validation:282", "boolq:validation:2843", "boolq:validation:2953", "boolq:validation:301", "boolq:validation:3089", "boolq:validation:313", "boolq:validation:3201", "boolq:validation:322", "boolq:validation:3231", "boolq:validation:39", "boolq:validation:461", "boolq:validation:558", "boolq:validation:589", "boolq:validation:731", "boolq:validation:883", "boolq:validation:915", "snli:validation:1156", "snli:validation:1330", "snli:validation:1559", "snli:validation:1735", "snli:validation:1876", "snli:validation:2251", "snli:validation:2257", "snli:validation:2377", "snli:validation:2881", "snli:validation:291", "snli:validation:300", "snli:validation:3049", "snli:validation:3496", "snli:validation:3871", "snli:validation:4414", "snli:validation:5140", "snli:validation:5440", "snli:validation:5806", "snli:validation:6442", "snli:validation:6772", "snli:validation:6949", "snli:validation:7228", "snli:validation:7465", "snli:validation:7510", "snli:validation:7726", "snli:validation:8095", "snli:validation:8143", "snli:validation:8623", "snli:validation:8962", "snli:validation:9451", "snli:validation:9574", "snli:validation:9655" ], "heldout_group_count": 128, "training_group_count": 384 }, { "fold": 1, "temperature_index": 47, "temperature": 1.2050359403717974, "training_macro_nll": 0.25585027399066257, "heldout_ids": [ "arc:validation:ARC-Challenge:101", "arc:validation:ARC-Challenge:129", "arc:validation:ARC-Challenge:148", "arc:validation:ARC-Challenge:156", "arc:validation:ARC-Challenge:167", "arc:validation:ARC-Challenge:249", "arc:validation:ARC-Challenge:263", "arc:validation:ARC-Challenge:266", "arc:validation:ARC-Challenge:267", "arc:validation:ARC-Challenge:28", "arc:validation:ARC-Challenge:281", "arc:validation:ARC-Challenge:57", "arc:validation:ARC-Challenge:63", "arc:validation:ARC-Challenge:64", "arc:validation:ARC-Challenge:86", "arc:validation:ARC-Challenge:99", "arc:validation:ARC-Easy:106", "arc:validation:ARC-Easy:151", "arc:validation:ARC-Easy:157", "arc:validation:ARC-Easy:166", "arc:validation:ARC-Easy:202", "arc:validation:ARC-Easy:219", "arc:validation:ARC-Easy:224", "arc:validation:ARC-Easy:322", "arc:validation:ARC-Easy:344", "arc:validation:ARC-Easy:382", "arc:validation:ARC-Easy:409", "arc:validation:ARC-Easy:455", "arc:validation:ARC-Easy:50", "arc:validation:ARC-Easy:538", "arc:validation:ARC-Easy:544", "arc:validation:ARC-Easy:552", "banking:train:1050", "banking:train:1131", "banking:train:1190", "banking:train:124", "banking:train:137", "banking:train:1493", "banking:train:1498", "banking:train:1624", "banking:train:2614", "banking:train:2739", "banking:train:3055", "banking:train:306", "banking:train:3312", "banking:train:3724", "banking:train:4070", "banking:train:476", "banking:train:4882", "banking:train:5075", "banking:train:5332", "banking:train:5445", "banking:train:6096", "banking:train:746", "banking:train:7828", "banking:train:8176", "banking:train:8315", "banking:train:8653", "banking:train:9019", "banking:train:9370", "banking:train:9454", "banking:train:9595", "banking:train:9862", "banking:train:9963", "boolq:validation:1189", "boolq:validation:1320", "boolq:validation:1326", "boolq:validation:1381", "boolq:validation:1410", "boolq:validation:149", "boolq:validation:1513", "boolq:validation:1732", "boolq:validation:1739", "boolq:validation:1741", "boolq:validation:1814", "boolq:validation:1848", "boolq:validation:1940", "boolq:validation:2077", "boolq:validation:2091", "boolq:validation:2148", "boolq:validation:2215", "boolq:validation:2251", "boolq:validation:2283", "boolq:validation:2321", "boolq:validation:2398", "boolq:validation:253", "boolq:validation:2575", "boolq:validation:2907", "boolq:validation:2909", "boolq:validation:2978", "boolq:validation:359", "boolq:validation:394", "boolq:validation:436", "boolq:validation:579", "boolq:validation:807", "boolq:validation:949", "snli:validation:2389", "snli:validation:2485", "snli:validation:255", "snli:validation:3169", "snli:validation:333", "snli:validation:3700", "snli:validation:3709", "snli:validation:4093", "snli:validation:420", "snli:validation:4492", "snli:validation:4516", "snli:validation:4648", "snli:validation:4888", "snli:validation:5185", "snli:validation:5332", "snli:validation:5947", "snli:validation:6124", "snli:validation:660", "snli:validation:6619", "snli:validation:6898", "snli:validation:7387", "snli:validation:7492", "snli:validation:7544", "snli:validation:7996", "snli:validation:8074", "snli:validation:8554", "snli:validation:8971", "snli:validation:9163", "snli:validation:9277", "snli:validation:9490", "snli:validation:9595", "snli:validation:9688" ], "heldout_group_count": 128, "training_group_count": 384 }, { "fold": 2, "temperature_index": 45, "temperature": 1.0839269140212033, "training_macro_nll": 0.22881493804363362, "heldout_ids": [ "arc:validation:ARC-Challenge:120", "arc:validation:ARC-Challenge:130", "arc:validation:ARC-Challenge:194", "arc:validation:ARC-Challenge:218", "arc:validation:ARC-Challenge:235", "arc:validation:ARC-Challenge:272", "arc:validation:ARC-Challenge:277", "arc:validation:ARC-Challenge:48", "arc:validation:ARC-Challenge:88", "arc:validation:ARC-Easy:115", "arc:validation:ARC-Easy:12", "arc:validation:ARC-Easy:131", "arc:validation:ARC-Easy:145", "arc:validation:ARC-Easy:179", "arc:validation:ARC-Easy:18", "arc:validation:ARC-Easy:186", "arc:validation:ARC-Easy:191", "arc:validation:ARC-Easy:215", "arc:validation:ARC-Easy:283", "arc:validation:ARC-Easy:314", "arc:validation:ARC-Easy:315", "arc:validation:ARC-Easy:378", "arc:validation:ARC-Easy:384", "arc:validation:ARC-Easy:403", "arc:validation:ARC-Easy:432", "arc:validation:ARC-Easy:439", "arc:validation:ARC-Easy:448", "arc:validation:ARC-Easy:461", "arc:validation:ARC-Easy:469", "arc:validation:ARC-Easy:470", "arc:validation:ARC-Easy:504", "arc:validation:ARC-Easy:561", "banking:train:1142", "banking:train:148", "banking:train:1633", "banking:train:1927", "banking:train:2186", "banking:train:2251", "banking:train:23", "banking:train:3048", "banking:train:3665", "banking:train:3952", "banking:train:3999", "banking:train:4364", "banking:train:4619", "banking:train:5049", "banking:train:5335", "banking:train:549", "banking:train:5957", "banking:train:5985", "banking:train:6132", "banking:train:6240", "banking:train:6488", "banking:train:7065", "banking:train:7749", "banking:train:8470", "banking:train:8761", "banking:train:895", "banking:train:9256", "banking:train:9406", "banking:train:9450", "banking:train:9467", "banking:train:9588", "banking:train:9716", "boolq:validation:1130", "boolq:validation:1167", "boolq:validation:124", "boolq:validation:1255", "boolq:validation:1256", "boolq:validation:1373", "boolq:validation:15", "boolq:validation:155", "boolq:validation:1568", "boolq:validation:16", "boolq:validation:1654", "boolq:validation:1702", "boolq:validation:1792", "boolq:validation:1818", "boolq:validation:210", "boolq:validation:2247", "boolq:validation:2259", "boolq:validation:2362", "boolq:validation:2443", "boolq:validation:2524", "boolq:validation:2553", "boolq:validation:2623", "boolq:validation:2660", "boolq:validation:2669", "boolq:validation:2679", "boolq:validation:2832", "boolq:validation:357", "boolq:validation:453", "boolq:validation:49", "boolq:validation:808", "boolq:validation:90", "boolq:validation:910", "snli:validation:1612", "snli:validation:168", "snli:validation:1693", "snli:validation:2134", "snli:validation:2248", "snli:validation:2422", "snli:validation:2587", "snli:validation:3007", "snli:validation:3097", "snli:validation:3772", "snli:validation:4012", "snli:validation:4168", "snli:validation:4537", "snli:validation:4681", "snli:validation:5050", "snli:validation:5632", "snli:validation:5761", "snli:validation:6055", "snli:validation:6283", "snli:validation:6715", "snli:validation:7117", "snli:validation:7192", "snli:validation:7507", "snli:validation:7594", "snli:validation:7924", "snli:validation:7954", "snli:validation:8035", "snli:validation:8110", "snli:validation:8995", "snli:validation:949", "snli:validation:9523", "snli:validation:9589" ], "heldout_group_count": 128, "training_group_count": 384 }, { "fold": 3, "temperature_index": 47, "temperature": 1.2050359403717974, "training_macro_nll": 0.24681503020373022, "heldout_ids": [ "arc:validation:ARC-Challenge:117", "arc:validation:ARC-Challenge:135", "arc:validation:ARC-Challenge:139", "arc:validation:ARC-Challenge:154", "arc:validation:ARC-Challenge:174", "arc:validation:ARC-Challenge:199", "arc:validation:ARC-Challenge:287", "arc:validation:ARC-Challenge:295", "arc:validation:ARC-Challenge:4", "arc:validation:ARC-Challenge:50", "arc:validation:ARC-Challenge:51", "arc:validation:ARC-Challenge:61", "arc:validation:ARC-Easy:107", "arc:validation:ARC-Easy:121", "arc:validation:ARC-Easy:124", "arc:validation:ARC-Easy:171", "arc:validation:ARC-Easy:235", "arc:validation:ARC-Easy:241", "arc:validation:ARC-Easy:279", "arc:validation:ARC-Easy:289", "arc:validation:ARC-Easy:37", "arc:validation:ARC-Easy:389", "arc:validation:ARC-Easy:394", "arc:validation:ARC-Easy:399", "arc:validation:ARC-Easy:423", "arc:validation:ARC-Easy:492", "arc:validation:ARC-Easy:499", "arc:validation:ARC-Easy:523", "arc:validation:ARC-Easy:533", "arc:validation:ARC-Easy:569", "arc:validation:ARC-Easy:69", "arc:validation:ARC-Easy:93", "banking:train:2049", "banking:train:2350", "banking:train:2497", "banking:train:2506", "banking:train:2720", "banking:train:2788", "banking:train:2967", "banking:train:3502", "banking:train:3726", "banking:train:4014", "banking:train:4735", "banking:train:4803", "banking:train:4917", "banking:train:5477", "banking:train:5851", "banking:train:6551", "banking:train:6839", "banking:train:6926", "banking:train:7809", "banking:train:8092", "banking:train:8186", "banking:train:8215", "banking:train:8667", "banking:train:8752", "banking:train:8763", "banking:train:8851", "banking:train:8917", "banking:train:9145", "banking:train:9279", "banking:train:9676", "banking:train:9837", "banking:train:9967", "boolq:validation:106", "boolq:validation:1065", "boolq:validation:1097", "boolq:validation:1336", "boolq:validation:1350", "boolq:validation:1383", "boolq:validation:1458", "boolq:validation:1641", "boolq:validation:1873", "boolq:validation:1930", "boolq:validation:1989", "boolq:validation:2103", "boolq:validation:234", "boolq:validation:2355", "boolq:validation:2384", "boolq:validation:2440", "boolq:validation:2444", "boolq:validation:2478", "boolq:validation:2872", "boolq:validation:2879", "boolq:validation:2924", "boolq:validation:2971", "boolq:validation:3164", "boolq:validation:4", "boolq:validation:415", "boolq:validation:419", "boolq:validation:428", "boolq:validation:512", "boolq:validation:563", "boolq:validation:643", "boolq:validation:696", "boolq:validation:874", "snli:validation:1102", "snli:validation:1240", "snli:validation:1261", "snli:validation:1738", "snli:validation:1978", "snli:validation:2563", "snli:validation:2659", "snli:validation:2767", "snli:validation:2836", "snli:validation:3232", "snli:validation:3655", "snli:validation:3913", "snli:validation:405", "snli:validation:4051", "snli:validation:4300", "snli:validation:4846", "snli:validation:498", "snli:validation:5224", "snli:validation:5575", "snli:validation:6598", "snli:validation:666", "snli:validation:6925", "snli:validation:6964", "snli:validation:7045", "snli:validation:7351", "snli:validation:7588", "snli:validation:7723", "snli:validation:7750", "snli:validation:7909", "snli:validation:8974", "snli:validation:9385", "snli:validation:9646" ], "heldout_group_count": 128, "training_group_count": 384 } ], "per_family": { "arc": { "count": 128, "score": 0.3379051798328397, "raw_macro_nll": 0.33955257651665527, "accuracy": 0.890625 }, "banking": { "count": 128, "score": 0.06793271948328157, "raw_macro_nll": 0.06537141992655102, "accuracy": 0.9609375 }, "boolq": { "count": 128, "score": 0.33084980282335186, "raw_macro_nll": 0.34265219288051024, "accuracy": 0.84375 }, "snli": { "count": 128, "score": 0.28448979293721316, "raw_macro_nll": 0.2795028088961736, "accuracy": 0.8984375 } }, "policy": { "id": "crossfit_temperature_nll_v1", "folds": 4, "seed": 431, "grouping": "source group, nested within task family; groups shuffled within sorted families and assigned round-robin", "temperature_grid": { "count": 101, "log10_min": -1.0, "log10_max": 1.3, "arithmetic": "Python float64" }, "temperature_fit": "minimum macro-family NLL on the other three folds; smallest temperature wins ties", "score": "macro-family mean of all held-out decision NLLs", "deployment_temperature": "fit afresh on reserved calibration only after model selection" } }, "small_file_list": [ "exit_code", "plan.json", "startup-verification.json", "state.json", "supervisor.log", "training/best_validation_predictions.json", "training/best_validation_selection.json", "training/correctness_final.json", "training/correctness_initial.json", "training/data_filter.json", "training/inherited_validation_selection.json", "training/initial_validation_selection.json", "training/manifest.json", "training/resumed_initial_predictions.json", "training/summary.json", "training/training.jsonl", "training/validation.jsonl", "training/validation_step_000040_predictions.json", "training/validation_step_000500_predictions.json", "training/validation_step_001000_predictions.json", "training/validation_step_001500_predictions.json", "training/validation_step_002000_predictions.json", "training/validation_step_002500_predictions.json", "training/validation_step_003000_predictions.json", "training/validation_step_003500_predictions.json", "training/validation_step_004000_predictions.json", "training/validation_step_004500_predictions.json", "training/validation_step_005000_predictions.json", "training/validation_step_005500_predictions.json", "training/validation_step_006000_predictions.json", "training.log", "training_exit_code" ], "log_error_lines": { "supervisor.log": [], "training.log": [] } }