| { |
| "checked_utc": "2026-09-17T02:11:24.308357+00:00", |
| "host": "spark-3e2a", |
| "campaign": "/home/andy/ai/opensysone/runs/20260916T193803Z-24h", |
| "plan": { |
| "resume_checkpoint": "/home/andy/ai/opensysone/runs/20260916T182352Z-train/artifacts/checkpoint.pt", |
| "resume_sha256": "af5790ae2f2b56477ebbdf6ab9c418d895e48d2bf5a6416e11b6e9863ad1db55", |
| "final_deadline": "2026-09-17T18:16:10+00:00", |
| "eval_reserve_seconds": 7200, |
| "evaluation_estimate": { |
| "pilot_validation_decisions": 512, |
| "pilot_validation_seconds": 130.97493624687195, |
| "evaluation_decisions": 6654, |
| "estimated_prediction_seconds": 1702.162550364621 |
| }, |
| "port": 18081, |
| "training_deadline": "2026-09-17T16:00:00+00:00", |
| "train_only": true, |
| "selection_metric": "crossfit_temperature_nll_v1", |
| "inference_max_tokens": 1024, |
| "created_utc": "2026-09-16T19:38:03.883204+00:00", |
| "source_commit": "4a60423c39d70f8d50472ce4f4f7fa4a4bd9fce1" |
| }, |
| "state": { |
| "supervisor_pid": 286092, |
| "campaign": "/home/andy/ai/opensysone/runs/20260916T193803Z-24h", |
| "stage": "training_complete", |
| "status": "training_complete", |
| "source_commit": "4a60423c39d70f8d50472ce4f4f7fa4a4bd9fce1", |
| "child_pid": null, |
| "child_command": [ |
| "/home/andy/ai/envs/opensysone/bin/python", |
| "/home/andy/projects/opensysone/experiment.py", |
| "train", |
| "--output", |
| "/home/andy/ai/opensysone/runs/20260916T193803Z-24h/training", |
| "--resume", |
| "/home/andy/ai/opensysone/runs/20260916T182352Z-train/artifacts/checkpoint.pt", |
| "--deadline", |
| "2026-09-17T16:00:00+00:00", |
| "--model", |
| "/home/andy/ai/models/opensysone/Qwen3.5-2B-15852e8c", |
| "--dataset", |
| "/home/andy/ai/opensysone/data/public-decisions-v1-20260916", |
| "--epochs", |
| "3", |
| "--rank", |
| "16", |
| "--alpha", |
| "32", |
| "--lr", |
| "0.0001", |
| "--head-lr", |
| "0.0001", |
| "--seed", |
| "431", |
| "--effective-batch", |
| "4", |
| "--branch-batch-size", |
| "2", |
| "--max-tokens", |
| "768", |
| "--save-steps", |
| "250", |
| "--save-seconds", |
| "900", |
| "--eval-steps", |
| "500", |
| "--validation-per-family", |
| "128", |
| "--patience", |
| "8", |
| "--selection-metric", |
| "crossfit_temperature_nll_v1" |
| ], |
| "stage_started_utc": "2026-09-16T19:38:04.584070+00:00", |
| "heartbeat_utc": "2026-09-17T01:59:24.923675+00:00", |
| "training_exit_code": 0, |
| "training_output": "/home/andy/ai/opensysone/runs/20260916T193803Z-24h/training", |
| "resume_checkpoint": "/home/andy/ai/opensysone/runs/20260916T193803Z-24h/training/checkpoint.pt", |
| "training_summary": { |
| "completed_steps": 6000, |
| "best_validation_macro_nll": 0.25676974726429014, |
| "status": "validation_early_stop", |
| "peak_cuda_allocated_bytes": 8786432512, |
| "selection_metric": "crossfit_temperature_nll_v1", |
| "best_validation_selection_score": 0.2552943737691716 |
| }, |
| "train_only": true, |
| "best_checkpoint": "/home/andy/ai/opensysone/runs/20260916T193803Z-24h/training/best.pt", |
| "best_sha256": "be08efdc9121a64547b5c9331d5908eeb0978c4fbc962885fd34f9758ec89d11", |
| "finished_utc": "2026-09-17T01:59:29.971800+00:00", |
| "selection": "Await cross-candidate validation selection before calibration/test/holdout", |
| "best_validation_predictions": "/home/andy/ai/opensysone/runs/20260916T193803Z-24h/training/best_validation_predictions.json", |
| "best_validation_predictions_sha256": "980c9df18461f5de79e54701bfee1b5fbff19dd3b85125e64f619a88c60f80e4" |
| }, |
| "summary": { |
| "status": "validation_early_stop", |
| "completed_steps": 6000, |
| "target_steps": 30703, |
| "initial_validation": { |
| "overall": { |
| "n": 512, |
| "accuracy": 0.828125, |
| "nll": 0.4981533203378369, |
| "brier_multiclass_sum": 0.2607127369212848, |
| "ece_top_label_10_equal_width_bins": 0.04818323429208249, |
| "reliability_bins": [ |
| { |
| "count": 0, |
| "confidence_sum": 0.0, |
| "correct_sum": 0.0 |
| }, |
| { |
| "count": 0, |
| "confidence_sum": 0.0, |
| "correct_sum": 0.0 |
| }, |
| { |
| "count": 2, |
| "confidence_sum": 0.5327933728694916, |
| "correct_sum": 1.0 |
| }, |
| { |
| "count": 4, |
| "confidence_sum": 1.4359374046325684, |
| "correct_sum": 3.0 |
| }, |
| { |
| "count": 17, |
| "confidence_sum": 7.9667291939258575, |
| "correct_sum": 7.0 |
| }, |
| { |
| "count": 55, |
| "confidence_sum": 30.154281735420227, |
| "correct_sum": 35.0 |
| }, |
| { |
| "count": 53, |
| "confidence_sum": 34.67930006980896, |
| "correct_sum": 34.0 |
| }, |
| { |
| "count": 46, |
| "confidence_sum": 34.177616238594055, |
| "correct_sum": 36.0 |
| }, |
| { |
| "count": 54, |
| "confidence_sum": 46.13812476396561, |
| "correct_sum": 44.0 |
| }, |
| { |
| "count": 281, |
| "confidence_sum": 276.18629068136215, |
| "correct_sum": 264.0 |
| } |
| ], |
| "accuracy_vs_coverage": { |
| "0.25": { |
| "n": 128, |
| "accuracy": 0.96875, |
| "min_confidence": 0.9978359341621399 |
| }, |
| "0.5": { |
| "n": 256, |
| "accuracy": 0.94140625, |
| "min_confidence": 0.9372246861457825 |
| }, |
| "0.75": { |
| "n": 384, |
| "accuracy": 0.9010416666666666, |
| "min_confidence": 0.6965566873550415 |
| }, |
| "1.0": { |
| "n": 512, |
| "accuracy": 0.828125, |
| "min_confidence": 0.2647492587566376 |
| } |
| } |
| }, |
| "per_family": { |
| "arc": { |
| "n": 128, |
| "accuracy": 0.8125, |
| "nll": 0.4990136543534618, |
| "brier_multiclass_sum": 0.26976813507500774, |
| "ece_top_label_10_equal_width_bins": 0.048822794342413545, |
| "reliability_bins": [ |
| { |
| "count": 0, |
| "confidence_sum": 0.0, |
| "correct_sum": 0.0 |
| }, |
| { |
| "count": 0, |
| "confidence_sum": 0.0, |
| "correct_sum": 0.0 |
| }, |
| { |
| "count": 2, |
| "confidence_sum": 0.5327933728694916, |
| "correct_sum": 1.0 |
| }, |
| { |
| "count": 4, |
| "confidence_sum": 1.4359374046325684, |
| "correct_sum": 3.0 |
| }, |
| { |
| "count": 8, |
| "confidence_sum": 3.675512135028839, |
| "correct_sum": 2.0 |
| }, |
| { |
| "count": 9, |
| "confidence_sum": 4.856119275093079, |
| "correct_sum": 6.0 |
| }, |
| { |
| "count": 13, |
| "confidence_sum": 8.502536118030548, |
| "correct_sum": 8.0 |
| }, |
| { |
| "count": 13, |
| "confidence_sum": 9.665025651454926, |
| "correct_sum": 10.0 |
| }, |
| { |
| "count": 28, |
| "confidence_sum": 24.128599762916565, |
| "correct_sum": 24.0 |
| }, |
| { |
| "count": 51, |
| "confidence_sum": 49.567454636096954, |
| "correct_sum": 50.0 |
| } |
| ], |
| "accuracy_vs_coverage": { |
| "0.25": { |
| "n": 32, |
| "accuracy": 1.0, |
| "min_confidence": 0.974073588848114 |
| }, |
| "0.5": { |
| "n": 64, |
| "accuracy": 0.953125, |
| "min_confidence": 0.8664795756340027 |
| }, |
| "0.75": { |
| "n": 96, |
| "accuracy": 0.8958333333333334, |
| "min_confidence": 0.6717178225517273 |
| }, |
| "1.0": { |
| "n": 128, |
| "accuracy": 0.8125, |
| "min_confidence": 0.2647492587566376 |
| } |
| } |
| }, |
| "banking": { |
| "n": 128, |
| "accuracy": 0.8828125, |
| "nll": 0.2938805352743866, |
| "brier_multiclass_sum": 0.15680863450549035, |
| "ece_top_label_10_equal_width_bins": 0.060145952040329576, |
| "reliability_bins": [ |
| { |
| "count": 0, |
| "confidence_sum": 0.0, |
| "correct_sum": 0.0 |
| }, |
| { |
| "count": 0, |
| "confidence_sum": 0.0, |
| "correct_sum": 0.0 |
| }, |
| { |
| "count": 0, |
| "confidence_sum": 0.0, |
| "correct_sum": 0.0 |
| }, |
| { |
| "count": 0, |
| "confidence_sum": 0.0, |
| "correct_sum": 0.0 |
| }, |
| { |
| "count": 1, |
| "confidence_sum": 0.47516098618507385, |
| "correct_sum": 0.0 |
| }, |
| { |
| "count": 6, |
| "confidence_sum": 3.3597115874290466, |
| "correct_sum": 1.0 |
| }, |
| { |
| "count": 5, |
| "confidence_sum": 3.223210096359253, |
| "correct_sum": 3.0 |
| }, |
| { |
| "count": 3, |
| "confidence_sum": 2.2593541145324707, |
| "correct_sum": 1.0 |
| }, |
| { |
| "count": 7, |
| "confidence_sum": 5.898111343383789, |
| "correct_sum": 4.0 |
| }, |
| { |
| "count": 106, |
| "confidence_sum": 105.48313373327255, |
| "correct_sum": 104.0 |
| } |
| ], |
| "accuracy_vs_coverage": { |
| "0.25": { |
| "n": 32, |
| "accuracy": 1.0, |
| "min_confidence": 0.9999927282333374 |
| }, |
| "0.5": { |
| "n": 64, |
| "accuracy": 1.0, |
| "min_confidence": 0.9998255372047424 |
| }, |
| "0.75": { |
| "n": 96, |
| "accuracy": 0.9895833333333334, |
| "min_confidence": 0.9847974181175232 |
| }, |
| "1.0": { |
| "n": 128, |
| "accuracy": 0.8828125, |
| "min_confidence": 0.47516098618507385 |
| } |
| } |
| }, |
| "boolq": { |
| "n": 128, |
| "accuracy": 0.796875, |
| "nll": 0.730819056238805, |
| "brier_multiclass_sum": 0.3320798331012733, |
| "ece_top_label_10_equal_width_bins": 0.1302144955843687, |
| "reliability_bins": [ |
| { |
| "count": 0, |
| "confidence_sum": 0.0, |
| "correct_sum": 0.0 |
| }, |
| { |
| "count": 0, |
| "confidence_sum": 0.0, |
| "correct_sum": 0.0 |
| }, |
| { |
| "count": 0, |
| "confidence_sum": 0.0, |
| "correct_sum": 0.0 |
| }, |
| { |
| "count": 0, |
| "confidence_sum": 0.0, |
| "correct_sum": 0.0 |
| }, |
| { |
| "count": 0, |
| "confidence_sum": 0.0, |
| "correct_sum": 0.0 |
| }, |
| { |
| "count": 9, |
| "confidence_sum": 5.026383757591248, |
| "correct_sum": 6.0 |
| }, |
| { |
| "count": 7, |
| "confidence_sum": 4.494362056255341, |
| "correct_sum": 4.0 |
| }, |
| { |
| "count": 6, |
| "confidence_sum": 4.508205056190491, |
| "correct_sum": 3.0 |
| }, |
| { |
| "count": 10, |
| "confidence_sum": 8.55346018075943, |
| "correct_sum": 7.0 |
| }, |
| { |
| "count": 96, |
| "confidence_sum": 94.13781189918518, |
| "correct_sum": 82.0 |
| } |
| ], |
| "accuracy_vs_coverage": { |
| "0.25": { |
| "n": 32, |
| "accuracy": 0.875, |
| "min_confidence": 0.9985519051551819 |
| }, |
| "0.5": { |
| "n": 64, |
| "accuracy": 0.890625, |
| "min_confidence": 0.9844649434089661 |
| }, |
| "0.75": { |
| "n": 96, |
| "accuracy": 0.8541666666666666, |
| "min_confidence": 0.9054988026618958 |
| }, |
| "1.0": { |
| "n": 128, |
| "accuracy": 0.796875, |
| "min_confidence": 0.5037087798118591 |
| } |
| } |
| }, |
| "snli": { |
| "n": 128, |
| "accuracy": 0.8203125, |
| "nll": 0.46890003548469394, |
| "brier_multiclass_sum": 0.284194345003368, |
| "ece_top_label_10_equal_width_bins": 0.10556101333349943, |
| "reliability_bins": [ |
| { |
| "count": 0, |
| "confidence_sum": 0.0, |
| "correct_sum": 0.0 |
| }, |
| { |
| "count": 0, |
| "confidence_sum": 0.0, |
| "correct_sum": 0.0 |
| }, |
| { |
| "count": 0, |
| "confidence_sum": 0.0, |
| "correct_sum": 0.0 |
| }, |
| { |
| "count": 0, |
| "confidence_sum": 0.0, |
| "correct_sum": 0.0 |
| }, |
| { |
| "count": 8, |
| "confidence_sum": 3.8160560727119446, |
| "correct_sum": 5.0 |
| }, |
| { |
| "count": 31, |
| "confidence_sum": 16.912067115306854, |
| "correct_sum": 22.0 |
| }, |
| { |
| "count": 28, |
| "confidence_sum": 18.45919179916382, |
| "correct_sum": 19.0 |
| }, |
| { |
| "count": 24, |
| "confidence_sum": 17.745031416416168, |
| "correct_sum": 22.0 |
| }, |
| { |
| "count": 9, |
| "confidence_sum": 7.557953476905823, |
| "correct_sum": 9.0 |
| }, |
| { |
| "count": 28, |
| "confidence_sum": 26.997890412807465, |
| "correct_sum": 28.0 |
| } |
| ], |
| "accuracy_vs_coverage": { |
| "0.25": { |
| "n": 32, |
| "accuracy": 1.0, |
| "min_confidence": 0.845478892326355 |
| }, |
| "0.5": { |
| "n": 64, |
| "accuracy": 0.96875, |
| "min_confidence": 0.695654571056366 |
| }, |
| "0.75": { |
| "n": 96, |
| "accuracy": 0.8645833333333334, |
| "min_confidence": 0.5729489326477051 |
| }, |
| "1.0": { |
| "n": 128, |
| "accuracy": 0.8203125, |
| "min_confidence": 0.4349120557308197 |
| } |
| } |
| } |
| } |
| }, |
| "best_validation_macro_nll": 0.25676974726429014, |
| "selection_metric": "crossfit_temperature_nll_v1", |
| "best_validation_selection_score": 0.2552943737691716, |
| "training_seconds": 22739.080890835998, |
| "median_step_seconds": 3.5843077870013076, |
| "processed_decisions": 23840, |
| "actual_branch_tokens": 9008732, |
| "padded_branch_tokens": 9051318, |
| "peak_cuda_allocated_bytes": 8786432512, |
| "peak_cuda_reserved_bytes": 8896118784, |
| "final_mem_available_bytes": 114020495360, |
| "correctness": { |
| "branch_chunks_1_probability_max_abs": 4.76837158203125e-07, |
| "branch_chunks_2_probability_max_abs": 0.0, |
| "branch_chunks_4_probability_max_abs": 6.556510925292969e-07, |
| "question_isolation_probability_max_abs": 2.0489096641540527e-08, |
| "candidate_permutation_probability_max_abs": 1.1920928955078125e-07, |
| "repeat_probability_max_abs": 0.0, |
| "tolerance_probability_abs": 0.0001 |
| }, |
| "final_correctness_status": "passed", |
| "checkpoint_sha256": "ab70dcb62432039d93ec71e200157250dc59c563404237e4dfad3662fd30f44f", |
| "best_sha256": "be08efdc9121a64547b5c9331d5908eeb0978c4fbc962885fd34f9758ec89d11" |
| }, |
| "manifest": { |
| "config": { |
| "command": "train", |
| "model": "/home/andy/ai/models/opensysone/Qwen3.5-2B-15852e8c", |
| "dataset": "/home/andy/ai/opensysone/data/public-decisions-v1-20260916", |
| "output": "/home/andy/ai/opensysone/runs/20260916T193803Z-24h/training", |
| "resume": "/home/andy/ai/opensysone/runs/20260916T182352Z-train/artifacts/checkpoint.pt", |
| "warm_start": null, |
| "steps": null, |
| "epochs": 3, |
| "rank": 16, |
| "alpha": 32.0, |
| "head_only": false, |
| "two_pass": false, |
| "lr": 0.0001, |
| "head_lr": 0.0001, |
| "seed": 431, |
| "effective_batch": 4, |
| "branch_batch_size": 2, |
| "max_tokens": 768, |
| "save_steps": 250, |
| "save_seconds": 900, |
| "eval_steps": 500, |
| "validation_per_family": 128, |
| "patience": 8, |
| "deadline": "2026-09-17T16:00:00+00:00", |
| "schedule_steps": null, |
| "selection_metric": "crossfit_temperature_nll_v1", |
| "adapters": true |
| }, |
| "pid": 286131, |
| "hostname": "spark-3e2a", |
| "started_utc": "2026-09-16T19:38:06.219880+00:00", |
| "git_commit": "4a60423c39d70f8d50472ce4f4f7fa4a4bd9fce1", |
| "git_status": "", |
| "source_sha256": { |
| "jev_harness.py": "4d4e979cb7ae352bcdacaaa6d64045e6b5e550b1a9721d4bad545045bee6c67f", |
| "smoke_data.py": "06b3cbac1c8c4a86b8aecbee4459073cc3e46d4ddcd576392f3cb4805924f815", |
| "experiment.py": "c036cbe9fbf6f71c1bb141dcfbecfb06a2f4bd697b070d2274b1931480e92031", |
| "training_model.py": "d5b0aefeeb5290816bc0b669aa0a8cbbe27f6a12b9cb23c141ac9b9ae9ee4e65", |
| "smoke_train.py": "8cdeb2b397177fc9c26638aaa871501ddab8e3871aa1573ecd98f66590f5c228", |
| "decision_model.py": "a3d8aeb02a1ac765c6cc30ff175acad0664560f01ab5403e22cade924d17371e", |
| "selection.py": "be0a7a8496b5b830aa572ceba93606320f442063fd38180503fd6980dc1c578f", |
| "scripts/download_model.py": "72ad9a5de44d09e2ee4ed8afb7c3c0ff6fb48987410a3f7f0368353572bf1f4d", |
| "scripts/download_candidate.py": "d06a2c01be0cf6577f927fb37e3bc1eab014949fd934e4f4d9825674adba608e", |
| "scripts/run_precision.sh": "766d82b30cf3e83951f662685b4472ee053c7fbc171c4ff140823bf4d1c2782f", |
| "scripts/campaign_status.py": "1500b5e24f06231c7aafd7277aefd840582e35997e265f79db93614828d34411", |
| "scripts/verify_artifact.py": "9833350e9d72c0065b15206bb71c5a8b5a6b3185219db90074e369ede563a985", |
| "scripts/start_spark_candidate.sh": "c2ca18b008a144de7cb264c9fcca634d68c4a8a317db3638e8a70b3dbcff064b", |
| "scripts/run_experiment.sh": "661a6309fc54a2a8aff918f14a553c72dcb21730bd6a3cfc55d6ccd4700d11d3", |
| "scripts/fleet_campaign.py": "9e8d0511a579ace145b7ad079c11ce9b7465950569e86b22099caec2831d5f21", |
| "scripts/diagnose_parity.py": "08b5d66d316ebda98a2226251a4f952701f86a1d5726ce7a4d7e8fb22755da5a", |
| "scripts/prepare_public_data.py": "32ea84aa719818e1141b958b6ef27a85f7ddb86bfcd7c1585fc487b25253d977", |
| "scripts/run_smoke.sh": "39d59f2120f362729d1c2e384391b82be1e580dcc1aca0eaa7ab231115225574", |
| "scripts/launch_24h.py": "39c26dc10535d3adac09209b0743edf2ed384563e732512ad62f8c6f37161a83", |
| "scripts/investigate_precision.py": "609b744a926d8a45b87ba8d225e5312ee0c71a7096b21bd3589c5846e1dc847c" |
| }, |
| "packages": { |
| "torch": "2.11.0+cu130", |
| "transformers": "5.15.0", |
| "pyarrow": "25.0.1", |
| "numpy": "2.5.2" |
| }, |
| "cuda": "13.0", |
| "gpu": "NVIDIA GB10", |
| "capability": [ |
| 12, |
| 1 |
| ], |
| "cuda_cap_bytes": 17179869184, |
| "initial_mem_available_bytes": 125287321600, |
| "oom_score_adj": "0", |
| "parent_checkpoint_sha256": "af5790ae2f2b56477ebbdf6ab9c418d895e48d2bf5a6416e11b6e9863ad1db55", |
| "initialization": { |
| "kind": "resume", |
| "parent_checkpoint": "/home/andy/ai/opensysone/runs/20260916T182352Z-train/artifacts/checkpoint.pt", |
| "parent_checkpoint_sha256": "af5790ae2f2b56477ebbdf6ab9c418d895e48d2bf5a6416e11b6e9863ad1db55", |
| "restores_optimizer": true, |
| "restores_rng": true, |
| "parent_step": 40, |
| "parent_source_commit": "f1c9322c5313021cf9f90984c8a451bea165f633", |
| "parent_initialization": null, |
| "selection_policy_change": { |
| "from": "raw_nll", |
| "to": "crossfit_temperature_nll_v1", |
| "optimizer_and_rng_unchanged": true |
| } |
| }, |
| "model_provenance": { |
| "model_id": "Qwen/Qwen3.5-2B", |
| "revision": "15852e8c16360a2fea060d615a32b45270f8a8fc", |
| "license": "apache-2.0" |
| }, |
| "data_signature": "dedef2b6406f252c165ee317925935079282f8a8ac96f034e89b07048d79f9e1", |
| "total_parameters": 1898646337, |
| "trainable_parameters": 16821249, |
| "adapter_modules": [ |
| "layers.0.linear_attn.out_proj", |
| "layers.0.linear_attn.in_proj_qkv", |
| "layers.0.linear_attn.in_proj_z", |
| "layers.0.linear_attn.in_proj_b", |
| "layers.0.linear_attn.in_proj_a", |
| "layers.0.mlp.gate_proj", |
| "layers.0.mlp.up_proj", |
| "layers.0.mlp.down_proj", |
| "layers.1.linear_attn.out_proj", |
| "layers.1.linear_attn.in_proj_qkv", |
| "layers.1.linear_attn.in_proj_z", |
| "layers.1.linear_attn.in_proj_b", |
| "layers.1.linear_attn.in_proj_a", |
| "layers.1.mlp.gate_proj", |
| "layers.1.mlp.up_proj", |
| "layers.1.mlp.down_proj", |
| "layers.2.linear_attn.out_proj", |
| "layers.2.linear_attn.in_proj_qkv", |
| "layers.2.linear_attn.in_proj_z", |
| "layers.2.linear_attn.in_proj_b", |
| "layers.2.linear_attn.in_proj_a", |
| "layers.2.mlp.gate_proj", |
| "layers.2.mlp.up_proj", |
| "layers.2.mlp.down_proj", |
| "layers.3.self_attn.q_proj", |
| "layers.3.self_attn.k_proj", |
| "layers.3.self_attn.v_proj", |
| "layers.3.self_attn.o_proj", |
| "layers.3.mlp.gate_proj", |
| "layers.3.mlp.up_proj", |
| "layers.3.mlp.down_proj", |
| "layers.4.linear_attn.out_proj", |
| "layers.4.linear_attn.in_proj_qkv", |
| "layers.4.linear_attn.in_proj_z", |
| "layers.4.linear_attn.in_proj_b", |
| "layers.4.linear_attn.in_proj_a", |
| "layers.4.mlp.gate_proj", |
| "layers.4.mlp.up_proj", |
| "layers.4.mlp.down_proj", |
| "layers.5.linear_attn.out_proj", |
| "layers.5.linear_attn.in_proj_qkv", |
| "layers.5.linear_attn.in_proj_z", |
| "layers.5.linear_attn.in_proj_b", |
| "layers.5.linear_attn.in_proj_a", |
| "layers.5.mlp.gate_proj", |
| "layers.5.mlp.up_proj", |
| "layers.5.mlp.down_proj", |
| "layers.6.linear_attn.out_proj", |
| "layers.6.linear_attn.in_proj_qkv", |
| "layers.6.linear_attn.in_proj_z", |
| "layers.6.linear_attn.in_proj_b", |
| "layers.6.linear_attn.in_proj_a", |
| "layers.6.mlp.gate_proj", |
| "layers.6.mlp.up_proj", |
| "layers.6.mlp.down_proj", |
| "layers.7.self_attn.q_proj", |
| "layers.7.self_attn.k_proj", |
| "layers.7.self_attn.v_proj", |
| "layers.7.self_attn.o_proj", |
| "layers.7.mlp.gate_proj", |
| "layers.7.mlp.up_proj", |
| "layers.7.mlp.down_proj", |
| "layers.8.linear_attn.out_proj", |
| "layers.8.linear_attn.in_proj_qkv", |
| "layers.8.linear_attn.in_proj_z", |
| "layers.8.linear_attn.in_proj_b", |
| "layers.8.linear_attn.in_proj_a", |
| "layers.8.mlp.gate_proj", |
| "layers.8.mlp.up_proj", |
| "layers.8.mlp.down_proj", |
| "layers.9.linear_attn.out_proj", |
| "layers.9.linear_attn.in_proj_qkv", |
| "layers.9.linear_attn.in_proj_z", |
| "layers.9.linear_attn.in_proj_b", |
| "layers.9.linear_attn.in_proj_a", |
| "layers.9.mlp.gate_proj", |
| "layers.9.mlp.up_proj", |
| "layers.9.mlp.down_proj", |
| "layers.10.linear_attn.out_proj", |
| "layers.10.linear_attn.in_proj_qkv", |
| "layers.10.linear_attn.in_proj_z", |
| "layers.10.linear_attn.in_proj_b", |
| "layers.10.linear_attn.in_proj_a", |
| "layers.10.mlp.gate_proj", |
| "layers.10.mlp.up_proj", |
| "layers.10.mlp.down_proj", |
| "layers.11.self_attn.q_proj", |
| "layers.11.self_attn.k_proj", |
| "layers.11.self_attn.v_proj", |
| "layers.11.self_attn.o_proj", |
| "layers.11.mlp.gate_proj", |
| "layers.11.mlp.up_proj", |
| "layers.11.mlp.down_proj", |
| "layers.12.linear_attn.out_proj", |
| "layers.12.linear_attn.in_proj_qkv", |
| "layers.12.linear_attn.in_proj_z", |
| "layers.12.linear_attn.in_proj_b", |
| "layers.12.linear_attn.in_proj_a", |
| "layers.12.mlp.gate_proj", |
| "layers.12.mlp.up_proj", |
| "layers.12.mlp.down_proj", |
| "layers.13.linear_attn.out_proj", |
| "layers.13.linear_attn.in_proj_qkv", |
| "layers.13.linear_attn.in_proj_z", |
| "layers.13.linear_attn.in_proj_b", |
| "layers.13.linear_attn.in_proj_a", |
| "layers.13.mlp.gate_proj", |
| "layers.13.mlp.up_proj", |
| "layers.13.mlp.down_proj", |
| "layers.14.linear_attn.out_proj", |
| "layers.14.linear_attn.in_proj_qkv", |
| "layers.14.linear_attn.in_proj_z", |
| "layers.14.linear_attn.in_proj_b", |
| "layers.14.linear_attn.in_proj_a", |
| "layers.14.mlp.gate_proj", |
| "layers.14.mlp.up_proj", |
| "layers.14.mlp.down_proj", |
| "layers.15.self_attn.q_proj", |
| "layers.15.self_attn.k_proj", |
| "layers.15.self_attn.v_proj", |
| "layers.15.self_attn.o_proj", |
| "layers.15.mlp.gate_proj", |
| "layers.15.mlp.up_proj", |
| "layers.15.mlp.down_proj", |
| "layers.16.linear_attn.out_proj", |
| "layers.16.linear_attn.in_proj_qkv", |
| "layers.16.linear_attn.in_proj_z", |
| "layers.16.linear_attn.in_proj_b", |
| "layers.16.linear_attn.in_proj_a", |
| "layers.16.mlp.gate_proj", |
| "layers.16.mlp.up_proj", |
| "layers.16.mlp.down_proj", |
| "layers.17.linear_attn.out_proj", |
| "layers.17.linear_attn.in_proj_qkv", |
| "layers.17.linear_attn.in_proj_z", |
| "layers.17.linear_attn.in_proj_b", |
| "layers.17.linear_attn.in_proj_a", |
| "layers.17.mlp.gate_proj", |
| "layers.17.mlp.up_proj", |
| "layers.17.mlp.down_proj", |
| "layers.18.linear_attn.out_proj", |
| "layers.18.linear_attn.in_proj_qkv", |
| "layers.18.linear_attn.in_proj_z", |
| "layers.18.linear_attn.in_proj_b", |
| "layers.18.linear_attn.in_proj_a", |
| "layers.18.mlp.gate_proj", |
| "layers.18.mlp.up_proj", |
| "layers.18.mlp.down_proj", |
| "layers.19.self_attn.q_proj", |
| "layers.19.self_attn.k_proj", |
| "layers.19.self_attn.v_proj", |
| "layers.19.self_attn.o_proj", |
| "layers.19.mlp.gate_proj", |
| "layers.19.mlp.up_proj", |
| "layers.19.mlp.down_proj", |
| "layers.20.linear_attn.out_proj", |
| "layers.20.linear_attn.in_proj_qkv", |
| "layers.20.linear_attn.in_proj_z", |
| "layers.20.linear_attn.in_proj_b", |
| "layers.20.linear_attn.in_proj_a", |
| "layers.20.mlp.gate_proj", |
| "layers.20.mlp.up_proj", |
| "layers.20.mlp.down_proj", |
| "layers.21.linear_attn.out_proj", |
| "layers.21.linear_attn.in_proj_qkv", |
| "layers.21.linear_attn.in_proj_z", |
| "layers.21.linear_attn.in_proj_b", |
| "layers.21.linear_attn.in_proj_a", |
| "layers.21.mlp.gate_proj", |
| "layers.21.mlp.up_proj", |
| "layers.21.mlp.down_proj", |
| "layers.22.linear_attn.out_proj", |
| "layers.22.linear_attn.in_proj_qkv", |
| "layers.22.linear_attn.in_proj_z", |
| "layers.22.linear_attn.in_proj_b", |
| "layers.22.linear_attn.in_proj_a", |
| "layers.22.mlp.gate_proj", |
| "layers.22.mlp.up_proj", |
| "layers.22.mlp.down_proj", |
| "layers.23.self_attn.q_proj", |
| "layers.23.self_attn.k_proj", |
| "layers.23.self_attn.v_proj", |
| "layers.23.self_attn.o_proj", |
| "layers.23.mlp.gate_proj", |
| "layers.23.mlp.up_proj", |
| "layers.23.mlp.down_proj" |
| ], |
| "load_and_tokenize_seconds": 2.767755454995495 |
| }, |
| "exit_codes": { |
| "exit_code": "0" |
| }, |
| "processes": " PID PPID STAT ELAPSED COMMAND\n", |
| "gpu_processes": "", |
| "memory": " total used free shared buff/cache available\nMem: 130661138432 4221775872 89653522432 4759552 37928640512 126439362560\nSwap: 0 0 0\n", |
| "source_commit": "4a60423c39d70f8d50472ce4f4f7fa4a4bd9fce1", |
| "source_status": "", |
| "training_audit": { |
| "rows": 5960, |
| "first_step": 41, |
| "last_step": 6000, |
| "steps_contiguous": true, |
| "finite_losses": true, |
| "finite_gradients": true, |
| "max_gradient_norm": 267.5868225097656, |
| "max_loss": 7.096554353829106, |
| "max_cuda_allocated_bytes": 8786432512, |
| "median_step_seconds": 3.5843077870013076, |
| "last_100_median_step_seconds": 3.610054773500451, |
| "last_update": { |
| "step": 6000, |
| "loss": 0.0019449404207989573, |
| "gradient_norm": 0.06868523359298706, |
| "seconds": 3.64171387499664, |
| "decisions": 4, |
| "branches": 14, |
| "actual_branch_tokens": 1412, |
| "padded_branch_tokens": 1423, |
| "peak_cuda_allocated_bytes": 8786432512, |
| "lr_factor": 0.911059698995469 |
| } |
| }, |
| "validation_history": [ |
| { |
| "step": 500, |
| "macro_nll": 0.45433305375755806, |
| "best_macro_nll": 0.45433305375755806, |
| "selection": { |
| "metric": "crossfit_temperature_nll_v1", |
| "score": 0.36986992229694793, |
| "raw_macro_nll": 0.45433305295880055, |
| "accuracy": 0.876953125, |
| "folds": [ |
| { |
| "fold": 0, |
| "temperature_index": 58, |
| "temperature": 2.1577444091526656, |
| "training_macro_nll": 0.3830001160949712, |
| "heldout_ids": [ |
| "arc:validation:ARC-Challenge:122", |
| "arc:validation:ARC-Challenge:126", |
| "arc:validation:ARC-Challenge:175", |
| "arc:validation:ARC-Challenge:187", |
| "arc:validation:ARC-Challenge:189", |
| "arc:validation:ARC-Challenge:192", |
| "arc:validation:ARC-Challenge:208", |
| "arc:validation:ARC-Challenge:238", |
| "arc:validation:ARC-Challenge:290", |
| "arc:validation:ARC-Challenge:293", |
| "arc:validation:ARC-Challenge:35", |
| "arc:validation:ARC-Challenge:43", |
| "arc:validation:ARC-Challenge:58", |
| "arc:validation:ARC-Challenge:7", |
| "arc:validation:ARC-Easy:160", |
| "arc:validation:ARC-Easy:201", |
| "arc:validation:ARC-Easy:229", |
| "arc:validation:ARC-Easy:238", |
| "arc:validation:ARC-Easy:253", |
| "arc:validation:ARC-Easy:259", |
| "arc:validation:ARC-Easy:272", |
| "arc:validation:ARC-Easy:298", |
| "arc:validation:ARC-Easy:329", |
| "arc:validation:ARC-Easy:331", |
| "arc:validation:ARC-Easy:337", |
| "arc:validation:ARC-Easy:362", |
| "arc:validation:ARC-Easy:443", |
| "arc:validation:ARC-Easy:500", |
| "arc:validation:ARC-Easy:545", |
| "arc:validation:ARC-Easy:68", |
| "arc:validation:ARC-Easy:7", |
| "arc:validation:ARC-Easy:74", |
| "banking:train:1053", |
| "banking:train:1594", |
| "banking:train:1747", |
| "banking:train:202", |
| "banking:train:2377", |
| "banking:train:2528", |
| "banking:train:2546", |
| "banking:train:2715", |
| "banking:train:3220", |
| "banking:train:3499", |
| "banking:train:3806", |
| "banking:train:3838", |
| "banking:train:3993", |
| "banking:train:4220", |
| "banking:train:5", |
| "banking:train:5192", |
| "banking:train:5394", |
| "banking:train:5846", |
| "banking:train:5863", |
| "banking:train:6032", |
| "banking:train:6508", |
| "banking:train:7579", |
| "banking:train:7600", |
| "banking:train:7987", |
| "banking:train:8155", |
| "banking:train:850", |
| "banking:train:8626", |
| "banking:train:8929", |
| "banking:train:9355", |
| "banking:train:9812", |
| "banking:train:987", |
| "banking:train:9972", |
| "boolq:validation:103", |
| "boolq:validation:1042", |
| "boolq:validation:1059", |
| "boolq:validation:1541", |
| "boolq:validation:1592", |
| "boolq:validation:1927", |
| "boolq:validation:2014", |
| "boolq:validation:2023", |
| "boolq:validation:2098", |
| "boolq:validation:2139", |
| "boolq:validation:2439", |
| "boolq:validation:2485", |
| "boolq:validation:2607", |
| "boolq:validation:2611", |
| "boolq:validation:2720", |
| "boolq:validation:2751", |
| "boolq:validation:282", |
| "boolq:validation:2843", |
| "boolq:validation:2953", |
| "boolq:validation:301", |
| "boolq:validation:3089", |
| "boolq:validation:313", |
| "boolq:validation:3201", |
| "boolq:validation:322", |
| "boolq:validation:3231", |
| "boolq:validation:39", |
| "boolq:validation:461", |
| "boolq:validation:558", |
| "boolq:validation:589", |
| "boolq:validation:731", |
| "boolq:validation:883", |
| "boolq:validation:915", |
| "snli:validation:1156", |
| "snli:validation:1330", |
| "snli:validation:1559", |
| "snli:validation:1735", |
| "snli:validation:1876", |
| "snli:validation:2251", |
| "snli:validation:2257", |
| "snli:validation:2377", |
| "snli:validation:2881", |
| "snli:validation:291", |
| "snli:validation:300", |
| "snli:validation:3049", |
| "snli:validation:3496", |
| "snli:validation:3871", |
| "snli:validation:4414", |
| "snli:validation:5140", |
| "snli:validation:5440", |
| "snli:validation:5806", |
| "snli:validation:6442", |
| "snli:validation:6772", |
| "snli:validation:6949", |
| "snli:validation:7228", |
| "snli:validation:7465", |
| "snli:validation:7510", |
| "snli:validation:7726", |
| "snli:validation:8095", |
| "snli:validation:8143", |
| "snli:validation:8623", |
| "snli:validation:8962", |
| "snli:validation:9451", |
| "snli:validation:9574", |
| "snli:validation:9655" |
| ], |
| "heldout_group_count": 128, |
| "training_group_count": 384 |
| }, |
| { |
| "fold": 1, |
| "temperature_index": 57, |
| "temperature": 2.046444636724674, |
| "training_macro_nll": 0.36597041087667326, |
| "heldout_ids": [ |
| "arc:validation:ARC-Challenge:101", |
| "arc:validation:ARC-Challenge:129", |
| "arc:validation:ARC-Challenge:148", |
| "arc:validation:ARC-Challenge:156", |
| "arc:validation:ARC-Challenge:167", |
| "arc:validation:ARC-Challenge:249", |
| "arc:validation:ARC-Challenge:263", |
| "arc:validation:ARC-Challenge:266", |
| "arc:validation:ARC-Challenge:267", |
| "arc:validation:ARC-Challenge:28", |
| "arc:validation:ARC-Challenge:281", |
| "arc:validation:ARC-Challenge:57", |
| "arc:validation:ARC-Challenge:63", |
| "arc:validation:ARC-Challenge:64", |
| "arc:validation:ARC-Challenge:86", |
| "arc:validation:ARC-Challenge:99", |
| "arc:validation:ARC-Easy:106", |
| "arc:validation:ARC-Easy:151", |
| "arc:validation:ARC-Easy:157", |
| "arc:validation:ARC-Easy:166", |
| "arc:validation:ARC-Easy:202", |
| "arc:validation:ARC-Easy:219", |
| "arc:validation:ARC-Easy:224", |
| "arc:validation:ARC-Easy:322", |
| "arc:validation:ARC-Easy:344", |
| "arc:validation:ARC-Easy:382", |
| "arc:validation:ARC-Easy:409", |
| "arc:validation:ARC-Easy:455", |
| "arc:validation:ARC-Easy:50", |
| "arc:validation:ARC-Easy:538", |
| "arc:validation:ARC-Easy:544", |
| "arc:validation:ARC-Easy:552", |
| "banking:train:1050", |
| "banking:train:1131", |
| "banking:train:1190", |
| "banking:train:124", |
| "banking:train:137", |
| "banking:train:1493", |
| "banking:train:1498", |
| "banking:train:1624", |
| "banking:train:2614", |
| "banking:train:2739", |
| "banking:train:3055", |
| "banking:train:306", |
| "banking:train:3312", |
| "banking:train:3724", |
| "banking:train:4070", |
| "banking:train:476", |
| "banking:train:4882", |
| "banking:train:5075", |
| "banking:train:5332", |
| "banking:train:5445", |
| "banking:train:6096", |
| "banking:train:746", |
| "banking:train:7828", |
| "banking:train:8176", |
| "banking:train:8315", |
| "banking:train:8653", |
| "banking:train:9019", |
| "banking:train:9370", |
| "banking:train:9454", |
| "banking:train:9595", |
| "banking:train:9862", |
| "banking:train:9963", |
| "boolq:validation:1189", |
| "boolq:validation:1320", |
| "boolq:validation:1326", |
| "boolq:validation:1381", |
| "boolq:validation:1410", |
| "boolq:validation:149", |
| "boolq:validation:1513", |
| "boolq:validation:1732", |
| "boolq:validation:1739", |
| "boolq:validation:1741", |
| "boolq:validation:1814", |
| "boolq:validation:1848", |
| "boolq:validation:1940", |
| "boolq:validation:2077", |
| "boolq:validation:2091", |
| "boolq:validation:2148", |
| "boolq:validation:2215", |
| "boolq:validation:2251", |
| "boolq:validation:2283", |
| "boolq:validation:2321", |
| "boolq:validation:2398", |
| "boolq:validation:253", |
| "boolq:validation:2575", |
| "boolq:validation:2907", |
| "boolq:validation:2909", |
| "boolq:validation:2978", |
| "boolq:validation:359", |
| "boolq:validation:394", |
| "boolq:validation:436", |
| "boolq:validation:579", |
| "boolq:validation:807", |
| "boolq:validation:949", |
| "snli:validation:2389", |
| "snli:validation:2485", |
| "snli:validation:255", |
| "snli:validation:3169", |
| "snli:validation:333", |
| "snli:validation:3700", |
| "snli:validation:3709", |
| "snli:validation:4093", |
| "snli:validation:420", |
| "snli:validation:4492", |
| "snli:validation:4516", |
| "snli:validation:4648", |
| "snli:validation:4888", |
| "snli:validation:5185", |
| "snli:validation:5332", |
| "snli:validation:5947", |
| "snli:validation:6124", |
| "snli:validation:660", |
| "snli:validation:6619", |
| "snli:validation:6898", |
| "snli:validation:7387", |
| "snli:validation:7492", |
| "snli:validation:7544", |
| "snli:validation:7996", |
| "snli:validation:8074", |
| "snli:validation:8554", |
| "snli:validation:8971", |
| "snli:validation:9163", |
| "snli:validation:9277", |
| "snli:validation:9490", |
| "snli:validation:9595", |
| "snli:validation:9688" |
| ], |
| "heldout_group_count": 128, |
| "training_group_count": 384 |
| }, |
| { |
| "fold": 2, |
| "temperature_index": 57, |
| "temperature": 2.046444636724674, |
| "training_macro_nll": 0.3553716993757123, |
| "heldout_ids": [ |
| "arc:validation:ARC-Challenge:120", |
| "arc:validation:ARC-Challenge:130", |
| "arc:validation:ARC-Challenge:194", |
| "arc:validation:ARC-Challenge:218", |
| "arc:validation:ARC-Challenge:235", |
| "arc:validation:ARC-Challenge:272", |
| "arc:validation:ARC-Challenge:277", |
| "arc:validation:ARC-Challenge:48", |
| "arc:validation:ARC-Challenge:88", |
| "arc:validation:ARC-Easy:115", |
| "arc:validation:ARC-Easy:12", |
| "arc:validation:ARC-Easy:131", |
| "arc:validation:ARC-Easy:145", |
| "arc:validation:ARC-Easy:179", |
| "arc:validation:ARC-Easy:18", |
| "arc:validation:ARC-Easy:186", |
| "arc:validation:ARC-Easy:191", |
| "arc:validation:ARC-Easy:215", |
| "arc:validation:ARC-Easy:283", |
| "arc:validation:ARC-Easy:314", |
| "arc:validation:ARC-Easy:315", |
| "arc:validation:ARC-Easy:378", |
| "arc:validation:ARC-Easy:384", |
| "arc:validation:ARC-Easy:403", |
| "arc:validation:ARC-Easy:432", |
| "arc:validation:ARC-Easy:439", |
| "arc:validation:ARC-Easy:448", |
| "arc:validation:ARC-Easy:461", |
| "arc:validation:ARC-Easy:469", |
| "arc:validation:ARC-Easy:470", |
| "arc:validation:ARC-Easy:504", |
| "arc:validation:ARC-Easy:561", |
| "banking:train:1142", |
| "banking:train:148", |
| "banking:train:1633", |
| "banking:train:1927", |
| "banking:train:2186", |
| "banking:train:2251", |
| "banking:train:23", |
| "banking:train:3048", |
| "banking:train:3665", |
| "banking:train:3952", |
| "banking:train:3999", |
| "banking:train:4364", |
| "banking:train:4619", |
| "banking:train:5049", |
| "banking:train:5335", |
| "banking:train:549", |
| "banking:train:5957", |
| "banking:train:5985", |
| "banking:train:6132", |
| "banking:train:6240", |
| "banking:train:6488", |
| "banking:train:7065", |
| "banking:train:7749", |
| "banking:train:8470", |
| "banking:train:8761", |
| "banking:train:895", |
| "banking:train:9256", |
| "banking:train:9406", |
| "banking:train:9450", |
| "banking:train:9467", |
| "banking:train:9588", |
| "banking:train:9716", |
| "boolq:validation:1130", |
| "boolq:validation:1167", |
| "boolq:validation:124", |
| "boolq:validation:1255", |
| "boolq:validation:1256", |
| "boolq:validation:1373", |
| "boolq:validation:15", |
| "boolq:validation:155", |
| "boolq:validation:1568", |
| "boolq:validation:16", |
| "boolq:validation:1654", |
| "boolq:validation:1702", |
| "boolq:validation:1792", |
| "boolq:validation:1818", |
| "boolq:validation:210", |
| "boolq:validation:2247", |
| "boolq:validation:2259", |
| "boolq:validation:2362", |
| "boolq:validation:2443", |
| "boolq:validation:2524", |
| "boolq:validation:2553", |
| "boolq:validation:2623", |
| "boolq:validation:2660", |
| "boolq:validation:2669", |
| "boolq:validation:2679", |
| "boolq:validation:2832", |
| "boolq:validation:357", |
| "boolq:validation:453", |
| "boolq:validation:49", |
| "boolq:validation:808", |
| "boolq:validation:90", |
| "boolq:validation:910", |
| "snli:validation:1612", |
| "snli:validation:168", |
| "snli:validation:1693", |
| "snli:validation:2134", |
| "snli:validation:2248", |
| "snli:validation:2422", |
| "snli:validation:2587", |
| "snli:validation:3007", |
| "snli:validation:3097", |
| "snli:validation:3772", |
| "snli:validation:4012", |
| "snli:validation:4168", |
| "snli:validation:4537", |
| "snli:validation:4681", |
| "snli:validation:5050", |
| "snli:validation:5632", |
| "snli:validation:5761", |
| "snli:validation:6055", |
| "snli:validation:6283", |
| "snli:validation:6715", |
| "snli:validation:7117", |
| "snli:validation:7192", |
| "snli:validation:7507", |
| "snli:validation:7594", |
| "snli:validation:7924", |
| "snli:validation:7954", |
| "snli:validation:8035", |
| "snli:validation:8110", |
| "snli:validation:8995", |
| "snli:validation:949", |
| "snli:validation:9523", |
| "snli:validation:9589" |
| ], |
| "heldout_group_count": 128, |
| "training_group_count": 384 |
| }, |
| { |
| "fold": 3, |
| "temperature_index": 57, |
| "temperature": 2.046444636724674, |
| "training_macro_nll": 0.3736932084232408, |
| "heldout_ids": [ |
| "arc:validation:ARC-Challenge:117", |
| "arc:validation:ARC-Challenge:135", |
| "arc:validation:ARC-Challenge:139", |
| "arc:validation:ARC-Challenge:154", |
| "arc:validation:ARC-Challenge:174", |
| "arc:validation:ARC-Challenge:199", |
| "arc:validation:ARC-Challenge:287", |
| "arc:validation:ARC-Challenge:295", |
| "arc:validation:ARC-Challenge:4", |
| "arc:validation:ARC-Challenge:50", |
| "arc:validation:ARC-Challenge:51", |
| "arc:validation:ARC-Challenge:61", |
| "arc:validation:ARC-Easy:107", |
| "arc:validation:ARC-Easy:121", |
| "arc:validation:ARC-Easy:124", |
| "arc:validation:ARC-Easy:171", |
| "arc:validation:ARC-Easy:235", |
| "arc:validation:ARC-Easy:241", |
| "arc:validation:ARC-Easy:279", |
| "arc:validation:ARC-Easy:289", |
| "arc:validation:ARC-Easy:37", |
| "arc:validation:ARC-Easy:389", |
| "arc:validation:ARC-Easy:394", |
| "arc:validation:ARC-Easy:399", |
| "arc:validation:ARC-Easy:423", |
| "arc:validation:ARC-Easy:492", |
| "arc:validation:ARC-Easy:499", |
| "arc:validation:ARC-Easy:523", |
| "arc:validation:ARC-Easy:533", |
| "arc:validation:ARC-Easy:569", |
| "arc:validation:ARC-Easy:69", |
| "arc:validation:ARC-Easy:93", |
| "banking:train:2049", |
| "banking:train:2350", |
| "banking:train:2497", |
| "banking:train:2506", |
| "banking:train:2720", |
| "banking:train:2788", |
| "banking:train:2967", |
| "banking:train:3502", |
| "banking:train:3726", |
| "banking:train:4014", |
| "banking:train:4735", |
| "banking:train:4803", |
| "banking:train:4917", |
| "banking:train:5477", |
| "banking:train:5851", |
| "banking:train:6551", |
| "banking:train:6839", |
| "banking:train:6926", |
| "banking:train:7809", |
| "banking:train:8092", |
| "banking:train:8186", |
| "banking:train:8215", |
| "banking:train:8667", |
| "banking:train:8752", |
| "banking:train:8763", |
| "banking:train:8851", |
| "banking:train:8917", |
| "banking:train:9145", |
| "banking:train:9279", |
| "banking:train:9676", |
| "banking:train:9837", |
| "banking:train:9967", |
| "boolq:validation:106", |
| "boolq:validation:1065", |
| "boolq:validation:1097", |
| "boolq:validation:1336", |
| "boolq:validation:1350", |
| "boolq:validation:1383", |
| "boolq:validation:1458", |
| "boolq:validation:1641", |
| "boolq:validation:1873", |
| "boolq:validation:1930", |
| "boolq:validation:1989", |
| "boolq:validation:2103", |
| "boolq:validation:234", |
| "boolq:validation:2355", |
| "boolq:validation:2384", |
| "boolq:validation:2440", |
| "boolq:validation:2444", |
| "boolq:validation:2478", |
| "boolq:validation:2872", |
| "boolq:validation:2879", |
| "boolq:validation:2924", |
| "boolq:validation:2971", |
| "boolq:validation:3164", |
| "boolq:validation:4", |
| "boolq:validation:415", |
| "boolq:validation:419", |
| "boolq:validation:428", |
| "boolq:validation:512", |
| "boolq:validation:563", |
| "boolq:validation:643", |
| "boolq:validation:696", |
| "boolq:validation:874", |
| "snli:validation:1102", |
| "snli:validation:1240", |
| "snli:validation:1261", |
| "snli:validation:1738", |
| "snli:validation:1978", |
| "snli:validation:2563", |
| "snli:validation:2659", |
| "snli:validation:2767", |
| "snli:validation:2836", |
| "snli:validation:3232", |
| "snli:validation:3655", |
| "snli:validation:3913", |
| "snli:validation:405", |
| "snli:validation:4051", |
| "snli:validation:4300", |
| "snli:validation:4846", |
| "snli:validation:498", |
| "snli:validation:5224", |
| "snli:validation:5575", |
| "snli:validation:6598", |
| "snli:validation:666", |
| "snli:validation:6925", |
| "snli:validation:6964", |
| "snli:validation:7045", |
| "snli:validation:7351", |
| "snli:validation:7588", |
| "snli:validation:7723", |
| "snli:validation:7750", |
| "snli:validation:7909", |
| "snli:validation:8974", |
| "snli:validation:9385", |
| "snli:validation:9646" |
| ], |
| "heldout_group_count": 128, |
| "training_group_count": 384 |
| } |
| ], |
| "per_family": { |
| "arc": { |
| "count": 128, |
| "score": 0.4232126018397948, |
| "raw_macro_nll": 0.5933205790802253, |
| "accuracy": 0.859375 |
| }, |
| "banking": { |
| "count": 128, |
| "score": 0.15421812640309446, |
| "raw_macro_nll": 0.2594607499213546, |
| "accuracy": 0.9453125 |
| }, |
| "boolq": { |
| "count": 128, |
| "score": 0.4010937069730148, |
| "raw_macro_nll": 0.43871330066590836, |
| "accuracy": 0.8515625 |
| }, |
| "snli": { |
| "count": 128, |
| "score": 0.5009552539718878, |
| "raw_macro_nll": 0.5258375821677139, |
| "accuracy": 0.8515625 |
| } |
| }, |
| "policy": { |
| "id": "crossfit_temperature_nll_v1", |
| "folds": 4, |
| "seed": 431, |
| "grouping": "source group, nested within task family; groups shuffled within sorted families and assigned round-robin", |
| "temperature_grid": { |
| "count": 101, |
| "log10_min": -1.0, |
| "log10_max": 1.3, |
| "arithmetic": "Python float64" |
| }, |
| "temperature_fit": "minimum macro-family NLL on the other three folds; smallest temperature wins ties", |
| "score": "macro-family mean of all held-out decision NLLs", |
| "deployment_temperature": "fit afresh on reserved calibration only after model selection" |
| } |
| }, |
| "best_selection_score": 0.36986992229694793, |
| "improved": true, |
| "metrics": { |
| "overall": { |
| "n": 512, |
| "accuracy": 0.876953125, |
| "nll": 0.45433305375755806, |
| "brier_multiclass_sum": 0.1898162259719647, |
| "ece_top_label_10_equal_width_bins": 0.05299526819726452, |
| "reliability_bins": [ |
| { |
| "count": 0, |
| "confidence_sum": 0.0, |
| "correct_sum": 0.0 |
| }, |
| { |
| "count": 0, |
| "confidence_sum": 0.0, |
| "correct_sum": 0.0 |
| }, |
| { |
| "count": 0, |
| "confidence_sum": 0.0, |
| "correct_sum": 0.0 |
| }, |
| { |
| "count": 0, |
| "confidence_sum": 0.0, |
| "correct_sum": 0.0 |
| }, |
| { |
| "count": 7, |
| "confidence_sum": 3.3479487597942352, |
| "correct_sum": 0.0 |
| }, |
| { |
| "count": 14, |
| "confidence_sum": 7.681548655033112, |
| "correct_sum": 9.0 |
| }, |
| { |
| "count": 27, |
| "confidence_sum": 17.5001118183136, |
| "correct_sum": 15.0 |
| }, |
| { |
| "count": 25, |
| "confidence_sum": 18.57009321451187, |
| "correct_sum": 17.0 |
| }, |
| { |
| "count": 43, |
| "confidence_sum": 37.1960312128067, |
| "correct_sum": 36.0 |
| }, |
| { |
| "count": 396, |
| "confidence_sum": 389.20094096660614, |
| "correct_sum": 372.0 |
| } |
| ], |
| "accuracy_vs_coverage": { |
| "0.25": { |
| "n": 128, |
| "accuracy": 1.0, |
| "min_confidence": 0.9999995231628418 |
| }, |
| "0.5": { |
| "n": 256, |
| "accuracy": 0.97265625, |
| "min_confidence": 0.993850588798523 |
| }, |
| "0.75": { |
| "n": 384, |
| "accuracy": 0.9427083333333334, |
| "min_confidence": 0.9114243984222412 |
| }, |
| "1.0": { |
| "n": 512, |
| "accuracy": 0.876953125, |
| "min_confidence": 0.4447111487388611 |
| } |
| } |
| }, |
| "per_family": { |
| "arc": { |
| "n": 128, |
| "accuracy": 0.859375, |
| "nll": 0.5933205873676035, |
| "brier_multiclass_sum": 0.20391137341560844, |
| "ece_top_label_10_equal_width_bins": 0.09224436432123184, |
| "reliability_bins": [ |
| { |
| "count": 0, |
| "confidence_sum": 0.0, |
| "correct_sum": 0.0 |
| }, |
| { |
| "count": 0, |
| "confidence_sum": 0.0, |
| "correct_sum": 0.0 |
| }, |
| { |
| "count": 0, |
| "confidence_sum": 0.0, |
| "correct_sum": 0.0 |
| }, |
| { |
| "count": 0, |
| "confidence_sum": 0.0, |
| "correct_sum": 0.0 |
| }, |
| { |
| "count": 3, |
| "confidence_sum": 1.3755568861961365, |
| "correct_sum": 0.0 |
| }, |
| { |
| "count": 5, |
| "confidence_sum": 2.768744707107544, |
| "correct_sum": 1.0 |
| }, |
| { |
| "count": 3, |
| "confidence_sum": 1.9373115301132202, |
| "correct_sum": 3.0 |
| }, |
| { |
| "count": 5, |
| "confidence_sum": 3.6977739930152893, |
| "correct_sum": 3.0 |
| }, |
| { |
| "count": 9, |
| "confidence_sum": 7.801501452922821, |
| "correct_sum": 6.0 |
| }, |
| { |
| "count": 103, |
| "confidence_sum": 102.1010131239891, |
| "correct_sum": 97.0 |
| } |
| ], |
| "accuracy_vs_coverage": { |
| "0.25": { |
| "n": 32, |
| "accuracy": 1.0, |
| "min_confidence": 0.9999940395355225 |
| }, |
| "0.5": { |
| "n": 64, |
| "accuracy": 1.0, |
| "min_confidence": 0.9995550513267517 |
| }, |
| "0.75": { |
| "n": 96, |
| "accuracy": 0.9583333333333334, |
| "min_confidence": 0.9565119743347168 |
| }, |
| "1.0": { |
| "n": 128, |
| "accuracy": 0.859375, |
| "min_confidence": 0.4447111487388611 |
| } |
| } |
| }, |
| "banking": { |
| "n": 128, |
| "accuracy": 0.9453125, |
| "nll": 0.25946074807588604, |
| "brier_multiclass_sum": 0.08189167536775005, |
| "ece_top_label_10_equal_width_bins": 0.04227270046249032, |
| "reliability_bins": [ |
| { |
| "count": 0, |
| "confidence_sum": 0.0, |
| "correct_sum": 0.0 |
| }, |
| { |
| "count": 0, |
| "confidence_sum": 0.0, |
| "correct_sum": 0.0 |
| }, |
| { |
| "count": 0, |
| "confidence_sum": 0.0, |
| "correct_sum": 0.0 |
| }, |
| { |
| "count": 0, |
| "confidence_sum": 0.0, |
| "correct_sum": 0.0 |
| }, |
| { |
| "count": 1, |
| "confidence_sum": 0.481073260307312, |
| "correct_sum": 0.0 |
| }, |
| { |
| "count": 0, |
| "confidence_sum": 0.0, |
| "correct_sum": 0.0 |
| }, |
| { |
| "count": 2, |
| "confidence_sum": 1.2654950618743896, |
| "correct_sum": 0.0 |
| }, |
| { |
| "count": 0, |
| "confidence_sum": 0.0, |
| "correct_sum": 0.0 |
| }, |
| { |
| "count": 0, |
| "confidence_sum": 0.0, |
| "correct_sum": 0.0 |
| }, |
| { |
| "count": 125, |
| "confidence_sum": 124.66433733701706, |
| "correct_sum": 121.0 |
| } |
| ], |
| "accuracy_vs_coverage": { |
| "0.25": { |
| "n": 32, |
| "accuracy": 1.0, |
| "min_confidence": 1.0 |
| }, |
| "0.5": { |
| "n": 64, |
| "accuracy": 1.0, |
| "min_confidence": 1.0 |
| }, |
| "0.75": { |
| "n": 96, |
| "accuracy": 1.0, |
| "min_confidence": 0.9999990463256836 |
| }, |
| "1.0": { |
| "n": 128, |
| "accuracy": 0.9453125, |
| "min_confidence": 0.481073260307312 |
| } |
| } |
| }, |
| "boolq": { |
| "n": 128, |
| "accuracy": 0.8515625, |
| "nll": 0.43871330363498373, |
| "brier_multiclass_sum": 0.23252863315887956, |
| "ece_top_label_10_equal_width_bins": 0.08019732777029276, |
| "reliability_bins": [ |
| { |
| "count": 0, |
| "confidence_sum": 0.0, |
| "correct_sum": 0.0 |
| }, |
| { |
| "count": 0, |
| "confidence_sum": 0.0, |
| "correct_sum": 0.0 |
| }, |
| { |
| "count": 0, |
| "confidence_sum": 0.0, |
| "correct_sum": 0.0 |
| }, |
| { |
| "count": 0, |
| "confidence_sum": 0.0, |
| "correct_sum": 0.0 |
| }, |
| { |
| "count": 0, |
| "confidence_sum": 0.0, |
| "correct_sum": 0.0 |
| }, |
| { |
| "count": 4, |
| "confidence_sum": 2.1604248881340027, |
| "correct_sum": 4.0 |
| }, |
| { |
| "count": 11, |
| "confidence_sum": 7.190720975399017, |
| "correct_sum": 5.0 |
| }, |
| { |
| "count": 8, |
| "confidence_sum": 5.9833953976631165, |
| "correct_sum": 4.0 |
| }, |
| { |
| "count": 17, |
| "confidence_sum": 14.755406379699707, |
| "correct_sum": 15.0 |
| }, |
| { |
| "count": 88, |
| "confidence_sum": 85.00697284936905, |
| "correct_sum": 81.0 |
| } |
| ], |
| "accuracy_vs_coverage": { |
| "0.25": { |
| "n": 32, |
| "accuracy": 0.9375, |
| "min_confidence": 0.9843237996101379 |
| }, |
| "0.5": { |
| "n": 64, |
| "accuracy": 0.921875, |
| "min_confidence": 0.9393304586410522 |
| }, |
| "0.75": { |
| "n": 96, |
| "accuracy": 0.9166666666666666, |
| "min_confidence": 0.8829596042633057 |
| }, |
| "1.0": { |
| "n": 128, |
| "accuracy": 0.8515625, |
| "min_confidence": 0.5294721722602844 |
| } |
| } |
| }, |
| "snli": { |
| "n": 128, |
| "accuracy": 0.8515625, |
| "nll": 0.5258375759517588, |
| "brier_multiclass_sum": 0.2409332219456208, |
| "ece_top_label_10_equal_width_bins": 0.06832886137999594, |
| "reliability_bins": [ |
| { |
| "count": 0, |
| "confidence_sum": 0.0, |
| "correct_sum": 0.0 |
| }, |
| { |
| "count": 0, |
| "confidence_sum": 0.0, |
| "correct_sum": 0.0 |
| }, |
| { |
| "count": 0, |
| "confidence_sum": 0.0, |
| "correct_sum": 0.0 |
| }, |
| { |
| "count": 0, |
| "confidence_sum": 0.0, |
| "correct_sum": 0.0 |
| }, |
| { |
| "count": 3, |
| "confidence_sum": 1.4913186132907867, |
| "correct_sum": 0.0 |
| }, |
| { |
| "count": 5, |
| "confidence_sum": 2.752379059791565, |
| "correct_sum": 4.0 |
| }, |
| { |
| "count": 11, |
| "confidence_sum": 7.106584250926971, |
| "correct_sum": 7.0 |
| }, |
| { |
| "count": 12, |
| "confidence_sum": 8.888923823833466, |
| "correct_sum": 10.0 |
| }, |
| { |
| "count": 17, |
| "confidence_sum": 14.639123380184174, |
| "correct_sum": 15.0 |
| }, |
| { |
| "count": 80, |
| "confidence_sum": 77.42861765623093, |
| "correct_sum": 73.0 |
| } |
| ], |
| "accuracy_vs_coverage": { |
| "0.25": { |
| "n": 32, |
| "accuracy": 0.9375, |
| "min_confidence": 0.9971461892127991 |
| }, |
| "0.5": { |
| "n": 64, |
| "accuracy": 0.921875, |
| "min_confidence": 0.9333295226097107 |
| }, |
| "0.75": { |
| "n": 96, |
| "accuracy": 0.90625, |
| "min_confidence": 0.8133017420768738 |
| }, |
| "1.0": { |
| "n": 128, |
| "accuracy": 0.8515625, |
| "min_confidence": 0.49619075655937195 |
| } |
| } |
| } |
| } |
| } |
| }, |
| { |
| "step": 1000, |
| "macro_nll": 0.4657101983683963, |
| "best_macro_nll": 0.4657101983683963, |
| "selection": { |
| "metric": "crossfit_temperature_nll_v1", |
| "score": 0.3281878376433132, |
| "raw_macro_nll": 0.4657101993268551, |
| "accuracy": 0.880859375, |
| "folds": [ |
| { |
| "fold": 0, |
| "temperature_index": 60, |
| "temperature": 2.39883291901949, |
| "training_macro_nll": 0.346048313200281, |
| "heldout_ids": [ |
| "arc:validation:ARC-Challenge:122", |
| "arc:validation:ARC-Challenge:126", |
| "arc:validation:ARC-Challenge:175", |
| "arc:validation:ARC-Challenge:187", |
| "arc:validation:ARC-Challenge:189", |
| "arc:validation:ARC-Challenge:192", |
| "arc:validation:ARC-Challenge:208", |
| "arc:validation:ARC-Challenge:238", |
| "arc:validation:ARC-Challenge:290", |
| "arc:validation:ARC-Challenge:293", |
| "arc:validation:ARC-Challenge:35", |
| "arc:validation:ARC-Challenge:43", |
| "arc:validation:ARC-Challenge:58", |
| "arc:validation:ARC-Challenge:7", |
| "arc:validation:ARC-Easy:160", |
| "arc:validation:ARC-Easy:201", |
| "arc:validation:ARC-Easy:229", |
| "arc:validation:ARC-Easy:238", |
| "arc:validation:ARC-Easy:253", |
| "arc:validation:ARC-Easy:259", |
| "arc:validation:ARC-Easy:272", |
| "arc:validation:ARC-Easy:298", |
| "arc:validation:ARC-Easy:329", |
| "arc:validation:ARC-Easy:331", |
| "arc:validation:ARC-Easy:337", |
| "arc:validation:ARC-Easy:362", |
| "arc:validation:ARC-Easy:443", |
| "arc:validation:ARC-Easy:500", |
| "arc:validation:ARC-Easy:545", |
| "arc:validation:ARC-Easy:68", |
| "arc:validation:ARC-Easy:7", |
| "arc:validation:ARC-Easy:74", |
| "banking:train:1053", |
| "banking:train:1594", |
| "banking:train:1747", |
| "banking:train:202", |
| "banking:train:2377", |
| "banking:train:2528", |
| "banking:train:2546", |
| "banking:train:2715", |
| "banking:train:3220", |
| "banking:train:3499", |
| "banking:train:3806", |
| "banking:train:3838", |
| "banking:train:3993", |
| "banking:train:4220", |
| "banking:train:5", |
| "banking:train:5192", |
| "banking:train:5394", |
| "banking:train:5846", |
| "banking:train:5863", |
| "banking:train:6032", |
| "banking:train:6508", |
| "banking:train:7579", |
| "banking:train:7600", |
| "banking:train:7987", |
| "banking:train:8155", |
| "banking:train:850", |
| "banking:train:8626", |
| "banking:train:8929", |
| "banking:train:9355", |
| "banking:train:9812", |
| "banking:train:987", |
| "banking:train:9972", |
| "boolq:validation:103", |
| "boolq:validation:1042", |
| "boolq:validation:1059", |
| "boolq:validation:1541", |
| "boolq:validation:1592", |
| "boolq:validation:1927", |
| "boolq:validation:2014", |
| "boolq:validation:2023", |
| "boolq:validation:2098", |
| "boolq:validation:2139", |
| "boolq:validation:2439", |
| "boolq:validation:2485", |
| "boolq:validation:2607", |
| "boolq:validation:2611", |
| "boolq:validation:2720", |
| "boolq:validation:2751", |
| "boolq:validation:282", |
| "boolq:validation:2843", |
| "boolq:validation:2953", |
| "boolq:validation:301", |
| "boolq:validation:3089", |
| "boolq:validation:313", |
| "boolq:validation:3201", |
| "boolq:validation:322", |
| "boolq:validation:3231", |
| "boolq:validation:39", |
| "boolq:validation:461", |
| "boolq:validation:558", |
| "boolq:validation:589", |
| "boolq:validation:731", |
| "boolq:validation:883", |
| "boolq:validation:915", |
| "snli:validation:1156", |
| "snli:validation:1330", |
| "snli:validation:1559", |
| "snli:validation:1735", |
| "snli:validation:1876", |
| "snli:validation:2251", |
| "snli:validation:2257", |
| "snli:validation:2377", |
| "snli:validation:2881", |
| "snli:validation:291", |
| "snli:validation:300", |
| "snli:validation:3049", |
| "snli:validation:3496", |
| "snli:validation:3871", |
| "snli:validation:4414", |
| "snli:validation:5140", |
| "snli:validation:5440", |
| "snli:validation:5806", |
| "snli:validation:6442", |
| "snli:validation:6772", |
| "snli:validation:6949", |
| "snli:validation:7228", |
| "snli:validation:7465", |
| "snli:validation:7510", |
| "snli:validation:7726", |
| "snli:validation:8095", |
| "snli:validation:8143", |
| "snli:validation:8623", |
| "snli:validation:8962", |
| "snli:validation:9451", |
| "snli:validation:9574", |
| "snli:validation:9655" |
| ], |
| "heldout_group_count": 128, |
| "training_group_count": 384 |
| }, |
| { |
| "fold": 1, |
| "temperature_index": 60, |
| "temperature": 2.39883291901949, |
| "training_macro_nll": 0.33008355176738974, |
| "heldout_ids": [ |
| "arc:validation:ARC-Challenge:101", |
| "arc:validation:ARC-Challenge:129", |
| "arc:validation:ARC-Challenge:148", |
| "arc:validation:ARC-Challenge:156", |
| "arc:validation:ARC-Challenge:167", |
| "arc:validation:ARC-Challenge:249", |
| "arc:validation:ARC-Challenge:263", |
| "arc:validation:ARC-Challenge:266", |
| "arc:validation:ARC-Challenge:267", |
| "arc:validation:ARC-Challenge:28", |
| "arc:validation:ARC-Challenge:281", |
| "arc:validation:ARC-Challenge:57", |
| "arc:validation:ARC-Challenge:63", |
| "arc:validation:ARC-Challenge:64", |
| "arc:validation:ARC-Challenge:86", |
| "arc:validation:ARC-Challenge:99", |
| "arc:validation:ARC-Easy:106", |
| "arc:validation:ARC-Easy:151", |
| "arc:validation:ARC-Easy:157", |
| "arc:validation:ARC-Easy:166", |
| "arc:validation:ARC-Easy:202", |
| "arc:validation:ARC-Easy:219", |
| "arc:validation:ARC-Easy:224", |
| "arc:validation:ARC-Easy:322", |
| "arc:validation:ARC-Easy:344", |
| "arc:validation:ARC-Easy:382", |
| "arc:validation:ARC-Easy:409", |
| "arc:validation:ARC-Easy:455", |
| "arc:validation:ARC-Easy:50", |
| "arc:validation:ARC-Easy:538", |
| "arc:validation:ARC-Easy:544", |
| "arc:validation:ARC-Easy:552", |
| "banking:train:1050", |
| "banking:train:1131", |
| "banking:train:1190", |
| "banking:train:124", |
| "banking:train:137", |
| "banking:train:1493", |
| "banking:train:1498", |
| "banking:train:1624", |
| "banking:train:2614", |
| "banking:train:2739", |
| "banking:train:3055", |
| "banking:train:306", |
| "banking:train:3312", |
| "banking:train:3724", |
| "banking:train:4070", |
| "banking:train:476", |
| "banking:train:4882", |
| "banking:train:5075", |
| "banking:train:5332", |
| "banking:train:5445", |
| "banking:train:6096", |
| "banking:train:746", |
| "banking:train:7828", |
| "banking:train:8176", |
| "banking:train:8315", |
| "banking:train:8653", |
| "banking:train:9019", |
| "banking:train:9370", |
| "banking:train:9454", |
| "banking:train:9595", |
| "banking:train:9862", |
| "banking:train:9963", |
| "boolq:validation:1189", |
| "boolq:validation:1320", |
| "boolq:validation:1326", |
| "boolq:validation:1381", |
| "boolq:validation:1410", |
| "boolq:validation:149", |
| "boolq:validation:1513", |
| "boolq:validation:1732", |
| "boolq:validation:1739", |
| "boolq:validation:1741", |
| "boolq:validation:1814", |
| "boolq:validation:1848", |
| "boolq:validation:1940", |
| "boolq:validation:2077", |
| "boolq:validation:2091", |
| "boolq:validation:2148", |
| "boolq:validation:2215", |
| "boolq:validation:2251", |
| "boolq:validation:2283", |
| "boolq:validation:2321", |
| "boolq:validation:2398", |
| "boolq:validation:253", |
| "boolq:validation:2575", |
| "boolq:validation:2907", |
| "boolq:validation:2909", |
| "boolq:validation:2978", |
| "boolq:validation:359", |
| "boolq:validation:394", |
| "boolq:validation:436", |
| "boolq:validation:579", |
| "boolq:validation:807", |
| "boolq:validation:949", |
| "snli:validation:2389", |
| "snli:validation:2485", |
| "snli:validation:255", |
| "snli:validation:3169", |
| "snli:validation:333", |
| "snli:validation:3700", |
| "snli:validation:3709", |
| "snli:validation:4093", |
| "snli:validation:420", |
| "snli:validation:4492", |
| "snli:validation:4516", |
| "snli:validation:4648", |
| "snli:validation:4888", |
| "snli:validation:5185", |
| "snli:validation:5332", |
| "snli:validation:5947", |
| "snli:validation:6124", |
| "snli:validation:660", |
| "snli:validation:6619", |
| "snli:validation:6898", |
| "snli:validation:7387", |
| "snli:validation:7492", |
| "snli:validation:7544", |
| "snli:validation:7996", |
| "snli:validation:8074", |
| "snli:validation:8554", |
| "snli:validation:8971", |
| "snli:validation:9163", |
| "snli:validation:9277", |
| "snli:validation:9490", |
| "snli:validation:9595", |
| "snli:validation:9688" |
| ], |
| "heldout_group_count": 128, |
| "training_group_count": 384 |
| }, |
| { |
| "fold": 2, |
| "temperature_index": 59, |
| "temperature": 2.2750974307720706, |
| "training_macro_nll": 0.30676367425506046, |
| "heldout_ids": [ |
| "arc:validation:ARC-Challenge:120", |
| "arc:validation:ARC-Challenge:130", |
| "arc:validation:ARC-Challenge:194", |
| "arc:validation:ARC-Challenge:218", |
| "arc:validation:ARC-Challenge:235", |
| "arc:validation:ARC-Challenge:272", |
| "arc:validation:ARC-Challenge:277", |
| "arc:validation:ARC-Challenge:48", |
| "arc:validation:ARC-Challenge:88", |
| "arc:validation:ARC-Easy:115", |
| "arc:validation:ARC-Easy:12", |
| "arc:validation:ARC-Easy:131", |
| "arc:validation:ARC-Easy:145", |
| "arc:validation:ARC-Easy:179", |
| "arc:validation:ARC-Easy:18", |
| "arc:validation:ARC-Easy:186", |
| "arc:validation:ARC-Easy:191", |
| "arc:validation:ARC-Easy:215", |
| "arc:validation:ARC-Easy:283", |
| "arc:validation:ARC-Easy:314", |
| "arc:validation:ARC-Easy:315", |
| "arc:validation:ARC-Easy:378", |
| "arc:validation:ARC-Easy:384", |
| "arc:validation:ARC-Easy:403", |
| "arc:validation:ARC-Easy:432", |
| "arc:validation:ARC-Easy:439", |
| "arc:validation:ARC-Easy:448", |
| "arc:validation:ARC-Easy:461", |
| "arc:validation:ARC-Easy:469", |
| "arc:validation:ARC-Easy:470", |
| "arc:validation:ARC-Easy:504", |
| "arc:validation:ARC-Easy:561", |
| "banking:train:1142", |
| "banking:train:148", |
| "banking:train:1633", |
| "banking:train:1927", |
| "banking:train:2186", |
| "banking:train:2251", |
| "banking:train:23", |
| "banking:train:3048", |
| "banking:train:3665", |
| "banking:train:3952", |
| "banking:train:3999", |
| "banking:train:4364", |
| "banking:train:4619", |
| "banking:train:5049", |
| "banking:train:5335", |
| "banking:train:549", |
| "banking:train:5957", |
| "banking:train:5985", |
| "banking:train:6132", |
| "banking:train:6240", |
| "banking:train:6488", |
| "banking:train:7065", |
| "banking:train:7749", |
| "banking:train:8470", |
| "banking:train:8761", |
| "banking:train:895", |
| "banking:train:9256", |
| "banking:train:9406", |
| "banking:train:9450", |
| "banking:train:9467", |
| "banking:train:9588", |
| "banking:train:9716", |
| "boolq:validation:1130", |
| "boolq:validation:1167", |
| "boolq:validation:124", |
| "boolq:validation:1255", |
| "boolq:validation:1256", |
| "boolq:validation:1373", |
| "boolq:validation:15", |
| "boolq:validation:155", |
| "boolq:validation:1568", |
| "boolq:validation:16", |
| "boolq:validation:1654", |
| "boolq:validation:1702", |
| "boolq:validation:1792", |
| "boolq:validation:1818", |
| "boolq:validation:210", |
| "boolq:validation:2247", |
| "boolq:validation:2259", |
| "boolq:validation:2362", |
| "boolq:validation:2443", |
| "boolq:validation:2524", |
| "boolq:validation:2553", |
| "boolq:validation:2623", |
| "boolq:validation:2660", |
| "boolq:validation:2669", |
| "boolq:validation:2679", |
| "boolq:validation:2832", |
| "boolq:validation:357", |
| "boolq:validation:453", |
| "boolq:validation:49", |
| "boolq:validation:808", |
| "boolq:validation:90", |
| "boolq:validation:910", |
| "snli:validation:1612", |
| "snli:validation:168", |
| "snli:validation:1693", |
| "snli:validation:2134", |
| "snli:validation:2248", |
| "snli:validation:2422", |
| "snli:validation:2587", |
| "snli:validation:3007", |
| "snli:validation:3097", |
| "snli:validation:3772", |
| "snli:validation:4012", |
| "snli:validation:4168", |
| "snli:validation:4537", |
| "snli:validation:4681", |
| "snli:validation:5050", |
| "snli:validation:5632", |
| "snli:validation:5761", |
| "snli:validation:6055", |
| "snli:validation:6283", |
| "snli:validation:6715", |
| "snli:validation:7117", |
| "snli:validation:7192", |
| "snli:validation:7507", |
| "snli:validation:7594", |
| "snli:validation:7924", |
| "snli:validation:7954", |
| "snli:validation:8035", |
| "snli:validation:8110", |
| "snli:validation:8995", |
| "snli:validation:949", |
| "snli:validation:9523", |
| "snli:validation:9589" |
| ], |
| "heldout_group_count": 128, |
| "training_group_count": 384 |
| }, |
| { |
| "fold": 3, |
| "temperature_index": 59, |
| "temperature": 2.2750974307720706, |
| "training_macro_nll": 0.32565293380222415, |
| "heldout_ids": [ |
| "arc:validation:ARC-Challenge:117", |
| "arc:validation:ARC-Challenge:135", |
| "arc:validation:ARC-Challenge:139", |
| "arc:validation:ARC-Challenge:154", |
| "arc:validation:ARC-Challenge:174", |
| "arc:validation:ARC-Challenge:199", |
| "arc:validation:ARC-Challenge:287", |
| "arc:validation:ARC-Challenge:295", |
| "arc:validation:ARC-Challenge:4", |
| "arc:validation:ARC-Challenge:50", |
| "arc:validation:ARC-Challenge:51", |
| "arc:validation:ARC-Challenge:61", |
| "arc:validation:ARC-Easy:107", |
| "arc:validation:ARC-Easy:121", |
| "arc:validation:ARC-Easy:124", |
| "arc:validation:ARC-Easy:171", |
| "arc:validation:ARC-Easy:235", |
| "arc:validation:ARC-Easy:241", |
| "arc:validation:ARC-Easy:279", |
| "arc:validation:ARC-Easy:289", |
| "arc:validation:ARC-Easy:37", |
| "arc:validation:ARC-Easy:389", |
| "arc:validation:ARC-Easy:394", |
| "arc:validation:ARC-Easy:399", |
| "arc:validation:ARC-Easy:423", |
| "arc:validation:ARC-Easy:492", |
| "arc:validation:ARC-Easy:499", |
| "arc:validation:ARC-Easy:523", |
| "arc:validation:ARC-Easy:533", |
| "arc:validation:ARC-Easy:569", |
| "arc:validation:ARC-Easy:69", |
| "arc:validation:ARC-Easy:93", |
| "banking:train:2049", |
| "banking:train:2350", |
| "banking:train:2497", |
| "banking:train:2506", |
| "banking:train:2720", |
| "banking:train:2788", |
| "banking:train:2967", |
| "banking:train:3502", |
| "banking:train:3726", |
| "banking:train:4014", |
| "banking:train:4735", |
| "banking:train:4803", |
| "banking:train:4917", |
| "banking:train:5477", |
| "banking:train:5851", |
| "banking:train:6551", |
| "banking:train:6839", |
| "banking:train:6926", |
| "banking:train:7809", |
| "banking:train:8092", |
| "banking:train:8186", |
| "banking:train:8215", |
| "banking:train:8667", |
| "banking:train:8752", |
| "banking:train:8763", |
| "banking:train:8851", |
| "banking:train:8917", |
| "banking:train:9145", |
| "banking:train:9279", |
| "banking:train:9676", |
| "banking:train:9837", |
| "banking:train:9967", |
| "boolq:validation:106", |
| "boolq:validation:1065", |
| "boolq:validation:1097", |
| "boolq:validation:1336", |
| "boolq:validation:1350", |
| "boolq:validation:1383", |
| "boolq:validation:1458", |
| "boolq:validation:1641", |
| "boolq:validation:1873", |
| "boolq:validation:1930", |
| "boolq:validation:1989", |
| "boolq:validation:2103", |
| "boolq:validation:234", |
| "boolq:validation:2355", |
| "boolq:validation:2384", |
| "boolq:validation:2440", |
| "boolq:validation:2444", |
| "boolq:validation:2478", |
| "boolq:validation:2872", |
| "boolq:validation:2879", |
| "boolq:validation:2924", |
| "boolq:validation:2971", |
| "boolq:validation:3164", |
| "boolq:validation:4", |
| "boolq:validation:415", |
| "boolq:validation:419", |
| "boolq:validation:428", |
| "boolq:validation:512", |
| "boolq:validation:563", |
| "boolq:validation:643", |
| "boolq:validation:696", |
| "boolq:validation:874", |
| "snli:validation:1102", |
| "snli:validation:1240", |
| "snli:validation:1261", |
| "snli:validation:1738", |
| "snli:validation:1978", |
| "snli:validation:2563", |
| "snli:validation:2659", |
| "snli:validation:2767", |
| "snli:validation:2836", |
| "snli:validation:3232", |
| "snli:validation:3655", |
| "snli:validation:3913", |
| "snli:validation:405", |
| "snli:validation:4051", |
| "snli:validation:4300", |
| "snli:validation:4846", |
| "snli:validation:498", |
| "snli:validation:5224", |
| "snli:validation:5575", |
| "snli:validation:6598", |
| "snli:validation:666", |
| "snli:validation:6925", |
| "snli:validation:6964", |
| "snli:validation:7045", |
| "snli:validation:7351", |
| "snli:validation:7588", |
| "snli:validation:7723", |
| "snli:validation:7750", |
| "snli:validation:7909", |
| "snli:validation:8974", |
| "snli:validation:9385", |
| "snli:validation:9646" |
| ], |
| "heldout_group_count": 128, |
| "training_group_count": 384 |
| } |
| ], |
| "per_family": { |
| "arc": { |
| "count": 128, |
| "score": 0.38706141779183195, |
| "raw_macro_nll": 0.5262972364291403, |
| "accuracy": 0.890625 |
| }, |
| "banking": { |
| "count": 128, |
| "score": 0.132631760684535, |
| "raw_macro_nll": 0.22976245986744437, |
| "accuracy": 0.953125 |
| }, |
| "boolq": { |
| "count": 128, |
| "score": 0.35232187333420917, |
| "raw_macro_nll": 0.5460250472998999, |
| "accuracy": 0.8359375 |
| }, |
| "snli": { |
| "count": 128, |
| "score": 0.4407362987626767, |
| "raw_macro_nll": 0.560756053710936, |
| "accuracy": 0.84375 |
| } |
| }, |
| "policy": { |
| "id": "crossfit_temperature_nll_v1", |
| "folds": 4, |
| "seed": 431, |
| "grouping": "source group, nested within task family; groups shuffled within sorted families and assigned round-robin", |
| "temperature_grid": { |
| "count": 101, |
| "log10_min": -1.0, |
| "log10_max": 1.3, |
| "arithmetic": "Python float64" |
| }, |
| "temperature_fit": "minimum macro-family NLL on the other three folds; smallest temperature wins ties", |
| "score": "macro-family mean of all held-out decision NLLs", |
| "deployment_temperature": "fit afresh on reserved calibration only after model selection" |
| } |
| }, |
| "best_selection_score": 0.3281878376433132, |
| "improved": true, |
| "metrics": { |
| "overall": { |
| "n": 512, |
| "accuracy": 0.880859375, |
| "nll": 0.4657101983683963, |
| "brier_multiclass_sum": 0.1966596386060834, |
| "ece_top_label_10_equal_width_bins": 0.08195119712036103, |
| "reliability_bins": [ |
| { |
| "count": 0, |
| "confidence_sum": 0.0, |
| "correct_sum": 0.0 |
| }, |
| { |
| "count": 0, |
| "confidence_sum": 0.0, |
| "correct_sum": 0.0 |
| }, |
| { |
| "count": 0, |
| "confidence_sum": 0.0, |
| "correct_sum": 0.0 |
| }, |
| { |
| "count": 0, |
| "confidence_sum": 0.0, |
| "correct_sum": 0.0 |
| }, |
| { |
| "count": 4, |
| "confidence_sum": 1.8117924332618713, |
| "correct_sum": 4.0 |
| }, |
| { |
| "count": 16, |
| "confidence_sum": 8.777001142501831, |
| "correct_sum": 6.0 |
| }, |
| { |
| "count": 6, |
| "confidence_sum": 3.8906716108322144, |
| "correct_sum": 2.0 |
| }, |
| { |
| "count": 17, |
| "confidence_sum": 12.783255934715271, |
| "correct_sum": 9.0 |
| }, |
| { |
| "count": 20, |
| "confidence_sum": 17.166792809963226, |
| "correct_sum": 17.0 |
| }, |
| { |
| "count": 449, |
| "confidence_sum": 444.1530838608742, |
| "correct_sum": 413.0 |
| } |
| ], |
| "accuracy_vs_coverage": { |
| "0.25": { |
| "n": 128, |
| "accuracy": 1.0, |
| "min_confidence": 0.999997615814209 |
| }, |
| "0.5": { |
| "n": 256, |
| "accuracy": 0.9921875, |
| "min_confidence": 0.999165415763855 |
| }, |
| "0.75": { |
| "n": 384, |
| "accuracy": 0.9453125, |
| "min_confidence": 0.9738333225250244 |
| }, |
| "1.0": { |
| "n": 512, |
| "accuracy": 0.880859375, |
| "min_confidence": 0.40963199734687805 |
| } |
| } |
| }, |
| "per_family": { |
| "arc": { |
| "n": 128, |
| "accuracy": 0.890625, |
| "nll": 0.5262972319795522, |
| "brier_multiclass_sum": 0.19759357566681648, |
| "ece_top_label_10_equal_width_bins": 0.08619366958737373, |
| "reliability_bins": [ |
| { |
| "count": 0, |
| "confidence_sum": 0.0, |
| "correct_sum": 0.0 |
| }, |
| { |
| "count": 0, |
| "confidence_sum": 0.0, |
| "correct_sum": 0.0 |
| }, |
| { |
| "count": 0, |
| "confidence_sum": 0.0, |
| "correct_sum": 0.0 |
| }, |
| { |
| "count": 0, |
| "confidence_sum": 0.0, |
| "correct_sum": 0.0 |
| }, |
| { |
| "count": 4, |
| "confidence_sum": 1.8117924332618713, |
| "correct_sum": 4.0 |
| }, |
| { |
| "count": 5, |
| "confidence_sum": 2.6929340958595276, |
| "correct_sum": 3.0 |
| }, |
| { |
| "count": 1, |
| "confidence_sum": 0.6008858680725098, |
| "correct_sum": 0.0 |
| }, |
| { |
| "count": 6, |
| "confidence_sum": 4.4938578605651855, |
| "correct_sum": 4.0 |
| }, |
| { |
| "count": 4, |
| "confidence_sum": 3.384382665157318, |
| "correct_sum": 3.0 |
| }, |
| { |
| "count": 108, |
| "confidence_sum": 107.05838984251022, |
| "correct_sum": 100.0 |
| } |
| ], |
| "accuracy_vs_coverage": { |
| "0.25": { |
| "n": 32, |
| "accuracy": 1.0, |
| "min_confidence": 0.999995231628418 |
| }, |
| "0.5": { |
| "n": 64, |
| "accuracy": 1.0, |
| "min_confidence": 0.9993054866790771 |
| }, |
| "0.75": { |
| "n": 96, |
| "accuracy": 0.9479166666666666, |
| "min_confidence": 0.9695820808410645 |
| }, |
| "1.0": { |
| "n": 128, |
| "accuracy": 0.890625, |
| "min_confidence": 0.40963199734687805 |
| } |
| } |
| }, |
| "banking": { |
| "n": 128, |
| "accuracy": 0.953125, |
| "nll": 0.22976245635574838, |
| "brier_multiclass_sum": 0.08413616641877218, |
| "ece_top_label_10_equal_width_bins": 0.04192498791962862, |
| "reliability_bins": [ |
| { |
| "count": 0, |
| "confidence_sum": 0.0, |
| "correct_sum": 0.0 |
| }, |
| { |
| "count": 0, |
| "confidence_sum": 0.0, |
| "correct_sum": 0.0 |
| }, |
| { |
| "count": 0, |
| "confidence_sum": 0.0, |
| "correct_sum": 0.0 |
| }, |
| { |
| "count": 0, |
| "confidence_sum": 0.0, |
| "correct_sum": 0.0 |
| }, |
| { |
| "count": 0, |
| "confidence_sum": 0.0, |
| "correct_sum": 0.0 |
| }, |
| { |
| "count": 0, |
| "confidence_sum": 0.0, |
| "correct_sum": 0.0 |
| }, |
| { |
| "count": 0, |
| "confidence_sum": 0.0, |
| "correct_sum": 0.0 |
| }, |
| { |
| "count": 2, |
| "confidence_sum": 1.5179685354232788, |
| "correct_sum": 1.0 |
| }, |
| { |
| "count": 1, |
| "confidence_sum": 0.8676369190216064, |
| "correct_sum": 1.0 |
| }, |
| { |
| "count": 125, |
| "confidence_sum": 124.71606683731079, |
| "correct_sum": 120.0 |
| } |
| ], |
| "accuracy_vs_coverage": { |
| "0.25": { |
| "n": 32, |
| "accuracy": 1.0, |
| "min_confidence": 1.0 |
| }, |
| "0.5": { |
| "n": 64, |
| "accuracy": 1.0, |
| "min_confidence": 1.0 |
| }, |
| "0.75": { |
| "n": 96, |
| "accuracy": 1.0, |
| "min_confidence": 0.9999890327453613 |
| }, |
| "1.0": { |
| "n": 128, |
| "accuracy": 0.953125, |
| "min_confidence": 0.7520317435264587 |
| } |
| } |
| }, |
| "boolq": { |
| "n": 128, |
| "accuracy": 0.8359375, |
| "nll": 0.5460250501834345, |
| "brier_multiclass_sum": 0.2505062007079596, |
| "ece_top_label_10_equal_width_bins": 0.11891159228980541, |
| "reliability_bins": [ |
| { |
| "count": 0, |
| "confidence_sum": 0.0, |
| "correct_sum": 0.0 |
| }, |
| { |
| "count": 0, |
| "confidence_sum": 0.0, |
| "correct_sum": 0.0 |
| }, |
| { |
| "count": 0, |
| "confidence_sum": 0.0, |
| "correct_sum": 0.0 |
| }, |
| { |
| "count": 0, |
| "confidence_sum": 0.0, |
| "correct_sum": 0.0 |
| }, |
| { |
| "count": 0, |
| "confidence_sum": 0.0, |
| "correct_sum": 0.0 |
| }, |
| { |
| "count": 4, |
| "confidence_sum": 2.2954535484313965, |
| "correct_sum": 0.0 |
| }, |
| { |
| "count": 1, |
| "confidence_sum": 0.6820338368415833, |
| "correct_sum": 0.0 |
| }, |
| { |
| "count": 6, |
| "confidence_sum": 4.4924063086509705, |
| "correct_sum": 3.0 |
| }, |
| { |
| "count": 6, |
| "confidence_sum": 5.07128119468689, |
| "correct_sum": 5.0 |
| }, |
| { |
| "count": 111, |
| "confidence_sum": 109.67950892448425, |
| "correct_sum": 99.0 |
| } |
| ], |
| "accuracy_vs_coverage": { |
| "0.25": { |
| "n": 32, |
| "accuracy": 1.0, |
| "min_confidence": 0.9998440742492676 |
| }, |
| "0.5": { |
| "n": 64, |
| "accuracy": 0.96875, |
| "min_confidence": 0.9980106949806213 |
| }, |
| "0.75": { |
| "n": 96, |
| "accuracy": 0.8958333333333334, |
| "min_confidence": 0.9715626835823059 |
| }, |
| "1.0": { |
| "n": 128, |
| "accuracy": 0.8359375, |
| "min_confidence": 0.5350096821784973 |
| } |
| } |
| }, |
| "snli": { |
| "n": 128, |
| "accuracy": 0.84375, |
| "nll": 0.56075605495485, |
| "brier_multiclass_sum": 0.2544026116307853, |
| "ece_top_label_10_equal_width_bins": 0.09008605359122157, |
| "reliability_bins": [ |
| { |
| "count": 0, |
| "confidence_sum": 0.0, |
| "correct_sum": 0.0 |
| }, |
| { |
| "count": 0, |
| "confidence_sum": 0.0, |
| "correct_sum": 0.0 |
| }, |
| { |
| "count": 0, |
| "confidence_sum": 0.0, |
| "correct_sum": 0.0 |
| }, |
| { |
| "count": 0, |
| "confidence_sum": 0.0, |
| "correct_sum": 0.0 |
| }, |
| { |
| "count": 0, |
| "confidence_sum": 0.0, |
| "correct_sum": 0.0 |
| }, |
| { |
| "count": 7, |
| "confidence_sum": 3.788613498210907, |
| "correct_sum": 3.0 |
| }, |
| { |
| "count": 4, |
| "confidence_sum": 2.6077519059181213, |
| "correct_sum": 2.0 |
| }, |
| { |
| "count": 3, |
| "confidence_sum": 2.279023230075836, |
| "correct_sum": 1.0 |
| }, |
| { |
| "count": 9, |
| "confidence_sum": 7.843492031097412, |
| "correct_sum": 8.0 |
| }, |
| { |
| "count": 105, |
| "confidence_sum": 102.69911825656891, |
| "correct_sum": 94.0 |
| } |
| ], |
| "accuracy_vs_coverage": { |
| "0.25": { |
| "n": 32, |
| "accuracy": 0.96875, |
| "min_confidence": 0.995185911655426 |
| }, |
| "0.5": { |
| "n": 64, |
| "accuracy": 0.96875, |
| "min_confidence": 0.980144739151001 |
| }, |
| "0.75": { |
| "n": 96, |
| "accuracy": 0.90625, |
| "min_confidence": 0.9447706341743469 |
| }, |
| "1.0": { |
| "n": 128, |
| "accuracy": 0.84375, |
| "min_confidence": 0.5119929313659668 |
| } |
| } |
| } |
| } |
| } |
| }, |
| { |
| "step": 1500, |
| "macro_nll": 0.3541424148692931, |
| "best_macro_nll": 0.4657101983683963, |
| "selection": { |
| "metric": "crossfit_temperature_nll_v1", |
| "score": 0.3300765088337009, |
| "raw_macro_nll": 0.35414241381908923, |
| "accuracy": 0.8828125, |
| "folds": [ |
| { |
| "fold": 0, |
| "temperature_index": 52, |
| "temperature": 1.5703628043335522, |
| "training_macro_nll": 0.3473758787149227, |
| "heldout_ids": [ |
| "arc:validation:ARC-Challenge:122", |
| "arc:validation:ARC-Challenge:126", |
| "arc:validation:ARC-Challenge:175", |
| "arc:validation:ARC-Challenge:187", |
| "arc:validation:ARC-Challenge:189", |
| "arc:validation:ARC-Challenge:192", |
| "arc:validation:ARC-Challenge:208", |
| "arc:validation:ARC-Challenge:238", |
| "arc:validation:ARC-Challenge:290", |
| "arc:validation:ARC-Challenge:293", |
| "arc:validation:ARC-Challenge:35", |
| "arc:validation:ARC-Challenge:43", |
| "arc:validation:ARC-Challenge:58", |
| "arc:validation:ARC-Challenge:7", |
| "arc:validation:ARC-Easy:160", |
| "arc:validation:ARC-Easy:201", |
| "arc:validation:ARC-Easy:229", |
| "arc:validation:ARC-Easy:238", |
| "arc:validation:ARC-Easy:253", |
| "arc:validation:ARC-Easy:259", |
| "arc:validation:ARC-Easy:272", |
| "arc:validation:ARC-Easy:298", |
| "arc:validation:ARC-Easy:329", |
| "arc:validation:ARC-Easy:331", |
| "arc:validation:ARC-Easy:337", |
| "arc:validation:ARC-Easy:362", |
| "arc:validation:ARC-Easy:443", |
| "arc:validation:ARC-Easy:500", |
| "arc:validation:ARC-Easy:545", |
| "arc:validation:ARC-Easy:68", |
| "arc:validation:ARC-Easy:7", |
| "arc:validation:ARC-Easy:74", |
| "banking:train:1053", |
| "banking:train:1594", |
| "banking:train:1747", |
| "banking:train:202", |
| "banking:train:2377", |
| "banking:train:2528", |
| "banking:train:2546", |
| "banking:train:2715", |
| "banking:train:3220", |
| "banking:train:3499", |
| "banking:train:3806", |
| "banking:train:3838", |
| "banking:train:3993", |
| "banking:train:4220", |
| "banking:train:5", |
| "banking:train:5192", |
| "banking:train:5394", |
| "banking:train:5846", |
| "banking:train:5863", |
| "banking:train:6032", |
| "banking:train:6508", |
| "banking:train:7579", |
| "banking:train:7600", |
| "banking:train:7987", |
| "banking:train:8155", |
| "banking:train:850", |
| "banking:train:8626", |
| "banking:train:8929", |
| "banking:train:9355", |
| "banking:train:9812", |
| "banking:train:987", |
| "banking:train:9972", |
| "boolq:validation:103", |
| "boolq:validation:1042", |
| "boolq:validation:1059", |
| "boolq:validation:1541", |
| "boolq:validation:1592", |
| "boolq:validation:1927", |
| "boolq:validation:2014", |
| "boolq:validation:2023", |
| "boolq:validation:2098", |
| "boolq:validation:2139", |
| "boolq:validation:2439", |
| "boolq:validation:2485", |
| "boolq:validation:2607", |
| "boolq:validation:2611", |
| "boolq:validation:2720", |
| "boolq:validation:2751", |
| "boolq:validation:282", |
| "boolq:validation:2843", |
| "boolq:validation:2953", |
| "boolq:validation:301", |
| "boolq:validation:3089", |
| "boolq:validation:313", |
| "boolq:validation:3201", |
| "boolq:validation:322", |
| "boolq:validation:3231", |
| "boolq:validation:39", |
| "boolq:validation:461", |
| "boolq:validation:558", |
| "boolq:validation:589", |
| "boolq:validation:731", |
| "boolq:validation:883", |
| "boolq:validation:915", |
| "snli:validation:1156", |
| "snli:validation:1330", |
| "snli:validation:1559", |
| "snli:validation:1735", |
| "snli:validation:1876", |
| "snli:validation:2251", |
| "snli:validation:2257", |
| "snli:validation:2377", |
| "snli:validation:2881", |
| "snli:validation:291", |
| "snli:validation:300", |
| "snli:validation:3049", |
| "snli:validation:3496", |
| "snli:validation:3871", |
| "snli:validation:4414", |
| "snli:validation:5140", |
| "snli:validation:5440", |
| "snli:validation:5806", |
| "snli:validation:6442", |
| "snli:validation:6772", |
| "snli:validation:6949", |
| "snli:validation:7228", |
| "snli:validation:7465", |
| "snli:validation:7510", |
| "snli:validation:7726", |
| "snli:validation:8095", |
| "snli:validation:8143", |
| "snli:validation:8623", |
| "snli:validation:8962", |
| "snli:validation:9451", |
| "snli:validation:9574", |
| "snli:validation:9655" |
| ], |
| "heldout_group_count": 128, |
| "training_group_count": 384 |
| }, |
| { |
| "fold": 1, |
| "temperature_index": 51, |
| "temperature": 1.4893610777109154, |
| "training_macro_nll": 0.33312313335211086, |
| "heldout_ids": [ |
| "arc:validation:ARC-Challenge:101", |
| "arc:validation:ARC-Challenge:129", |
| "arc:validation:ARC-Challenge:148", |
| "arc:validation:ARC-Challenge:156", |
| "arc:validation:ARC-Challenge:167", |
| "arc:validation:ARC-Challenge:249", |
| "arc:validation:ARC-Challenge:263", |
| "arc:validation:ARC-Challenge:266", |
| "arc:validation:ARC-Challenge:267", |
| "arc:validation:ARC-Challenge:28", |
| "arc:validation:ARC-Challenge:281", |
| "arc:validation:ARC-Challenge:57", |
| "arc:validation:ARC-Challenge:63", |
| "arc:validation:ARC-Challenge:64", |
| "arc:validation:ARC-Challenge:86", |
| "arc:validation:ARC-Challenge:99", |
| "arc:validation:ARC-Easy:106", |
| "arc:validation:ARC-Easy:151", |
| "arc:validation:ARC-Easy:157", |
| "arc:validation:ARC-Easy:166", |
| "arc:validation:ARC-Easy:202", |
| "arc:validation:ARC-Easy:219", |
| "arc:validation:ARC-Easy:224", |
| "arc:validation:ARC-Easy:322", |
| "arc:validation:ARC-Easy:344", |
| "arc:validation:ARC-Easy:382", |
| "arc:validation:ARC-Easy:409", |
| "arc:validation:ARC-Easy:455", |
| "arc:validation:ARC-Easy:50", |
| "arc:validation:ARC-Easy:538", |
| "arc:validation:ARC-Easy:544", |
| "arc:validation:ARC-Easy:552", |
| "banking:train:1050", |
| "banking:train:1131", |
| "banking:train:1190", |
| "banking:train:124", |
| "banking:train:137", |
| "banking:train:1493", |
| "banking:train:1498", |
| "banking:train:1624", |
| "banking:train:2614", |
| "banking:train:2739", |
| "banking:train:3055", |
| "banking:train:306", |
| "banking:train:3312", |
| "banking:train:3724", |
| "banking:train:4070", |
| "banking:train:476", |
| "banking:train:4882", |
| "banking:train:5075", |
| "banking:train:5332", |
| "banking:train:5445", |
| "banking:train:6096", |
| "banking:train:746", |
| "banking:train:7828", |
| "banking:train:8176", |
| "banking:train:8315", |
| "banking:train:8653", |
| "banking:train:9019", |
| "banking:train:9370", |
| "banking:train:9454", |
| "banking:train:9595", |
| "banking:train:9862", |
| "banking:train:9963", |
| "boolq:validation:1189", |
| "boolq:validation:1320", |
| "boolq:validation:1326", |
| "boolq:validation:1381", |
| "boolq:validation:1410", |
| "boolq:validation:149", |
| "boolq:validation:1513", |
| "boolq:validation:1732", |
| "boolq:validation:1739", |
| "boolq:validation:1741", |
| "boolq:validation:1814", |
| "boolq:validation:1848", |
| "boolq:validation:1940", |
| "boolq:validation:2077", |
| "boolq:validation:2091", |
| "boolq:validation:2148", |
| "boolq:validation:2215", |
| "boolq:validation:2251", |
| "boolq:validation:2283", |
| "boolq:validation:2321", |
| "boolq:validation:2398", |
| "boolq:validation:253", |
| "boolq:validation:2575", |
| "boolq:validation:2907", |
| "boolq:validation:2909", |
| "boolq:validation:2978", |
| "boolq:validation:359", |
| "boolq:validation:394", |
| "boolq:validation:436", |
| "boolq:validation:579", |
| "boolq:validation:807", |
| "boolq:validation:949", |
| "snli:validation:2389", |
| "snli:validation:2485", |
| "snli:validation:255", |
| "snli:validation:3169", |
| "snli:validation:333", |
| "snli:validation:3700", |
| "snli:validation:3709", |
| "snli:validation:4093", |
| "snli:validation:420", |
| "snli:validation:4492", |
| "snli:validation:4516", |
| "snli:validation:4648", |
| "snli:validation:4888", |
| "snli:validation:5185", |
| "snli:validation:5332", |
| "snli:validation:5947", |
| "snli:validation:6124", |
| "snli:validation:660", |
| "snli:validation:6619", |
| "snli:validation:6898", |
| "snli:validation:7387", |
| "snli:validation:7492", |
| "snli:validation:7544", |
| "snli:validation:7996", |
| "snli:validation:8074", |
| "snli:validation:8554", |
| "snli:validation:8971", |
| "snli:validation:9163", |
| "snli:validation:9277", |
| "snli:validation:9490", |
| "snli:validation:9595", |
| "snli:validation:9688" |
| ], |
| "heldout_group_count": 128, |
| "training_group_count": 384 |
| }, |
| { |
| "fold": 2, |
| "temperature_index": 51, |
| "temperature": 1.4893610777109154, |
| "training_macro_nll": 0.32299109700120576, |
| "heldout_ids": [ |
| "arc:validation:ARC-Challenge:120", |
| "arc:validation:ARC-Challenge:130", |
| "arc:validation:ARC-Challenge:194", |
| "arc:validation:ARC-Challenge:218", |
| "arc:validation:ARC-Challenge:235", |
| "arc:validation:ARC-Challenge:272", |
| "arc:validation:ARC-Challenge:277", |
| "arc:validation:ARC-Challenge:48", |
| "arc:validation:ARC-Challenge:88", |
| "arc:validation:ARC-Easy:115", |
| "arc:validation:ARC-Easy:12", |
| "arc:validation:ARC-Easy:131", |
| "arc:validation:ARC-Easy:145", |
| "arc:validation:ARC-Easy:179", |
| "arc:validation:ARC-Easy:18", |
| "arc:validation:ARC-Easy:186", |
| "arc:validation:ARC-Easy:191", |
| "arc:validation:ARC-Easy:215", |
| "arc:validation:ARC-Easy:283", |
| "arc:validation:ARC-Easy:314", |
| "arc:validation:ARC-Easy:315", |
| "arc:validation:ARC-Easy:378", |
| "arc:validation:ARC-Easy:384", |
| "arc:validation:ARC-Easy:403", |
| "arc:validation:ARC-Easy:432", |
| "arc:validation:ARC-Easy:439", |
| "arc:validation:ARC-Easy:448", |
| "arc:validation:ARC-Easy:461", |
| "arc:validation:ARC-Easy:469", |
| "arc:validation:ARC-Easy:470", |
| "arc:validation:ARC-Easy:504", |
| "arc:validation:ARC-Easy:561", |
| "banking:train:1142", |
| "banking:train:148", |
| "banking:train:1633", |
| "banking:train:1927", |
| "banking:train:2186", |
| "banking:train:2251", |
| "banking:train:23", |
| "banking:train:3048", |
| "banking:train:3665", |
| "banking:train:3952", |
| "banking:train:3999", |
| "banking:train:4364", |
| "banking:train:4619", |
| "banking:train:5049", |
| "banking:train:5335", |
| "banking:train:549", |
| "banking:train:5957", |
| "banking:train:5985", |
| "banking:train:6132", |
| "banking:train:6240", |
| "banking:train:6488", |
| "banking:train:7065", |
| "banking:train:7749", |
| "banking:train:8470", |
| "banking:train:8761", |
| "banking:train:895", |
| "banking:train:9256", |
| "banking:train:9406", |
| "banking:train:9450", |
| "banking:train:9467", |
| "banking:train:9588", |
| "banking:train:9716", |
| "boolq:validation:1130", |
| "boolq:validation:1167", |
| "boolq:validation:124", |
| "boolq:validation:1255", |
| "boolq:validation:1256", |
| "boolq:validation:1373", |
| "boolq:validation:15", |
| "boolq:validation:155", |
| "boolq:validation:1568", |
| "boolq:validation:16", |
| "boolq:validation:1654", |
| "boolq:validation:1702", |
| "boolq:validation:1792", |
| "boolq:validation:1818", |
| "boolq:validation:210", |
| "boolq:validation:2247", |
| "boolq:validation:2259", |
| "boolq:validation:2362", |
| "boolq:validation:2443", |
| "boolq:validation:2524", |
| "boolq:validation:2553", |
| "boolq:validation:2623", |
| "boolq:validation:2660", |
| "boolq:validation:2669", |
| "boolq:validation:2679", |
| "boolq:validation:2832", |
| "boolq:validation:357", |
| "boolq:validation:453", |
| "boolq:validation:49", |
| "boolq:validation:808", |
| "boolq:validation:90", |
| "boolq:validation:910", |
| "snli:validation:1612", |
| "snli:validation:168", |
| "snli:validation:1693", |
| "snli:validation:2134", |
| "snli:validation:2248", |
| "snli:validation:2422", |
| "snli:validation:2587", |
| "snli:validation:3007", |
| "snli:validation:3097", |
| "snli:validation:3772", |
| "snli:validation:4012", |
| "snli:validation:4168", |
| "snli:validation:4537", |
| "snli:validation:4681", |
| "snli:validation:5050", |
| "snli:validation:5632", |
| "snli:validation:5761", |
| "snli:validation:6055", |
| "snli:validation:6283", |
| "snli:validation:6715", |
| "snli:validation:7117", |
| "snli:validation:7192", |
| "snli:validation:7507", |
| "snli:validation:7594", |
| "snli:validation:7924", |
| "snli:validation:7954", |
| "snli:validation:8035", |
| "snli:validation:8110", |
| "snli:validation:8995", |
| "snli:validation:949", |
| "snli:validation:9523", |
| "snli:validation:9589" |
| ], |
| "heldout_group_count": 128, |
| "training_group_count": 384 |
| }, |
| { |
| "fold": 3, |
| "temperature_index": 49, |
| "temperature": 1.3396766874259345, |
| "training_macro_nll": 0.30270275653699436, |
| "heldout_ids": [ |
| "arc:validation:ARC-Challenge:117", |
| "arc:validation:ARC-Challenge:135", |
| "arc:validation:ARC-Challenge:139", |
| "arc:validation:ARC-Challenge:154", |
| "arc:validation:ARC-Challenge:174", |
| "arc:validation:ARC-Challenge:199", |
| "arc:validation:ARC-Challenge:287", |
| "arc:validation:ARC-Challenge:295", |
| "arc:validation:ARC-Challenge:4", |
| "arc:validation:ARC-Challenge:50", |
| "arc:validation:ARC-Challenge:51", |
| "arc:validation:ARC-Challenge:61", |
| "arc:validation:ARC-Easy:107", |
| "arc:validation:ARC-Easy:121", |
| "arc:validation:ARC-Easy:124", |
| "arc:validation:ARC-Easy:171", |
| "arc:validation:ARC-Easy:235", |
| "arc:validation:ARC-Easy:241", |
| "arc:validation:ARC-Easy:279", |
| "arc:validation:ARC-Easy:289", |
| "arc:validation:ARC-Easy:37", |
| "arc:validation:ARC-Easy:389", |
| "arc:validation:ARC-Easy:394", |
| "arc:validation:ARC-Easy:399", |
| "arc:validation:ARC-Easy:423", |
| "arc:validation:ARC-Easy:492", |
| "arc:validation:ARC-Easy:499", |
| "arc:validation:ARC-Easy:523", |
| "arc:validation:ARC-Easy:533", |
| "arc:validation:ARC-Easy:569", |
| "arc:validation:ARC-Easy:69", |
| "arc:validation:ARC-Easy:93", |
| "banking:train:2049", |
| "banking:train:2350", |
| "banking:train:2497", |
| "banking:train:2506", |
| "banking:train:2720", |
| "banking:train:2788", |
| "banking:train:2967", |
| "banking:train:3502", |
| "banking:train:3726", |
| "banking:train:4014", |
| "banking:train:4735", |
| "banking:train:4803", |
| "banking:train:4917", |
| "banking:train:5477", |
| "banking:train:5851", |
| "banking:train:6551", |
| "banking:train:6839", |
| "banking:train:6926", |
| "banking:train:7809", |
| "banking:train:8092", |
| "banking:train:8186", |
| "banking:train:8215", |
| "banking:train:8667", |
| "banking:train:8752", |
| "banking:train:8763", |
| "banking:train:8851", |
| "banking:train:8917", |
| "banking:train:9145", |
| "banking:train:9279", |
| "banking:train:9676", |
| "banking:train:9837", |
| "banking:train:9967", |
| "boolq:validation:106", |
| "boolq:validation:1065", |
| "boolq:validation:1097", |
| "boolq:validation:1336", |
| "boolq:validation:1350", |
| "boolq:validation:1383", |
| "boolq:validation:1458", |
| "boolq:validation:1641", |
| "boolq:validation:1873", |
| "boolq:validation:1930", |
| "boolq:validation:1989", |
| "boolq:validation:2103", |
| "boolq:validation:234", |
| "boolq:validation:2355", |
| "boolq:validation:2384", |
| "boolq:validation:2440", |
| "boolq:validation:2444", |
| "boolq:validation:2478", |
| "boolq:validation:2872", |
| "boolq:validation:2879", |
| "boolq:validation:2924", |
| "boolq:validation:2971", |
| "boolq:validation:3164", |
| "boolq:validation:4", |
| "boolq:validation:415", |
| "boolq:validation:419", |
| "boolq:validation:428", |
| "boolq:validation:512", |
| "boolq:validation:563", |
| "boolq:validation:643", |
| "boolq:validation:696", |
| "boolq:validation:874", |
| "snli:validation:1102", |
| "snli:validation:1240", |
| "snli:validation:1261", |
| "snli:validation:1738", |
| "snli:validation:1978", |
| "snli:validation:2563", |
| "snli:validation:2659", |
| "snli:validation:2767", |
| "snli:validation:2836", |
| "snli:validation:3232", |
| "snli:validation:3655", |
| "snli:validation:3913", |
| "snli:validation:405", |
| "snli:validation:4051", |
| "snli:validation:4300", |
| "snli:validation:4846", |
| "snli:validation:498", |
| "snli:validation:5224", |
| "snli:validation:5575", |
| "snli:validation:6598", |
| "snli:validation:666", |
| "snli:validation:6925", |
| "snli:validation:6964", |
| "snli:validation:7045", |
| "snli:validation:7351", |
| "snli:validation:7588", |
| "snli:validation:7723", |
| "snli:validation:7750", |
| "snli:validation:7909", |
| "snli:validation:8974", |
| "snli:validation:9385", |
| "snli:validation:9646" |
| ], |
| "heldout_group_count": 128, |
| "training_group_count": 384 |
| } |
| ], |
| "per_family": { |
| "arc": { |
| "count": 128, |
| "score": 0.39971936917627643, |
| "raw_macro_nll": 0.44148110277449976, |
| "accuracy": 0.8828125 |
| }, |
| "banking": { |
| "count": 128, |
| "score": 0.13308767041911446, |
| "raw_macro_nll": 0.1467244923439624, |
| "accuracy": 0.9375 |
| }, |
| "boolq": { |
| "count": 128, |
| "score": 0.41748072217353377, |
| "raw_macro_nll": 0.4907469603329647, |
| "accuracy": 0.8203125 |
| }, |
| "snli": { |
| "count": 128, |
| "score": 0.3700182735658788, |
| "raw_macro_nll": 0.3376170998249301, |
| "accuracy": 0.890625 |
| } |
| }, |
| "policy": { |
| "id": "crossfit_temperature_nll_v1", |
| "folds": 4, |
| "seed": 431, |
| "grouping": "source group, nested within task family; groups shuffled within sorted families and assigned round-robin", |
| "temperature_grid": { |
| "count": 101, |
| "log10_min": -1.0, |
| "log10_max": 1.3, |
| "arithmetic": "Python float64" |
| }, |
| "temperature_fit": "minimum macro-family NLL on the other three folds; smallest temperature wins ties", |
| "score": "macro-family mean of all held-out decision NLLs", |
| "deployment_temperature": "fit afresh on reserved calibration only after model selection" |
| } |
| }, |
| "best_selection_score": 0.3281878376433132, |
| "improved": false, |
| "metrics": { |
| "overall": { |
| "n": 512, |
| "accuracy": 0.8828125, |
| "nll": 0.3541424148692931, |
| "brier_multiclass_sum": 0.1805443581641838, |
| "ece_top_label_10_equal_width_bins": 0.04613619588781148, |
| "reliability_bins": [ |
| { |
| "count": 0, |
| "confidence_sum": 0.0, |
| "correct_sum": 0.0 |
| }, |
| { |
| "count": 0, |
| "confidence_sum": 0.0, |
| "correct_sum": 0.0 |
| }, |
| { |
| "count": 0, |
| "confidence_sum": 0.0, |
| "correct_sum": 0.0 |
| }, |
| { |
| "count": 3, |
| "confidence_sum": 1.1111035645008087, |
| "correct_sum": 1.0 |
| }, |
| { |
| "count": 7, |
| "confidence_sum": 3.200128525495529, |
| "correct_sum": 4.0 |
| }, |
| { |
| "count": 13, |
| "confidence_sum": 7.04876035451889, |
| "correct_sum": 5.0 |
| }, |
| { |
| "count": 17, |
| "confidence_sum": 11.040592074394226, |
| "correct_sum": 11.0 |
| }, |
| { |
| "count": 22, |
| "confidence_sum": 16.702092111110687, |
| "correct_sum": 14.0 |
| }, |
| { |
| "count": 57, |
| "confidence_sum": 48.59177905321121, |
| "correct_sum": 45.0 |
| }, |
| { |
| "count": 393, |
| "confidence_sum": 386.3275336623192, |
| "correct_sum": 372.0 |
| } |
| ], |
| "accuracy_vs_coverage": { |
| "0.25": { |
| "n": 128, |
| "accuracy": 1.0, |
| "min_confidence": 0.9998112320899963 |
| }, |
| "0.5": { |
| "n": 256, |
| "accuracy": 0.984375, |
| "min_confidence": 0.9876569509506226 |
| }, |
| "0.75": { |
| "n": 384, |
| "accuracy": 0.9479166666666666, |
| "min_confidence": 0.9104589223861694 |
| }, |
| "1.0": { |
| "n": 512, |
| "accuracy": 0.8828125, |
| "min_confidence": 0.3282764256000519 |
| } |
| } |
| }, |
| "per_family": { |
| "arc": { |
| "n": 128, |
| "accuracy": 0.8828125, |
| "nll": 0.4414811164252181, |
| "brier_multiclass_sum": 0.18197369388678808, |
| "ece_top_label_10_equal_width_bins": 0.0416633328422904, |
| "reliability_bins": [ |
| { |
| "count": 0, |
| "confidence_sum": 0.0, |
| "correct_sum": 0.0 |
| }, |
| { |
| "count": 0, |
| "confidence_sum": 0.0, |
| "correct_sum": 0.0 |
| }, |
| { |
| "count": 0, |
| "confidence_sum": 0.0, |
| "correct_sum": 0.0 |
| }, |
| { |
| "count": 3, |
| "confidence_sum": 1.1111035645008087, |
| "correct_sum": 1.0 |
| }, |
| { |
| "count": 5, |
| "confidence_sum": 2.229231983423233, |
| "correct_sum": 3.0 |
| }, |
| { |
| "count": 6, |
| "confidence_sum": 3.2244449257850647, |
| "correct_sum": 3.0 |
| }, |
| { |
| "count": 0, |
| "confidence_sum": 0.0, |
| "correct_sum": 0.0 |
| }, |
| { |
| "count": 3, |
| "confidence_sum": 2.3234559297561646, |
| "correct_sum": 2.0 |
| }, |
| { |
| "count": 12, |
| "confidence_sum": 10.301778793334961, |
| "correct_sum": 10.0 |
| }, |
| { |
| "count": 99, |
| "confidence_sum": 97.6013553738594, |
| "correct_sum": 94.0 |
| } |
| ], |
| "accuracy_vs_coverage": { |
| "0.25": { |
| "n": 32, |
| "accuracy": 1.0, |
| "min_confidence": 0.9998188614845276 |
| }, |
| "0.5": { |
| "n": 64, |
| "accuracy": 0.984375, |
| "min_confidence": 0.9946053624153137 |
| }, |
| "0.75": { |
| "n": 96, |
| "accuracy": 0.9479166666666666, |
| "min_confidence": 0.91451096534729 |
| }, |
| "1.0": { |
| "n": 128, |
| "accuracy": 0.8828125, |
| "min_confidence": 0.3282764256000519 |
| } |
| } |
| }, |
| "banking": { |
| "n": 128, |
| "accuracy": 0.9375, |
| "nll": 0.14672449013245747, |
| "brier_multiclass_sum": 0.08483707779111284, |
| "ece_top_label_10_equal_width_bins": 0.028193810489028692, |
| "reliability_bins": [ |
| { |
| "count": 0, |
| "confidence_sum": 0.0, |
| "correct_sum": 0.0 |
| }, |
| { |
| "count": 0, |
| "confidence_sum": 0.0, |
| "correct_sum": 0.0 |
| }, |
| { |
| "count": 0, |
| "confidence_sum": 0.0, |
| "correct_sum": 0.0 |
| }, |
| { |
| "count": 0, |
| "confidence_sum": 0.0, |
| "correct_sum": 0.0 |
| }, |
| { |
| "count": 2, |
| "confidence_sum": 0.9708965420722961, |
| "correct_sum": 1.0 |
| }, |
| { |
| "count": 2, |
| "confidence_sum": 1.1783032417297363, |
| "correct_sum": 0.0 |
| }, |
| { |
| "count": 3, |
| "confidence_sum": 1.9266068935394287, |
| "correct_sum": 2.0 |
| }, |
| { |
| "count": 3, |
| "confidence_sum": 2.2358115315437317, |
| "correct_sum": 2.0 |
| }, |
| { |
| "count": 3, |
| "confidence_sum": 2.4956213235855103, |
| "correct_sum": 2.0 |
| }, |
| { |
| "count": 115, |
| "confidence_sum": 114.59657508134842, |
| "correct_sum": 113.0 |
| } |
| ], |
| "accuracy_vs_coverage": { |
| "0.25": { |
| "n": 32, |
| "accuracy": 1.0, |
| "min_confidence": 1.0 |
| }, |
| "0.5": { |
| "n": 64, |
| "accuracy": 1.0, |
| "min_confidence": 0.999998927116394 |
| }, |
| "0.75": { |
| "n": 96, |
| "accuracy": 1.0, |
| "min_confidence": 0.9996840953826904 |
| }, |
| "1.0": { |
| "n": 128, |
| "accuracy": 0.9375, |
| "min_confidence": 0.47107839584350586 |
| } |
| } |
| }, |
| "boolq": { |
| "n": 128, |
| "accuracy": 0.8203125, |
| "nll": 0.49074695723096795, |
| "brier_multiclass_sum": 0.2747688886485723, |
| "ece_top_label_10_equal_width_bins": 0.1055659051053226, |
| "reliability_bins": [ |
| { |
| "count": 0, |
| "confidence_sum": 0.0, |
| "correct_sum": 0.0 |
| }, |
| { |
| "count": 0, |
| "confidence_sum": 0.0, |
| "correct_sum": 0.0 |
| }, |
| { |
| "count": 0, |
| "confidence_sum": 0.0, |
| "correct_sum": 0.0 |
| }, |
| { |
| "count": 0, |
| "confidence_sum": 0.0, |
| "correct_sum": 0.0 |
| }, |
| { |
| "count": 0, |
| "confidence_sum": 0.0, |
| "correct_sum": 0.0 |
| }, |
| { |
| "count": 3, |
| "confidence_sum": 1.5206995010375977, |
| "correct_sum": 1.0 |
| }, |
| { |
| "count": 6, |
| "confidence_sum": 3.793248772621155, |
| "correct_sum": 3.0 |
| }, |
| { |
| "count": 9, |
| "confidence_sum": 6.839642345905304, |
| "correct_sum": 5.0 |
| }, |
| { |
| "count": 9, |
| "confidence_sum": 7.7483291029930115, |
| "correct_sum": 6.0 |
| }, |
| { |
| "count": 101, |
| "confidence_sum": 98.61051613092422, |
| "correct_sum": 90.0 |
| } |
| ], |
| "accuracy_vs_coverage": { |
| "0.25": { |
| "n": 32, |
| "accuracy": 0.9375, |
| "min_confidence": 0.9944236874580383 |
| }, |
| "0.5": { |
| "n": 64, |
| "accuracy": 0.9375, |
| "min_confidence": 0.9793309569358826 |
| }, |
| "0.75": { |
| "n": 96, |
| "accuracy": 0.8958333333333334, |
| "min_confidence": 0.9120692014694214 |
| }, |
| "1.0": { |
| "n": 128, |
| "accuracy": 0.8203125, |
| "min_confidence": 0.5004621744155884 |
| } |
| } |
| }, |
| "snli": { |
| "n": 128, |
| "accuracy": 0.890625, |
| "nll": 0.33761709568852893, |
| "brier_multiclass_sum": 0.18059777233026203, |
| "ece_top_label_10_equal_width_bins": 0.020881996024399996, |
| "reliability_bins": [ |
| { |
| "count": 0, |
| "confidence_sum": 0.0, |
| "correct_sum": 0.0 |
| }, |
| { |
| "count": 0, |
| "confidence_sum": 0.0, |
| "correct_sum": 0.0 |
| }, |
| { |
| "count": 0, |
| "confidence_sum": 0.0, |
| "correct_sum": 0.0 |
| }, |
| { |
| "count": 0, |
| "confidence_sum": 0.0, |
| "correct_sum": 0.0 |
| }, |
| { |
| "count": 0, |
| "confidence_sum": 0.0, |
| "correct_sum": 0.0 |
| }, |
| { |
| "count": 2, |
| "confidence_sum": 1.1253126859664917, |
| "correct_sum": 1.0 |
| }, |
| { |
| "count": 8, |
| "confidence_sum": 5.320736408233643, |
| "correct_sum": 6.0 |
| }, |
| { |
| "count": 7, |
| "confidence_sum": 5.303182303905487, |
| "correct_sum": 5.0 |
| }, |
| { |
| "count": 33, |
| "confidence_sum": 28.04604983329773, |
| "correct_sum": 27.0 |
| }, |
| { |
| "count": 78, |
| "confidence_sum": 75.51908707618713, |
| "correct_sum": 75.0 |
| } |
| ], |
| "accuracy_vs_coverage": { |
| "0.25": { |
| "n": 32, |
| "accuracy": 0.96875, |
| "min_confidence": 0.9792487621307373 |
| }, |
| "0.5": { |
| "n": 64, |
| "accuracy": 0.96875, |
| "min_confidence": 0.9421285390853882 |
| }, |
| "0.75": { |
| "n": 96, |
| "accuracy": 0.9479166666666666, |
| "min_confidence": 0.847649335861206 |
| }, |
| "1.0": { |
| "n": 128, |
| "accuracy": 0.890625, |
| "min_confidence": 0.5262982249259949 |
| } |
| } |
| } |
| } |
| } |
| }, |
| { |
| "step": 2000, |
| "macro_nll": 0.25676974726429014, |
| "best_macro_nll": 0.25676974726429014, |
| "selection": { |
| "metric": "crossfit_temperature_nll_v1", |
| "score": 0.2552943737691716, |
| "raw_macro_nll": 0.25676974955497256, |
| "accuracy": 0.8984375, |
| "folds": [ |
| { |
| "fold": 0, |
| "temperature_index": 48, |
| "temperature": 1.2705741052085413, |
| "training_macro_nll": 0.27533710550694335, |
| "heldout_ids": [ |
| "arc:validation:ARC-Challenge:122", |
| "arc:validation:ARC-Challenge:126", |
| "arc:validation:ARC-Challenge:175", |
| "arc:validation:ARC-Challenge:187", |
| "arc:validation:ARC-Challenge:189", |
| "arc:validation:ARC-Challenge:192", |
| "arc:validation:ARC-Challenge:208", |
| "arc:validation:ARC-Challenge:238", |
| "arc:validation:ARC-Challenge:290", |
| "arc:validation:ARC-Challenge:293", |
| "arc:validation:ARC-Challenge:35", |
| "arc:validation:ARC-Challenge:43", |
| "arc:validation:ARC-Challenge:58", |
| "arc:validation:ARC-Challenge:7", |
| "arc:validation:ARC-Easy:160", |
| "arc:validation:ARC-Easy:201", |
| "arc:validation:ARC-Easy:229", |
| "arc:validation:ARC-Easy:238", |
| "arc:validation:ARC-Easy:253", |
| "arc:validation:ARC-Easy:259", |
| "arc:validation:ARC-Easy:272", |
| "arc:validation:ARC-Easy:298", |
| "arc:validation:ARC-Easy:329", |
| "arc:validation:ARC-Easy:331", |
| "arc:validation:ARC-Easy:337", |
| "arc:validation:ARC-Easy:362", |
| "arc:validation:ARC-Easy:443", |
| "arc:validation:ARC-Easy:500", |
| "arc:validation:ARC-Easy:545", |
| "arc:validation:ARC-Easy:68", |
| "arc:validation:ARC-Easy:7", |
| "arc:validation:ARC-Easy:74", |
| "banking:train:1053", |
| "banking:train:1594", |
| "banking:train:1747", |
| "banking:train:202", |
| "banking:train:2377", |
| "banking:train:2528", |
| "banking:train:2546", |
| "banking:train:2715", |
| "banking:train:3220", |
| "banking:train:3499", |
| "banking:train:3806", |
| "banking:train:3838", |
| "banking:train:3993", |
| "banking:train:4220", |
| "banking:train:5", |
| "banking:train:5192", |
| "banking:train:5394", |
| "banking:train:5846", |
| "banking:train:5863", |
| "banking:train:6032", |
| "banking:train:6508", |
| "banking:train:7579", |
| "banking:train:7600", |
| "banking:train:7987", |
| "banking:train:8155", |
| "banking:train:850", |
| "banking:train:8626", |
| "banking:train:8929", |
| "banking:train:9355", |
| "banking:train:9812", |
| "banking:train:987", |
| "banking:train:9972", |
| "boolq:validation:103", |
| "boolq:validation:1042", |
| "boolq:validation:1059", |
| "boolq:validation:1541", |
| "boolq:validation:1592", |
| "boolq:validation:1927", |
| "boolq:validation:2014", |
| "boolq:validation:2023", |
| "boolq:validation:2098", |
| "boolq:validation:2139", |
| "boolq:validation:2439", |
| "boolq:validation:2485", |
| "boolq:validation:2607", |
| "boolq:validation:2611", |
| "boolq:validation:2720", |
| "boolq:validation:2751", |
| "boolq:validation:282", |
| "boolq:validation:2843", |
| "boolq:validation:2953", |
| "boolq:validation:301", |
| "boolq:validation:3089", |
| "boolq:validation:313", |
| "boolq:validation:3201", |
| "boolq:validation:322", |
| "boolq:validation:3231", |
| "boolq:validation:39", |
| "boolq:validation:461", |
| "boolq:validation:558", |
| "boolq:validation:589", |
| "boolq:validation:731", |
| "boolq:validation:883", |
| "boolq:validation:915", |
| "snli:validation:1156", |
| "snli:validation:1330", |
| "snli:validation:1559", |
| "snli:validation:1735", |
| "snli:validation:1876", |
| "snli:validation:2251", |
| "snli:validation:2257", |
| "snli:validation:2377", |
| "snli:validation:2881", |
| "snli:validation:291", |
| "snli:validation:300", |
| "snli:validation:3049", |
| "snli:validation:3496", |
| "snli:validation:3871", |
| "snli:validation:4414", |
| "snli:validation:5140", |
| "snli:validation:5440", |
| "snli:validation:5806", |
| "snli:validation:6442", |
| "snli:validation:6772", |
| "snli:validation:6949", |
| "snli:validation:7228", |
| "snli:validation:7465", |
| "snli:validation:7510", |
| "snli:validation:7726", |
| "snli:validation:8095", |
| "snli:validation:8143", |
| "snli:validation:8623", |
| "snli:validation:8962", |
| "snli:validation:9451", |
| "snli:validation:9574", |
| "snli:validation:9655" |
| ], |
| "heldout_group_count": 128, |
| "training_group_count": 384 |
| }, |
| { |
| "fold": 1, |
| "temperature_index": 47, |
| "temperature": 1.2050359403717974, |
| "training_macro_nll": 0.25585027399066257, |
| "heldout_ids": [ |
| "arc:validation:ARC-Challenge:101", |
| "arc:validation:ARC-Challenge:129", |
| "arc:validation:ARC-Challenge:148", |
| "arc:validation:ARC-Challenge:156", |
| "arc:validation:ARC-Challenge:167", |
| "arc:validation:ARC-Challenge:249", |
| "arc:validation:ARC-Challenge:263", |
| "arc:validation:ARC-Challenge:266", |
| "arc:validation:ARC-Challenge:267", |
| "arc:validation:ARC-Challenge:28", |
| "arc:validation:ARC-Challenge:281", |
| "arc:validation:ARC-Challenge:57", |
| "arc:validation:ARC-Challenge:63", |
| "arc:validation:ARC-Challenge:64", |
| "arc:validation:ARC-Challenge:86", |
| "arc:validation:ARC-Challenge:99", |
| "arc:validation:ARC-Easy:106", |
| "arc:validation:ARC-Easy:151", |
| "arc:validation:ARC-Easy:157", |
| "arc:validation:ARC-Easy:166", |
| "arc:validation:ARC-Easy:202", |
| "arc:validation:ARC-Easy:219", |
| "arc:validation:ARC-Easy:224", |
| "arc:validation:ARC-Easy:322", |
| "arc:validation:ARC-Easy:344", |
| "arc:validation:ARC-Easy:382", |
| "arc:validation:ARC-Easy:409", |
| "arc:validation:ARC-Easy:455", |
| "arc:validation:ARC-Easy:50", |
| "arc:validation:ARC-Easy:538", |
| "arc:validation:ARC-Easy:544", |
| "arc:validation:ARC-Easy:552", |
| "banking:train:1050", |
| "banking:train:1131", |
| "banking:train:1190", |
| "banking:train:124", |
| "banking:train:137", |
| "banking:train:1493", |
| "banking:train:1498", |
| "banking:train:1624", |
| "banking:train:2614", |
| "banking:train:2739", |
| "banking:train:3055", |
| "banking:train:306", |
| "banking:train:3312", |
| "banking:train:3724", |
| "banking:train:4070", |
| "banking:train:476", |
| "banking:train:4882", |
| "banking:train:5075", |
| "banking:train:5332", |
| "banking:train:5445", |
| "banking:train:6096", |
| "banking:train:746", |
| "banking:train:7828", |
| "banking:train:8176", |
| "banking:train:8315", |
| "banking:train:8653", |
| "banking:train:9019", |
| "banking:train:9370", |
| "banking:train:9454", |
| "banking:train:9595", |
| "banking:train:9862", |
| "banking:train:9963", |
| "boolq:validation:1189", |
| "boolq:validation:1320", |
| "boolq:validation:1326", |
| "boolq:validation:1381", |
| "boolq:validation:1410", |
| "boolq:validation:149", |
| "boolq:validation:1513", |
| "boolq:validation:1732", |
| "boolq:validation:1739", |
| "boolq:validation:1741", |
| "boolq:validation:1814", |
| "boolq:validation:1848", |
| "boolq:validation:1940", |
| "boolq:validation:2077", |
| "boolq:validation:2091", |
| "boolq:validation:2148", |
| "boolq:validation:2215", |
| "boolq:validation:2251", |
| "boolq:validation:2283", |
| "boolq:validation:2321", |
| "boolq:validation:2398", |
| "boolq:validation:253", |
| "boolq:validation:2575", |
| "boolq:validation:2907", |
| "boolq:validation:2909", |
| "boolq:validation:2978", |
| "boolq:validation:359", |
| "boolq:validation:394", |
| "boolq:validation:436", |
| "boolq:validation:579", |
| "boolq:validation:807", |
| "boolq:validation:949", |
| "snli:validation:2389", |
| "snli:validation:2485", |
| "snli:validation:255", |
| "snli:validation:3169", |
| "snli:validation:333", |
| "snli:validation:3700", |
| "snli:validation:3709", |
| "snli:validation:4093", |
| "snli:validation:420", |
| "snli:validation:4492", |
| "snli:validation:4516", |
| "snli:validation:4648", |
| "snli:validation:4888", |
| "snli:validation:5185", |
| "snli:validation:5332", |
| "snli:validation:5947", |
| "snli:validation:6124", |
| "snli:validation:660", |
| "snli:validation:6619", |
| "snli:validation:6898", |
| "snli:validation:7387", |
| "snli:validation:7492", |
| "snli:validation:7544", |
| "snli:validation:7996", |
| "snli:validation:8074", |
| "snli:validation:8554", |
| "snli:validation:8971", |
| "snli:validation:9163", |
| "snli:validation:9277", |
| "snli:validation:9490", |
| "snli:validation:9595", |
| "snli:validation:9688" |
| ], |
| "heldout_group_count": 128, |
| "training_group_count": 384 |
| }, |
| { |
| "fold": 2, |
| "temperature_index": 45, |
| "temperature": 1.0839269140212033, |
| "training_macro_nll": 0.22881493804363362, |
| "heldout_ids": [ |
| "arc:validation:ARC-Challenge:120", |
| "arc:validation:ARC-Challenge:130", |
| "arc:validation:ARC-Challenge:194", |
| "arc:validation:ARC-Challenge:218", |
| "arc:validation:ARC-Challenge:235", |
| "arc:validation:ARC-Challenge:272", |
| "arc:validation:ARC-Challenge:277", |
| "arc:validation:ARC-Challenge:48", |
| "arc:validation:ARC-Challenge:88", |
| "arc:validation:ARC-Easy:115", |
| "arc:validation:ARC-Easy:12", |
| "arc:validation:ARC-Easy:131", |
| "arc:validation:ARC-Easy:145", |
| "arc:validation:ARC-Easy:179", |
| "arc:validation:ARC-Easy:18", |
| "arc:validation:ARC-Easy:186", |
| "arc:validation:ARC-Easy:191", |
| "arc:validation:ARC-Easy:215", |
| "arc:validation:ARC-Easy:283", |
| "arc:validation:ARC-Easy:314", |
| "arc:validation:ARC-Easy:315", |
| "arc:validation:ARC-Easy:378", |
| "arc:validation:ARC-Easy:384", |
| "arc:validation:ARC-Easy:403", |
| "arc:validation:ARC-Easy:432", |
| "arc:validation:ARC-Easy:439", |
| "arc:validation:ARC-Easy:448", |
| "arc:validation:ARC-Easy:461", |
| "arc:validation:ARC-Easy:469", |
| "arc:validation:ARC-Easy:470", |
| "arc:validation:ARC-Easy:504", |
| "arc:validation:ARC-Easy:561", |
| "banking:train:1142", |
| "banking:train:148", |
| "banking:train:1633", |
| "banking:train:1927", |
| "banking:train:2186", |
| "banking:train:2251", |
| "banking:train:23", |
| "banking:train:3048", |
| "banking:train:3665", |
| "banking:train:3952", |
| "banking:train:3999", |
| "banking:train:4364", |
| "banking:train:4619", |
| "banking:train:5049", |
| "banking:train:5335", |
| "banking:train:549", |
| "banking:train:5957", |
| "banking:train:5985", |
| "banking:train:6132", |
| "banking:train:6240", |
| "banking:train:6488", |
| "banking:train:7065", |
| "banking:train:7749", |
| "banking:train:8470", |
| "banking:train:8761", |
| "banking:train:895", |
| "banking:train:9256", |
| "banking:train:9406", |
| "banking:train:9450", |
| "banking:train:9467", |
| "banking:train:9588", |
| "banking:train:9716", |
| "boolq:validation:1130", |
| "boolq:validation:1167", |
| "boolq:validation:124", |
| "boolq:validation:1255", |
| "boolq:validation:1256", |
| "boolq:validation:1373", |
| "boolq:validation:15", |
| "boolq:validation:155", |
| "boolq:validation:1568", |
| "boolq:validation:16", |
| "boolq:validation:1654", |
| "boolq:validation:1702", |
| "boolq:validation:1792", |
| "boolq:validation:1818", |
| "boolq:validation:210", |
| "boolq:validation:2247", |
| "boolq:validation:2259", |
| "boolq:validation:2362", |
| "boolq:validation:2443", |
| "boolq:validation:2524", |
| "boolq:validation:2553", |
| "boolq:validation:2623", |
| "boolq:validation:2660", |
| "boolq:validation:2669", |
| "boolq:validation:2679", |
| "boolq:validation:2832", |
| "boolq:validation:357", |
| "boolq:validation:453", |
| "boolq:validation:49", |
| "boolq:validation:808", |
| "boolq:validation:90", |
| "boolq:validation:910", |
| "snli:validation:1612", |
| "snli:validation:168", |
| "snli:validation:1693", |
| "snli:validation:2134", |
| "snli:validation:2248", |
| "snli:validation:2422", |
| "snli:validation:2587", |
| "snli:validation:3007", |
| "snli:validation:3097", |
| "snli:validation:3772", |
| "snli:validation:4012", |
| "snli:validation:4168", |
| "snli:validation:4537", |
| "snli:validation:4681", |
| "snli:validation:5050", |
| "snli:validation:5632", |
| "snli:validation:5761", |
| "snli:validation:6055", |
| "snli:validation:6283", |
| "snli:validation:6715", |
| "snli:validation:7117", |
| "snli:validation:7192", |
| "snli:validation:7507", |
| "snli:validation:7594", |
| "snli:validation:7924", |
| "snli:validation:7954", |
| "snli:validation:8035", |
| "snli:validation:8110", |
| "snli:validation:8995", |
| "snli:validation:949", |
| "snli:validation:9523", |
| "snli:validation:9589" |
| ], |
| "heldout_group_count": 128, |
| "training_group_count": 384 |
| }, |
| { |
| "fold": 3, |
| "temperature_index": 47, |
| "temperature": 1.2050359403717974, |
| "training_macro_nll": 0.24681503020373022, |
| "heldout_ids": [ |
| "arc:validation:ARC-Challenge:117", |
| "arc:validation:ARC-Challenge:135", |
| "arc:validation:ARC-Challenge:139", |
| "arc:validation:ARC-Challenge:154", |
| "arc:validation:ARC-Challenge:174", |
| "arc:validation:ARC-Challenge:199", |
| "arc:validation:ARC-Challenge:287", |
| "arc:validation:ARC-Challenge:295", |
| "arc:validation:ARC-Challenge:4", |
| "arc:validation:ARC-Challenge:50", |
| "arc:validation:ARC-Challenge:51", |
| "arc:validation:ARC-Challenge:61", |
| "arc:validation:ARC-Easy:107", |
| "arc:validation:ARC-Easy:121", |
| "arc:validation:ARC-Easy:124", |
| "arc:validation:ARC-Easy:171", |
| "arc:validation:ARC-Easy:235", |
| "arc:validation:ARC-Easy:241", |
| "arc:validation:ARC-Easy:279", |
| "arc:validation:ARC-Easy:289", |
| "arc:validation:ARC-Easy:37", |
| "arc:validation:ARC-Easy:389", |
| "arc:validation:ARC-Easy:394", |
| "arc:validation:ARC-Easy:399", |
| "arc:validation:ARC-Easy:423", |
| "arc:validation:ARC-Easy:492", |
| "arc:validation:ARC-Easy:499", |
| "arc:validation:ARC-Easy:523", |
| "arc:validation:ARC-Easy:533", |
| "arc:validation:ARC-Easy:569", |
| "arc:validation:ARC-Easy:69", |
| "arc:validation:ARC-Easy:93", |
| "banking:train:2049", |
| "banking:train:2350", |
| "banking:train:2497", |
| "banking:train:2506", |
| "banking:train:2720", |
| "banking:train:2788", |
| "banking:train:2967", |
| "banking:train:3502", |
| "banking:train:3726", |
| "banking:train:4014", |
| "banking:train:4735", |
| "banking:train:4803", |
| "banking:train:4917", |
| "banking:train:5477", |
| "banking:train:5851", |
| "banking:train:6551", |
| "banking:train:6839", |
| "banking:train:6926", |
| "banking:train:7809", |
| "banking:train:8092", |
| "banking:train:8186", |
| "banking:train:8215", |
| "banking:train:8667", |
| "banking:train:8752", |
| "banking:train:8763", |
| "banking:train:8851", |
| "banking:train:8917", |
| "banking:train:9145", |
| "banking:train:9279", |
| "banking:train:9676", |
| "banking:train:9837", |
| "banking:train:9967", |
| "boolq:validation:106", |
| "boolq:validation:1065", |
| "boolq:validation:1097", |
| "boolq:validation:1336", |
| "boolq:validation:1350", |
| "boolq:validation:1383", |
| "boolq:validation:1458", |
| "boolq:validation:1641", |
| "boolq:validation:1873", |
| "boolq:validation:1930", |
| "boolq:validation:1989", |
| "boolq:validation:2103", |
| "boolq:validation:234", |
| "boolq:validation:2355", |
| "boolq:validation:2384", |
| "boolq:validation:2440", |
| "boolq:validation:2444", |
| "boolq:validation:2478", |
| "boolq:validation:2872", |
| "boolq:validation:2879", |
| "boolq:validation:2924", |
| "boolq:validation:2971", |
| "boolq:validation:3164", |
| "boolq:validation:4", |
| "boolq:validation:415", |
| "boolq:validation:419", |
| "boolq:validation:428", |
| "boolq:validation:512", |
| "boolq:validation:563", |
| "boolq:validation:643", |
| "boolq:validation:696", |
| "boolq:validation:874", |
| "snli:validation:1102", |
| "snli:validation:1240", |
| "snli:validation:1261", |
| "snli:validation:1738", |
| "snli:validation:1978", |
| "snli:validation:2563", |
| "snli:validation:2659", |
| "snli:validation:2767", |
| "snli:validation:2836", |
| "snli:validation:3232", |
| "snli:validation:3655", |
| "snli:validation:3913", |
| "snli:validation:405", |
| "snli:validation:4051", |
| "snli:validation:4300", |
| "snli:validation:4846", |
| "snli:validation:498", |
| "snli:validation:5224", |
| "snli:validation:5575", |
| "snli:validation:6598", |
| "snli:validation:666", |
| "snli:validation:6925", |
| "snli:validation:6964", |
| "snli:validation:7045", |
| "snli:validation:7351", |
| "snli:validation:7588", |
| "snli:validation:7723", |
| "snli:validation:7750", |
| "snli:validation:7909", |
| "snli:validation:8974", |
| "snli:validation:9385", |
| "snli:validation:9646" |
| ], |
| "heldout_group_count": 128, |
| "training_group_count": 384 |
| } |
| ], |
| "per_family": { |
| "arc": { |
| "count": 128, |
| "score": 0.3379051798328397, |
| "raw_macro_nll": 0.33955257651665527, |
| "accuracy": 0.890625 |
| }, |
| "banking": { |
| "count": 128, |
| "score": 0.06793271948328157, |
| "raw_macro_nll": 0.06537141992655102, |
| "accuracy": 0.9609375 |
| }, |
| "boolq": { |
| "count": 128, |
| "score": 0.33084980282335186, |
| "raw_macro_nll": 0.34265219288051024, |
| "accuracy": 0.84375 |
| }, |
| "snli": { |
| "count": 128, |
| "score": 0.28448979293721316, |
| "raw_macro_nll": 0.2795028088961736, |
| "accuracy": 0.8984375 |
| } |
| }, |
| "policy": { |
| "id": "crossfit_temperature_nll_v1", |
| "folds": 4, |
| "seed": 431, |
| "grouping": "source group, nested within task family; groups shuffled within sorted families and assigned round-robin", |
| "temperature_grid": { |
| "count": 101, |
| "log10_min": -1.0, |
| "log10_max": 1.3, |
| "arithmetic": "Python float64" |
| }, |
| "temperature_fit": "minimum macro-family NLL on the other three folds; smallest temperature wins ties", |
| "score": "macro-family mean of all held-out decision NLLs", |
| "deployment_temperature": "fit afresh on reserved calibration only after model selection" |
| } |
| }, |
| "best_selection_score": 0.2552943737691716, |
| "improved": true, |
| "metrics": { |
| "overall": { |
| "n": 512, |
| "accuracy": 0.8984375, |
| "nll": 0.25676974726429014, |
| "brier_multiclass_sum": 0.14451170508324393, |
| "ece_top_label_10_equal_width_bins": 0.030249507224652916, |
| "reliability_bins": [ |
| { |
| "count": 0, |
| "confidence_sum": 0.0, |
| "correct_sum": 0.0 |
| }, |
| { |
| "count": 0, |
| "confidence_sum": 0.0, |
| "correct_sum": 0.0 |
| }, |
| { |
| "count": 0, |
| "confidence_sum": 0.0, |
| "correct_sum": 0.0 |
| }, |
| { |
| "count": 3, |
| "confidence_sum": 1.0243161022663116, |
| "correct_sum": 0.0 |
| }, |
| { |
| "count": 5, |
| "confidence_sum": 2.3768064975738525, |
| "correct_sum": 4.0 |
| }, |
| { |
| "count": 20, |
| "confidence_sum": 10.845634400844574, |
| "correct_sum": 10.0 |
| }, |
| { |
| "count": 23, |
| "confidence_sum": 15.07276839017868, |
| "correct_sum": 13.0 |
| }, |
| { |
| "count": 17, |
| "confidence_sum": 12.791166067123413, |
| "correct_sum": 12.0 |
| }, |
| { |
| "count": 40, |
| "confidence_sum": 34.14968603849411, |
| "correct_sum": 30.0 |
| }, |
| { |
| "count": 404, |
| "confidence_sum": 395.98098319768906, |
| "correct_sum": 391.0 |
| } |
| ], |
| "accuracy_vs_coverage": { |
| "0.25": { |
| "n": 128, |
| "accuracy": 1.0, |
| "min_confidence": 0.9993056058883667 |
| }, |
| "0.5": { |
| "n": 256, |
| "accuracy": 1.0, |
| "min_confidence": 0.9854145050048828 |
| }, |
| "0.75": { |
| "n": 384, |
| "accuracy": 0.9739583333333334, |
| "min_confidence": 0.9169918894767761 |
| }, |
| "1.0": { |
| "n": 512, |
| "accuracy": 0.8984375, |
| "min_confidence": 0.3054785430431366 |
| } |
| } |
| }, |
| "per_family": { |
| "arc": { |
| "n": 128, |
| "accuracy": 0.890625, |
| "nll": 0.33955256911365517, |
| "brier_multiclass_sum": 0.16622784887383055, |
| "ece_top_label_10_equal_width_bins": 0.059738497948274016, |
| "reliability_bins": [ |
| { |
| "count": 0, |
| "confidence_sum": 0.0, |
| "correct_sum": 0.0 |
| }, |
| { |
| "count": 0, |
| "confidence_sum": 0.0, |
| "correct_sum": 0.0 |
| }, |
| { |
| "count": 0, |
| "confidence_sum": 0.0, |
| "correct_sum": 0.0 |
| }, |
| { |
| "count": 3, |
| "confidence_sum": 1.0243161022663116, |
| "correct_sum": 0.0 |
| }, |
| { |
| "count": 2, |
| "confidence_sum": 0.9794403314590454, |
| "correct_sum": 2.0 |
| }, |
| { |
| "count": 6, |
| "confidence_sum": 3.2076597809791565, |
| "correct_sum": 4.0 |
| }, |
| { |
| "count": 6, |
| "confidence_sum": 3.9651999473571777, |
| "correct_sum": 5.0 |
| }, |
| { |
| "count": 5, |
| "confidence_sum": 3.7508853673934937, |
| "correct_sum": 3.0 |
| }, |
| { |
| "count": 8, |
| "confidence_sum": 6.78725129365921, |
| "correct_sum": 5.0 |
| }, |
| { |
| "count": 98, |
| "confidence_sum": 96.23637503385544, |
| "correct_sum": 95.0 |
| } |
| ], |
| "accuracy_vs_coverage": { |
| "0.25": { |
| "n": 32, |
| "accuracy": 1.0, |
| "min_confidence": 0.9996716976165771 |
| }, |
| "0.5": { |
| "n": 64, |
| "accuracy": 1.0, |
| "min_confidence": 0.9919483661651611 |
| }, |
| "0.75": { |
| "n": 96, |
| "accuracy": 0.96875, |
| "min_confidence": 0.9052539467811584 |
| }, |
| "1.0": { |
| "n": 128, |
| "accuracy": 0.890625, |
| "min_confidence": 0.3054785430431366 |
| } |
| } |
| }, |
| "banking": { |
| "n": 128, |
| "accuracy": 0.9609375, |
| "nll": 0.06537142041187444, |
| "brier_multiclass_sum": 0.04192916462498076, |
| "ece_top_label_10_equal_width_bins": 0.02170603978447616, |
| "reliability_bins": [ |
| { |
| "count": 0, |
| "confidence_sum": 0.0, |
| "correct_sum": 0.0 |
| }, |
| { |
| "count": 0, |
| "confidence_sum": 0.0, |
| "correct_sum": 0.0 |
| }, |
| { |
| "count": 0, |
| "confidence_sum": 0.0, |
| "correct_sum": 0.0 |
| }, |
| { |
| "count": 0, |
| "confidence_sum": 0.0, |
| "correct_sum": 0.0 |
| }, |
| { |
| "count": 1, |
| "confidence_sum": 0.48401013016700745, |
| "correct_sum": 0.0 |
| }, |
| { |
| "count": 2, |
| "confidence_sum": 1.0103490948677063, |
| "correct_sum": 1.0 |
| }, |
| { |
| "count": 1, |
| "confidence_sum": 0.6197635531425476, |
| "correct_sum": 0.0 |
| }, |
| { |
| "count": 4, |
| "confidence_sum": 3.016280710697174, |
| "correct_sum": 2.0 |
| }, |
| { |
| "count": 1, |
| "confidence_sum": 0.8994449973106384, |
| "correct_sum": 1.0 |
| }, |
| { |
| "count": 119, |
| "confidence_sum": 118.45258539915085, |
| "correct_sum": 119.0 |
| } |
| ], |
| "accuracy_vs_coverage": { |
| "0.25": { |
| "n": 32, |
| "accuracy": 1.0, |
| "min_confidence": 0.9999996423721313 |
| }, |
| "0.5": { |
| "n": 64, |
| "accuracy": 1.0, |
| "min_confidence": 0.9999728202819824 |
| }, |
| "0.75": { |
| "n": 96, |
| "accuracy": 1.0, |
| "min_confidence": 0.9991067051887512 |
| }, |
| "1.0": { |
| "n": 128, |
| "accuracy": 0.9609375, |
| "min_confidence": 0.48401013016700745 |
| } |
| } |
| }, |
| "boolq": { |
| "n": 128, |
| "accuracy": 0.84375, |
| "nll": 0.342652197339703, |
| "brier_multiclass_sum": 0.21037350762899634, |
| "ece_top_label_10_equal_width_bins": 0.06018189946189523, |
| "reliability_bins": [ |
| { |
| "count": 0, |
| "confidence_sum": 0.0, |
| "correct_sum": 0.0 |
| }, |
| { |
| "count": 0, |
| "confidence_sum": 0.0, |
| "correct_sum": 0.0 |
| }, |
| { |
| "count": 0, |
| "confidence_sum": 0.0, |
| "correct_sum": 0.0 |
| }, |
| { |
| "count": 0, |
| "confidence_sum": 0.0, |
| "correct_sum": 0.0 |
| }, |
| { |
| "count": 0, |
| "confidence_sum": 0.0, |
| "correct_sum": 0.0 |
| }, |
| { |
| "count": 8, |
| "confidence_sum": 4.464165925979614, |
| "correct_sum": 3.0 |
| }, |
| { |
| "count": 13, |
| "confidence_sum": 8.543370068073273, |
| "correct_sum": 7.0 |
| }, |
| { |
| "count": 3, |
| "confidence_sum": 2.237887918949127, |
| "correct_sum": 3.0 |
| }, |
| { |
| "count": 18, |
| "confidence_sum": 15.334045886993408, |
| "correct_sum": 14.0 |
| }, |
| { |
| "count": 86, |
| "confidence_sum": 83.59958916902542, |
| "correct_sum": 81.0 |
| } |
| ], |
| "accuracy_vs_coverage": { |
| "0.25": { |
| "n": 32, |
| "accuracy": 1.0, |
| "min_confidence": 0.9892238974571228 |
| }, |
| "0.5": { |
| "n": 64, |
| "accuracy": 0.953125, |
| "min_confidence": 0.9593857526779175 |
| }, |
| "0.75": { |
| "n": 96, |
| "accuracy": 0.9166666666666666, |
| "min_confidence": 0.8476907014846802 |
| }, |
| "1.0": { |
| "n": 128, |
| "accuracy": 0.84375, |
| "min_confidence": 0.5223926901817322 |
| } |
| } |
| }, |
| "snli": { |
| "n": 128, |
| "accuracy": 0.8984375, |
| "nll": 0.2795028021919279, |
| "brier_multiclass_sum": 0.1595162992051681, |
| "ece_top_label_10_equal_width_bins": 0.04085784195922315, |
| "reliability_bins": [ |
| { |
| "count": 0, |
| "confidence_sum": 0.0, |
| "correct_sum": 0.0 |
| }, |
| { |
| "count": 0, |
| "confidence_sum": 0.0, |
| "correct_sum": 0.0 |
| }, |
| { |
| "count": 0, |
| "confidence_sum": 0.0, |
| "correct_sum": 0.0 |
| }, |
| { |
| "count": 0, |
| "confidence_sum": 0.0, |
| "correct_sum": 0.0 |
| }, |
| { |
| "count": 2, |
| "confidence_sum": 0.9133560359477997, |
| "correct_sum": 2.0 |
| }, |
| { |
| "count": 4, |
| "confidence_sum": 2.163459599018097, |
| "correct_sum": 2.0 |
| }, |
| { |
| "count": 3, |
| "confidence_sum": 1.9444348216056824, |
| "correct_sum": 1.0 |
| }, |
| { |
| "count": 5, |
| "confidence_sum": 3.786112070083618, |
| "correct_sum": 4.0 |
| }, |
| { |
| "count": 13, |
| "confidence_sum": 11.128943860530853, |
| "correct_sum": 10.0 |
| }, |
| { |
| "count": 101, |
| "confidence_sum": 97.69243359565735, |
| "correct_sum": 96.0 |
| } |
| ], |
| "accuracy_vs_coverage": { |
| "0.25": { |
| "n": 32, |
| "accuracy": 1.0, |
| "min_confidence": 0.9896841049194336 |
| }, |
| "0.5": { |
| "n": 64, |
| "accuracy": 0.984375, |
| "min_confidence": 0.9620543718338013 |
| }, |
| "0.75": { |
| "n": 96, |
| "accuracy": 0.9479166666666666, |
| "min_confidence": 0.9089819192886353 |
| }, |
| "1.0": { |
| "n": 128, |
| "accuracy": 0.8984375, |
| "min_confidence": 0.43042799830436707 |
| } |
| } |
| } |
| } |
| } |
| }, |
| { |
| "step": 2500, |
| "macro_nll": 0.34164803258559817, |
| "best_macro_nll": 0.25676974726429014, |
| "selection": { |
| "metric": "crossfit_temperature_nll_v1", |
| "score": 0.2873950432295314, |
| "raw_macro_nll": 0.3416480337510342, |
| "accuracy": 0.908203125, |
| "folds": [ |
| { |
| "fold": 0, |
| "temperature_index": 54, |
| "temperature": 1.7458221529205038, |
| "training_macro_nll": 0.2923693492865519, |
| "heldout_ids": [ |
| "arc:validation:ARC-Challenge:122", |
| "arc:validation:ARC-Challenge:126", |
| "arc:validation:ARC-Challenge:175", |
| "arc:validation:ARC-Challenge:187", |
| "arc:validation:ARC-Challenge:189", |
| "arc:validation:ARC-Challenge:192", |
| "arc:validation:ARC-Challenge:208", |
| "arc:validation:ARC-Challenge:238", |
| "arc:validation:ARC-Challenge:290", |
| "arc:validation:ARC-Challenge:293", |
| "arc:validation:ARC-Challenge:35", |
| "arc:validation:ARC-Challenge:43", |
| "arc:validation:ARC-Challenge:58", |
| "arc:validation:ARC-Challenge:7", |
| "arc:validation:ARC-Easy:160", |
| "arc:validation:ARC-Easy:201", |
| "arc:validation:ARC-Easy:229", |
| "arc:validation:ARC-Easy:238", |
| "arc:validation:ARC-Easy:253", |
| "arc:validation:ARC-Easy:259", |
| "arc:validation:ARC-Easy:272", |
| "arc:validation:ARC-Easy:298", |
| "arc:validation:ARC-Easy:329", |
| "arc:validation:ARC-Easy:331", |
| "arc:validation:ARC-Easy:337", |
| "arc:validation:ARC-Easy:362", |
| "arc:validation:ARC-Easy:443", |
| "arc:validation:ARC-Easy:500", |
| "arc:validation:ARC-Easy:545", |
| "arc:validation:ARC-Easy:68", |
| "arc:validation:ARC-Easy:7", |
| "arc:validation:ARC-Easy:74", |
| "banking:train:1053", |
| "banking:train:1594", |
| "banking:train:1747", |
| "banking:train:202", |
| "banking:train:2377", |
| "banking:train:2528", |
| "banking:train:2546", |
| "banking:train:2715", |
| "banking:train:3220", |
| "banking:train:3499", |
| "banking:train:3806", |
| "banking:train:3838", |
| "banking:train:3993", |
| "banking:train:4220", |
| "banking:train:5", |
| "banking:train:5192", |
| "banking:train:5394", |
| "banking:train:5846", |
| "banking:train:5863", |
| "banking:train:6032", |
| "banking:train:6508", |
| "banking:train:7579", |
| "banking:train:7600", |
| "banking:train:7987", |
| "banking:train:8155", |
| "banking:train:850", |
| "banking:train:8626", |
| "banking:train:8929", |
| "banking:train:9355", |
| "banking:train:9812", |
| "banking:train:987", |
| "banking:train:9972", |
| "boolq:validation:103", |
| "boolq:validation:1042", |
| "boolq:validation:1059", |
| "boolq:validation:1541", |
| "boolq:validation:1592", |
| "boolq:validation:1927", |
| "boolq:validation:2014", |
| "boolq:validation:2023", |
| "boolq:validation:2098", |
| "boolq:validation:2139", |
| "boolq:validation:2439", |
| "boolq:validation:2485", |
| "boolq:validation:2607", |
| "boolq:validation:2611", |
| "boolq:validation:2720", |
| "boolq:validation:2751", |
| "boolq:validation:282", |
| "boolq:validation:2843", |
| "boolq:validation:2953", |
| "boolq:validation:301", |
| "boolq:validation:3089", |
| "boolq:validation:313", |
| "boolq:validation:3201", |
| "boolq:validation:322", |
| "boolq:validation:3231", |
| "boolq:validation:39", |
| "boolq:validation:461", |
| "boolq:validation:558", |
| "boolq:validation:589", |
| "boolq:validation:731", |
| "boolq:validation:883", |
| "boolq:validation:915", |
| "snli:validation:1156", |
| "snli:validation:1330", |
| "snli:validation:1559", |
| "snli:validation:1735", |
| "snli:validation:1876", |
| "snli:validation:2251", |
| "snli:validation:2257", |
| "snli:validation:2377", |
| "snli:validation:2881", |
| "snli:validation:291", |
| "snli:validation:300", |
| "snli:validation:3049", |
| "snli:validation:3496", |
| "snli:validation:3871", |
| "snli:validation:4414", |
| "snli:validation:5140", |
| "snli:validation:5440", |
| "snli:validation:5806", |
| "snli:validation:6442", |
| "snli:validation:6772", |
| "snli:validation:6949", |
| "snli:validation:7228", |
| "snli:validation:7465", |
| "snli:validation:7510", |
| "snli:validation:7726", |
| "snli:validation:8095", |
| "snli:validation:8143", |
| "snli:validation:8623", |
| "snli:validation:8962", |
| "snli:validation:9451", |
| "snli:validation:9574", |
| "snli:validation:9655" |
| ], |
| "heldout_group_count": 128, |
| "training_group_count": 384 |
| }, |
| { |
| "fold": 1, |
| "temperature_index": 54, |
| "temperature": 1.7458221529205038, |
| "training_macro_nll": 0.29119915448817607, |
| "heldout_ids": [ |
| "arc:validation:ARC-Challenge:101", |
| "arc:validation:ARC-Challenge:129", |
| "arc:validation:ARC-Challenge:148", |
| "arc:validation:ARC-Challenge:156", |
| "arc:validation:ARC-Challenge:167", |
| "arc:validation:ARC-Challenge:249", |
| "arc:validation:ARC-Challenge:263", |
| "arc:validation:ARC-Challenge:266", |
| "arc:validation:ARC-Challenge:267", |
| "arc:validation:ARC-Challenge:28", |
| "arc:validation:ARC-Challenge:281", |
| "arc:validation:ARC-Challenge:57", |
| "arc:validation:ARC-Challenge:63", |
| "arc:validation:ARC-Challenge:64", |
| "arc:validation:ARC-Challenge:86", |
| "arc:validation:ARC-Challenge:99", |
| "arc:validation:ARC-Easy:106", |
| "arc:validation:ARC-Easy:151", |
| "arc:validation:ARC-Easy:157", |
| "arc:validation:ARC-Easy:166", |
| "arc:validation:ARC-Easy:202", |
| "arc:validation:ARC-Easy:219", |
| "arc:validation:ARC-Easy:224", |
| "arc:validation:ARC-Easy:322", |
| "arc:validation:ARC-Easy:344", |
| "arc:validation:ARC-Easy:382", |
| "arc:validation:ARC-Easy:409", |
| "arc:validation:ARC-Easy:455", |
| "arc:validation:ARC-Easy:50", |
| "arc:validation:ARC-Easy:538", |
| "arc:validation:ARC-Easy:544", |
| "arc:validation:ARC-Easy:552", |
| "banking:train:1050", |
| "banking:train:1131", |
| "banking:train:1190", |
| "banking:train:124", |
| "banking:train:137", |
| "banking:train:1493", |
| "banking:train:1498", |
| "banking:train:1624", |
| "banking:train:2614", |
| "banking:train:2739", |
| "banking:train:3055", |
| "banking:train:306", |
| "banking:train:3312", |
| "banking:train:3724", |
| "banking:train:4070", |
| "banking:train:476", |
| "banking:train:4882", |
| "banking:train:5075", |
| "banking:train:5332", |
| "banking:train:5445", |
| "banking:train:6096", |
| "banking:train:746", |
| "banking:train:7828", |
| "banking:train:8176", |
| "banking:train:8315", |
| "banking:train:8653", |
| "banking:train:9019", |
| "banking:train:9370", |
| "banking:train:9454", |
| "banking:train:9595", |
| "banking:train:9862", |
| "banking:train:9963", |
| "boolq:validation:1189", |
| "boolq:validation:1320", |
| "boolq:validation:1326", |
| "boolq:validation:1381", |
| "boolq:validation:1410", |
| "boolq:validation:149", |
| "boolq:validation:1513", |
| "boolq:validation:1732", |
| "boolq:validation:1739", |
| "boolq:validation:1741", |
| "boolq:validation:1814", |
| "boolq:validation:1848", |
| "boolq:validation:1940", |
| "boolq:validation:2077", |
| "boolq:validation:2091", |
| "boolq:validation:2148", |
| "boolq:validation:2215", |
| "boolq:validation:2251", |
| "boolq:validation:2283", |
| "boolq:validation:2321", |
| "boolq:validation:2398", |
| "boolq:validation:253", |
| "boolq:validation:2575", |
| "boolq:validation:2907", |
| "boolq:validation:2909", |
| "boolq:validation:2978", |
| "boolq:validation:359", |
| "boolq:validation:394", |
| "boolq:validation:436", |
| "boolq:validation:579", |
| "boolq:validation:807", |
| "boolq:validation:949", |
| "snli:validation:2389", |
| "snli:validation:2485", |
| "snli:validation:255", |
| "snli:validation:3169", |
| "snli:validation:333", |
| "snli:validation:3700", |
| "snli:validation:3709", |
| "snli:validation:4093", |
| "snli:validation:420", |
| "snli:validation:4492", |
| "snli:validation:4516", |
| "snli:validation:4648", |
| "snli:validation:4888", |
| "snli:validation:5185", |
| "snli:validation:5332", |
| "snli:validation:5947", |
| "snli:validation:6124", |
| "snli:validation:660", |
| "snli:validation:6619", |
| "snli:validation:6898", |
| "snli:validation:7387", |
| "snli:validation:7492", |
| "snli:validation:7544", |
| "snli:validation:7996", |
| "snli:validation:8074", |
| "snli:validation:8554", |
| "snli:validation:8971", |
| "snli:validation:9163", |
| "snli:validation:9277", |
| "snli:validation:9490", |
| "snli:validation:9595", |
| "snli:validation:9688" |
| ], |
| "heldout_group_count": 128, |
| "training_group_count": 384 |
| }, |
| { |
| "fold": 2, |
| "temperature_index": 54, |
| "temperature": 1.7458221529205038, |
| "training_macro_nll": 0.27996013831109845, |
| "heldout_ids": [ |
| "arc:validation:ARC-Challenge:120", |
| "arc:validation:ARC-Challenge:130", |
| "arc:validation:ARC-Challenge:194", |
| "arc:validation:ARC-Challenge:218", |
| "arc:validation:ARC-Challenge:235", |
| "arc:validation:ARC-Challenge:272", |
| "arc:validation:ARC-Challenge:277", |
| "arc:validation:ARC-Challenge:48", |
| "arc:validation:ARC-Challenge:88", |
| "arc:validation:ARC-Easy:115", |
| "arc:validation:ARC-Easy:12", |
| "arc:validation:ARC-Easy:131", |
| "arc:validation:ARC-Easy:145", |
| "arc:validation:ARC-Easy:179", |
| "arc:validation:ARC-Easy:18", |
| "arc:validation:ARC-Easy:186", |
| "arc:validation:ARC-Easy:191", |
| "arc:validation:ARC-Easy:215", |
| "arc:validation:ARC-Easy:283", |
| "arc:validation:ARC-Easy:314", |
| "arc:validation:ARC-Easy:315", |
| "arc:validation:ARC-Easy:378", |
| "arc:validation:ARC-Easy:384", |
| "arc:validation:ARC-Easy:403", |
| "arc:validation:ARC-Easy:432", |
| "arc:validation:ARC-Easy:439", |
| "arc:validation:ARC-Easy:448", |
| "arc:validation:ARC-Easy:461", |
| "arc:validation:ARC-Easy:469", |
| "arc:validation:ARC-Easy:470", |
| "arc:validation:ARC-Easy:504", |
| "arc:validation:ARC-Easy:561", |
| "banking:train:1142", |
| "banking:train:148", |
| "banking:train:1633", |
| "banking:train:1927", |
| "banking:train:2186", |
| "banking:train:2251", |
| "banking:train:23", |
| "banking:train:3048", |
| "banking:train:3665", |
| "banking:train:3952", |
| "banking:train:3999", |
| "banking:train:4364", |
| "banking:train:4619", |
| "banking:train:5049", |
| "banking:train:5335", |
| "banking:train:549", |
| "banking:train:5957", |
| "banking:train:5985", |
| "banking:train:6132", |
| "banking:train:6240", |
| "banking:train:6488", |
| "banking:train:7065", |
| "banking:train:7749", |
| "banking:train:8470", |
| "banking:train:8761", |
| "banking:train:895", |
| "banking:train:9256", |
| "banking:train:9406", |
| "banking:train:9450", |
| "banking:train:9467", |
| "banking:train:9588", |
| "banking:train:9716", |
| "boolq:validation:1130", |
| "boolq:validation:1167", |
| "boolq:validation:124", |
| "boolq:validation:1255", |
| "boolq:validation:1256", |
| "boolq:validation:1373", |
| "boolq:validation:15", |
| "boolq:validation:155", |
| "boolq:validation:1568", |
| "boolq:validation:16", |
| "boolq:validation:1654", |
| "boolq:validation:1702", |
| "boolq:validation:1792", |
| "boolq:validation:1818", |
| "boolq:validation:210", |
| "boolq:validation:2247", |
| "boolq:validation:2259", |
| "boolq:validation:2362", |
| "boolq:validation:2443", |
| "boolq:validation:2524", |
| "boolq:validation:2553", |
| "boolq:validation:2623", |
| "boolq:validation:2660", |
| "boolq:validation:2669", |
| "boolq:validation:2679", |
| "boolq:validation:2832", |
| "boolq:validation:357", |
| "boolq:validation:453", |
| "boolq:validation:49", |
| "boolq:validation:808", |
| "boolq:validation:90", |
| "boolq:validation:910", |
| "snli:validation:1612", |
| "snli:validation:168", |
| "snli:validation:1693", |
| "snli:validation:2134", |
| "snli:validation:2248", |
| "snli:validation:2422", |
| "snli:validation:2587", |
| "snli:validation:3007", |
| "snli:validation:3097", |
| "snli:validation:3772", |
| "snli:validation:4012", |
| "snli:validation:4168", |
| "snli:validation:4537", |
| "snli:validation:4681", |
| "snli:validation:5050", |
| "snli:validation:5632", |
| "snli:validation:5761", |
| "snli:validation:6055", |
| "snli:validation:6283", |
| "snli:validation:6715", |
| "snli:validation:7117", |
| "snli:validation:7192", |
| "snli:validation:7507", |
| "snli:validation:7594", |
| "snli:validation:7924", |
| "snli:validation:7954", |
| "snli:validation:8035", |
| "snli:validation:8110", |
| "snli:validation:8995", |
| "snli:validation:949", |
| "snli:validation:9523", |
| "snli:validation:9589" |
| ], |
| "heldout_group_count": 128, |
| "training_group_count": 384 |
| }, |
| { |
| "fold": 3, |
| "temperature_index": 53, |
| "temperature": 1.6557699634695275, |
| "training_macro_nll": 0.2844243480235682, |
| "heldout_ids": [ |
| "arc:validation:ARC-Challenge:117", |
| "arc:validation:ARC-Challenge:135", |
| "arc:validation:ARC-Challenge:139", |
| "arc:validation:ARC-Challenge:154", |
| "arc:validation:ARC-Challenge:174", |
| "arc:validation:ARC-Challenge:199", |
| "arc:validation:ARC-Challenge:287", |
| "arc:validation:ARC-Challenge:295", |
| "arc:validation:ARC-Challenge:4", |
| "arc:validation:ARC-Challenge:50", |
| "arc:validation:ARC-Challenge:51", |
| "arc:validation:ARC-Challenge:61", |
| "arc:validation:ARC-Easy:107", |
| "arc:validation:ARC-Easy:121", |
| "arc:validation:ARC-Easy:124", |
| "arc:validation:ARC-Easy:171", |
| "arc:validation:ARC-Easy:235", |
| "arc:validation:ARC-Easy:241", |
| "arc:validation:ARC-Easy:279", |
| "arc:validation:ARC-Easy:289", |
| "arc:validation:ARC-Easy:37", |
| "arc:validation:ARC-Easy:389", |
| "arc:validation:ARC-Easy:394", |
| "arc:validation:ARC-Easy:399", |
| "arc:validation:ARC-Easy:423", |
| "arc:validation:ARC-Easy:492", |
| "arc:validation:ARC-Easy:499", |
| "arc:validation:ARC-Easy:523", |
| "arc:validation:ARC-Easy:533", |
| "arc:validation:ARC-Easy:569", |
| "arc:validation:ARC-Easy:69", |
| "arc:validation:ARC-Easy:93", |
| "banking:train:2049", |
| "banking:train:2350", |
| "banking:train:2497", |
| "banking:train:2506", |
| "banking:train:2720", |
| "banking:train:2788", |
| "banking:train:2967", |
| "banking:train:3502", |
| "banking:train:3726", |
| "banking:train:4014", |
| "banking:train:4735", |
| "banking:train:4803", |
| "banking:train:4917", |
| "banking:train:5477", |
| "banking:train:5851", |
| "banking:train:6551", |
| "banking:train:6839", |
| "banking:train:6926", |
| "banking:train:7809", |
| "banking:train:8092", |
| "banking:train:8186", |
| "banking:train:8215", |
| "banking:train:8667", |
| "banking:train:8752", |
| "banking:train:8763", |
| "banking:train:8851", |
| "banking:train:8917", |
| "banking:train:9145", |
| "banking:train:9279", |
| "banking:train:9676", |
| "banking:train:9837", |
| "banking:train:9967", |
| "boolq:validation:106", |
| "boolq:validation:1065", |
| "boolq:validation:1097", |
| "boolq:validation:1336", |
| "boolq:validation:1350", |
| "boolq:validation:1383", |
| "boolq:validation:1458", |
| "boolq:validation:1641", |
| "boolq:validation:1873", |
| "boolq:validation:1930", |
| "boolq:validation:1989", |
| "boolq:validation:2103", |
| "boolq:validation:234", |
| "boolq:validation:2355", |
| "boolq:validation:2384", |
| "boolq:validation:2440", |
| "boolq:validation:2444", |
| "boolq:validation:2478", |
| "boolq:validation:2872", |
| "boolq:validation:2879", |
| "boolq:validation:2924", |
| "boolq:validation:2971", |
| "boolq:validation:3164", |
| "boolq:validation:4", |
| "boolq:validation:415", |
| "boolq:validation:419", |
| "boolq:validation:428", |
| "boolq:validation:512", |
| "boolq:validation:563", |
| "boolq:validation:643", |
| "boolq:validation:696", |
| "boolq:validation:874", |
| "snli:validation:1102", |
| "snli:validation:1240", |
| "snli:validation:1261", |
| "snli:validation:1738", |
| "snli:validation:1978", |
| "snli:validation:2563", |
| "snli:validation:2659", |
| "snli:validation:2767", |
| "snli:validation:2836", |
| "snli:validation:3232", |
| "snli:validation:3655", |
| "snli:validation:3913", |
| "snli:validation:405", |
| "snli:validation:4051", |
| "snli:validation:4300", |
| "snli:validation:4846", |
| "snli:validation:498", |
| "snli:validation:5224", |
| "snli:validation:5575", |
| "snli:validation:6598", |
| "snli:validation:666", |
| "snli:validation:6925", |
| "snli:validation:6964", |
| "snli:validation:7045", |
| "snli:validation:7351", |
| "snli:validation:7588", |
| "snli:validation:7723", |
| "snli:validation:7750", |
| "snli:validation:7909", |
| "snli:validation:8974", |
| "snli:validation:9385", |
| "snli:validation:9646" |
| ], |
| "heldout_group_count": 128, |
| "training_group_count": 384 |
| } |
| ], |
| "per_family": { |
| "arc": { |
| "count": 128, |
| "score": 0.37032308473279407, |
| "raw_macro_nll": 0.4008364961392736, |
| "accuracy": 0.890625 |
| }, |
| "banking": { |
| "count": 128, |
| "score": 0.08469598977961221, |
| "raw_macro_nll": 0.08458895137039663, |
| "accuracy": 0.9765625 |
| }, |
| "boolq": { |
| "count": 128, |
| "score": 0.4290453198465801, |
| "raw_macro_nll": 0.6039625253530503, |
| "accuracy": 0.8359375 |
| }, |
| "snli": { |
| "count": 128, |
| "score": 0.2655157785591392, |
| "raw_macro_nll": 0.27720416214141624, |
| "accuracy": 0.9296875 |
| } |
| }, |
| "policy": { |
| "id": "crossfit_temperature_nll_v1", |
| "folds": 4, |
| "seed": 431, |
| "grouping": "source group, nested within task family; groups shuffled within sorted families and assigned round-robin", |
| "temperature_grid": { |
| "count": 101, |
| "log10_min": -1.0, |
| "log10_max": 1.3, |
| "arithmetic": "Python float64" |
| }, |
| "temperature_fit": "minimum macro-family NLL on the other three folds; smallest temperature wins ties", |
| "score": "macro-family mean of all held-out decision NLLs", |
| "deployment_temperature": "fit afresh on reserved calibration only after model selection" |
| } |
| }, |
| "best_selection_score": 0.2552943737691716, |
| "improved": false, |
| "metrics": { |
| "overall": { |
| "n": 512, |
| "accuracy": 0.908203125, |
| "nll": 0.34164803258559817, |
| "brier_multiclass_sum": 0.15958235995790504, |
| "ece_top_label_10_equal_width_bins": 0.06509242358151823, |
| "reliability_bins": [ |
| { |
| "count": 0, |
| "confidence_sum": 0.0, |
| "correct_sum": 0.0 |
| }, |
| { |
| "count": 0, |
| "confidence_sum": 0.0, |
| "correct_sum": 0.0 |
| }, |
| { |
| "count": 0, |
| "confidence_sum": 0.0, |
| "correct_sum": 0.0 |
| }, |
| { |
| "count": 4, |
| "confidence_sum": 1.4218776524066925, |
| "correct_sum": 1.0 |
| }, |
| { |
| "count": 7, |
| "confidence_sum": 3.1050805151462555, |
| "correct_sum": 5.0 |
| }, |
| { |
| "count": 8, |
| "confidence_sum": 4.550839066505432, |
| "correct_sum": 8.0 |
| }, |
| { |
| "count": 13, |
| "confidence_sum": 8.378304839134216, |
| "correct_sum": 7.0 |
| }, |
| { |
| "count": 15, |
| "confidence_sum": 11.172361552715302, |
| "correct_sum": 12.0 |
| }, |
| { |
| "count": 18, |
| "confidence_sum": 15.334031105041504, |
| "correct_sum": 11.0 |
| }, |
| { |
| "count": 447, |
| "confidence_sum": 442.0213884115219, |
| "correct_sum": 421.0 |
| } |
| ], |
| "accuracy_vs_coverage": { |
| "0.25": { |
| "n": 128, |
| "accuracy": 1.0, |
| "min_confidence": 0.9997068047523499 |
| }, |
| "0.5": { |
| "n": 256, |
| "accuracy": 0.984375, |
| "min_confidence": 0.9974879026412964 |
| }, |
| "0.75": { |
| "n": 384, |
| "accuracy": 0.96875, |
| "min_confidence": 0.9728125333786011 |
| }, |
| "1.0": { |
| "n": 512, |
| "accuracy": 0.908203125, |
| "min_confidence": 0.32846468687057495 |
| } |
| } |
| }, |
| "per_family": { |
| "arc": { |
| "n": 128, |
| "accuracy": 0.890625, |
| "nll": 0.4008364924734096, |
| "brier_multiclass_sum": 0.1836061792924404, |
| "ece_top_label_10_equal_width_bins": 0.07639543549157679, |
| "reliability_bins": [ |
| { |
| "count": 0, |
| "confidence_sum": 0.0, |
| "correct_sum": 0.0 |
| }, |
| { |
| "count": 0, |
| "confidence_sum": 0.0, |
| "correct_sum": 0.0 |
| }, |
| { |
| "count": 0, |
| "confidence_sum": 0.0, |
| "correct_sum": 0.0 |
| }, |
| { |
| "count": 3, |
| "confidence_sum": 1.0558361113071442, |
| "correct_sum": 1.0 |
| }, |
| { |
| "count": 6, |
| "confidence_sum": 2.6552467942237854, |
| "correct_sum": 4.0 |
| }, |
| { |
| "count": 3, |
| "confidence_sum": 1.7100982666015625, |
| "correct_sum": 3.0 |
| }, |
| { |
| "count": 4, |
| "confidence_sum": 2.608660101890564, |
| "correct_sum": 2.0 |
| }, |
| { |
| "count": 3, |
| "confidence_sum": 2.2052063941955566, |
| "correct_sum": 3.0 |
| }, |
| { |
| "count": 6, |
| "confidence_sum": 5.102949798107147, |
| "correct_sum": 4.0 |
| }, |
| { |
| "count": 103, |
| "confidence_sum": 101.58172118663788, |
| "correct_sum": 97.0 |
| } |
| ], |
| "accuracy_vs_coverage": { |
| "0.25": { |
| "n": 32, |
| "accuracy": 1.0, |
| "min_confidence": 0.9998447895050049 |
| }, |
| "0.5": { |
| "n": 64, |
| "accuracy": 1.0, |
| "min_confidence": 0.9945265650749207 |
| }, |
| "0.75": { |
| "n": 96, |
| "accuracy": 0.96875, |
| "min_confidence": 0.9348756074905396 |
| }, |
| "1.0": { |
| "n": 128, |
| "accuracy": 0.890625, |
| "min_confidence": 0.32846468687057495 |
| } |
| } |
| }, |
| "banking": { |
| "n": 128, |
| "accuracy": 0.9765625, |
| "nll": 0.08458894538692885, |
| "brier_multiclass_sum": 0.03869071261468688, |
| "ece_top_label_10_equal_width_bins": 0.01209510350599885, |
| "reliability_bins": [ |
| { |
| "count": 0, |
| "confidence_sum": 0.0, |
| "correct_sum": 0.0 |
| }, |
| { |
| "count": 0, |
| "confidence_sum": 0.0, |
| "correct_sum": 0.0 |
| }, |
| { |
| "count": 0, |
| "confidence_sum": 0.0, |
| "correct_sum": 0.0 |
| }, |
| { |
| "count": 0, |
| "confidence_sum": 0.0, |
| "correct_sum": 0.0 |
| }, |
| { |
| "count": 0, |
| "confidence_sum": 0.0, |
| "correct_sum": 0.0 |
| }, |
| { |
| "count": 0, |
| "confidence_sum": 0.0, |
| "correct_sum": 0.0 |
| }, |
| { |
| "count": 5, |
| "confidence_sum": 3.2286646962165833, |
| "correct_sum": 3.0 |
| }, |
| { |
| "count": 3, |
| "confidence_sum": 2.218995213508606, |
| "correct_sum": 3.0 |
| }, |
| { |
| "count": 0, |
| "confidence_sum": 0.0, |
| "correct_sum": 0.0 |
| }, |
| { |
| "count": 120, |
| "confidence_sum": 119.53850376605988, |
| "correct_sum": 119.0 |
| } |
| ], |
| "accuracy_vs_coverage": { |
| "0.25": { |
| "n": 32, |
| "accuracy": 1.0, |
| "min_confidence": 0.9999996423721313 |
| }, |
| "0.5": { |
| "n": 64, |
| "accuracy": 1.0, |
| "min_confidence": 0.9999823570251465 |
| }, |
| "0.75": { |
| "n": 96, |
| "accuracy": 1.0, |
| "min_confidence": 0.9979609251022339 |
| }, |
| "1.0": { |
| "n": 128, |
| "accuracy": 0.9765625, |
| "min_confidence": 0.6143197417259216 |
| } |
| } |
| }, |
| "boolq": { |
| "n": 128, |
| "accuracy": 0.8359375, |
| "nll": 0.6039625278771723, |
| "brier_multiclass_sum": 0.2873108505217812, |
| "ece_top_label_10_equal_width_bins": 0.1287803715094924, |
| "reliability_bins": [ |
| { |
| "count": 0, |
| "confidence_sum": 0.0, |
| "correct_sum": 0.0 |
| }, |
| { |
| "count": 0, |
| "confidence_sum": 0.0, |
| "correct_sum": 0.0 |
| }, |
| { |
| "count": 0, |
| "confidence_sum": 0.0, |
| "correct_sum": 0.0 |
| }, |
| { |
| "count": 0, |
| "confidence_sum": 0.0, |
| "correct_sum": 0.0 |
| }, |
| { |
| "count": 0, |
| "confidence_sum": 0.0, |
| "correct_sum": 0.0 |
| }, |
| { |
| "count": 2, |
| "confidence_sum": 1.103356122970581, |
| "correct_sum": 2.0 |
| }, |
| { |
| "count": 4, |
| "confidence_sum": 2.540980041027069, |
| "correct_sum": 2.0 |
| }, |
| { |
| "count": 7, |
| "confidence_sum": 5.245690405368805, |
| "correct_sum": 5.0 |
| }, |
| { |
| "count": 6, |
| "confidence_sum": 5.077808380126953, |
| "correct_sum": 2.0 |
| }, |
| { |
| "count": 109, |
| "confidence_sum": 107.72276484966278, |
| "correct_sum": 96.0 |
| } |
| ], |
| "accuracy_vs_coverage": { |
| "0.25": { |
| "n": 32, |
| "accuracy": 1.0, |
| "min_confidence": 0.9992809891700745 |
| }, |
| "0.5": { |
| "n": 64, |
| "accuracy": 0.9375, |
| "min_confidence": 0.9971458315849304 |
| }, |
| "0.75": { |
| "n": 96, |
| "accuracy": 0.90625, |
| "min_confidence": 0.9584174156188965 |
| }, |
| "1.0": { |
| "n": 128, |
| "accuracy": 0.8359375, |
| "min_confidence": 0.5258947014808655 |
| } |
| } |
| }, |
| "snli": { |
| "n": 128, |
| "accuracy": 0.9296875, |
| "nll": 0.27720416460488195, |
| "brier_multiclass_sum": 0.12872169740271175, |
| "ece_top_label_10_equal_width_bins": 0.05478878295980394, |
| "reliability_bins": [ |
| { |
| "count": 0, |
| "confidence_sum": 0.0, |
| "correct_sum": 0.0 |
| }, |
| { |
| "count": 0, |
| "confidence_sum": 0.0, |
| "correct_sum": 0.0 |
| }, |
| { |
| "count": 0, |
| "confidence_sum": 0.0, |
| "correct_sum": 0.0 |
| }, |
| { |
| "count": 1, |
| "confidence_sum": 0.36604154109954834, |
| "correct_sum": 0.0 |
| }, |
| { |
| "count": 1, |
| "confidence_sum": 0.4498337209224701, |
| "correct_sum": 1.0 |
| }, |
| { |
| "count": 3, |
| "confidence_sum": 1.7373846769332886, |
| "correct_sum": 3.0 |
| }, |
| { |
| "count": 0, |
| "confidence_sum": 0.0, |
| "correct_sum": 0.0 |
| }, |
| { |
| "count": 2, |
| "confidence_sum": 1.502469539642334, |
| "correct_sum": 1.0 |
| }, |
| { |
| "count": 6, |
| "confidence_sum": 5.153272926807404, |
| "correct_sum": 5.0 |
| }, |
| { |
| "count": 115, |
| "confidence_sum": 113.17839860916138, |
| "correct_sum": 109.0 |
| } |
| ], |
| "accuracy_vs_coverage": { |
| "0.25": { |
| "n": 32, |
| "accuracy": 1.0, |
| "min_confidence": 0.9986096620559692 |
| }, |
| "0.5": { |
| "n": 64, |
| "accuracy": 0.96875, |
| "min_confidence": 0.9895986318588257 |
| }, |
| "0.75": { |
| "n": 96, |
| "accuracy": 0.96875, |
| "min_confidence": 0.9699889421463013 |
| }, |
| "1.0": { |
| "n": 128, |
| "accuracy": 0.9296875, |
| "min_confidence": 0.36604154109954834 |
| } |
| } |
| } |
| } |
| } |
| }, |
| { |
| "step": 3000, |
| "macro_nll": 0.343850982809214, |
| "best_macro_nll": 0.25676974726429014, |
| "selection": { |
| "metric": "crossfit_temperature_nll_v1", |
| "score": 0.26419258441127697, |
| "raw_macro_nll": 0.3438509819060138, |
| "accuracy": 0.91015625, |
| "folds": [ |
| { |
| "fold": 0, |
| "temperature_index": 57, |
| "temperature": 2.046444636724674, |
| "training_macro_nll": 0.27731212139717715, |
| "heldout_ids": [ |
| "arc:validation:ARC-Challenge:122", |
| "arc:validation:ARC-Challenge:126", |
| "arc:validation:ARC-Challenge:175", |
| "arc:validation:ARC-Challenge:187", |
| "arc:validation:ARC-Challenge:189", |
| "arc:validation:ARC-Challenge:192", |
| "arc:validation:ARC-Challenge:208", |
| "arc:validation:ARC-Challenge:238", |
| "arc:validation:ARC-Challenge:290", |
| "arc:validation:ARC-Challenge:293", |
| "arc:validation:ARC-Challenge:35", |
| "arc:validation:ARC-Challenge:43", |
| "arc:validation:ARC-Challenge:58", |
| "arc:validation:ARC-Challenge:7", |
| "arc:validation:ARC-Easy:160", |
| "arc:validation:ARC-Easy:201", |
| "arc:validation:ARC-Easy:229", |
| "arc:validation:ARC-Easy:238", |
| "arc:validation:ARC-Easy:253", |
| "arc:validation:ARC-Easy:259", |
| "arc:validation:ARC-Easy:272", |
| "arc:validation:ARC-Easy:298", |
| "arc:validation:ARC-Easy:329", |
| "arc:validation:ARC-Easy:331", |
| "arc:validation:ARC-Easy:337", |
| "arc:validation:ARC-Easy:362", |
| "arc:validation:ARC-Easy:443", |
| "arc:validation:ARC-Easy:500", |
| "arc:validation:ARC-Easy:545", |
| "arc:validation:ARC-Easy:68", |
| "arc:validation:ARC-Easy:7", |
| "arc:validation:ARC-Easy:74", |
| "banking:train:1053", |
| "banking:train:1594", |
| "banking:train:1747", |
| "banking:train:202", |
| "banking:train:2377", |
| "banking:train:2528", |
| "banking:train:2546", |
| "banking:train:2715", |
| "banking:train:3220", |
| "banking:train:3499", |
| "banking:train:3806", |
| "banking:train:3838", |
| "banking:train:3993", |
| "banking:train:4220", |
| "banking:train:5", |
| "banking:train:5192", |
| "banking:train:5394", |
| "banking:train:5846", |
| "banking:train:5863", |
| "banking:train:6032", |
| "banking:train:6508", |
| "banking:train:7579", |
| "banking:train:7600", |
| "banking:train:7987", |
| "banking:train:8155", |
| "banking:train:850", |
| "banking:train:8626", |
| "banking:train:8929", |
| "banking:train:9355", |
| "banking:train:9812", |
| "banking:train:987", |
| "banking:train:9972", |
| "boolq:validation:103", |
| "boolq:validation:1042", |
| "boolq:validation:1059", |
| "boolq:validation:1541", |
| "boolq:validation:1592", |
| "boolq:validation:1927", |
| "boolq:validation:2014", |
| "boolq:validation:2023", |
| "boolq:validation:2098", |
| "boolq:validation:2139", |
| "boolq:validation:2439", |
| "boolq:validation:2485", |
| "boolq:validation:2607", |
| "boolq:validation:2611", |
| "boolq:validation:2720", |
| "boolq:validation:2751", |
| "boolq:validation:282", |
| "boolq:validation:2843", |
| "boolq:validation:2953", |
| "boolq:validation:301", |
| "boolq:validation:3089", |
| "boolq:validation:313", |
| "boolq:validation:3201", |
| "boolq:validation:322", |
| "boolq:validation:3231", |
| "boolq:validation:39", |
| "boolq:validation:461", |
| "boolq:validation:558", |
| "boolq:validation:589", |
| "boolq:validation:731", |
| "boolq:validation:883", |
| "boolq:validation:915", |
| "snli:validation:1156", |
| "snli:validation:1330", |
| "snli:validation:1559", |
| "snli:validation:1735", |
| "snli:validation:1876", |
| "snli:validation:2251", |
| "snli:validation:2257", |
| "snli:validation:2377", |
| "snli:validation:2881", |
| "snli:validation:291", |
| "snli:validation:300", |
| "snli:validation:3049", |
| "snli:validation:3496", |
| "snli:validation:3871", |
| "snli:validation:4414", |
| "snli:validation:5140", |
| "snli:validation:5440", |
| "snli:validation:5806", |
| "snli:validation:6442", |
| "snli:validation:6772", |
| "snli:validation:6949", |
| "snli:validation:7228", |
| "snli:validation:7465", |
| "snli:validation:7510", |
| "snli:validation:7726", |
| "snli:validation:8095", |
| "snli:validation:8143", |
| "snli:validation:8623", |
| "snli:validation:8962", |
| "snli:validation:9451", |
| "snli:validation:9574", |
| "snli:validation:9655" |
| ], |
| "heldout_group_count": 128, |
| "training_group_count": 384 |
| }, |
| { |
| "fold": 1, |
| "temperature_index": 56, |
| "temperature": 1.9408858775927773, |
| "training_macro_nll": 0.25991780083545074, |
| "heldout_ids": [ |
| "arc:validation:ARC-Challenge:101", |
| "arc:validation:ARC-Challenge:129", |
| "arc:validation:ARC-Challenge:148", |
| "arc:validation:ARC-Challenge:156", |
| "arc:validation:ARC-Challenge:167", |
| "arc:validation:ARC-Challenge:249", |
| "arc:validation:ARC-Challenge:263", |
| "arc:validation:ARC-Challenge:266", |
| "arc:validation:ARC-Challenge:267", |
| "arc:validation:ARC-Challenge:28", |
| "arc:validation:ARC-Challenge:281", |
| "arc:validation:ARC-Challenge:57", |
| "arc:validation:ARC-Challenge:63", |
| "arc:validation:ARC-Challenge:64", |
| "arc:validation:ARC-Challenge:86", |
| "arc:validation:ARC-Challenge:99", |
| "arc:validation:ARC-Easy:106", |
| "arc:validation:ARC-Easy:151", |
| "arc:validation:ARC-Easy:157", |
| "arc:validation:ARC-Easy:166", |
| "arc:validation:ARC-Easy:202", |
| "arc:validation:ARC-Easy:219", |
| "arc:validation:ARC-Easy:224", |
| "arc:validation:ARC-Easy:322", |
| "arc:validation:ARC-Easy:344", |
| "arc:validation:ARC-Easy:382", |
| "arc:validation:ARC-Easy:409", |
| "arc:validation:ARC-Easy:455", |
| "arc:validation:ARC-Easy:50", |
| "arc:validation:ARC-Easy:538", |
| "arc:validation:ARC-Easy:544", |
| "arc:validation:ARC-Easy:552", |
| "banking:train:1050", |
| "banking:train:1131", |
| "banking:train:1190", |
| "banking:train:124", |
| "banking:train:137", |
| "banking:train:1493", |
| "banking:train:1498", |
| "banking:train:1624", |
| "banking:train:2614", |
| "banking:train:2739", |
| "banking:train:3055", |
| "banking:train:306", |
| "banking:train:3312", |
| "banking:train:3724", |
| "banking:train:4070", |
| "banking:train:476", |
| "banking:train:4882", |
| "banking:train:5075", |
| "banking:train:5332", |
| "banking:train:5445", |
| "banking:train:6096", |
| "banking:train:746", |
| "banking:train:7828", |
| "banking:train:8176", |
| "banking:train:8315", |
| "banking:train:8653", |
| "banking:train:9019", |
| "banking:train:9370", |
| "banking:train:9454", |
| "banking:train:9595", |
| "banking:train:9862", |
| "banking:train:9963", |
| "boolq:validation:1189", |
| "boolq:validation:1320", |
| "boolq:validation:1326", |
| "boolq:validation:1381", |
| "boolq:validation:1410", |
| "boolq:validation:149", |
| "boolq:validation:1513", |
| "boolq:validation:1732", |
| "boolq:validation:1739", |
| "boolq:validation:1741", |
| "boolq:validation:1814", |
| "boolq:validation:1848", |
| "boolq:validation:1940", |
| "boolq:validation:2077", |
| "boolq:validation:2091", |
| "boolq:validation:2148", |
| "boolq:validation:2215", |
| "boolq:validation:2251", |
| "boolq:validation:2283", |
| "boolq:validation:2321", |
| "boolq:validation:2398", |
| "boolq:validation:253", |
| "boolq:validation:2575", |
| "boolq:validation:2907", |
| "boolq:validation:2909", |
| "boolq:validation:2978", |
| "boolq:validation:359", |
| "boolq:validation:394", |
| "boolq:validation:436", |
| "boolq:validation:579", |
| "boolq:validation:807", |
| "boolq:validation:949", |
| "snli:validation:2389", |
| "snli:validation:2485", |
| "snli:validation:255", |
| "snli:validation:3169", |
| "snli:validation:333", |
| "snli:validation:3700", |
| "snli:validation:3709", |
| "snli:validation:4093", |
| "snli:validation:420", |
| "snli:validation:4492", |
| "snli:validation:4516", |
| "snli:validation:4648", |
| "snli:validation:4888", |
| "snli:validation:5185", |
| "snli:validation:5332", |
| "snli:validation:5947", |
| "snli:validation:6124", |
| "snli:validation:660", |
| "snli:validation:6619", |
| "snli:validation:6898", |
| "snli:validation:7387", |
| "snli:validation:7492", |
| "snli:validation:7544", |
| "snli:validation:7996", |
| "snli:validation:8074", |
| "snli:validation:8554", |
| "snli:validation:8971", |
| "snli:validation:9163", |
| "snli:validation:9277", |
| "snli:validation:9490", |
| "snli:validation:9595", |
| "snli:validation:9688" |
| ], |
| "heldout_group_count": 128, |
| "training_group_count": 384 |
| }, |
| { |
| "fold": 2, |
| "temperature_index": 57, |
| "temperature": 2.046444636724674, |
| "training_macro_nll": 0.2569765609800613, |
| "heldout_ids": [ |
| "arc:validation:ARC-Challenge:120", |
| "arc:validation:ARC-Challenge:130", |
| "arc:validation:ARC-Challenge:194", |
| "arc:validation:ARC-Challenge:218", |
| "arc:validation:ARC-Challenge:235", |
| "arc:validation:ARC-Challenge:272", |
| "arc:validation:ARC-Challenge:277", |
| "arc:validation:ARC-Challenge:48", |
| "arc:validation:ARC-Challenge:88", |
| "arc:validation:ARC-Easy:115", |
| "arc:validation:ARC-Easy:12", |
| "arc:validation:ARC-Easy:131", |
| "arc:validation:ARC-Easy:145", |
| "arc:validation:ARC-Easy:179", |
| "arc:validation:ARC-Easy:18", |
| "arc:validation:ARC-Easy:186", |
| "arc:validation:ARC-Easy:191", |
| "arc:validation:ARC-Easy:215", |
| "arc:validation:ARC-Easy:283", |
| "arc:validation:ARC-Easy:314", |
| "arc:validation:ARC-Easy:315", |
| "arc:validation:ARC-Easy:378", |
| "arc:validation:ARC-Easy:384", |
| "arc:validation:ARC-Easy:403", |
| "arc:validation:ARC-Easy:432", |
| "arc:validation:ARC-Easy:439", |
| "arc:validation:ARC-Easy:448", |
| "arc:validation:ARC-Easy:461", |
| "arc:validation:ARC-Easy:469", |
| "arc:validation:ARC-Easy:470", |
| "arc:validation:ARC-Easy:504", |
| "arc:validation:ARC-Easy:561", |
| "banking:train:1142", |
| "banking:train:148", |
| "banking:train:1633", |
| "banking:train:1927", |
| "banking:train:2186", |
| "banking:train:2251", |
| "banking:train:23", |
| "banking:train:3048", |
| "banking:train:3665", |
| "banking:train:3952", |
| "banking:train:3999", |
| "banking:train:4364", |
| "banking:train:4619", |
| "banking:train:5049", |
| "banking:train:5335", |
| "banking:train:549", |
| "banking:train:5957", |
| "banking:train:5985", |
| "banking:train:6132", |
| "banking:train:6240", |
| "banking:train:6488", |
| "banking:train:7065", |
| "banking:train:7749", |
| "banking:train:8470", |
| "banking:train:8761", |
| "banking:train:895", |
| "banking:train:9256", |
| "banking:train:9406", |
| "banking:train:9450", |
| "banking:train:9467", |
| "banking:train:9588", |
| "banking:train:9716", |
| "boolq:validation:1130", |
| "boolq:validation:1167", |
| "boolq:validation:124", |
| "boolq:validation:1255", |
| "boolq:validation:1256", |
| "boolq:validation:1373", |
| "boolq:validation:15", |
| "boolq:validation:155", |
| "boolq:validation:1568", |
| "boolq:validation:16", |
| "boolq:validation:1654", |
| "boolq:validation:1702", |
| "boolq:validation:1792", |
| "boolq:validation:1818", |
| "boolq:validation:210", |
| "boolq:validation:2247", |
| "boolq:validation:2259", |
| "boolq:validation:2362", |
| "boolq:validation:2443", |
| "boolq:validation:2524", |
| "boolq:validation:2553", |
| "boolq:validation:2623", |
| "boolq:validation:2660", |
| "boolq:validation:2669", |
| "boolq:validation:2679", |
| "boolq:validation:2832", |
| "boolq:validation:357", |
| "boolq:validation:453", |
| "boolq:validation:49", |
| "boolq:validation:808", |
| "boolq:validation:90", |
| "boolq:validation:910", |
| "snli:validation:1612", |
| "snli:validation:168", |
| "snli:validation:1693", |
| "snli:validation:2134", |
| "snli:validation:2248", |
| "snli:validation:2422", |
| "snli:validation:2587", |
| "snli:validation:3007", |
| "snli:validation:3097", |
| "snli:validation:3772", |
| "snli:validation:4012", |
| "snli:validation:4168", |
| "snli:validation:4537", |
| "snli:validation:4681", |
| "snli:validation:5050", |
| "snli:validation:5632", |
| "snli:validation:5761", |
| "snli:validation:6055", |
| "snli:validation:6283", |
| "snli:validation:6715", |
| "snli:validation:7117", |
| "snli:validation:7192", |
| "snli:validation:7507", |
| "snli:validation:7594", |
| "snli:validation:7924", |
| "snli:validation:7954", |
| "snli:validation:8035", |
| "snli:validation:8110", |
| "snli:validation:8995", |
| "snli:validation:949", |
| "snli:validation:9523", |
| "snli:validation:9589" |
| ], |
| "heldout_group_count": 128, |
| "training_group_count": 384 |
| }, |
| { |
| "fold": 3, |
| "temperature_index": 56, |
| "temperature": 1.9408858775927773, |
| "training_macro_nll": 0.2592448225500369, |
| "heldout_ids": [ |
| "arc:validation:ARC-Challenge:117", |
| "arc:validation:ARC-Challenge:135", |
| "arc:validation:ARC-Challenge:139", |
| "arc:validation:ARC-Challenge:154", |
| "arc:validation:ARC-Challenge:174", |
| "arc:validation:ARC-Challenge:199", |
| "arc:validation:ARC-Challenge:287", |
| "arc:validation:ARC-Challenge:295", |
| "arc:validation:ARC-Challenge:4", |
| "arc:validation:ARC-Challenge:50", |
| "arc:validation:ARC-Challenge:51", |
| "arc:validation:ARC-Challenge:61", |
| "arc:validation:ARC-Easy:107", |
| "arc:validation:ARC-Easy:121", |
| "arc:validation:ARC-Easy:124", |
| "arc:validation:ARC-Easy:171", |
| "arc:validation:ARC-Easy:235", |
| "arc:validation:ARC-Easy:241", |
| "arc:validation:ARC-Easy:279", |
| "arc:validation:ARC-Easy:289", |
| "arc:validation:ARC-Easy:37", |
| "arc:validation:ARC-Easy:389", |
| "arc:validation:ARC-Easy:394", |
| "arc:validation:ARC-Easy:399", |
| "arc:validation:ARC-Easy:423", |
| "arc:validation:ARC-Easy:492", |
| "arc:validation:ARC-Easy:499", |
| "arc:validation:ARC-Easy:523", |
| "arc:validation:ARC-Easy:533", |
| "arc:validation:ARC-Easy:569", |
| "arc:validation:ARC-Easy:69", |
| "arc:validation:ARC-Easy:93", |
| "banking:train:2049", |
| "banking:train:2350", |
| "banking:train:2497", |
| "banking:train:2506", |
| "banking:train:2720", |
| "banking:train:2788", |
| "banking:train:2967", |
| "banking:train:3502", |
| "banking:train:3726", |
| "banking:train:4014", |
| "banking:train:4735", |
| "banking:train:4803", |
| "banking:train:4917", |
| "banking:train:5477", |
| "banking:train:5851", |
| "banking:train:6551", |
| "banking:train:6839", |
| "banking:train:6926", |
| "banking:train:7809", |
| "banking:train:8092", |
| "banking:train:8186", |
| "banking:train:8215", |
| "banking:train:8667", |
| "banking:train:8752", |
| "banking:train:8763", |
| "banking:train:8851", |
| "banking:train:8917", |
| "banking:train:9145", |
| "banking:train:9279", |
| "banking:train:9676", |
| "banking:train:9837", |
| "banking:train:9967", |
| "boolq:validation:106", |
| "boolq:validation:1065", |
| "boolq:validation:1097", |
| "boolq:validation:1336", |
| "boolq:validation:1350", |
| "boolq:validation:1383", |
| "boolq:validation:1458", |
| "boolq:validation:1641", |
| "boolq:validation:1873", |
| "boolq:validation:1930", |
| "boolq:validation:1989", |
| "boolq:validation:2103", |
| "boolq:validation:234", |
| "boolq:validation:2355", |
| "boolq:validation:2384", |
| "boolq:validation:2440", |
| "boolq:validation:2444", |
| "boolq:validation:2478", |
| "boolq:validation:2872", |
| "boolq:validation:2879", |
| "boolq:validation:2924", |
| "boolq:validation:2971", |
| "boolq:validation:3164", |
| "boolq:validation:4", |
| "boolq:validation:415", |
| "boolq:validation:419", |
| "boolq:validation:428", |
| "boolq:validation:512", |
| "boolq:validation:563", |
| "boolq:validation:643", |
| "boolq:validation:696", |
| "boolq:validation:874", |
| "snli:validation:1102", |
| "snli:validation:1240", |
| "snli:validation:1261", |
| "snli:validation:1738", |
| "snli:validation:1978", |
| "snli:validation:2563", |
| "snli:validation:2659", |
| "snli:validation:2767", |
| "snli:validation:2836", |
| "snli:validation:3232", |
| "snli:validation:3655", |
| "snli:validation:3913", |
| "snli:validation:405", |
| "snli:validation:4051", |
| "snli:validation:4300", |
| "snli:validation:4846", |
| "snli:validation:498", |
| "snli:validation:5224", |
| "snli:validation:5575", |
| "snli:validation:6598", |
| "snli:validation:666", |
| "snli:validation:6925", |
| "snli:validation:6964", |
| "snli:validation:7045", |
| "snli:validation:7351", |
| "snli:validation:7588", |
| "snli:validation:7723", |
| "snli:validation:7750", |
| "snli:validation:7909", |
| "snli:validation:8974", |
| "snli:validation:9385", |
| "snli:validation:9646" |
| ], |
| "heldout_group_count": 128, |
| "training_group_count": 384 |
| } |
| ], |
| "per_family": { |
| "arc": { |
| "count": 128, |
| "score": 0.35546071765134163, |
| "raw_macro_nll": 0.5363843868768633, |
| "accuracy": 0.9140625 |
| }, |
| "banking": { |
| "count": 128, |
| "score": 0.022742246686383003, |
| "raw_macro_nll": 0.030751403774512704, |
| "accuracy": 0.9921875 |
| }, |
| "boolq": { |
| "count": 128, |
| "score": 0.33967224778120886, |
| "raw_macro_nll": 0.4241840175504803, |
| "accuracy": 0.8515625 |
| }, |
| "snli": { |
| "count": 128, |
| "score": 0.3388951255261744, |
| "raw_macro_nll": 0.38408411942219894, |
| "accuracy": 0.8828125 |
| } |
| }, |
| "policy": { |
| "id": "crossfit_temperature_nll_v1", |
| "folds": 4, |
| "seed": 431, |
| "grouping": "source group, nested within task family; groups shuffled within sorted families and assigned round-robin", |
| "temperature_grid": { |
| "count": 101, |
| "log10_min": -1.0, |
| "log10_max": 1.3, |
| "arithmetic": "Python float64" |
| }, |
| "temperature_fit": "minimum macro-family NLL on the other three folds; smallest temperature wins ties", |
| "score": "macro-family mean of all held-out decision NLLs", |
| "deployment_temperature": "fit afresh on reserved calibration only after model selection" |
| } |
| }, |
| "best_selection_score": 0.2552943737691716, |
| "improved": false, |
| "metrics": { |
| "overall": { |
| "n": 512, |
| "accuracy": 0.91015625, |
| "nll": 0.343850982809214, |
| "brier_multiclass_sum": 0.1506205599699681, |
| "ece_top_label_10_equal_width_bins": 0.05876007775077596, |
| "reliability_bins": [ |
| { |
| "count": 0, |
| "confidence_sum": 0.0, |
| "correct_sum": 0.0 |
| }, |
| { |
| "count": 0, |
| "confidence_sum": 0.0, |
| "correct_sum": 0.0 |
| }, |
| { |
| "count": 0, |
| "confidence_sum": 0.0, |
| "correct_sum": 0.0 |
| }, |
| { |
| "count": 0, |
| "confidence_sum": 0.0, |
| "correct_sum": 0.0 |
| }, |
| { |
| "count": 2, |
| "confidence_sum": 0.8661302030086517, |
| "correct_sum": 1.0 |
| }, |
| { |
| "count": 7, |
| "confidence_sum": 3.8665531277656555, |
| "correct_sum": 5.0 |
| }, |
| { |
| "count": 9, |
| "confidence_sum": 5.823009371757507, |
| "correct_sum": 4.0 |
| }, |
| { |
| "count": 12, |
| "confidence_sum": 8.999069094657898, |
| "correct_sum": 8.0 |
| }, |
| { |
| "count": 27, |
| "confidence_sum": 23.208462595939636, |
| "correct_sum": 17.0 |
| }, |
| { |
| "count": 455, |
| "confidence_sum": 450.78730207681656, |
| "correct_sum": 431.0 |
| } |
| ], |
| "accuracy_vs_coverage": { |
| "0.25": { |
| "n": 128, |
| "accuracy": 1.0, |
| "min_confidence": 0.9999998807907104 |
| }, |
| "0.5": { |
| "n": 256, |
| "accuracy": 0.99609375, |
| "min_confidence": 0.9987045526504517 |
| }, |
| "0.75": { |
| "n": 384, |
| "accuracy": 0.9713541666666666, |
| "min_confidence": 0.9814466238021851 |
| }, |
| "1.0": { |
| "n": 512, |
| "accuracy": 0.91015625, |
| "min_confidence": 0.42066824436187744 |
| } |
| } |
| }, |
| "per_family": { |
| "arc": { |
| "n": 128, |
| "accuracy": 0.9140625, |
| "nll": 0.5363843899576184, |
| "brier_multiclass_sum": 0.14942062283772675, |
| "ece_top_label_10_equal_width_bins": 0.060488176764920354, |
| "reliability_bins": [ |
| { |
| "count": 0, |
| "confidence_sum": 0.0, |
| "correct_sum": 0.0 |
| }, |
| { |
| "count": 0, |
| "confidence_sum": 0.0, |
| "correct_sum": 0.0 |
| }, |
| { |
| "count": 0, |
| "confidence_sum": 0.0, |
| "correct_sum": 0.0 |
| }, |
| { |
| "count": 0, |
| "confidence_sum": 0.0, |
| "correct_sum": 0.0 |
| }, |
| { |
| "count": 2, |
| "confidence_sum": 0.8661302030086517, |
| "correct_sum": 1.0 |
| }, |
| { |
| "count": 4, |
| "confidence_sum": 2.261072337627411, |
| "correct_sum": 2.0 |
| }, |
| { |
| "count": 1, |
| "confidence_sum": 0.6171885132789612, |
| "correct_sum": 1.0 |
| }, |
| { |
| "count": 4, |
| "confidence_sum": 3.027863085269928, |
| "correct_sum": 3.0 |
| }, |
| { |
| "count": 4, |
| "confidence_sum": 3.4061849117279053, |
| "correct_sum": 4.0 |
| }, |
| { |
| "count": 113, |
| "confidence_sum": 112.34305483102798, |
| "correct_sum": 106.0 |
| } |
| ], |
| "accuracy_vs_coverage": { |
| "0.25": { |
| "n": 32, |
| "accuracy": 1.0, |
| "min_confidence": 0.9999995231628418 |
| }, |
| "0.5": { |
| "n": 64, |
| "accuracy": 0.984375, |
| "min_confidence": 0.9999145269393921 |
| }, |
| "0.75": { |
| "n": 96, |
| "accuracy": 0.9479166666666666, |
| "min_confidence": 0.993623673915863 |
| }, |
| "1.0": { |
| "n": 128, |
| "accuracy": 0.9140625, |
| "min_confidence": 0.42066824436187744 |
| } |
| } |
| }, |
| "banking": { |
| "n": 128, |
| "accuracy": 0.9921875, |
| "nll": 0.030751401536174172, |
| "brier_multiclass_sum": 0.015997644546320234, |
| "ece_top_label_10_equal_width_bins": 0.009750935714691877, |
| "reliability_bins": [ |
| { |
| "count": 0, |
| "confidence_sum": 0.0, |
| "correct_sum": 0.0 |
| }, |
| { |
| "count": 0, |
| "confidence_sum": 0.0, |
| "correct_sum": 0.0 |
| }, |
| { |
| "count": 0, |
| "confidence_sum": 0.0, |
| "correct_sum": 0.0 |
| }, |
| { |
| "count": 0, |
| "confidence_sum": 0.0, |
| "correct_sum": 0.0 |
| }, |
| { |
| "count": 0, |
| "confidence_sum": 0.0, |
| "correct_sum": 0.0 |
| }, |
| { |
| "count": 0, |
| "confidence_sum": 0.0, |
| "correct_sum": 0.0 |
| }, |
| { |
| "count": 1, |
| "confidence_sum": 0.6965740919113159, |
| "correct_sum": 1.0 |
| }, |
| { |
| "count": 0, |
| "confidence_sum": 0.0, |
| "correct_sum": 0.0 |
| }, |
| { |
| "count": 0, |
| "confidence_sum": 0.0, |
| "correct_sum": 0.0 |
| }, |
| { |
| "count": 127, |
| "confidence_sum": 126.94469386339188, |
| "correct_sum": 126.0 |
| } |
| ], |
| "accuracy_vs_coverage": { |
| "0.25": { |
| "n": 32, |
| "accuracy": 1.0, |
| "min_confidence": 1.0 |
| }, |
| "0.5": { |
| "n": 64, |
| "accuracy": 1.0, |
| "min_confidence": 1.0 |
| }, |
| "0.75": { |
| "n": 96, |
| "accuracy": 1.0, |
| "min_confidence": 1.0 |
| }, |
| "1.0": { |
| "n": 128, |
| "accuracy": 0.9921875, |
| "min_confidence": 0.6965740919113159 |
| } |
| } |
| }, |
| "boolq": { |
| "n": 128, |
| "accuracy": 0.8515625, |
| "nll": 0.42418402199575667, |
| "brier_multiclass_sum": 0.24188416272422544, |
| "ece_top_label_10_equal_width_bins": 0.10255556367337704, |
| "reliability_bins": [ |
| { |
| "count": 0, |
| "confidence_sum": 0.0, |
| "correct_sum": 0.0 |
| }, |
| { |
| "count": 0, |
| "confidence_sum": 0.0, |
| "correct_sum": 0.0 |
| }, |
| { |
| "count": 0, |
| "confidence_sum": 0.0, |
| "correct_sum": 0.0 |
| }, |
| { |
| "count": 0, |
| "confidence_sum": 0.0, |
| "correct_sum": 0.0 |
| }, |
| { |
| "count": 0, |
| "confidence_sum": 0.0, |
| "correct_sum": 0.0 |
| }, |
| { |
| "count": 1, |
| "confidence_sum": 0.5557354688644409, |
| "correct_sum": 1.0 |
| }, |
| { |
| "count": 3, |
| "confidence_sum": 1.84639310836792, |
| "correct_sum": 1.0 |
| }, |
| { |
| "count": 6, |
| "confidence_sum": 4.499466836452484, |
| "correct_sum": 4.0 |
| }, |
| { |
| "count": 14, |
| "confidence_sum": 11.978712320327759, |
| "correct_sum": 8.0 |
| }, |
| { |
| "count": 104, |
| "confidence_sum": 102.35827535390854, |
| "correct_sum": 95.0 |
| } |
| ], |
| "accuracy_vs_coverage": { |
| "0.25": { |
| "n": 32, |
| "accuracy": 1.0, |
| "min_confidence": 0.9973174929618835 |
| }, |
| "0.5": { |
| "n": 64, |
| "accuracy": 0.96875, |
| "min_confidence": 0.9890179634094238 |
| }, |
| "0.75": { |
| "n": 96, |
| "accuracy": 0.9270833333333334, |
| "min_confidence": 0.9507815837860107 |
| }, |
| "1.0": { |
| "n": 128, |
| "accuracy": 0.8515625, |
| "min_confidence": 0.5557354688644409 |
| } |
| } |
| }, |
| "snli": { |
| "n": 128, |
| "accuracy": 0.8828125, |
| "nll": 0.38408411774730666, |
| "brier_multiclass_sum": 0.19517980977159996, |
| "ece_top_label_10_equal_width_bins": 0.08632571017369628, |
| "reliability_bins": [ |
| { |
| "count": 0, |
| "confidence_sum": 0.0, |
| "correct_sum": 0.0 |
| }, |
| { |
| "count": 0, |
| "confidence_sum": 0.0, |
| "correct_sum": 0.0 |
| }, |
| { |
| "count": 0, |
| "confidence_sum": 0.0, |
| "correct_sum": 0.0 |
| }, |
| { |
| "count": 0, |
| "confidence_sum": 0.0, |
| "correct_sum": 0.0 |
| }, |
| { |
| "count": 0, |
| "confidence_sum": 0.0, |
| "correct_sum": 0.0 |
| }, |
| { |
| "count": 2, |
| "confidence_sum": 1.0497453212738037, |
| "correct_sum": 2.0 |
| }, |
| { |
| "count": 4, |
| "confidence_sum": 2.6628536581993103, |
| "correct_sum": 1.0 |
| }, |
| { |
| "count": 2, |
| "confidence_sum": 1.4717391729354858, |
| "correct_sum": 1.0 |
| }, |
| { |
| "count": 9, |
| "confidence_sum": 7.823565363883972, |
| "correct_sum": 5.0 |
| }, |
| { |
| "count": 111, |
| "confidence_sum": 109.14127802848816, |
| "correct_sum": 104.0 |
| } |
| ], |
| "accuracy_vs_coverage": { |
| "0.25": { |
| "n": 32, |
| "accuracy": 1.0, |
| "min_confidence": 0.9989041090011597 |
| }, |
| "0.5": { |
| "n": 64, |
| "accuracy": 0.96875, |
| "min_confidence": 0.9890118837356567 |
| }, |
| "0.75": { |
| "n": 96, |
| "accuracy": 0.9375, |
| "min_confidence": 0.9636436700820923 |
| }, |
| "1.0": { |
| "n": 128, |
| "accuracy": 0.8828125, |
| "min_confidence": 0.5159494876861572 |
| } |
| } |
| } |
| } |
| } |
| }, |
| { |
| "step": 3500, |
| "macro_nll": 0.32187738830487217, |
| "best_macro_nll": 0.25676974726429014, |
| "selection": { |
| "metric": "crossfit_temperature_nll_v1", |
| "score": 0.2805558053510232, |
| "raw_macro_nll": 0.32187738794463494, |
| "accuracy": 0.9140625, |
| "folds": [ |
| { |
| "fold": 0, |
| "temperature_index": 54, |
| "temperature": 1.7458221529205038, |
| "training_macro_nll": 0.3014060109823255, |
| "heldout_ids": [ |
| "arc:validation:ARC-Challenge:122", |
| "arc:validation:ARC-Challenge:126", |
| "arc:validation:ARC-Challenge:175", |
| "arc:validation:ARC-Challenge:187", |
| "arc:validation:ARC-Challenge:189", |
| "arc:validation:ARC-Challenge:192", |
| "arc:validation:ARC-Challenge:208", |
| "arc:validation:ARC-Challenge:238", |
| "arc:validation:ARC-Challenge:290", |
| "arc:validation:ARC-Challenge:293", |
| "arc:validation:ARC-Challenge:35", |
| "arc:validation:ARC-Challenge:43", |
| "arc:validation:ARC-Challenge:58", |
| "arc:validation:ARC-Challenge:7", |
| "arc:validation:ARC-Easy:160", |
| "arc:validation:ARC-Easy:201", |
| "arc:validation:ARC-Easy:229", |
| "arc:validation:ARC-Easy:238", |
| "arc:validation:ARC-Easy:253", |
| "arc:validation:ARC-Easy:259", |
| "arc:validation:ARC-Easy:272", |
| "arc:validation:ARC-Easy:298", |
| "arc:validation:ARC-Easy:329", |
| "arc:validation:ARC-Easy:331", |
| "arc:validation:ARC-Easy:337", |
| "arc:validation:ARC-Easy:362", |
| "arc:validation:ARC-Easy:443", |
| "arc:validation:ARC-Easy:500", |
| "arc:validation:ARC-Easy:545", |
| "arc:validation:ARC-Easy:68", |
| "arc:validation:ARC-Easy:7", |
| "arc:validation:ARC-Easy:74", |
| "banking:train:1053", |
| "banking:train:1594", |
| "banking:train:1747", |
| "banking:train:202", |
| "banking:train:2377", |
| "banking:train:2528", |
| "banking:train:2546", |
| "banking:train:2715", |
| "banking:train:3220", |
| "banking:train:3499", |
| "banking:train:3806", |
| "banking:train:3838", |
| "banking:train:3993", |
| "banking:train:4220", |
| "banking:train:5", |
| "banking:train:5192", |
| "banking:train:5394", |
| "banking:train:5846", |
| "banking:train:5863", |
| "banking:train:6032", |
| "banking:train:6508", |
| "banking:train:7579", |
| "banking:train:7600", |
| "banking:train:7987", |
| "banking:train:8155", |
| "banking:train:850", |
| "banking:train:8626", |
| "banking:train:8929", |
| "banking:train:9355", |
| "banking:train:9812", |
| "banking:train:987", |
| "banking:train:9972", |
| "boolq:validation:103", |
| "boolq:validation:1042", |
| "boolq:validation:1059", |
| "boolq:validation:1541", |
| "boolq:validation:1592", |
| "boolq:validation:1927", |
| "boolq:validation:2014", |
| "boolq:validation:2023", |
| "boolq:validation:2098", |
| "boolq:validation:2139", |
| "boolq:validation:2439", |
| "boolq:validation:2485", |
| "boolq:validation:2607", |
| "boolq:validation:2611", |
| "boolq:validation:2720", |
| "boolq:validation:2751", |
| "boolq:validation:282", |
| "boolq:validation:2843", |
| "boolq:validation:2953", |
| "boolq:validation:301", |
| "boolq:validation:3089", |
| "boolq:validation:313", |
| "boolq:validation:3201", |
| "boolq:validation:322", |
| "boolq:validation:3231", |
| "boolq:validation:39", |
| "boolq:validation:461", |
| "boolq:validation:558", |
| "boolq:validation:589", |
| "boolq:validation:731", |
| "boolq:validation:883", |
| "boolq:validation:915", |
| "snli:validation:1156", |
| "snli:validation:1330", |
| "snli:validation:1559", |
| "snli:validation:1735", |
| "snli:validation:1876", |
| "snli:validation:2251", |
| "snli:validation:2257", |
| "snli:validation:2377", |
| "snli:validation:2881", |
| "snli:validation:291", |
| "snli:validation:300", |
| "snli:validation:3049", |
| "snli:validation:3496", |
| "snli:validation:3871", |
| "snli:validation:4414", |
| "snli:validation:5140", |
| "snli:validation:5440", |
| "snli:validation:5806", |
| "snli:validation:6442", |
| "snli:validation:6772", |
| "snli:validation:6949", |
| "snli:validation:7228", |
| "snli:validation:7465", |
| "snli:validation:7510", |
| "snli:validation:7726", |
| "snli:validation:8095", |
| "snli:validation:8143", |
| "snli:validation:8623", |
| "snli:validation:8962", |
| "snli:validation:9451", |
| "snli:validation:9574", |
| "snli:validation:9655" |
| ], |
| "heldout_group_count": 128, |
| "training_group_count": 384 |
| }, |
| { |
| "fold": 1, |
| "temperature_index": 52, |
| "temperature": 1.5703628043335522, |
| "training_macro_nll": 0.2684132604949462, |
| "heldout_ids": [ |
| "arc:validation:ARC-Challenge:101", |
| "arc:validation:ARC-Challenge:129", |
| "arc:validation:ARC-Challenge:148", |
| "arc:validation:ARC-Challenge:156", |
| "arc:validation:ARC-Challenge:167", |
| "arc:validation:ARC-Challenge:249", |
| "arc:validation:ARC-Challenge:263", |
| "arc:validation:ARC-Challenge:266", |
| "arc:validation:ARC-Challenge:267", |
| "arc:validation:ARC-Challenge:28", |
| "arc:validation:ARC-Challenge:281", |
| "arc:validation:ARC-Challenge:57", |
| "arc:validation:ARC-Challenge:63", |
| "arc:validation:ARC-Challenge:64", |
| "arc:validation:ARC-Challenge:86", |
| "arc:validation:ARC-Challenge:99", |
| "arc:validation:ARC-Easy:106", |
| "arc:validation:ARC-Easy:151", |
| "arc:validation:ARC-Easy:157", |
| "arc:validation:ARC-Easy:166", |
| "arc:validation:ARC-Easy:202", |
| "arc:validation:ARC-Easy:219", |
| "arc:validation:ARC-Easy:224", |
| "arc:validation:ARC-Easy:322", |
| "arc:validation:ARC-Easy:344", |
| "arc:validation:ARC-Easy:382", |
| "arc:validation:ARC-Easy:409", |
| "arc:validation:ARC-Easy:455", |
| "arc:validation:ARC-Easy:50", |
| "arc:validation:ARC-Easy:538", |
| "arc:validation:ARC-Easy:544", |
| "arc:validation:ARC-Easy:552", |
| "banking:train:1050", |
| "banking:train:1131", |
| "banking:train:1190", |
| "banking:train:124", |
| "banking:train:137", |
| "banking:train:1493", |
| "banking:train:1498", |
| "banking:train:1624", |
| "banking:train:2614", |
| "banking:train:2739", |
| "banking:train:3055", |
| "banking:train:306", |
| "banking:train:3312", |
| "banking:train:3724", |
| "banking:train:4070", |
| "banking:train:476", |
| "banking:train:4882", |
| "banking:train:5075", |
| "banking:train:5332", |
| "banking:train:5445", |
| "banking:train:6096", |
| "banking:train:746", |
| "banking:train:7828", |
| "banking:train:8176", |
| "banking:train:8315", |
| "banking:train:8653", |
| "banking:train:9019", |
| "banking:train:9370", |
| "banking:train:9454", |
| "banking:train:9595", |
| "banking:train:9862", |
| "banking:train:9963", |
| "boolq:validation:1189", |
| "boolq:validation:1320", |
| "boolq:validation:1326", |
| "boolq:validation:1381", |
| "boolq:validation:1410", |
| "boolq:validation:149", |
| "boolq:validation:1513", |
| "boolq:validation:1732", |
| "boolq:validation:1739", |
| "boolq:validation:1741", |
| "boolq:validation:1814", |
| "boolq:validation:1848", |
| "boolq:validation:1940", |
| "boolq:validation:2077", |
| "boolq:validation:2091", |
| "boolq:validation:2148", |
| "boolq:validation:2215", |
| "boolq:validation:2251", |
| "boolq:validation:2283", |
| "boolq:validation:2321", |
| "boolq:validation:2398", |
| "boolq:validation:253", |
| "boolq:validation:2575", |
| "boolq:validation:2907", |
| "boolq:validation:2909", |
| "boolq:validation:2978", |
| "boolq:validation:359", |
| "boolq:validation:394", |
| "boolq:validation:436", |
| "boolq:validation:579", |
| "boolq:validation:807", |
| "boolq:validation:949", |
| "snli:validation:2389", |
| "snli:validation:2485", |
| "snli:validation:255", |
| "snli:validation:3169", |
| "snli:validation:333", |
| "snli:validation:3700", |
| "snli:validation:3709", |
| "snli:validation:4093", |
| "snli:validation:420", |
| "snli:validation:4492", |
| "snli:validation:4516", |
| "snli:validation:4648", |
| "snli:validation:4888", |
| "snli:validation:5185", |
| "snli:validation:5332", |
| "snli:validation:5947", |
| "snli:validation:6124", |
| "snli:validation:660", |
| "snli:validation:6619", |
| "snli:validation:6898", |
| "snli:validation:7387", |
| "snli:validation:7492", |
| "snli:validation:7544", |
| "snli:validation:7996", |
| "snli:validation:8074", |
| "snli:validation:8554", |
| "snli:validation:8971", |
| "snli:validation:9163", |
| "snli:validation:9277", |
| "snli:validation:9490", |
| "snli:validation:9595", |
| "snli:validation:9688" |
| ], |
| "heldout_group_count": 128, |
| "training_group_count": 384 |
| }, |
| { |
| "fold": 2, |
| "temperature_index": 53, |
| "temperature": 1.6557699634695275, |
| "training_macro_nll": 0.2710780825791906, |
| "heldout_ids": [ |
| "arc:validation:ARC-Challenge:120", |
| "arc:validation:ARC-Challenge:130", |
| "arc:validation:ARC-Challenge:194", |
| "arc:validation:ARC-Challenge:218", |
| "arc:validation:ARC-Challenge:235", |
| "arc:validation:ARC-Challenge:272", |
| "arc:validation:ARC-Challenge:277", |
| "arc:validation:ARC-Challenge:48", |
| "arc:validation:ARC-Challenge:88", |
| "arc:validation:ARC-Easy:115", |
| "arc:validation:ARC-Easy:12", |
| "arc:validation:ARC-Easy:131", |
| "arc:validation:ARC-Easy:145", |
| "arc:validation:ARC-Easy:179", |
| "arc:validation:ARC-Easy:18", |
| "arc:validation:ARC-Easy:186", |
| "arc:validation:ARC-Easy:191", |
| "arc:validation:ARC-Easy:215", |
| "arc:validation:ARC-Easy:283", |
| "arc:validation:ARC-Easy:314", |
| "arc:validation:ARC-Easy:315", |
| "arc:validation:ARC-Easy:378", |
| "arc:validation:ARC-Easy:384", |
| "arc:validation:ARC-Easy:403", |
| "arc:validation:ARC-Easy:432", |
| "arc:validation:ARC-Easy:439", |
| "arc:validation:ARC-Easy:448", |
| "arc:validation:ARC-Easy:461", |
| "arc:validation:ARC-Easy:469", |
| "arc:validation:ARC-Easy:470", |
| "arc:validation:ARC-Easy:504", |
| "arc:validation:ARC-Easy:561", |
| "banking:train:1142", |
| "banking:train:148", |
| "banking:train:1633", |
| "banking:train:1927", |
| "banking:train:2186", |
| "banking:train:2251", |
| "banking:train:23", |
| "banking:train:3048", |
| "banking:train:3665", |
| "banking:train:3952", |
| "banking:train:3999", |
| "banking:train:4364", |
| "banking:train:4619", |
| "banking:train:5049", |
| "banking:train:5335", |
| "banking:train:549", |
| "banking:train:5957", |
| "banking:train:5985", |
| "banking:train:6132", |
| "banking:train:6240", |
| "banking:train:6488", |
| "banking:train:7065", |
| "banking:train:7749", |
| "banking:train:8470", |
| "banking:train:8761", |
| "banking:train:895", |
| "banking:train:9256", |
| "banking:train:9406", |
| "banking:train:9450", |
| "banking:train:9467", |
| "banking:train:9588", |
| "banking:train:9716", |
| "boolq:validation:1130", |
| "boolq:validation:1167", |
| "boolq:validation:124", |
| "boolq:validation:1255", |
| "boolq:validation:1256", |
| "boolq:validation:1373", |
| "boolq:validation:15", |
| "boolq:validation:155", |
| "boolq:validation:1568", |
| "boolq:validation:16", |
| "boolq:validation:1654", |
| "boolq:validation:1702", |
| "boolq:validation:1792", |
| "boolq:validation:1818", |
| "boolq:validation:210", |
| "boolq:validation:2247", |
| "boolq:validation:2259", |
| "boolq:validation:2362", |
| "boolq:validation:2443", |
| "boolq:validation:2524", |
| "boolq:validation:2553", |
| "boolq:validation:2623", |
| "boolq:validation:2660", |
| "boolq:validation:2669", |
| "boolq:validation:2679", |
| "boolq:validation:2832", |
| "boolq:validation:357", |
| "boolq:validation:453", |
| "boolq:validation:49", |
| "boolq:validation:808", |
| "boolq:validation:90", |
| "boolq:validation:910", |
| "snli:validation:1612", |
| "snli:validation:168", |
| "snli:validation:1693", |
| "snli:validation:2134", |
| "snli:validation:2248", |
| "snli:validation:2422", |
| "snli:validation:2587", |
| "snli:validation:3007", |
| "snli:validation:3097", |
| "snli:validation:3772", |
| "snli:validation:4012", |
| "snli:validation:4168", |
| "snli:validation:4537", |
| "snli:validation:4681", |
| "snli:validation:5050", |
| "snli:validation:5632", |
| "snli:validation:5761", |
| "snli:validation:6055", |
| "snli:validation:6283", |
| "snli:validation:6715", |
| "snli:validation:7117", |
| "snli:validation:7192", |
| "snli:validation:7507", |
| "snli:validation:7594", |
| "snli:validation:7924", |
| "snli:validation:7954", |
| "snli:validation:8035", |
| "snli:validation:8110", |
| "snli:validation:8995", |
| "snli:validation:949", |
| "snli:validation:9523", |
| "snli:validation:9589" |
| ], |
| "heldout_group_count": 128, |
| "training_group_count": 384 |
| }, |
| { |
| "fold": 3, |
| "temperature_index": 52, |
| "temperature": 1.5703628043335522, |
| "training_macro_nll": 0.27198244895421647, |
| "heldout_ids": [ |
| "arc:validation:ARC-Challenge:117", |
| "arc:validation:ARC-Challenge:135", |
| "arc:validation:ARC-Challenge:139", |
| "arc:validation:ARC-Challenge:154", |
| "arc:validation:ARC-Challenge:174", |
| "arc:validation:ARC-Challenge:199", |
| "arc:validation:ARC-Challenge:287", |
| "arc:validation:ARC-Challenge:295", |
| "arc:validation:ARC-Challenge:4", |
| "arc:validation:ARC-Challenge:50", |
| "arc:validation:ARC-Challenge:51", |
| "arc:validation:ARC-Challenge:61", |
| "arc:validation:ARC-Easy:107", |
| "arc:validation:ARC-Easy:121", |
| "arc:validation:ARC-Easy:124", |
| "arc:validation:ARC-Easy:171", |
| "arc:validation:ARC-Easy:235", |
| "arc:validation:ARC-Easy:241", |
| "arc:validation:ARC-Easy:279", |
| "arc:validation:ARC-Easy:289", |
| "arc:validation:ARC-Easy:37", |
| "arc:validation:ARC-Easy:389", |
| "arc:validation:ARC-Easy:394", |
| "arc:validation:ARC-Easy:399", |
| "arc:validation:ARC-Easy:423", |
| "arc:validation:ARC-Easy:492", |
| "arc:validation:ARC-Easy:499", |
| "arc:validation:ARC-Easy:523", |
| "arc:validation:ARC-Easy:533", |
| "arc:validation:ARC-Easy:569", |
| "arc:validation:ARC-Easy:69", |
| "arc:validation:ARC-Easy:93", |
| "banking:train:2049", |
| "banking:train:2350", |
| "banking:train:2497", |
| "banking:train:2506", |
| "banking:train:2720", |
| "banking:train:2788", |
| "banking:train:2967", |
| "banking:train:3502", |
| "banking:train:3726", |
| "banking:train:4014", |
| "banking:train:4735", |
| "banking:train:4803", |
| "banking:train:4917", |
| "banking:train:5477", |
| "banking:train:5851", |
| "banking:train:6551", |
| "banking:train:6839", |
| "banking:train:6926", |
| "banking:train:7809", |
| "banking:train:8092", |
| "banking:train:8186", |
| "banking:train:8215", |
| "banking:train:8667", |
| "banking:train:8752", |
| "banking:train:8763", |
| "banking:train:8851", |
| "banking:train:8917", |
| "banking:train:9145", |
| "banking:train:9279", |
| "banking:train:9676", |
| "banking:train:9837", |
| "banking:train:9967", |
| "boolq:validation:106", |
| "boolq:validation:1065", |
| "boolq:validation:1097", |
| "boolq:validation:1336", |
| "boolq:validation:1350", |
| "boolq:validation:1383", |
| "boolq:validation:1458", |
| "boolq:validation:1641", |
| "boolq:validation:1873", |
| "boolq:validation:1930", |
| "boolq:validation:1989", |
| "boolq:validation:2103", |
| "boolq:validation:234", |
| "boolq:validation:2355", |
| "boolq:validation:2384", |
| "boolq:validation:2440", |
| "boolq:validation:2444", |
| "boolq:validation:2478", |
| "boolq:validation:2872", |
| "boolq:validation:2879", |
| "boolq:validation:2924", |
| "boolq:validation:2971", |
| "boolq:validation:3164", |
| "boolq:validation:4", |
| "boolq:validation:415", |
| "boolq:validation:419", |
| "boolq:validation:428", |
| "boolq:validation:512", |
| "boolq:validation:563", |
| "boolq:validation:643", |
| "boolq:validation:696", |
| "boolq:validation:874", |
| "snli:validation:1102", |
| "snli:validation:1240", |
| "snli:validation:1261", |
| "snli:validation:1738", |
| "snli:validation:1978", |
| "snli:validation:2563", |
| "snli:validation:2659", |
| "snli:validation:2767", |
| "snli:validation:2836", |
| "snli:validation:3232", |
| "snli:validation:3655", |
| "snli:validation:3913", |
| "snli:validation:405", |
| "snli:validation:4051", |
| "snli:validation:4300", |
| "snli:validation:4846", |
| "snli:validation:498", |
| "snli:validation:5224", |
| "snli:validation:5575", |
| "snli:validation:6598", |
| "snli:validation:666", |
| "snli:validation:6925", |
| "snli:validation:6964", |
| "snli:validation:7045", |
| "snli:validation:7351", |
| "snli:validation:7588", |
| "snli:validation:7723", |
| "snli:validation:7750", |
| "snli:validation:7909", |
| "snli:validation:8974", |
| "snli:validation:9385", |
| "snli:validation:9646" |
| ], |
| "heldout_group_count": 128, |
| "training_group_count": 384 |
| } |
| ], |
| "per_family": { |
| "arc": { |
| "count": 128, |
| "score": 0.3012568093094588, |
| "raw_macro_nll": 0.33594514952792776, |
| "accuracy": 0.90625 |
| }, |
| "banking": { |
| "count": 128, |
| "score": 0.04181559751379856, |
| "raw_macro_nll": 0.03417229952375239, |
| "accuracy": 0.9765625 |
| }, |
| "boolq": { |
| "count": 128, |
| "score": 0.43627154539585905, |
| "raw_macro_nll": 0.5853439575679401, |
| "accuracy": 0.859375 |
| }, |
| "snli": { |
| "count": 128, |
| "score": 0.3428792691849764, |
| "raw_macro_nll": 0.33204814515891956, |
| "accuracy": 0.9140625 |
| } |
| }, |
| "policy": { |
| "id": "crossfit_temperature_nll_v1", |
| "folds": 4, |
| "seed": 431, |
| "grouping": "source group, nested within task family; groups shuffled within sorted families and assigned round-robin", |
| "temperature_grid": { |
| "count": 101, |
| "log10_min": -1.0, |
| "log10_max": 1.3, |
| "arithmetic": "Python float64" |
| }, |
| "temperature_fit": "minimum macro-family NLL on the other three folds; smallest temperature wins ties", |
| "score": "macro-family mean of all held-out decision NLLs", |
| "deployment_temperature": "fit afresh on reserved calibration only after model selection" |
| } |
| }, |
| "best_selection_score": 0.2552943737691716, |
| "improved": false, |
| "metrics": { |
| "overall": { |
| "n": 512, |
| "accuracy": 0.9140625, |
| "nll": 0.32187738830487217, |
| "brier_multiclass_sum": 0.14736465059578374, |
| "ece_top_label_10_equal_width_bins": 0.04893872793763876, |
| "reliability_bins": [ |
| { |
| "count": 0, |
| "confidence_sum": 0.0, |
| "correct_sum": 0.0 |
| }, |
| { |
| "count": 0, |
| "confidence_sum": 0.0, |
| "correct_sum": 0.0 |
| }, |
| { |
| "count": 0, |
| "confidence_sum": 0.0, |
| "correct_sum": 0.0 |
| }, |
| { |
| "count": 2, |
| "confidence_sum": 0.7409681677818298, |
| "correct_sum": 0.0 |
| }, |
| { |
| "count": 2, |
| "confidence_sum": 0.8548331260681152, |
| "correct_sum": 1.0 |
| }, |
| { |
| "count": 15, |
| "confidence_sum": 8.04920494556427, |
| "correct_sum": 10.0 |
| }, |
| { |
| "count": 7, |
| "confidence_sum": 4.55422168970108, |
| "correct_sum": 6.0 |
| }, |
| { |
| "count": 18, |
| "confidence_sum": 13.7101429104805, |
| "correct_sum": 14.0 |
| }, |
| { |
| "count": 28, |
| "confidence_sum": 24.01894336938858, |
| "correct_sum": 19.0 |
| }, |
| { |
| "count": 440, |
| "confidence_sum": 433.4651198387146, |
| "correct_sum": 418.0 |
| } |
| ], |
| "accuracy_vs_coverage": { |
| "0.25": { |
| "n": 128, |
| "accuracy": 1.0, |
| "min_confidence": 0.9999927282333374 |
| }, |
| "0.5": { |
| "n": 256, |
| "accuracy": 0.98828125, |
| "min_confidence": 0.9974467754364014 |
| }, |
| "0.75": { |
| "n": 384, |
| "accuracy": 0.9557291666666666, |
| "min_confidence": 0.9531290531158447 |
| }, |
| "1.0": { |
| "n": 512, |
| "accuracy": 0.9140625, |
| "min_confidence": 0.3422239124774933 |
| } |
| } |
| }, |
| "per_family": { |
| "arc": { |
| "n": 128, |
| "accuracy": 0.90625, |
| "nll": 0.33594515031489874, |
| "brier_multiclass_sum": 0.15290327723897154, |
| "ece_top_label_10_equal_width_bins": 0.049574420088902116, |
| "reliability_bins": [ |
| { |
| "count": 0, |
| "confidence_sum": 0.0, |
| "correct_sum": 0.0 |
| }, |
| { |
| "count": 0, |
| "confidence_sum": 0.0, |
| "correct_sum": 0.0 |
| }, |
| { |
| "count": 0, |
| "confidence_sum": 0.0, |
| "correct_sum": 0.0 |
| }, |
| { |
| "count": 1, |
| "confidence_sum": 0.3422239124774933, |
| "correct_sum": 0.0 |
| }, |
| { |
| "count": 2, |
| "confidence_sum": 0.8548331260681152, |
| "correct_sum": 1.0 |
| }, |
| { |
| "count": 7, |
| "confidence_sum": 3.7372488379478455, |
| "correct_sum": 5.0 |
| }, |
| { |
| "count": 4, |
| "confidence_sum": 2.650745928287506, |
| "correct_sum": 3.0 |
| }, |
| { |
| "count": 4, |
| "confidence_sum": 3.025138795375824, |
| "correct_sum": 3.0 |
| }, |
| { |
| "count": 6, |
| "confidence_sum": 5.112928211688995, |
| "correct_sum": 4.0 |
| }, |
| { |
| "count": 104, |
| "confidence_sum": 103.10806274414062, |
| "correct_sum": 100.0 |
| } |
| ], |
| "accuracy_vs_coverage": { |
| "0.25": { |
| "n": 32, |
| "accuracy": 1.0, |
| "min_confidence": 0.9999889135360718 |
| }, |
| "0.5": { |
| "n": 64, |
| "accuracy": 1.0, |
| "min_confidence": 0.9987491369247437 |
| }, |
| "0.75": { |
| "n": 96, |
| "accuracy": 0.96875, |
| "min_confidence": 0.959827721118927 |
| }, |
| "1.0": { |
| "n": 128, |
| "accuracy": 0.90625, |
| "min_confidence": 0.3422239124774933 |
| } |
| } |
| }, |
| "banking": { |
| "n": 128, |
| "accuracy": 0.9765625, |
| "nll": 0.034172295577444056, |
| "brier_multiclass_sum": 0.02131978426881908, |
| "ece_top_label_10_equal_width_bins": 0.008815546752884984, |
| "reliability_bins": [ |
| { |
| "count": 0, |
| "confidence_sum": 0.0, |
| "correct_sum": 0.0 |
| }, |
| { |
| "count": 0, |
| "confidence_sum": 0.0, |
| "correct_sum": 0.0 |
| }, |
| { |
| "count": 0, |
| "confidence_sum": 0.0, |
| "correct_sum": 0.0 |
| }, |
| { |
| "count": 1, |
| "confidence_sum": 0.39874425530433655, |
| "correct_sum": 0.0 |
| }, |
| { |
| "count": 0, |
| "confidence_sum": 0.0, |
| "correct_sum": 0.0 |
| }, |
| { |
| "count": 1, |
| "confidence_sum": 0.5265122652053833, |
| "correct_sum": 0.0 |
| }, |
| { |
| "count": 0, |
| "confidence_sum": 0.0, |
| "correct_sum": 0.0 |
| }, |
| { |
| "count": 4, |
| "confidence_sum": 3.018770754337311, |
| "correct_sum": 3.0 |
| }, |
| { |
| "count": 0, |
| "confidence_sum": 0.0, |
| "correct_sum": 0.0 |
| }, |
| { |
| "count": 122, |
| "confidence_sum": 121.81563729047775, |
| "correct_sum": 122.0 |
| } |
| ], |
| "accuracy_vs_coverage": { |
| "0.25": { |
| "n": 32, |
| "accuracy": 1.0, |
| "min_confidence": 1.0 |
| }, |
| "0.5": { |
| "n": 64, |
| "accuracy": 1.0, |
| "min_confidence": 1.0 |
| }, |
| "0.75": { |
| "n": 96, |
| "accuracy": 1.0, |
| "min_confidence": 0.9999971389770508 |
| }, |
| "1.0": { |
| "n": 128, |
| "accuracy": 0.9765625, |
| "min_confidence": 0.39874425530433655 |
| } |
| } |
| }, |
| "boolq": { |
| "n": 128, |
| "accuracy": 0.859375, |
| "nll": 0.5853439643498177, |
| "brier_multiclass_sum": 0.2555628278708776, |
| "ece_top_label_10_equal_width_bins": 0.10254310397431254, |
| "reliability_bins": [ |
| { |
| "count": 0, |
| "confidence_sum": 0.0, |
| "correct_sum": 0.0 |
| }, |
| { |
| "count": 0, |
| "confidence_sum": 0.0, |
| "correct_sum": 0.0 |
| }, |
| { |
| "count": 0, |
| "confidence_sum": 0.0, |
| "correct_sum": 0.0 |
| }, |
| { |
| "count": 0, |
| "confidence_sum": 0.0, |
| "correct_sum": 0.0 |
| }, |
| { |
| "count": 0, |
| "confidence_sum": 0.0, |
| "correct_sum": 0.0 |
| }, |
| { |
| "count": 6, |
| "confidence_sum": 3.190328001976013, |
| "correct_sum": 4.0 |
| }, |
| { |
| "count": 2, |
| "confidence_sum": 1.281350314617157, |
| "correct_sum": 2.0 |
| }, |
| { |
| "count": 8, |
| "confidence_sum": 6.086918532848358, |
| "correct_sum": 6.0 |
| }, |
| { |
| "count": 11, |
| "confidence_sum": 9.494810223579407, |
| "correct_sum": 8.0 |
| }, |
| { |
| "count": 101, |
| "confidence_sum": 100.01546686887741, |
| "correct_sum": 90.0 |
| } |
| ], |
| "accuracy_vs_coverage": { |
| "0.25": { |
| "n": 32, |
| "accuracy": 0.96875, |
| "min_confidence": 0.9992445707321167 |
| }, |
| "0.5": { |
| "n": 64, |
| "accuracy": 0.921875, |
| "min_confidence": 0.9960445165634155 |
| }, |
| "0.75": { |
| "n": 96, |
| "accuracy": 0.8854166666666666, |
| "min_confidence": 0.9512931704521179 |
| }, |
| "1.0": { |
| "n": 128, |
| "accuracy": 0.859375, |
| "min_confidence": 0.5006566047668457 |
| } |
| } |
| }, |
| "snli": { |
| "n": 128, |
| "accuracy": 0.9140625, |
| "nll": 0.3320481429773281, |
| "brier_multiclass_sum": 0.15967271300446675, |
| "ece_top_label_10_equal_width_bins": 0.04797345120459795, |
| "reliability_bins": [ |
| { |
| "count": 0, |
| "confidence_sum": 0.0, |
| "correct_sum": 0.0 |
| }, |
| { |
| "count": 0, |
| "confidence_sum": 0.0, |
| "correct_sum": 0.0 |
| }, |
| { |
| "count": 0, |
| "confidence_sum": 0.0, |
| "correct_sum": 0.0 |
| }, |
| { |
| "count": 0, |
| "confidence_sum": 0.0, |
| "correct_sum": 0.0 |
| }, |
| { |
| "count": 0, |
| "confidence_sum": 0.0, |
| "correct_sum": 0.0 |
| }, |
| { |
| "count": 1, |
| "confidence_sum": 0.5951158404350281, |
| "correct_sum": 1.0 |
| }, |
| { |
| "count": 1, |
| "confidence_sum": 0.6221254467964172, |
| "correct_sum": 1.0 |
| }, |
| { |
| "count": 2, |
| "confidence_sum": 1.5793148279190063, |
| "correct_sum": 2.0 |
| }, |
| { |
| "count": 11, |
| "confidence_sum": 9.411204934120178, |
| "correct_sum": 7.0 |
| }, |
| { |
| "count": 113, |
| "confidence_sum": 108.52595293521881, |
| "correct_sum": 106.0 |
| } |
| ], |
| "accuracy_vs_coverage": { |
| "0.25": { |
| "n": 32, |
| "accuracy": 0.9375, |
| "min_confidence": 0.9775771498680115 |
| }, |
| "0.5": { |
| "n": 64, |
| "accuracy": 0.953125, |
| "min_confidence": 0.9560697078704834 |
| }, |
| "0.75": { |
| "n": 96, |
| "accuracy": 0.9479166666666666, |
| "min_confidence": 0.9285557866096497 |
| }, |
| "1.0": { |
| "n": 128, |
| "accuracy": 0.9140625, |
| "min_confidence": 0.5951158404350281 |
| } |
| } |
| } |
| } |
| } |
| }, |
| { |
| "step": 4000, |
| "macro_nll": 0.6221038123596042, |
| "best_macro_nll": 0.25676974726429014, |
| "selection": { |
| "metric": "crossfit_temperature_nll_v1", |
| "score": 0.31337868570193705, |
| "raw_macro_nll": 0.6221038143217102, |
| "accuracy": 0.896484375, |
| "folds": [ |
| { |
| "fold": 0, |
| "temperature_index": 67, |
| "temperature": 3.4753616144320576, |
| "training_macro_nll": 0.32442736125181126, |
| "heldout_ids": [ |
| "arc:validation:ARC-Challenge:122", |
| "arc:validation:ARC-Challenge:126", |
| "arc:validation:ARC-Challenge:175", |
| "arc:validation:ARC-Challenge:187", |
| "arc:validation:ARC-Challenge:189", |
| "arc:validation:ARC-Challenge:192", |
| "arc:validation:ARC-Challenge:208", |
| "arc:validation:ARC-Challenge:238", |
| "arc:validation:ARC-Challenge:290", |
| "arc:validation:ARC-Challenge:293", |
| "arc:validation:ARC-Challenge:35", |
| "arc:validation:ARC-Challenge:43", |
| "arc:validation:ARC-Challenge:58", |
| "arc:validation:ARC-Challenge:7", |
| "arc:validation:ARC-Easy:160", |
| "arc:validation:ARC-Easy:201", |
| "arc:validation:ARC-Easy:229", |
| "arc:validation:ARC-Easy:238", |
| "arc:validation:ARC-Easy:253", |
| "arc:validation:ARC-Easy:259", |
| "arc:validation:ARC-Easy:272", |
| "arc:validation:ARC-Easy:298", |
| "arc:validation:ARC-Easy:329", |
| "arc:validation:ARC-Easy:331", |
| "arc:validation:ARC-Easy:337", |
| "arc:validation:ARC-Easy:362", |
| "arc:validation:ARC-Easy:443", |
| "arc:validation:ARC-Easy:500", |
| "arc:validation:ARC-Easy:545", |
| "arc:validation:ARC-Easy:68", |
| "arc:validation:ARC-Easy:7", |
| "arc:validation:ARC-Easy:74", |
| "banking:train:1053", |
| "banking:train:1594", |
| "banking:train:1747", |
| "banking:train:202", |
| "banking:train:2377", |
| "banking:train:2528", |
| "banking:train:2546", |
| "banking:train:2715", |
| "banking:train:3220", |
| "banking:train:3499", |
| "banking:train:3806", |
| "banking:train:3838", |
| "banking:train:3993", |
| "banking:train:4220", |
| "banking:train:5", |
| "banking:train:5192", |
| "banking:train:5394", |
| "banking:train:5846", |
| "banking:train:5863", |
| "banking:train:6032", |
| "banking:train:6508", |
| "banking:train:7579", |
| "banking:train:7600", |
| "banking:train:7987", |
| "banking:train:8155", |
| "banking:train:850", |
| "banking:train:8626", |
| "banking:train:8929", |
| "banking:train:9355", |
| "banking:train:9812", |
| "banking:train:987", |
| "banking:train:9972", |
| "boolq:validation:103", |
| "boolq:validation:1042", |
| "boolq:validation:1059", |
| "boolq:validation:1541", |
| "boolq:validation:1592", |
| "boolq:validation:1927", |
| "boolq:validation:2014", |
| "boolq:validation:2023", |
| "boolq:validation:2098", |
| "boolq:validation:2139", |
| "boolq:validation:2439", |
| "boolq:validation:2485", |
| "boolq:validation:2607", |
| "boolq:validation:2611", |
| "boolq:validation:2720", |
| "boolq:validation:2751", |
| "boolq:validation:282", |
| "boolq:validation:2843", |
| "boolq:validation:2953", |
| "boolq:validation:301", |
| "boolq:validation:3089", |
| "boolq:validation:313", |
| "boolq:validation:3201", |
| "boolq:validation:322", |
| "boolq:validation:3231", |
| "boolq:validation:39", |
| "boolq:validation:461", |
| "boolq:validation:558", |
| "boolq:validation:589", |
| "boolq:validation:731", |
| "boolq:validation:883", |
| "boolq:validation:915", |
| "snli:validation:1156", |
| "snli:validation:1330", |
| "snli:validation:1559", |
| "snli:validation:1735", |
| "snli:validation:1876", |
| "snli:validation:2251", |
| "snli:validation:2257", |
| "snli:validation:2377", |
| "snli:validation:2881", |
| "snli:validation:291", |
| "snli:validation:300", |
| "snli:validation:3049", |
| "snli:validation:3496", |
| "snli:validation:3871", |
| "snli:validation:4414", |
| "snli:validation:5140", |
| "snli:validation:5440", |
| "snli:validation:5806", |
| "snli:validation:6442", |
| "snli:validation:6772", |
| "snli:validation:6949", |
| "snli:validation:7228", |
| "snli:validation:7465", |
| "snli:validation:7510", |
| "snli:validation:7726", |
| "snli:validation:8095", |
| "snli:validation:8143", |
| "snli:validation:8623", |
| "snli:validation:8962", |
| "snli:validation:9451", |
| "snli:validation:9574", |
| "snli:validation:9655" |
| ], |
| "heldout_group_count": 128, |
| "training_group_count": 384 |
| }, |
| { |
| "fold": 1, |
| "temperature_index": 67, |
| "temperature": 3.4753616144320576, |
| "training_macro_nll": 0.3142750882453091, |
| "heldout_ids": [ |
| "arc:validation:ARC-Challenge:101", |
| "arc:validation:ARC-Challenge:129", |
| "arc:validation:ARC-Challenge:148", |
| "arc:validation:ARC-Challenge:156", |
| "arc:validation:ARC-Challenge:167", |
| "arc:validation:ARC-Challenge:249", |
| "arc:validation:ARC-Challenge:263", |
| "arc:validation:ARC-Challenge:266", |
| "arc:validation:ARC-Challenge:267", |
| "arc:validation:ARC-Challenge:28", |
| "arc:validation:ARC-Challenge:281", |
| "arc:validation:ARC-Challenge:57", |
| "arc:validation:ARC-Challenge:63", |
| "arc:validation:ARC-Challenge:64", |
| "arc:validation:ARC-Challenge:86", |
| "arc:validation:ARC-Challenge:99", |
| "arc:validation:ARC-Easy:106", |
| "arc:validation:ARC-Easy:151", |
| "arc:validation:ARC-Easy:157", |
| "arc:validation:ARC-Easy:166", |
| "arc:validation:ARC-Easy:202", |
| "arc:validation:ARC-Easy:219", |
| "arc:validation:ARC-Easy:224", |
| "arc:validation:ARC-Easy:322", |
| "arc:validation:ARC-Easy:344", |
| "arc:validation:ARC-Easy:382", |
| "arc:validation:ARC-Easy:409", |
| "arc:validation:ARC-Easy:455", |
| "arc:validation:ARC-Easy:50", |
| "arc:validation:ARC-Easy:538", |
| "arc:validation:ARC-Easy:544", |
| "arc:validation:ARC-Easy:552", |
| "banking:train:1050", |
| "banking:train:1131", |
| "banking:train:1190", |
| "banking:train:124", |
| "banking:train:137", |
| "banking:train:1493", |
| "banking:train:1498", |
| "banking:train:1624", |
| "banking:train:2614", |
| "banking:train:2739", |
| "banking:train:3055", |
| "banking:train:306", |
| "banking:train:3312", |
| "banking:train:3724", |
| "banking:train:4070", |
| "banking:train:476", |
| "banking:train:4882", |
| "banking:train:5075", |
| "banking:train:5332", |
| "banking:train:5445", |
| "banking:train:6096", |
| "banking:train:746", |
| "banking:train:7828", |
| "banking:train:8176", |
| "banking:train:8315", |
| "banking:train:8653", |
| "banking:train:9019", |
| "banking:train:9370", |
| "banking:train:9454", |
| "banking:train:9595", |
| "banking:train:9862", |
| "banking:train:9963", |
| "boolq:validation:1189", |
| "boolq:validation:1320", |
| "boolq:validation:1326", |
| "boolq:validation:1381", |
| "boolq:validation:1410", |
| "boolq:validation:149", |
| "boolq:validation:1513", |
| "boolq:validation:1732", |
| "boolq:validation:1739", |
| "boolq:validation:1741", |
| "boolq:validation:1814", |
| "boolq:validation:1848", |
| "boolq:validation:1940", |
| "boolq:validation:2077", |
| "boolq:validation:2091", |
| "boolq:validation:2148", |
| "boolq:validation:2215", |
| "boolq:validation:2251", |
| "boolq:validation:2283", |
| "boolq:validation:2321", |
| "boolq:validation:2398", |
| "boolq:validation:253", |
| "boolq:validation:2575", |
| "boolq:validation:2907", |
| "boolq:validation:2909", |
| "boolq:validation:2978", |
| "boolq:validation:359", |
| "boolq:validation:394", |
| "boolq:validation:436", |
| "boolq:validation:579", |
| "boolq:validation:807", |
| "boolq:validation:949", |
| "snli:validation:2389", |
| "snli:validation:2485", |
| "snli:validation:255", |
| "snli:validation:3169", |
| "snli:validation:333", |
| "snli:validation:3700", |
| "snli:validation:3709", |
| "snli:validation:4093", |
| "snli:validation:420", |
| "snli:validation:4492", |
| "snli:validation:4516", |
| "snli:validation:4648", |
| "snli:validation:4888", |
| "snli:validation:5185", |
| "snli:validation:5332", |
| "snli:validation:5947", |
| "snli:validation:6124", |
| "snli:validation:660", |
| "snli:validation:6619", |
| "snli:validation:6898", |
| "snli:validation:7387", |
| "snli:validation:7492", |
| "snli:validation:7544", |
| "snli:validation:7996", |
| "snli:validation:8074", |
| "snli:validation:8554", |
| "snli:validation:8971", |
| "snli:validation:9163", |
| "snli:validation:9277", |
| "snli:validation:9490", |
| "snli:validation:9595", |
| "snli:validation:9688" |
| ], |
| "heldout_group_count": 128, |
| "training_group_count": 384 |
| }, |
| { |
| "fold": 2, |
| "temperature_index": 65, |
| "temperature": 3.126079367123956, |
| "training_macro_nll": 0.2825683057725843, |
| "heldout_ids": [ |
| "arc:validation:ARC-Challenge:120", |
| "arc:validation:ARC-Challenge:130", |
| "arc:validation:ARC-Challenge:194", |
| "arc:validation:ARC-Challenge:218", |
| "arc:validation:ARC-Challenge:235", |
| "arc:validation:ARC-Challenge:272", |
| "arc:validation:ARC-Challenge:277", |
| "arc:validation:ARC-Challenge:48", |
| "arc:validation:ARC-Challenge:88", |
| "arc:validation:ARC-Easy:115", |
| "arc:validation:ARC-Easy:12", |
| "arc:validation:ARC-Easy:131", |
| "arc:validation:ARC-Easy:145", |
| "arc:validation:ARC-Easy:179", |
| "arc:validation:ARC-Easy:18", |
| "arc:validation:ARC-Easy:186", |
| "arc:validation:ARC-Easy:191", |
| "arc:validation:ARC-Easy:215", |
| "arc:validation:ARC-Easy:283", |
| "arc:validation:ARC-Easy:314", |
| "arc:validation:ARC-Easy:315", |
| "arc:validation:ARC-Easy:378", |
| "arc:validation:ARC-Easy:384", |
| "arc:validation:ARC-Easy:403", |
| "arc:validation:ARC-Easy:432", |
| "arc:validation:ARC-Easy:439", |
| "arc:validation:ARC-Easy:448", |
| "arc:validation:ARC-Easy:461", |
| "arc:validation:ARC-Easy:469", |
| "arc:validation:ARC-Easy:470", |
| "arc:validation:ARC-Easy:504", |
| "arc:validation:ARC-Easy:561", |
| "banking:train:1142", |
| "banking:train:148", |
| "banking:train:1633", |
| "banking:train:1927", |
| "banking:train:2186", |
| "banking:train:2251", |
| "banking:train:23", |
| "banking:train:3048", |
| "banking:train:3665", |
| "banking:train:3952", |
| "banking:train:3999", |
| "banking:train:4364", |
| "banking:train:4619", |
| "banking:train:5049", |
| "banking:train:5335", |
| "banking:train:549", |
| "banking:train:5957", |
| "banking:train:5985", |
| "banking:train:6132", |
| "banking:train:6240", |
| "banking:train:6488", |
| "banking:train:7065", |
| "banking:train:7749", |
| "banking:train:8470", |
| "banking:train:8761", |
| "banking:train:895", |
| "banking:train:9256", |
| "banking:train:9406", |
| "banking:train:9450", |
| "banking:train:9467", |
| "banking:train:9588", |
| "banking:train:9716", |
| "boolq:validation:1130", |
| "boolq:validation:1167", |
| "boolq:validation:124", |
| "boolq:validation:1255", |
| "boolq:validation:1256", |
| "boolq:validation:1373", |
| "boolq:validation:15", |
| "boolq:validation:155", |
| "boolq:validation:1568", |
| "boolq:validation:16", |
| "boolq:validation:1654", |
| "boolq:validation:1702", |
| "boolq:validation:1792", |
| "boolq:validation:1818", |
| "boolq:validation:210", |
| "boolq:validation:2247", |
| "boolq:validation:2259", |
| "boolq:validation:2362", |
| "boolq:validation:2443", |
| "boolq:validation:2524", |
| "boolq:validation:2553", |
| "boolq:validation:2623", |
| "boolq:validation:2660", |
| "boolq:validation:2669", |
| "boolq:validation:2679", |
| "boolq:validation:2832", |
| "boolq:validation:357", |
| "boolq:validation:453", |
| "boolq:validation:49", |
| "boolq:validation:808", |
| "boolq:validation:90", |
| "boolq:validation:910", |
| "snli:validation:1612", |
| "snli:validation:168", |
| "snli:validation:1693", |
| "snli:validation:2134", |
| "snli:validation:2248", |
| "snli:validation:2422", |
| "snli:validation:2587", |
| "snli:validation:3007", |
| "snli:validation:3097", |
| "snli:validation:3772", |
| "snli:validation:4012", |
| "snli:validation:4168", |
| "snli:validation:4537", |
| "snli:validation:4681", |
| "snli:validation:5050", |
| "snli:validation:5632", |
| "snli:validation:5761", |
| "snli:validation:6055", |
| "snli:validation:6283", |
| "snli:validation:6715", |
| "snli:validation:7117", |
| "snli:validation:7192", |
| "snli:validation:7507", |
| "snli:validation:7594", |
| "snli:validation:7924", |
| "snli:validation:7954", |
| "snli:validation:8035", |
| "snli:validation:8110", |
| "snli:validation:8995", |
| "snli:validation:949", |
| "snli:validation:9523", |
| "snli:validation:9589" |
| ], |
| "heldout_group_count": 128, |
| "training_group_count": 384 |
| }, |
| { |
| "fold": 3, |
| "temperature_index": 67, |
| "temperature": 3.4753616144320576, |
| "training_macro_nll": 0.3223639562301427, |
| "heldout_ids": [ |
| "arc:validation:ARC-Challenge:117", |
| "arc:validation:ARC-Challenge:135", |
| "arc:validation:ARC-Challenge:139", |
| "arc:validation:ARC-Challenge:154", |
| "arc:validation:ARC-Challenge:174", |
| "arc:validation:ARC-Challenge:199", |
| "arc:validation:ARC-Challenge:287", |
| "arc:validation:ARC-Challenge:295", |
| "arc:validation:ARC-Challenge:4", |
| "arc:validation:ARC-Challenge:50", |
| "arc:validation:ARC-Challenge:51", |
| "arc:validation:ARC-Challenge:61", |
| "arc:validation:ARC-Easy:107", |
| "arc:validation:ARC-Easy:121", |
| "arc:validation:ARC-Easy:124", |
| "arc:validation:ARC-Easy:171", |
| "arc:validation:ARC-Easy:235", |
| "arc:validation:ARC-Easy:241", |
| "arc:validation:ARC-Easy:279", |
| "arc:validation:ARC-Easy:289", |
| "arc:validation:ARC-Easy:37", |
| "arc:validation:ARC-Easy:389", |
| "arc:validation:ARC-Easy:394", |
| "arc:validation:ARC-Easy:399", |
| "arc:validation:ARC-Easy:423", |
| "arc:validation:ARC-Easy:492", |
| "arc:validation:ARC-Easy:499", |
| "arc:validation:ARC-Easy:523", |
| "arc:validation:ARC-Easy:533", |
| "arc:validation:ARC-Easy:569", |
| "arc:validation:ARC-Easy:69", |
| "arc:validation:ARC-Easy:93", |
| "banking:train:2049", |
| "banking:train:2350", |
| "banking:train:2497", |
| "banking:train:2506", |
| "banking:train:2720", |
| "banking:train:2788", |
| "banking:train:2967", |
| "banking:train:3502", |
| "banking:train:3726", |
| "banking:train:4014", |
| "banking:train:4735", |
| "banking:train:4803", |
| "banking:train:4917", |
| "banking:train:5477", |
| "banking:train:5851", |
| "banking:train:6551", |
| "banking:train:6839", |
| "banking:train:6926", |
| "banking:train:7809", |
| "banking:train:8092", |
| "banking:train:8186", |
| "banking:train:8215", |
| "banking:train:8667", |
| "banking:train:8752", |
| "banking:train:8763", |
| "banking:train:8851", |
| "banking:train:8917", |
| "banking:train:9145", |
| "banking:train:9279", |
| "banking:train:9676", |
| "banking:train:9837", |
| "banking:train:9967", |
| "boolq:validation:106", |
| "boolq:validation:1065", |
| "boolq:validation:1097", |
| "boolq:validation:1336", |
| "boolq:validation:1350", |
| "boolq:validation:1383", |
| "boolq:validation:1458", |
| "boolq:validation:1641", |
| "boolq:validation:1873", |
| "boolq:validation:1930", |
| "boolq:validation:1989", |
| "boolq:validation:2103", |
| "boolq:validation:234", |
| "boolq:validation:2355", |
| "boolq:validation:2384", |
| "boolq:validation:2440", |
| "boolq:validation:2444", |
| "boolq:validation:2478", |
| "boolq:validation:2872", |
| "boolq:validation:2879", |
| "boolq:validation:2924", |
| "boolq:validation:2971", |
| "boolq:validation:3164", |
| "boolq:validation:4", |
| "boolq:validation:415", |
| "boolq:validation:419", |
| "boolq:validation:428", |
| "boolq:validation:512", |
| "boolq:validation:563", |
| "boolq:validation:643", |
| "boolq:validation:696", |
| "boolq:validation:874", |
| "snli:validation:1102", |
| "snli:validation:1240", |
| "snli:validation:1261", |
| "snli:validation:1738", |
| "snli:validation:1978", |
| "snli:validation:2563", |
| "snli:validation:2659", |
| "snli:validation:2767", |
| "snli:validation:2836", |
| "snli:validation:3232", |
| "snli:validation:3655", |
| "snli:validation:3913", |
| "snli:validation:405", |
| "snli:validation:4051", |
| "snli:validation:4300", |
| "snli:validation:4846", |
| "snli:validation:498", |
| "snli:validation:5224", |
| "snli:validation:5575", |
| "snli:validation:6598", |
| "snli:validation:666", |
| "snli:validation:6925", |
| "snli:validation:6964", |
| "snli:validation:7045", |
| "snli:validation:7351", |
| "snli:validation:7588", |
| "snli:validation:7723", |
| "snli:validation:7750", |
| "snli:validation:7909", |
| "snli:validation:8974", |
| "snli:validation:9385", |
| "snli:validation:9646" |
| ], |
| "heldout_group_count": 128, |
| "training_group_count": 384 |
| } |
| ], |
| "per_family": { |
| "arc": { |
| "count": 128, |
| "score": 0.4118695814660189, |
| "raw_macro_nll": 0.8736200777910155, |
| "accuracy": 0.875 |
| }, |
| "banking": { |
| "count": 128, |
| "score": 0.06946594441189612, |
| "raw_macro_nll": 0.106002592628879, |
| "accuracy": 0.9765625 |
| }, |
| "boolq": { |
| "count": 128, |
| "score": 0.3332506839528764, |
| "raw_macro_nll": 0.7364644020524043, |
| "accuracy": 0.875 |
| }, |
| "snli": { |
| "count": 128, |
| "score": 0.43892853297695694, |
| "raw_macro_nll": 0.7723281848145425, |
| "accuracy": 0.859375 |
| } |
| }, |
| "policy": { |
| "id": "crossfit_temperature_nll_v1", |
| "folds": 4, |
| "seed": 431, |
| "grouping": "source group, nested within task family; groups shuffled within sorted families and assigned round-robin", |
| "temperature_grid": { |
| "count": 101, |
| "log10_min": -1.0, |
| "log10_max": 1.3, |
| "arithmetic": "Python float64" |
| }, |
| "temperature_fit": "minimum macro-family NLL on the other three folds; smallest temperature wins ties", |
| "score": "macro-family mean of all held-out decision NLLs", |
| "deployment_temperature": "fit afresh on reserved calibration only after model selection" |
| } |
| }, |
| "best_selection_score": 0.2552943737691716, |
| "improved": false, |
| "metrics": { |
| "overall": { |
| "n": 512, |
| "accuracy": 0.896484375, |
| "nll": 0.6221038123596042, |
| "brier_multiclass_sum": 0.19169821909519502, |
| "ece_top_label_10_equal_width_bins": 0.08989131019916385, |
| "reliability_bins": [ |
| { |
| "count": 0, |
| "confidence_sum": 0.0, |
| "correct_sum": 0.0 |
| }, |
| { |
| "count": 0, |
| "confidence_sum": 0.0, |
| "correct_sum": 0.0 |
| }, |
| { |
| "count": 0, |
| "confidence_sum": 0.0, |
| "correct_sum": 0.0 |
| }, |
| { |
| "count": 1, |
| "confidence_sum": 0.39359351992607117, |
| "correct_sum": 1.0 |
| }, |
| { |
| "count": 1, |
| "confidence_sum": 0.42377254366874695, |
| "correct_sum": 0.0 |
| }, |
| { |
| "count": 7, |
| "confidence_sum": 3.8932124972343445, |
| "correct_sum": 5.0 |
| }, |
| { |
| "count": 9, |
| "confidence_sum": 5.776679337024689, |
| "correct_sum": 7.0 |
| }, |
| { |
| "count": 9, |
| "confidence_sum": 6.852575600147247, |
| "correct_sum": 7.0 |
| }, |
| { |
| "count": 12, |
| "confidence_sum": 10.222121059894562, |
| "correct_sum": 6.0 |
| }, |
| { |
| "count": 473, |
| "confidence_sum": 471.29451817274094, |
| "correct_sum": 433.0 |
| } |
| ], |
| "accuracy_vs_coverage": { |
| "0.25": { |
| "n": 128, |
| "accuracy": 1.0, |
| "min_confidence": 1.0 |
| }, |
| "0.5": { |
| "n": 256, |
| "accuracy": 0.98828125, |
| "min_confidence": 0.9999780654907227 |
| }, |
| "0.75": { |
| "n": 384, |
| "accuracy": 0.953125, |
| "min_confidence": 0.9980484247207642 |
| }, |
| "1.0": { |
| "n": 512, |
| "accuracy": 0.896484375, |
| "min_confidence": 0.39359351992607117 |
| } |
| } |
| }, |
| "per_family": { |
| "arc": { |
| "n": 128, |
| "accuracy": 0.875, |
| "nll": 0.8736200741430675, |
| "brier_multiclass_sum": 0.21730879220548308, |
| "ece_top_label_10_equal_width_bins": 0.11237058648839593, |
| "reliability_bins": [ |
| { |
| "count": 0, |
| "confidence_sum": 0.0, |
| "correct_sum": 0.0 |
| }, |
| { |
| "count": 0, |
| "confidence_sum": 0.0, |
| "correct_sum": 0.0 |
| }, |
| { |
| "count": 0, |
| "confidence_sum": 0.0, |
| "correct_sum": 0.0 |
| }, |
| { |
| "count": 1, |
| "confidence_sum": 0.39359351992607117, |
| "correct_sum": 1.0 |
| }, |
| { |
| "count": 1, |
| "confidence_sum": 0.42377254366874695, |
| "correct_sum": 0.0 |
| }, |
| { |
| "count": 2, |
| "confidence_sum": 1.1089537739753723, |
| "correct_sum": 2.0 |
| }, |
| { |
| "count": 6, |
| "confidence_sum": 3.8407031297683716, |
| "correct_sum": 4.0 |
| }, |
| { |
| "count": 2, |
| "confidence_sum": 1.4807611107826233, |
| "correct_sum": 2.0 |
| }, |
| { |
| "count": 5, |
| "confidence_sum": 4.257317125797272, |
| "correct_sum": 1.0 |
| }, |
| { |
| "count": 111, |
| "confidence_sum": 110.5263569355011, |
| "correct_sum": 102.0 |
| } |
| ], |
| "accuracy_vs_coverage": { |
| "0.25": { |
| "n": 32, |
| "accuracy": 1.0, |
| "min_confidence": 1.0 |
| }, |
| "0.5": { |
| "n": 64, |
| "accuracy": 0.984375, |
| "min_confidence": 0.9999995231628418 |
| }, |
| "0.75": { |
| "n": 96, |
| "accuracy": 0.9583333333333334, |
| "min_confidence": 0.9975321292877197 |
| }, |
| "1.0": { |
| "n": 128, |
| "accuracy": 0.875, |
| "min_confidence": 0.39359351992607117 |
| } |
| } |
| }, |
| "banking": { |
| "n": 128, |
| "accuracy": 0.9765625, |
| "nll": 0.10600259243186233, |
| "brier_multiclass_sum": 0.04318536812967019, |
| "ece_top_label_10_equal_width_bins": 0.025164072401821613, |
| "reliability_bins": [ |
| { |
| "count": 0, |
| "confidence_sum": 0.0, |
| "correct_sum": 0.0 |
| }, |
| { |
| "count": 0, |
| "confidence_sum": 0.0, |
| "correct_sum": 0.0 |
| }, |
| { |
| "count": 0, |
| "confidence_sum": 0.0, |
| "correct_sum": 0.0 |
| }, |
| { |
| "count": 0, |
| "confidence_sum": 0.0, |
| "correct_sum": 0.0 |
| }, |
| { |
| "count": 0, |
| "confidence_sum": 0.0, |
| "correct_sum": 0.0 |
| }, |
| { |
| "count": 2, |
| "confidence_sum": 1.099687397480011, |
| "correct_sum": 2.0 |
| }, |
| { |
| "count": 0, |
| "confidence_sum": 0.0, |
| "correct_sum": 0.0 |
| }, |
| { |
| "count": 2, |
| "confidence_sum": 1.5330036878585815, |
| "correct_sum": 1.0 |
| }, |
| { |
| "count": 1, |
| "confidence_sum": 0.8130332827568054, |
| "correct_sum": 0.0 |
| }, |
| { |
| "count": 123, |
| "confidence_sum": 122.97465169429779, |
| "correct_sum": 122.0 |
| } |
| ], |
| "accuracy_vs_coverage": { |
| "0.25": { |
| "n": 32, |
| "accuracy": 1.0, |
| "min_confidence": 1.0 |
| }, |
| "0.5": { |
| "n": 64, |
| "accuracy": 1.0, |
| "min_confidence": 1.0 |
| }, |
| "0.75": { |
| "n": 96, |
| "accuracy": 1.0, |
| "min_confidence": 1.0 |
| }, |
| "1.0": { |
| "n": 128, |
| "accuracy": 0.9765625, |
| "min_confidence": 0.545770525932312 |
| } |
| } |
| }, |
| "boolq": { |
| "n": 128, |
| "accuracy": 0.875, |
| "nll": 0.7364644002551142, |
| "brier_multiclass_sum": 0.24680240271797868, |
| "ece_top_label_10_equal_width_bins": 0.13037992408499122, |
| "reliability_bins": [ |
| { |
| "count": 0, |
| "confidence_sum": 0.0, |
| "correct_sum": 0.0 |
| }, |
| { |
| "count": 0, |
| "confidence_sum": 0.0, |
| "correct_sum": 0.0 |
| }, |
| { |
| "count": 0, |
| "confidence_sum": 0.0, |
| "correct_sum": 0.0 |
| }, |
| { |
| "count": 0, |
| "confidence_sum": 0.0, |
| "correct_sum": 0.0 |
| }, |
| { |
| "count": 0, |
| "confidence_sum": 0.0, |
| "correct_sum": 0.0 |
| }, |
| { |
| "count": 1, |
| "confidence_sum": 0.5227815508842468, |
| "correct_sum": 1.0 |
| }, |
| { |
| "count": 2, |
| "confidence_sum": 1.2790811657905579, |
| "correct_sum": 2.0 |
| }, |
| { |
| "count": 2, |
| "confidence_sum": 1.5391781330108643, |
| "correct_sum": 1.0 |
| }, |
| { |
| "count": 3, |
| "confidence_sum": 2.5299994945526123, |
| "correct_sum": 3.0 |
| }, |
| { |
| "count": 120, |
| "confidence_sum": 119.48131436109543, |
| "correct_sum": 105.0 |
| } |
| ], |
| "accuracy_vs_coverage": { |
| "0.25": { |
| "n": 32, |
| "accuracy": 0.96875, |
| "min_confidence": 0.9999909400939941 |
| }, |
| "0.5": { |
| "n": 64, |
| "accuracy": 0.96875, |
| "min_confidence": 0.9999333620071411 |
| }, |
| "0.75": { |
| "n": 96, |
| "accuracy": 0.9270833333333334, |
| "min_confidence": 0.9974094033241272 |
| }, |
| "1.0": { |
| "n": 128, |
| "accuracy": 0.875, |
| "min_confidence": 0.5227815508842468 |
| } |
| } |
| }, |
| "snli": { |
| "n": 128, |
| "accuracy": 0.859375, |
| "nll": 0.7723281826083728, |
| "brier_multiclass_sum": 0.25949631332764816, |
| "ece_top_label_10_equal_width_bins": 0.1339002219028771, |
| "reliability_bins": [ |
| { |
| "count": 0, |
| "confidence_sum": 0.0, |
| "correct_sum": 0.0 |
| }, |
| { |
| "count": 0, |
| "confidence_sum": 0.0, |
| "correct_sum": 0.0 |
| }, |
| { |
| "count": 0, |
| "confidence_sum": 0.0, |
| "correct_sum": 0.0 |
| }, |
| { |
| "count": 0, |
| "confidence_sum": 0.0, |
| "correct_sum": 0.0 |
| }, |
| { |
| "count": 0, |
| "confidence_sum": 0.0, |
| "correct_sum": 0.0 |
| }, |
| { |
| "count": 2, |
| "confidence_sum": 1.1617897748947144, |
| "correct_sum": 0.0 |
| }, |
| { |
| "count": 1, |
| "confidence_sum": 0.6568950414657593, |
| "correct_sum": 1.0 |
| }, |
| { |
| "count": 3, |
| "confidence_sum": 2.2996326684951782, |
| "correct_sum": 3.0 |
| }, |
| { |
| "count": 3, |
| "confidence_sum": 2.6217711567878723, |
| "correct_sum": 2.0 |
| }, |
| { |
| "count": 119, |
| "confidence_sum": 118.31219518184662, |
| "correct_sum": 104.0 |
| } |
| ], |
| "accuracy_vs_coverage": { |
| "0.25": { |
| "n": 32, |
| "accuracy": 0.9375, |
| "min_confidence": 0.9998804330825806 |
| }, |
| "0.5": { |
| "n": 64, |
| "accuracy": 0.953125, |
| "min_confidence": 0.999064028263092 |
| }, |
| "0.75": { |
| "n": 96, |
| "accuracy": 0.8958333333333334, |
| "min_confidence": 0.9940833449363708 |
| }, |
| "1.0": { |
| "n": 128, |
| "accuracy": 0.859375, |
| "min_confidence": 0.5666037201881409 |
| } |
| } |
| } |
| } |
| } |
| }, |
| { |
| "step": 4500, |
| "macro_nll": 0.4348410999061469, |
| "best_macro_nll": 0.25676974726429014, |
| "selection": { |
| "metric": "crossfit_temperature_nll_v1", |
| "score": 0.2841199300221572, |
| "raw_macro_nll": 0.4348410990047701, |
| "accuracy": 0.8984375, |
| "folds": [ |
| { |
| "fold": 0, |
| "temperature_index": 62, |
| "temperature": 2.666858664521479, |
| "training_macro_nll": 0.30822706909779624, |
| "heldout_ids": [ |
| "arc:validation:ARC-Challenge:122", |
| "arc:validation:ARC-Challenge:126", |
| "arc:validation:ARC-Challenge:175", |
| "arc:validation:ARC-Challenge:187", |
| "arc:validation:ARC-Challenge:189", |
| "arc:validation:ARC-Challenge:192", |
| "arc:validation:ARC-Challenge:208", |
| "arc:validation:ARC-Challenge:238", |
| "arc:validation:ARC-Challenge:290", |
| "arc:validation:ARC-Challenge:293", |
| "arc:validation:ARC-Challenge:35", |
| "arc:validation:ARC-Challenge:43", |
| "arc:validation:ARC-Challenge:58", |
| "arc:validation:ARC-Challenge:7", |
| "arc:validation:ARC-Easy:160", |
| "arc:validation:ARC-Easy:201", |
| "arc:validation:ARC-Easy:229", |
| "arc:validation:ARC-Easy:238", |
| "arc:validation:ARC-Easy:253", |
| "arc:validation:ARC-Easy:259", |
| "arc:validation:ARC-Easy:272", |
| "arc:validation:ARC-Easy:298", |
| "arc:validation:ARC-Easy:329", |
| "arc:validation:ARC-Easy:331", |
| "arc:validation:ARC-Easy:337", |
| "arc:validation:ARC-Easy:362", |
| "arc:validation:ARC-Easy:443", |
| "arc:validation:ARC-Easy:500", |
| "arc:validation:ARC-Easy:545", |
| "arc:validation:ARC-Easy:68", |
| "arc:validation:ARC-Easy:7", |
| "arc:validation:ARC-Easy:74", |
| "banking:train:1053", |
| "banking:train:1594", |
| "banking:train:1747", |
| "banking:train:202", |
| "banking:train:2377", |
| "banking:train:2528", |
| "banking:train:2546", |
| "banking:train:2715", |
| "banking:train:3220", |
| "banking:train:3499", |
| "banking:train:3806", |
| "banking:train:3838", |
| "banking:train:3993", |
| "banking:train:4220", |
| "banking:train:5", |
| "banking:train:5192", |
| "banking:train:5394", |
| "banking:train:5846", |
| "banking:train:5863", |
| "banking:train:6032", |
| "banking:train:6508", |
| "banking:train:7579", |
| "banking:train:7600", |
| "banking:train:7987", |
| "banking:train:8155", |
| "banking:train:850", |
| "banking:train:8626", |
| "banking:train:8929", |
| "banking:train:9355", |
| "banking:train:9812", |
| "banking:train:987", |
| "banking:train:9972", |
| "boolq:validation:103", |
| "boolq:validation:1042", |
| "boolq:validation:1059", |
| "boolq:validation:1541", |
| "boolq:validation:1592", |
| "boolq:validation:1927", |
| "boolq:validation:2014", |
| "boolq:validation:2023", |
| "boolq:validation:2098", |
| "boolq:validation:2139", |
| "boolq:validation:2439", |
| "boolq:validation:2485", |
| "boolq:validation:2607", |
| "boolq:validation:2611", |
| "boolq:validation:2720", |
| "boolq:validation:2751", |
| "boolq:validation:282", |
| "boolq:validation:2843", |
| "boolq:validation:2953", |
| "boolq:validation:301", |
| "boolq:validation:3089", |
| "boolq:validation:313", |
| "boolq:validation:3201", |
| "boolq:validation:322", |
| "boolq:validation:3231", |
| "boolq:validation:39", |
| "boolq:validation:461", |
| "boolq:validation:558", |
| "boolq:validation:589", |
| "boolq:validation:731", |
| "boolq:validation:883", |
| "boolq:validation:915", |
| "snli:validation:1156", |
| "snli:validation:1330", |
| "snli:validation:1559", |
| "snli:validation:1735", |
| "snli:validation:1876", |
| "snli:validation:2251", |
| "snli:validation:2257", |
| "snli:validation:2377", |
| "snli:validation:2881", |
| "snli:validation:291", |
| "snli:validation:300", |
| "snli:validation:3049", |
| "snli:validation:3496", |
| "snli:validation:3871", |
| "snli:validation:4414", |
| "snli:validation:5140", |
| "snli:validation:5440", |
| "snli:validation:5806", |
| "snli:validation:6442", |
| "snli:validation:6772", |
| "snli:validation:6949", |
| "snli:validation:7228", |
| "snli:validation:7465", |
| "snli:validation:7510", |
| "snli:validation:7726", |
| "snli:validation:8095", |
| "snli:validation:8143", |
| "snli:validation:8623", |
| "snli:validation:8962", |
| "snli:validation:9451", |
| "snli:validation:9574", |
| "snli:validation:9655" |
| ], |
| "heldout_group_count": 128, |
| "training_group_count": 384 |
| }, |
| { |
| "fold": 1, |
| "temperature_index": 60, |
| "temperature": 2.39883291901949, |
| "training_macro_nll": 0.27078229308933455, |
| "heldout_ids": [ |
| "arc:validation:ARC-Challenge:101", |
| "arc:validation:ARC-Challenge:129", |
| "arc:validation:ARC-Challenge:148", |
| "arc:validation:ARC-Challenge:156", |
| "arc:validation:ARC-Challenge:167", |
| "arc:validation:ARC-Challenge:249", |
| "arc:validation:ARC-Challenge:263", |
| "arc:validation:ARC-Challenge:266", |
| "arc:validation:ARC-Challenge:267", |
| "arc:validation:ARC-Challenge:28", |
| "arc:validation:ARC-Challenge:281", |
| "arc:validation:ARC-Challenge:57", |
| "arc:validation:ARC-Challenge:63", |
| "arc:validation:ARC-Challenge:64", |
| "arc:validation:ARC-Challenge:86", |
| "arc:validation:ARC-Challenge:99", |
| "arc:validation:ARC-Easy:106", |
| "arc:validation:ARC-Easy:151", |
| "arc:validation:ARC-Easy:157", |
| "arc:validation:ARC-Easy:166", |
| "arc:validation:ARC-Easy:202", |
| "arc:validation:ARC-Easy:219", |
| "arc:validation:ARC-Easy:224", |
| "arc:validation:ARC-Easy:322", |
| "arc:validation:ARC-Easy:344", |
| "arc:validation:ARC-Easy:382", |
| "arc:validation:ARC-Easy:409", |
| "arc:validation:ARC-Easy:455", |
| "arc:validation:ARC-Easy:50", |
| "arc:validation:ARC-Easy:538", |
| "arc:validation:ARC-Easy:544", |
| "arc:validation:ARC-Easy:552", |
| "banking:train:1050", |
| "banking:train:1131", |
| "banking:train:1190", |
| "banking:train:124", |
| "banking:train:137", |
| "banking:train:1493", |
| "banking:train:1498", |
| "banking:train:1624", |
| "banking:train:2614", |
| "banking:train:2739", |
| "banking:train:3055", |
| "banking:train:306", |
| "banking:train:3312", |
| "banking:train:3724", |
| "banking:train:4070", |
| "banking:train:476", |
| "banking:train:4882", |
| "banking:train:5075", |
| "banking:train:5332", |
| "banking:train:5445", |
| "banking:train:6096", |
| "banking:train:746", |
| "banking:train:7828", |
| "banking:train:8176", |
| "banking:train:8315", |
| "banking:train:8653", |
| "banking:train:9019", |
| "banking:train:9370", |
| "banking:train:9454", |
| "banking:train:9595", |
| "banking:train:9862", |
| "banking:train:9963", |
| "boolq:validation:1189", |
| "boolq:validation:1320", |
| "boolq:validation:1326", |
| "boolq:validation:1381", |
| "boolq:validation:1410", |
| "boolq:validation:149", |
| "boolq:validation:1513", |
| "boolq:validation:1732", |
| "boolq:validation:1739", |
| "boolq:validation:1741", |
| "boolq:validation:1814", |
| "boolq:validation:1848", |
| "boolq:validation:1940", |
| "boolq:validation:2077", |
| "boolq:validation:2091", |
| "boolq:validation:2148", |
| "boolq:validation:2215", |
| "boolq:validation:2251", |
| "boolq:validation:2283", |
| "boolq:validation:2321", |
| "boolq:validation:2398", |
| "boolq:validation:253", |
| "boolq:validation:2575", |
| "boolq:validation:2907", |
| "boolq:validation:2909", |
| "boolq:validation:2978", |
| "boolq:validation:359", |
| "boolq:validation:394", |
| "boolq:validation:436", |
| "boolq:validation:579", |
| "boolq:validation:807", |
| "boolq:validation:949", |
| "snli:validation:2389", |
| "snli:validation:2485", |
| "snli:validation:255", |
| "snli:validation:3169", |
| "snli:validation:333", |
| "snli:validation:3700", |
| "snli:validation:3709", |
| "snli:validation:4093", |
| "snli:validation:420", |
| "snli:validation:4492", |
| "snli:validation:4516", |
| "snli:validation:4648", |
| "snli:validation:4888", |
| "snli:validation:5185", |
| "snli:validation:5332", |
| "snli:validation:5947", |
| "snli:validation:6124", |
| "snli:validation:660", |
| "snli:validation:6619", |
| "snli:validation:6898", |
| "snli:validation:7387", |
| "snli:validation:7492", |
| "snli:validation:7544", |
| "snli:validation:7996", |
| "snli:validation:8074", |
| "snli:validation:8554", |
| "snli:validation:8971", |
| "snli:validation:9163", |
| "snli:validation:9277", |
| "snli:validation:9490", |
| "snli:validation:9595", |
| "snli:validation:9688" |
| ], |
| "heldout_group_count": 128, |
| "training_group_count": 384 |
| }, |
| { |
| "fold": 2, |
| "temperature_index": 60, |
| "temperature": 2.39883291901949, |
| "training_macro_nll": 0.25779374717908343, |
| "heldout_ids": [ |
| "arc:validation:ARC-Challenge:120", |
| "arc:validation:ARC-Challenge:130", |
| "arc:validation:ARC-Challenge:194", |
| "arc:validation:ARC-Challenge:218", |
| "arc:validation:ARC-Challenge:235", |
| "arc:validation:ARC-Challenge:272", |
| "arc:validation:ARC-Challenge:277", |
| "arc:validation:ARC-Challenge:48", |
| "arc:validation:ARC-Challenge:88", |
| "arc:validation:ARC-Easy:115", |
| "arc:validation:ARC-Easy:12", |
| "arc:validation:ARC-Easy:131", |
| "arc:validation:ARC-Easy:145", |
| "arc:validation:ARC-Easy:179", |
| "arc:validation:ARC-Easy:18", |
| "arc:validation:ARC-Easy:186", |
| "arc:validation:ARC-Easy:191", |
| "arc:validation:ARC-Easy:215", |
| "arc:validation:ARC-Easy:283", |
| "arc:validation:ARC-Easy:314", |
| "arc:validation:ARC-Easy:315", |
| "arc:validation:ARC-Easy:378", |
| "arc:validation:ARC-Easy:384", |
| "arc:validation:ARC-Easy:403", |
| "arc:validation:ARC-Easy:432", |
| "arc:validation:ARC-Easy:439", |
| "arc:validation:ARC-Easy:448", |
| "arc:validation:ARC-Easy:461", |
| "arc:validation:ARC-Easy:469", |
| "arc:validation:ARC-Easy:470", |
| "arc:validation:ARC-Easy:504", |
| "arc:validation:ARC-Easy:561", |
| "banking:train:1142", |
| "banking:train:148", |
| "banking:train:1633", |
| "banking:train:1927", |
| "banking:train:2186", |
| "banking:train:2251", |
| "banking:train:23", |
| "banking:train:3048", |
| "banking:train:3665", |
| "banking:train:3952", |
| "banking:train:3999", |
| "banking:train:4364", |
| "banking:train:4619", |
| "banking:train:5049", |
| "banking:train:5335", |
| "banking:train:549", |
| "banking:train:5957", |
| "banking:train:5985", |
| "banking:train:6132", |
| "banking:train:6240", |
| "banking:train:6488", |
| "banking:train:7065", |
| "banking:train:7749", |
| "banking:train:8470", |
| "banking:train:8761", |
| "banking:train:895", |
| "banking:train:9256", |
| "banking:train:9406", |
| "banking:train:9450", |
| "banking:train:9467", |
| "banking:train:9588", |
| "banking:train:9716", |
| "boolq:validation:1130", |
| "boolq:validation:1167", |
| "boolq:validation:124", |
| "boolq:validation:1255", |
| "boolq:validation:1256", |
| "boolq:validation:1373", |
| "boolq:validation:15", |
| "boolq:validation:155", |
| "boolq:validation:1568", |
| "boolq:validation:16", |
| "boolq:validation:1654", |
| "boolq:validation:1702", |
| "boolq:validation:1792", |
| "boolq:validation:1818", |
| "boolq:validation:210", |
| "boolq:validation:2247", |
| "boolq:validation:2259", |
| "boolq:validation:2362", |
| "boolq:validation:2443", |
| "boolq:validation:2524", |
| "boolq:validation:2553", |
| "boolq:validation:2623", |
| "boolq:validation:2660", |
| "boolq:validation:2669", |
| "boolq:validation:2679", |
| "boolq:validation:2832", |
| "boolq:validation:357", |
| "boolq:validation:453", |
| "boolq:validation:49", |
| "boolq:validation:808", |
| "boolq:validation:90", |
| "boolq:validation:910", |
| "snli:validation:1612", |
| "snli:validation:168", |
| "snli:validation:1693", |
| "snli:validation:2134", |
| "snli:validation:2248", |
| "snli:validation:2422", |
| "snli:validation:2587", |
| "snli:validation:3007", |
| "snli:validation:3097", |
| "snli:validation:3772", |
| "snli:validation:4012", |
| "snli:validation:4168", |
| "snli:validation:4537", |
| "snli:validation:4681", |
| "snli:validation:5050", |
| "snli:validation:5632", |
| "snli:validation:5761", |
| "snli:validation:6055", |
| "snli:validation:6283", |
| "snli:validation:6715", |
| "snli:validation:7117", |
| "snli:validation:7192", |
| "snli:validation:7507", |
| "snli:validation:7594", |
| "snli:validation:7924", |
| "snli:validation:7954", |
| "snli:validation:8035", |
| "snli:validation:8110", |
| "snli:validation:8995", |
| "snli:validation:949", |
| "snli:validation:9523", |
| "snli:validation:9589" |
| ], |
| "heldout_group_count": 128, |
| "training_group_count": 384 |
| }, |
| { |
| "fold": 3, |
| "temperature_index": 61, |
| "temperature": 2.5292979964461435, |
| "training_macro_nll": 0.2911054608892844, |
| "heldout_ids": [ |
| "arc:validation:ARC-Challenge:117", |
| "arc:validation:ARC-Challenge:135", |
| "arc:validation:ARC-Challenge:139", |
| "arc:validation:ARC-Challenge:154", |
| "arc:validation:ARC-Challenge:174", |
| "arc:validation:ARC-Challenge:199", |
| "arc:validation:ARC-Challenge:287", |
| "arc:validation:ARC-Challenge:295", |
| "arc:validation:ARC-Challenge:4", |
| "arc:validation:ARC-Challenge:50", |
| "arc:validation:ARC-Challenge:51", |
| "arc:validation:ARC-Challenge:61", |
| "arc:validation:ARC-Easy:107", |
| "arc:validation:ARC-Easy:121", |
| "arc:validation:ARC-Easy:124", |
| "arc:validation:ARC-Easy:171", |
| "arc:validation:ARC-Easy:235", |
| "arc:validation:ARC-Easy:241", |
| "arc:validation:ARC-Easy:279", |
| "arc:validation:ARC-Easy:289", |
| "arc:validation:ARC-Easy:37", |
| "arc:validation:ARC-Easy:389", |
| "arc:validation:ARC-Easy:394", |
| "arc:validation:ARC-Easy:399", |
| "arc:validation:ARC-Easy:423", |
| "arc:validation:ARC-Easy:492", |
| "arc:validation:ARC-Easy:499", |
| "arc:validation:ARC-Easy:523", |
| "arc:validation:ARC-Easy:533", |
| "arc:validation:ARC-Easy:569", |
| "arc:validation:ARC-Easy:69", |
| "arc:validation:ARC-Easy:93", |
| "banking:train:2049", |
| "banking:train:2350", |
| "banking:train:2497", |
| "banking:train:2506", |
| "banking:train:2720", |
| "banking:train:2788", |
| "banking:train:2967", |
| "banking:train:3502", |
| "banking:train:3726", |
| "banking:train:4014", |
| "banking:train:4735", |
| "banking:train:4803", |
| "banking:train:4917", |
| "banking:train:5477", |
| "banking:train:5851", |
| "banking:train:6551", |
| "banking:train:6839", |
| "banking:train:6926", |
| "banking:train:7809", |
| "banking:train:8092", |
| "banking:train:8186", |
| "banking:train:8215", |
| "banking:train:8667", |
| "banking:train:8752", |
| "banking:train:8763", |
| "banking:train:8851", |
| "banking:train:8917", |
| "banking:train:9145", |
| "banking:train:9279", |
| "banking:train:9676", |
| "banking:train:9837", |
| "banking:train:9967", |
| "boolq:validation:106", |
| "boolq:validation:1065", |
| "boolq:validation:1097", |
| "boolq:validation:1336", |
| "boolq:validation:1350", |
| "boolq:validation:1383", |
| "boolq:validation:1458", |
| "boolq:validation:1641", |
| "boolq:validation:1873", |
| "boolq:validation:1930", |
| "boolq:validation:1989", |
| "boolq:validation:2103", |
| "boolq:validation:234", |
| "boolq:validation:2355", |
| "boolq:validation:2384", |
| "boolq:validation:2440", |
| "boolq:validation:2444", |
| "boolq:validation:2478", |
| "boolq:validation:2872", |
| "boolq:validation:2879", |
| "boolq:validation:2924", |
| "boolq:validation:2971", |
| "boolq:validation:3164", |
| "boolq:validation:4", |
| "boolq:validation:415", |
| "boolq:validation:419", |
| "boolq:validation:428", |
| "boolq:validation:512", |
| "boolq:validation:563", |
| "boolq:validation:643", |
| "boolq:validation:696", |
| "boolq:validation:874", |
| "snli:validation:1102", |
| "snli:validation:1240", |
| "snli:validation:1261", |
| "snli:validation:1738", |
| "snli:validation:1978", |
| "snli:validation:2563", |
| "snli:validation:2659", |
| "snli:validation:2767", |
| "snli:validation:2836", |
| "snli:validation:3232", |
| "snli:validation:3655", |
| "snli:validation:3913", |
| "snli:validation:405", |
| "snli:validation:4051", |
| "snli:validation:4300", |
| "snli:validation:4846", |
| "snli:validation:498", |
| "snli:validation:5224", |
| "snli:validation:5575", |
| "snli:validation:6598", |
| "snli:validation:666", |
| "snli:validation:6925", |
| "snli:validation:6964", |
| "snli:validation:7045", |
| "snli:validation:7351", |
| "snli:validation:7588", |
| "snli:validation:7723", |
| "snli:validation:7750", |
| "snli:validation:7909", |
| "snli:validation:8974", |
| "snli:validation:9385", |
| "snli:validation:9646" |
| ], |
| "heldout_group_count": 128, |
| "training_group_count": 384 |
| } |
| ], |
| "per_family": { |
| "arc": { |
| "count": 128, |
| "score": 0.45409842583654353, |
| "raw_macro_nll": 0.7961190826926177, |
| "accuracy": 0.8515625 |
| }, |
| "banking": { |
| "count": 128, |
| "score": 0.055047587673440465, |
| "raw_macro_nll": 0.09841930191488779, |
| "accuracy": 0.9765625 |
| }, |
| "boolq": { |
| "count": 128, |
| "score": 0.3088651912550437, |
| "raw_macro_nll": 0.5010539159559688, |
| "accuracy": 0.8515625 |
| }, |
| "snli": { |
| "count": 128, |
| "score": 0.318468515323601, |
| "raw_macro_nll": 0.3437720954556061, |
| "accuracy": 0.9140625 |
| } |
| }, |
| "policy": { |
| "id": "crossfit_temperature_nll_v1", |
| "folds": 4, |
| "seed": 431, |
| "grouping": "source group, nested within task family; groups shuffled within sorted families and assigned round-robin", |
| "temperature_grid": { |
| "count": 101, |
| "log10_min": -1.0, |
| "log10_max": 1.3, |
| "arithmetic": "Python float64" |
| }, |
| "temperature_fit": "minimum macro-family NLL on the other three folds; smallest temperature wins ties", |
| "score": "macro-family mean of all held-out decision NLLs", |
| "deployment_temperature": "fit afresh on reserved calibration only after model selection" |
| } |
| }, |
| "best_selection_score": 0.2552943737691716, |
| "improved": false, |
| "metrics": { |
| "overall": { |
| "n": 512, |
| "accuracy": 0.8984375, |
| "nll": 0.4348410999061469, |
| "brier_multiclass_sum": 0.16255039658970658, |
| "ece_top_label_10_equal_width_bins": 0.0713859559618868, |
| "reliability_bins": [ |
| { |
| "count": 0, |
| "confidence_sum": 0.0, |
| "correct_sum": 0.0 |
| }, |
| { |
| "count": 0, |
| "confidence_sum": 0.0, |
| "correct_sum": 0.0 |
| }, |
| { |
| "count": 0, |
| "confidence_sum": 0.0, |
| "correct_sum": 0.0 |
| }, |
| { |
| "count": 0, |
| "confidence_sum": 0.0, |
| "correct_sum": 0.0 |
| }, |
| { |
| "count": 1, |
| "confidence_sum": 0.4954971969127655, |
| "correct_sum": 1.0 |
| }, |
| { |
| "count": 13, |
| "confidence_sum": 7.076289176940918, |
| "correct_sum": 4.0 |
| }, |
| { |
| "count": 9, |
| "confidence_sum": 5.850665271282196, |
| "correct_sum": 3.0 |
| }, |
| { |
| "count": 4, |
| "confidence_sum": 2.993383467197418, |
| "correct_sum": 2.0 |
| }, |
| { |
| "count": 20, |
| "confidence_sum": 17.163459360599518, |
| "correct_sum": 14.0 |
| }, |
| { |
| "count": 465, |
| "confidence_sum": 461.96130937337875, |
| "correct_sum": 436.0 |
| } |
| ], |
| "accuracy_vs_coverage": { |
| "0.25": { |
| "n": 128, |
| "accuracy": 1.0, |
| "min_confidence": 1.0 |
| }, |
| "0.5": { |
| "n": 256, |
| "accuracy": 0.99609375, |
| "min_confidence": 0.9998538494110107 |
| }, |
| "0.75": { |
| "n": 384, |
| "accuracy": 0.9713541666666666, |
| "min_confidence": 0.989825427532196 |
| }, |
| "1.0": { |
| "n": 512, |
| "accuracy": 0.8984375, |
| "min_confidence": 0.4954971969127655 |
| } |
| } |
| }, |
| "per_family": { |
| "arc": { |
| "n": 128, |
| "accuracy": 0.8515625, |
| "nll": 0.7961190888847145, |
| "brier_multiclass_sum": 0.2236221681839995, |
| "ece_top_label_10_equal_width_bins": 0.11635206383652985, |
| "reliability_bins": [ |
| { |
| "count": 0, |
| "confidence_sum": 0.0, |
| "correct_sum": 0.0 |
| }, |
| { |
| "count": 0, |
| "confidence_sum": 0.0, |
| "correct_sum": 0.0 |
| }, |
| { |
| "count": 0, |
| "confidence_sum": 0.0, |
| "correct_sum": 0.0 |
| }, |
| { |
| "count": 0, |
| "confidence_sum": 0.0, |
| "correct_sum": 0.0 |
| }, |
| { |
| "count": 1, |
| "confidence_sum": 0.4954971969127655, |
| "correct_sum": 1.0 |
| }, |
| { |
| "count": 6, |
| "confidence_sum": 3.2659605145454407, |
| "correct_sum": 0.0 |
| }, |
| { |
| "count": 1, |
| "confidence_sum": 0.6727883815765381, |
| "correct_sum": 0.0 |
| }, |
| { |
| "count": 2, |
| "confidence_sum": 1.5027006268501282, |
| "correct_sum": 2.0 |
| }, |
| { |
| "count": 10, |
| "confidence_sum": 8.412083506584167, |
| "correct_sum": 6.0 |
| }, |
| { |
| "count": 108, |
| "confidence_sum": 107.54042959213257, |
| "correct_sum": 100.0 |
| } |
| ], |
| "accuracy_vs_coverage": { |
| "0.25": { |
| "n": 32, |
| "accuracy": 1.0, |
| "min_confidence": 1.0 |
| }, |
| "0.5": { |
| "n": 64, |
| "accuracy": 0.984375, |
| "min_confidence": 0.9999865293502808 |
| }, |
| "0.75": { |
| "n": 96, |
| "accuracy": 0.9583333333333334, |
| "min_confidence": 0.985666811466217 |
| }, |
| "1.0": { |
| "n": 128, |
| "accuracy": 0.8515625, |
| "min_confidence": 0.4954971969127655 |
| } |
| } |
| }, |
| "banking": { |
| "n": 128, |
| "accuracy": 0.9765625, |
| "nll": 0.09841930049332331, |
| "brier_multiclass_sum": 0.042800634513240954, |
| "ece_top_label_10_equal_width_bins": 0.020601469557732344, |
| "reliability_bins": [ |
| { |
| "count": 0, |
| "confidence_sum": 0.0, |
| "correct_sum": 0.0 |
| }, |
| { |
| "count": 0, |
| "confidence_sum": 0.0, |
| "correct_sum": 0.0 |
| }, |
| { |
| "count": 0, |
| "confidence_sum": 0.0, |
| "correct_sum": 0.0 |
| }, |
| { |
| "count": 0, |
| "confidence_sum": 0.0, |
| "correct_sum": 0.0 |
| }, |
| { |
| "count": 0, |
| "confidence_sum": 0.0, |
| "correct_sum": 0.0 |
| }, |
| { |
| "count": 0, |
| "confidence_sum": 0.0, |
| "correct_sum": 0.0 |
| }, |
| { |
| "count": 0, |
| "confidence_sum": 0.0, |
| "correct_sum": 0.0 |
| }, |
| { |
| "count": 0, |
| "confidence_sum": 0.0, |
| "correct_sum": 0.0 |
| }, |
| { |
| "count": 1, |
| "confidence_sum": 0.8777487277984619, |
| "correct_sum": 0.0 |
| }, |
| { |
| "count": 127, |
| "confidence_sum": 126.75923937559128, |
| "correct_sum": 125.0 |
| } |
| ], |
| "accuracy_vs_coverage": { |
| "0.25": { |
| "n": 32, |
| "accuracy": 1.0, |
| "min_confidence": 1.0 |
| }, |
| "0.5": { |
| "n": 64, |
| "accuracy": 1.0, |
| "min_confidence": 1.0 |
| }, |
| "0.75": { |
| "n": 96, |
| "accuracy": 1.0, |
| "min_confidence": 1.0 |
| }, |
| "1.0": { |
| "n": 128, |
| "accuracy": 0.9765625, |
| "min_confidence": 0.8777487277984619 |
| } |
| } |
| }, |
| "boolq": { |
| "n": 128, |
| "accuracy": 0.8515625, |
| "nll": 0.5010539169356036, |
| "brier_multiclass_sum": 0.22973198097830894, |
| "ece_top_label_10_equal_width_bins": 0.10703934449702501, |
| "reliability_bins": [ |
| { |
| "count": 0, |
| "confidence_sum": 0.0, |
| "correct_sum": 0.0 |
| }, |
| { |
| "count": 0, |
| "confidence_sum": 0.0, |
| "correct_sum": 0.0 |
| }, |
| { |
| "count": 0, |
| "confidence_sum": 0.0, |
| "correct_sum": 0.0 |
| }, |
| { |
| "count": 0, |
| "confidence_sum": 0.0, |
| "correct_sum": 0.0 |
| }, |
| { |
| "count": 0, |
| "confidence_sum": 0.0, |
| "correct_sum": 0.0 |
| }, |
| { |
| "count": 6, |
| "confidence_sum": 3.307970881462097, |
| "correct_sum": 3.0 |
| }, |
| { |
| "count": 5, |
| "confidence_sum": 3.20867782831192, |
| "correct_sum": 2.0 |
| }, |
| { |
| "count": 2, |
| "confidence_sum": 1.49068284034729, |
| "correct_sum": 0.0 |
| }, |
| { |
| "count": 3, |
| "confidence_sum": 2.652133047580719, |
| "correct_sum": 3.0 |
| }, |
| { |
| "count": 112, |
| "confidence_sum": 111.34583759307861, |
| "correct_sum": 101.0 |
| } |
| ], |
| "accuracy_vs_coverage": { |
| "0.25": { |
| "n": 32, |
| "accuracy": 1.0, |
| "min_confidence": 0.9999068975448608 |
| }, |
| "0.5": { |
| "n": 64, |
| "accuracy": 1.0, |
| "min_confidence": 0.9995142221450806 |
| }, |
| "0.75": { |
| "n": 96, |
| "accuracy": 0.9583333333333334, |
| "min_confidence": 0.9929304122924805 |
| }, |
| "1.0": { |
| "n": 128, |
| "accuracy": 0.8515625, |
| "min_confidence": 0.5254567861557007 |
| } |
| } |
| }, |
| "snli": { |
| "n": 128, |
| "accuracy": 0.9140625, |
| "nll": 0.34377209331094605, |
| "brier_multiclass_sum": 0.15404680268327697, |
| "ece_top_label_10_equal_width_bins": 0.06253232946619391, |
| "reliability_bins": [ |
| { |
| "count": 0, |
| "confidence_sum": 0.0, |
| "correct_sum": 0.0 |
| }, |
| { |
| "count": 0, |
| "confidence_sum": 0.0, |
| "correct_sum": 0.0 |
| }, |
| { |
| "count": 0, |
| "confidence_sum": 0.0, |
| "correct_sum": 0.0 |
| }, |
| { |
| "count": 0, |
| "confidence_sum": 0.0, |
| "correct_sum": 0.0 |
| }, |
| { |
| "count": 0, |
| "confidence_sum": 0.0, |
| "correct_sum": 0.0 |
| }, |
| { |
| "count": 1, |
| "confidence_sum": 0.5023577809333801, |
| "correct_sum": 1.0 |
| }, |
| { |
| "count": 3, |
| "confidence_sum": 1.9691990613937378, |
| "correct_sum": 1.0 |
| }, |
| { |
| "count": 0, |
| "confidence_sum": 0.0, |
| "correct_sum": 0.0 |
| }, |
| { |
| "count": 6, |
| "confidence_sum": 5.221494078636169, |
| "correct_sum": 5.0 |
| }, |
| { |
| "count": 118, |
| "confidence_sum": 116.3158028125763, |
| "correct_sum": 110.0 |
| } |
| ], |
| "accuracy_vs_coverage": { |
| "0.25": { |
| "n": 32, |
| "accuracy": 1.0, |
| "min_confidence": 0.9988596439361572 |
| }, |
| "0.5": { |
| "n": 64, |
| "accuracy": 1.0, |
| "min_confidence": 0.9949784874916077 |
| }, |
| "0.75": { |
| "n": 96, |
| "accuracy": 0.9479166666666666, |
| "min_confidence": 0.9736598134040833 |
| }, |
| "1.0": { |
| "n": 128, |
| "accuracy": 0.9140625, |
| "min_confidence": 0.5023577809333801 |
| } |
| } |
| } |
| } |
| } |
| }, |
| { |
| "step": 5000, |
| "macro_nll": 0.2879291938128496, |
| "best_macro_nll": 0.25676974726429014, |
| "selection": { |
| "metric": "crossfit_temperature_nll_v1", |
| "score": 0.279832908363464, |
| "raw_macro_nll": 0.2879291926537506, |
| "accuracy": 0.90625, |
| "folds": [ |
| { |
| "fold": 0, |
| "temperature_index": 48, |
| "temperature": 1.2705741052085413, |
| "training_macro_nll": 0.2949829510321852, |
| "heldout_ids": [ |
| "arc:validation:ARC-Challenge:122", |
| "arc:validation:ARC-Challenge:126", |
| "arc:validation:ARC-Challenge:175", |
| "arc:validation:ARC-Challenge:187", |
| "arc:validation:ARC-Challenge:189", |
| "arc:validation:ARC-Challenge:192", |
| "arc:validation:ARC-Challenge:208", |
| "arc:validation:ARC-Challenge:238", |
| "arc:validation:ARC-Challenge:290", |
| "arc:validation:ARC-Challenge:293", |
| "arc:validation:ARC-Challenge:35", |
| "arc:validation:ARC-Challenge:43", |
| "arc:validation:ARC-Challenge:58", |
| "arc:validation:ARC-Challenge:7", |
| "arc:validation:ARC-Easy:160", |
| "arc:validation:ARC-Easy:201", |
| "arc:validation:ARC-Easy:229", |
| "arc:validation:ARC-Easy:238", |
| "arc:validation:ARC-Easy:253", |
| "arc:validation:ARC-Easy:259", |
| "arc:validation:ARC-Easy:272", |
| "arc:validation:ARC-Easy:298", |
| "arc:validation:ARC-Easy:329", |
| "arc:validation:ARC-Easy:331", |
| "arc:validation:ARC-Easy:337", |
| "arc:validation:ARC-Easy:362", |
| "arc:validation:ARC-Easy:443", |
| "arc:validation:ARC-Easy:500", |
| "arc:validation:ARC-Easy:545", |
| "arc:validation:ARC-Easy:68", |
| "arc:validation:ARC-Easy:7", |
| "arc:validation:ARC-Easy:74", |
| "banking:train:1053", |
| "banking:train:1594", |
| "banking:train:1747", |
| "banking:train:202", |
| "banking:train:2377", |
| "banking:train:2528", |
| "banking:train:2546", |
| "banking:train:2715", |
| "banking:train:3220", |
| "banking:train:3499", |
| "banking:train:3806", |
| "banking:train:3838", |
| "banking:train:3993", |
| "banking:train:4220", |
| "banking:train:5", |
| "banking:train:5192", |
| "banking:train:5394", |
| "banking:train:5846", |
| "banking:train:5863", |
| "banking:train:6032", |
| "banking:train:6508", |
| "banking:train:7579", |
| "banking:train:7600", |
| "banking:train:7987", |
| "banking:train:8155", |
| "banking:train:850", |
| "banking:train:8626", |
| "banking:train:8929", |
| "banking:train:9355", |
| "banking:train:9812", |
| "banking:train:987", |
| "banking:train:9972", |
| "boolq:validation:103", |
| "boolq:validation:1042", |
| "boolq:validation:1059", |
| "boolq:validation:1541", |
| "boolq:validation:1592", |
| "boolq:validation:1927", |
| "boolq:validation:2014", |
| "boolq:validation:2023", |
| "boolq:validation:2098", |
| "boolq:validation:2139", |
| "boolq:validation:2439", |
| "boolq:validation:2485", |
| "boolq:validation:2607", |
| "boolq:validation:2611", |
| "boolq:validation:2720", |
| "boolq:validation:2751", |
| "boolq:validation:282", |
| "boolq:validation:2843", |
| "boolq:validation:2953", |
| "boolq:validation:301", |
| "boolq:validation:3089", |
| "boolq:validation:313", |
| "boolq:validation:3201", |
| "boolq:validation:322", |
| "boolq:validation:3231", |
| "boolq:validation:39", |
| "boolq:validation:461", |
| "boolq:validation:558", |
| "boolq:validation:589", |
| "boolq:validation:731", |
| "boolq:validation:883", |
| "boolq:validation:915", |
| "snli:validation:1156", |
| "snli:validation:1330", |
| "snli:validation:1559", |
| "snli:validation:1735", |
| "snli:validation:1876", |
| "snli:validation:2251", |
| "snli:validation:2257", |
| "snli:validation:2377", |
| "snli:validation:2881", |
| "snli:validation:291", |
| "snli:validation:300", |
| "snli:validation:3049", |
| "snli:validation:3496", |
| "snli:validation:3871", |
| "snli:validation:4414", |
| "snli:validation:5140", |
| "snli:validation:5440", |
| "snli:validation:5806", |
| "snli:validation:6442", |
| "snli:validation:6772", |
| "snli:validation:6949", |
| "snli:validation:7228", |
| "snli:validation:7465", |
| "snli:validation:7510", |
| "snli:validation:7726", |
| "snli:validation:8095", |
| "snli:validation:8143", |
| "snli:validation:8623", |
| "snli:validation:8962", |
| "snli:validation:9451", |
| "snli:validation:9574", |
| "snli:validation:9655" |
| ], |
| "heldout_group_count": 128, |
| "training_group_count": 384 |
| }, |
| { |
| "fold": 1, |
| "temperature_index": 48, |
| "temperature": 1.2705741052085413, |
| "training_macro_nll": 0.26910043519983107, |
| "heldout_ids": [ |
| "arc:validation:ARC-Challenge:101", |
| "arc:validation:ARC-Challenge:129", |
| "arc:validation:ARC-Challenge:148", |
| "arc:validation:ARC-Challenge:156", |
| "arc:validation:ARC-Challenge:167", |
| "arc:validation:ARC-Challenge:249", |
| "arc:validation:ARC-Challenge:263", |
| "arc:validation:ARC-Challenge:266", |
| "arc:validation:ARC-Challenge:267", |
| "arc:validation:ARC-Challenge:28", |
| "arc:validation:ARC-Challenge:281", |
| "arc:validation:ARC-Challenge:57", |
| "arc:validation:ARC-Challenge:63", |
| "arc:validation:ARC-Challenge:64", |
| "arc:validation:ARC-Challenge:86", |
| "arc:validation:ARC-Challenge:99", |
| "arc:validation:ARC-Easy:106", |
| "arc:validation:ARC-Easy:151", |
| "arc:validation:ARC-Easy:157", |
| "arc:validation:ARC-Easy:166", |
| "arc:validation:ARC-Easy:202", |
| "arc:validation:ARC-Easy:219", |
| "arc:validation:ARC-Easy:224", |
| "arc:validation:ARC-Easy:322", |
| "arc:validation:ARC-Easy:344", |
| "arc:validation:ARC-Easy:382", |
| "arc:validation:ARC-Easy:409", |
| "arc:validation:ARC-Easy:455", |
| "arc:validation:ARC-Easy:50", |
| "arc:validation:ARC-Easy:538", |
| "arc:validation:ARC-Easy:544", |
| "arc:validation:ARC-Easy:552", |
| "banking:train:1050", |
| "banking:train:1131", |
| "banking:train:1190", |
| "banking:train:124", |
| "banking:train:137", |
| "banking:train:1493", |
| "banking:train:1498", |
| "banking:train:1624", |
| "banking:train:2614", |
| "banking:train:2739", |
| "banking:train:3055", |
| "banking:train:306", |
| "banking:train:3312", |
| "banking:train:3724", |
| "banking:train:4070", |
| "banking:train:476", |
| "banking:train:4882", |
| "banking:train:5075", |
| "banking:train:5332", |
| "banking:train:5445", |
| "banking:train:6096", |
| "banking:train:746", |
| "banking:train:7828", |
| "banking:train:8176", |
| "banking:train:8315", |
| "banking:train:8653", |
| "banking:train:9019", |
| "banking:train:9370", |
| "banking:train:9454", |
| "banking:train:9595", |
| "banking:train:9862", |
| "banking:train:9963", |
| "boolq:validation:1189", |
| "boolq:validation:1320", |
| "boolq:validation:1326", |
| "boolq:validation:1381", |
| "boolq:validation:1410", |
| "boolq:validation:149", |
| "boolq:validation:1513", |
| "boolq:validation:1732", |
| "boolq:validation:1739", |
| "boolq:validation:1741", |
| "boolq:validation:1814", |
| "boolq:validation:1848", |
| "boolq:validation:1940", |
| "boolq:validation:2077", |
| "boolq:validation:2091", |
| "boolq:validation:2148", |
| "boolq:validation:2215", |
| "boolq:validation:2251", |
| "boolq:validation:2283", |
| "boolq:validation:2321", |
| "boolq:validation:2398", |
| "boolq:validation:253", |
| "boolq:validation:2575", |
| "boolq:validation:2907", |
| "boolq:validation:2909", |
| "boolq:validation:2978", |
| "boolq:validation:359", |
| "boolq:validation:394", |
| "boolq:validation:436", |
| "boolq:validation:579", |
| "boolq:validation:807", |
| "boolq:validation:949", |
| "snli:validation:2389", |
| "snli:validation:2485", |
| "snli:validation:255", |
| "snli:validation:3169", |
| "snli:validation:333", |
| "snli:validation:3700", |
| "snli:validation:3709", |
| "snli:validation:4093", |
| "snli:validation:420", |
| "snli:validation:4492", |
| "snli:validation:4516", |
| "snli:validation:4648", |
| "snli:validation:4888", |
| "snli:validation:5185", |
| "snli:validation:5332", |
| "snli:validation:5947", |
| "snli:validation:6124", |
| "snli:validation:660", |
| "snli:validation:6619", |
| "snli:validation:6898", |
| "snli:validation:7387", |
| "snli:validation:7492", |
| "snli:validation:7544", |
| "snli:validation:7996", |
| "snli:validation:8074", |
| "snli:validation:8554", |
| "snli:validation:8971", |
| "snli:validation:9163", |
| "snli:validation:9277", |
| "snli:validation:9490", |
| "snli:validation:9595", |
| "snli:validation:9688" |
| ], |
| "heldout_group_count": 128, |
| "training_group_count": 384 |
| }, |
| { |
| "fold": 2, |
| "temperature_index": 47, |
| "temperature": 1.2050359403717974, |
| "training_macro_nll": 0.25473528099447806, |
| "heldout_ids": [ |
| "arc:validation:ARC-Challenge:120", |
| "arc:validation:ARC-Challenge:130", |
| "arc:validation:ARC-Challenge:194", |
| "arc:validation:ARC-Challenge:218", |
| "arc:validation:ARC-Challenge:235", |
| "arc:validation:ARC-Challenge:272", |
| "arc:validation:ARC-Challenge:277", |
| "arc:validation:ARC-Challenge:48", |
| "arc:validation:ARC-Challenge:88", |
| "arc:validation:ARC-Easy:115", |
| "arc:validation:ARC-Easy:12", |
| "arc:validation:ARC-Easy:131", |
| "arc:validation:ARC-Easy:145", |
| "arc:validation:ARC-Easy:179", |
| "arc:validation:ARC-Easy:18", |
| "arc:validation:ARC-Easy:186", |
| "arc:validation:ARC-Easy:191", |
| "arc:validation:ARC-Easy:215", |
| "arc:validation:ARC-Easy:283", |
| "arc:validation:ARC-Easy:314", |
| "arc:validation:ARC-Easy:315", |
| "arc:validation:ARC-Easy:378", |
| "arc:validation:ARC-Easy:384", |
| "arc:validation:ARC-Easy:403", |
| "arc:validation:ARC-Easy:432", |
| "arc:validation:ARC-Easy:439", |
| "arc:validation:ARC-Easy:448", |
| "arc:validation:ARC-Easy:461", |
| "arc:validation:ARC-Easy:469", |
| "arc:validation:ARC-Easy:470", |
| "arc:validation:ARC-Easy:504", |
| "arc:validation:ARC-Easy:561", |
| "banking:train:1142", |
| "banking:train:148", |
| "banking:train:1633", |
| "banking:train:1927", |
| "banking:train:2186", |
| "banking:train:2251", |
| "banking:train:23", |
| "banking:train:3048", |
| "banking:train:3665", |
| "banking:train:3952", |
| "banking:train:3999", |
| "banking:train:4364", |
| "banking:train:4619", |
| "banking:train:5049", |
| "banking:train:5335", |
| "banking:train:549", |
| "banking:train:5957", |
| "banking:train:5985", |
| "banking:train:6132", |
| "banking:train:6240", |
| "banking:train:6488", |
| "banking:train:7065", |
| "banking:train:7749", |
| "banking:train:8470", |
| "banking:train:8761", |
| "banking:train:895", |
| "banking:train:9256", |
| "banking:train:9406", |
| "banking:train:9450", |
| "banking:train:9467", |
| "banking:train:9588", |
| "banking:train:9716", |
| "boolq:validation:1130", |
| "boolq:validation:1167", |
| "boolq:validation:124", |
| "boolq:validation:1255", |
| "boolq:validation:1256", |
| "boolq:validation:1373", |
| "boolq:validation:15", |
| "boolq:validation:155", |
| "boolq:validation:1568", |
| "boolq:validation:16", |
| "boolq:validation:1654", |
| "boolq:validation:1702", |
| "boolq:validation:1792", |
| "boolq:validation:1818", |
| "boolq:validation:210", |
| "boolq:validation:2247", |
| "boolq:validation:2259", |
| "boolq:validation:2362", |
| "boolq:validation:2443", |
| "boolq:validation:2524", |
| "boolq:validation:2553", |
| "boolq:validation:2623", |
| "boolq:validation:2660", |
| "boolq:validation:2669", |
| "boolq:validation:2679", |
| "boolq:validation:2832", |
| "boolq:validation:357", |
| "boolq:validation:453", |
| "boolq:validation:49", |
| "boolq:validation:808", |
| "boolq:validation:90", |
| "boolq:validation:910", |
| "snli:validation:1612", |
| "snli:validation:168", |
| "snli:validation:1693", |
| "snli:validation:2134", |
| "snli:validation:2248", |
| "snli:validation:2422", |
| "snli:validation:2587", |
| "snli:validation:3007", |
| "snli:validation:3097", |
| "snli:validation:3772", |
| "snli:validation:4012", |
| "snli:validation:4168", |
| "snli:validation:4537", |
| "snli:validation:4681", |
| "snli:validation:5050", |
| "snli:validation:5632", |
| "snli:validation:5761", |
| "snli:validation:6055", |
| "snli:validation:6283", |
| "snli:validation:6715", |
| "snli:validation:7117", |
| "snli:validation:7192", |
| "snli:validation:7507", |
| "snli:validation:7594", |
| "snli:validation:7924", |
| "snli:validation:7954", |
| "snli:validation:8035", |
| "snli:validation:8110", |
| "snli:validation:8995", |
| "snli:validation:949", |
| "snli:validation:9523", |
| "snli:validation:9589" |
| ], |
| "heldout_group_count": 128, |
| "training_group_count": 384 |
| }, |
| { |
| "fold": 3, |
| "temperature_index": 49, |
| "temperature": 1.3396766874259345, |
| "training_macro_nll": 0.2946301125911753, |
| "heldout_ids": [ |
| "arc:validation:ARC-Challenge:117", |
| "arc:validation:ARC-Challenge:135", |
| "arc:validation:ARC-Challenge:139", |
| "arc:validation:ARC-Challenge:154", |
| "arc:validation:ARC-Challenge:174", |
| "arc:validation:ARC-Challenge:199", |
| "arc:validation:ARC-Challenge:287", |
| "arc:validation:ARC-Challenge:295", |
| "arc:validation:ARC-Challenge:4", |
| "arc:validation:ARC-Challenge:50", |
| "arc:validation:ARC-Challenge:51", |
| "arc:validation:ARC-Challenge:61", |
| "arc:validation:ARC-Easy:107", |
| "arc:validation:ARC-Easy:121", |
| "arc:validation:ARC-Easy:124", |
| "arc:validation:ARC-Easy:171", |
| "arc:validation:ARC-Easy:235", |
| "arc:validation:ARC-Easy:241", |
| "arc:validation:ARC-Easy:279", |
| "arc:validation:ARC-Easy:289", |
| "arc:validation:ARC-Easy:37", |
| "arc:validation:ARC-Easy:389", |
| "arc:validation:ARC-Easy:394", |
| "arc:validation:ARC-Easy:399", |
| "arc:validation:ARC-Easy:423", |
| "arc:validation:ARC-Easy:492", |
| "arc:validation:ARC-Easy:499", |
| "arc:validation:ARC-Easy:523", |
| "arc:validation:ARC-Easy:533", |
| "arc:validation:ARC-Easy:569", |
| "arc:validation:ARC-Easy:69", |
| "arc:validation:ARC-Easy:93", |
| "banking:train:2049", |
| "banking:train:2350", |
| "banking:train:2497", |
| "banking:train:2506", |
| "banking:train:2720", |
| "banking:train:2788", |
| "banking:train:2967", |
| "banking:train:3502", |
| "banking:train:3726", |
| "banking:train:4014", |
| "banking:train:4735", |
| "banking:train:4803", |
| "banking:train:4917", |
| "banking:train:5477", |
| "banking:train:5851", |
| "banking:train:6551", |
| "banking:train:6839", |
| "banking:train:6926", |
| "banking:train:7809", |
| "banking:train:8092", |
| "banking:train:8186", |
| "banking:train:8215", |
| "banking:train:8667", |
| "banking:train:8752", |
| "banking:train:8763", |
| "banking:train:8851", |
| "banking:train:8917", |
| "banking:train:9145", |
| "banking:train:9279", |
| "banking:train:9676", |
| "banking:train:9837", |
| "banking:train:9967", |
| "boolq:validation:106", |
| "boolq:validation:1065", |
| "boolq:validation:1097", |
| "boolq:validation:1336", |
| "boolq:validation:1350", |
| "boolq:validation:1383", |
| "boolq:validation:1458", |
| "boolq:validation:1641", |
| "boolq:validation:1873", |
| "boolq:validation:1930", |
| "boolq:validation:1989", |
| "boolq:validation:2103", |
| "boolq:validation:234", |
| "boolq:validation:2355", |
| "boolq:validation:2384", |
| "boolq:validation:2440", |
| "boolq:validation:2444", |
| "boolq:validation:2478", |
| "boolq:validation:2872", |
| "boolq:validation:2879", |
| "boolq:validation:2924", |
| "boolq:validation:2971", |
| "boolq:validation:3164", |
| "boolq:validation:4", |
| "boolq:validation:415", |
| "boolq:validation:419", |
| "boolq:validation:428", |
| "boolq:validation:512", |
| "boolq:validation:563", |
| "boolq:validation:643", |
| "boolq:validation:696", |
| "boolq:validation:874", |
| "snli:validation:1102", |
| "snli:validation:1240", |
| "snli:validation:1261", |
| "snli:validation:1738", |
| "snli:validation:1978", |
| "snli:validation:2563", |
| "snli:validation:2659", |
| "snli:validation:2767", |
| "snli:validation:2836", |
| "snli:validation:3232", |
| "snli:validation:3655", |
| "snli:validation:3913", |
| "snli:validation:405", |
| "snli:validation:4051", |
| "snli:validation:4300", |
| "snli:validation:4846", |
| "snli:validation:498", |
| "snli:validation:5224", |
| "snli:validation:5575", |
| "snli:validation:6598", |
| "snli:validation:666", |
| "snli:validation:6925", |
| "snli:validation:6964", |
| "snli:validation:7045", |
| "snli:validation:7351", |
| "snli:validation:7588", |
| "snli:validation:7723", |
| "snli:validation:7750", |
| "snli:validation:7909", |
| "snli:validation:8974", |
| "snli:validation:9385", |
| "snli:validation:9646" |
| ], |
| "heldout_group_count": 128, |
| "training_group_count": 384 |
| } |
| ], |
| "per_family": { |
| "arc": { |
| "count": 128, |
| "score": 0.4283105853506469, |
| "raw_macro_nll": 0.4595502587159514, |
| "accuracy": 0.859375 |
| }, |
| "banking": { |
| "count": 128, |
| "score": 0.05047459421857176, |
| "raw_macro_nll": 0.0495573852832617, |
| "accuracy": 0.9921875 |
| }, |
| "boolq": { |
| "count": 128, |
| "score": 0.3196884642718922, |
| "raw_macro_nll": 0.3320905225271662, |
| "accuracy": 0.8828125 |
| }, |
| "snli": { |
| "count": 128, |
| "score": 0.32085798961274503, |
| "raw_macro_nll": 0.3105186040886232, |
| "accuracy": 0.890625 |
| } |
| }, |
| "policy": { |
| "id": "crossfit_temperature_nll_v1", |
| "folds": 4, |
| "seed": 431, |
| "grouping": "source group, nested within task family; groups shuffled within sorted families and assigned round-robin", |
| "temperature_grid": { |
| "count": 101, |
| "log10_min": -1.0, |
| "log10_max": 1.3, |
| "arithmetic": "Python float64" |
| }, |
| "temperature_fit": "minimum macro-family NLL on the other three folds; smallest temperature wins ties", |
| "score": "macro-family mean of all held-out decision NLLs", |
| "deployment_temperature": "fit afresh on reserved calibration only after model selection" |
| } |
| }, |
| "best_selection_score": 0.2552943737691716, |
| "improved": false, |
| "metrics": { |
| "overall": { |
| "n": 512, |
| "accuracy": 0.90625, |
| "nll": 0.2879291938128496, |
| "brier_multiclass_sum": 0.1459957167817959, |
| "ece_top_label_10_equal_width_bins": 0.03560248663416132, |
| "reliability_bins": [ |
| { |
| "count": 0, |
| "confidence_sum": 0.0, |
| "correct_sum": 0.0 |
| }, |
| { |
| "count": 0, |
| "confidence_sum": 0.0, |
| "correct_sum": 0.0 |
| }, |
| { |
| "count": 0, |
| "confidence_sum": 0.0, |
| "correct_sum": 0.0 |
| }, |
| { |
| "count": 2, |
| "confidence_sum": 0.7507493495941162, |
| "correct_sum": 0.0 |
| }, |
| { |
| "count": 6, |
| "confidence_sum": 2.8289709389209747, |
| "correct_sum": 1.0 |
| }, |
| { |
| "count": 12, |
| "confidence_sum": 6.604806303977966, |
| "correct_sum": 8.0 |
| }, |
| { |
| "count": 22, |
| "confidence_sum": 14.299658298492432, |
| "correct_sum": 17.0 |
| }, |
| { |
| "count": 24, |
| "confidence_sum": 18.386729657649994, |
| "correct_sum": 16.0 |
| }, |
| { |
| "count": 46, |
| "confidence_sum": 39.259098529815674, |
| "correct_sum": 33.0 |
| }, |
| { |
| "count": 400, |
| "confidence_sum": 391.90738928318024, |
| "correct_sum": 389.0 |
| } |
| ], |
| "accuracy_vs_coverage": { |
| "0.25": { |
| "n": 128, |
| "accuracy": 1.0, |
| "min_confidence": 0.9997970461845398 |
| }, |
| "0.5": { |
| "n": 256, |
| "accuracy": 0.98828125, |
| "min_confidence": 0.9788828492164612 |
| }, |
| "0.75": { |
| "n": 384, |
| "accuracy": 0.9739583333333334, |
| "min_confidence": 0.9226401448249817 |
| }, |
| "1.0": { |
| "n": 512, |
| "accuracy": 0.90625, |
| "min_confidence": 0.36794614791870117 |
| } |
| } |
| }, |
| "per_family": { |
| "arc": { |
| "n": 128, |
| "accuracy": 0.859375, |
| "nll": 0.4595502640510675, |
| "brier_multiclass_sum": 0.19233675665365213, |
| "ece_top_label_10_equal_width_bins": 0.04693025094456971, |
| "reliability_bins": [ |
| { |
| "count": 0, |
| "confidence_sum": 0.0, |
| "correct_sum": 0.0 |
| }, |
| { |
| "count": 0, |
| "confidence_sum": 0.0, |
| "correct_sum": 0.0 |
| }, |
| { |
| "count": 0, |
| "confidence_sum": 0.0, |
| "correct_sum": 0.0 |
| }, |
| { |
| "count": 2, |
| "confidence_sum": 0.7507493495941162, |
| "correct_sum": 0.0 |
| }, |
| { |
| "count": 5, |
| "confidence_sum": 2.3596443235874176, |
| "correct_sum": 1.0 |
| }, |
| { |
| "count": 2, |
| "confidence_sum": 1.1291241645812988, |
| "correct_sum": 1.0 |
| }, |
| { |
| "count": 9, |
| "confidence_sum": 5.849187076091766, |
| "correct_sum": 5.0 |
| }, |
| { |
| "count": 5, |
| "confidence_sum": 3.8443750739097595, |
| "correct_sum": 4.0 |
| }, |
| { |
| "count": 16, |
| "confidence_sum": 13.591759443283081, |
| "correct_sum": 13.0 |
| }, |
| { |
| "count": 89, |
| "confidence_sum": 88.170982837677, |
| "correct_sum": 86.0 |
| } |
| ], |
| "accuracy_vs_coverage": { |
| "0.25": { |
| "n": 32, |
| "accuracy": 1.0, |
| "min_confidence": 0.9997974038124084 |
| }, |
| "0.5": { |
| "n": 64, |
| "accuracy": 1.0, |
| "min_confidence": 0.9940166473388672 |
| }, |
| "0.75": { |
| "n": 96, |
| "accuracy": 0.9479166666666666, |
| "min_confidence": 0.8691695928573608 |
| }, |
| "1.0": { |
| "n": 128, |
| "accuracy": 0.859375, |
| "min_confidence": 0.36794614791870117 |
| } |
| } |
| }, |
| "banking": { |
| "n": 128, |
| "accuracy": 0.9921875, |
| "nll": 0.04955738214622285, |
| "brier_multiclass_sum": 0.02729644176736861, |
| "ece_top_label_10_equal_width_bins": 0.018931797705590725, |
| "reliability_bins": [ |
| { |
| "count": 0, |
| "confidence_sum": 0.0, |
| "correct_sum": 0.0 |
| }, |
| { |
| "count": 0, |
| "confidence_sum": 0.0, |
| "correct_sum": 0.0 |
| }, |
| { |
| "count": 0, |
| "confidence_sum": 0.0, |
| "correct_sum": 0.0 |
| }, |
| { |
| "count": 0, |
| "confidence_sum": 0.0, |
| "correct_sum": 0.0 |
| }, |
| { |
| "count": 0, |
| "confidence_sum": 0.0, |
| "correct_sum": 0.0 |
| }, |
| { |
| "count": 4, |
| "confidence_sum": 2.1734386682510376, |
| "correct_sum": 4.0 |
| }, |
| { |
| "count": 0, |
| "confidence_sum": 0.0, |
| "correct_sum": 0.0 |
| }, |
| { |
| "count": 0, |
| "confidence_sum": 0.0, |
| "correct_sum": 0.0 |
| }, |
| { |
| "count": 1, |
| "confidence_sum": 0.8818942308425903, |
| "correct_sum": 1.0 |
| }, |
| { |
| "count": 123, |
| "confidence_sum": 122.47860300540924, |
| "correct_sum": 122.0 |
| } |
| ], |
| "accuracy_vs_coverage": { |
| "0.25": { |
| "n": 32, |
| "accuracy": 1.0, |
| "min_confidence": 0.9999998807907104 |
| }, |
| "0.5": { |
| "n": 64, |
| "accuracy": 1.0, |
| "min_confidence": 0.9999978542327881 |
| }, |
| "0.75": { |
| "n": 96, |
| "accuracy": 1.0, |
| "min_confidence": 0.9997970461845398 |
| }, |
| "1.0": { |
| "n": 128, |
| "accuracy": 0.9921875, |
| "min_confidence": 0.5277213454246521 |
| } |
| } |
| }, |
| "boolq": { |
| "n": 128, |
| "accuracy": 0.8828125, |
| "nll": 0.33209052507845627, |
| "brier_multiclass_sum": 0.1964803123360663, |
| "ece_top_label_10_equal_width_bins": 0.06582438806071877, |
| "reliability_bins": [ |
| { |
| "count": 0, |
| "confidence_sum": 0.0, |
| "correct_sum": 0.0 |
| }, |
| { |
| "count": 0, |
| "confidence_sum": 0.0, |
| "correct_sum": 0.0 |
| }, |
| { |
| "count": 0, |
| "confidence_sum": 0.0, |
| "correct_sum": 0.0 |
| }, |
| { |
| "count": 0, |
| "confidence_sum": 0.0, |
| "correct_sum": 0.0 |
| }, |
| { |
| "count": 0, |
| "confidence_sum": 0.0, |
| "correct_sum": 0.0 |
| }, |
| { |
| "count": 6, |
| "confidence_sum": 3.30224347114563, |
| "correct_sum": 3.0 |
| }, |
| { |
| "count": 9, |
| "confidence_sum": 5.869049251079559, |
| "correct_sum": 8.0 |
| }, |
| { |
| "count": 7, |
| "confidence_sum": 5.441415011882782, |
| "correct_sum": 6.0 |
| }, |
| { |
| "count": 11, |
| "confidence_sum": 9.470021307468414, |
| "correct_sum": 7.0 |
| }, |
| { |
| "count": 95, |
| "confidence_sum": 91.9637211561203, |
| "correct_sum": 89.0 |
| } |
| ], |
| "accuracy_vs_coverage": { |
| "0.25": { |
| "n": 32, |
| "accuracy": 0.96875, |
| "min_confidence": 0.9858549237251282 |
| }, |
| "0.5": { |
| "n": 64, |
| "accuracy": 0.984375, |
| "min_confidence": 0.9649767875671387 |
| }, |
| "0.75": { |
| "n": 96, |
| "accuracy": 0.9270833333333334, |
| "min_confidence": 0.8953233957290649 |
| }, |
| "1.0": { |
| "n": 128, |
| "accuracy": 0.8828125, |
| "min_confidence": 0.512544572353363 |
| } |
| } |
| }, |
| "snli": { |
| "n": 128, |
| "accuracy": 0.890625, |
| "nll": 0.3105186039756518, |
| "brier_multiclass_sum": 0.1678693563700965, |
| "ece_top_label_10_equal_width_bins": 0.0860170740634203, |
| "reliability_bins": [ |
| { |
| "count": 0, |
| "confidence_sum": 0.0, |
| "correct_sum": 0.0 |
| }, |
| { |
| "count": 0, |
| "confidence_sum": 0.0, |
| "correct_sum": 0.0 |
| }, |
| { |
| "count": 0, |
| "confidence_sum": 0.0, |
| "correct_sum": 0.0 |
| }, |
| { |
| "count": 0, |
| "confidence_sum": 0.0, |
| "correct_sum": 0.0 |
| }, |
| { |
| "count": 1, |
| "confidence_sum": 0.46932661533355713, |
| "correct_sum": 0.0 |
| }, |
| { |
| "count": 0, |
| "confidence_sum": 0.0, |
| "correct_sum": 0.0 |
| }, |
| { |
| "count": 4, |
| "confidence_sum": 2.581421971321106, |
| "correct_sum": 4.0 |
| }, |
| { |
| "count": 12, |
| "confidence_sum": 9.100939571857452, |
| "correct_sum": 6.0 |
| }, |
| { |
| "count": 18, |
| "confidence_sum": 15.315423548221588, |
| "correct_sum": 12.0 |
| }, |
| { |
| "count": 93, |
| "confidence_sum": 89.2940822839737, |
| "correct_sum": 92.0 |
| } |
| ], |
| "accuracy_vs_coverage": { |
| "0.25": { |
| "n": 32, |
| "accuracy": 1.0, |
| "min_confidence": 0.9734072685241699 |
| }, |
| "0.5": { |
| "n": 64, |
| "accuracy": 1.0, |
| "min_confidence": 0.9560357928276062 |
| }, |
| "0.75": { |
| "n": 96, |
| "accuracy": 0.9791666666666666, |
| "min_confidence": 0.8869620561599731 |
| }, |
| "1.0": { |
| "n": 128, |
| "accuracy": 0.890625, |
| "min_confidence": 0.46932661533355713 |
| } |
| } |
| } |
| } |
| } |
| }, |
| { |
| "step": 5500, |
| "macro_nll": 0.388565277749868, |
| "best_macro_nll": 0.25676974726429014, |
| "selection": { |
| "metric": "crossfit_temperature_nll_v1", |
| "score": 0.30398422164076616, |
| "raw_macro_nll": 0.38856527338080316, |
| "accuracy": 0.896484375, |
| "folds": [ |
| { |
| "fold": 0, |
| "temperature_index": 58, |
| "temperature": 2.1577444091526656, |
| "training_macro_nll": 0.3199929810058706, |
| "heldout_ids": [ |
| "arc:validation:ARC-Challenge:122", |
| "arc:validation:ARC-Challenge:126", |
| "arc:validation:ARC-Challenge:175", |
| "arc:validation:ARC-Challenge:187", |
| "arc:validation:ARC-Challenge:189", |
| "arc:validation:ARC-Challenge:192", |
| "arc:validation:ARC-Challenge:208", |
| "arc:validation:ARC-Challenge:238", |
| "arc:validation:ARC-Challenge:290", |
| "arc:validation:ARC-Challenge:293", |
| "arc:validation:ARC-Challenge:35", |
| "arc:validation:ARC-Challenge:43", |
| "arc:validation:ARC-Challenge:58", |
| "arc:validation:ARC-Challenge:7", |
| "arc:validation:ARC-Easy:160", |
| "arc:validation:ARC-Easy:201", |
| "arc:validation:ARC-Easy:229", |
| "arc:validation:ARC-Easy:238", |
| "arc:validation:ARC-Easy:253", |
| "arc:validation:ARC-Easy:259", |
| "arc:validation:ARC-Easy:272", |
| "arc:validation:ARC-Easy:298", |
| "arc:validation:ARC-Easy:329", |
| "arc:validation:ARC-Easy:331", |
| "arc:validation:ARC-Easy:337", |
| "arc:validation:ARC-Easy:362", |
| "arc:validation:ARC-Easy:443", |
| "arc:validation:ARC-Easy:500", |
| "arc:validation:ARC-Easy:545", |
| "arc:validation:ARC-Easy:68", |
| "arc:validation:ARC-Easy:7", |
| "arc:validation:ARC-Easy:74", |
| "banking:train:1053", |
| "banking:train:1594", |
| "banking:train:1747", |
| "banking:train:202", |
| "banking:train:2377", |
| "banking:train:2528", |
| "banking:train:2546", |
| "banking:train:2715", |
| "banking:train:3220", |
| "banking:train:3499", |
| "banking:train:3806", |
| "banking:train:3838", |
| "banking:train:3993", |
| "banking:train:4220", |
| "banking:train:5", |
| "banking:train:5192", |
| "banking:train:5394", |
| "banking:train:5846", |
| "banking:train:5863", |
| "banking:train:6032", |
| "banking:train:6508", |
| "banking:train:7579", |
| "banking:train:7600", |
| "banking:train:7987", |
| "banking:train:8155", |
| "banking:train:850", |
| "banking:train:8626", |
| "banking:train:8929", |
| "banking:train:9355", |
| "banking:train:9812", |
| "banking:train:987", |
| "banking:train:9972", |
| "boolq:validation:103", |
| "boolq:validation:1042", |
| "boolq:validation:1059", |
| "boolq:validation:1541", |
| "boolq:validation:1592", |
| "boolq:validation:1927", |
| "boolq:validation:2014", |
| "boolq:validation:2023", |
| "boolq:validation:2098", |
| "boolq:validation:2139", |
| "boolq:validation:2439", |
| "boolq:validation:2485", |
| "boolq:validation:2607", |
| "boolq:validation:2611", |
| "boolq:validation:2720", |
| "boolq:validation:2751", |
| "boolq:validation:282", |
| "boolq:validation:2843", |
| "boolq:validation:2953", |
| "boolq:validation:301", |
| "boolq:validation:3089", |
| "boolq:validation:313", |
| "boolq:validation:3201", |
| "boolq:validation:322", |
| "boolq:validation:3231", |
| "boolq:validation:39", |
| "boolq:validation:461", |
| "boolq:validation:558", |
| "boolq:validation:589", |
| "boolq:validation:731", |
| "boolq:validation:883", |
| "boolq:validation:915", |
| "snli:validation:1156", |
| "snli:validation:1330", |
| "snli:validation:1559", |
| "snli:validation:1735", |
| "snli:validation:1876", |
| "snli:validation:2251", |
| "snli:validation:2257", |
| "snli:validation:2377", |
| "snli:validation:2881", |
| "snli:validation:291", |
| "snli:validation:300", |
| "snli:validation:3049", |
| "snli:validation:3496", |
| "snli:validation:3871", |
| "snli:validation:4414", |
| "snli:validation:5140", |
| "snli:validation:5440", |
| "snli:validation:5806", |
| "snli:validation:6442", |
| "snli:validation:6772", |
| "snli:validation:6949", |
| "snli:validation:7228", |
| "snli:validation:7465", |
| "snli:validation:7510", |
| "snli:validation:7726", |
| "snli:validation:8095", |
| "snli:validation:8143", |
| "snli:validation:8623", |
| "snli:validation:8962", |
| "snli:validation:9451", |
| "snli:validation:9574", |
| "snli:validation:9655" |
| ], |
| "heldout_group_count": 128, |
| "training_group_count": 384 |
| }, |
| { |
| "fold": 1, |
| "temperature_index": 57, |
| "temperature": 2.046444636724674, |
| "training_macro_nll": 0.30257257772561696, |
| "heldout_ids": [ |
| "arc:validation:ARC-Challenge:101", |
| "arc:validation:ARC-Challenge:129", |
| "arc:validation:ARC-Challenge:148", |
| "arc:validation:ARC-Challenge:156", |
| "arc:validation:ARC-Challenge:167", |
| "arc:validation:ARC-Challenge:249", |
| "arc:validation:ARC-Challenge:263", |
| "arc:validation:ARC-Challenge:266", |
| "arc:validation:ARC-Challenge:267", |
| "arc:validation:ARC-Challenge:28", |
| "arc:validation:ARC-Challenge:281", |
| "arc:validation:ARC-Challenge:57", |
| "arc:validation:ARC-Challenge:63", |
| "arc:validation:ARC-Challenge:64", |
| "arc:validation:ARC-Challenge:86", |
| "arc:validation:ARC-Challenge:99", |
| "arc:validation:ARC-Easy:106", |
| "arc:validation:ARC-Easy:151", |
| "arc:validation:ARC-Easy:157", |
| "arc:validation:ARC-Easy:166", |
| "arc:validation:ARC-Easy:202", |
| "arc:validation:ARC-Easy:219", |
| "arc:validation:ARC-Easy:224", |
| "arc:validation:ARC-Easy:322", |
| "arc:validation:ARC-Easy:344", |
| "arc:validation:ARC-Easy:382", |
| "arc:validation:ARC-Easy:409", |
| "arc:validation:ARC-Easy:455", |
| "arc:validation:ARC-Easy:50", |
| "arc:validation:ARC-Easy:538", |
| "arc:validation:ARC-Easy:544", |
| "arc:validation:ARC-Easy:552", |
| "banking:train:1050", |
| "banking:train:1131", |
| "banking:train:1190", |
| "banking:train:124", |
| "banking:train:137", |
| "banking:train:1493", |
| "banking:train:1498", |
| "banking:train:1624", |
| "banking:train:2614", |
| "banking:train:2739", |
| "banking:train:3055", |
| "banking:train:306", |
| "banking:train:3312", |
| "banking:train:3724", |
| "banking:train:4070", |
| "banking:train:476", |
| "banking:train:4882", |
| "banking:train:5075", |
| "banking:train:5332", |
| "banking:train:5445", |
| "banking:train:6096", |
| "banking:train:746", |
| "banking:train:7828", |
| "banking:train:8176", |
| "banking:train:8315", |
| "banking:train:8653", |
| "banking:train:9019", |
| "banking:train:9370", |
| "banking:train:9454", |
| "banking:train:9595", |
| "banking:train:9862", |
| "banking:train:9963", |
| "boolq:validation:1189", |
| "boolq:validation:1320", |
| "boolq:validation:1326", |
| "boolq:validation:1381", |
| "boolq:validation:1410", |
| "boolq:validation:149", |
| "boolq:validation:1513", |
| "boolq:validation:1732", |
| "boolq:validation:1739", |
| "boolq:validation:1741", |
| "boolq:validation:1814", |
| "boolq:validation:1848", |
| "boolq:validation:1940", |
| "boolq:validation:2077", |
| "boolq:validation:2091", |
| "boolq:validation:2148", |
| "boolq:validation:2215", |
| "boolq:validation:2251", |
| "boolq:validation:2283", |
| "boolq:validation:2321", |
| "boolq:validation:2398", |
| "boolq:validation:253", |
| "boolq:validation:2575", |
| "boolq:validation:2907", |
| "boolq:validation:2909", |
| "boolq:validation:2978", |
| "boolq:validation:359", |
| "boolq:validation:394", |
| "boolq:validation:436", |
| "boolq:validation:579", |
| "boolq:validation:807", |
| "boolq:validation:949", |
| "snli:validation:2389", |
| "snli:validation:2485", |
| "snli:validation:255", |
| "snli:validation:3169", |
| "snli:validation:333", |
| "snli:validation:3700", |
| "snli:validation:3709", |
| "snli:validation:4093", |
| "snli:validation:420", |
| "snli:validation:4492", |
| "snli:validation:4516", |
| "snli:validation:4648", |
| "snli:validation:4888", |
| "snli:validation:5185", |
| "snli:validation:5332", |
| "snli:validation:5947", |
| "snli:validation:6124", |
| "snli:validation:660", |
| "snli:validation:6619", |
| "snli:validation:6898", |
| "snli:validation:7387", |
| "snli:validation:7492", |
| "snli:validation:7544", |
| "snli:validation:7996", |
| "snli:validation:8074", |
| "snli:validation:8554", |
| "snli:validation:8971", |
| "snli:validation:9163", |
| "snli:validation:9277", |
| "snli:validation:9490", |
| "snli:validation:9595", |
| "snli:validation:9688" |
| ], |
| "heldout_group_count": 128, |
| "training_group_count": 384 |
| }, |
| { |
| "fold": 2, |
| "temperature_index": 57, |
| "temperature": 2.046444636724674, |
| "training_macro_nll": 0.29578256432862426, |
| "heldout_ids": [ |
| "arc:validation:ARC-Challenge:120", |
| "arc:validation:ARC-Challenge:130", |
| "arc:validation:ARC-Challenge:194", |
| "arc:validation:ARC-Challenge:218", |
| "arc:validation:ARC-Challenge:235", |
| "arc:validation:ARC-Challenge:272", |
| "arc:validation:ARC-Challenge:277", |
| "arc:validation:ARC-Challenge:48", |
| "arc:validation:ARC-Challenge:88", |
| "arc:validation:ARC-Easy:115", |
| "arc:validation:ARC-Easy:12", |
| "arc:validation:ARC-Easy:131", |
| "arc:validation:ARC-Easy:145", |
| "arc:validation:ARC-Easy:179", |
| "arc:validation:ARC-Easy:18", |
| "arc:validation:ARC-Easy:186", |
| "arc:validation:ARC-Easy:191", |
| "arc:validation:ARC-Easy:215", |
| "arc:validation:ARC-Easy:283", |
| "arc:validation:ARC-Easy:314", |
| "arc:validation:ARC-Easy:315", |
| "arc:validation:ARC-Easy:378", |
| "arc:validation:ARC-Easy:384", |
| "arc:validation:ARC-Easy:403", |
| "arc:validation:ARC-Easy:432", |
| "arc:validation:ARC-Easy:439", |
| "arc:validation:ARC-Easy:448", |
| "arc:validation:ARC-Easy:461", |
| "arc:validation:ARC-Easy:469", |
| "arc:validation:ARC-Easy:470", |
| "arc:validation:ARC-Easy:504", |
| "arc:validation:ARC-Easy:561", |
| "banking:train:1142", |
| "banking:train:148", |
| "banking:train:1633", |
| "banking:train:1927", |
| "banking:train:2186", |
| "banking:train:2251", |
| "banking:train:23", |
| "banking:train:3048", |
| "banking:train:3665", |
| "banking:train:3952", |
| "banking:train:3999", |
| "banking:train:4364", |
| "banking:train:4619", |
| "banking:train:5049", |
| "banking:train:5335", |
| "banking:train:549", |
| "banking:train:5957", |
| "banking:train:5985", |
| "banking:train:6132", |
| "banking:train:6240", |
| "banking:train:6488", |
| "banking:train:7065", |
| "banking:train:7749", |
| "banking:train:8470", |
| "banking:train:8761", |
| "banking:train:895", |
| "banking:train:9256", |
| "banking:train:9406", |
| "banking:train:9450", |
| "banking:train:9467", |
| "banking:train:9588", |
| "banking:train:9716", |
| "boolq:validation:1130", |
| "boolq:validation:1167", |
| "boolq:validation:124", |
| "boolq:validation:1255", |
| "boolq:validation:1256", |
| "boolq:validation:1373", |
| "boolq:validation:15", |
| "boolq:validation:155", |
| "boolq:validation:1568", |
| "boolq:validation:16", |
| "boolq:validation:1654", |
| "boolq:validation:1702", |
| "boolq:validation:1792", |
| "boolq:validation:1818", |
| "boolq:validation:210", |
| "boolq:validation:2247", |
| "boolq:validation:2259", |
| "boolq:validation:2362", |
| "boolq:validation:2443", |
| "boolq:validation:2524", |
| "boolq:validation:2553", |
| "boolq:validation:2623", |
| "boolq:validation:2660", |
| "boolq:validation:2669", |
| "boolq:validation:2679", |
| "boolq:validation:2832", |
| "boolq:validation:357", |
| "boolq:validation:453", |
| "boolq:validation:49", |
| "boolq:validation:808", |
| "boolq:validation:90", |
| "boolq:validation:910", |
| "snli:validation:1612", |
| "snli:validation:168", |
| "snli:validation:1693", |
| "snli:validation:2134", |
| "snli:validation:2248", |
| "snli:validation:2422", |
| "snli:validation:2587", |
| "snli:validation:3007", |
| "snli:validation:3097", |
| "snli:validation:3772", |
| "snli:validation:4012", |
| "snli:validation:4168", |
| "snli:validation:4537", |
| "snli:validation:4681", |
| "snli:validation:5050", |
| "snli:validation:5632", |
| "snli:validation:5761", |
| "snli:validation:6055", |
| "snli:validation:6283", |
| "snli:validation:6715", |
| "snli:validation:7117", |
| "snli:validation:7192", |
| "snli:validation:7507", |
| "snli:validation:7594", |
| "snli:validation:7924", |
| "snli:validation:7954", |
| "snli:validation:8035", |
| "snli:validation:8110", |
| "snli:validation:8995", |
| "snli:validation:949", |
| "snli:validation:9523", |
| "snli:validation:9589" |
| ], |
| "heldout_group_count": 128, |
| "training_group_count": 384 |
| }, |
| { |
| "fold": 3, |
| "temperature_index": 56, |
| "temperature": 1.9408858775927773, |
| "training_macro_nll": 0.29129510768094824, |
| "heldout_ids": [ |
| "arc:validation:ARC-Challenge:117", |
| "arc:validation:ARC-Challenge:135", |
| "arc:validation:ARC-Challenge:139", |
| "arc:validation:ARC-Challenge:154", |
| "arc:validation:ARC-Challenge:174", |
| "arc:validation:ARC-Challenge:199", |
| "arc:validation:ARC-Challenge:287", |
| "arc:validation:ARC-Challenge:295", |
| "arc:validation:ARC-Challenge:4", |
| "arc:validation:ARC-Challenge:50", |
| "arc:validation:ARC-Challenge:51", |
| "arc:validation:ARC-Challenge:61", |
| "arc:validation:ARC-Easy:107", |
| "arc:validation:ARC-Easy:121", |
| "arc:validation:ARC-Easy:124", |
| "arc:validation:ARC-Easy:171", |
| "arc:validation:ARC-Easy:235", |
| "arc:validation:ARC-Easy:241", |
| "arc:validation:ARC-Easy:279", |
| "arc:validation:ARC-Easy:289", |
| "arc:validation:ARC-Easy:37", |
| "arc:validation:ARC-Easy:389", |
| "arc:validation:ARC-Easy:394", |
| "arc:validation:ARC-Easy:399", |
| "arc:validation:ARC-Easy:423", |
| "arc:validation:ARC-Easy:492", |
| "arc:validation:ARC-Easy:499", |
| "arc:validation:ARC-Easy:523", |
| "arc:validation:ARC-Easy:533", |
| "arc:validation:ARC-Easy:569", |
| "arc:validation:ARC-Easy:69", |
| "arc:validation:ARC-Easy:93", |
| "banking:train:2049", |
| "banking:train:2350", |
| "banking:train:2497", |
| "banking:train:2506", |
| "banking:train:2720", |
| "banking:train:2788", |
| "banking:train:2967", |
| "banking:train:3502", |
| "banking:train:3726", |
| "banking:train:4014", |
| "banking:train:4735", |
| "banking:train:4803", |
| "banking:train:4917", |
| "banking:train:5477", |
| "banking:train:5851", |
| "banking:train:6551", |
| "banking:train:6839", |
| "banking:train:6926", |
| "banking:train:7809", |
| "banking:train:8092", |
| "banking:train:8186", |
| "banking:train:8215", |
| "banking:train:8667", |
| "banking:train:8752", |
| "banking:train:8763", |
| "banking:train:8851", |
| "banking:train:8917", |
| "banking:train:9145", |
| "banking:train:9279", |
| "banking:train:9676", |
| "banking:train:9837", |
| "banking:train:9967", |
| "boolq:validation:106", |
| "boolq:validation:1065", |
| "boolq:validation:1097", |
| "boolq:validation:1336", |
| "boolq:validation:1350", |
| "boolq:validation:1383", |
| "boolq:validation:1458", |
| "boolq:validation:1641", |
| "boolq:validation:1873", |
| "boolq:validation:1930", |
| "boolq:validation:1989", |
| "boolq:validation:2103", |
| "boolq:validation:234", |
| "boolq:validation:2355", |
| "boolq:validation:2384", |
| "boolq:validation:2440", |
| "boolq:validation:2444", |
| "boolq:validation:2478", |
| "boolq:validation:2872", |
| "boolq:validation:2879", |
| "boolq:validation:2924", |
| "boolq:validation:2971", |
| "boolq:validation:3164", |
| "boolq:validation:4", |
| "boolq:validation:415", |
| "boolq:validation:419", |
| "boolq:validation:428", |
| "boolq:validation:512", |
| "boolq:validation:563", |
| "boolq:validation:643", |
| "boolq:validation:696", |
| "boolq:validation:874", |
| "snli:validation:1102", |
| "snli:validation:1240", |
| "snli:validation:1261", |
| "snli:validation:1738", |
| "snli:validation:1978", |
| "snli:validation:2563", |
| "snli:validation:2659", |
| "snli:validation:2767", |
| "snli:validation:2836", |
| "snli:validation:3232", |
| "snli:validation:3655", |
| "snli:validation:3913", |
| "snli:validation:405", |
| "snli:validation:4051", |
| "snli:validation:4300", |
| "snli:validation:4846", |
| "snli:validation:498", |
| "snli:validation:5224", |
| "snli:validation:5575", |
| "snli:validation:6598", |
| "snli:validation:666", |
| "snli:validation:6925", |
| "snli:validation:6964", |
| "snli:validation:7045", |
| "snli:validation:7351", |
| "snli:validation:7588", |
| "snli:validation:7723", |
| "snli:validation:7750", |
| "snli:validation:7909", |
| "snli:validation:8974", |
| "snli:validation:9385", |
| "snli:validation:9646" |
| ], |
| "heldout_group_count": 128, |
| "training_group_count": 384 |
| } |
| ], |
| "per_family": { |
| "arc": { |
| "count": 128, |
| "score": 0.3994756698273555, |
| "raw_macro_nll": 0.5594607852608507, |
| "accuracy": 0.8828125 |
| }, |
| "banking": { |
| "count": 128, |
| "score": 0.029118518016223807, |
| "raw_macro_nll": 0.021053720166049603, |
| "accuracy": 0.9921875 |
| }, |
| "boolq": { |
| "count": 128, |
| "score": 0.36662860933042185, |
| "raw_macro_nll": 0.5107546041960799, |
| "accuracy": 0.84375 |
| }, |
| "snli": { |
| "count": 128, |
| "score": 0.4207140893890635, |
| "raw_macro_nll": 0.4629919839002324, |
| "accuracy": 0.8671875 |
| } |
| }, |
| "policy": { |
| "id": "crossfit_temperature_nll_v1", |
| "folds": 4, |
| "seed": 431, |
| "grouping": "source group, nested within task family; groups shuffled within sorted families and assigned round-robin", |
| "temperature_grid": { |
| "count": 101, |
| "log10_min": -1.0, |
| "log10_max": 1.3, |
| "arithmetic": "Python float64" |
| }, |
| "temperature_fit": "minimum macro-family NLL on the other three folds; smallest temperature wins ties", |
| "score": "macro-family mean of all held-out decision NLLs", |
| "deployment_temperature": "fit afresh on reserved calibration only after model selection" |
| } |
| }, |
| "best_selection_score": 0.2552943737691716, |
| "improved": false, |
| "metrics": { |
| "overall": { |
| "n": 512, |
| "accuracy": 0.896484375, |
| "nll": 0.388565277749868, |
| "brier_multiclass_sum": 0.1666528894976072, |
| "ece_top_label_10_equal_width_bins": 0.05876279535004869, |
| "reliability_bins": [ |
| { |
| "count": 0, |
| "confidence_sum": 0.0, |
| "correct_sum": 0.0 |
| }, |
| { |
| "count": 0, |
| "confidence_sum": 0.0, |
| "correct_sum": 0.0 |
| }, |
| { |
| "count": 0, |
| "confidence_sum": 0.0, |
| "correct_sum": 0.0 |
| }, |
| { |
| "count": 1, |
| "confidence_sum": 0.3927515745162964, |
| "correct_sum": 0.0 |
| }, |
| { |
| "count": 4, |
| "confidence_sum": 1.8952177464962006, |
| "correct_sum": 2.0 |
| }, |
| { |
| "count": 11, |
| "confidence_sum": 6.0297017097473145, |
| "correct_sum": 7.0 |
| }, |
| { |
| "count": 15, |
| "confidence_sum": 9.80898916721344, |
| "correct_sum": 5.0 |
| }, |
| { |
| "count": 10, |
| "confidence_sum": 7.650386393070221, |
| "correct_sum": 6.0 |
| }, |
| { |
| "count": 34, |
| "confidence_sum": 29.33014488220215, |
| "correct_sum": 24.0 |
| }, |
| { |
| "count": 437, |
| "confidence_sum": 431.82919865846634, |
| "correct_sum": 415.0 |
| } |
| ], |
| "accuracy_vs_coverage": { |
| "0.25": { |
| "n": 128, |
| "accuracy": 1.0, |
| "min_confidence": 0.9999988079071045 |
| }, |
| "0.5": { |
| "n": 256, |
| "accuracy": 0.984375, |
| "min_confidence": 0.998248815536499 |
| }, |
| "0.75": { |
| "n": 384, |
| "accuracy": 0.953125, |
| "min_confidence": 0.9573226571083069 |
| }, |
| "1.0": { |
| "n": 512, |
| "accuracy": 0.896484375, |
| "min_confidence": 0.3927515745162964 |
| } |
| } |
| }, |
| "per_family": { |
| "arc": { |
| "n": 128, |
| "accuracy": 0.8828125, |
| "nll": 0.559460803464001, |
| "brier_multiclass_sum": 0.18063573367263372, |
| "ece_top_label_10_equal_width_bins": 0.0859838502947241, |
| "reliability_bins": [ |
| { |
| "count": 0, |
| "confidence_sum": 0.0, |
| "correct_sum": 0.0 |
| }, |
| { |
| "count": 0, |
| "confidence_sum": 0.0, |
| "correct_sum": 0.0 |
| }, |
| { |
| "count": 0, |
| "confidence_sum": 0.0, |
| "correct_sum": 0.0 |
| }, |
| { |
| "count": 1, |
| "confidence_sum": 0.3927515745162964, |
| "correct_sum": 0.0 |
| }, |
| { |
| "count": 3, |
| "confidence_sum": 1.4031347334384918, |
| "correct_sum": 1.0 |
| }, |
| { |
| "count": 5, |
| "confidence_sum": 2.770816743373871, |
| "correct_sum": 4.0 |
| }, |
| { |
| "count": 8, |
| "confidence_sum": 5.263402879238129, |
| "correct_sum": 2.0 |
| }, |
| { |
| "count": 2, |
| "confidence_sum": 1.5604954361915588, |
| "correct_sum": 2.0 |
| }, |
| { |
| "count": 6, |
| "confidence_sum": 5.152871131896973, |
| "correct_sum": 6.0 |
| }, |
| { |
| "count": 103, |
| "confidence_sum": 102.43082696199417, |
| "correct_sum": 98.0 |
| } |
| ], |
| "accuracy_vs_coverage": { |
| "0.25": { |
| "n": 32, |
| "accuracy": 1.0, |
| "min_confidence": 0.9999986886978149 |
| }, |
| "0.5": { |
| "n": 64, |
| "accuracy": 0.96875, |
| "min_confidence": 0.9996984004974365 |
| }, |
| "0.75": { |
| "n": 96, |
| "accuracy": 0.9479166666666666, |
| "min_confidence": 0.9729017615318298 |
| }, |
| "1.0": { |
| "n": 128, |
| "accuracy": 0.8828125, |
| "min_confidence": 0.3927515745162964 |
| } |
| } |
| }, |
| "banking": { |
| "n": 128, |
| "accuracy": 0.9921875, |
| "nll": 0.02105371563040176, |
| "brier_multiclass_sum": 0.013577903502986103, |
| "ece_top_label_10_equal_width_bins": 0.01120622968301177, |
| "reliability_bins": [ |
| { |
| "count": 0, |
| "confidence_sum": 0.0, |
| "correct_sum": 0.0 |
| }, |
| { |
| "count": 0, |
| "confidence_sum": 0.0, |
| "correct_sum": 0.0 |
| }, |
| { |
| "count": 0, |
| "confidence_sum": 0.0, |
| "correct_sum": 0.0 |
| }, |
| { |
| "count": 0, |
| "confidence_sum": 0.0, |
| "correct_sum": 0.0 |
| }, |
| { |
| "count": 0, |
| "confidence_sum": 0.0, |
| "correct_sum": 0.0 |
| }, |
| { |
| "count": 0, |
| "confidence_sum": 0.0, |
| "correct_sum": 0.0 |
| }, |
| { |
| "count": 1, |
| "confidence_sum": 0.685830295085907, |
| "correct_sum": 1.0 |
| }, |
| { |
| "count": 1, |
| "confidence_sum": 0.7017979025840759, |
| "correct_sum": 1.0 |
| }, |
| { |
| "count": 2, |
| "confidence_sum": 1.6746965050697327, |
| "correct_sum": 1.0 |
| }, |
| { |
| "count": 124, |
| "confidence_sum": 123.85267090797424, |
| "correct_sum": 124.0 |
| } |
| ], |
| "accuracy_vs_coverage": { |
| "0.25": { |
| "n": 32, |
| "accuracy": 1.0, |
| "min_confidence": 1.0 |
| }, |
| "0.5": { |
| "n": 64, |
| "accuracy": 1.0, |
| "min_confidence": 1.0 |
| }, |
| "0.75": { |
| "n": 96, |
| "accuracy": 1.0, |
| "min_confidence": 0.9999998807907104 |
| }, |
| "1.0": { |
| "n": 128, |
| "accuracy": 0.9921875, |
| "min_confidence": 0.685830295085907 |
| } |
| } |
| }, |
| "boolq": { |
| "n": 128, |
| "accuracy": 0.84375, |
| "nll": 0.510754603942928, |
| "brier_multiclass_sum": 0.25073990983894767, |
| "ece_top_label_10_equal_width_bins": 0.10163650894537568, |
| "reliability_bins": [ |
| { |
| "count": 0, |
| "confidence_sum": 0.0, |
| "correct_sum": 0.0 |
| }, |
| { |
| "count": 0, |
| "confidence_sum": 0.0, |
| "correct_sum": 0.0 |
| }, |
| { |
| "count": 0, |
| "confidence_sum": 0.0, |
| "correct_sum": 0.0 |
| }, |
| { |
| "count": 0, |
| "confidence_sum": 0.0, |
| "correct_sum": 0.0 |
| }, |
| { |
| "count": 0, |
| "confidence_sum": 0.0, |
| "correct_sum": 0.0 |
| }, |
| { |
| "count": 4, |
| "confidence_sum": 2.1253692507743835, |
| "correct_sum": 2.0 |
| }, |
| { |
| "count": 5, |
| "confidence_sum": 3.2207350730895996, |
| "correct_sum": 2.0 |
| }, |
| { |
| "count": 4, |
| "confidence_sum": 3.1362109780311584, |
| "correct_sum": 2.0 |
| }, |
| { |
| "count": 9, |
| "confidence_sum": 7.752904176712036, |
| "correct_sum": 7.0 |
| }, |
| { |
| "count": 106, |
| "confidence_sum": 104.77425366640091, |
| "correct_sum": 95.0 |
| } |
| ], |
| "accuracy_vs_coverage": { |
| "0.25": { |
| "n": 32, |
| "accuracy": 0.96875, |
| "min_confidence": 0.9983727335929871 |
| }, |
| "0.5": { |
| "n": 64, |
| "accuracy": 0.96875, |
| "min_confidence": 0.9950273633003235 |
| }, |
| "0.75": { |
| "n": 96, |
| "accuracy": 0.8958333333333334, |
| "min_confidence": 0.9621425271034241 |
| }, |
| "1.0": { |
| "n": 128, |
| "accuracy": 0.84375, |
| "min_confidence": 0.5126386284828186 |
| } |
| } |
| }, |
| "snli": { |
| "n": 128, |
| "accuracy": 0.8671875, |
| "nll": 0.46299198796214114, |
| "brier_multiclass_sum": 0.22165801097586132, |
| "ece_top_label_10_equal_width_bins": 0.07854262413457036, |
| "reliability_bins": [ |
| { |
| "count": 0, |
| "confidence_sum": 0.0, |
| "correct_sum": 0.0 |
| }, |
| { |
| "count": 0, |
| "confidence_sum": 0.0, |
| "correct_sum": 0.0 |
| }, |
| { |
| "count": 0, |
| "confidence_sum": 0.0, |
| "correct_sum": 0.0 |
| }, |
| { |
| "count": 0, |
| "confidence_sum": 0.0, |
| "correct_sum": 0.0 |
| }, |
| { |
| "count": 1, |
| "confidence_sum": 0.49208301305770874, |
| "correct_sum": 1.0 |
| }, |
| { |
| "count": 2, |
| "confidence_sum": 1.13351571559906, |
| "correct_sum": 1.0 |
| }, |
| { |
| "count": 1, |
| "confidence_sum": 0.6390209197998047, |
| "correct_sum": 0.0 |
| }, |
| { |
| "count": 3, |
| "confidence_sum": 2.2518820762634277, |
| "correct_sum": 1.0 |
| }, |
| { |
| "count": 17, |
| "confidence_sum": 14.749673068523407, |
| "correct_sum": 10.0 |
| }, |
| { |
| "count": 104, |
| "confidence_sum": 100.77144712209702, |
| "correct_sum": 98.0 |
| } |
| ], |
| "accuracy_vs_coverage": { |
| "0.25": { |
| "n": 32, |
| "accuracy": 0.96875, |
| "min_confidence": 0.9963485598564148 |
| }, |
| "0.5": { |
| "n": 64, |
| "accuracy": 0.953125, |
| "min_confidence": 0.9578976631164551 |
| }, |
| "0.75": { |
| "n": 96, |
| "accuracy": 0.9479166666666666, |
| "min_confidence": 0.9151947498321533 |
| }, |
| "1.0": { |
| "n": 128, |
| "accuracy": 0.8671875, |
| "min_confidence": 0.49208301305770874 |
| } |
| } |
| } |
| } |
| } |
| }, |
| { |
| "step": 6000, |
| "macro_nll": 0.40940693625302227, |
| "best_macro_nll": 0.25676974726429014, |
| "selection": { |
| "metric": "crossfit_temperature_nll_v1", |
| "score": 0.35159254105589344, |
| "raw_macro_nll": 0.40940693985472393, |
| "accuracy": 0.869140625, |
| "folds": [ |
| { |
| "fold": 0, |
| "temperature_index": 54, |
| "temperature": 1.7458221529205038, |
| "training_macro_nll": 0.34777796230347097, |
| "heldout_ids": [ |
| "arc:validation:ARC-Challenge:122", |
| "arc:validation:ARC-Challenge:126", |
| "arc:validation:ARC-Challenge:175", |
| "arc:validation:ARC-Challenge:187", |
| "arc:validation:ARC-Challenge:189", |
| "arc:validation:ARC-Challenge:192", |
| "arc:validation:ARC-Challenge:208", |
| "arc:validation:ARC-Challenge:238", |
| "arc:validation:ARC-Challenge:290", |
| "arc:validation:ARC-Challenge:293", |
| "arc:validation:ARC-Challenge:35", |
| "arc:validation:ARC-Challenge:43", |
| "arc:validation:ARC-Challenge:58", |
| "arc:validation:ARC-Challenge:7", |
| "arc:validation:ARC-Easy:160", |
| "arc:validation:ARC-Easy:201", |
| "arc:validation:ARC-Easy:229", |
| "arc:validation:ARC-Easy:238", |
| "arc:validation:ARC-Easy:253", |
| "arc:validation:ARC-Easy:259", |
| "arc:validation:ARC-Easy:272", |
| "arc:validation:ARC-Easy:298", |
| "arc:validation:ARC-Easy:329", |
| "arc:validation:ARC-Easy:331", |
| "arc:validation:ARC-Easy:337", |
| "arc:validation:ARC-Easy:362", |
| "arc:validation:ARC-Easy:443", |
| "arc:validation:ARC-Easy:500", |
| "arc:validation:ARC-Easy:545", |
| "arc:validation:ARC-Easy:68", |
| "arc:validation:ARC-Easy:7", |
| "arc:validation:ARC-Easy:74", |
| "banking:train:1053", |
| "banking:train:1594", |
| "banking:train:1747", |
| "banking:train:202", |
| "banking:train:2377", |
| "banking:train:2528", |
| "banking:train:2546", |
| "banking:train:2715", |
| "banking:train:3220", |
| "banking:train:3499", |
| "banking:train:3806", |
| "banking:train:3838", |
| "banking:train:3993", |
| "banking:train:4220", |
| "banking:train:5", |
| "banking:train:5192", |
| "banking:train:5394", |
| "banking:train:5846", |
| "banking:train:5863", |
| "banking:train:6032", |
| "banking:train:6508", |
| "banking:train:7579", |
| "banking:train:7600", |
| "banking:train:7987", |
| "banking:train:8155", |
| "banking:train:850", |
| "banking:train:8626", |
| "banking:train:8929", |
| "banking:train:9355", |
| "banking:train:9812", |
| "banking:train:987", |
| "banking:train:9972", |
| "boolq:validation:103", |
| "boolq:validation:1042", |
| "boolq:validation:1059", |
| "boolq:validation:1541", |
| "boolq:validation:1592", |
| "boolq:validation:1927", |
| "boolq:validation:2014", |
| "boolq:validation:2023", |
| "boolq:validation:2098", |
| "boolq:validation:2139", |
| "boolq:validation:2439", |
| "boolq:validation:2485", |
| "boolq:validation:2607", |
| "boolq:validation:2611", |
| "boolq:validation:2720", |
| "boolq:validation:2751", |
| "boolq:validation:282", |
| "boolq:validation:2843", |
| "boolq:validation:2953", |
| "boolq:validation:301", |
| "boolq:validation:3089", |
| "boolq:validation:313", |
| "boolq:validation:3201", |
| "boolq:validation:322", |
| "boolq:validation:3231", |
| "boolq:validation:39", |
| "boolq:validation:461", |
| "boolq:validation:558", |
| "boolq:validation:589", |
| "boolq:validation:731", |
| "boolq:validation:883", |
| "boolq:validation:915", |
| "snli:validation:1156", |
| "snli:validation:1330", |
| "snli:validation:1559", |
| "snli:validation:1735", |
| "snli:validation:1876", |
| "snli:validation:2251", |
| "snli:validation:2257", |
| "snli:validation:2377", |
| "snli:validation:2881", |
| "snli:validation:291", |
| "snli:validation:300", |
| "snli:validation:3049", |
| "snli:validation:3496", |
| "snli:validation:3871", |
| "snli:validation:4414", |
| "snli:validation:5140", |
| "snli:validation:5440", |
| "snli:validation:5806", |
| "snli:validation:6442", |
| "snli:validation:6772", |
| "snli:validation:6949", |
| "snli:validation:7228", |
| "snli:validation:7465", |
| "snli:validation:7510", |
| "snli:validation:7726", |
| "snli:validation:8095", |
| "snli:validation:8143", |
| "snli:validation:8623", |
| "snli:validation:8962", |
| "snli:validation:9451", |
| "snli:validation:9574", |
| "snli:validation:9655" |
| ], |
| "heldout_group_count": 128, |
| "training_group_count": 384 |
| }, |
| { |
| "fold": 1, |
| "temperature_index": 55, |
| "temperature": 1.8407720014689555, |
| "training_macro_nll": 0.3529714060051017, |
| "heldout_ids": [ |
| "arc:validation:ARC-Challenge:101", |
| "arc:validation:ARC-Challenge:129", |
| "arc:validation:ARC-Challenge:148", |
| "arc:validation:ARC-Challenge:156", |
| "arc:validation:ARC-Challenge:167", |
| "arc:validation:ARC-Challenge:249", |
| "arc:validation:ARC-Challenge:263", |
| "arc:validation:ARC-Challenge:266", |
| "arc:validation:ARC-Challenge:267", |
| "arc:validation:ARC-Challenge:28", |
| "arc:validation:ARC-Challenge:281", |
| "arc:validation:ARC-Challenge:57", |
| "arc:validation:ARC-Challenge:63", |
| "arc:validation:ARC-Challenge:64", |
| "arc:validation:ARC-Challenge:86", |
| "arc:validation:ARC-Challenge:99", |
| "arc:validation:ARC-Easy:106", |
| "arc:validation:ARC-Easy:151", |
| "arc:validation:ARC-Easy:157", |
| "arc:validation:ARC-Easy:166", |
| "arc:validation:ARC-Easy:202", |
| "arc:validation:ARC-Easy:219", |
| "arc:validation:ARC-Easy:224", |
| "arc:validation:ARC-Easy:322", |
| "arc:validation:ARC-Easy:344", |
| "arc:validation:ARC-Easy:382", |
| "arc:validation:ARC-Easy:409", |
| "arc:validation:ARC-Easy:455", |
| "arc:validation:ARC-Easy:50", |
| "arc:validation:ARC-Easy:538", |
| "arc:validation:ARC-Easy:544", |
| "arc:validation:ARC-Easy:552", |
| "banking:train:1050", |
| "banking:train:1131", |
| "banking:train:1190", |
| "banking:train:124", |
| "banking:train:137", |
| "banking:train:1493", |
| "banking:train:1498", |
| "banking:train:1624", |
| "banking:train:2614", |
| "banking:train:2739", |
| "banking:train:3055", |
| "banking:train:306", |
| "banking:train:3312", |
| "banking:train:3724", |
| "banking:train:4070", |
| "banking:train:476", |
| "banking:train:4882", |
| "banking:train:5075", |
| "banking:train:5332", |
| "banking:train:5445", |
| "banking:train:6096", |
| "banking:train:746", |
| "banking:train:7828", |
| "banking:train:8176", |
| "banking:train:8315", |
| "banking:train:8653", |
| "banking:train:9019", |
| "banking:train:9370", |
| "banking:train:9454", |
| "banking:train:9595", |
| "banking:train:9862", |
| "banking:train:9963", |
| "boolq:validation:1189", |
| "boolq:validation:1320", |
| "boolq:validation:1326", |
| "boolq:validation:1381", |
| "boolq:validation:1410", |
| "boolq:validation:149", |
| "boolq:validation:1513", |
| "boolq:validation:1732", |
| "boolq:validation:1739", |
| "boolq:validation:1741", |
| "boolq:validation:1814", |
| "boolq:validation:1848", |
| "boolq:validation:1940", |
| "boolq:validation:2077", |
| "boolq:validation:2091", |
| "boolq:validation:2148", |
| "boolq:validation:2215", |
| "boolq:validation:2251", |
| "boolq:validation:2283", |
| "boolq:validation:2321", |
| "boolq:validation:2398", |
| "boolq:validation:253", |
| "boolq:validation:2575", |
| "boolq:validation:2907", |
| "boolq:validation:2909", |
| "boolq:validation:2978", |
| "boolq:validation:359", |
| "boolq:validation:394", |
| "boolq:validation:436", |
| "boolq:validation:579", |
| "boolq:validation:807", |
| "boolq:validation:949", |
| "snli:validation:2389", |
| "snli:validation:2485", |
| "snli:validation:255", |
| "snli:validation:3169", |
| "snli:validation:333", |
| "snli:validation:3700", |
| "snli:validation:3709", |
| "snli:validation:4093", |
| "snli:validation:420", |
| "snli:validation:4492", |
| "snli:validation:4516", |
| "snli:validation:4648", |
| "snli:validation:4888", |
| "snli:validation:5185", |
| "snli:validation:5332", |
| "snli:validation:5947", |
| "snli:validation:6124", |
| "snli:validation:660", |
| "snli:validation:6619", |
| "snli:validation:6898", |
| "snli:validation:7387", |
| "snli:validation:7492", |
| "snli:validation:7544", |
| "snli:validation:7996", |
| "snli:validation:8074", |
| "snli:validation:8554", |
| "snli:validation:8971", |
| "snli:validation:9163", |
| "snli:validation:9277", |
| "snli:validation:9490", |
| "snli:validation:9595", |
| "snli:validation:9688" |
| ], |
| "heldout_group_count": 128, |
| "training_group_count": 384 |
| }, |
| { |
| "fold": 2, |
| "temperature_index": 55, |
| "temperature": 1.8407720014689555, |
| "training_macro_nll": 0.3578657995753054, |
| "heldout_ids": [ |
| "arc:validation:ARC-Challenge:120", |
| "arc:validation:ARC-Challenge:130", |
| "arc:validation:ARC-Challenge:194", |
| "arc:validation:ARC-Challenge:218", |
| "arc:validation:ARC-Challenge:235", |
| "arc:validation:ARC-Challenge:272", |
| "arc:validation:ARC-Challenge:277", |
| "arc:validation:ARC-Challenge:48", |
| "arc:validation:ARC-Challenge:88", |
| "arc:validation:ARC-Easy:115", |
| "arc:validation:ARC-Easy:12", |
| "arc:validation:ARC-Easy:131", |
| "arc:validation:ARC-Easy:145", |
| "arc:validation:ARC-Easy:179", |
| "arc:validation:ARC-Easy:18", |
| "arc:validation:ARC-Easy:186", |
| "arc:validation:ARC-Easy:191", |
| "arc:validation:ARC-Easy:215", |
| "arc:validation:ARC-Easy:283", |
| "arc:validation:ARC-Easy:314", |
| "arc:validation:ARC-Easy:315", |
| "arc:validation:ARC-Easy:378", |
| "arc:validation:ARC-Easy:384", |
| "arc:validation:ARC-Easy:403", |
| "arc:validation:ARC-Easy:432", |
| "arc:validation:ARC-Easy:439", |
| "arc:validation:ARC-Easy:448", |
| "arc:validation:ARC-Easy:461", |
| "arc:validation:ARC-Easy:469", |
| "arc:validation:ARC-Easy:470", |
| "arc:validation:ARC-Easy:504", |
| "arc:validation:ARC-Easy:561", |
| "banking:train:1142", |
| "banking:train:148", |
| "banking:train:1633", |
| "banking:train:1927", |
| "banking:train:2186", |
| "banking:train:2251", |
| "banking:train:23", |
| "banking:train:3048", |
| "banking:train:3665", |
| "banking:train:3952", |
| "banking:train:3999", |
| "banking:train:4364", |
| "banking:train:4619", |
| "banking:train:5049", |
| "banking:train:5335", |
| "banking:train:549", |
| "banking:train:5957", |
| "banking:train:5985", |
| "banking:train:6132", |
| "banking:train:6240", |
| "banking:train:6488", |
| "banking:train:7065", |
| "banking:train:7749", |
| "banking:train:8470", |
| "banking:train:8761", |
| "banking:train:895", |
| "banking:train:9256", |
| "banking:train:9406", |
| "banking:train:9450", |
| "banking:train:9467", |
| "banking:train:9588", |
| "banking:train:9716", |
| "boolq:validation:1130", |
| "boolq:validation:1167", |
| "boolq:validation:124", |
| "boolq:validation:1255", |
| "boolq:validation:1256", |
| "boolq:validation:1373", |
| "boolq:validation:15", |
| "boolq:validation:155", |
| "boolq:validation:1568", |
| "boolq:validation:16", |
| "boolq:validation:1654", |
| "boolq:validation:1702", |
| "boolq:validation:1792", |
| "boolq:validation:1818", |
| "boolq:validation:210", |
| "boolq:validation:2247", |
| "boolq:validation:2259", |
| "boolq:validation:2362", |
| "boolq:validation:2443", |
| "boolq:validation:2524", |
| "boolq:validation:2553", |
| "boolq:validation:2623", |
| "boolq:validation:2660", |
| "boolq:validation:2669", |
| "boolq:validation:2679", |
| "boolq:validation:2832", |
| "boolq:validation:357", |
| "boolq:validation:453", |
| "boolq:validation:49", |
| "boolq:validation:808", |
| "boolq:validation:90", |
| "boolq:validation:910", |
| "snli:validation:1612", |
| "snli:validation:168", |
| "snli:validation:1693", |
| "snli:validation:2134", |
| "snli:validation:2248", |
| "snli:validation:2422", |
| "snli:validation:2587", |
| "snli:validation:3007", |
| "snli:validation:3097", |
| "snli:validation:3772", |
| "snli:validation:4012", |
| "snli:validation:4168", |
| "snli:validation:4537", |
| "snli:validation:4681", |
| "snli:validation:5050", |
| "snli:validation:5632", |
| "snli:validation:5761", |
| "snli:validation:6055", |
| "snli:validation:6283", |
| "snli:validation:6715", |
| "snli:validation:7117", |
| "snli:validation:7192", |
| "snli:validation:7507", |
| "snli:validation:7594", |
| "snli:validation:7924", |
| "snli:validation:7954", |
| "snli:validation:8035", |
| "snli:validation:8110", |
| "snli:validation:8995", |
| "snli:validation:949", |
| "snli:validation:9523", |
| "snli:validation:9589" |
| ], |
| "heldout_group_count": 128, |
| "training_group_count": 384 |
| }, |
| { |
| "fold": 3, |
| "temperature_index": 52, |
| "temperature": 1.5703628043335522, |
| "training_macro_nll": 0.32735544756754187, |
| "heldout_ids": [ |
| "arc:validation:ARC-Challenge:117", |
| "arc:validation:ARC-Challenge:135", |
| "arc:validation:ARC-Challenge:139", |
| "arc:validation:ARC-Challenge:154", |
| "arc:validation:ARC-Challenge:174", |
| "arc:validation:ARC-Challenge:199", |
| "arc:validation:ARC-Challenge:287", |
| "arc:validation:ARC-Challenge:295", |
| "arc:validation:ARC-Challenge:4", |
| "arc:validation:ARC-Challenge:50", |
| "arc:validation:ARC-Challenge:51", |
| "arc:validation:ARC-Challenge:61", |
| "arc:validation:ARC-Easy:107", |
| "arc:validation:ARC-Easy:121", |
| "arc:validation:ARC-Easy:124", |
| "arc:validation:ARC-Easy:171", |
| "arc:validation:ARC-Easy:235", |
| "arc:validation:ARC-Easy:241", |
| "arc:validation:ARC-Easy:279", |
| "arc:validation:ARC-Easy:289", |
| "arc:validation:ARC-Easy:37", |
| "arc:validation:ARC-Easy:389", |
| "arc:validation:ARC-Easy:394", |
| "arc:validation:ARC-Easy:399", |
| "arc:validation:ARC-Easy:423", |
| "arc:validation:ARC-Easy:492", |
| "arc:validation:ARC-Easy:499", |
| "arc:validation:ARC-Easy:523", |
| "arc:validation:ARC-Easy:533", |
| "arc:validation:ARC-Easy:569", |
| "arc:validation:ARC-Easy:69", |
| "arc:validation:ARC-Easy:93", |
| "banking:train:2049", |
| "banking:train:2350", |
| "banking:train:2497", |
| "banking:train:2506", |
| "banking:train:2720", |
| "banking:train:2788", |
| "banking:train:2967", |
| "banking:train:3502", |
| "banking:train:3726", |
| "banking:train:4014", |
| "banking:train:4735", |
| "banking:train:4803", |
| "banking:train:4917", |
| "banking:train:5477", |
| "banking:train:5851", |
| "banking:train:6551", |
| "banking:train:6839", |
| "banking:train:6926", |
| "banking:train:7809", |
| "banking:train:8092", |
| "banking:train:8186", |
| "banking:train:8215", |
| "banking:train:8667", |
| "banking:train:8752", |
| "banking:train:8763", |
| "banking:train:8851", |
| "banking:train:8917", |
| "banking:train:9145", |
| "banking:train:9279", |
| "banking:train:9676", |
| "banking:train:9837", |
| "banking:train:9967", |
| "boolq:validation:106", |
| "boolq:validation:1065", |
| "boolq:validation:1097", |
| "boolq:validation:1336", |
| "boolq:validation:1350", |
| "boolq:validation:1383", |
| "boolq:validation:1458", |
| "boolq:validation:1641", |
| "boolq:validation:1873", |
| "boolq:validation:1930", |
| "boolq:validation:1989", |
| "boolq:validation:2103", |
| "boolq:validation:234", |
| "boolq:validation:2355", |
| "boolq:validation:2384", |
| "boolq:validation:2440", |
| "boolq:validation:2444", |
| "boolq:validation:2478", |
| "boolq:validation:2872", |
| "boolq:validation:2879", |
| "boolq:validation:2924", |
| "boolq:validation:2971", |
| "boolq:validation:3164", |
| "boolq:validation:4", |
| "boolq:validation:415", |
| "boolq:validation:419", |
| "boolq:validation:428", |
| "boolq:validation:512", |
| "boolq:validation:563", |
| "boolq:validation:643", |
| "boolq:validation:696", |
| "boolq:validation:874", |
| "snli:validation:1102", |
| "snli:validation:1240", |
| "snli:validation:1261", |
| "snli:validation:1738", |
| "snli:validation:1978", |
| "snli:validation:2563", |
| "snli:validation:2659", |
| "snli:validation:2767", |
| "snli:validation:2836", |
| "snli:validation:3232", |
| "snli:validation:3655", |
| "snli:validation:3913", |
| "snli:validation:405", |
| "snli:validation:4051", |
| "snli:validation:4300", |
| "snli:validation:4846", |
| "snli:validation:498", |
| "snli:validation:5224", |
| "snli:validation:5575", |
| "snli:validation:6598", |
| "snli:validation:666", |
| "snli:validation:6925", |
| "snli:validation:6964", |
| "snli:validation:7045", |
| "snli:validation:7351", |
| "snli:validation:7588", |
| "snli:validation:7723", |
| "snli:validation:7750", |
| "snli:validation:7909", |
| "snli:validation:8974", |
| "snli:validation:9385", |
| "snli:validation:9646" |
| ], |
| "heldout_group_count": 128, |
| "training_group_count": 384 |
| } |
| ], |
| "per_family": { |
| "arc": { |
| "count": 128, |
| "score": 0.4975187847350728, |
| "raw_macro_nll": 0.4627253663720225, |
| "accuracy": 0.8125 |
| }, |
| "banking": { |
| "count": 128, |
| "score": 0.028150747289219766, |
| "raw_macro_nll": 0.02860187647413226, |
| "accuracy": 0.984375 |
| }, |
| "boolq": { |
| "count": 128, |
| "score": 0.4220618375304737, |
| "raw_macro_nll": 0.5700947268218147, |
| "accuracy": 0.8359375 |
| }, |
| "snli": { |
| "count": 128, |
| "score": 0.45863879466880736, |
| "raw_macro_nll": 0.5762057897509263, |
| "accuracy": 0.84375 |
| } |
| }, |
| "policy": { |
| "id": "crossfit_temperature_nll_v1", |
| "folds": 4, |
| "seed": 431, |
| "grouping": "source group, nested within task family; groups shuffled within sorted families and assigned round-robin", |
| "temperature_grid": { |
| "count": 101, |
| "log10_min": -1.0, |
| "log10_max": 1.3, |
| "arithmetic": "Python float64" |
| }, |
| "temperature_fit": "minimum macro-family NLL on the other three folds; smallest temperature wins ties", |
| "score": "macro-family mean of all held-out decision NLLs", |
| "deployment_temperature": "fit afresh on reserved calibration only after model selection" |
| } |
| }, |
| "best_selection_score": 0.2552943737691716, |
| "improved": false, |
| "metrics": { |
| "overall": { |
| "n": 512, |
| "accuracy": 0.869140625, |
| "nll": 0.40940693625302227, |
| "brier_multiclass_sum": 0.20434636874989778, |
| "ece_top_label_10_equal_width_bins": 0.073907851066906, |
| "reliability_bins": [ |
| { |
| "count": 0, |
| "confidence_sum": 0.0, |
| "correct_sum": 0.0 |
| }, |
| { |
| "count": 0, |
| "confidence_sum": 0.0, |
| "correct_sum": 0.0 |
| }, |
| { |
| "count": 0, |
| "confidence_sum": 0.0, |
| "correct_sum": 0.0 |
| }, |
| { |
| "count": 8, |
| "confidence_sum": 2.964146226644516, |
| "correct_sum": 1.0 |
| }, |
| { |
| "count": 4, |
| "confidence_sum": 1.775023639202118, |
| "correct_sum": 1.0 |
| }, |
| { |
| "count": 12, |
| "confidence_sum": 6.521916568279266, |
| "correct_sum": 4.0 |
| }, |
| { |
| "count": 23, |
| "confidence_sum": 15.011595368385315, |
| "correct_sum": 17.0 |
| }, |
| { |
| "count": 13, |
| "confidence_sum": 9.764744758605957, |
| "correct_sum": 10.0 |
| }, |
| { |
| "count": 31, |
| "confidence_sum": 26.539382338523865, |
| "correct_sum": 21.0 |
| }, |
| { |
| "count": 421, |
| "confidence_sum": 415.8166911005974, |
| "correct_sum": 391.0 |
| } |
| ], |
| "accuracy_vs_coverage": { |
| "0.25": { |
| "n": 128, |
| "accuracy": 1.0, |
| "min_confidence": 0.9999061822891235 |
| }, |
| "0.5": { |
| "n": 256, |
| "accuracy": 0.9765625, |
| "min_confidence": 0.992233395576477 |
| }, |
| "0.75": { |
| "n": 384, |
| "accuracy": 0.9479166666666666, |
| "min_confidence": 0.9575716257095337 |
| }, |
| "1.0": { |
| "n": 512, |
| "accuracy": 0.869140625, |
| "min_confidence": 0.3231365382671356 |
| } |
| } |
| }, |
| "per_family": { |
| "arc": { |
| "n": 128, |
| "accuracy": 0.8125, |
| "nll": 0.46272536673021847, |
| "brier_multiclass_sum": 0.2405542295009345, |
| "ece_top_label_10_equal_width_bins": 0.07007392193190753, |
| "reliability_bins": [ |
| { |
| "count": 0, |
| "confidence_sum": 0.0, |
| "correct_sum": 0.0 |
| }, |
| { |
| "count": 0, |
| "confidence_sum": 0.0, |
| "correct_sum": 0.0 |
| }, |
| { |
| "count": 0, |
| "confidence_sum": 0.0, |
| "correct_sum": 0.0 |
| }, |
| { |
| "count": 8, |
| "confidence_sum": 2.964146226644516, |
| "correct_sum": 1.0 |
| }, |
| { |
| "count": 4, |
| "confidence_sum": 1.775023639202118, |
| "correct_sum": 1.0 |
| }, |
| { |
| "count": 9, |
| "confidence_sum": 4.836179673671722, |
| "correct_sum": 4.0 |
| }, |
| { |
| "count": 14, |
| "confidence_sum": 9.198149383068085, |
| "correct_sum": 10.0 |
| }, |
| { |
| "count": 8, |
| "confidence_sum": 6.019686222076416, |
| "correct_sum": 7.0 |
| }, |
| { |
| "count": 12, |
| "confidence_sum": 10.27308714389801, |
| "correct_sum": 8.0 |
| }, |
| { |
| "count": 73, |
| "confidence_sum": 71.6611390709877, |
| "correct_sum": 73.0 |
| } |
| ], |
| "accuracy_vs_coverage": { |
| "0.25": { |
| "n": 32, |
| "accuracy": 1.0, |
| "min_confidence": 0.9957687854766846 |
| }, |
| "0.5": { |
| "n": 64, |
| "accuracy": 1.0, |
| "min_confidence": 0.9485200643539429 |
| }, |
| "0.75": { |
| "n": 96, |
| "accuracy": 0.9375, |
| "min_confidence": 0.6923663020133972 |
| }, |
| "1.0": { |
| "n": 128, |
| "accuracy": 0.8125, |
| "min_confidence": 0.3231365382671356 |
| } |
| } |
| }, |
| "banking": { |
| "n": 128, |
| "accuracy": 0.984375, |
| "nll": 0.028601872775282866, |
| "brier_multiclass_sum": 0.01876134001342418, |
| "ece_top_label_10_equal_width_bins": 0.011103796772658825, |
| "reliability_bins": [ |
| { |
| "count": 0, |
| "confidence_sum": 0.0, |
| "correct_sum": 0.0 |
| }, |
| { |
| "count": 0, |
| "confidence_sum": 0.0, |
| "correct_sum": 0.0 |
| }, |
| { |
| "count": 0, |
| "confidence_sum": 0.0, |
| "correct_sum": 0.0 |
| }, |
| { |
| "count": 0, |
| "confidence_sum": 0.0, |
| "correct_sum": 0.0 |
| }, |
| { |
| "count": 0, |
| "confidence_sum": 0.0, |
| "correct_sum": 0.0 |
| }, |
| { |
| "count": 1, |
| "confidence_sum": 0.5795621275901794, |
| "correct_sum": 0.0 |
| }, |
| { |
| "count": 0, |
| "confidence_sum": 0.0, |
| "correct_sum": 0.0 |
| }, |
| { |
| "count": 0, |
| "confidence_sum": 0.0, |
| "correct_sum": 0.0 |
| }, |
| { |
| "count": 1, |
| "confidence_sum": 0.8920316100120544, |
| "correct_sum": 1.0 |
| }, |
| { |
| "count": 126, |
| "confidence_sum": 125.7337554693222, |
| "correct_sum": 125.0 |
| } |
| ], |
| "accuracy_vs_coverage": { |
| "0.25": { |
| "n": 32, |
| "accuracy": 1.0, |
| "min_confidence": 1.0 |
| }, |
| "0.5": { |
| "n": 64, |
| "accuracy": 1.0, |
| "min_confidence": 1.0 |
| }, |
| "0.75": { |
| "n": 96, |
| "accuracy": 1.0, |
| "min_confidence": 0.9999983310699463 |
| }, |
| "1.0": { |
| "n": 128, |
| "accuracy": 0.984375, |
| "min_confidence": 0.5795621275901794 |
| } |
| } |
| }, |
| "boolq": { |
| "n": 128, |
| "accuracy": 0.8359375, |
| "nll": 0.5700947275718136, |
| "brier_multiclass_sum": 0.29235693626842657, |
| "ece_top_label_10_equal_width_bins": 0.1428935481235385, |
| "reliability_bins": [ |
| { |
| "count": 0, |
| "confidence_sum": 0.0, |
| "correct_sum": 0.0 |
| }, |
| { |
| "count": 0, |
| "confidence_sum": 0.0, |
| "correct_sum": 0.0 |
| }, |
| { |
| "count": 0, |
| "confidence_sum": 0.0, |
| "correct_sum": 0.0 |
| }, |
| { |
| "count": 0, |
| "confidence_sum": 0.0, |
| "correct_sum": 0.0 |
| }, |
| { |
| "count": 0, |
| "confidence_sum": 0.0, |
| "correct_sum": 0.0 |
| }, |
| { |
| "count": 1, |
| "confidence_sum": 0.527448296546936, |
| "correct_sum": 0.0 |
| }, |
| { |
| "count": 4, |
| "confidence_sum": 2.6020392775535583, |
| "correct_sum": 4.0 |
| }, |
| { |
| "count": 1, |
| "confidence_sum": 0.7223353981971741, |
| "correct_sum": 0.0 |
| }, |
| { |
| "count": 12, |
| "confidence_sum": 10.259824514389038, |
| "correct_sum": 9.0 |
| }, |
| { |
| "count": 110, |
| "confidence_sum": 108.38280522823334, |
| "correct_sum": 94.0 |
| } |
| ], |
| "accuracy_vs_coverage": { |
| "0.25": { |
| "n": 32, |
| "accuracy": 0.96875, |
| "min_confidence": 0.9951808452606201 |
| }, |
| "0.5": { |
| "n": 64, |
| "accuracy": 0.9375, |
| "min_confidence": 0.9884833693504333 |
| }, |
| "0.75": { |
| "n": 96, |
| "accuracy": 0.8958333333333334, |
| "min_confidence": 0.9682738780975342 |
| }, |
| "1.0": { |
| "n": 128, |
| "accuracy": 0.8359375, |
| "min_confidence": 0.527448296546936 |
| } |
| } |
| }, |
| "snli": { |
| "n": 128, |
| "accuracy": 0.84375, |
| "nll": 0.5762057779347742, |
| "brier_multiclass_sum": 0.26571296921680587, |
| "ece_top_label_10_equal_width_bins": 0.10911161499097943, |
| "reliability_bins": [ |
| { |
| "count": 0, |
| "confidence_sum": 0.0, |
| "correct_sum": 0.0 |
| }, |
| { |
| "count": 0, |
| "confidence_sum": 0.0, |
| "correct_sum": 0.0 |
| }, |
| { |
| "count": 0, |
| "confidence_sum": 0.0, |
| "correct_sum": 0.0 |
| }, |
| { |
| "count": 0, |
| "confidence_sum": 0.0, |
| "correct_sum": 0.0 |
| }, |
| { |
| "count": 0, |
| "confidence_sum": 0.0, |
| "correct_sum": 0.0 |
| }, |
| { |
| "count": 1, |
| "confidence_sum": 0.5787264704704285, |
| "correct_sum": 0.0 |
| }, |
| { |
| "count": 5, |
| "confidence_sum": 3.211406707763672, |
| "correct_sum": 3.0 |
| }, |
| { |
| "count": 4, |
| "confidence_sum": 3.022723138332367, |
| "correct_sum": 3.0 |
| }, |
| { |
| "count": 6, |
| "confidence_sum": 5.114439070224762, |
| "correct_sum": 3.0 |
| }, |
| { |
| "count": 112, |
| "confidence_sum": 110.03899133205414, |
| "correct_sum": 99.0 |
| } |
| ], |
| "accuracy_vs_coverage": { |
| "0.25": { |
| "n": 32, |
| "accuracy": 0.9375, |
| "min_confidence": 0.9953944683074951 |
| }, |
| "0.5": { |
| "n": 64, |
| "accuracy": 0.9375, |
| "min_confidence": 0.9888361096382141 |
| }, |
| "0.75": { |
| "n": 96, |
| "accuracy": 0.9166666666666666, |
| "min_confidence": 0.9661396741867065 |
| }, |
| "1.0": { |
| "n": 128, |
| "accuracy": 0.84375, |
| "min_confidence": 0.5787264704704285 |
| } |
| } |
| } |
| } |
| } |
| } |
| ], |
| "checkpoints": { |
| "checkpoint.pt": { |
| "path": "/home/andy/ai/opensysone/runs/20260916T193803Z-24h/training/checkpoint.pt", |
| "bytes": 202309085, |
| "modified_utc": "2026-09-17T01:59:13.373975+00:00", |
| "sha256": "ab70dcb62432039d93ec71e200157250dc59c563404237e4dfad3662fd30f44f", |
| "metadata": { |
| "step": 6000, |
| "best_validation_macro_nll": 0.25676974726429014, |
| "selection_metric": "crossfit_temperature_nll_v1", |
| "best_validation_selection_score": 0.2552943737691716, |
| "stale_evaluations": 8, |
| "source_commit": "4a60423c39d70f8d50472ce4f4f7fa4a4bd9fce1", |
| "config": { |
| "command": "train", |
| "model": "/home/andy/ai/models/opensysone/Qwen3.5-2B-15852e8c", |
| "dataset": "/home/andy/ai/opensysone/data/public-decisions-v1-20260916", |
| "output": "/home/andy/ai/opensysone/runs/20260916T193803Z-24h/training", |
| "resume": "/home/andy/ai/opensysone/runs/20260916T182352Z-train/artifacts/checkpoint.pt", |
| "warm_start": null, |
| "steps": null, |
| "epochs": 3, |
| "rank": 16, |
| "alpha": 32.0, |
| "head_only": false, |
| "two_pass": false, |
| "lr": 0.0001, |
| "head_lr": 0.0001, |
| "seed": 431, |
| "effective_batch": 4, |
| "branch_batch_size": 2, |
| "max_tokens": 768, |
| "save_steps": 250, |
| "save_seconds": 900, |
| "eval_steps": 500, |
| "validation_per_family": 128, |
| "patience": 8, |
| "deadline": "2026-09-17T16:00:00+00:00", |
| "schedule_steps": null, |
| "selection_metric": "crossfit_temperature_nll_v1", |
| "adapters": true |
| }, |
| "initialization": { |
| "kind": "resume", |
| "parent_checkpoint": "/home/andy/ai/opensysone/runs/20260916T182352Z-train/artifacts/checkpoint.pt", |
| "parent_checkpoint_sha256": "af5790ae2f2b56477ebbdf6ab9c418d895e48d2bf5a6416e11b6e9863ad1db55", |
| "restores_optimizer": true, |
| "restores_rng": true, |
| "parent_step": 40, |
| "parent_source_commit": "f1c9322c5313021cf9f90984c8a451bea165f633", |
| "parent_initialization": null, |
| "selection_policy_change": { |
| "from": "raw_nll", |
| "to": "crossfit_temperature_nll_v1", |
| "optimizer_and_rng_unchanged": true |
| } |
| } |
| }, |
| "has_optimizer": true, |
| "has_rng": true |
| }, |
| "best.pt": { |
| "path": "/home/andy/ai/opensysone/runs/20260916T193803Z-24h/training/best.pt", |
| "bytes": 67383903, |
| "modified_utc": "2026-09-16T21:45:31.514410+00:00", |
| "sha256": "be08efdc9121a64547b5c9331d5908eeb0978c4fbc962885fd34f9758ec89d11", |
| "metadata": { |
| "step": 2000, |
| "best_validation_macro_nll": 0.25676974726429014, |
| "selection_metric": "crossfit_temperature_nll_v1", |
| "best_validation_selection_score": 0.2552943737691716, |
| "stale_evaluations": 0, |
| "source_commit": "4a60423c39d70f8d50472ce4f4f7fa4a4bd9fce1", |
| "config": { |
| "command": "train", |
| "model": "/home/andy/ai/models/opensysone/Qwen3.5-2B-15852e8c", |
| "dataset": "/home/andy/ai/opensysone/data/public-decisions-v1-20260916", |
| "output": "/home/andy/ai/opensysone/runs/20260916T193803Z-24h/training", |
| "resume": "/home/andy/ai/opensysone/runs/20260916T182352Z-train/artifacts/checkpoint.pt", |
| "warm_start": null, |
| "steps": null, |
| "epochs": 3, |
| "rank": 16, |
| "alpha": 32.0, |
| "head_only": false, |
| "two_pass": false, |
| "lr": 0.0001, |
| "head_lr": 0.0001, |
| "seed": 431, |
| "effective_batch": 4, |
| "branch_batch_size": 2, |
| "max_tokens": 768, |
| "save_steps": 250, |
| "save_seconds": 900, |
| "eval_steps": 500, |
| "validation_per_family": 128, |
| "patience": 8, |
| "deadline": "2026-09-17T16:00:00+00:00", |
| "schedule_steps": null, |
| "selection_metric": "crossfit_temperature_nll_v1", |
| "adapters": true |
| }, |
| "initialization": { |
| "kind": "resume", |
| "parent_checkpoint": "/home/andy/ai/opensysone/runs/20260916T182352Z-train/artifacts/checkpoint.pt", |
| "parent_checkpoint_sha256": "af5790ae2f2b56477ebbdf6ab9c418d895e48d2bf5a6416e11b6e9863ad1db55", |
| "restores_optimizer": true, |
| "restores_rng": true, |
| "parent_step": 40, |
| "parent_source_commit": "f1c9322c5313021cf9f90984c8a451bea165f633", |
| "parent_initialization": null, |
| "selection_policy_change": { |
| "from": "raw_nll", |
| "to": "crossfit_temperature_nll_v1", |
| "optimizer_and_rng_unchanged": true |
| } |
| } |
| }, |
| "has_optimizer": false, |
| "has_rng": false |
| } |
| }, |
| "final_correctness": { |
| "branch_chunks_1_probability_max_abs": 4.76837158203125e-07, |
| "branch_chunks_2_probability_max_abs": 0.0, |
| "branch_chunks_4_probability_max_abs": 6.556510925292969e-07, |
| "question_isolation_probability_max_abs": 2.0489096641540527e-08, |
| "candidate_permutation_probability_max_abs": 1.1920928955078125e-07, |
| "repeat_probability_max_abs": 0.0, |
| "tolerance_probability_abs": 0.0001 |
| }, |
| "best_selection": { |
| "metric": "crossfit_temperature_nll_v1", |
| "score": 0.2552943737691716, |
| "raw_macro_nll": 0.25676974955497256, |
| "accuracy": 0.8984375, |
| "folds": [ |
| { |
| "fold": 0, |
| "temperature_index": 48, |
| "temperature": 1.2705741052085413, |
| "training_macro_nll": 0.27533710550694335, |
| "heldout_ids": [ |
| "arc:validation:ARC-Challenge:122", |
| "arc:validation:ARC-Challenge:126", |
| "arc:validation:ARC-Challenge:175", |
| "arc:validation:ARC-Challenge:187", |
| "arc:validation:ARC-Challenge:189", |
| "arc:validation:ARC-Challenge:192", |
| "arc:validation:ARC-Challenge:208", |
| "arc:validation:ARC-Challenge:238", |
| "arc:validation:ARC-Challenge:290", |
| "arc:validation:ARC-Challenge:293", |
| "arc:validation:ARC-Challenge:35", |
| "arc:validation:ARC-Challenge:43", |
| "arc:validation:ARC-Challenge:58", |
| "arc:validation:ARC-Challenge:7", |
| "arc:validation:ARC-Easy:160", |
| "arc:validation:ARC-Easy:201", |
| "arc:validation:ARC-Easy:229", |
| "arc:validation:ARC-Easy:238", |
| "arc:validation:ARC-Easy:253", |
| "arc:validation:ARC-Easy:259", |
| "arc:validation:ARC-Easy:272", |
| "arc:validation:ARC-Easy:298", |
| "arc:validation:ARC-Easy:329", |
| "arc:validation:ARC-Easy:331", |
| "arc:validation:ARC-Easy:337", |
| "arc:validation:ARC-Easy:362", |
| "arc:validation:ARC-Easy:443", |
| "arc:validation:ARC-Easy:500", |
| "arc:validation:ARC-Easy:545", |
| "arc:validation:ARC-Easy:68", |
| "arc:validation:ARC-Easy:7", |
| "arc:validation:ARC-Easy:74", |
| "banking:train:1053", |
| "banking:train:1594", |
| "banking:train:1747", |
| "banking:train:202", |
| "banking:train:2377", |
| "banking:train:2528", |
| "banking:train:2546", |
| "banking:train:2715", |
| "banking:train:3220", |
| "banking:train:3499", |
| "banking:train:3806", |
| "banking:train:3838", |
| "banking:train:3993", |
| "banking:train:4220", |
| "banking:train:5", |
| "banking:train:5192", |
| "banking:train:5394", |
| "banking:train:5846", |
| "banking:train:5863", |
| "banking:train:6032", |
| "banking:train:6508", |
| "banking:train:7579", |
| "banking:train:7600", |
| "banking:train:7987", |
| "banking:train:8155", |
| "banking:train:850", |
| "banking:train:8626", |
| "banking:train:8929", |
| "banking:train:9355", |
| "banking:train:9812", |
| "banking:train:987", |
| "banking:train:9972", |
| "boolq:validation:103", |
| "boolq:validation:1042", |
| "boolq:validation:1059", |
| "boolq:validation:1541", |
| "boolq:validation:1592", |
| "boolq:validation:1927", |
| "boolq:validation:2014", |
| "boolq:validation:2023", |
| "boolq:validation:2098", |
| "boolq:validation:2139", |
| "boolq:validation:2439", |
| "boolq:validation:2485", |
| "boolq:validation:2607", |
| "boolq:validation:2611", |
| "boolq:validation:2720", |
| "boolq:validation:2751", |
| "boolq:validation:282", |
| "boolq:validation:2843", |
| "boolq:validation:2953", |
| "boolq:validation:301", |
| "boolq:validation:3089", |
| "boolq:validation:313", |
| "boolq:validation:3201", |
| "boolq:validation:322", |
| "boolq:validation:3231", |
| "boolq:validation:39", |
| "boolq:validation:461", |
| "boolq:validation:558", |
| "boolq:validation:589", |
| "boolq:validation:731", |
| "boolq:validation:883", |
| "boolq:validation:915", |
| "snli:validation:1156", |
| "snli:validation:1330", |
| "snli:validation:1559", |
| "snli:validation:1735", |
| "snli:validation:1876", |
| "snli:validation:2251", |
| "snli:validation:2257", |
| "snli:validation:2377", |
| "snli:validation:2881", |
| "snli:validation:291", |
| "snli:validation:300", |
| "snli:validation:3049", |
| "snli:validation:3496", |
| "snli:validation:3871", |
| "snli:validation:4414", |
| "snli:validation:5140", |
| "snli:validation:5440", |
| "snli:validation:5806", |
| "snli:validation:6442", |
| "snli:validation:6772", |
| "snli:validation:6949", |
| "snli:validation:7228", |
| "snli:validation:7465", |
| "snli:validation:7510", |
| "snli:validation:7726", |
| "snli:validation:8095", |
| "snli:validation:8143", |
| "snli:validation:8623", |
| "snli:validation:8962", |
| "snli:validation:9451", |
| "snli:validation:9574", |
| "snli:validation:9655" |
| ], |
| "heldout_group_count": 128, |
| "training_group_count": 384 |
| }, |
| { |
| "fold": 1, |
| "temperature_index": 47, |
| "temperature": 1.2050359403717974, |
| "training_macro_nll": 0.25585027399066257, |
| "heldout_ids": [ |
| "arc:validation:ARC-Challenge:101", |
| "arc:validation:ARC-Challenge:129", |
| "arc:validation:ARC-Challenge:148", |
| "arc:validation:ARC-Challenge:156", |
| "arc:validation:ARC-Challenge:167", |
| "arc:validation:ARC-Challenge:249", |
| "arc:validation:ARC-Challenge:263", |
| "arc:validation:ARC-Challenge:266", |
| "arc:validation:ARC-Challenge:267", |
| "arc:validation:ARC-Challenge:28", |
| "arc:validation:ARC-Challenge:281", |
| "arc:validation:ARC-Challenge:57", |
| "arc:validation:ARC-Challenge:63", |
| "arc:validation:ARC-Challenge:64", |
| "arc:validation:ARC-Challenge:86", |
| "arc:validation:ARC-Challenge:99", |
| "arc:validation:ARC-Easy:106", |
| "arc:validation:ARC-Easy:151", |
| "arc:validation:ARC-Easy:157", |
| "arc:validation:ARC-Easy:166", |
| "arc:validation:ARC-Easy:202", |
| "arc:validation:ARC-Easy:219", |
| "arc:validation:ARC-Easy:224", |
| "arc:validation:ARC-Easy:322", |
| "arc:validation:ARC-Easy:344", |
| "arc:validation:ARC-Easy:382", |
| "arc:validation:ARC-Easy:409", |
| "arc:validation:ARC-Easy:455", |
| "arc:validation:ARC-Easy:50", |
| "arc:validation:ARC-Easy:538", |
| "arc:validation:ARC-Easy:544", |
| "arc:validation:ARC-Easy:552", |
| "banking:train:1050", |
| "banking:train:1131", |
| "banking:train:1190", |
| "banking:train:124", |
| "banking:train:137", |
| "banking:train:1493", |
| "banking:train:1498", |
| "banking:train:1624", |
| "banking:train:2614", |
| "banking:train:2739", |
| "banking:train:3055", |
| "banking:train:306", |
| "banking:train:3312", |
| "banking:train:3724", |
| "banking:train:4070", |
| "banking:train:476", |
| "banking:train:4882", |
| "banking:train:5075", |
| "banking:train:5332", |
| "banking:train:5445", |
| "banking:train:6096", |
| "banking:train:746", |
| "banking:train:7828", |
| "banking:train:8176", |
| "banking:train:8315", |
| "banking:train:8653", |
| "banking:train:9019", |
| "banking:train:9370", |
| "banking:train:9454", |
| "banking:train:9595", |
| "banking:train:9862", |
| "banking:train:9963", |
| "boolq:validation:1189", |
| "boolq:validation:1320", |
| "boolq:validation:1326", |
| "boolq:validation:1381", |
| "boolq:validation:1410", |
| "boolq:validation:149", |
| "boolq:validation:1513", |
| "boolq:validation:1732", |
| "boolq:validation:1739", |
| "boolq:validation:1741", |
| "boolq:validation:1814", |
| "boolq:validation:1848", |
| "boolq:validation:1940", |
| "boolq:validation:2077", |
| "boolq:validation:2091", |
| "boolq:validation:2148", |
| "boolq:validation:2215", |
| "boolq:validation:2251", |
| "boolq:validation:2283", |
| "boolq:validation:2321", |
| "boolq:validation:2398", |
| "boolq:validation:253", |
| "boolq:validation:2575", |
| "boolq:validation:2907", |
| "boolq:validation:2909", |
| "boolq:validation:2978", |
| "boolq:validation:359", |
| "boolq:validation:394", |
| "boolq:validation:436", |
| "boolq:validation:579", |
| "boolq:validation:807", |
| "boolq:validation:949", |
| "snli:validation:2389", |
| "snli:validation:2485", |
| "snli:validation:255", |
| "snli:validation:3169", |
| "snli:validation:333", |
| "snli:validation:3700", |
| "snli:validation:3709", |
| "snli:validation:4093", |
| "snli:validation:420", |
| "snli:validation:4492", |
| "snli:validation:4516", |
| "snli:validation:4648", |
| "snli:validation:4888", |
| "snli:validation:5185", |
| "snli:validation:5332", |
| "snli:validation:5947", |
| "snli:validation:6124", |
| "snli:validation:660", |
| "snli:validation:6619", |
| "snli:validation:6898", |
| "snli:validation:7387", |
| "snli:validation:7492", |
| "snli:validation:7544", |
| "snli:validation:7996", |
| "snli:validation:8074", |
| "snli:validation:8554", |
| "snli:validation:8971", |
| "snli:validation:9163", |
| "snli:validation:9277", |
| "snli:validation:9490", |
| "snli:validation:9595", |
| "snli:validation:9688" |
| ], |
| "heldout_group_count": 128, |
| "training_group_count": 384 |
| }, |
| { |
| "fold": 2, |
| "temperature_index": 45, |
| "temperature": 1.0839269140212033, |
| "training_macro_nll": 0.22881493804363362, |
| "heldout_ids": [ |
| "arc:validation:ARC-Challenge:120", |
| "arc:validation:ARC-Challenge:130", |
| "arc:validation:ARC-Challenge:194", |
| "arc:validation:ARC-Challenge:218", |
| "arc:validation:ARC-Challenge:235", |
| "arc:validation:ARC-Challenge:272", |
| "arc:validation:ARC-Challenge:277", |
| "arc:validation:ARC-Challenge:48", |
| "arc:validation:ARC-Challenge:88", |
| "arc:validation:ARC-Easy:115", |
| "arc:validation:ARC-Easy:12", |
| "arc:validation:ARC-Easy:131", |
| "arc:validation:ARC-Easy:145", |
| "arc:validation:ARC-Easy:179", |
| "arc:validation:ARC-Easy:18", |
| "arc:validation:ARC-Easy:186", |
| "arc:validation:ARC-Easy:191", |
| "arc:validation:ARC-Easy:215", |
| "arc:validation:ARC-Easy:283", |
| "arc:validation:ARC-Easy:314", |
| "arc:validation:ARC-Easy:315", |
| "arc:validation:ARC-Easy:378", |
| "arc:validation:ARC-Easy:384", |
| "arc:validation:ARC-Easy:403", |
| "arc:validation:ARC-Easy:432", |
| "arc:validation:ARC-Easy:439", |
| "arc:validation:ARC-Easy:448", |
| "arc:validation:ARC-Easy:461", |
| "arc:validation:ARC-Easy:469", |
| "arc:validation:ARC-Easy:470", |
| "arc:validation:ARC-Easy:504", |
| "arc:validation:ARC-Easy:561", |
| "banking:train:1142", |
| "banking:train:148", |
| "banking:train:1633", |
| "banking:train:1927", |
| "banking:train:2186", |
| "banking:train:2251", |
| "banking:train:23", |
| "banking:train:3048", |
| "banking:train:3665", |
| "banking:train:3952", |
| "banking:train:3999", |
| "banking:train:4364", |
| "banking:train:4619", |
| "banking:train:5049", |
| "banking:train:5335", |
| "banking:train:549", |
| "banking:train:5957", |
| "banking:train:5985", |
| "banking:train:6132", |
| "banking:train:6240", |
| "banking:train:6488", |
| "banking:train:7065", |
| "banking:train:7749", |
| "banking:train:8470", |
| "banking:train:8761", |
| "banking:train:895", |
| "banking:train:9256", |
| "banking:train:9406", |
| "banking:train:9450", |
| "banking:train:9467", |
| "banking:train:9588", |
| "banking:train:9716", |
| "boolq:validation:1130", |
| "boolq:validation:1167", |
| "boolq:validation:124", |
| "boolq:validation:1255", |
| "boolq:validation:1256", |
| "boolq:validation:1373", |
| "boolq:validation:15", |
| "boolq:validation:155", |
| "boolq:validation:1568", |
| "boolq:validation:16", |
| "boolq:validation:1654", |
| "boolq:validation:1702", |
| "boolq:validation:1792", |
| "boolq:validation:1818", |
| "boolq:validation:210", |
| "boolq:validation:2247", |
| "boolq:validation:2259", |
| "boolq:validation:2362", |
| "boolq:validation:2443", |
| "boolq:validation:2524", |
| "boolq:validation:2553", |
| "boolq:validation:2623", |
| "boolq:validation:2660", |
| "boolq:validation:2669", |
| "boolq:validation:2679", |
| "boolq:validation:2832", |
| "boolq:validation:357", |
| "boolq:validation:453", |
| "boolq:validation:49", |
| "boolq:validation:808", |
| "boolq:validation:90", |
| "boolq:validation:910", |
| "snli:validation:1612", |
| "snli:validation:168", |
| "snli:validation:1693", |
| "snli:validation:2134", |
| "snli:validation:2248", |
| "snli:validation:2422", |
| "snli:validation:2587", |
| "snli:validation:3007", |
| "snli:validation:3097", |
| "snli:validation:3772", |
| "snli:validation:4012", |
| "snli:validation:4168", |
| "snli:validation:4537", |
| "snli:validation:4681", |
| "snli:validation:5050", |
| "snli:validation:5632", |
| "snli:validation:5761", |
| "snli:validation:6055", |
| "snli:validation:6283", |
| "snli:validation:6715", |
| "snli:validation:7117", |
| "snli:validation:7192", |
| "snli:validation:7507", |
| "snli:validation:7594", |
| "snli:validation:7924", |
| "snli:validation:7954", |
| "snli:validation:8035", |
| "snli:validation:8110", |
| "snli:validation:8995", |
| "snli:validation:949", |
| "snli:validation:9523", |
| "snli:validation:9589" |
| ], |
| "heldout_group_count": 128, |
| "training_group_count": 384 |
| }, |
| { |
| "fold": 3, |
| "temperature_index": 47, |
| "temperature": 1.2050359403717974, |
| "training_macro_nll": 0.24681503020373022, |
| "heldout_ids": [ |
| "arc:validation:ARC-Challenge:117", |
| "arc:validation:ARC-Challenge:135", |
| "arc:validation:ARC-Challenge:139", |
| "arc:validation:ARC-Challenge:154", |
| "arc:validation:ARC-Challenge:174", |
| "arc:validation:ARC-Challenge:199", |
| "arc:validation:ARC-Challenge:287", |
| "arc:validation:ARC-Challenge:295", |
| "arc:validation:ARC-Challenge:4", |
| "arc:validation:ARC-Challenge:50", |
| "arc:validation:ARC-Challenge:51", |
| "arc:validation:ARC-Challenge:61", |
| "arc:validation:ARC-Easy:107", |
| "arc:validation:ARC-Easy:121", |
| "arc:validation:ARC-Easy:124", |
| "arc:validation:ARC-Easy:171", |
| "arc:validation:ARC-Easy:235", |
| "arc:validation:ARC-Easy:241", |
| "arc:validation:ARC-Easy:279", |
| "arc:validation:ARC-Easy:289", |
| "arc:validation:ARC-Easy:37", |
| "arc:validation:ARC-Easy:389", |
| "arc:validation:ARC-Easy:394", |
| "arc:validation:ARC-Easy:399", |
| "arc:validation:ARC-Easy:423", |
| "arc:validation:ARC-Easy:492", |
| "arc:validation:ARC-Easy:499", |
| "arc:validation:ARC-Easy:523", |
| "arc:validation:ARC-Easy:533", |
| "arc:validation:ARC-Easy:569", |
| "arc:validation:ARC-Easy:69", |
| "arc:validation:ARC-Easy:93", |
| "banking:train:2049", |
| "banking:train:2350", |
| "banking:train:2497", |
| "banking:train:2506", |
| "banking:train:2720", |
| "banking:train:2788", |
| "banking:train:2967", |
| "banking:train:3502", |
| "banking:train:3726", |
| "banking:train:4014", |
| "banking:train:4735", |
| "banking:train:4803", |
| "banking:train:4917", |
| "banking:train:5477", |
| "banking:train:5851", |
| "banking:train:6551", |
| "banking:train:6839", |
| "banking:train:6926", |
| "banking:train:7809", |
| "banking:train:8092", |
| "banking:train:8186", |
| "banking:train:8215", |
| "banking:train:8667", |
| "banking:train:8752", |
| "banking:train:8763", |
| "banking:train:8851", |
| "banking:train:8917", |
| "banking:train:9145", |
| "banking:train:9279", |
| "banking:train:9676", |
| "banking:train:9837", |
| "banking:train:9967", |
| "boolq:validation:106", |
| "boolq:validation:1065", |
| "boolq:validation:1097", |
| "boolq:validation:1336", |
| "boolq:validation:1350", |
| "boolq:validation:1383", |
| "boolq:validation:1458", |
| "boolq:validation:1641", |
| "boolq:validation:1873", |
| "boolq:validation:1930", |
| "boolq:validation:1989", |
| "boolq:validation:2103", |
| "boolq:validation:234", |
| "boolq:validation:2355", |
| "boolq:validation:2384", |
| "boolq:validation:2440", |
| "boolq:validation:2444", |
| "boolq:validation:2478", |
| "boolq:validation:2872", |
| "boolq:validation:2879", |
| "boolq:validation:2924", |
| "boolq:validation:2971", |
| "boolq:validation:3164", |
| "boolq:validation:4", |
| "boolq:validation:415", |
| "boolq:validation:419", |
| "boolq:validation:428", |
| "boolq:validation:512", |
| "boolq:validation:563", |
| "boolq:validation:643", |
| "boolq:validation:696", |
| "boolq:validation:874", |
| "snli:validation:1102", |
| "snli:validation:1240", |
| "snli:validation:1261", |
| "snli:validation:1738", |
| "snli:validation:1978", |
| "snli:validation:2563", |
| "snli:validation:2659", |
| "snli:validation:2767", |
| "snli:validation:2836", |
| "snli:validation:3232", |
| "snli:validation:3655", |
| "snli:validation:3913", |
| "snli:validation:405", |
| "snli:validation:4051", |
| "snli:validation:4300", |
| "snli:validation:4846", |
| "snli:validation:498", |
| "snli:validation:5224", |
| "snli:validation:5575", |
| "snli:validation:6598", |
| "snli:validation:666", |
| "snli:validation:6925", |
| "snli:validation:6964", |
| "snli:validation:7045", |
| "snli:validation:7351", |
| "snli:validation:7588", |
| "snli:validation:7723", |
| "snli:validation:7750", |
| "snli:validation:7909", |
| "snli:validation:8974", |
| "snli:validation:9385", |
| "snli:validation:9646" |
| ], |
| "heldout_group_count": 128, |
| "training_group_count": 384 |
| } |
| ], |
| "per_family": { |
| "arc": { |
| "count": 128, |
| "score": 0.3379051798328397, |
| "raw_macro_nll": 0.33955257651665527, |
| "accuracy": 0.890625 |
| }, |
| "banking": { |
| "count": 128, |
| "score": 0.06793271948328157, |
| "raw_macro_nll": 0.06537141992655102, |
| "accuracy": 0.9609375 |
| }, |
| "boolq": { |
| "count": 128, |
| "score": 0.33084980282335186, |
| "raw_macro_nll": 0.34265219288051024, |
| "accuracy": 0.84375 |
| }, |
| "snli": { |
| "count": 128, |
| "score": 0.28448979293721316, |
| "raw_macro_nll": 0.2795028088961736, |
| "accuracy": 0.8984375 |
| } |
| }, |
| "policy": { |
| "id": "crossfit_temperature_nll_v1", |
| "folds": 4, |
| "seed": 431, |
| "grouping": "source group, nested within task family; groups shuffled within sorted families and assigned round-robin", |
| "temperature_grid": { |
| "count": 101, |
| "log10_min": -1.0, |
| "log10_max": 1.3, |
| "arithmetic": "Python float64" |
| }, |
| "temperature_fit": "minimum macro-family NLL on the other three folds; smallest temperature wins ties", |
| "score": "macro-family mean of all held-out decision NLLs", |
| "deployment_temperature": "fit afresh on reserved calibration only after model selection" |
| } |
| }, |
| "small_file_list": [ |
| "exit_code", |
| "plan.json", |
| "startup-verification.json", |
| "state.json", |
| "supervisor.log", |
| "training/best_validation_predictions.json", |
| "training/best_validation_selection.json", |
| "training/correctness_final.json", |
| "training/correctness_initial.json", |
| "training/data_filter.json", |
| "training/inherited_validation_selection.json", |
| "training/initial_validation_selection.json", |
| "training/manifest.json", |
| "training/resumed_initial_predictions.json", |
| "training/summary.json", |
| "training/training.jsonl", |
| "training/validation.jsonl", |
| "training/validation_step_000040_predictions.json", |
| "training/validation_step_000500_predictions.json", |
| "training/validation_step_001000_predictions.json", |
| "training/validation_step_001500_predictions.json", |
| "training/validation_step_002000_predictions.json", |
| "training/validation_step_002500_predictions.json", |
| "training/validation_step_003000_predictions.json", |
| "training/validation_step_003500_predictions.json", |
| "training/validation_step_004000_predictions.json", |
| "training/validation_step_004500_predictions.json", |
| "training/validation_step_005000_predictions.json", |
| "training/validation_step_005500_predictions.json", |
| "training/validation_step_006000_predictions.json", |
| "training.log", |
| "training_exit_code" |
| ], |
| "log_error_lines": { |
| "supervisor.log": [], |
| "training.log": [] |
| } |
| } |
|
|
