| { |
| "status": "completed_step_target", |
| "completed_steps": 41, |
| "target_steps": 41, |
| "initial_validation": { |
| "overall": { |
| "n": 512, |
| "accuracy": 0.828125, |
| "nll": 0.4981533203378369, |
| "brier_multiclass_sum": 0.2607127369212848, |
| "ece_top_label_10_equal_width_bins": 0.04818323429208249, |
| "reliability_bins": [ |
| { |
| "count": 0, |
| "confidence_sum": 0.0, |
| "correct_sum": 0.0 |
| }, |
| { |
| "count": 0, |
| "confidence_sum": 0.0, |
| "correct_sum": 0.0 |
| }, |
| { |
| "count": 2, |
| "confidence_sum": 0.5327933728694916, |
| "correct_sum": 1.0 |
| }, |
| { |
| "count": 4, |
| "confidence_sum": 1.4359374046325684, |
| "correct_sum": 3.0 |
| }, |
| { |
| "count": 17, |
| "confidence_sum": 7.9667291939258575, |
| "correct_sum": 7.0 |
| }, |
| { |
| "count": 55, |
| "confidence_sum": 30.154281735420227, |
| "correct_sum": 35.0 |
| }, |
| { |
| "count": 53, |
| "confidence_sum": 34.67930006980896, |
| "correct_sum": 34.0 |
| }, |
| { |
| "count": 46, |
| "confidence_sum": 34.177616238594055, |
| "correct_sum": 36.0 |
| }, |
| { |
| "count": 54, |
| "confidence_sum": 46.13812476396561, |
| "correct_sum": 44.0 |
| }, |
| { |
| "count": 281, |
| "confidence_sum": 276.18629068136215, |
| "correct_sum": 264.0 |
| } |
| ], |
| "accuracy_vs_coverage": { |
| "0.25": { |
| "n": 128, |
| "accuracy": 0.96875, |
| "min_confidence": 0.9978359341621399 |
| }, |
| "0.5": { |
| "n": 256, |
| "accuracy": 0.94140625, |
| "min_confidence": 0.9372246861457825 |
| }, |
| "0.75": { |
| "n": 384, |
| "accuracy": 0.9010416666666666, |
| "min_confidence": 0.6965566873550415 |
| }, |
| "1.0": { |
| "n": 512, |
| "accuracy": 0.828125, |
| "min_confidence": 0.2647492587566376 |
| } |
| } |
| }, |
| "per_family": { |
| "arc": { |
| "n": 128, |
| "accuracy": 0.8125, |
| "nll": 0.4990136543534618, |
| "brier_multiclass_sum": 0.26976813507500774, |
| "ece_top_label_10_equal_width_bins": 0.048822794342413545, |
| "reliability_bins": [ |
| { |
| "count": 0, |
| "confidence_sum": 0.0, |
| "correct_sum": 0.0 |
| }, |
| { |
| "count": 0, |
| "confidence_sum": 0.0, |
| "correct_sum": 0.0 |
| }, |
| { |
| "count": 2, |
| "confidence_sum": 0.5327933728694916, |
| "correct_sum": 1.0 |
| }, |
| { |
| "count": 4, |
| "confidence_sum": 1.4359374046325684, |
| "correct_sum": 3.0 |
| }, |
| { |
| "count": 8, |
| "confidence_sum": 3.675512135028839, |
| "correct_sum": 2.0 |
| }, |
| { |
| "count": 9, |
| "confidence_sum": 4.856119275093079, |
| "correct_sum": 6.0 |
| }, |
| { |
| "count": 13, |
| "confidence_sum": 8.502536118030548, |
| "correct_sum": 8.0 |
| }, |
| { |
| "count": 13, |
| "confidence_sum": 9.665025651454926, |
| "correct_sum": 10.0 |
| }, |
| { |
| "count": 28, |
| "confidence_sum": 24.128599762916565, |
| "correct_sum": 24.0 |
| }, |
| { |
| "count": 51, |
| "confidence_sum": 49.567454636096954, |
| "correct_sum": 50.0 |
| } |
| ], |
| "accuracy_vs_coverage": { |
| "0.25": { |
| "n": 32, |
| "accuracy": 1.0, |
| "min_confidence": 0.974073588848114 |
| }, |
| "0.5": { |
| "n": 64, |
| "accuracy": 0.953125, |
| "min_confidence": 0.8664795756340027 |
| }, |
| "0.75": { |
| "n": 96, |
| "accuracy": 0.8958333333333334, |
| "min_confidence": 0.6717178225517273 |
| }, |
| "1.0": { |
| "n": 128, |
| "accuracy": 0.8125, |
| "min_confidence": 0.2647492587566376 |
| } |
| } |
| }, |
| "banking": { |
| "n": 128, |
| "accuracy": 0.8828125, |
| "nll": 0.2938805352743866, |
| "brier_multiclass_sum": 0.15680863450549035, |
| "ece_top_label_10_equal_width_bins": 0.060145952040329576, |
| "reliability_bins": [ |
| { |
| "count": 0, |
| "confidence_sum": 0.0, |
| "correct_sum": 0.0 |
| }, |
| { |
| "count": 0, |
| "confidence_sum": 0.0, |
| "correct_sum": 0.0 |
| }, |
| { |
| "count": 0, |
| "confidence_sum": 0.0, |
| "correct_sum": 0.0 |
| }, |
| { |
| "count": 0, |
| "confidence_sum": 0.0, |
| "correct_sum": 0.0 |
| }, |
| { |
| "count": 1, |
| "confidence_sum": 0.47516098618507385, |
| "correct_sum": 0.0 |
| }, |
| { |
| "count": 6, |
| "confidence_sum": 3.3597115874290466, |
| "correct_sum": 1.0 |
| }, |
| { |
| "count": 5, |
| "confidence_sum": 3.223210096359253, |
| "correct_sum": 3.0 |
| }, |
| { |
| "count": 3, |
| "confidence_sum": 2.2593541145324707, |
| "correct_sum": 1.0 |
| }, |
| { |
| "count": 7, |
| "confidence_sum": 5.898111343383789, |
| "correct_sum": 4.0 |
| }, |
| { |
| "count": 106, |
| "confidence_sum": 105.48313373327255, |
| "correct_sum": 104.0 |
| } |
| ], |
| "accuracy_vs_coverage": { |
| "0.25": { |
| "n": 32, |
| "accuracy": 1.0, |
| "min_confidence": 0.9999927282333374 |
| }, |
| "0.5": { |
| "n": 64, |
| "accuracy": 1.0, |
| "min_confidence": 0.9998255372047424 |
| }, |
| "0.75": { |
| "n": 96, |
| "accuracy": 0.9895833333333334, |
| "min_confidence": 0.9847974181175232 |
| }, |
| "1.0": { |
| "n": 128, |
| "accuracy": 0.8828125, |
| "min_confidence": 0.47516098618507385 |
| } |
| } |
| }, |
| "boolq": { |
| "n": 128, |
| "accuracy": 0.796875, |
| "nll": 0.730819056238805, |
| "brier_multiclass_sum": 0.3320798331012733, |
| "ece_top_label_10_equal_width_bins": 0.1302144955843687, |
| "reliability_bins": [ |
| { |
| "count": 0, |
| "confidence_sum": 0.0, |
| "correct_sum": 0.0 |
| }, |
| { |
| "count": 0, |
| "confidence_sum": 0.0, |
| "correct_sum": 0.0 |
| }, |
| { |
| "count": 0, |
| "confidence_sum": 0.0, |
| "correct_sum": 0.0 |
| }, |
| { |
| "count": 0, |
| "confidence_sum": 0.0, |
| "correct_sum": 0.0 |
| }, |
| { |
| "count": 0, |
| "confidence_sum": 0.0, |
| "correct_sum": 0.0 |
| }, |
| { |
| "count": 9, |
| "confidence_sum": 5.026383757591248, |
| "correct_sum": 6.0 |
| }, |
| { |
| "count": 7, |
| "confidence_sum": 4.494362056255341, |
| "correct_sum": 4.0 |
| }, |
| { |
| "count": 6, |
| "confidence_sum": 4.508205056190491, |
| "correct_sum": 3.0 |
| }, |
| { |
| "count": 10, |
| "confidence_sum": 8.55346018075943, |
| "correct_sum": 7.0 |
| }, |
| { |
| "count": 96, |
| "confidence_sum": 94.13781189918518, |
| "correct_sum": 82.0 |
| } |
| ], |
| "accuracy_vs_coverage": { |
| "0.25": { |
| "n": 32, |
| "accuracy": 0.875, |
| "min_confidence": 0.9985519051551819 |
| }, |
| "0.5": { |
| "n": 64, |
| "accuracy": 0.890625, |
| "min_confidence": 0.9844649434089661 |
| }, |
| "0.75": { |
| "n": 96, |
| "accuracy": 0.8541666666666666, |
| "min_confidence": 0.9054988026618958 |
| }, |
| "1.0": { |
| "n": 128, |
| "accuracy": 0.796875, |
| "min_confidence": 0.5037087798118591 |
| } |
| } |
| }, |
| "snli": { |
| "n": 128, |
| "accuracy": 0.8203125, |
| "nll": 0.46890003548469394, |
| "brier_multiclass_sum": 0.284194345003368, |
| "ece_top_label_10_equal_width_bins": 0.10556101333349943, |
| "reliability_bins": [ |
| { |
| "count": 0, |
| "confidence_sum": 0.0, |
| "correct_sum": 0.0 |
| }, |
| { |
| "count": 0, |
| "confidence_sum": 0.0, |
| "correct_sum": 0.0 |
| }, |
| { |
| "count": 0, |
| "confidence_sum": 0.0, |
| "correct_sum": 0.0 |
| }, |
| { |
| "count": 0, |
| "confidence_sum": 0.0, |
| "correct_sum": 0.0 |
| }, |
| { |
| "count": 8, |
| "confidence_sum": 3.8160560727119446, |
| "correct_sum": 5.0 |
| }, |
| { |
| "count": 31, |
| "confidence_sum": 16.912067115306854, |
| "correct_sum": 22.0 |
| }, |
| { |
| "count": 28, |
| "confidence_sum": 18.45919179916382, |
| "correct_sum": 19.0 |
| }, |
| { |
| "count": 24, |
| "confidence_sum": 17.745031416416168, |
| "correct_sum": 22.0 |
| }, |
| { |
| "count": 9, |
| "confidence_sum": 7.557953476905823, |
| "correct_sum": 9.0 |
| }, |
| { |
| "count": 28, |
| "confidence_sum": 26.997890412807465, |
| "correct_sum": 28.0 |
| } |
| ], |
| "accuracy_vs_coverage": { |
| "0.25": { |
| "n": 32, |
| "accuracy": 1.0, |
| "min_confidence": 0.845478892326355 |
| }, |
| "0.5": { |
| "n": 64, |
| "accuracy": 0.96875, |
| "min_confidence": 0.695654571056366 |
| }, |
| "0.75": { |
| "n": 96, |
| "accuracy": 0.8645833333333334, |
| "min_confidence": 0.5729489326477051 |
| }, |
| "1.0": { |
| "n": 128, |
| "accuracy": 0.8203125, |
| "min_confidence": 0.4349120557308197 |
| } |
| } |
| } |
| } |
| }, |
| "best_validation_macro_nll": 0.49447758084491444, |
| "training_seconds": 151.7393543109938, |
| "median_step_seconds": 4.051890700997319, |
| "processed_decisions": 4, |
| "actual_branch_tokens": 1730, |
| "padded_branch_tokens": 1738, |
| "peak_cuda_allocated_bytes": 8221573120, |
| "peak_cuda_reserved_bytes": 8319401984, |
| "final_mem_available_bytes": 115780268032, |
| "correctness": { |
| "branch_chunks_1_probability_max_abs": 3.039836883544922e-06, |
| "branch_chunks_2_probability_max_abs": 0.0, |
| "branch_chunks_4_probability_max_abs": 3.159046173095703e-06, |
| "question_isolation_probability_max_abs": 1.1324882507324219e-06, |
| "candidate_permutation_probability_max_abs": 2.384185791015625e-06, |
| "repeat_probability_max_abs": 0.0, |
| "tolerance_probability_abs": 0.0001 |
| }, |
| "checkpoint_sha256": "86e19be6e02ef38a017e75bfea917786fa8440197549fc60f7d725d94868257e", |
| "best_sha256": "e2280674990dd4976a5d439a831f184ebb5fc492f5349dc1bfb6513e51fec7d3" |
| } |
|
|
