| { |
| "status": "completed_step_target", |
| "completed_steps": 40, |
| "target_steps": 40, |
| "initial_validation": { |
| "overall": { |
| "n": 512, |
| "accuracy": 0.865234375, |
| "nll": 1.4361620344958197, |
| "brier_multiclass_sum": 0.24158708247170743, |
| "ece_top_label_10_equal_width_bins": 0.11415780044626445, |
| "reliability_bins": [ |
| { |
| "count": 0, |
| "confidence_sum": 0.0, |
| "correct_sum": 0.0 |
| }, |
| { |
| "count": 0, |
| "confidence_sum": 0.0, |
| "correct_sum": 0.0 |
| }, |
| { |
| "count": 0, |
| "confidence_sum": 0.0, |
| "correct_sum": 0.0 |
| }, |
| { |
| "count": 1, |
| "confidence_sum": 0.3630107641220093, |
| "correct_sum": 0.0 |
| }, |
| { |
| "count": 1, |
| "confidence_sum": 0.49957096576690674, |
| "correct_sum": 0.0 |
| }, |
| { |
| "count": 7, |
| "confidence_sum": 3.892404556274414, |
| "correct_sum": 2.0 |
| }, |
| { |
| "count": 5, |
| "confidence_sum": 3.242408335208893, |
| "correct_sum": 3.0 |
| }, |
| { |
| "count": 6, |
| "confidence_sum": 4.528022885322571, |
| "correct_sum": 2.0 |
| }, |
| { |
| "count": 10, |
| "confidence_sum": 8.651127338409424, |
| "correct_sum": 6.0 |
| }, |
| { |
| "count": 482, |
| "confidence_sum": 480.2722489833832, |
| "correct_sum": 430.0 |
| } |
| ], |
| "accuracy_vs_coverage": { |
| "0.25": { |
| "n": 128, |
| "accuracy": 0.9921875, |
| "min_confidence": 1.0 |
| }, |
| "0.5": { |
| "n": 256, |
| "accuracy": 0.96484375, |
| "min_confidence": 1.0 |
| }, |
| "0.75": { |
| "n": 384, |
| "accuracy": 0.9270833333333334, |
| "min_confidence": 0.9998980760574341 |
| }, |
| "1.0": { |
| "n": 512, |
| "accuracy": 0.865234375, |
| "min_confidence": 0.3630107641220093 |
| } |
| } |
| }, |
| "per_family": { |
| "arc": { |
| "n": 128, |
| "accuracy": 0.90625, |
| "nll": 1.010106345727131, |
| "brier_multiclass_sum": 0.15493916615196046, |
| "ece_top_label_10_equal_width_bins": 0.06394888879731297, |
| "reliability_bins": [ |
| { |
| "count": 0, |
| "confidence_sum": 0.0, |
| "correct_sum": 0.0 |
| }, |
| { |
| "count": 0, |
| "confidence_sum": 0.0, |
| "correct_sum": 0.0 |
| }, |
| { |
| "count": 0, |
| "confidence_sum": 0.0, |
| "correct_sum": 0.0 |
| }, |
| { |
| "count": 1, |
| "confidence_sum": 0.3630107641220093, |
| "correct_sum": 0.0 |
| }, |
| { |
| "count": 0, |
| "confidence_sum": 0.0, |
| "correct_sum": 0.0 |
| }, |
| { |
| "count": 3, |
| "confidence_sum": 1.6491040587425232, |
| "correct_sum": 1.0 |
| }, |
| { |
| "count": 3, |
| "confidence_sum": 1.978573203086853, |
| "correct_sum": 2.0 |
| }, |
| { |
| "count": 0, |
| "confidence_sum": 0.0, |
| "correct_sum": 0.0 |
| }, |
| { |
| "count": 1, |
| "confidence_sum": 0.8926308155059814, |
| "correct_sum": 0.0 |
| }, |
| { |
| "count": 120, |
| "confidence_sum": 119.2592853307724, |
| "correct_sum": 113.0 |
| } |
| ], |
| "accuracy_vs_coverage": { |
| "0.25": { |
| "n": 32, |
| "accuracy": 1.0, |
| "min_confidence": 1.0 |
| }, |
| "0.5": { |
| "n": 64, |
| "accuracy": 0.984375, |
| "min_confidence": 1.0 |
| }, |
| "0.75": { |
| "n": 96, |
| "accuracy": 0.96875, |
| "min_confidence": 0.9995371103286743 |
| }, |
| "1.0": { |
| "n": 128, |
| "accuracy": 0.90625, |
| "min_confidence": 0.3630107641220093 |
| } |
| } |
| }, |
| "banking": { |
| "n": 128, |
| "accuracy": 0.90625, |
| "nll": 0.4191825529246126, |
| "brier_multiclass_sum": 0.15350192836631016, |
| "ece_top_label_10_equal_width_bins": 0.06889990298077464, |
| "reliability_bins": [ |
| { |
| "count": 0, |
| "confidence_sum": 0.0, |
| "correct_sum": 0.0 |
| }, |
| { |
| "count": 0, |
| "confidence_sum": 0.0, |
| "correct_sum": 0.0 |
| }, |
| { |
| "count": 0, |
| "confidence_sum": 0.0, |
| "correct_sum": 0.0 |
| }, |
| { |
| "count": 0, |
| "confidence_sum": 0.0, |
| "correct_sum": 0.0 |
| }, |
| { |
| "count": 1, |
| "confidence_sum": 0.49957096576690674, |
| "correct_sum": 0.0 |
| }, |
| { |
| "count": 3, |
| "confidence_sum": 1.6872268319129944, |
| "correct_sum": 1.0 |
| }, |
| { |
| "count": 1, |
| "confidence_sum": 0.6297667622566223, |
| "correct_sum": 1.0 |
| }, |
| { |
| "count": 4, |
| "confidence_sum": 3.0497742295265198, |
| "correct_sum": 2.0 |
| }, |
| { |
| "count": 3, |
| "confidence_sum": 2.5167407393455505, |
| "correct_sum": 2.0 |
| }, |
| { |
| "count": 116, |
| "confidence_sum": 115.6956415772438, |
| "correct_sum": 110.0 |
| } |
| ], |
| "accuracy_vs_coverage": { |
| "0.25": { |
| "n": 32, |
| "accuracy": 1.0, |
| "min_confidence": 1.0 |
| }, |
| "0.5": { |
| "n": 64, |
| "accuracy": 1.0, |
| "min_confidence": 1.0 |
| }, |
| "0.75": { |
| "n": 96, |
| "accuracy": 0.9791666666666666, |
| "min_confidence": 0.9997592568397522 |
| }, |
| "1.0": { |
| "n": 128, |
| "accuracy": 0.90625, |
| "min_confidence": 0.49957096576690674 |
| } |
| } |
| }, |
| "boolq": { |
| "n": 128, |
| "accuracy": 0.8203125, |
| "nll": 2.8586640789486135, |
| "brier_multiclass_sum": 0.34384347342040844, |
| "ece_top_label_10_equal_width_bins": 0.17410107515752316, |
| "reliability_bins": [ |
| { |
| "count": 0, |
| "confidence_sum": 0.0, |
| "correct_sum": 0.0 |
| }, |
| { |
| "count": 0, |
| "confidence_sum": 0.0, |
| "correct_sum": 0.0 |
| }, |
| { |
| "count": 0, |
| "confidence_sum": 0.0, |
| "correct_sum": 0.0 |
| }, |
| { |
| "count": 0, |
| "confidence_sum": 0.0, |
| "correct_sum": 0.0 |
| }, |
| { |
| "count": 0, |
| "confidence_sum": 0.0, |
| "correct_sum": 0.0 |
| }, |
| { |
| "count": 1, |
| "confidence_sum": 0.5560736656188965, |
| "correct_sum": 0.0 |
| }, |
| { |
| "count": 0, |
| "confidence_sum": 0.0, |
| "correct_sum": 0.0 |
| }, |
| { |
| "count": 0, |
| "confidence_sum": 0.0, |
| "correct_sum": 0.0 |
| }, |
| { |
| "count": 0, |
| "confidence_sum": 0.0, |
| "correct_sum": 0.0 |
| }, |
| { |
| "count": 127, |
| "confidence_sum": 126.72886395454407, |
| "correct_sum": 105.0 |
| } |
| ], |
| "accuracy_vs_coverage": { |
| "0.25": { |
| "n": 32, |
| "accuracy": 0.875, |
| "min_confidence": 1.0 |
| }, |
| "0.5": { |
| "n": 64, |
| "accuracy": 0.90625, |
| "min_confidence": 1.0 |
| }, |
| "0.75": { |
| "n": 96, |
| "accuracy": 0.8958333333333334, |
| "min_confidence": 0.999998927116394 |
| }, |
| "1.0": { |
| "n": 128, |
| "accuracy": 0.8203125, |
| "min_confidence": 0.5560736656188965 |
| } |
| } |
| }, |
| "snli": { |
| "n": 128, |
| "accuracy": 0.828125, |
| "nll": 1.4566951603829215, |
| "brier_multiclass_sum": 0.3140637619481507, |
| "ece_top_label_10_equal_width_bins": 0.1558010228909552, |
| "reliability_bins": [ |
| { |
| "count": 0, |
| "confidence_sum": 0.0, |
| "correct_sum": 0.0 |
| }, |
| { |
| "count": 0, |
| "confidence_sum": 0.0, |
| "correct_sum": 0.0 |
| }, |
| { |
| "count": 0, |
| "confidence_sum": 0.0, |
| "correct_sum": 0.0 |
| }, |
| { |
| "count": 0, |
| "confidence_sum": 0.0, |
| "correct_sum": 0.0 |
| }, |
| { |
| "count": 0, |
| "confidence_sum": 0.0, |
| "correct_sum": 0.0 |
| }, |
| { |
| "count": 0, |
| "confidence_sum": 0.0, |
| "correct_sum": 0.0 |
| }, |
| { |
| "count": 1, |
| "confidence_sum": 0.6340683698654175, |
| "correct_sum": 0.0 |
| }, |
| { |
| "count": 2, |
| "confidence_sum": 1.478248655796051, |
| "correct_sum": 0.0 |
| }, |
| { |
| "count": 6, |
| "confidence_sum": 5.241755783557892, |
| "correct_sum": 4.0 |
| }, |
| { |
| "count": 119, |
| "confidence_sum": 118.5884581208229, |
| "correct_sum": 102.0 |
| } |
| ], |
| "accuracy_vs_coverage": { |
| "0.25": { |
| "n": 32, |
| "accuracy": 1.0, |
| "min_confidence": 1.0 |
| }, |
| "0.5": { |
| "n": 64, |
| "accuracy": 0.984375, |
| "min_confidence": 0.9999997615814209 |
| }, |
| "0.75": { |
| "n": 96, |
| "accuracy": 0.8958333333333334, |
| "min_confidence": 0.999707043170929 |
| }, |
| "1.0": { |
| "n": 128, |
| "accuracy": 0.828125, |
| "min_confidence": 0.6340683698654175 |
| } |
| } |
| } |
| } |
| }, |
| "best_validation_macro_nll": 0.39566109077634387, |
| "training_seconds": 724.0232084460004, |
| "median_step_seconds": 8.95618181649479, |
| "processed_decisions": 160, |
| "actual_branch_tokens": 55268, |
| "padded_branch_tokens": 55268, |
| "peak_cuda_allocated_bytes": 16654055424, |
| "peak_cuda_reserved_bytes": 16754147328, |
| "final_mem_available_bytes": 106445496320, |
| "correctness": { |
| "branch_chunks_1_probability_max_abs": 0.0, |
| "branch_chunks_2_probability_max_abs": 1.430511474609375e-06, |
| "branch_chunks_4_probability_max_abs": 1.6689300537109375e-06, |
| "question_isolation_probability_max_abs": 0.0, |
| "candidate_permutation_probability_max_abs": 0.0, |
| "repeat_probability_max_abs": 0.0, |
| "tolerance_probability_abs": 0.0001 |
| }, |
| "checkpoint_sha256": "e26f75b2396de88311873fac4eb91e1e40d0ec940778ec99f282bcfd96a2e258", |
| "best_sha256": "64977ee0b1a6147c6faf59283edea9adf564dd36d53f4580bc20940b94c6764f" |
| } |
|
|
