| { |
| "status": "completed_step_target", |
| "completed_steps": 8, |
| "target_steps": 8, |
| "initial_validation": { |
| "overall": { |
| "n": 512, |
| "accuracy": 0.935546875, |
| "nll": 0.30729043451049853, |
| "brier_multiclass_sum": 0.11201944144105896, |
| "ece_top_label_10_equal_width_bins": 0.04742210410768166, |
| "reliability_bins": [ |
| { |
| "count": 0, |
| "confidence_sum": 0.0, |
| "correct_sum": 0.0 |
| }, |
| { |
| "count": 0, |
| "confidence_sum": 0.0, |
| "correct_sum": 0.0 |
| }, |
| { |
| "count": 0, |
| "confidence_sum": 0.0, |
| "correct_sum": 0.0 |
| }, |
| { |
| "count": 0, |
| "confidence_sum": 0.0, |
| "correct_sum": 0.0 |
| }, |
| { |
| "count": 1, |
| "confidence_sum": 0.4343312680721283, |
| "correct_sum": 0.0 |
| }, |
| { |
| "count": 6, |
| "confidence_sum": 3.341103494167328, |
| "correct_sum": 2.0 |
| }, |
| { |
| "count": 7, |
| "confidence_sum": 4.585636496543884, |
| "correct_sum": 5.0 |
| }, |
| { |
| "count": 6, |
| "confidence_sum": 4.494804680347443, |
| "correct_sum": 4.0 |
| }, |
| { |
| "count": 6, |
| "confidence_sum": 5.045445084571838, |
| "correct_sum": 4.0 |
| }, |
| { |
| "count": 486, |
| "confidence_sum": 484.55006927251816, |
| "correct_sum": 464.0 |
| } |
| ], |
| "accuracy_vs_coverage": { |
| "0.25": { |
| "n": 128, |
| "accuracy": 1.0, |
| "min_confidence": 1.0 |
| }, |
| "0.5": { |
| "n": 256, |
| "accuracy": 1.0, |
| "min_confidence": 0.9999840259552002 |
| }, |
| "0.75": { |
| "n": 384, |
| "accuracy": 0.9817708333333334, |
| "min_confidence": 0.9996139407157898 |
| }, |
| "1.0": { |
| "n": 512, |
| "accuracy": 0.935546875, |
| "min_confidence": 0.4343312680721283 |
| } |
| } |
| }, |
| "per_family": { |
| "arc": { |
| "n": 128, |
| "accuracy": 0.953125, |
| "nll": 0.10747530940477185, |
| "brier_multiclass_sum": 0.057378651952895626, |
| "ece_top_label_10_equal_width_bins": 0.03795913537032902, |
| "reliability_bins": [ |
| { |
| "count": 0, |
| "confidence_sum": 0.0, |
| "correct_sum": 0.0 |
| }, |
| { |
| "count": 0, |
| "confidence_sum": 0.0, |
| "correct_sum": 0.0 |
| }, |
| { |
| "count": 0, |
| "confidence_sum": 0.0, |
| "correct_sum": 0.0 |
| }, |
| { |
| "count": 0, |
| "confidence_sum": 0.0, |
| "correct_sum": 0.0 |
| }, |
| { |
| "count": 1, |
| "confidence_sum": 0.4343312680721283, |
| "correct_sum": 0.0 |
| }, |
| { |
| "count": 3, |
| "confidence_sum": 1.6741251349449158, |
| "correct_sum": 0.0 |
| }, |
| { |
| "count": 3, |
| "confidence_sum": 1.9875115156173706, |
| "correct_sum": 3.0 |
| }, |
| { |
| "count": 2, |
| "confidence_sum": 1.4926129579544067, |
| "correct_sum": 1.0 |
| }, |
| { |
| "count": 2, |
| "confidence_sum": 1.6912723183631897, |
| "correct_sum": 1.0 |
| }, |
| { |
| "count": 117, |
| "confidence_sum": 116.44606083631516, |
| "correct_sum": 117.0 |
| } |
| ], |
| "accuracy_vs_coverage": { |
| "0.25": { |
| "n": 32, |
| "accuracy": 1.0, |
| "min_confidence": 1.0 |
| }, |
| "0.5": { |
| "n": 64, |
| "accuracy": 1.0, |
| "min_confidence": 0.9999973773956299 |
| }, |
| "0.75": { |
| "n": 96, |
| "accuracy": 1.0, |
| "min_confidence": 0.9985412359237671 |
| }, |
| "1.0": { |
| "n": 128, |
| "accuracy": 0.953125, |
| "min_confidence": 0.4343312680721283 |
| } |
| } |
| }, |
| "banking": { |
| "n": 128, |
| "accuracy": 0.9765625, |
| "nll": 0.0933483044127848, |
| "brier_multiclass_sum": 0.042689504809822275, |
| "ece_top_label_10_equal_width_bins": 0.024542495142668486, |
| "reliability_bins": [ |
| { |
| "count": 0, |
| "confidence_sum": 0.0, |
| "correct_sum": 0.0 |
| }, |
| { |
| "count": 0, |
| "confidence_sum": 0.0, |
| "correct_sum": 0.0 |
| }, |
| { |
| "count": 0, |
| "confidence_sum": 0.0, |
| "correct_sum": 0.0 |
| }, |
| { |
| "count": 0, |
| "confidence_sum": 0.0, |
| "correct_sum": 0.0 |
| }, |
| { |
| "count": 0, |
| "confidence_sum": 0.0, |
| "correct_sum": 0.0 |
| }, |
| { |
| "count": 1, |
| "confidence_sum": 0.5531935095787048, |
| "correct_sum": 1.0 |
| }, |
| { |
| "count": 1, |
| "confidence_sum": 0.6562426686286926, |
| "correct_sum": 0.0 |
| }, |
| { |
| "count": 0, |
| "confidence_sum": 0.0, |
| "correct_sum": 0.0 |
| }, |
| { |
| "count": 1, |
| "confidence_sum": 0.828097939491272, |
| "correct_sum": 1.0 |
| }, |
| { |
| "count": 125, |
| "confidence_sum": 124.86648815870285, |
| "correct_sum": 123.0 |
| } |
| ], |
| "accuracy_vs_coverage": { |
| "0.25": { |
| "n": 32, |
| "accuracy": 1.0, |
| "min_confidence": 1.0 |
| }, |
| "0.5": { |
| "n": 64, |
| "accuracy": 1.0, |
| "min_confidence": 1.0 |
| }, |
| "0.75": { |
| "n": 96, |
| "accuracy": 1.0, |
| "min_confidence": 1.0 |
| }, |
| "1.0": { |
| "n": 128, |
| "accuracy": 0.9765625, |
| "min_confidence": 0.5531935095787048 |
| } |
| } |
| }, |
| "boolq": { |
| "n": 128, |
| "accuracy": 0.890625, |
| "nll": 0.6294207111478052, |
| "brier_multiclass_sum": 0.20177014021701117, |
| "ece_top_label_10_equal_width_bins": 0.09753888379782438, |
| "reliability_bins": [ |
| { |
| "count": 0, |
| "confidence_sum": 0.0, |
| "correct_sum": 0.0 |
| }, |
| { |
| "count": 0, |
| "confidence_sum": 0.0, |
| "correct_sum": 0.0 |
| }, |
| { |
| "count": 0, |
| "confidence_sum": 0.0, |
| "correct_sum": 0.0 |
| }, |
| { |
| "count": 0, |
| "confidence_sum": 0.0, |
| "correct_sum": 0.0 |
| }, |
| { |
| "count": 0, |
| "confidence_sum": 0.0, |
| "correct_sum": 0.0 |
| }, |
| { |
| "count": 0, |
| "confidence_sum": 0.0, |
| "correct_sum": 0.0 |
| }, |
| { |
| "count": 2, |
| "confidence_sum": 1.3022648096084595, |
| "correct_sum": 1.0 |
| }, |
| { |
| "count": 2, |
| "confidence_sum": 1.5202099680900574, |
| "correct_sum": 1.0 |
| }, |
| { |
| "count": 1, |
| "confidence_sum": 0.8283403515815735, |
| "correct_sum": 1.0 |
| }, |
| { |
| "count": 123, |
| "confidence_sum": 122.49084270000458, |
| "correct_sum": 111.0 |
| } |
| ], |
| "accuracy_vs_coverage": { |
| "0.25": { |
| "n": 32, |
| "accuracy": 1.0, |
| "min_confidence": 0.9999921321868896 |
| }, |
| "0.5": { |
| "n": 64, |
| "accuracy": 0.96875, |
| "min_confidence": 0.9999270439147949 |
| }, |
| "0.75": { |
| "n": 96, |
| "accuracy": 0.9375, |
| "min_confidence": 0.9993771910667419 |
| }, |
| "1.0": { |
| "n": 128, |
| "accuracy": 0.890625, |
| "min_confidence": 0.6442157626152039 |
| } |
| } |
| }, |
| "snli": { |
| "n": 128, |
| "accuracy": 0.921875, |
| "nll": 0.3989174130766323, |
| "brier_multiclass_sum": 0.14623946878450678, |
| "ece_top_label_10_equal_width_bins": 0.07372341910377145, |
| "reliability_bins": [ |
| { |
| "count": 0, |
| "confidence_sum": 0.0, |
| "correct_sum": 0.0 |
| }, |
| { |
| "count": 0, |
| "confidence_sum": 0.0, |
| "correct_sum": 0.0 |
| }, |
| { |
| "count": 0, |
| "confidence_sum": 0.0, |
| "correct_sum": 0.0 |
| }, |
| { |
| "count": 0, |
| "confidence_sum": 0.0, |
| "correct_sum": 0.0 |
| }, |
| { |
| "count": 0, |
| "confidence_sum": 0.0, |
| "correct_sum": 0.0 |
| }, |
| { |
| "count": 2, |
| "confidence_sum": 1.1137848496437073, |
| "correct_sum": 1.0 |
| }, |
| { |
| "count": 1, |
| "confidence_sum": 0.6396175026893616, |
| "correct_sum": 1.0 |
| }, |
| { |
| "count": 2, |
| "confidence_sum": 1.4819817543029785, |
| "correct_sum": 2.0 |
| }, |
| { |
| "count": 2, |
| "confidence_sum": 1.6977344751358032, |
| "correct_sum": 1.0 |
| }, |
| { |
| "count": 121, |
| "confidence_sum": 120.74667757749557, |
| "correct_sum": 113.0 |
| } |
| ], |
| "accuracy_vs_coverage": { |
| "0.25": { |
| "n": 32, |
| "accuracy": 1.0, |
| "min_confidence": 0.9999723434448242 |
| }, |
| "0.5": { |
| "n": 64, |
| "accuracy": 1.0, |
| "min_confidence": 0.9998985528945923 |
| }, |
| "0.75": { |
| "n": 96, |
| "accuracy": 0.9791666666666666, |
| "min_confidence": 0.9993452429771423 |
| }, |
| "1.0": { |
| "n": 128, |
| "accuracy": 0.921875, |
| "min_confidence": 0.528592586517334 |
| } |
| } |
| } |
| } |
| }, |
| "best_validation_macro_nll": 0.30729043451049853, |
| "selection_metric": "crossfit_temperature_nll_v1", |
| "best_validation_selection_score": 0.18864030037724333, |
| "training_seconds": 422.01792606700474, |
| "median_step_seconds": 9.330743154998345, |
| "processed_decisions": 32, |
| "actual_branch_tokens": 11404, |
| "padded_branch_tokens": 11404, |
| "peak_cuda_allocated_bytes": 16668094976, |
| "peak_cuda_reserved_bytes": 16775118848, |
| "final_mem_available_bytes": 106346242048, |
| "correctness": { |
| "branch_chunks_1_probability_max_abs": 0.0, |
| "branch_chunks_2_probability_max_abs": 3.637978807091713e-09, |
| "branch_chunks_4_probability_max_abs": 2.153683453798294e-09, |
| "question_isolation_probability_max_abs": 0.0, |
| "candidate_permutation_probability_max_abs": 0.0, |
| "repeat_probability_max_abs": 0.0, |
| "tolerance_probability_abs": 0.0001 |
| }, |
| "final_correctness_status": "passed", |
| "checkpoint_sha256": "ece94b096b7fa37eec9e180a5f652da86a9f4f42d49f7f124850f046915237a8", |
| "best_sha256": "a2b38db7e46440cdef7e5dee4dea5cc1f8310105abc1172c4c765900da4ce6a9" |
| } |
|
|
