File size: 5,232 Bytes
854f51d | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 | {
"description": "Best MATH DCF results aggregated from multiple runs",
"date_generated": "2026-01-15",
"source_runs": {
"math_anneal": "results/math_anneal/math/hyperparams.json",
"math_anneal_low_alpha": "results/math_anneal_low_alpha/math/hyperparams.json",
"math": "results/math/hyperparams.json"
},
"alpha_to_source": {
"0.03": "math_anneal",
"0.04": "math_anneal",
"0.05": "math_anneal_low_alpha",
"0.06": "math_anneal",
"0.07": "math_anneal",
"0.08": "math_anneal",
"0.09": "math_anneal",
"0.10": "math_anneal"
},
"best_results": {
"0.03": {
"source": "math_anneal",
"gamma": 4.0,
"lambda_": 1.35,
"squash_temp": 1.0,
"temp": 1.0,
"cutoff_temp": 0.5,
"hard_coverage_mean": 0.9654545454545455,
"hard_coverage_std": 0.05674358739755357,
"hard_avg_claims_retained_mean": 1.7604545454545453,
"hard_avg_claims_retained_std": 0.9699291743760393,
"baseline_coverage": 0.9708910891089109,
"baseline_retention": 0.7255445544554455,
"meets_baseline": false,
"retention_improvement": 1.0349099909990998
},
"0.04": {
"source": "math_anneal",
"gamma": 5.0,
"lambda_": 1.3,
"squash_temp": 2.0,
"temp": 0.1,
"cutoff_temp": 0.1,
"hard_coverage_mean": 0.9559090909090908,
"hard_coverage_std": 0.05815873121491669,
"hard_avg_claims_retained_mean": 2.2209090909090907,
"hard_avg_claims_retained_std": 0.8840370397236226,
"baseline_coverage": 0.9580198019801978,
"baseline_retention": 1.1380198019801981,
"meets_baseline": false,
"retention_improvement": 1.0828892889288926
},
"0.05": {
"source": "math_anneal_low_alpha",
"gamma": 6.0,
"lambda_": 1.6,
"squash_temp": 2.0,
"temp": 0.1,
"cutoff_temp": 0.1,
"hard_coverage_mean": 0.9554545454545454,
"hard_coverage_std": 0.07368124333800775,
"hard_avg_claims_retained_mean": 2.3086363636363636,
"hard_avg_claims_retained_std": 1.1840694466734012,
"baseline_coverage": 0.9485148514851486,
"baseline_retention": 1.4352475247524752,
"meets_baseline": true,
"retention_improvement": 0.8733888388838884
},
"0.06": {
"source": "math_anneal",
"gamma": 6.0,
"lambda_": 1.6,
"squash_temp": 0.4,
"temp": 0.1,
"cutoff_temp": 0.1,
"hard_coverage_mean": 0.9413636363636364,
"hard_coverage_std": 0.08442684740047723,
"hard_avg_claims_retained_mean": 3.557727272727272,
"hard_avg_claims_retained_std": 1.6173523793559226,
"baseline_coverage": 0.9409900990099007,
"baseline_retention": 1.738415841584158,
"meets_baseline": true,
"retention_improvement": 1.8193114311431142
},
"0.07": {
"source": "math_anneal",
"gamma": 1.5,
"lambda_": 1.7,
"squash_temp": 0.4,
"temp": 0.1,
"cutoff_temp": 0.1,
"hard_coverage_mean": 0.9313636363636364,
"hard_coverage_std": 0.0886293703534906,
"hard_avg_claims_retained_mean": 3.6304545454545454,
"hard_avg_claims_retained_std": 1.4842764745466264,
"baseline_coverage": 0.9299009900990097,
"baseline_retention": 2.0138613861386134,
"meets_baseline": true,
"retention_improvement": 1.616593159315932
},
"0.08": {
"source": "math_anneal",
"gamma": 6.0,
"lambda_": 1.75,
"squash_temp": 1.0,
"temp": 0.1,
"cutoff_temp": 0.1,
"hard_coverage_mean": 0.9413636363636364,
"hard_coverage_std": 0.08442684740047723,
"hard_avg_claims_retained_mean": 3.6909090909090905,
"hard_avg_claims_retained_std": 1.2174862328883755,
"baseline_coverage": 0.9182178217821783,
"baseline_retention": 2.3081188118811884,
"meets_baseline": true,
"retention_improvement": 1.3827902790279021
},
"0.09": {
"source": "math_anneal",
"gamma": 2.0,
"lambda_": 1.7,
"squash_temp": 0.4,
"temp": 0.1,
"cutoff_temp": 0.01,
"hard_coverage_mean": 0.9063636363636365,
"hard_coverage_std": 0.09642793738039832,
"hard_avg_claims_retained_mean": 5.566818181818181,
"hard_avg_claims_retained_std": 1.4568370442687537,
"baseline_coverage": 0.905940594059406,
"baseline_retention": 2.7489108910891087,
"meets_baseline": true,
"retention_improvement": 2.8179072907290723
},
"0.10": {
"source": "math_anneal",
"gamma": 5.0,
"lambda_": 1.9,
"squash_temp": 2.0,
"temp": 0.1,
"cutoff_temp": 0.5,
"hard_coverage_mean": 0.8963636363636365,
"hard_coverage_std": 0.10161505712595326,
"hard_avg_claims_retained_mean": 5.989545454545455,
"hard_avg_claims_retained_std": 1.3708544569533119,
"baseline_coverage": 0.896831683168317,
"baseline_retention": 3.3269306930693068,
"meets_baseline": false,
"retention_improvement": 2.6626147614761482
}
},
"summary": {
"meets_baseline_count": 5,
"total_alphas": 8,
"alphas_meeting_baseline": ["0.05", "0.06", "0.07", "0.08", "0.09"],
"alphas_not_meeting": ["0.03", "0.04", "0.10"]
}
}
|