repro-differentiable-conformal-training-for-llm-reasoning-factuality / source_current /results /felm_best_optimization_results.json
| { | |
| "description": "Best FELM DCF results aggregated from multiple runs", | |
| "date_generated": "2026-01-16", | |
| "source_runs": { | |
| "felm_anneal": "results/felm_anneal/dataset/hyperparams.json", | |
| "felm_anneal_low_lambda_v2": "results/felm_anneal_low_lambda_v2/dataset/hyperparams.json", | |
| "felm_hone": "results/felm_hone/dataset/hyperparams.json" | |
| }, | |
| "alpha_to_source": { | |
| "0.01": "felm_anneal", | |
| "0.02": "felm_anneal", | |
| "0.03": "felm_anneal", | |
| "0.04": "felm_anneal", | |
| "0.05": "felm_anneal", | |
| "0.06": "felm_anneal_low_lambda_v2", | |
| "0.07": "felm_anneal_low_lambda_v2", | |
| "0.08": "felm_anneal", | |
| "0.09": "felm_hone", | |
| "0.1": "felm_hone" | |
| }, | |
| "alpha_to_features": { | |
| "0.01": [ | |
| "frequency-score", | |
| "coherent_to_ancestors", | |
| "inference_gap_size", | |
| "has_missing_dependencies", | |
| "nx_pagerank", | |
| "nx_reachability", | |
| "nx_out_degree" | |
| ], | |
| "0.02": [ | |
| "frequency-score", | |
| "coherent_to_ancestors", | |
| "inference_gap_size", | |
| "has_missing_dependencies", | |
| "nx_pagerank", | |
| "nx_reachability", | |
| "nx_out_degree" | |
| ], | |
| "0.03": [ | |
| "frequency-score", | |
| "coherent_to_ancestors", | |
| "inference_gap_size", | |
| "has_missing_dependencies", | |
| "nx_pagerank", | |
| "nx_reachability", | |
| "nx_out_degree" | |
| ], | |
| "0.04": [ | |
| "frequency-score", | |
| "coherent_to_ancestors", | |
| "inference_gap_size", | |
| "has_missing_dependencies", | |
| "nx_pagerank", | |
| "nx_reachability", | |
| "nx_out_degree" | |
| ], | |
| "0.05": [ | |
| "frequency-score", | |
| "coherent_to_ancestors", | |
| "inference_gap_size", | |
| "has_missing_dependencies", | |
| "nx_pagerank", | |
| "nx_reachability", | |
| "nx_out_degree" | |
| ], | |
| "0.06": [ | |
| "frequency-score", | |
| "coherent_to_ancestors", | |
| "inference_gap_size", | |
| "has_missing_dependencies", | |
| "nx_pagerank", | |
| "nx_reachability", | |
| "nx_out_degree" | |
| ], | |
| "0.07": [ | |
| "frequency-score", | |
| "coherent_to_ancestors", | |
| "inference_gap_size", | |
| "has_missing_dependencies", | |
| "nx_pagerank", | |
| "nx_reachability", | |
| "nx_out_degree" | |
| ], | |
| "0.08": [ | |
| "frequency-score", | |
| "coherent_to_ancestors", | |
| "inference_gap_size", | |
| "has_missing_dependencies", | |
| "nx_pagerank", | |
| "nx_reachability", | |
| "nx_out_degree" | |
| ], | |
| "0.09": [ | |
| "frequency-score", | |
| "claim_index", | |
| "coherent_to_ancestors", | |
| "inference_gap_size", | |
| "has_missing_dependencies", | |
| "has_unnecessary_dependencies", | |
| "problem_relevance", | |
| "uses_problem_data", | |
| "domain_science", | |
| "domain_reasoning", | |
| "nx_in_degree", | |
| "nx_out_degree", | |
| "nx_pagerank", | |
| "nx_betweenness", | |
| "nx_closeness", | |
| "nx_clustering", | |
| "nx_is_source", | |
| "nx_is_sink", | |
| "nx_reachability", | |
| "nx_depth_from_sources" | |
| ], | |
| "0.1": [ | |
| "frequency-score", | |
| "claim_index", | |
| "coherent_to_ancestors", | |
| "inference_gap_size", | |
| "has_missing_dependencies", | |
| "has_unnecessary_dependencies", | |
| "problem_relevance", | |
| "uses_problem_data", | |
| "domain_science", | |
| "domain_reasoning", | |
| "nx_in_degree", | |
| "nx_out_degree", | |
| "nx_pagerank", | |
| "nx_betweenness", | |
| "nx_closeness", | |
| "nx_clustering", | |
| "nx_is_source", | |
| "nx_is_sink", | |
| "nx_reachability", | |
| "nx_depth_from_sources" | |
| ] | |
| }, | |
| "results": { | |
| "0.01": { | |
| "alpha": 0.01, | |
| "target_coverage": 0.99, | |
| "source": "felm_anneal", | |
| "feature_set": "7_features", | |
| "features": [ | |
| "frequency-score", | |
| "coherent_to_ancestors", | |
| "inference_gap_size", | |
| "has_missing_dependencies", | |
| "nx_pagerank", | |
| "nx_reachability", | |
| "nx_out_degree" | |
| ], | |
| "learned": { | |
| "coverage": 0.9915476190476191, | |
| "coverage_std": 0.015760064391671432, | |
| "avg_claims_retained": 0.7153174603174602, | |
| "avg_claims_retained_std": 0.19439424360066407, | |
| "precision": 0.2608035714285714, | |
| "precision_std": 0.09359210216558873 | |
| }, | |
| "baseline": { | |
| "coverage": 0.9918873239436619, | |
| "coverage_std": 0.006704935457718004, | |
| "avg_claims_retained": 0.4446760563380282, | |
| "avg_claims_retained_std": 0.17468878212716243, | |
| "precision": 0.13114084507042253, | |
| "precision_std": 0.046563578384589195 | |
| }, | |
| "hyperparams": { | |
| "temp": 1.0, | |
| "beta": 1, | |
| "gamma": 1.5, | |
| "lambda_": 1.2, | |
| "violation_mode": "exponential", | |
| "squash_temp": 0.8, | |
| "cutoff_temp": 0.01, | |
| "lr": 0.015, | |
| "hard_coverage_mean": 0.9915476190476191, | |
| "hard_coverage_std": 0.015760064391671432, | |
| "hard_avg_claims_retained_mean": 0.7153174603174602, | |
| "hard_avg_claims_retained_std": 0.19439424360066407, | |
| "hard_precision_mean": 0.2608035714285714, | |
| "hard_precision_std": 0.09359210216558873 | |
| }, | |
| "meets_coverage": true | |
| }, | |
| "0.02": { | |
| "alpha": 0.02, | |
| "target_coverage": 0.98, | |
| "source": "felm_anneal", | |
| "feature_set": "7_features", | |
| "features": [ | |
| "frequency-score", | |
| "coherent_to_ancestors", | |
| "inference_gap_size", | |
| "has_missing_dependencies", | |
| "nx_pagerank", | |
| "nx_reachability", | |
| "nx_out_degree" | |
| ], | |
| "learned": { | |
| "coverage": 0.9774603174603176, | |
| "coverage_std": 0.020982121749398376, | |
| "avg_claims_retained": 1.4280555555555556, | |
| "avg_claims_retained_std": 0.4606316849650614, | |
| "precision": 0.4873743386243386, | |
| "precision_std": 0.15545133977830713 | |
| }, | |
| "baseline": { | |
| "coverage": 0.9808450704225352, | |
| "coverage_std": 0.009643517052745727, | |
| "avg_claims_retained": 0.8967887323943661, | |
| "avg_claims_retained_std": 0.20397459885189215, | |
| "precision": 0.2575446009389671, | |
| "precision_std": 0.05132522708138355 | |
| }, | |
| "hyperparams": { | |
| "temp": 1.0, | |
| "beta": 1, | |
| "gamma": 1.0, | |
| "lambda_": 1.5, | |
| "violation_mode": "exponential", | |
| "squash_temp": 0.8, | |
| "cutoff_temp": 0.01, | |
| "lr": 0.015, | |
| "hard_coverage_mean": 0.9774603174603176, | |
| "hard_coverage_std": 0.020982121749398376, | |
| "hard_avg_claims_retained_mean": 1.4280555555555556, | |
| "hard_avg_claims_retained_std": 0.4606316849650614, | |
| "hard_precision_mean": 0.4873743386243386, | |
| "hard_precision_std": 0.15545133977830713 | |
| }, | |
| "meets_coverage": true | |
| }, | |
| "0.03": { | |
| "alpha": 0.03, | |
| "target_coverage": 0.97, | |
| "source": "felm_anneal", | |
| "feature_set": "7_features", | |
| "features": [ | |
| "frequency-score", | |
| "coherent_to_ancestors", | |
| "inference_gap_size", | |
| "has_missing_dependencies", | |
| "nx_pagerank", | |
| "nx_reachability", | |
| "nx_out_degree" | |
| ], | |
| "learned": { | |
| "coverage": 0.9732142857142858, | |
| "coverage_std": 0.024502964120548153, | |
| "avg_claims_retained": 1.4577380952380952, | |
| "avg_claims_retained_std": 0.3746905899990178, | |
| "precision": 0.5040410052910053, | |
| "precision_std": 0.09771194976933983 | |
| }, | |
| "baseline": { | |
| "coverage": 0.9710985915492957, | |
| "coverage_std": 0.011279008314819218, | |
| "avg_claims_retained": 1.180281690140845, | |
| "avg_claims_retained_std": 0.1856609439647076, | |
| "precision": 0.33245070422535217, | |
| "precision_std": 0.042476707374746105 | |
| }, | |
| "hyperparams": { | |
| "temp": 1.0, | |
| "beta": 1, | |
| "gamma": 1.25, | |
| "lambda_": 1.3, | |
| "violation_mode": "exponential", | |
| "squash_temp": 0.8, | |
| "cutoff_temp": 0.01, | |
| "lr": 0.015, | |
| "hard_coverage_mean": 0.9732142857142858, | |
| "hard_coverage_std": 0.024502964120548153, | |
| "hard_avg_claims_retained_mean": 1.4577380952380952, | |
| "hard_avg_claims_retained_std": 0.3746905899990178, | |
| "hard_precision_mean": 0.5040410052910053, | |
| "hard_precision_std": 0.09771194976933983 | |
| }, | |
| "meets_coverage": true | |
| }, | |
| "0.04": { | |
| "alpha": 0.04, | |
| "target_coverage": 0.96, | |
| "source": "felm_anneal", | |
| "feature_set": "7_features", | |
| "features": [ | |
| "frequency-score", | |
| "coherent_to_ancestors", | |
| "inference_gap_size", | |
| "has_missing_dependencies", | |
| "nx_pagerank", | |
| "nx_reachability", | |
| "nx_out_degree" | |
| ], | |
| "learned": { | |
| "coverage": 0.9647222222222223, | |
| "coverage_std": 0.034487277697772496, | |
| "avg_claims_retained": 1.6817460317460315, | |
| "avg_claims_retained_std": 0.355912253759144, | |
| "precision": 0.5549007936507937, | |
| "precision_std": 0.08089042865566827 | |
| }, | |
| "baseline": { | |
| "coverage": 0.9588732394366198, | |
| "coverage_std": 0.013079647060556534, | |
| "avg_claims_retained": 1.447492957746479, | |
| "avg_claims_retained_std": 0.17841908409564786, | |
| "precision": 0.40163380281690136, | |
| "precision_std": 0.03931126367158381 | |
| }, | |
| "hyperparams": { | |
| "temp": 0.5, | |
| "beta": 1, | |
| "gamma": 1.5, | |
| "lambda_": 1.45, | |
| "violation_mode": "exponential", | |
| "squash_temp": 0.8, | |
| "cutoff_temp": 0.01, | |
| "lr": 0.015, | |
| "hard_coverage_mean": 0.9647222222222223, | |
| "hard_coverage_std": 0.034487277697772496, | |
| "hard_avg_claims_retained_mean": 1.6817460317460315, | |
| "hard_avg_claims_retained_std": 0.355912253759144, | |
| "hard_precision_mean": 0.5549007936507937, | |
| "hard_precision_std": 0.08089042865566827 | |
| }, | |
| "meets_coverage": true | |
| }, | |
| "0.05": { | |
| "alpha": 0.05, | |
| "target_coverage": 0.95, | |
| "source": "felm_anneal", | |
| "feature_set": "7_features", | |
| "features": [ | |
| "frequency-score", | |
| "coherent_to_ancestors", | |
| "inference_gap_size", | |
| "has_missing_dependencies", | |
| "nx_pagerank", | |
| "nx_reachability", | |
| "nx_out_degree" | |
| ], | |
| "learned": { | |
| "coverage": 0.9534523809523809, | |
| "coverage_std": 0.03620150468970789, | |
| "avg_claims_retained": 1.8526984126984125, | |
| "avg_claims_retained_std": 0.39758074143016564, | |
| "precision": 0.5965702003023432, | |
| "precision_std": 0.09169126576443776 | |
| }, | |
| "baseline": { | |
| "coverage": 0.9509859154929577, | |
| "coverage_std": 0.014862992629039931, | |
| "avg_claims_retained": 1.6611830985915492, | |
| "avg_claims_retained_std": 0.1976028048345379, | |
| "precision": 0.45214741784037554, | |
| "precision_std": 0.038844472303419496 | |
| }, | |
| "hyperparams": { | |
| "temp": 0.5, | |
| "beta": 1, | |
| "gamma": 1.25, | |
| "lambda_": 1.45, | |
| "violation_mode": "exponential", | |
| "squash_temp": 0.8, | |
| "cutoff_temp": 0.01, | |
| "lr": 0.015, | |
| "hard_coverage_mean": 0.9534523809523809, | |
| "hard_coverage_std": 0.03620150468970789, | |
| "hard_avg_claims_retained_mean": 1.8526984126984125, | |
| "hard_avg_claims_retained_std": 0.39758074143016564, | |
| "hard_precision_mean": 0.5965702003023432, | |
| "hard_precision_std": 0.09169126576443776 | |
| }, | |
| "meets_coverage": true | |
| }, | |
| "0.06": { | |
| "alpha": 0.06, | |
| "target_coverage": 0.94, | |
| "source": "felm_anneal_low_lambda_v2", | |
| "feature_set": "7_features", | |
| "features": [ | |
| "frequency-score", | |
| "coherent_to_ancestors", | |
| "inference_gap_size", | |
| "has_missing_dependencies", | |
| "nx_pagerank", | |
| "nx_reachability", | |
| "nx_out_degree" | |
| ], | |
| "learned": { | |
| "coverage": 0.9491666666666667, | |
| "coverage_std": 0.039838167051758806, | |
| "avg_claims_retained": 1.9226587301587301, | |
| "avg_claims_retained_std": 0.4188284415837419, | |
| "precision": 0.6038770786092214, | |
| "precision_std": 0.08327533165270261 | |
| }, | |
| "baseline": { | |
| "coverage": 0.9394366197183099, | |
| "coverage_std": 0.016762338644295032, | |
| "avg_claims_retained": 1.9679999999999997, | |
| "avg_claims_retained_std": 0.23461035494430724, | |
| "precision": 0.5214323272971161, | |
| "precision_std": 0.04593742293818858 | |
| }, | |
| "hyperparams": { | |
| "temp": 1.0, | |
| "beta": 1, | |
| "gamma": 1.0, | |
| "lambda_": 1.0, | |
| "violation_mode": "exponential", | |
| "squash_temp": 0.8, | |
| "cutoff_temp": 0.01, | |
| "lr": 0.015, | |
| "hard_coverage_mean": 0.9491666666666667, | |
| "hard_coverage_std": 0.039838167051758806, | |
| "hard_avg_claims_retained_mean": 1.9226587301587301, | |
| "hard_avg_claims_retained_std": 0.4188284415837419, | |
| "hard_precision_mean": 0.6038770786092214, | |
| "hard_precision_std": 0.08327533165270261 | |
| }, | |
| "meets_coverage": true | |
| }, | |
| "0.07": { | |
| "alpha": 0.07, | |
| "target_coverage": 0.9299999999999999, | |
| "source": "felm_anneal_low_lambda_v2", | |
| "feature_set": "7_features", | |
| "features": [ | |
| "frequency-score", | |
| "coherent_to_ancestors", | |
| "inference_gap_size", | |
| "has_missing_dependencies", | |
| "nx_pagerank", | |
| "nx_reachability", | |
| "nx_out_degree" | |
| ], | |
| "learned": { | |
| "coverage": 0.9421031746031746, | |
| "coverage_std": 0.041659088502089124, | |
| "avg_claims_retained": 1.9847222222222225, | |
| "avg_claims_retained_std": 0.45195000881583347, | |
| "precision": 0.6110926870748299, | |
| "precision_std": 0.11778145514506322 | |
| }, | |
| "baseline": { | |
| "coverage": 0.9312676056338028, | |
| "coverage_std": 0.01731879002697396, | |
| "avg_claims_retained": 2.2001690140845067, | |
| "avg_claims_retained_std": 0.25667794209696926, | |
| "precision": 0.5760910798122065, | |
| "precision_std": 0.05309294787977286 | |
| }, | |
| "hyperparams": { | |
| "temp": 1.0, | |
| "beta": 1, | |
| "gamma": 1.25, | |
| "lambda_": 0.9, | |
| "violation_mode": "exponential", | |
| "squash_temp": 0.8, | |
| "cutoff_temp": 0.01, | |
| "lr": 0.015, | |
| "hard_coverage_mean": 0.9421031746031746, | |
| "hard_coverage_std": 0.041659088502089124, | |
| "hard_avg_claims_retained_mean": 1.9847222222222225, | |
| "hard_avg_claims_retained_std": 0.45195000881583347, | |
| "hard_precision_mean": 0.6110926870748299, | |
| "hard_precision_std": 0.11778145514506322 | |
| }, | |
| "meets_coverage": true | |
| }, | |
| "0.08": { | |
| "alpha": 0.08, | |
| "target_coverage": 0.92, | |
| "source": "felm_anneal", | |
| "feature_set": "7_features", | |
| "features": [ | |
| "frequency-score", | |
| "coherent_to_ancestors", | |
| "inference_gap_size", | |
| "has_missing_dependencies", | |
| "nx_pagerank", | |
| "nx_reachability", | |
| "nx_out_degree" | |
| ], | |
| "learned": { | |
| "coverage": 0.9363888888888889, | |
| "coverage_std": 0.04032361142303625, | |
| "avg_claims_retained": 2.10952380952381, | |
| "avg_claims_retained_std": 0.3633810567311269, | |
| "precision": 0.6353406084656085, | |
| "precision_std": 0.07939856335930871 | |
| }, | |
| "baseline": { | |
| "coverage": 0.9214647887323943, | |
| "coverage_std": 0.018217666054599003, | |
| "avg_claims_retained": 2.4501971830985916, | |
| "avg_claims_retained_std": 0.23481517800994994, | |
| "precision": 0.6371147999105745, | |
| "precision_std": 0.05024683848853584 | |
| }, | |
| "hyperparams": { | |
| "temp": 1.0, | |
| "beta": 1, | |
| "gamma": 1.25, | |
| "lambda_": 1.5, | |
| "violation_mode": "exponential", | |
| "squash_temp": 0.8, | |
| "cutoff_temp": 0.01, | |
| "lr": 0.015, | |
| "hard_coverage_mean": 0.9363888888888889, | |
| "hard_coverage_std": 0.04032361142303625, | |
| "hard_avg_claims_retained_mean": 2.10952380952381, | |
| "hard_avg_claims_retained_std": 0.3633810567311269, | |
| "hard_precision_mean": 0.6353406084656085, | |
| "hard_precision_std": 0.07939856335930871 | |
| }, | |
| "meets_coverage": true | |
| }, | |
| "0.09": { | |
| "alpha": 0.09, | |
| "target_coverage": 0.91, | |
| "source": "felm_hone", | |
| "feature_set": "20_features", | |
| "features": [ | |
| "frequency-score", | |
| "claim_index", | |
| "coherent_to_ancestors", | |
| "inference_gap_size", | |
| "has_missing_dependencies", | |
| "has_unnecessary_dependencies", | |
| "problem_relevance", | |
| "uses_problem_data", | |
| "domain_science", | |
| "domain_reasoning", | |
| "nx_in_degree", | |
| "nx_out_degree", | |
| "nx_pagerank", | |
| "nx_betweenness", | |
| "nx_closeness", | |
| "nx_clustering", | |
| "nx_is_source", | |
| "nx_is_sink", | |
| "nx_reachability", | |
| "nx_depth_from_sources" | |
| ], | |
| "learned": { | |
| "coverage": 0.9139285714285714, | |
| "coverage_std": 0.04530729530451504, | |
| "avg_claims_retained": 2.760357142857143, | |
| "avg_claims_retained_std": 0.444718557405786, | |
| "precision": 0.7079765419765419, | |
| "precision_std": 0.05769065357895684 | |
| }, | |
| "baseline": { | |
| "coverage": 0.9079999999999998, | |
| "coverage_std": 0.019979040793232657, | |
| "avg_claims_retained": 2.6562816901408444, | |
| "avg_claims_retained_std": 0.1811952994981004, | |
| "precision": 0.6821681198300918, | |
| "precision_std": 0.03279157111570292 | |
| }, | |
| "hyperparams": { | |
| "temp": 1.0, | |
| "beta": 1, | |
| "gamma": 1.0, | |
| "lambda_": 1.65, | |
| "violation_mode": "exponential", | |
| "squash_temp": 0.8, | |
| "cutoff_temp": 0.01, | |
| "lr": 0.015, | |
| "hard_coverage_mean": 0.9139285714285714, | |
| "hard_coverage_std": 0.04530729530451504, | |
| "hard_avg_claims_retained_mean": 2.760357142857143, | |
| "hard_avg_claims_retained_std": 0.444718557405786, | |
| "hard_precision_mean": 0.7079765419765419, | |
| "hard_precision_std": 0.05769065357895684 | |
| }, | |
| "meets_coverage": true | |
| }, | |
| "0.1": { | |
| "alpha": 0.1, | |
| "target_coverage": 0.9, | |
| "source": "felm_hone", | |
| "feature_set": "20_features", | |
| "features": [ | |
| "frequency-score", | |
| "claim_index", | |
| "coherent_to_ancestors", | |
| "inference_gap_size", | |
| "has_missing_dependencies", | |
| "has_unnecessary_dependencies", | |
| "problem_relevance", | |
| "uses_problem_data", | |
| "domain_science", | |
| "domain_reasoning", | |
| "nx_in_degree", | |
| "nx_out_degree", | |
| "nx_pagerank", | |
| "nx_betweenness", | |
| "nx_closeness", | |
| "nx_clustering", | |
| "nx_is_source", | |
| "nx_is_sink", | |
| "nx_reachability", | |
| "nx_depth_from_sources" | |
| ], | |
| "learned": { | |
| "coverage": 0.9053571428571429, | |
| "coverage_std": 0.04800168687197146, | |
| "avg_claims_retained": 2.846746031746032, | |
| "avg_claims_retained_std": 0.4311378112567819, | |
| "precision": 0.7341042568542568, | |
| "precision_std": 0.058453704655857086 | |
| }, | |
| "baseline": { | |
| "coverage": 0.9007887323943662, | |
| "coverage_std": 0.020213421880577218, | |
| "avg_claims_retained": 2.7548732394366198, | |
| "avg_claims_retained_std": 0.15754765135642929, | |
| "precision": 0.7018582606751622, | |
| "precision_std": 0.022396892802106768 | |
| }, | |
| "hyperparams": { | |
| "temp": 0.5, | |
| "beta": 1, | |
| "gamma": 1.25, | |
| "lambda_": 1.9, | |
| "violation_mode": "exponential", | |
| "squash_temp": 0.8, | |
| "cutoff_temp": 0.01, | |
| "lr": 0.015, | |
| "hard_coverage_mean": 0.9053571428571429, | |
| "hard_coverage_std": 0.04800168687197146, | |
| "hard_avg_claims_retained_mean": 2.846746031746032, | |
| "hard_avg_claims_retained_std": 0.4311378112567819, | |
| "hard_precision_mean": 0.7341042568542568, | |
| "hard_precision_std": 0.058453704655857086 | |
| }, | |
| "meets_coverage": true | |
| } | |
| } | |
| } |