| { |
| "method": "Wilcoxon signed-rank on teacher-forced species token log-probabilities", |
| "description": "Teacher-force through '### [Name] is a' prefix, then extract log P(species_token). Species token is the discriminative one requiring multi-hop reasoning. Comparison: m3 vs m5.", |
| "comparison": "m3_vs_m5", |
| "bonferroni_k": 5, |
| "adjusted_alpha": 0.01, |
| "results": { |
| "prosqa_id": { |
| "n": 500, |
| "n_ties": 0, |
| "n_nonzero": 500, |
| "median_diff": -0.0001227500616778343, |
| "mean_diff": -0.06978289037453805, |
| "std_diff": 1.0628266559166086, |
| "wilcoxon_stat": 19908.0, |
| "p_value": 7.135644023141254e-40, |
| "p_bonferroni": 3.567822011570627e-39, |
| "effect_size_r": 0.5910176620488471, |
| "z_score": 13.215556682442198, |
| "m3_median_lp": -1.9490528757160064e-05, |
| "m5_median_lp": -0.0002181292074965313, |
| "m3_mean_lp": -0.11297737371596042, |
| "m5_mean_lp": -0.18276026409049848, |
| "m3_prob_median": 0.999980509661182, |
| "m5_prob_median": 0.9997818945809493, |
| "direction": "m3>m5" |
| }, |
| "7hop": { |
| "n": 1000, |
| "n_ties": 0, |
| "n_nonzero": 1000, |
| "median_diff": -0.00032740284859755775, |
| "mean_diff": 0.2813422611065431, |
| "std_diff": 2.906805478757986, |
| "wilcoxon_stat": 248577.0, |
| "p_value": 0.8546955264303868, |
| "p_bonferroni": 1.0, |
| "effect_size_r": 0.005791098858817567, |
| "z_score": 0.18313062549065387, |
| "m3_median_lp": -0.026303524151444435, |
| "m5_median_lp": -0.037403758615255356, |
| "m3_mean_lp": -1.3909491490889976, |
| "m5_mean_lp": -1.1096068879824543, |
| "m3_prob_median": 0.9740394002541536, |
| "m5_prob_median": 0.9632871213462526, |
| "direction": "m3>m5" |
| }, |
| "8hop": { |
| "n": 1000, |
| "n_ties": 0, |
| "n_nonzero": 1000, |
| "median_diff": -0.00027873104806985793, |
| "mean_diff": 0.21474395308568736, |
| "std_diff": 2.776021865502376, |
| "wilcoxon_stat": 248628.0, |
| "p_value": 0.8590779324092401, |
| "p_bonferroni": 1.0, |
| "effect_size_r": 0.005614562073521871, |
| "z_score": 0.17754804216726866, |
| "m3_median_lp": -0.02322522271424532, |
| "m5_median_lp": -0.04081491939723492, |
| "m3_mean_lp": -1.3103880740790188, |
| "m5_mean_lp": -1.0956441209933316, |
| "m3_prob_median": 0.9770424068482431, |
| "m5_prob_median": 0.960006792142127, |
| "direction": "m3>m5" |
| }, |
| "dag": { |
| "n": 1000, |
| "n_ties": 0, |
| "n_nonzero": 1000, |
| "median_diff": -0.0010396945101547317, |
| "mean_diff": 0.16994009311438624, |
| "std_diff": 2.963397551627641, |
| "wilcoxon_stat": 249483.0, |
| "p_value": 0.9330900836102644, |
| "p_bonferroni": 1.0, |
| "effect_size_r": 0.0026549747906234756, |
| "z_score": 0.08395767468698836, |
| "m3_median_lp": -0.22533510625362396, |
| "m5_median_lp": -0.34515246748924255, |
| "m3_mean_lp": -1.8029823147184267, |
| "m5_mean_lp": -1.6330422216040406, |
| "m3_prob_median": 0.7982486758110119, |
| "m5_prob_median": 0.7081123811200218, |
| "direction": "m3>m5" |
| }, |
| "dense": { |
| "n": 1000, |
| "n_ties": 0, |
| "n_nonzero": 1000, |
| "median_diff": 0.00026002148661063984, |
| "mean_diff": 0.5418239545176224, |
| "std_diff": 2.9624931634035607, |
| "wilcoxon_stat": 217527.0, |
| "p_value": 0.0003410538200066114, |
| "p_bonferroni": 0.001705269100033057, |
| "effect_size_r": 0.11327084755354884, |
| "z_score": 3.581938707669256, |
| "m3_median_lp": -0.14667461067438126, |
| "m5_median_lp": -0.1057770662009716, |
| "m3_mean_lp": -1.6534713189456967, |
| "m5_mean_lp": -1.111647364428074, |
| "m3_prob_median": 0.8635749297684109, |
| "m5_prob_median": 0.8996251825817885, |
| "direction": "m5>m3" |
| } |
| } |
| } |
