{
  "method": "Wilcoxon signed-rank on teacher-forced species token log-probabilities",
  "description": "Teacher-force through '### [Name] is a' prefix, then extract log P(species_token). Species token is the discriminative one requiring multi-hop reasoning. Comparison: m3 vs m5.",
  "comparison": "m3_vs_m5",
  "bonferroni_k": 5,
  "adjusted_alpha": 0.01,
  "results": {
    "prosqa_id": {
      "n": 500,
      "n_ties": 0,
      "n_nonzero": 500,
      "median_diff": -0.0001227500616778343,
      "mean_diff": -0.06978289037453805,
      "std_diff": 1.0628266559166086,
      "wilcoxon_stat": 19908.0,
      "p_value": 7.135644023141254e-40,
      "p_bonferroni": 3.567822011570627e-39,
      "effect_size_r": 0.5910176620488471,
      "z_score": 13.215556682442198,
      "m3_median_lp": -1.9490528757160064e-05,
      "m5_median_lp": -0.0002181292074965313,
      "m3_mean_lp": -0.11297737371596042,
      "m5_mean_lp": -0.18276026409049848,
      "m3_prob_median": 0.999980509661182,
      "m5_prob_median": 0.9997818945809493,
      "direction": "m3>m5"
    },
    "7hop": {
      "n": 1000,
      "n_ties": 0,
      "n_nonzero": 1000,
      "median_diff": -0.00032740284859755775,
      "mean_diff": 0.2813422611065431,
      "std_diff": 2.906805478757986,
      "wilcoxon_stat": 248577.0,
      "p_value": 0.8546955264303868,
      "p_bonferroni": 1.0,
      "effect_size_r": 0.005791098858817567,
      "z_score": 0.18313062549065387,
      "m3_median_lp": -0.026303524151444435,
      "m5_median_lp": -0.037403758615255356,
      "m3_mean_lp": -1.3909491490889976,
      "m5_mean_lp": -1.1096068879824543,
      "m3_prob_median": 0.9740394002541536,
      "m5_prob_median": 0.9632871213462526,
      "direction": "m3>m5"
    },
    "8hop": {
      "n": 1000,
      "n_ties": 0,
      "n_nonzero": 1000,
      "median_diff": -0.00027873104806985793,
      "mean_diff": 0.21474395308568736,
      "std_diff": 2.776021865502376,
      "wilcoxon_stat": 248628.0,
      "p_value": 0.8590779324092401,
      "p_bonferroni": 1.0,
      "effect_size_r": 0.005614562073521871,
      "z_score": 0.17754804216726866,
      "m3_median_lp": -0.02322522271424532,
      "m5_median_lp": -0.04081491939723492,
      "m3_mean_lp": -1.3103880740790188,
      "m5_mean_lp": -1.0956441209933316,
      "m3_prob_median": 0.9770424068482431,
      "m5_prob_median": 0.960006792142127,
      "direction": "m3>m5"
    },
    "dag": {
      "n": 1000,
      "n_ties": 0,
      "n_nonzero": 1000,
      "median_diff": -0.0010396945101547317,
      "mean_diff": 0.16994009311438624,
      "std_diff": 2.963397551627641,
      "wilcoxon_stat": 249483.0,
      "p_value": 0.9330900836102644,
      "p_bonferroni": 1.0,
      "effect_size_r": 0.0026549747906234756,
      "z_score": 0.08395767468698836,
      "m3_median_lp": -0.22533510625362396,
      "m5_median_lp": -0.34515246748924255,
      "m3_mean_lp": -1.8029823147184267,
      "m5_mean_lp": -1.6330422216040406,
      "m3_prob_median": 0.7982486758110119,
      "m5_prob_median": 0.7081123811200218,
      "direction": "m3>m5"
    },
    "dense": {
      "n": 1000,
      "n_ties": 0,
      "n_nonzero": 1000,
      "median_diff": 0.00026002148661063984,
      "mean_diff": 0.5418239545176224,
      "std_diff": 2.9624931634035607,
      "wilcoxon_stat": 217527.0,
      "p_value": 0.0003410538200066114,
      "p_bonferroni": 0.001705269100033057,
      "effect_size_r": 0.11327084755354884,
      "z_score": 3.581938707669256,
      "m3_median_lp": -0.14667461067438126,
      "m5_median_lp": -0.1057770662009716,
      "m3_mean_lp": -1.6534713189456967,
      "m5_mean_lp": -1.111647364428074,
      "m3_prob_median": 0.8635749297684109,
      "m5_prob_median": 0.8996251825817885,
      "direction": "m5>m3"
    }
  }
}