{
  "diagnostic_1_probability_scale": {
    "prosqa_id": {
      "m3_median_prob": 1.8015017595499927e-16,
      "m5_median_prob": 7.91679494328894e-14,
      "ratio_m5_over_m3": 439.4552989649325
    },
    "7hop": {
      "m3_median_prob": 1.124004303514157e-15,
      "m5_median_prob": 8.064306124379313e-14,
      "ratio_m5_over_m3": 71.74622107020912
    },
    "8hop": {
      "m3_median_prob": 1.374099505807729e-15,
      "m5_median_prob": 1.1024880913607591e-13,
      "ratio_m5_over_m3": 80.23349740692105
    },
    "dag": {
      "m3_median_prob": 2.452494473518025e-16,
      "m5_median_prob": 6.406020952062677e-14,
      "ratio_m5_over_m3": 261.20429714459027
    },
    "dense": {
      "m3_median_prob": 9.287058992349747e-16,
      "m5_median_prob": 6.880599064425863e-14,
      "ratio_m5_over_m3": 74.08803012981595
    }
  },
  "diagnostic_2_entropy": {
    "prosqa_id": {
      "n": 500,
      "m3_mean_entropy": 8.245684373986754e-08,
      "m5_mean_entropy": 7.790865615004972e-07,
      "m3_std_entropy": 5.380577548978713e-07,
      "m5_std_entropy": 2.5222992223835106e-06,
      "m3_median_entropy": 3.2982319098096013e-09,
      "m5_median_entropy": 9.583473570273782e-08,
      "entropy_diff_mean": -6.966297177606296e-07,
      "entropy_diff_p_value": 3.2357157290647014e-09,
      "direction": "M5 > M3"
    },
    "dag": {
      "n": 1000,
      "m3_mean_entropy": 5.54877669094885e-08,
      "m5_mean_entropy": 9.562996519757449e-07,
      "m3_std_entropy": 1.9521541945357018e-07,
      "m5_std_entropy": 4.271332058753898e-06,
      "m3_median_entropy": 8.63850679877487e-09,
      "m5_median_entropy": 2.2466630866802006e-07,
      "entropy_diff_mean": -9.008118850662564e-07,
      "entropy_diff_p_value": 3.5962255165486594e-11,
      "direction": "M5 > M3"
    }
  },
  "diagnostic_3_subsampling": {
    "prosqa_id": {
      "50": {
        "median_r": 1.0111228967991008,
        "mean_r": 1.0051491497447864,
        "std_r": 0.06029258401254942,
        "min_r": 0.8318166676389841,
        "max_r": 1.1251537019813398,
        "n_successful": 200
      },
      "100": {
        "median_r": 0.8386074293556046,
        "mean_r": 0.836016651585947,
        "std_r": 0.015510156365912298,
        "min_r": 0.7880640541545902,
        "max_r": 0.8616442058077676,
        "n_successful": 200
      },
      "200": {
        "median_r": 0.8358284039529764,
        "mean_r": 0.8359841376581776,
        "std_r": 0.009534859818316218,
        "min_r": 0.8060189883590921,
        "max_r": 0.8577864249910184,
        "n_successful": 200
      },
      "500": {
        "median_r": 0.8347188840290514,
        "mean_r": 0.8347188840290514,
        "std_r": 0.0,
        "min_r": 0.8347188840290514,
        "max_r": 0.8347188840290514,
        "n_successful": 200
      }
    },
    "7hop": {
      "50": {
        "median_r": 0.9481486999740725,
        "mean_r": 0.9531135622917283,
        "std_r": 0.06293261512730695,
        "min_r": 0.7928139676366417,
        "max_r": 1.1251537019813398,
        "n_successful": 200
      },
      "100": {
        "median_r": 0.819008977747048,
        "mean_r": 0.8156927134353896,
        "std_r": 0.02111324353944751,
        "min_r": 0.7382083439222973,
        "max_r": 0.8575182159954398,
        "n_successful": 200
      },
      "200": {
        "median_r": 0.8189177079865471,
        "mean_r": 0.8169341523729338,
        "std_r": 0.015910137227400267,
        "min_r": 0.7601185278787841,
        "max_r": 0.8519194488394003,
        "n_successful": 200
      },
      "500": {
        "median_r": 0.8164834911273876,
        "mean_r": 0.8167422178703777,
        "std_r": 0.007052610683216399,
        "min_r": 0.8009183795838579,
        "max_r": 0.8364206695578106,
        "n_successful": 200
      },
      "1000": {
        "median_r": 0.8170330113820354,
        "mean_r": 0.8170330113820353,
        "std_r": 1.1102230246251565e-16,
        "min_r": 0.8170330113820354,
        "max_r": 0.8170330113820354,
        "n_successful": 200
      }
    },
    "8hop": {
      "50": {
        "median_r": 0.9739360145177666,
        "mean_r": 0.9693214129779848,
        "std_r": 0.06744026838119704,
        "min_r": 0.75397404039132,
        "max_r": 1.1251537019813398,
        "n_successful": 200
      },
      "100": {
        "median_r": 0.8214158051375724,
        "mean_r": 0.8201642548944997,
        "std_r": 0.019890168055637775,
        "min_r": 0.7581506280152144,
        "max_r": 0.8633633682295706,
        "n_successful": 200
      },
      "200": {
        "median_r": 0.8193922428223398,
        "mean_r": 0.8200169032243652,
        "std_r": 0.0133925371240747,
        "min_r": 0.7714210848767548,
        "max_r": 0.8509703791678149,
        "n_successful": 200
      },
      "500": {
        "median_r": 0.8197694591199104,
        "mean_r": 0.819961774720315,
        "std_r": 0.007195415498732053,
        "min_r": 0.801264270951492,
        "max_r": 0.839699719722981,
        "n_successful": 200
      },
      "1000": {
        "median_r": 0.8202902881460207,
        "mean_r": 0.8202902881460207,
        "std_r": 0.0,
        "min_r": 0.8202902881460207,
        "max_r": 0.8202902881460207,
        "n_successful": 200
      }
    },
    "dag": {
      "50": {
        "median_r": 1.0673612216706767,
        "mean_r": 1.0618530277163836,
        "std_r": 0.04394317499378165,
        "min_r": 0.9294377393890321,
        "max_r": 1.1251537019813398,
        "n_successful": 200
      },
      "100": {
        "median_r": 0.8544237236361942,
        "mean_r": 0.8535108483902165,
        "std_r": 0.00847521599972869,
        "min_r": 0.8245102974968181,
        "max_r": 0.8678331905262591,
        "n_successful": 200
      },
      "200": {
        "median_r": 0.8526959603888792,
        "mean_r": 0.8522434267136549,
        "std_r": 0.006530786353488246,
        "min_r": 0.8320752647971617,
        "max_r": 0.8655515404858075,
        "n_successful": 200
      },
      "500": {
        "median_r": 0.8517021501798804,
        "mean_r": 0.8516264691486423,
        "std_r": 0.003518625131681763,
        "min_r": 0.8409449286464634,
        "max_r": 0.862016630762726,
        "n_successful": 200
      },
      "1000": {
        "median_r": 0.8518384501300393,
        "mean_r": 0.851838450130039,
        "std_r": 3.3306690738754696e-16,
        "min_r": 0.8518384501300393,
        "max_r": 0.8518384501300393,
        "n_successful": 200
      }
    },
    "dense": {
      "50": {
        "median_r": 0.9893792786083057,
        "mean_r": 0.9841720531880661,
        "std_r": 0.06501513630435339,
        "min_r": 0.8275968732060146,
        "max_r": 1.1251537019813398,
        "n_successful": 200
      },
      "100": {
        "median_r": 0.8289801197935066,
        "mean_r": 0.8266678463361812,
        "std_r": 0.020971271348192252,
        "min_r": 0.7578067955308537,
        "max_r": 0.8661140281044558,
        "n_successful": 200
      },
      "200": {
        "median_r": 0.8226708471423619,
        "mean_r": 0.8233761784664719,
        "std_r": 0.013990142267202568,
        "min_r": 0.780135270043129,
        "max_r": 0.8511429372899214,
        "n_successful": 200
      },
      "500": {
        "median_r": 0.823643442437411,
        "mean_r": 0.8231771116955668,
        "std_r": 0.007503303921795524,
        "min_r": 0.8019007110679384,
        "max_r": 0.8400041041264988,
        "n_successful": 200
      },
      "1000": {
        "median_r": 0.82315641477788,
        "mean_r": 0.8231564147778804,
        "std_r": 3.3306690738754696e-16,
        "min_r": 0.82315641477788,
        "max_r": 0.82315641477788,
        "n_successful": 200
      }
    }
  },
  "diagnostic_4_species_token": {
    "dag": {
      "first_token_wilcoxon": {
        "n": 1000,
        "direction": "M5 > M3",
        "p": 7.992937269512403e-160,
        "r": 0.8518384501300393,
        "m3_median_lp": -35.94425582885742,
        "m5_median_lp": -30.378952980041504
      },
      "species_token_wilcoxon": {
        "n": 1000,
        "direction": "M5 > M3",
        "p": 1.2486659436314225e-164,
        "r": 0.8647152509398429,
        "m3_median_lp": -51.22725296020508,
        "m5_median_lp": -38.59332847595215
      },
      "species_same_as_first": false,
      "n_species_valid": 1000
    }
  },
  "diagnostic_5_quartile_accuracy": {
    "prosqa_id": {
      "by_m3_confidence": {
        "Q1": {
          "n": 125,
          "m3_acc": 0.984,
          "m5_acc": 0.976,
          "diff_m3_minus_m5": 0.008000000000000007,
          "m3_lp_range": [
            -50.72520446777344,
            -38.92578887939453
          ]
        },
        "Q2": {
          "n": 125,
          "m3_acc": 0.984,
          "m5_acc": 0.944,
          "diff_m3_minus_m5": 0.040000000000000036,
          "m3_lp_range": [
            -38.88484191894531,
            -36.26152801513672
          ]
        },
        "Q3": {
          "n": 125,
          "m3_acc": 0.96,
          "m5_acc": 0.984,
          "diff_m3_minus_m5": -0.02400000000000002,
          "m3_lp_range": [
            -36.243953704833984,
            -33.15000915527344
          ]
        },
        "Q4": {
          "n": 125,
          "m3_acc": 0.952,
          "m5_acc": 0.96,
          "diff_m3_minus_m5": -0.008000000000000007,
          "m3_lp_range": [
            -33.142913818359375,
            -25.003509521484375
          ]
        }
      },
      "by_m5_confidence": {}
    },
    "7hop": {
      "by_m3_confidence": {
        "Q1": {
          "n": 250,
          "m3_acc": 0.66,
          "m5_acc": 0.772,
          "diff_m3_minus_m5": -0.11199999999999999,
          "m3_lp_range": [
            -47.91731262207031,
            -38.02136993408203
          ]
        },
        "Q2": {
          "n": 250,
          "m3_acc": 0.712,
          "m5_acc": 0.74,
          "diff_m3_minus_m5": -0.028000000000000025,
          "m3_lp_range": [
            -38.012367248535156,
            -34.42694091796875
          ]
        },
        "Q3": {
          "n": 250,
          "m3_acc": 0.608,
          "m5_acc": 0.756,
          "diff_m3_minus_m5": -0.14800000000000002,
          "m3_lp_range": [
            -34.41681671142578,
            -31.62476348876953
          ]
        },
        "Q4": {
          "n": 250,
          "m3_acc": 0.66,
          "m5_acc": 0.748,
          "diff_m3_minus_m5": -0.08799999999999997,
          "m3_lp_range": [
            -31.606525421142578,
            -21.095178604125977
          ]
        }
      },
      "by_m5_confidence": {}
    },
    "8hop": {
      "by_m3_confidence": {
        "Q1": {
          "n": 250,
          "m3_acc": 0.676,
          "m5_acc": 0.776,
          "diff_m3_minus_m5": -0.09999999999999998,
          "m3_lp_range": [
            -47.47686767578125,
            -37.30986022949219
          ]
        },
        "Q2": {
          "n": 250,
          "m3_acc": 0.652,
          "m5_acc": 0.744,
          "diff_m3_minus_m5": -0.09199999999999997,
          "m3_lp_range": [
            -37.29325866699219,
            -34.222503662109375
          ]
        },
        "Q3": {
          "n": 250,
          "m3_acc": 0.676,
          "m5_acc": 0.732,
          "diff_m3_minus_m5": -0.05599999999999994,
          "m3_lp_range": [
            -34.219451904296875,
            -31.44131088256836
          ]
        },
        "Q4": {
          "n": 250,
          "m3_acc": 0.696,
          "m5_acc": 0.752,
          "diff_m3_minus_m5": -0.05600000000000005,
          "m3_lp_range": [
            -31.43999481201172,
            -23.230289459228516
          ]
        }
      },
      "by_m5_confidence": {}
    },
    "dag": {
      "by_m3_confidence": {
        "Q1": {
          "n": 250,
          "m3_acc": 0.568,
          "m5_acc": 0.516,
          "diff_m3_minus_m5": 0.051999999999999935,
          "m3_lp_range": [
            -50.762939453125,
            -39.221099853515625
          ]
        },
        "Q2": {
          "n": 250,
          "m3_acc": 0.556,
          "m5_acc": 0.5,
          "diff_m3_minus_m5": 0.05600000000000005,
          "m3_lp_range": [
            -39.2105598449707,
            -35.946693420410156
          ]
        },
        "Q3": {
          "n": 250,
          "m3_acc": 0.636,
          "m5_acc": 0.56,
          "diff_m3_minus_m5": 0.07599999999999996,
          "m3_lp_range": [
            -35.94181823730469,
            -32.98561096191406
          ]
        },
        "Q4": {
          "n": 250,
          "m3_acc": 0.608,
          "m5_acc": 0.5,
          "diff_m3_minus_m5": 0.10799999999999998,
          "m3_lp_range": [
            -32.979331970214844,
            -25.686195373535156
          ]
        }
      },
      "by_m5_confidence": {}
    },
    "dense": {
      "by_m3_confidence": {
        "Q1": {
          "n": 250,
          "m3_acc": 0.648,
          "m5_acc": 0.692,
          "diff_m3_minus_m5": -0.04399999999999993,
          "m3_lp_range": [
            -48.519752502441406,
            -37.903221130371094
          ]
        },
        "Q2": {
          "n": 250,
          "m3_acc": 0.632,
          "m5_acc": 0.696,
          "diff_m3_minus_m5": -0.06399999999999995,
          "m3_lp_range": [
            -37.88905334472656,
            -34.615211486816406
          ]
        },
        "Q3": {
          "n": 250,
          "m3_acc": 0.568,
          "m5_acc": 0.648,
          "diff_m3_minus_m5": -0.08000000000000007,
          "m3_lp_range": [
            -34.610267639160156,
            -31.857315063476562
          ]
        },
        "Q4": {
          "n": 250,
          "m3_acc": 0.6,
          "m5_acc": 0.7,
          "diff_m3_minus_m5": -0.09999999999999998,
          "m3_lp_range": [
            -31.839698791503906,
            -22.86867332458496
          ]
        }
      },
      "by_m5_confidence": {}
    }
  }
}