{
  "queries": 970,
  "models": {
    "upstream": {
      "1": {
        "scored_queries": 970,
        "excluded_queries": 0,
        "hit": 0.534020618556701,
        "precision": 0.534020618556701,
        "macro_precision": 0.534020618556701,
        "ndcg": 0.4592047128129602,
        "known_positive_recall": 0.014444554022554491,
        "recall_queries": 925,
        "useful": 518,
        "retained": 970,
        "excluded_positions": 0,
        "mean_useful": 0.534020618556701,
        "mean_retained": 1.0
      },
      "3": {
        "scored_queries": 970,
        "excluded_queries": 0,
        "hit": 0.7298969072164948,
        "precision": 0.5082530949105915,
        "macro_precision": 0.5082474226804123,
        "ndcg": 0.4413745306026501,
        "known_positive_recall": 0.03938697535513605,
        "recall_queries": 925,
        "useful": 1478,
        "retained": 2908,
        "excluded_positions": 2,
        "mean_useful": 1.5237113402061855,
        "mean_retained": 2.997938144329897
      },
      "5": {
        "scored_queries": 970,
        "excluded_queries": 0,
        "hit": 0.7948453608247422,
        "precision": 0.49948379103861246,
        "macro_precision": 0.4996219931271478,
        "ndcg": 0.43462885647387695,
        "known_positive_recall": 0.06172226483192968,
        "recall_queries": 925,
        "useful": 2419,
        "retained": 4843,
        "excluded_positions": 7,
        "mean_useful": 2.4938144329896907,
        "mean_retained": 4.992783505154639
      },
      "10": {
        "scored_queries": 970,
        "excluded_queries": 0,
        "hit": 0.8484536082474227,
        "precision": 0.4739561802397685,
        "macro_precision": 0.47407543773523153,
        "ndcg": 0.43629894440575245,
        "known_positive_recall": 0.11180689389376004,
        "recall_queries": 925,
        "useful": 4586,
        "retained": 9676,
        "excluded_positions": 24,
        "mean_useful": 4.727835051546392,
        "mean_retained": 9.975257731958763
      },
      "20": {
        "scored_queries": 970,
        "excluded_queries": 0,
        "hit": 0.8876288659793814,
        "precision": 0.4393610421836228,
        "macro_precision": 0.44005992535723476,
        "ndcg": 0.4525661889832219,
        "known_positive_recall": 0.19611550928151497,
        "recall_queries": 925,
        "useful": 8499,
        "retained": 19344,
        "excluded_positions": 56,
        "mean_useful": 8.761855670103094,
        "mean_retained": 19.942268041237114
      },
      "50": {
        "scored_queries": 970,
        "excluded_queries": 0,
        "hit": 0.931958762886598,
        "precision": 0.38598560079443894,
        "macro_precision": 0.3867178488702632,
        "ndcg": 0.5107049932805754,
        "known_positive_recall": 0.41118693689165037,
        "recall_queries": 925,
        "useful": 18657,
        "retained": 48336,
        "excluded_positions": 164,
        "mean_useful": 19.234020618556702,
        "mean_retained": 49.83092783505155
      }
    },
    "finetuned": {
      "1": {
        "scored_queries": 970,
        "excluded_queries": 0,
        "hit": 0.6474226804123712,
        "precision": 0.6474226804123712,
        "macro_precision": 0.6474226804123712,
        "ndcg": 0.5312714776632302,
        "known_positive_recall": 0.017869431761799028,
        "recall_queries": 925,
        "useful": 628,
        "retained": 970,
        "excluded_positions": 0,
        "mean_useful": 0.6474226804123712,
        "mean_retained": 1.0
      },
      "3": {
        "scored_queries": 970,
        "excluded_queries": 0,
        "hit": 0.7938144329896907,
        "precision": 0.6308009625300791,
        "macro_precision": 0.6309278350515464,
        "ndcg": 0.5406982958852574,
        "known_positive_recall": 0.05034823759106709,
        "recall_queries": 925,
        "useful": 1835,
        "retained": 2909,
        "excluded_positions": 1,
        "mean_useful": 1.8917525773195876,
        "mean_retained": 2.9989690721649485
      },
      "5": {
        "scored_queries": 970,
        "excluded_queries": 0,
        "hit": 0.8474226804123711,
        "precision": 0.6266005782734407,
        "macro_precision": 0.6269759450171821,
        "ndcg": 0.5453498298950709,
        "known_positive_recall": 0.08055790386567772,
        "recall_queries": 925,
        "useful": 3034,
        "retained": 4842,
        "excluded_positions": 8,
        "mean_useful": 3.1278350515463917,
        "mean_retained": 4.991752577319588
      },
      "10": {
        "scored_queries": 970,
        "excluded_queries": 0,
        "hit": 0.8814432989690721,
        "precision": 0.6117999586691465,
        "macro_precision": 0.6123367697594502,
        "ndcg": 0.5574064002423099,
        "known_positive_recall": 0.15074270958466607,
        "recall_queries": 925,
        "useful": 5921,
        "retained": 9678,
        "excluded_positions": 22,
        "mean_useful": 6.104123711340206,
        "mean_retained": 9.977319587628866
      },
      "20": {
        "scored_queries": 970,
        "excluded_queries": 0,
        "hit": 0.9051546391752577,
        "precision": 0.5810720008270016,
        "macro_precision": 0.5815832221977094,
        "ndcg": 0.5818739813983459,
        "known_positive_recall": 0.27556154002694083,
        "recall_queries": 925,
        "useful": 11242,
        "retained": 19347,
        "excluded_positions": 53,
        "mean_useful": 11.589690721649484,
        "mean_retained": 19.945360824742266
      },
      "50": {
        "scored_queries": 970,
        "excluded_queries": 0,
        "hit": 0.9443298969072165,
        "precision": 0.5045207208325575,
        "macro_precision": 0.5051647365119746,
        "ndcg": 0.6449716439245425,
        "known_positive_recall": 0.5791516627126307,
        "recall_queries": 925,
        "useful": 24385,
        "retained": 48333,
        "excluded_positions": 167,
        "mean_useful": 25.13917525773196,
        "mean_retained": 49.827835051546394
      }
    }
  },
  "answerable": {
    "queries": 925,
    "models": {
      "upstream": {
        "1": {
          "scored_queries": 925,
          "excluded_queries": 0,
          "hit": 0.56,
          "precision": 0.56,
          "macro_precision": 0.56,
          "ndcg": 0.46208494208494205,
          "known_positive_recall": 0.014444554022554491,
          "recall_queries": 925,
          "useful": 518,
          "retained": 925,
          "excluded_positions": 0,
          "mean_useful": 0.56,
          "mean_retained": 1.0
        },
        "3": {
          "scored_queries": 925,
          "excluded_queries": 0,
          "hit": 0.7654054054054054,
          "precision": 0.5329967544175983,
          "macro_precision": 0.532972972972973,
          "ndcg": 0.44657619780045693,
          "known_positive_recall": 0.03938697535513605,
          "recall_queries": 925,
          "useful": 1478,
          "retained": 2773,
          "excluded_positions": 2,
          "mean_useful": 1.597837837837838,
          "mean_retained": 2.997837837837838
        },
        "5": {
          "scored_queries": 925,
          "excluded_queries": 0,
          "hit": 0.8335135135135135,
          "precision": 0.5238198354265916,
          "macro_precision": 0.523927927927928,
          "ndcg": 0.4398171919899157,
          "known_positive_recall": 0.06172226483192968,
          "recall_queries": 925,
          "useful": 2419,
          "retained": 4618,
          "excluded_positions": 7,
          "mean_useful": 2.615135135135135,
          "mean_retained": 4.992432432432432
        },
        "10": {
          "scored_queries": 925,
          "excluded_queries": 0,
          "hit": 0.8897297297297297,
          "precision": 0.49707348796878387,
          "macro_precision": 0.49713856713856713,
          "ndcg": 0.4417676357044915,
          "known_positive_recall": 0.11180689389376004,
          "recall_queries": 925,
          "useful": 4586,
          "retained": 9226,
          "excluded_positions": 24,
          "mean_useful": 4.957837837837838,
          "mean_retained": 9.974054054054054
        },
        "20": {
          "scored_queries": 925,
          "excluded_queries": 0,
          "hit": 0.9308108108108109,
          "precision": 0.46080026024723486,
          "macro_precision": 0.4614682460502894,
          "ndcg": 0.4579770106113412,
          "known_positive_recall": 0.19611550928151497,
          "recall_queries": 925,
          "useful": 8499,
          "retained": 18444,
          "excluded_positions": 56,
          "mean_useful": 9.188108108108109,
          "mean_retained": 19.93945945945946
        },
        "50": {
          "scored_queries": 925,
          "excluded_queries": 0,
          "hit": 0.9772972972972973,
          "precision": 0.4048301002473636,
          "macro_precision": 0.4055311496261139,
          "ndcg": 0.5145143143872781,
          "known_positive_recall": 0.41118693689165037,
          "recall_queries": 925,
          "useful": 18657,
          "retained": 46086,
          "excluded_positions": 164,
          "mean_useful": 20.16972972972973,
          "mean_retained": 49.8227027027027
        }
      },
      "finetuned": {
        "1": {
          "scored_queries": 925,
          "excluded_queries": 0,
          "hit": 0.6789189189189189,
          "precision": 0.6789189189189189,
          "macro_precision": 0.6789189189189189,
          "ndcg": 0.5365765765765765,
          "known_positive_recall": 0.017869431761799028,
          "recall_queries": 925,
          "useful": 628,
          "retained": 925,
          "excluded_positions": 0,
          "mean_useful": 0.6789189189189189,
          "mean_retained": 1.0
        },
        "3": {
          "scored_queries": 925,
          "excluded_queries": 0,
          "hit": 0.8324324324324325,
          "precision": 0.6614996395097332,
          "macro_precision": 0.6616216216216216,
          "ndcg": 0.5439797460516042,
          "known_positive_recall": 0.05034823759106709,
          "recall_queries": 925,
          "useful": 1835,
          "retained": 2774,
          "excluded_positions": 1,
          "mean_useful": 1.9837837837837837,
          "mean_retained": 2.998918918918919
        },
        "5": {
          "scored_queries": 925,
          "excluded_queries": 0,
          "hit": 0.8886486486486487,
          "precision": 0.6571366688325753,
          "macro_precision": 0.6574774774774775,
          "ndcg": 0.5486144310934794,
          "known_positive_recall": 0.08055790386567772,
          "recall_queries": 925,
          "useful": 3034,
          "retained": 4617,
          "excluded_positions": 8,
          "mean_useful": 3.28,
          "mean_retained": 4.991351351351351
        },
        "10": {
          "scored_queries": 925,
          "excluded_queries": 0,
          "hit": 0.9243243243243243,
          "precision": 0.6416341569137408,
          "macro_precision": 0.6421261261261262,
          "ndcg": 0.5620625351358739,
          "known_positive_recall": 0.15074270958466607,
          "recall_queries": 925,
          "useful": 5921,
          "retained": 9228,
          "excluded_positions": 22,
          "mean_useful": 6.401081081081081,
          "mean_retained": 9.976216216216216
        },
        "20": {
          "scored_queries": 925,
          "excluded_queries": 0,
          "hit": 0.9491891891891892,
          "precision": 0.6094215861657722,
          "macro_precision": 0.6098764600343548,
          "ndcg": 0.5888460119487677,
          "known_positive_recall": 0.27556154002694083,
          "recall_queries": 925,
          "useful": 11242,
          "retained": 18447,
          "excluded_positions": 53,
          "mean_useful": 12.153513513513513,
          "mean_retained": 19.942702702702704
        },
        "50": {
          "scored_queries": 925,
          "excluded_queries": 0,
          "hit": 0.9902702702702703,
          "precision": 0.5291539179306902,
          "macro_precision": 0.5297403182882328,
          "ndcg": 0.6524763537042031,
          "known_positive_recall": 0.5791516627126307,
          "recall_queries": 925,
          "useful": 24385,
          "retained": 46083,
          "excluded_positions": 167,
          "mean_useful": 26.36216216216216,
          "mean_retained": 49.81945945945946
        }
      }
    }
  },
  "corpus_passages": 82719
}