{
  "queries": 970,
  "models": {
    "production": {
      "1": {
        "scored_queries": 964,
        "excluded_queries": 6,
        "hit": 0.7261410788381742,
        "precision": 0.7261410788381742,
        "macro_precision": 0.7261410788381742,
        "ndcg": 0.6822762299940723,
        "known_positive_recall": 0.03374565629667812,
        "recall_queries": 889,
        "useful": 700,
        "retained": 964,
        "excluded_positions": 0,
        "mean_useful": 0.7261410788381742,
        "mean_retained": 1.0
      },
      "3": {
        "scored_queries": 970,
        "excluded_queries": 0,
        "hit": 0.8391752577319588,
        "precision": 0.7108890420399724,
        "macro_precision": 0.7109965635738832,
        "ndcg": 0.6737359241548024,
        "known_positive_recall": 0.09151322333990437,
        "recall_queries": 895,
        "useful": 2063,
        "retained": 2902,
        "excluded_positions": 8,
        "mean_useful": 2.12680412371134,
        "mean_retained": 2.9917525773195877
      },
      "5": {
        "scored_queries": 970,
        "excluded_queries": 0,
        "hit": 0.8587628865979381,
        "precision": 0.6968194960760017,
        "macro_precision": 0.6970103092783505,
        "ndcg": 0.6720934096988599,
        "known_positive_recall": 0.14456571515697228,
        "recall_queries": 895,
        "useful": 3374,
        "retained": 4842,
        "excluded_positions": 8,
        "mean_useful": 3.4783505154639176,
        "mean_retained": 4.991752577319588
      },
      "10": {
        "scored_queries": 970,
        "excluded_queries": 0,
        "hit": 0.8783505154639175,
        "precision": 0.6640181611804767,
        "macro_precision": 0.6641809851088202,
        "ndcg": 0.6764288375969912,
        "known_positive_recall": 0.2647168884118442,
        "recall_queries": 895,
        "useful": 6435,
        "retained": 9691,
        "excluded_positions": 9,
        "mean_useful": 6.634020618556701,
        "mean_retained": 9.990721649484536
      },
      "20": {
        "scored_queries": 970,
        "excluded_queries": 0,
        "hit": 0.9010309278350516,
        "precision": 0.6041172221648953,
        "macro_precision": 0.604256841821554,
        "ndcg": 0.6943443750382963,
        "known_positive_recall": 0.4574631459961227,
        "recall_queries": 895,
        "useful": 11709,
        "retained": 19382,
        "excluded_positions": 18,
        "mean_useful": 12.071134020618556,
        "mean_retained": 19.981443298969072
      },
      "selected": {
        "scored_queries": 970,
        "excluded_queries": 0,
        "hit": 0.8525773195876288,
        "precision": 0.8040927303949628,
        "macro_precision": 0.7015133181852246,
        "ndcg": 0.67618405088445,
        "known_positive_recall": 0.19797868631055085,
        "recall_queries": 895,
        "useful": 5619,
        "retained": 6988,
        "excluded_positions": 8,
        "mean_useful": 5.792783505154639,
        "mean_retained": 7.204123711340206
      }
    },
    "candidate": {
      "1": {
        "scored_queries": 964,
        "excluded_queries": 6,
        "hit": 0.7230290456431535,
        "precision": 0.7230290456431535,
        "macro_precision": 0.7230290456431535,
        "ndcg": 0.6648389646314957,
        "known_positive_recall": 0.03456403636248349,
        "recall_queries": 889,
        "useful": 697,
        "retained": 964,
        "excluded_positions": 0,
        "mean_useful": 0.7230290456431535,
        "mean_retained": 1.0
      },
      "3": {
        "scored_queries": 970,
        "excluded_queries": 0,
        "hit": 0.8536082474226804,
        "precision": 0.7157640565712314,
        "macro_precision": 0.7166666666666667,
        "ndcg": 0.6576832474683237,
        "known_positive_recall": 0.09526530468586906,
        "recall_queries": 895,
        "useful": 2075,
        "retained": 2899,
        "excluded_positions": 11,
        "mean_useful": 2.1391752577319587,
        "mean_retained": 2.988659793814433
      },
      "5": {
        "scored_queries": 970,
        "excluded_queries": 0,
        "hit": 0.8845360824742268,
        "precision": 0.7028311634635255,
        "macro_precision": 0.7031786941580757,
        "ndcg": 0.6554718901879515,
        "known_positive_recall": 0.15391852985862914,
        "recall_queries": 895,
        "useful": 3401,
        "retained": 4839,
        "excluded_positions": 11,
        "mean_useful": 3.5061855670103093,
        "mean_retained": 4.988659793814433
      },
      "10": {
        "scored_queries": 970,
        "excluded_queries": 0,
        "hit": 0.8989690721649485,
        "precision": 0.67180070291503,
        "macro_precision": 0.6722631320569465,
        "ndcg": 0.6610872123112499,
        "known_positive_recall": 0.28045885967323153,
        "recall_queries": 895,
        "useful": 6499,
        "retained": 9674,
        "excluded_positions": 26,
        "mean_useful": 6.7,
        "mean_retained": 9.97319587628866
      },
      "20": {
        "scored_queries": 970,
        "excluded_queries": 0,
        "hit": 0.9103092783505154,
        "precision": 0.61794500723589,
        "macro_precision": 0.6184956360634603,
        "ndcg": 0.684223528795109,
        "known_positive_recall": 0.4887591886594314,
        "recall_queries": 895,
        "useful": 11956,
        "retained": 19348,
        "excluded_positions": 52,
        "mean_useful": 12.32577319587629,
        "mean_retained": 19.94639175257732
      },
      "selected": {
        "scored_queries": 970,
        "excluded_queries": 0,
        "hit": 0.8618556701030928,
        "precision": 0.8033770583310076,
        "macro_precision": 0.7040325511660611,
        "ndcg": 0.6599128936120324,
        "known_positive_recall": 0.20805438995183476,
        "recall_queries": 895,
        "useful": 5757,
        "retained": 7166,
        "excluded_positions": 17,
        "mean_useful": 5.935051546391753,
        "mean_retained": 7.387628865979382
      }
    }
  },
  "public": {
    "scifact": {
      "queries": 300,
      "ndcg@10": {
        "upstream": 0.7875641458078616,
        "production": 0.7678535660256963,
        "candidate": 0.7782845713792622
      }
    },
    "fiqa": {
      "queries": 648,
      "ndcg@10": {
        "upstream": 0.474145026242454,
        "production": 0.4008741437969072,
        "candidate": 0.446838124778398
      }
    },
    "nfcorpus": {
      "queries": 323,
      "ndcg@10": {
        "upstream": 0.3932480325545885,
        "candidate": 0.3889646280478944
      }
    },
    "scidocs": {
      "queries": 1000,
      "ndcg@10": {
        "upstream": 0.19447454865159183,
        "candidate": 0.18521860884727676
      }
    },
    "arguana": {
      "queries": 1406,
      "ndcg@10": {
        "upstream": 0.6431541443325537,
        "candidate": 0.6258634411077135
      }
    }
  }
}