{
  "queries": 970,
  "models": {
    "cuda": {
      "1": {
        "scored_queries": 965,
        "excluded_queries": 5,
        "hit": 0.7233160621761658,
        "precision": 0.7233160621761658,
        "macro_precision": 0.7233160621761658,
        "ndcg": 0.6458425857389588,
        "known_positive_recall": 0.021209009540392988,
        "recall_queries": 920,
        "useful": 698,
        "retained": 965,
        "excluded_positions": 0,
        "mean_useful": 0.7233160621761658,
        "mean_retained": 1.0
      },
      "3": {
        "scored_queries": 970,
        "excluded_queries": 0,
        "hit": 0.8536082474226804,
        "precision": 0.7157640565712314,
        "macro_precision": 0.7166666666666667,
        "ndcg": 0.6374467700901159,
        "known_positive_recall": 0.0589083117861917,
        "recall_queries": 925,
        "useful": 2075,
        "retained": 2899,
        "excluded_positions": 11,
        "mean_useful": 2.1391752577319587,
        "mean_retained": 2.988659793814433
      },
      "5": {
        "scored_queries": 970,
        "excluded_queries": 0,
        "hit": 0.8845360824742268,
        "precision": 0.7028311634635255,
        "macro_precision": 0.7031786941580757,
        "ndcg": 0.632593817782834,
        "known_positive_recall": 0.09351524011264825,
        "recall_queries": 925,
        "useful": 3401,
        "retained": 4839,
        "excluded_positions": 11,
        "mean_useful": 3.5061855670103093,
        "mean_retained": 4.988659793814433
      },
      "10": {
        "scored_queries": 970,
        "excluded_queries": 0,
        "hit": 0.8989690721649485,
        "precision": 0.67180070291503,
        "macro_precision": 0.6722631320569465,
        "ndcg": 0.6315462046841925,
        "known_positive_recall": 0.17039667688880922,
        "recall_queries": 925,
        "useful": 6499,
        "retained": 9674,
        "excluded_positions": 26,
        "mean_useful": 6.7,
        "mean_retained": 9.97319587628866
      },
      "20": {
        "scored_queries": 970,
        "excluded_queries": 0,
        "hit": 0.9103092783505154,
        "precision": 0.61794500723589,
        "macro_precision": 0.6184956360634603,
        "ndcg": 0.6416731640651591,
        "known_positive_recall": 0.29717756138114526,
        "recall_queries": 925,
        "useful": 11956,
        "retained": 19348,
        "excluded_positions": 52,
        "mean_useful": 12.32577319587629,
        "mean_retained": 19.94639175257732
      },
      "50": {
        "scored_queries": 970,
        "excluded_queries": 0,
        "hit": 0.9309278350515464,
        "precision": 0.4043216890119198,
        "macro_precision": 0.4044939351610387,
        "ndcg": 0.6051209510262134,
        "known_positive_recall": 0.4598077844640693,
        "recall_queries": 925,
        "useful": 19572,
        "retained": 48407,
        "excluded_positions": 93,
        "mean_useful": 20.177319587628865,
        "mean_retained": 49.904123711340205
      },
      "selected": {
        "scored_queries": 970,
        "excluded_queries": 0,
        "hit": 0.8618556701030928,
        "precision": 0.8033770583310076,
        "macro_precision": 0.7040325511660611,
        "ndcg": 0.6359990281117442,
        "known_positive_recall": 0.128026594631235,
        "recall_queries": 925,
        "useful": 5757,
        "retained": 7166,
        "excluded_positions": 17,
        "mean_useful": 5.935051546391753,
        "mean_retained": 7.387628865979382
      }
    },
    "vulkan": {
      "1": {
        "scored_queries": 965,
        "excluded_queries": 5,
        "hit": 0.7243523316062176,
        "precision": 0.7243523316062176,
        "macro_precision": 0.7243523316062176,
        "ndcg": 0.6461386627189736,
        "known_positive_recall": 0.021228078953055077,
        "recall_queries": 920,
        "useful": 699,
        "retained": 965,
        "excluded_positions": 0,
        "mean_useful": 0.7243523316062176,
        "mean_retained": 1.0
      },
      "3": {
        "scored_queries": 970,
        "excluded_queries": 0,
        "hit": 0.8525773195876288,
        "precision": 0.7157640565712314,
        "macro_precision": 0.7166666666666667,
        "ndcg": 0.6373247400469291,
        "known_positive_recall": 0.05893952330659241,
        "recall_queries": 925,
        "useful": 2075,
        "retained": 2899,
        "excluded_positions": 11,
        "mean_useful": 2.1391752577319587,
        "mean_retained": 2.988659793814433
      },
      "5": {
        "scored_queries": 970,
        "excluded_queries": 0,
        "hit": 0.8845360824742268,
        "precision": 0.7023563455973543,
        "macro_precision": 0.702766323024055,
        "ndcg": 0.6321994825191731,
        "known_positive_recall": 0.09344587015679974,
        "recall_queries": 925,
        "useful": 3398,
        "retained": 4838,
        "excluded_positions": 12,
        "mean_useful": 3.5030927835051546,
        "mean_retained": 4.9876288659793815
      },
      "10": {
        "scored_queries": 970,
        "excluded_queries": 0,
        "hit": 0.8989690721649485,
        "precision": 0.6723514211886304,
        "macro_precision": 0.6727589592538046,
        "ndcg": 0.631891212140629,
        "known_positive_recall": 0.17053838380216288,
        "recall_queries": 925,
        "useful": 6505,
        "retained": 9675,
        "excluded_positions": 25,
        "mean_useful": 6.706185567010309,
        "mean_retained": 9.974226804123711
      },
      "20": {
        "scored_queries": 970,
        "excluded_queries": 0,
        "hit": 0.9103092783505154,
        "precision": 0.6178416373785404,
        "macro_precision": 0.6183925432799551,
        "ndcg": 0.6416203428445276,
        "known_positive_recall": 0.2971249330393808,
        "recall_queries": 925,
        "useful": 11954,
        "retained": 19348,
        "excluded_positions": 52,
        "mean_useful": 12.323711340206186,
        "mean_retained": 19.94639175257732
      },
      "50": {
        "scored_queries": 970,
        "excluded_queries": 0,
        "hit": 0.9309278350515464,
        "precision": 0.4043216890119198,
        "macro_precision": 0.4044939351610387,
        "ndcg": 0.6050905860403181,
        "known_positive_recall": 0.4598077844640693,
        "recall_queries": 925,
        "useful": 19572,
        "retained": 48407,
        "excluded_positions": 93,
        "mean_useful": 20.177319587628865,
        "mean_retained": 49.904123711340205
      },
      "selected": {
        "scored_queries": 970,
        "excluded_queries": 0,
        "hit": 0.8608247422680413,
        "precision": 0.8027855153203343,
        "macro_precision": 0.7038533836490732,
        "ndcg": 0.6358997417721292,
        "known_positive_recall": 0.12825382711378852,
        "recall_queries": 925,
        "useful": 5764,
        "retained": 7180,
        "excluded_positions": 17,
        "mean_useful": 5.942268041237114,
        "mean_retained": 7.402061855670103
      }
    }
  },
  "public": {},
  "answerable": {
    "queries": 925,
    "models": {
      "cuda": {
        "1": {
          "scored_queries": 920,
          "excluded_queries": 5,
          "hit": 0.758695652173913,
          "precision": 0.758695652173913,
          "macro_precision": 0.758695652173913,
          "ndcg": 0.6513457556935818,
          "known_positive_recall": 0.021209009540392988,
          "recall_queries": 920,
          "useful": 698,
          "retained": 920,
          "excluded_positions": 0,
          "mean_useful": 0.758695652173913,
          "mean_retained": 1.0
        },
        "3": {
          "scored_queries": 925,
          "excluded_queries": 0,
          "hit": 0.8951351351351351,
          "precision": 0.7507235890014472,
          "macro_precision": 0.7515315315315315,
          "ndcg": 0.6422807686434969,
          "known_positive_recall": 0.0589083117861917,
          "recall_queries": 925,
          "useful": 2075,
          "retained": 2764,
          "excluded_positions": 11,
          "mean_useful": 2.2432432432432434,
          "mean_retained": 2.9881081081081082
        },
        "5": {
          "scored_queries": 925,
          "excluded_queries": 0,
          "hit": 0.9275675675675675,
          "precision": 0.7371044646727352,
          "macro_precision": 0.7373873873873874,
          "ndcg": 0.6375956845322198,
          "known_positive_recall": 0.09351524011264825,
          "recall_queries": 925,
          "useful": 3401,
          "retained": 4614,
          "excluded_positions": 11,
          "mean_useful": 3.676756756756757,
          "mean_retained": 4.988108108108108
        },
        "10": {
          "scored_queries": 925,
          "excluded_queries": 0,
          "hit": 0.9427027027027027,
          "precision": 0.7045750216825672,
          "macro_precision": 0.704967824967825,
          "ndcg": 0.6377512015628823,
          "known_positive_recall": 0.17039667688880922,
          "recall_queries": 925,
          "useful": 6499,
          "retained": 9224,
          "excluded_positions": 26,
          "mean_useful": 7.025945945945946,
          "mean_retained": 9.971891891891891
        },
        "20": {
          "scored_queries": 925,
          "excluded_queries": 0,
          "hit": 0.9545945945945946,
          "precision": 0.6480919340849957,
          "macro_precision": 0.648584612953034,
          "ndcg": 0.6491335337062211,
          "known_positive_recall": 0.29717756138114526,
          "recall_queries": 925,
          "useful": 11956,
          "retained": 18448,
          "excluded_positions": 52,
          "mean_useful": 12.925405405405405,
          "mean_retained": 19.943783783783783
        },
        "50": {
          "scored_queries": 925,
          "excluded_queries": 0,
          "hit": 0.9762162162162162,
          "precision": 0.424031024546656,
          "macro_precision": 0.42417201849319736,
          "ndcg": 0.6129768665312807,
          "known_positive_recall": 0.4598077844640693,
          "recall_queries": 925,
          "useful": 19572,
          "retained": 46157,
          "excluded_positions": 93,
          "mean_useful": 21.158918918918918,
          "mean_retained": 49.89945945945946
        },
        "selected": {
          "scored_queries": 925,
          "excluded_queries": 0,
          "hit": 0.9037837837837838,
          "precision": 0.8188024463092021,
          "macro_precision": 0.7382827833849506,
          "ndcg": 0.6407625959742856,
          "known_positive_recall": 0.128026594631235,
          "recall_queries": 925,
          "useful": 5757,
          "retained": 7031,
          "excluded_positions": 17,
          "mean_useful": 6.223783783783784,
          "mean_retained": 7.601081081081081
        }
      },
      "vulkan": {
        "1": {
          "scored_queries": 920,
          "excluded_queries": 5,
          "hit": 0.7597826086956522,
          "precision": 0.7597826086956522,
          "macro_precision": 0.7597826086956522,
          "ndcg": 0.6516563146997929,
          "known_positive_recall": 0.021228078953055077,
          "recall_queries": 920,
          "useful": 699,
          "retained": 920,
          "excluded_positions": 0,
          "mean_useful": 0.7597826086956522,
          "mean_retained": 1.0
        },
        "3": {
          "scored_queries": 925,
          "excluded_queries": 0,
          "hit": 0.894054054054054,
          "precision": 0.7507235890014472,
          "macro_precision": 0.7515315315315315,
          "ndcg": 0.6421528020036147,
          "known_positive_recall": 0.05893952330659241,
          "recall_queries": 925,
          "useful": 2075,
          "retained": 2764,
          "excluded_positions": 11,
          "mean_useful": 2.2432432432432434,
          "mean_retained": 2.9881081081081082
        },
        "5": {
          "scored_queries": 925,
          "excluded_queries": 0,
          "hit": 0.9275675675675675,
          "precision": 0.7366139171905485,
          "macro_precision": 0.736954954954955,
          "ndcg": 0.6371821653908674,
          "known_positive_recall": 0.09344587015679974,
          "recall_queries": 925,
          "useful": 3398,
          "retained": 4613,
          "excluded_positions": 12,
          "mean_useful": 3.6735135135135133,
          "mean_retained": 4.987027027027027
        },
        "10": {
          "scored_queries": 925,
          "excluded_queries": 0,
          "hit": 0.9427027027027027,
          "precision": 0.7051490514905149,
          "macro_precision": 0.7054877734877735,
          "ndcg": 0.6381124055299272,
          "known_positive_recall": 0.17053838380216288,
          "recall_queries": 925,
          "useful": 6505,
          "retained": 9225,
          "excluded_positions": 25,
          "mean_useful": 7.032432432432432,
          "mean_retained": 9.972972972972974
        },
        "20": {
          "scored_queries": 925,
          "excluded_queries": 0,
          "hit": 0.9545945945945946,
          "precision": 0.6479835212489159,
          "macro_precision": 0.6484765048449259,
          "ndcg": 0.6490652861835569,
          "known_positive_recall": 0.2971249330393808,
          "recall_queries": 925,
          "useful": 11954,
          "retained": 18448,
          "excluded_positions": 52,
          "mean_useful": 12.923243243243244,
          "mean_retained": 19.943783783783783
        },
        "50": {
          "scored_queries": 925,
          "excluded_queries": 0,
          "hit": 0.9762162162162162,
          "precision": 0.424031024546656,
          "macro_precision": 0.42417201849319736,
          "ndcg": 0.6129402668287812,
          "known_positive_recall": 0.4598077844640693,
          "recall_queries": 925,
          "useful": 19572,
          "retained": 46157,
          "excluded_positions": 93,
          "mean_useful": 21.158918918918918,
          "mean_retained": 49.89945945945946
        },
        "selected": {
          "scored_queries": 925,
          "excluded_queries": 0,
          "hit": 0.9027027027027027,
          "precision": 0.8181689141234918,
          "macro_precision": 0.7380948996103794,
          "ndcg": 0.640658479488419,
          "known_positive_recall": 0.12825382711378852,
          "recall_queries": 925,
          "useful": 5764,
          "retained": 7045,
          "excluded_positions": 17,
          "mean_useful": 6.231351351351352,
          "mean_retained": 7.616216216216216
        }
      }
    }
  }
}