| { | |
| "queries": 925, | |
| "models": { | |
| "upstream_gemma": { | |
| "1": { | |
| "scored_queries": 919, | |
| "excluded_queries": 6, | |
| "hit": 0.7399347116430903, | |
| "precision": 0.7399347116430903, | |
| "macro_precision": 0.7399347116430903, | |
| "ndcg": 0.6295144826156795, | |
| "known_positive_recall": 0.02009148065967933, | |
| "recall_queries": 919, | |
| "useful": 680, | |
| "retained": 919, | |
| "excluded_positions": 0, | |
| "mean_useful": 0.7399347116430903, | |
| "mean_retained": 1.0 | |
| }, | |
| "3": { | |
| "scored_queries": 925, | |
| "excluded_queries": 0, | |
| "hit": 0.8681081081081081, | |
| "precision": 0.7149566473988439, | |
| "macro_precision": 0.7153153153153153, | |
| "ndcg": 0.6120552491562434, | |
| "known_positive_recall": 0.05489711276806684, | |
| "recall_queries": 925, | |
| "useful": 1979, | |
| "retained": 2768, | |
| "excluded_positions": 7, | |
| "mean_useful": 2.1394594594594594, | |
| "mean_retained": 2.9924324324324325 | |
| }, | |
| "5": { | |
| "scored_queries": 925, | |
| "excluded_queries": 0, | |
| "hit": 0.8994594594594595, | |
| "precision": 0.7009527934170636, | |
| "macro_precision": 0.7011351351351351, | |
| "ndcg": 0.60538850988017, | |
| "known_positive_recall": 0.08526196518888282, | |
| "recall_queries": 925, | |
| "useful": 3237, | |
| "retained": 4618, | |
| "excluded_positions": 7, | |
| "mean_useful": 3.4994594594594592, | |
| "mean_retained": 4.992432432432432 | |
| }, | |
| "10": { | |
| "scored_queries": 925, | |
| "excluded_queries": 0, | |
| "hit": 0.9308108108108109, | |
| "precision": 0.6519761775852734, | |
| "macro_precision": 0.6521381381381381, | |
| "ndcg": 0.5958896253275177, | |
| "known_positive_recall": 0.15250767729827347, | |
| "recall_queries": 925, | |
| "useful": 6021, | |
| "retained": 9235, | |
| "excluded_positions": 15, | |
| "mean_useful": 6.509189189189189, | |
| "mean_retained": 9.983783783783784 | |
| }, | |
| "20": { | |
| "scored_queries": 925, | |
| "excluded_queries": 0, | |
| "hit": 0.9567567567567568, | |
| "precision": 0.5685743678596568, | |
| "macro_precision": 0.5687161650814901, | |
| "ndcg": 0.5873176954516321, | |
| "known_positive_recall": 0.2518432399920873, | |
| "recall_queries": 925, | |
| "useful": 10501, | |
| "retained": 18469, | |
| "excluded_positions": 31, | |
| "mean_useful": 11.352432432432433, | |
| "mean_retained": 19.966486486486488 | |
| }, | |
| "50": { | |
| "scored_queries": 925, | |
| "excluded_queries": 0, | |
| "hit": 0.9697297297297297, | |
| "precision": 0.35454801464376234, | |
| "macro_precision": 0.35468757089595926, | |
| "ndcg": 0.5373826502849983, | |
| "known_positive_recall": 0.36848413838619143, | |
| "recall_queries": 925, | |
| "useful": 16367, | |
| "retained": 46163, | |
| "excluded_positions": 87, | |
| "mean_useful": 17.694054054054053, | |
| "mean_retained": 49.905945945945945 | |
| } | |
| }, | |
| "upstream_ettin": { | |
| "1": { | |
| "scored_queries": 919, | |
| "excluded_queries": 6, | |
| "hit": 0.5669205658324266, | |
| "precision": 0.5669205658324266, | |
| "macro_precision": 0.5669205658324266, | |
| "ndcg": 0.4731851391263796, | |
| "known_positive_recall": 0.01443636631579192, | |
| "recall_queries": 919, | |
| "useful": 521, | |
| "retained": 919, | |
| "excluded_positions": 0, | |
| "mean_useful": 0.5669205658324266, | |
| "mean_retained": 1.0 | |
| }, | |
| "3": { | |
| "scored_queries": 925, | |
| "excluded_queries": 0, | |
| "hit": 0.7902702702702703, | |
| "precision": 0.5603603603603604, | |
| "macro_precision": 0.5603603603603603, | |
| "ndcg": 0.4727968405371722, | |
| "known_positive_recall": 0.04228504528013372, | |
| "recall_queries": 925, | |
| "useful": 1555, | |
| "retained": 2775, | |
| "excluded_positions": 0, | |
| "mean_useful": 1.681081081081081, | |
| "mean_retained": 3.0 | |
| }, | |
| "5": { | |
| "scored_queries": 925, | |
| "excluded_queries": 0, | |
| "hit": 0.8562162162162162, | |
| "precision": 0.5582954791261086, | |
| "macro_precision": 0.5582162162162162, | |
| "ndcg": 0.47778481822398317, | |
| "known_positive_recall": 0.06742273722612116, | |
| "recall_queries": 925, | |
| "useful": 2581, | |
| "retained": 4623, | |
| "excluded_positions": 2, | |
| "mean_useful": 2.79027027027027, | |
| "mean_retained": 4.997837837837838 | |
| }, | |
| "10": { | |
| "scored_queries": 925, | |
| "excluded_queries": 0, | |
| "hit": 0.9113513513513514, | |
| "precision": 0.5447462395844606, | |
| "macro_precision": 0.5447507507507507, | |
| "ndcg": 0.49027329580696355, | |
| "known_positive_recall": 0.12832495703473729, | |
| "recall_queries": 925, | |
| "useful": 5034, | |
| "retained": 9241, | |
| "excluded_positions": 9, | |
| "mean_useful": 5.4421621621621625, | |
| "mean_retained": 9.99027027027027 | |
| }, | |
| "20": { | |
| "scored_queries": 925, | |
| "excluded_queries": 0, | |
| "hit": 0.9502702702702702, | |
| "precision": 0.5220568335588633, | |
| "macro_precision": 0.5220833774951422, | |
| "ndcg": 0.5201039924104931, | |
| "known_positive_recall": 0.23627621886671868, | |
| "recall_queries": 925, | |
| "useful": 9645, | |
| "retained": 18475, | |
| "excluded_positions": 25, | |
| "mean_useful": 10.427027027027027, | |
| "mean_retained": 19.972972972972972 | |
| }, | |
| "50": { | |
| "scored_queries": 925, | |
| "excluded_queries": 0, | |
| "hit": 0.9762162162162162, | |
| "precision": 0.424031024546656, | |
| "macro_precision": 0.42417201849319736, | |
| "ndcg": 0.5598208131173884, | |
| "known_positive_recall": 0.4583210768438434, | |
| "recall_queries": 925, | |
| "useful": 19572, | |
| "retained": 46157, | |
| "excluded_positions": 93, | |
| "mean_useful": 21.158918918918918, | |
| "mean_retained": 49.89945945945946 | |
| } | |
| }, | |
| "finetuned": { | |
| "1": { | |
| "scored_queries": 919, | |
| "excluded_queries": 6, | |
| "hit": 0.7595212187159956, | |
| "precision": 0.7595212187159956, | |
| "macro_precision": 0.7595212187159956, | |
| "ndcg": 0.6526763044717343, | |
| "known_positive_recall": 0.021181970033131128, | |
| "recall_queries": 919, | |
| "useful": 698, | |
| "retained": 919, | |
| "excluded_positions": 0, | |
| "mean_useful": 0.7595212187159956, | |
| "mean_retained": 1.0 | |
| }, | |
| "3": { | |
| "scored_queries": 925, | |
| "excluded_queries": 0, | |
| "hit": 0.894054054054054, | |
| "precision": 0.7510853835021708, | |
| "macro_precision": 0.7518918918918919, | |
| "ndcg": 0.6424089559634334, | |
| "known_positive_recall": 0.058858902936843926, | |
| "recall_queries": 925, | |
| "useful": 2076, | |
| "retained": 2764, | |
| "excluded_positions": 11, | |
| "mean_useful": 2.2443243243243245, | |
| "mean_retained": 2.9881081081081082 | |
| }, | |
| "5": { | |
| "scored_queries": 925, | |
| "excluded_queries": 0, | |
| "hit": 0.9275675675675675, | |
| "precision": 0.7366710013003901, | |
| "macro_precision": 0.736954954954955, | |
| "ndcg": 0.6373095878906122, | |
| "known_positive_recall": 0.0932034345948595, | |
| "recall_queries": 925, | |
| "useful": 3399, | |
| "retained": 4614, | |
| "excluded_positions": 11, | |
| "mean_useful": 3.674594594594595, | |
| "mean_retained": 4.988108108108108 | |
| }, | |
| "10": { | |
| "scored_queries": 925, | |
| "excluded_queries": 0, | |
| "hit": 0.9427027027027027, | |
| "precision": 0.7049002601908065, | |
| "macro_precision": 0.7052921492921493, | |
| "ndcg": 0.6377369246501603, | |
| "known_positive_recall": 0.16987439814587393, | |
| "recall_queries": 925, | |
| "useful": 6502, | |
| "retained": 9224, | |
| "excluded_positions": 26, | |
| "mean_useful": 7.029189189189189, | |
| "mean_retained": 9.971891891891891 | |
| }, | |
| "20": { | |
| "scored_queries": 925, | |
| "excluded_queries": 0, | |
| "hit": 0.9545945945945946, | |
| "precision": 0.6479835212489159, | |
| "macro_precision": 0.6484765048449259, | |
| "ndcg": 0.6488792776497584, | |
| "known_positive_recall": 0.2962743880804353, | |
| "recall_queries": 925, | |
| "useful": 11954, | |
| "retained": 18448, | |
| "excluded_positions": 52, | |
| "mean_useful": 12.923243243243244, | |
| "mean_retained": 19.943783783783783 | |
| }, | |
| "50": { | |
| "scored_queries": 925, | |
| "excluded_queries": 0, | |
| "hit": 0.9762162162162162, | |
| "precision": 0.424031024546656, | |
| "macro_precision": 0.42417201849319736, | |
| "ndcg": 0.612510329000538, | |
| "known_positive_recall": 0.4583210768438434, | |
| "recall_queries": 925, | |
| "useful": 19572, | |
| "retained": 46157, | |
| "excluded_positions": 93, | |
| "mean_useful": 21.158918918918918, | |
| "mean_retained": 49.89945945945946 | |
| } | |
| } | |
| }, | |
| "corpus_passages": 82719, | |
| "reference_pairs": 122783 | |
| } | |