| { | |
| "method": "Unchanged published BF16-fidelity benchmark: 100 EN/DE text pairs, 50 code pairs, same reference and prefixes", | |
| "hq": { | |
| "model_label": "Sakura Native AutoRoundV2 HQ Q4_K PLE-Q8_0 Retrieval256", | |
| "total_items": 300, | |
| "total_encoding_time_s": 22.847290992736816, | |
| "avg_latency_ms": 76.15763664245605, | |
| "dimensions": { | |
| "768": { | |
| "combined_fidelity": { | |
| "mean_cosine": 0.9940026998519897, | |
| "median_cosine": 0.9947526454925537, | |
| "min_cosine": 0.9814159870147705, | |
| "mean_l2_drift": 0.10663197934627533, | |
| "spearman_rank_correlation": 0.9885723946118131 | |
| }, | |
| "text_queries_fidelity": { | |
| "mean_cosine": 0.9957579970359802, | |
| "median_cosine": 0.9961777925491333, | |
| "min_cosine": 0.9838103652000427, | |
| "mean_l2_drift": 0.09061824530363083, | |
| "spearman_rank_correlation": 0.9924962422922609 | |
| }, | |
| "text_docs_fidelity": { | |
| "mean_cosine": 0.9920704364776611, | |
| "median_cosine": 0.9925398826599121, | |
| "min_cosine": 0.9814159870147705, | |
| "mean_l2_drift": 0.12406869977712631, | |
| "spearman_rank_correlation": 0.9851379756327648 | |
| }, | |
| "code_queries_fidelity": { | |
| "mean_cosine": 0.996131181716919, | |
| "median_cosine": 0.9963771104812622, | |
| "min_cosine": 0.992403507232666, | |
| "mean_l2_drift": 0.08698179572820663, | |
| "spearman_rank_correlation": 0.9911986228614428 | |
| }, | |
| "code_snippets_fidelity": { | |
| "mean_cosine": 0.992228090763092, | |
| "median_cosine": 0.9927085638046265, | |
| "min_cosine": 0.9859081506729126, | |
| "mean_l2_drift": 0.12343620508909225, | |
| "spearman_rank_correlation": 0.9849673089593854 | |
| }, | |
| "text_retrieval": { | |
| "top1_agreement": 1.0, | |
| "top5_agreement": 0.8999999999999999, | |
| "recall_at_1": 0.94, | |
| "recall_at_5": 1.0, | |
| "recall_at_10": 1.0, | |
| "ndcg_at_10": 0.9766891680410028 | |
| }, | |
| "code_retrieval": { | |
| "top1_agreement": 1.0, | |
| "top5_agreement": 0.8759999999999998, | |
| "recall_at_1": 0.98, | |
| "recall_at_5": 1.0, | |
| "recall_at_10": 1.0, | |
| "ndcg_at_10": 0.9754085149830157 | |
| } | |
| }, | |
| "512": { | |
| "combined_fidelity": { | |
| "mean_cosine": 0.9941583275794983, | |
| "median_cosine": 0.9949311017990112, | |
| "min_cosine": 0.9818282723426819, | |
| "mean_l2_drift": 0.10525313019752502, | |
| "spearman_rank_correlation": 0.9886308739750488 | |
| }, | |
| "text_queries_fidelity": { | |
| "mean_cosine": 0.9958592057228088, | |
| "median_cosine": 0.9962297081947327, | |
| "min_cosine": 0.9844546914100647, | |
| "mean_l2_drift": 0.08952875435352325, | |
| "spearman_rank_correlation": 0.992443436748103 | |
| }, | |
| "text_docs_fidelity": { | |
| "mean_cosine": 0.9922857880592346, | |
| "median_cosine": 0.9928115010261536, | |
| "min_cosine": 0.9818282723426819, | |
| "mean_l2_drift": 0.12236985564231873, | |
| "spearman_rank_correlation": 0.9854122725737566 | |
| }, | |
| "code_queries_fidelity": { | |
| "mean_cosine": 0.9962114691734314, | |
| "median_cosine": 0.9965128898620605, | |
| "min_cosine": 0.9924570322036743, | |
| "mean_l2_drift": 0.08604688942432404, | |
| "spearman_rank_correlation": 0.9910154061108948 | |
| }, | |
| "code_snippets_fidelity": { | |
| "mean_cosine": 0.9924482703208923, | |
| "median_cosine": 0.992801308631897, | |
| "min_cosine": 0.9862302541732788, | |
| "mean_l2_drift": 0.12167471647262573, | |
| "spearman_rank_correlation": 0.9850584190956796 | |
| }, | |
| "text_retrieval": { | |
| "top1_agreement": 0.99, | |
| "top5_agreement": 0.9039999999999999, | |
| "recall_at_1": 0.93, | |
| "recall_at_5": 1.0, | |
| "recall_at_10": 1.0, | |
| "ndcg_at_10": 0.9754695318518756 | |
| }, | |
| "code_retrieval": { | |
| "top1_agreement": 1.0, | |
| "top5_agreement": 0.892, | |
| "recall_at_1": 0.98, | |
| "recall_at_5": 1.0, | |
| "recall_at_10": 1.0, | |
| "ndcg_at_10": 0.97317023314118 | |
| } | |
| }, | |
| "256": { | |
| "combined_fidelity": { | |
| "mean_cosine": 0.994737446308136, | |
| "median_cosine": 0.9953632354736328, | |
| "min_cosine": 0.9837870001792908, | |
| "mean_l2_drift": 0.09981194883584976, | |
| "spearman_rank_correlation": 0.9893874709986344 | |
| }, | |
| "text_queries_fidelity": { | |
| "mean_cosine": 0.9962558746337891, | |
| "median_cosine": 0.9966042041778564, | |
| "min_cosine": 0.9857850074768066, | |
| "mean_l2_drift": 0.0849779024720192, | |
| "spearman_rank_correlation": 0.9928425364313742 | |
| }, | |
| "text_docs_fidelity": { | |
| "mean_cosine": 0.9930971264839172, | |
| "median_cosine": 0.9935889840126038, | |
| "min_cosine": 0.9837870001792908, | |
| "mean_l2_drift": 0.11578014492988586, | |
| "spearman_rank_correlation": 0.9866534910386169 | |
| }, | |
| "code_queries_fidelity": { | |
| "mean_cosine": 0.9966413974761963, | |
| "median_cosine": 0.9968907833099365, | |
| "min_cosine": 0.993119478225708, | |
| "mean_l2_drift": 0.08096438646316528, | |
| "spearman_rank_correlation": 0.9914799620741137 | |
| }, | |
| "code_snippets_fidelity": { | |
| "mean_cosine": 0.9930777549743652, | |
| "median_cosine": 0.9935734272003174, | |
| "min_cosine": 0.9868944883346558, | |
| "mean_l2_drift": 0.11639125645160675, | |
| "spearman_rank_correlation": 0.9858528089777109 | |
| }, | |
| "text_retrieval": { | |
| "top1_agreement": 0.99, | |
| "top5_agreement": 0.898, | |
| "recall_at_1": 0.93, | |
| "recall_at_5": 1.0, | |
| "recall_at_10": 1.0, | |
| "ndcg_at_10": 0.9737128486737014 | |
| }, | |
| "code_retrieval": { | |
| "top1_agreement": 1.0, | |
| "top5_agreement": 0.88, | |
| "recall_at_1": 0.98, | |
| "recall_at_5": 1.0, | |
| "recall_at_10": 1.0, | |
| "ndcg_at_10": 0.9683037099581555 | |
| } | |
| }, | |
| "128": { | |
| "combined_fidelity": { | |
| "mean_cosine": 0.9962875247001648, | |
| "median_cosine": 0.9966529607772827, | |
| "min_cosine": 0.988479733467102, | |
| "mean_l2_drift": 0.08378876000642776, | |
| "spearman_rank_correlation": 0.9891410623012039 | |
| }, | |
| "text_queries_fidelity": { | |
| "mean_cosine": 0.9971998333930969, | |
| "median_cosine": 0.9975807070732117, | |
| "min_cosine": 0.988479733467102, | |
| "mean_l2_drift": 0.0732530951499939, | |
| "spearman_rank_correlation": 0.992236139997222 | |
| }, | |
| "text_docs_fidelity": { | |
| "mean_cosine": 0.9951152801513672, | |
| "median_cosine": 0.9954268932342529, | |
| "min_cosine": 0.9892120361328125, | |
| "mean_l2_drift": 0.09740488231182098, | |
| "spearman_rank_correlation": 0.9864264951778177 | |
| }, | |
| "code_queries_fidelity": { | |
| "mean_cosine": 0.9978145360946655, | |
| "median_cosine": 0.9979954957962036, | |
| "min_cosine": 0.9955280423164368, | |
| "mean_l2_drift": 0.06525526195764542, | |
| "spearman_rank_correlation": 0.9913819287262265 | |
| }, | |
| "code_snippets_fidelity": { | |
| "mean_cosine": 0.9952802062034607, | |
| "median_cosine": 0.9955841898918152, | |
| "min_cosine": 0.9906960725784302, | |
| "mean_l2_drift": 0.09616132825613022, | |
| "spearman_rank_correlation": 0.9861391747309171 | |
| }, | |
| "text_retrieval": { | |
| "top1_agreement": 1.0, | |
| "top5_agreement": 0.8739999999999999, | |
| "recall_at_1": 0.89, | |
| "recall_at_5": 1.0, | |
| "recall_at_10": 1.0, | |
| "ndcg_at_10": 0.9627996512731696 | |
| }, | |
| "code_retrieval": { | |
| "top1_agreement": 1.0, | |
| "top5_agreement": 0.848, | |
| "recall_at_1": 0.98, | |
| "recall_at_5": 1.0, | |
| "recall_at_10": 1.0, | |
| "ndcg_at_10": 0.9599789863148916 | |
| } | |
| } | |
| } | |
| }, | |
| "hq_language_views": { | |
| "model_label": "Sakura Native AutoRoundV2 HQ Q4_K PLE-Q8_0 Retrieval256", | |
| "method": "Same metrics and full document pool; queries grouped by existing lang field", | |
| "dimensions": { | |
| "768": { | |
| "en": { | |
| "pairs": 50, | |
| "fidelity": { | |
| "mean_cosine": 0.9935376644134521, | |
| "median_cosine": 0.9945746660232544, | |
| "min_cosine": 0.9814159870147705, | |
| "mean_l2_drift": 0.10984587669372559, | |
| "spearman_rank_correlation": 0.9876961077283617 | |
| }, | |
| "retrieval": { | |
| "top1_agreement": 1.0, | |
| "top5_agreement": 0.8919999999999999, | |
| "recall_at_1": 1.0, | |
| "recall_at_5": 1.0, | |
| "recall_at_10": 1.0, | |
| "ndcg_at_10": 0.9763361764358859 | |
| } | |
| }, | |
| "de": { | |
| "pairs": 50, | |
| "fidelity": { | |
| "mean_cosine": 0.9942905902862549, | |
| "median_cosine": 0.9950199127197266, | |
| "min_cosine": 0.9838103652000427, | |
| "mean_l2_drift": 0.10484106838703156, | |
| "spearman_rank_correlation": 0.9899381101966641 | |
| }, | |
| "retrieval": { | |
| "top1_agreement": 1.0, | |
| "top5_agreement": 0.9079999999999998, | |
| "recall_at_1": 0.88, | |
| "recall_at_5": 1.0, | |
| "recall_at_10": 1.0, | |
| "ndcg_at_10": 0.9770421596461197 | |
| } | |
| }, | |
| "code": { | |
| "queries_fidelity": { | |
| "mean_cosine": 0.996131181716919, | |
| "median_cosine": 0.9963771104812622, | |
| "min_cosine": 0.992403507232666, | |
| "mean_l2_drift": 0.08698179572820663, | |
| "spearman_rank_correlation": 0.9911986228614428 | |
| }, | |
| "snippets_fidelity": { | |
| "mean_cosine": 0.992228090763092, | |
| "median_cosine": 0.9927085638046265, | |
| "min_cosine": 0.9859081506729126, | |
| "mean_l2_drift": 0.12343620508909225, | |
| "spearman_rank_correlation": 0.9849673089593854 | |
| }, | |
| "retrieval": { | |
| "top1_agreement": 1.0, | |
| "top5_agreement": 0.8759999999999998, | |
| "recall_at_1": 0.98, | |
| "recall_at_5": 1.0, | |
| "recall_at_10": 1.0, | |
| "ndcg_at_10": 0.9754085149830157 | |
| }, | |
| "fidelity": { | |
| "mean_cosine": 0.9941796064376831, | |
| "median_cosine": 0.9943437576293945, | |
| "min_cosine": 0.9859081506729126, | |
| "mean_l2_drift": 0.10520900040864944, | |
| "spearman_rank_correlation": 0.9880829659104141 | |
| } | |
| } | |
| }, | |
| "512": { | |
| "en": { | |
| "pairs": 50, | |
| "fidelity": { | |
| "mean_cosine": 0.9937088489532471, | |
| "median_cosine": 0.9946292638778687, | |
| "min_cosine": 0.9818282723426819, | |
| "mean_l2_drift": 0.10839315503835678, | |
| "spearman_rank_correlation": 0.9877108709908913 | |
| }, | |
| "retrieval": { | |
| "top1_agreement": 1.0, | |
| "top5_agreement": 0.9039999999999999, | |
| "recall_at_1": 1.0, | |
| "recall_at_5": 1.0, | |
| "recall_at_10": 1.0, | |
| "ndcg_at_10": 0.9748837250602671 | |
| } | |
| }, | |
| "de": { | |
| "pairs": 50, | |
| "fidelity": { | |
| "mean_cosine": 0.9944360256195068, | |
| "median_cosine": 0.9951354265213013, | |
| "min_cosine": 0.9844546914100647, | |
| "mean_l2_drift": 0.10350543260574341, | |
| "spearman_rank_correlation": 0.9901448383309682 | |
| }, | |
| "retrieval": { | |
| "top1_agreement": 0.98, | |
| "top5_agreement": 0.9039999999999999, | |
| "recall_at_1": 0.86, | |
| "recall_at_5": 1.0, | |
| "recall_at_10": 1.0, | |
| "ndcg_at_10": 0.9760553386434845 | |
| } | |
| }, | |
| "code": { | |
| "queries_fidelity": { | |
| "mean_cosine": 0.9962114691734314, | |
| "median_cosine": 0.9965128898620605, | |
| "min_cosine": 0.9924570322036743, | |
| "mean_l2_drift": 0.08604688942432404, | |
| "spearman_rank_correlation": 0.9910154061108948 | |
| }, | |
| "snippets_fidelity": { | |
| "mean_cosine": 0.9924482703208923, | |
| "median_cosine": 0.992801308631897, | |
| "min_cosine": 0.9862302541732788, | |
| "mean_l2_drift": 0.12167471647262573, | |
| "spearman_rank_correlation": 0.9850584190956796 | |
| }, | |
| "retrieval": { | |
| "top1_agreement": 1.0, | |
| "top5_agreement": 0.892, | |
| "recall_at_1": 0.98, | |
| "recall_at_5": 1.0, | |
| "recall_at_10": 1.0, | |
| "ndcg_at_10": 0.97317023314118 | |
| }, | |
| "fidelity": { | |
| "mean_cosine": 0.9943298101425171, | |
| "median_cosine": 0.9944037199020386, | |
| "min_cosine": 0.9862302541732788, | |
| "mean_l2_drift": 0.10386079549789429, | |
| "spearman_rank_correlation": 0.9880369126032873 | |
| } | |
| } | |
| }, | |
| "256": { | |
| "en": { | |
| "pairs": 50, | |
| "fidelity": { | |
| "mean_cosine": 0.994392991065979, | |
| "median_cosine": 0.9951571226119995, | |
| "min_cosine": 0.9837870001792908, | |
| "mean_l2_drift": 0.10231548547744751, | |
| "spearman_rank_correlation": 0.9885983689456543 | |
| }, | |
| "retrieval": { | |
| "top1_agreement": 1.0, | |
| "top5_agreement": 0.892, | |
| "recall_at_1": 1.0, | |
| "recall_at_5": 1.0, | |
| "recall_at_10": 1.0, | |
| "ndcg_at_10": 0.9728269292652085 | |
| } | |
| }, | |
| "de": { | |
| "pairs": 50, | |
| "fidelity": { | |
| "mean_cosine": 0.9949600100517273, | |
| "median_cosine": 0.9954547882080078, | |
| "min_cosine": 0.9857850074768066, | |
| "mean_l2_drift": 0.09844255447387695, | |
| "spearman_rank_correlation": 0.990897658524337 | |
| }, | |
| "retrieval": { | |
| "top1_agreement": 0.98, | |
| "top5_agreement": 0.904, | |
| "recall_at_1": 0.86, | |
| "recall_at_5": 1.0, | |
| "recall_at_10": 1.0, | |
| "ndcg_at_10": 0.9745987680821948 | |
| } | |
| }, | |
| "code": { | |
| "queries_fidelity": { | |
| "mean_cosine": 0.9966413974761963, | |
| "median_cosine": 0.9968907833099365, | |
| "min_cosine": 0.993119478225708, | |
| "mean_l2_drift": 0.08096438646316528, | |
| "spearman_rank_correlation": 0.9914799620741137 | |
| }, | |
| "snippets_fidelity": { | |
| "mean_cosine": 0.9930777549743652, | |
| "median_cosine": 0.9935734272003174, | |
| "min_cosine": 0.9868944883346558, | |
| "mean_l2_drift": 0.11639125645160675, | |
| "spearman_rank_correlation": 0.9858528089777109 | |
| }, | |
| "retrieval": { | |
| "top1_agreement": 1.0, | |
| "top5_agreement": 0.88, | |
| "recall_at_1": 0.98, | |
| "recall_at_5": 1.0, | |
| "recall_at_10": 1.0, | |
| "ndcg_at_10": 0.9683037099581555 | |
| }, | |
| "fidelity": { | |
| "mean_cosine": 0.994859516620636, | |
| "median_cosine": 0.9950376749038696, | |
| "min_cosine": 0.9868944883346558, | |
| "mean_l2_drift": 0.09867781400680542, | |
| "spearman_rank_correlation": 0.9886663855259122 | |
| } | |
| } | |
| }, | |
| "128": { | |
| "en": { | |
| "pairs": 50, | |
| "fidelity": { | |
| "mean_cosine": 0.9961062669754028, | |
| "median_cosine": 0.9966549873352051, | |
| "min_cosine": 0.9892120361328125, | |
| "mean_l2_drift": 0.08542466163635254, | |
| "spearman_rank_correlation": 0.9887060374159149 | |
| }, | |
| "retrieval": { | |
| "top1_agreement": 1.0, | |
| "top5_agreement": 0.8679999999999999, | |
| "recall_at_1": 1.0, | |
| "recall_at_5": 1.0, | |
| "recall_at_10": 1.0, | |
| "ndcg_at_10": 0.9640552852487183 | |
| } | |
| }, | |
| "de": { | |
| "pairs": 50, | |
| "fidelity": { | |
| "mean_cosine": 0.9962089657783508, | |
| "median_cosine": 0.9966347217559814, | |
| "min_cosine": 0.988479733467102, | |
| "mean_l2_drift": 0.08523330837488174, | |
| "spearman_rank_correlation": 0.9899565977591248 | |
| }, | |
| "retrieval": { | |
| "top1_agreement": 1.0, | |
| "top5_agreement": 0.8799999999999999, | |
| "recall_at_1": 0.78, | |
| "recall_at_5": 1.0, | |
| "recall_at_10": 1.0, | |
| "ndcg_at_10": 0.9615440172976208 | |
| } | |
| }, | |
| "code": { | |
| "queries_fidelity": { | |
| "mean_cosine": 0.9978145360946655, | |
| "median_cosine": 0.9979954957962036, | |
| "min_cosine": 0.9955280423164368, | |
| "mean_l2_drift": 0.06525526195764542, | |
| "spearman_rank_correlation": 0.9913819287262265 | |
| }, | |
| "snippets_fidelity": { | |
| "mean_cosine": 0.9952802062034607, | |
| "median_cosine": 0.9955841898918152, | |
| "min_cosine": 0.9906960725784302, | |
| "mean_l2_drift": 0.09616132825613022, | |
| "spearman_rank_correlation": 0.9861391747309171 | |
| }, | |
| "retrieval": { | |
| "top1_agreement": 1.0, | |
| "top5_agreement": 0.848, | |
| "recall_at_1": 0.98, | |
| "recall_at_5": 1.0, | |
| "recall_at_10": 1.0, | |
| "ndcg_at_10": 0.9599789863148916 | |
| }, | |
| "fidelity": { | |
| "mean_cosine": 0.9965473413467407, | |
| "median_cosine": 0.9966987371444702, | |
| "min_cosine": 0.9906960725784302, | |
| "mean_l2_drift": 0.08070829510688782, | |
| "spearman_rank_correlation": 0.9887605517285718 | |
| } | |
| } | |
| } | |
| } | |
| }, | |
| "current_sakura_control": { | |
| "model_label": "Current Sakura control, unchanged file", | |
| "total_items": 300, | |
| "total_encoding_time_s": 6.6283392906188965, | |
| "avg_latency_ms": 22.09446430206299, | |
| "dimensions": { | |
| "768": { | |
| "combined_fidelity": { | |
| "mean_cosine": 0.986284613609314, | |
| "median_cosine": 0.9876355528831482, | |
| "min_cosine": 0.958172082901001, | |
| "mean_l2_drift": 0.16195423901081085, | |
| "spearman_rank_correlation": 0.9741920246403909 | |
| }, | |
| "text_queries_fidelity": { | |
| "mean_cosine": 0.9903071522712708, | |
| "median_cosine": 0.9909560084342957, | |
| "min_cosine": 0.979697585105896, | |
| "mean_l2_drift": 0.13742855191230774, | |
| "spearman_rank_correlation": 0.983096065478136 | |
| }, | |
| "text_docs_fidelity": { | |
| "mean_cosine": 0.9827121496200562, | |
| "median_cosine": 0.9837750196456909, | |
| "min_cosine": 0.958172082901001, | |
| "mean_l2_drift": 0.18353427946567535, | |
| "spearman_rank_correlation": 0.9680007720597849 | |
| }, | |
| "code_queries_fidelity": { | |
| "mean_cosine": 0.9903720617294312, | |
| "median_cosine": 0.9909213781356812, | |
| "min_cosine": 0.985554039478302, | |
| "mean_l2_drift": 0.13808852434158325, | |
| "spearman_rank_correlation": 0.9789676419249733 | |
| }, | |
| "code_snippets_fidelity": { | |
| "mean_cosine": 0.9812971353530884, | |
| "median_cosine": 0.9823829531669617, | |
| "min_cosine": 0.9640132188796997, | |
| "mean_l2_drift": 0.19171124696731567, | |
| "spearman_rank_correlation": 0.9639908308415314 | |
| }, | |
| "text_retrieval": { | |
| "top1_agreement": 1.0, | |
| "top5_agreement": 0.8619999999999999, | |
| "recall_at_1": 0.94, | |
| "recall_at_5": 1.0, | |
| "recall_at_10": 1.0, | |
| "ndcg_at_10": 0.956836099421053 | |
| }, | |
| "code_retrieval": { | |
| "top1_agreement": 1.0, | |
| "top5_agreement": 0.8279999999999998, | |
| "recall_at_1": 0.98, | |
| "recall_at_5": 1.0, | |
| "recall_at_10": 1.0, | |
| "ndcg_at_10": 0.9437929615877698 | |
| } | |
| }, | |
| "512": { | |
| "combined_fidelity": { | |
| "mean_cosine": 0.9865731000900269, | |
| "median_cosine": 0.9878156185150146, | |
| "min_cosine": 0.9575186967849731, | |
| "mean_l2_drift": 0.16019409894943237, | |
| "spearman_rank_correlation": 0.9743931759982976 | |
| }, | |
| "text_queries_fidelity": { | |
| "mean_cosine": 0.9904858469963074, | |
| "median_cosine": 0.9911065101623535, | |
| "min_cosine": 0.9796087741851807, | |
| "mean_l2_drift": 0.13611041009426117, | |
| "spearman_rank_correlation": 0.9830556550965539 | |
| }, | |
| "text_docs_fidelity": { | |
| "mean_cosine": 0.9830740094184875, | |
| "median_cosine": 0.9843037128448486, | |
| "min_cosine": 0.9575186967849731, | |
| "mean_l2_drift": 0.18152324855327606, | |
| "spearman_rank_correlation": 0.9685502847273751 | |
| }, | |
| "code_queries_fidelity": { | |
| "mean_cosine": 0.9905858635902405, | |
| "median_cosine": 0.9910707473754883, | |
| "min_cosine": 0.9858636856079102, | |
| "mean_l2_drift": 0.13650040328502655, | |
| "spearman_rank_correlation": 0.9788778511045352 | |
| }, | |
| "code_snippets_fidelity": { | |
| "mean_cosine": 0.9817330837249756, | |
| "median_cosine": 0.9827720522880554, | |
| "min_cosine": 0.9656853675842285, | |
| "mean_l2_drift": 0.18939685821533203, | |
| "spearman_rank_correlation": 0.9642693252373931 | |
| }, | |
| "text_retrieval": { | |
| "top1_agreement": 1.0, | |
| "top5_agreement": 0.856, | |
| "recall_at_1": 0.94, | |
| "recall_at_5": 1.0, | |
| "recall_at_10": 1.0, | |
| "ndcg_at_10": 0.9492304772444666 | |
| }, | |
| "code_retrieval": { | |
| "top1_agreement": 1.0, | |
| "top5_agreement": 0.8279999999999998, | |
| "recall_at_1": 0.98, | |
| "recall_at_5": 1.0, | |
| "recall_at_10": 1.0, | |
| "ndcg_at_10": 0.9494146991987843 | |
| } | |
| }, | |
| "256": { | |
| "combined_fidelity": { | |
| "mean_cosine": 0.9878397583961487, | |
| "median_cosine": 0.9890769124031067, | |
| "min_cosine": 0.9613763093948364, | |
| "mean_l2_drift": 0.15224973857402802, | |
| "spearman_rank_correlation": 0.9762566788481293 | |
| }, | |
| "text_queries_fidelity": { | |
| "mean_cosine": 0.9913231730461121, | |
| "median_cosine": 0.9920037984848022, | |
| "min_cosine": 0.9795629978179932, | |
| "mean_l2_drift": 0.12969373166561127, | |
| "spearman_rank_correlation": 0.984101699924683 | |
| }, | |
| "text_docs_fidelity": { | |
| "mean_cosine": 0.9847172498703003, | |
| "median_cosine": 0.9862321615219116, | |
| "min_cosine": 0.9613763093948364, | |
| "mean_l2_drift": 0.17230598628520966, | |
| "spearman_rank_correlation": 0.9714100119065244 | |
| }, | |
| "code_queries_fidelity": { | |
| "mean_cosine": 0.9916158318519592, | |
| "median_cosine": 0.991854190826416, | |
| "min_cosine": 0.9869277477264404, | |
| "mean_l2_drift": 0.128679558634758, | |
| "spearman_rank_correlation": 0.9802603520850244 | |
| }, | |
| "code_snippets_fidelity": { | |
| "mean_cosine": 0.9833423495292664, | |
| "median_cosine": 0.9837037324905396, | |
| "min_cosine": 0.9680748581886292, | |
| "mean_l2_drift": 0.18081946671009064, | |
| "spearman_rank_correlation": 0.9662562973413368 | |
| }, | |
| "text_retrieval": { | |
| "top1_agreement": 1.0, | |
| "top5_agreement": 0.8499999999999999, | |
| "recall_at_1": 0.94, | |
| "recall_at_5": 1.0, | |
| "recall_at_10": 1.0, | |
| "ndcg_at_10": 0.952191329392561 | |
| }, | |
| "code_retrieval": { | |
| "top1_agreement": 1.0, | |
| "top5_agreement": 0.8519999999999999, | |
| "recall_at_1": 0.98, | |
| "recall_at_5": 1.0, | |
| "recall_at_10": 1.0, | |
| "ndcg_at_10": 0.9479355114978034 | |
| } | |
| }, | |
| "128": { | |
| "combined_fidelity": { | |
| "mean_cosine": 0.9913265109062195, | |
| "median_cosine": 0.9921226501464844, | |
| "min_cosine": 0.9699418544769287, | |
| "mean_l2_drift": 0.12861783802509308, | |
| "spearman_rank_correlation": 0.9760275762402219 | |
| }, | |
| "text_queries_fidelity": { | |
| "mean_cosine": 0.9934825301170349, | |
| "median_cosine": 0.993942141532898, | |
| "min_cosine": 0.9848894476890564, | |
| "mean_l2_drift": 0.11236220598220825, | |
| "spearman_rank_correlation": 0.9831001868491966 | |
| }, | |
| "text_docs_fidelity": { | |
| "mean_cosine": 0.9891209602355957, | |
| "median_cosine": 0.9902327060699463, | |
| "min_cosine": 0.9699418544769287, | |
| "mean_l2_drift": 0.14522407948970795, | |
| "spearman_rank_correlation": 0.9709582457608106 | |
| }, | |
| "code_queries_fidelity": { | |
| "mean_cosine": 0.9943042993545532, | |
| "median_cosine": 0.994342565536499, | |
| "min_cosine": 0.9914700984954834, | |
| "mean_l2_drift": 0.1060643270611763, | |
| "spearman_rank_correlation": 0.9802328293138136 | |
| }, | |
| "code_snippets_fidelity": { | |
| "mean_cosine": 0.9884477257728577, | |
| "median_cosine": 0.9884743094444275, | |
| "min_cosine": 0.9777816534042358, | |
| "mean_l2_drift": 0.15047010779380798, | |
| "spearman_rank_correlation": 0.967815762907503 | |
| }, | |
| "text_retrieval": { | |
| "top1_agreement": 0.98, | |
| "top5_agreement": 0.8019999999999999, | |
| "recall_at_1": 0.87, | |
| "recall_at_5": 1.0, | |
| "recall_at_10": 1.0, | |
| "ndcg_at_10": 0.9281811877674093 | |
| }, | |
| "code_retrieval": { | |
| "top1_agreement": 0.98, | |
| "top5_agreement": 0.8000000000000002, | |
| "recall_at_1": 0.96, | |
| "recall_at_5": 1.0, | |
| "recall_at_10": 1.0, | |
| "ndcg_at_10": 0.935597356929814 | |
| } | |
| } | |
| } | |
| }, | |
| "unsloth_fresh_control": { | |
| "model_label": "Unsloth control, unchanged file", | |
| "total_items": 300, | |
| "total_encoding_time_s": 6.860053539276123, | |
| "avg_latency_ms": 22.86684513092041, | |
| "dimensions": { | |
| "768": { | |
| "combined_fidelity": { | |
| "mean_cosine": 0.9935415387153625, | |
| "median_cosine": 0.9943074584007263, | |
| "min_cosine": 0.9768133163452148, | |
| "mean_l2_drift": 0.11076759546995163, | |
| "spearman_rank_correlation": 0.9877099696258076 | |
| }, | |
| "text_queries_fidelity": { | |
| "mean_cosine": 0.9953446388244629, | |
| "median_cosine": 0.9958416819572449, | |
| "min_cosine": 0.9806791543960571, | |
| "mean_l2_drift": 0.09459321200847626, | |
| "spearman_rank_correlation": 0.9917900807128084 | |
| }, | |
| "text_docs_fidelity": { | |
| "mean_cosine": 0.9916658997535706, | |
| "median_cosine": 0.9921389818191528, | |
| "min_cosine": 0.9768133163452148, | |
| "mean_l2_drift": 0.12721379101276398, | |
| "spearman_rank_correlation": 0.9844453890770951 | |
| }, | |
| "code_queries_fidelity": { | |
| "mean_cosine": 0.9956746697425842, | |
| "median_cosine": 0.9959942102432251, | |
| "min_cosine": 0.9922338724136353, | |
| "mean_l2_drift": 0.09241673350334167, | |
| "spearman_rank_correlation": 0.9901718653695548 | |
| }, | |
| "code_snippets_fidelity": { | |
| "mean_cosine": 0.9915536642074585, | |
| "median_cosine": 0.9920182824134827, | |
| "min_cosine": 0.9823362827301025, | |
| "mean_l2_drift": 0.12857486307621002, | |
| "spearman_rank_correlation": 0.9836170128054836 | |
| }, | |
| "text_retrieval": { | |
| "top1_agreement": 1.0, | |
| "top5_agreement": 0.8960000000000001, | |
| "recall_at_1": 0.94, | |
| "recall_at_5": 1.0, | |
| "recall_at_10": 1.0, | |
| "ndcg_at_10": 0.973523320444983 | |
| }, | |
| "code_retrieval": { | |
| "top1_agreement": 1.0, | |
| "top5_agreement": 0.8839999999999999, | |
| "recall_at_1": 0.98, | |
| "recall_at_5": 1.0, | |
| "recall_at_10": 1.0, | |
| "ndcg_at_10": 0.9761568702778369 | |
| } | |
| }, | |
| "512": { | |
| "combined_fidelity": { | |
| "mean_cosine": 0.9937021732330322, | |
| "median_cosine": 0.994452953338623, | |
| "min_cosine": 0.9778607487678528, | |
| "mean_l2_drift": 0.10937415808439255, | |
| "spearman_rank_correlation": 0.9877799143799625 | |
| }, | |
| "text_queries_fidelity": { | |
| "mean_cosine": 0.9954435229301453, | |
| "median_cosine": 0.995930552482605, | |
| "min_cosine": 0.981410026550293, | |
| "mean_l2_drift": 0.09359025955200195, | |
| "spearman_rank_correlation": 0.9917398610802381 | |
| }, | |
| "text_docs_fidelity": { | |
| "mean_cosine": 0.9918571710586548, | |
| "median_cosine": 0.9923776388168335, | |
| "min_cosine": 0.9778607487678528, | |
| "mean_l2_drift": 0.12571796774864197, | |
| "spearman_rank_correlation": 0.984640227025979 | |
| }, | |
| "code_queries_fidelity": { | |
| "mean_cosine": 0.9957953095436096, | |
| "median_cosine": 0.9961422085762024, | |
| "min_cosine": 0.9922667741775513, | |
| "mean_l2_drift": 0.09108803421258926, | |
| "spearman_rank_correlation": 0.990114584934537 | |
| }, | |
| "code_snippets_fidelity": { | |
| "mean_cosine": 0.9918164014816284, | |
| "median_cosine": 0.9923561215400696, | |
| "min_cosine": 0.9825160503387451, | |
| "mean_l2_drift": 0.12654057145118713, | |
| "spearman_rank_correlation": 0.9838047251328041 | |
| }, | |
| "text_retrieval": { | |
| "top1_agreement": 0.99, | |
| "top5_agreement": 0.8959999999999998, | |
| "recall_at_1": 0.95, | |
| "recall_at_5": 1.0, | |
| "recall_at_10": 1.0, | |
| "ndcg_at_10": 0.9717382253618233 | |
| }, | |
| "code_retrieval": { | |
| "top1_agreement": 1.0, | |
| "top5_agreement": 0.9039999999999998, | |
| "recall_at_1": 0.98, | |
| "recall_at_5": 1.0, | |
| "recall_at_10": 1.0, | |
| "ndcg_at_10": 0.9768270027203616 | |
| } | |
| }, | |
| "256": { | |
| "combined_fidelity": { | |
| "mean_cosine": 0.9943554401397705, | |
| "median_cosine": 0.9950957298278809, | |
| "min_cosine": 0.9821154475212097, | |
| "mean_l2_drift": 0.10344351083040237, | |
| "spearman_rank_correlation": 0.9886171133613281 | |
| }, | |
| "text_queries_fidelity": { | |
| "mean_cosine": 0.9959253072738647, | |
| "median_cosine": 0.9963704347610474, | |
| "min_cosine": 0.9829124212265015, | |
| "mean_l2_drift": 0.08834270387887955, | |
| "spearman_rank_correlation": 0.9922134186957315 | |
| }, | |
| "text_docs_fidelity": { | |
| "mean_cosine": 0.9927664995193481, | |
| "median_cosine": 0.9931821823120117, | |
| "min_cosine": 0.9821154475212097, | |
| "mean_l2_drift": 0.11844141781330109, | |
| "spearman_rank_correlation": 0.9860210978438405 | |
| }, | |
| "code_queries_fidelity": { | |
| "mean_cosine": 0.9962533712387085, | |
| "median_cosine": 0.9965182542800903, | |
| "min_cosine": 0.9930441379547119, | |
| "mean_l2_drift": 0.08597713708877563, | |
| "spearman_rank_correlation": 0.9906471708059155 | |
| }, | |
| "code_snippets_fidelity": { | |
| "mean_cosine": 0.9924961924552917, | |
| "median_cosine": 0.9929518699645996, | |
| "min_cosine": 0.9838738441467285, | |
| "mean_l2_drift": 0.12111572176218033, | |
| "spearman_rank_correlation": 0.9845864762829086 | |
| }, | |
| "text_retrieval": { | |
| "top1_agreement": 0.99, | |
| "top5_agreement": 0.8999999999999999, | |
| "recall_at_1": 0.95, | |
| "recall_at_5": 1.0, | |
| "recall_at_10": 1.0, | |
| "ndcg_at_10": 0.9692165261715496 | |
| }, | |
| "code_retrieval": { | |
| "top1_agreement": 1.0, | |
| "top5_agreement": 0.8959999999999999, | |
| "recall_at_1": 0.98, | |
| "recall_at_5": 1.0, | |
| "recall_at_10": 1.0, | |
| "ndcg_at_10": 0.9709378677534456 | |
| } | |
| }, | |
| "128": { | |
| "combined_fidelity": { | |
| "mean_cosine": 0.9960004687309265, | |
| "median_cosine": 0.9965834617614746, | |
| "min_cosine": 0.9861778020858765, | |
| "mean_l2_drift": 0.08701489865779877, | |
| "spearman_rank_correlation": 0.9884302353382209 | |
| }, | |
| "text_queries_fidelity": { | |
| "mean_cosine": 0.9969315528869629, | |
| "median_cosine": 0.9973435401916504, | |
| "min_cosine": 0.9879273176193237, | |
| "mean_l2_drift": 0.07647903263568878, | |
| "spearman_rank_correlation": 0.991550187075518 | |
| }, | |
| "text_docs_fidelity": { | |
| "mean_cosine": 0.9948344230651855, | |
| "median_cosine": 0.9952946901321411, | |
| "min_cosine": 0.9861778020858765, | |
| "mean_l2_drift": 0.10000750422477722, | |
| "spearman_rank_correlation": 0.9857375367969347 | |
| }, | |
| "code_queries_fidelity": { | |
| "mean_cosine": 0.9975305795669556, | |
| "median_cosine": 0.9977264404296875, | |
| "min_cosine": 0.9946298599243164, | |
| "mean_l2_drift": 0.06972154229879379, | |
| "spearman_rank_correlation": 0.9906695924597985 | |
| }, | |
| "code_snippets_fidelity": { | |
| "mean_cosine": 0.9949401617050171, | |
| "median_cosine": 0.9952267408370972, | |
| "min_cosine": 0.9888275861740112, | |
| "mean_l2_drift": 0.09939484298229218, | |
| "spearman_rank_correlation": 0.9853363718246215 | |
| }, | |
| "text_retrieval": { | |
| "top1_agreement": 0.98, | |
| "top5_agreement": 0.87, | |
| "recall_at_1": 0.89, | |
| "recall_at_5": 1.0, | |
| "recall_at_10": 1.0, | |
| "ndcg_at_10": 0.9581580178688701 | |
| }, | |
| "code_retrieval": { | |
| "top1_agreement": 1.0, | |
| "top5_agreement": 0.8599999999999999, | |
| "recall_at_1": 0.98, | |
| "recall_at_5": 1.0, | |
| "recall_at_10": 1.0, | |
| "ndcg_at_10": 0.9665498988919873 | |
| } | |
| } | |
| } | |
| }, | |
| "runtime_smoke": { | |
| "server_load": "PASS", | |
| "llama_cpp_commit": "51ce9c11a6f2dfa895696c0048c4333e8953b728", | |
| "tests": [ | |
| { | |
| "text": "task: search result | query: mixed precision embeddings", | |
| "dimension": 768, | |
| "finite": true, | |
| "mrl": "PASS" | |
| }, | |
| { | |
| "text": "Ein deutscher Test f\u00fcr semantische Suche.", | |
| "dimension": 768, | |
| "finite": true, | |
| "mrl": "PASS" | |
| }, | |
| { | |
| "text": "def add(a, b): return a + b", | |
| "dimension": 768, | |
| "finite": true, | |
| "mrl": "PASS" | |
| } | |
| ], | |
| "status": "PASS" | |
| }, | |
| "ndcg_definition": "Reference-ranking graded nDCG@10; not independent human relevance judgments", | |
| "source_snapshot": "914f7f89142e33e77833254d9c9b90c3cef7303b", | |
| "runtime_flags": "--embedding --pooling mean -ngl 99 -c 2048 -b 2048 -ub 2048 --parallel 1", | |
| "published_unsloth_baseline": { | |
| "mean_cosine": 0.9934849143028259, | |
| "spearman": 0.9876057425348118, | |
| "text_top5": 0.896, | |
| "recall_at_5": 1.0 | |
| }, | |
| "bf16_reference_sha256": "f9e06dfc9a2395d7e7b948c78338e63f3b8c9f79a528ee5f8176f0d601f28a70" | |
| } |