diff --git "a/scores.json" "b/scores.json" --- "a/scores.json" +++ "b/scores.json" @@ -4,27 +4,6 @@ "revision_semantics": "self denotes the Hugging Face snapshot containing this file; the native artifact fingerprint binds its evaluated model files.", "score_scale": "0-1; multiply by 100 for percentages", "matrix": { - "embedding": { - "model": "llm-semantic-router/Vela-1.0-Encoder-307M-Embedding", - "revision": "972c180aecd2aa3fca97159098ab00d25d53fffb", - "artifact_fingerprint_sha256": "2b4747393b1e3e477fb1391a5d6817917362ab68fc5ae2436c769a7357a541ce", - "metrics": { - "text.banking77.accuracy": { - "numerator": 2464, - "denominator": 3080, - "value": 0.8 - }, - "text.massive-en.accuracy": { - "numerator": 2248, - "denominator": 2972, - "value": 0.756393001345895 - } - }, - "not_applicable": [ - "image", - "audio" - ] - }, "small": { "model": "llm-semantic-router/multi-modal-embed-small", "revision": "fdf8e01b7b0f3a69ac1ac8e2a64dcb1ede177ba4", @@ -102,159 +81,83 @@ } } }, - "large": { - "model": "llm-semantic-router/multi-modal-embed-large", - "revision": "e21cde3ccc414c56f504b322662f42c603a939ee", - "artifact_fingerprint_sha256": "8439b0c46c36d13a6371081cf7e40cf2c1cc7df68a2bf69e41d514452f65d549", - "metrics": { - "text.banking77.accuracy": { - "numerator": 2334, - "denominator": 3080, - "value": 0.7577922077922078 - }, - "text.massive-en.accuracy": { - "numerator": 2149, - "denominator": 2972, - "value": 0.7230820995962315 - }, - "image.image_to_text.recall@1": { - "numerator": 350, - "denominator": 823, - "value": 0.425273390036452 - }, - "image.image_to_text.recall@5": { - "numerator": 619, - "denominator": 823, - "value": 0.7521263669501823 - }, - "image.image_to_text.recall@10": { - "numerator": 721, - "denominator": 823, - "value": 0.8760631834750912 - }, - "image.text_to_image.recall@1": { - "numerator": 1442, - "denominator": 4115, - "value": 0.35042527339003643 - }, - "image.text_to_image.recall@5": { - "numerator": 2884, - "denominator": 4115, - "value": 0.7008505467800729 - }, - "image.text_to_image.recall@10": { - "numerator": 3453, - "denominator": 4115, - "value": 0.8391251518833536 - }, - "audio.audio_to_text.recall@1": { - "numerator": 1488, - "denominator": 2611, - "value": 0.5698965913443125 - }, - "audio.audio_to_text.recall@5": { - "numerator": 2137, - "denominator": 2611, - "value": 0.8184603600153199 - }, - "audio.audio_to_text.recall@10": { - "numerator": 2296, - "denominator": 2611, - "value": 0.8793565683646113 - }, - "audio.text_to_audio.recall@1": { - "numerator": 2051, - "denominator": 2610, - "value": 0.785823754789272 - }, - "audio.text_to_audio.recall@5": { - "numerator": 2454, - "denominator": 2610, - "value": 0.9402298850574713 - }, - "audio.text_to_audio.recall@10": { - "numerator": 2532, - "denominator": 2610, - "value": 0.9701149425287356 - } - } - }, "candidate": { "model": "llm-semantic-router/Vela-1.0-Omni-Nano", "revision": "self", - "artifact_fingerprint_sha256": "fb9cd39c9980a085b75456c715c2295b997784fe733314be68c13635f6384edf", + "artifact_fingerprint_sha256": "96d76f97478e114b816df3ab1b3c6947e4c5ef0babfb40ab865d7e3c838d34c4", "metrics": { "audio.audio_to_text.recall@1": { "denominator": 2611, - "numerator": 235, - "value": 0.09000382995021065 + "numerator": 448, + "value": 0.17158176943699732 }, "audio.audio_to_text.recall@10": { "denominator": 2611, - "numerator": 822, - "value": 0.31482190731520493 + "numerator": 1227, + "value": 0.469934890846419 }, "audio.audio_to_text.recall@5": { "denominator": 2611, - "numerator": 582, - "value": 0.22290310225967064 + "numerator": 957, + "value": 0.36652623515894295 }, "audio.text_to_audio.recall@1": { "denominator": 2610, - "numerator": 366, - "value": 0.14022988505747128 + "numerator": 664, + "value": 0.25440613026819925 }, "audio.text_to_audio.recall@10": { "denominator": 2610, - "numerator": 1055, - "value": 0.4042145593869732 + "numerator": 1459, + "value": 0.5590038314176246 }, "audio.text_to_audio.recall@5": { "denominator": 2610, - "numerator": 827, - "value": 0.3168582375478927 + "numerator": 1204, + "value": 0.46130268199233715 }, "image.image_to_text.recall@1": { "denominator": 823, - "numerator": 540, - "value": 0.6561360874848117 + "numerator": 501, + "value": 0.6087484811664642 }, "image.image_to_text.recall@10": { "denominator": 823, - "numerator": 791, - "value": 0.9611178614823815 + "numerator": 781, + "value": 0.9489671931956257 }, "image.image_to_text.recall@5": { "denominator": 823, - "numerator": 731, - "value": 0.8882138517618469 + "numerator": 717, + "value": 0.8712029161603888 }, "image.text_to_image.recall@1": { "denominator": 4115, - "numerator": 2184, - "value": 0.5307411907654921 + "numerator": 2297, + "value": 0.5582017010935602 }, "image.text_to_image.recall@10": { "denominator": 4115, - "numerator": 3847, - "value": 0.934872417982989 + "numerator": 3879, + "value": 0.9426488456865127 }, "image.text_to_image.recall@5": { "denominator": 4115, - "numerator": 3465, - "value": 0.8420413122721749 + "numerator": 3507, + "value": 0.8522478736330498 }, "text.banking77.accuracy": { "denominator": 3080, - "numerator": 2366, - "value": 0.7681818181818182 + "numerator": 2710, + "value": 0.8798701298701299 }, "text.massive-en.accuracy": { "denominator": 2972, - "numerator": 2061, - "value": 0.6934724091520862 + "numerator": 2436, + "value": 0.819650067294751 } - } + }, + "parameters": 144554112 } }, "datasets": { @@ -312,102 +215,103 @@ "similarity": "FP32 cosine of L2-normalized embeddings", "ties": "lexicographic candidate ID", "retrieval": "complete candidate pools; all annotated positives", - "classification": "nearest normalized mean class prototype; Vela Omni uses supervised training-split intent-label adaptation; comparison models evaluated as published", + "classification": "Nearest normalized mean class prototype from labeled training examples; the original small comparator has task-adapted history, while current GIST is frozen.", "aggregation": "Each family is the mean of its two primary metrics; utility is the mean of all six primaries. Retrieval primaries are R@1; R@5 and R@10 are reported separately.", - "evaluation_scope": "fixed test-split evaluation pools reused across releases" + "evaluation_scope": "fixed test-split evaluation pools reused across releases", + "text_training_scope": "Current GIST text tower is frozen; supervised training prototypes are recomputed under the fixed common protocol. The earlier small comparator has task-adapted history. This is not the official MTEB classification protocol." }, "paired_uncertainty": { "original": { "intervals": { "audio.audio_to_text.recall@1": { - "lower": 0.0385732787440121, - "point_delta": 0.04787437763309077, - "upper": 0.05823913815293125 + "lower": 0.11352472177122043, + "point_delta": 0.12945231711987745, + "upper": 0.14665221762155492 }, "audio.audio_to_text.recall@10": { - "lower": 0.10851901960700777, - "point_delta": 0.124473381846036, - "upper": 0.1413847124579227 + "lower": 0.25338950715018704, + "point_delta": 0.2795863653772501, + "upper": 0.30833452722063037 }, "audio.audio_to_text.recall@5": { - "lower": 0.08853185417610582, - "point_delta": 0.10302566066641133, - "upper": 0.11698281043825656 + "lower": 0.22198472459131105, + "point_delta": 0.24664879356568364, + "upper": 0.2736971153456769 }, "audio.text_to_audio.recall@1": { - "lower": 0.03460956365067202, - "point_delta": 0.044444444444444446, - "upper": 0.054626828684714246 + "lower": 0.14202610349345837, + "point_delta": 0.15862068965517243, + "upper": 0.17613425616168824 }, "audio.text_to_audio.recall@10": { - "lower": 0.08066331826013635, - "point_delta": 0.09731800766283524, - "upper": 0.11225668523126753 + "lower": 0.23515433278434666, + "point_delta": 0.25210727969348656, + "upper": 0.2707672328928673 }, "audio.text_to_audio.recall@5": { - "lower": 0.07566159737268849, - "point_delta": 0.09157088122605364, - "upper": 0.10606682928157289 + "lower": 0.21796104940042046, + "point_delta": 0.23601532567049807, + "upper": 0.2547919081648684 }, "family.audio": { - "lower": 0.03960904589415151, - "point_delta": 0.04615941103876761, - "upper": 0.05346161856895934 + "lower": 0.1322499685832721, + "point_delta": 0.14403650338752494, + "upper": 0.15644009735751394 }, "family.image": { - "lower": 0.21105710814094775, - "point_delta": 0.23839611178614822, - "upper": 0.26537059538274604 + "lower": 0.20084750911300123, + "point_delta": 0.2284325637910085, + "upper": 0.25637910085054677 }, "family.text": { - "lower": 0.04221878867362225, - "point_delta": 0.04897244411039835, - "upper": 0.05518266838476903 + "lower": 0.15747009669727144, + "point_delta": 0.16790542902588662, + "upper": 0.17869785413173905 }, "image.image_to_text.recall@1": { - "lower": 0.2065613608748481, - "point_delta": 0.24787363304981774, - "upper": 0.28797083839611176 + "lower": 0.1591737545565006, + "point_delta": 0.20048602673147023, + "upper": 0.24301336573511542 }, "image.image_to_text.recall@10": { - "lower": 0.14702308626974483, - "point_delta": 0.17618469015795868, - "upper": 0.20534629404617255 + "lower": 0.1336573511543135, + "point_delta": 0.16403402187120292, + "upper": 0.19441069258809235 }, "image.image_to_text.recall@5": { - "lower": 0.181044957472661, - "point_delta": 0.21628189550425272, - "upper": 0.25273390036452004 + "lower": 0.16160388821385177, + "point_delta": 0.19927095990279464, + "upper": 0.23572296476306198 }, "image.text_to_image.recall@1": { - "lower": 0.20267314702308628, - "point_delta": 0.22891859052247873, - "upper": 0.257606318347509 + "lower": 0.23013365735115432, + "point_delta": 0.25637910085054677, + "upper": 0.2831166464155528 }, "image.text_to_image.recall@10": { - "lower": 0.16889428918590524, - "point_delta": 0.1919805589307412, - "upper": 0.21482381530984204 + "lower": 0.17618469015795868, + "point_delta": 0.1997569866342649, + "upper": 0.22235722964763063 }, "image.text_to_image.recall@5": { - "lower": 0.221865127582017, - "point_delta": 0.24787363304981774, - "upper": 0.27314702308626976 + "lower": 0.23207776427703525, + "point_delta": 0.2580801944106926, + "upper": 0.28360267314702303 }, "text.banking77.accuracy": { - "lower": 0.05357142857142857, - "point_delta": 0.06396103896103895, - "upper": 0.0737012987012987 + "lower": 0.16066212268743915, + "point_delta": 0.17564935064935064, + "upper": 0.19058596303826084 }, "text.massive-en.accuracy": { - "lower": 0.0258750247573777, - "point_delta": 0.03398384925975774, - "upper": 0.042381789402061346 + "lower": 0.14554621848739496, + "point_delta": 0.1601615074024226, + "upper": 0.17569841804106362 }, "utility": { - "lower": 0.10172624493008055, - "point_delta": 0.11117598897843806, - "upper": 0.12072061135783345 + "lower": 0.16995251624973068, + "point_delta": 0.18012483206814003, + "upper": 0.1910563694034248 } }, "method": "paired stratified group percentile; PCG64; fixed candidate pools", @@ -415,7 +319,7 @@ "seed": 20260918, "comparison": { "candidate": "llm-semantic-router/Vela-1.0-Omni-Nano", - "candidate_artifact_fingerprint_sha256": "fb9cd39c9980a085b75456c715c2295b997784fe733314be68c13635f6384edf", + "candidate_artifact_fingerprint_sha256": "96d76f97478e114b816df3ab1b3c6947e4c5ef0babfb40ab865d7e3c838d34c4", "baseline": "llm-semantic-router/multi-modal-embed-small", "baseline_revision": "fdf8e01b7b0f3a69ac1ac8e2a64dcb1ede177ba4", "baseline_artifact_fingerprint_sha256": "684fe0c48a4ddd3944167f6e5dc75e97ada24e5b120bd9a2921f00dc03112eeb" @@ -424,94 +328,94 @@ "previous": { "intervals": { "audio.audio_to_text.recall@1": { - "lower": 0.0, - "point_delta": 0.0, - "upper": 0.0 + "lower": 0.06612466882994839, + "point_delta": 0.08157793948678667, + "upper": 0.0967748551165683 }, "audio.audio_to_text.recall@10": { - "lower": 0.0, - "point_delta": 0.0, - "upper": 0.0 + "lower": 0.13256556125090607, + "point_delta": 0.15511298353121408, + "upper": 0.18059527375151307 }, "audio.audio_to_text.recall@5": { - "lower": 0.0, - "point_delta": 0.0, - "upper": 0.0 + "lower": 0.12086266595311207, + "point_delta": 0.1436231328992723, + "upper": 0.16909569992403942 }, "audio.text_to_audio.recall@1": { - "lower": 0.0, - "point_delta": 0.0, - "upper": 0.0 + "lower": 0.09860745201354912, + "point_delta": 0.11417624521072797, + "upper": 0.13014135201210755 }, "audio.text_to_audio.recall@10": { - "lower": 0.0, - "point_delta": 0.0, - "upper": 0.0 + "lower": 0.13604064880650404, + "point_delta": 0.15478927203065135, + "upper": 0.17574363317277317 }, "audio.text_to_audio.recall@5": { - "lower": 0.0, - "point_delta": 0.0, - "upper": 0.0 + "lower": 0.1272585913379362, + "point_delta": 0.14444444444444443, + "upper": 0.16333503134796237 }, "family.audio": { - "lower": 0.0, - "point_delta": 0.0, - "upper": 0.0 + "lower": 0.08555566687871614, + "point_delta": 0.09787709234875733, + "upper": 0.11042213832911257 }, "family.image": { - "lower": 0.14191373025516402, - "point_delta": 0.16780072904009719, - "upper": 0.19465370595382747 + "lower": -0.028797083839611177, + "point_delta": -0.00996354799513973, + "upper": 0.008630012150668272 }, "family.text": { - "lower": 0.0, - "point_delta": 0.0, - "upper": 0.0 + "lower": 0.10934917350884273, + "point_delta": 0.11893298491548829, + "upper": 0.12872853420036084 }, "image.image_to_text.recall@1": { - "lower": 0.1421628189550425, - "point_delta": 0.18347509113001215, - "upper": 0.22235722964763063 + "lower": -0.08140947752126367, + "point_delta": -0.04738760631834751, + "upper": -0.012150668286755772 }, "image.image_to_text.recall@10": { - "lower": 0.07776427703523693, - "point_delta": 0.10206561360874848, - "upper": 0.12636695018226002 + "lower": -0.027946537059538274, + "point_delta": -0.012150668286755772, + "upper": 0.002430133657351154 }, "image.image_to_text.recall@5": { - "lower": 0.10692588092345079, - "point_delta": 0.13973268529769137, - "upper": 0.17253948967193194 + "lower": -0.040097205346294046, + "point_delta": -0.01701093560145808, + "upper": 0.004860267314702308 }, "image.text_to_image.recall@1": { - "lower": 0.1292831105710814, - "point_delta": 0.15212636695018225, - "upper": 0.17619076549210202 + "lower": 0.01117861482381531, + "point_delta": 0.027460510328068045, + "upper": 0.042770352369380316 }, "image.text_to_image.recall@10": { - "lower": 0.0865127582017011, - "point_delta": 0.10376670716889429, - "upper": 0.12102673147023083 + "lower": 0.001215066828675577, + "point_delta": 0.007776427703523694, + "upper": 0.014580801944106925 }, "image.text_to_image.recall@5": { - "lower": 0.11713244228432564, - "point_delta": 0.13778857837181044, - "upper": 0.15868772782503038 + "lower": -0.001701093560145808, + "point_delta": 0.010206561360874849, + "upper": 0.02187120291616039 }, "text.banking77.accuracy": { - "lower": 0.0, - "point_delta": 0.0, - "upper": 0.0 + "lower": 0.0990581357583631, + "point_delta": 0.11168831168831168, + "upper": 0.12439713161970098 }, "text.massive-en.accuracy": { - "lower": 0.0, - "point_delta": 0.0, - "upper": 0.0 + "lower": 0.1117468865701784, + "point_delta": 0.12617765814266488, + "upper": 0.14069455352356733 }, "utility": { - "lower": 0.04730457675172134, - "point_delta": 0.055933576346699064, - "upper": 0.06488456865127583 + "lower": 0.060760442670994376, + "point_delta": 0.06894884308970196, + "upper": 0.07704865461098383 } }, "method": "paired stratified group percentile; PCG64; fixed candidate pools", @@ -519,70 +423,44 @@ "seed": 20260918, "comparison": { "candidate": "llm-semantic-router/Vela-1.0-Omni-Nano", - "candidate_artifact_fingerprint_sha256": "fb9cd39c9980a085b75456c715c2295b997784fe733314be68c13635f6384edf", + "candidate_artifact_fingerprint_sha256": "96d76f97478e114b816df3ab1b3c6947e4c5ef0babfb40ab865d7e3c838d34c4", "baseline": "llm-semantic-router/Vela-1.0-Omni-Nano", - "baseline_revision": "70c3412bb0ddfb19386d673d4a102c753a574f1b", - "baseline_artifact_fingerprint_sha256": "b596f1dbab89369555e0a72a0d6854d780e305686750949b308c8f167993bdeb" + "baseline_revision": "3a1efc48995e2b71622d65c6cf71cfc00bbf3aa3", + "baseline_artifact_fingerprint_sha256": "fb9cd39c9980a085b75456c715c2295b997784fe733314be68c13635f6384edf" } } }, "training_scope": { - "text": "Banking77 and MASSIVE English training examples and intent labels; task-adapted classification, not zero-shot transfer or standard MTEB classification", - "image": "Image-only alignment using all 16,509 eligible COCO training images, 81,998 canonical training-caption texts and all 82,522 positive image-caption associations; the vision tower is frozen and the image affine projection is optimized. Evaluation examples are excluded.", - "audio": "LibriSpeech train-clean-100 speech-transcript subset", - "inherited_history": { - "text": "Banking77 and MASSIVE English training examples and intent labels; task-adapted classification, not zero-shot transfer or standard MTEB classification", - "audio": "LibriSpeech train-clean-100 speech-transcript subset", - "previous_image_alignment": "COCO CC-BY2 training image-caption subset", - "source_revision": "70c3412bb0ddfb19386d673d4a102c753a574f1b", - "source_native_artifact_sha256": "b596f1dbab89369555e0a72a0d6854d780e305686750949b308c8f167993bdeb", - "text_and_audio_parameters_unchanged_in_this_update": true - } + "text": "The complete text tower is the frozen official avsolatorio/GIST-small-Embedding-v0 checkpoint at 75e62fd210b9fde790430e0b2f040b0b00a021b1. Its model card discloses MTEB classification training data; no claim of independent pretraining data is made.", + "image": "Frozen current vision encoder and initial image head; an orthogonal shared-space rotation is trained on 16,509 COCO training images, 81,998 unique caption texts and 82,522 complete positive associations, then folded into the image affine head.", + "audio": "Frozen current speech encoder and initial audio head; a separate orthogonal shared-space rotation is trained on 28,535 LibriSpeech train-clean-100 clips and all positive transcript associations, then folded into the audio affine head.", + "procedure": "One fixed 300-step run; only the final artifact is reported. Encoders remain frozen. The two rotations preserve within-modality inner products, up to floating-point rounding.", + "evaluation_scope": "Common text classification uses labeled training prototypes with the earlier task-adapted comparator. It is separate from the official frozen-encoder MTEB classification protocols." }, "descriptive_metrics": { "definition": "Macro-F1 is the unweighted mean of per-class F1 across all fixed prototype classes; zero_division=0.", "scope": "Same models, examples, class prototypes and predictions as the accuracy matrix.", "metrics": { "text.banking77.macro_f1": { - "embedding": { - "value": 0.799544379259159, - "classes": 77, - "queries": 3080 - }, "small": { "value": 0.7020781051899517, "classes": 77, "queries": 3080 }, - "large": { - "value": 0.758420975737861, - "classes": 77, - "queries": 3080 - }, "candidate": { - "value": 0.7658421887247635, + "value": 0.8791681858119436, "classes": 77, "queries": 3080 } }, "text.massive-en.macro_f1": { - "embedding": { - "value": 0.7289382215405605, - "classes": 60, - "queries": 2972 - }, "small": { "value": 0.6171775818092488, "classes": 60, "queries": 2972 }, - "large": { - "value": 0.6819549711767595, - "classes": 60, - "queries": 2972 - }, "candidate": { - "value": 0.6524376423162815, + "value": 0.7923223418549947, "classes": 60, "queries": 2972 } @@ -597,23 +475,20 @@ "nano": { "repo_id": "llm-semantic-router/Vela-1.0-Omni-Nano", "revision": "self", - "native_artifact_sha256": "fb9cd39c9980a085b75456c715c2295b997784fe733314be68c13635f6384edf", - "parameters": 133956480, + "native_artifact_sha256": "96d76f97478e114b816df3ab1b3c6947e4c5ef0babfb40ab865d7e3c838d34c4", + "parameters": 144554112, "embedding_dimensions": 384, "max_text_tokens": 512 }, "mini": { "repo_id": "llm-semantic-router/Vela-1.0-Omni-Mini", - "revision": "d6db3d0fc206089cf8bc0a152a811ebb702adbe9", - "native_artifact_sha256": "035f6084c6c984f465e4f91c50caf8f13790177d4c0c29e4ce5d6c0c233fd1a0", - "parameters": 1044152640, + "revision": "self", + "native_artifact_sha256": "d93ebb5f2f4951d52423ae13da71b037ce44d02957671e1e71bfe25623f3cff2", + "evaluated_native_artifact_sha256": "cf5f45cf6e26cd225e0fc0c1ece629b13b8bd0dfe57dcedef739ca8f529f80ab", + "parameters": 1332891200, "embedding_dimensions": 768, - "max_text_tokens": 32768 - }, - "embedding_307m": { - "repo_id": "llm-semantic-router/Vela-1.0-Encoder-307M-Embedding", - "revision": "972c180aecd2aa3fca97159098ab00d25d53fffb", - "evaluation_max_text_tokens": 128 + "max_text_tokens": 32768, + "applicability": "NOTICE-only update; computation identical to evaluated native artifact." } }, "protocol": { @@ -649,21 +524,20 @@ "evaluation_pairs": 9927 }, "scores": { - "nano": 0.7169618925026362, - "mini": 0.7601793833329842, - "embedding_307m": 0.797141583022696 + "nano": 0.8053174565838819, + "mini": 0.8146451087159461 }, "metrics": { "nano": { - "pearson": 0.8194169233518556, - "spearman": 0.7169617707661863, - "cosine_pearson": 0.819416922085493, - "cosine_spearman": 0.7169618925026362, - "manhattan_pearson": 0.7935695063285152, - "manhattan_spearman": 0.7160084195066224, - "euclidean_pearson": 0.7944328514895539, - "euclidean_spearman": 0.7169617703551949, - "main_score": 0.7169618925026362, + "pearson": 0.8603345909155973, + "spearman": 0.8053174398238511, + "cosine_pearson": 0.8603345867277692, + "cosine_spearman": 0.8053174565838819, + "manhattan_pearson": 0.8332190477069294, + "manhattan_spearman": 0.8044409124391882, + "euclidean_pearson": 0.834049309726515, + "euclidean_spearman": 0.8053172111347874, + "main_score": 0.8053174565838819, "hf_subset": "default", "languages": [ "eng-Latn" @@ -671,36 +545,20 @@ "mteb_version": "2.21.0" }, "mini": { - "pearson": 0.8222796468165929, - "spearman": 0.7601794770327241, - "cosine_pearson": 0.822279648868016, - "cosine_spearman": 0.7601793833329842, - "manhattan_pearson": 0.8156838993848485, - "manhattan_spearman": 0.7596954927006786, - "euclidean_pearson": 0.8157312727340976, - "euclidean_spearman": 0.7601793999456663, - "main_score": 0.7601793833329842, - "hf_subset": "default", - "languages": [ - "eng-Latn" - ], - "mteb_version": "2.21.0" - }, - "embedding_307m": { - "pearson": 0.8494850160221006, - "spearman": 0.7971411972793244, - "cosine_pearson": 0.8494850171658237, - "cosine_spearman": 0.797141583022696, - "manhattan_pearson": 0.8234333216483684, - "manhattan_spearman": 0.798087517353178, - "euclidean_pearson": 0.822143223218851, - "euclidean_spearman": 0.7971415143850773, - "main_score": 0.797141583022696, + "cosine_pearson": 0.869527246505987, + "cosine_spearman": 0.8146451087159461, + "euclidean_pearson": 0.8491906422857852, + "euclidean_spearman": 0.8146451562244098, "hf_subset": "default", "languages": [ "eng-Latn" ], - "mteb_version": "2.21.0" + "main_score": 0.8146451087159461, + "manhattan_pearson": 0.8477780403118136, + "manhattan_spearman": 0.8130049916492924, + "mteb_version": "2.21.0", + "pearson": 0.8695272491639148, + "spearman": 0.8146454047686934 } } }, @@ -721,21 +579,20 @@ "evaluation_pairs": 1379 }, "scores": { - "nano": 0.7792400226361769, - "mini": 0.752513369430476, - "embedding_307m": 0.8132364632269824 + "nano": 0.870781967302773, + "mini": 0.8478075996581739 }, "metrics": { "nano": { - "pearson": 0.7717062133633645, - "spearman": 0.7792400217447207, - "cosine_pearson": 0.7717062156025293, - "cosine_spearman": 0.7792400226361769, - "manhattan_pearson": 0.7928914389432686, - "manhattan_spearman": 0.7777756255094064, - "euclidean_pearson": 0.7949414830254274, - "euclidean_spearman": 0.779239966513648, - "main_score": 0.7792400226361769, + "pearson": 0.8541596951761798, + "spearman": 0.8707819111802441, + "cosine_pearson": 0.854159691434247, + "cosine_spearman": 0.870781967302773, + "manhattan_pearson": 0.867700912246326, + "manhattan_spearman": 0.8690795304788012, + "euclidean_pearson": 0.8699231026532712, + "euclidean_spearman": 0.8707819663065923, + "main_score": 0.870781967302773, "hf_subset": "default", "languages": [ "eng-Latn" @@ -743,36 +600,20 @@ "mteb_version": "2.21.0" }, "mini": { - "pearson": 0.7489705566859157, - "spearman": 0.752513369430476, - "cosine_pearson": 0.748970546211057, - "cosine_spearman": 0.752513369430476, - "manhattan_pearson": 0.7714427226961128, - "manhattan_spearman": 0.7504103299982545, - "euclidean_pearson": 0.7731599391587733, - "euclidean_spearman": 0.752513369430476, - "main_score": 0.752513369430476, + "cosine_pearson": 0.8430290028334904, + "cosine_spearman": 0.8478075996581739, + "euclidean_pearson": 0.8485456920737019, + "euclidean_spearman": 0.8478075996581739, "hf_subset": "default", "languages": [ "eng-Latn" ], - "mteb_version": "2.21.0" - }, - "embedding_307m": { - "pearson": 0.8189125304331548, - "spearman": 0.8132383072529291, - "cosine_pearson": 0.8189125359017332, - "cosine_spearman": 0.8132364632269824, - "manhattan_pearson": 0.8232193272020889, - "manhattan_spearman": 0.8115378530431893, - "euclidean_pearson": 0.8249272816677745, - "euclidean_spearman": 0.8132371300401904, - "main_score": 0.8132364632269824, - "hf_subset": "default", - "languages": [ - "eng-Latn" - ], - "mteb_version": "2.21.0" + "main_score": 0.8478075996581739, + "manhattan_pearson": 0.8465050093624056, + "manhattan_spearman": 0.8462432939446499, + "mteb_version": "2.21.0", + "pearson": 0.843029007299188, + "spearman": 0.8478075996581739 } } }, @@ -794,160 +635,160 @@ "documents": 8674 }, "scores": { - "nano": 0.37514, - "mini": 0.36413 + "nano": 0.59246, + "mini": 0.68209 }, "metrics": { "nano": { - "ndcg_at_1": 0.17425, - "ndcg_at_3": 0.27012, - "ndcg_at_5": 0.31718, - "ndcg_at_10": 0.37514, - "ndcg_at_20": 0.40855, - "ndcg_at_100": 0.43759, - "ndcg_at_1000": 0.44701, - "map_at_1": 0.17425, - "map_at_3": 0.24561, - "map_at_5": 0.2715, - "map_at_10": 0.29556, - "map_at_20": 0.30489, - "map_at_100": 0.3093, - "map_at_1000": 0.30967, - "recall_at_1": 0.17425, - "recall_at_3": 0.34139, - "recall_at_5": 0.45661, - "recall_at_10": 0.63514, - "recall_at_20": 0.766, - "recall_at_100": 0.91679, - "recall_at_1000": 0.99004, - "accuracy": 0.17425, - "precision_at_1": 0.17425, - "precision_at_3": 0.1138, - "precision_at_5": 0.09132, - "precision_at_10": 0.06351, - "precision_at_20": 0.0383, - "precision_at_100": 0.00917, - "precision_at_1000": 0.00099, - "mrr_at_1": 0.17425320056899005, - "mrr_at_3": 0.24561403508771928, - "mrr_at_5": 0.2715030820293978, - "mrr_at_10": 0.2955561764772291, - "mrr_at_20": 0.30489450370601584, - "mrr_at_100": 0.3092982324964941, - "mrr_at_1000": 0.30966870165383564, - "nauc_ndcg_at_1_max": -0.012854409302840068, - "nauc_ndcg_at_1_std": 0.08676381700185752, - "nauc_ndcg_at_1_diff1": 0.11334791281377288, - "nauc_ndcg_at_3_max": 0.0007934267314863777, - "nauc_ndcg_at_3_std": 0.05484641820257593, - "nauc_ndcg_at_3_diff1": 0.07348303062976835, - "nauc_ndcg_at_5_max": 0.020607224558072373, - "nauc_ndcg_at_5_std": 0.0741284488966682, - "nauc_ndcg_at_5_diff1": 0.08730888772052957, - "nauc_ndcg_at_10_max": 0.035048132350077, - "nauc_ndcg_at_10_std": 0.10523053259120095, - "nauc_ndcg_at_10_diff1": 0.08020955970997308, - "nauc_ndcg_at_20_max": 0.04122292301118309, - "nauc_ndcg_at_20_std": 0.12537385022454034, - "nauc_ndcg_at_20_diff1": 0.0842700247341082, - "nauc_ndcg_at_100_max": 0.04333008607419203, - "nauc_ndcg_at_100_std": 0.12294409372370381, - "nauc_ndcg_at_100_diff1": 0.08845423517529309, - "nauc_ndcg_at_1000_max": 0.030127261370811845, - "nauc_ndcg_at_1000_std": 0.10766365950024334, - "nauc_ndcg_at_1000_diff1": 0.0861546357811246, - "nauc_map_at_1_max": -0.012854409302840068, - "nauc_map_at_1_std": 0.08676381700185752, - "nauc_map_at_1_diff1": 0.11334791281377288, - "nauc_map_at_3_max": -0.0029405293797002733, - "nauc_map_at_3_std": 0.06167764879997172, - "nauc_map_at_3_diff1": 0.07950290715235987, - "nauc_map_at_5_max": 0.00830536326209409, - "nauc_map_at_5_std": 0.07256833511730693, - "nauc_map_at_5_diff1": 0.08747148302006987, - "nauc_map_at_10_max": 0.015140286093876175, - "nauc_map_at_10_std": 0.08580662522686537, - "nauc_map_at_10_diff1": 0.08508762518017199, - "nauc_map_at_20_max": 0.0163028127500422, - "nauc_map_at_20_std": 0.09050711291668995, - "nauc_map_at_20_diff1": 0.08629712373980958, - "nauc_map_at_100_max": 0.016183246425002705, - "nauc_map_at_100_std": 0.09000154689011092, - "nauc_map_at_100_diff1": 0.08699050038257111, - "nauc_map_at_1000_max": 0.015701662616224514, - "nauc_map_at_1000_std": 0.0894576065792463, - "nauc_map_at_1000_diff1": 0.08694453879028893, - "nauc_recall_at_1_max": -0.012854409302840068, - "nauc_recall_at_1_std": 0.08676381700185752, - "nauc_recall_at_1_diff1": 0.11334791281377288, - "nauc_recall_at_3_max": 0.010338756866368223, - "nauc_recall_at_3_std": 0.03789278134763109, - "nauc_recall_at_3_diff1": 0.059403567851730316, - "nauc_recall_at_5_max": 0.05398176066498534, - "nauc_recall_at_5_std": 0.08064232472596307, - "nauc_recall_at_5_diff1": 0.08949514479783123, - "nauc_recall_at_10_max": 0.09984307699040616, - "nauc_recall_at_10_std": 0.17703668336587267, - "nauc_recall_at_10_diff1": 0.06498907396830537, - "nauc_recall_at_20_max": 0.15799758987163673, - "nauc_recall_at_20_std": 0.31157412853790695, - "nauc_recall_at_20_diff1": 0.07941702046000772, - "nauc_recall_at_100_max": 0.37265902510843735, - "nauc_recall_at_100_std": 0.5476974790176777, - "nauc_recall_at_100_diff1": 0.12977852219557848, - "nauc_recall_at_1000_max": 0.3004922701567, - "nauc_recall_at_1000_std": 0.7475055455618029, - "nauc_recall_at_1000_diff1": -0.05500901218402981, - "nauc_precision_at_1_max": -0.012854409302840068, - "nauc_precision_at_1_std": 0.08676381700185752, - "nauc_precision_at_1_diff1": 0.11334791281377288, - "nauc_precision_at_3_max": 0.010338756866368036, - "nauc_precision_at_3_std": 0.037892781347630994, - "nauc_precision_at_3_diff1": 0.05940356785173012, - "nauc_precision_at_5_max": 0.05398176066498592, - "nauc_precision_at_5_std": 0.08064232472596324, - "nauc_precision_at_5_diff1": 0.08949514479783155, - "nauc_precision_at_10_max": 0.09984307699040575, - "nauc_precision_at_10_std": 0.17703668336587192, - "nauc_precision_at_10_diff1": 0.06498907396830521, - "nauc_precision_at_20_max": 0.1579975898716373, - "nauc_precision_at_20_std": 0.31157412853790756, - "nauc_precision_at_20_diff1": 0.0794170204600081, - "nauc_precision_at_100_max": 0.3726590251084365, - "nauc_precision_at_100_std": 0.5476974790176794, - "nauc_precision_at_100_diff1": 0.1297785221955785, - "nauc_precision_at_1000_max": 0.3004922701567069, - "nauc_precision_at_1000_std": 0.7475055455618104, - "nauc_precision_at_1000_diff1": -0.055009012184042826, - "nauc_mrr_at_1_max": -0.012854409302840068, - "nauc_mrr_at_1_std": 0.08676381700185752, - "nauc_mrr_at_1_diff1": 0.11334791281377288, - "nauc_mrr_at_3_max": -0.0029405293797002733, - "nauc_mrr_at_3_std": 0.06167764879997172, - "nauc_mrr_at_3_diff1": 0.07950290715235987, - "nauc_mrr_at_5_max": 0.00830536326209409, - "nauc_mrr_at_5_std": 0.07256833511730693, - "nauc_mrr_at_5_diff1": 0.08747148302006987, - "nauc_mrr_at_10_max": 0.015140286093876175, - "nauc_mrr_at_10_std": 0.08580662522686537, - "nauc_mrr_at_10_diff1": 0.08508762518017199, - "nauc_mrr_at_20_max": 0.0163028127500422, - "nauc_mrr_at_20_std": 0.09050711291668995, - "nauc_mrr_at_20_diff1": 0.08629712373980958, - "nauc_mrr_at_100_max": 0.016183246425002705, - "nauc_mrr_at_100_std": 0.09000154689011092, - "nauc_mrr_at_100_diff1": 0.08699050038257111, - "nauc_mrr_at_1000_max": 0.015701662616224514, - "nauc_mrr_at_1000_std": 0.0894576065792463, - "nauc_mrr_at_1000_diff1": 0.08694453879028893, - "hit_rate_at_1": 0.17425, - "hit_rate_at_3": 0.34139, - "hit_rate_at_5": 0.45661, - "hit_rate_at_10": 0.63514, - "hit_rate_at_20": 0.766, - "hit_rate_at_100": 0.91679, - "hit_rate_at_1000": 0.99004, - "main_score": 0.37514, + "ndcg_at_1": 0.35349, + "ndcg_at_3": 0.49665, + "ndcg_at_5": 0.54581, + "ndcg_at_10": 0.59246, + "ndcg_at_20": 0.6132, + "ndcg_at_100": 0.62189, + "ndcg_at_1000": 0.62242, + "map_at_1": 0.35349, + "map_at_3": 0.46005, + "map_at_5": 0.48729, + "map_at_10": 0.50679, + "map_at_20": 0.51261, + "map_at_100": 0.51395, + "map_at_1000": 0.51397, + "recall_at_1": 0.35349, + "recall_at_3": 0.60313, + "recall_at_5": 0.72262, + "recall_at_10": 0.86558, + "recall_at_20": 0.94666, + "recall_at_100": 0.99147, + "recall_at_1000": 0.99573, + "accuracy": 0.35349, + "precision_at_1": 0.35349, + "precision_at_3": 0.20104, + "precision_at_5": 0.14452, + "precision_at_10": 0.08656, + "precision_at_20": 0.04733, + "precision_at_100": 0.00991, + "precision_at_1000": 0.001, + "mrr_at_1": 0.3534850640113798, + "mrr_at_3": 0.4600521574205785, + "mrr_at_5": 0.48729255571360836, + "mrr_at_10": 0.5067903429745535, + "mrr_at_20": 0.5126066994732466, + "mrr_at_100": 0.5139513666778618, + "mrr_at_1000": 0.5139717342260931, + "nauc_ndcg_at_1_max": -0.11747299439249903, + "nauc_ndcg_at_1_std": -0.11436695908554317, + "nauc_ndcg_at_1_diff1": 0.12406213904460131, + "nauc_ndcg_at_3_max": -0.0810374905617702, + "nauc_ndcg_at_3_std": -0.10902178800246173, + "nauc_ndcg_at_3_diff1": 0.09457199601062195, + "nauc_ndcg_at_5_max": -0.08485395980147033, + "nauc_ndcg_at_5_std": -0.10485663186916257, + "nauc_ndcg_at_5_diff1": 0.08582902365914505, + "nauc_ndcg_at_10_max": -0.08309974111344137, + "nauc_ndcg_at_10_std": -0.11506227102720018, + "nauc_ndcg_at_10_diff1": 0.08990562536626041, + "nauc_ndcg_at_20_max": -0.0791935521931642, + "nauc_ndcg_at_20_std": -0.10319594721919796, + "nauc_ndcg_at_20_diff1": 0.09971433495514617, + "nauc_ndcg_at_100_max": -0.08635224865444148, + "nauc_ndcg_at_100_std": -0.10462787774486179, + "nauc_ndcg_at_100_diff1": 0.09791984963152253, + "nauc_ndcg_at_1000_max": -0.0879373318368675, + "nauc_ndcg_at_1000_std": -0.10553141137460995, + "nauc_ndcg_at_1000_diff1": 0.09704987719570578, + "nauc_map_at_1_max": -0.11747299439249903, + "nauc_map_at_1_std": -0.11436695908554317, + "nauc_map_at_1_diff1": 0.12406213904460131, + "nauc_map_at_3_max": -0.09064151313996319, + "nauc_map_at_3_std": -0.1105170515941861, + "nauc_map_at_3_diff1": 0.09940664377727629, + "nauc_map_at_5_max": -0.09342459992783216, + "nauc_map_at_5_std": -0.10864465771524179, + "nauc_map_at_5_diff1": 0.09517709876339578, + "nauc_map_at_10_max": -0.09304942751697344, + "nauc_map_at_10_std": -0.11264010939720397, + "nauc_map_at_10_diff1": 0.09707068152210184, + "nauc_map_at_20_max": -0.09221792922323616, + "nauc_map_at_20_std": -0.10961211427408772, + "nauc_map_at_20_diff1": 0.09943255521491648, + "nauc_map_at_100_max": -0.09316799067959038, + "nauc_map_at_100_std": -0.10968589627543163, + "nauc_map_at_100_diff1": 0.09910011165040408, + "nauc_map_at_1000_max": -0.09321385399598632, + "nauc_map_at_1000_std": -0.10970513219025511, + "nauc_map_at_1000_diff1": 0.09907801272427635, + "nauc_recall_at_1_max": -0.11747299439249903, + "nauc_recall_at_1_std": -0.11436695908554317, + "nauc_recall_at_1_diff1": 0.12406213904460131, + "nauc_recall_at_3_max": -0.05070404620274185, + "nauc_recall_at_3_std": -0.10426214370217692, + "nauc_recall_at_3_diff1": 0.0805612409902579, + "nauc_recall_at_5_max": -0.051784236042595515, + "nauc_recall_at_5_std": -0.0889570674134141, + "nauc_recall_at_5_diff1": 0.049390111709494505, + "nauc_recall_at_10_max": -0.013727385294040483, + "nauc_recall_at_10_std": -0.13798296178002195, + "nauc_recall_at_10_diff1": 0.044023090224588696, + "nauc_recall_at_20_max": 0.15251226818625876, + "nauc_recall_at_20_std": 0.03363980242022382, + "nauc_recall_at_20_diff1": 0.15796490326794294, + "nauc_recall_at_100_max": 0.3311354440025815, + "nauc_recall_at_100_std": 0.4782526238266158, + "nauc_recall_at_100_diff1": 0.25005005298352057, + "nauc_recall_at_1000_max": 0.02172236447995406, + "nauc_recall_at_1000_std": 0.609819560470231, + "nauc_recall_at_1000_diff1": -0.005495147382062338, + "nauc_precision_at_1_max": -0.11747299439249903, + "nauc_precision_at_1_std": -0.11436695908554317, + "nauc_precision_at_1_diff1": 0.12406213904460131, + "nauc_precision_at_3_max": -0.05070404620274239, + "nauc_precision_at_3_std": -0.10426214370217736, + "nauc_precision_at_3_diff1": 0.08056124099025751, + "nauc_precision_at_5_max": -0.05178423604259566, + "nauc_precision_at_5_std": -0.08895706741341412, + "nauc_precision_at_5_diff1": 0.04939011170949452, + "nauc_precision_at_10_max": -0.013727385294041244, + "nauc_precision_at_10_std": -0.13798296178002226, + "nauc_precision_at_10_diff1": 0.044023090224589216, + "nauc_precision_at_20_max": 0.15251226818625896, + "nauc_precision_at_20_std": 0.03363980242022203, + "nauc_precision_at_20_diff1": 0.15796490326794188, + "nauc_precision_at_100_max": 0.3311354440025917, + "nauc_precision_at_100_std": 0.4782526238266313, + "nauc_precision_at_100_diff1": 0.25005005298354305, + "nauc_precision_at_1000_max": 0.0217223644799946, + "nauc_precision_at_1000_std": 0.6098195604702127, + "nauc_precision_at_1000_diff1": -0.0054951473820258456, + "nauc_mrr_at_1_max": -0.11747299439249903, + "nauc_mrr_at_1_std": -0.11436695908554317, + "nauc_mrr_at_1_diff1": 0.12406213904460131, + "nauc_mrr_at_3_max": -0.09064151313996319, + "nauc_mrr_at_3_std": -0.1105170515941861, + "nauc_mrr_at_3_diff1": 0.09940664377727629, + "nauc_mrr_at_5_max": -0.09342459992783216, + "nauc_mrr_at_5_std": -0.10864465771524179, + "nauc_mrr_at_5_diff1": 0.09517709876339578, + "nauc_mrr_at_10_max": -0.09304942751697344, + "nauc_mrr_at_10_std": -0.11264010939720397, + "nauc_mrr_at_10_diff1": 0.09707068152210184, + "nauc_mrr_at_20_max": -0.09221792922323616, + "nauc_mrr_at_20_std": -0.10961211427408772, + "nauc_mrr_at_20_diff1": 0.09943255521491648, + "nauc_mrr_at_100_max": -0.09316799067959038, + "nauc_mrr_at_100_std": -0.10968589627543163, + "nauc_mrr_at_100_diff1": 0.09910011165040408, + "nauc_mrr_at_1000_max": -0.09321385399598632, + "nauc_mrr_at_1000_std": -0.10970513219025511, + "nauc_mrr_at_1000_diff1": 0.09907801272427635, + "hit_rate_at_1": 0.35349, + "hit_rate_at_3": 0.60313, + "hit_rate_at_5": 0.72262, + "hit_rate_at_10": 0.86558, + "hit_rate_at_20": 0.94666, + "hit_rate_at_100": 0.99147, + "hit_rate_at_1000": 0.99573, + "main_score": 0.59246, "hf_subset": "default", "languages": [ "eng-Latn" @@ -955,160 +796,160 @@ "mteb_version": "2.21.0" }, "mini": { - "ndcg_at_1": 0.15789, - "ndcg_at_3": 0.26711, - "ndcg_at_5": 0.31361, - "ndcg_at_10": 0.36413, - "ndcg_at_20": 0.40172, - "ndcg_at_100": 0.43138, - "ndcg_at_1000": 0.44017, - "map_at_1": 0.15789, - "map_at_3": 0.23957, - "map_at_5": 0.26549, - "map_at_10": 0.28631, - "map_at_20": 0.29672, - "map_at_100": 0.30104, - "map_at_1000": 0.30138, - "recall_at_1": 0.15789, - "recall_at_3": 0.34708, - "recall_at_5": 0.45946, - "recall_at_10": 0.61593, - "recall_at_20": 0.76387, - "recall_at_100": 0.92034, - "recall_at_1000": 0.98933, - "accuracy": 0.15789, - "precision_at_1": 0.15789, - "precision_at_3": 0.11569, - "precision_at_5": 0.09189, - "precision_at_10": 0.06159, - "precision_at_20": 0.03819, - "precision_at_100": 0.0092, - "precision_at_1000": 0.00099, - "mrr_at_1": 0.15789473684210525, - "mrr_at_3": 0.23956851588430533, - "mrr_at_5": 0.26549312470365105, - "mrr_at_10": 0.28631178170651855, - "mrr_at_20": 0.2967216276070358, - "mrr_at_100": 0.3010408041354957, - "mrr_at_1000": 0.3013763630745192, - "nauc_ndcg_at_1_max": -0.06337192571116465, - "nauc_ndcg_at_1_std": -0.04928315143299763, - "nauc_ndcg_at_1_diff1": 0.0842660849840467, - "nauc_ndcg_at_3_max": -0.02089703214535498, - "nauc_ndcg_at_3_std": -0.0251177327797788, - "nauc_ndcg_at_3_diff1": 0.057661612732888716, - "nauc_ndcg_at_5_max": -0.010967446322962574, - "nauc_ndcg_at_5_std": -0.024585407638390625, - "nauc_ndcg_at_5_diff1": 0.06056592946035405, - "nauc_ndcg_at_10_max": 0.0020933122978897514, - "nauc_ndcg_at_10_std": -0.017859954377114125, - "nauc_ndcg_at_10_diff1": 0.06508583402285395, - "nauc_ndcg_at_20_max": 0.010235926091525314, - "nauc_ndcg_at_20_std": 0.0007794838375854261, - "nauc_ndcg_at_20_diff1": 0.06735954725329767, - "nauc_ndcg_at_100_max": 0.002381905099738436, - "nauc_ndcg_at_100_std": 0.008052425240155718, - "nauc_ndcg_at_100_diff1": 0.06042516679725263, - "nauc_ndcg_at_1000_max": -0.0058738012172331894, - "nauc_ndcg_at_1000_std": -0.0043094568403283335, - "nauc_ndcg_at_1000_diff1": 0.05930574538573805, - "nauc_map_at_1_max": -0.06337192571116465, - "nauc_map_at_1_std": -0.04928315143299763, - "nauc_map_at_1_diff1": 0.0842660849840467, - "nauc_map_at_3_max": -0.03138183227204495, - "nauc_map_at_3_std": -0.030340202848035035, - "nauc_map_at_3_diff1": 0.059249068788292975, - "nauc_map_at_5_max": -0.025354535991427198, - "nauc_map_at_5_std": -0.029672509747791915, - "nauc_map_at_5_diff1": 0.06073863109812545, - "nauc_map_at_10_max": -0.020210372452168703, - "nauc_map_at_10_std": -0.027271139934316947, - "nauc_map_at_10_diff1": 0.06258482981328098, - "nauc_map_at_20_max": -0.018612164552598766, - "nauc_map_at_20_std": -0.022182947475984684, - "nauc_map_at_20_diff1": 0.06280318106790861, - "nauc_map_at_100_max": -0.01990750206820275, - "nauc_map_at_100_std": -0.021394589062622812, - "nauc_map_at_100_diff1": 0.06162894610543603, - "nauc_map_at_1000_max": -0.020249905464431128, - "nauc_map_at_1000_std": -0.021811489290011532, - "nauc_map_at_1000_diff1": 0.06155979145434957, - "nauc_recall_at_1_max": -0.06337192571116465, - "nauc_recall_at_1_std": -0.04928315143299763, - "nauc_recall_at_1_diff1": 0.0842660849840467, - "nauc_recall_at_3_max": 0.004877033795042247, - "nauc_recall_at_3_std": -0.01248368933424835, - "nauc_recall_at_3_diff1": 0.05512708750315134, - "nauc_recall_at_5_max": 0.025099525510702818, - "nauc_recall_at_5_std": -0.012595407351099602, - "nauc_recall_at_5_diff1": 0.06198849352143125, - "nauc_recall_at_10_max": 0.07009179337568387, - "nauc_recall_at_10_std": 0.011069121969996607, - "nauc_recall_at_10_diff1": 0.07592849845298667, - "nauc_recall_at_20_max": 0.14098722845572492, - "nauc_recall_at_20_std": 0.1134519461262802, - "nauc_recall_at_20_diff1": 0.09496328915300539, - "nauc_recall_at_100_max": 0.22720540744726367, - "nauc_recall_at_100_std": 0.4097676886656483, - "nauc_recall_at_100_diff1": 0.03700534055726312, - "nauc_recall_at_1000_max": 0.32659903720033334, - "nauc_recall_at_1000_std": 0.8135553709558353, - "nauc_recall_at_1000_diff1": -0.250485096885085, - "nauc_precision_at_1_max": -0.06337192571116465, - "nauc_precision_at_1_std": -0.04928315143299763, - "nauc_precision_at_1_diff1": 0.0842660849840467, - "nauc_precision_at_3_max": 0.00487703379504197, - "nauc_precision_at_3_std": -0.012483689334248491, - "nauc_precision_at_3_diff1": 0.05512708750315098, - "nauc_precision_at_5_max": 0.025099525510702995, - "nauc_precision_at_5_std": -0.01259540735109937, - "nauc_precision_at_5_diff1": 0.061988493521431785, - "nauc_precision_at_10_max": 0.07009179337568382, - "nauc_precision_at_10_std": 0.011069121969996348, - "nauc_precision_at_10_diff1": 0.07592849845298615, - "nauc_precision_at_20_max": 0.14098722845572442, - "nauc_precision_at_20_std": 0.11345194612627922, - "nauc_precision_at_20_diff1": 0.09496328915300546, - "nauc_precision_at_100_max": 0.22720540744726378, - "nauc_precision_at_100_std": 0.40976768866564856, - "nauc_precision_at_100_diff1": 0.03700534055726116, - "nauc_precision_at_1000_max": 0.32659903720033095, - "nauc_precision_at_1000_std": 0.813555370955819, - "nauc_precision_at_1000_diff1": -0.25048509688509923, - "nauc_mrr_at_1_max": -0.06337192571116465, - "nauc_mrr_at_1_std": -0.04928315143299763, - "nauc_mrr_at_1_diff1": 0.0842660849840467, - "nauc_mrr_at_3_max": -0.03138183227204495, - "nauc_mrr_at_3_std": -0.030340202848035035, - "nauc_mrr_at_3_diff1": 0.059249068788292975, - "nauc_mrr_at_5_max": -0.025354535991427198, - "nauc_mrr_at_5_std": -0.029672509747791915, - "nauc_mrr_at_5_diff1": 0.06073863109812545, - "nauc_mrr_at_10_max": -0.020210372452168703, - "nauc_mrr_at_10_std": -0.027271139934316947, - "nauc_mrr_at_10_diff1": 0.06258482981328098, - "nauc_mrr_at_20_max": -0.018612164552598766, - "nauc_mrr_at_20_std": -0.022182947475984684, - "nauc_mrr_at_20_diff1": 0.06280318106790861, - "nauc_mrr_at_100_max": -0.01990750206820275, - "nauc_mrr_at_100_std": -0.021394589062622812, - "nauc_mrr_at_100_diff1": 0.06162894610543603, - "nauc_mrr_at_1000_max": -0.020249905464431128, - "nauc_mrr_at_1000_std": -0.021811489290011532, - "nauc_mrr_at_1000_diff1": 0.06155979145434957, - "hit_rate_at_1": 0.15789, - "hit_rate_at_3": 0.34708, - "hit_rate_at_5": 0.45946, - "hit_rate_at_10": 0.61593, - "hit_rate_at_20": 0.76387, - "hit_rate_at_100": 0.92034, - "hit_rate_at_1000": 0.98933, - "main_score": 0.36413, + "accuracy": 0.43741, "hf_subset": "default", + "hit_rate_at_1": 0.43741, + "hit_rate_at_10": 0.93314, + "hit_rate_at_100": 0.99431, + "hit_rate_at_1000": 0.99644, + "hit_rate_at_20": 0.97368, + "hit_rate_at_3": 0.72048, + "hit_rate_at_5": 0.83428, "languages": [ "eng-Latn" ], - "mteb_version": "2.21.0" + "main_score": 0.68209, + "map_at_1": 0.43741, + "map_at_10": 0.60181, + "map_at_100": 0.60545, + "map_at_1000": 0.60546, + "map_at_20": 0.60483, + "map_at_3": 0.56223, + "map_at_5": 0.58826, + "mrr_at_1": 0.43741109530583216, + "mrr_at_10": 0.6018057531215426, + "mrr_at_100": 0.6054513268620654, + "mrr_at_1000": 0.6054647076144088, + "mrr_at_20": 0.6048322233235464, + "mrr_at_3": 0.5622332859174964, + "mrr_at_5": 0.5882645803698435, + "mteb_version": "2.21.0", + "nauc_map_at_1000_diff1": 0.11519320922485428, + "nauc_map_at_1000_max": -0.10172486978320612, + "nauc_map_at_1000_std": -0.18735274249432798, + "nauc_map_at_100_diff1": 0.11518793608384066, + "nauc_map_at_100_max": -0.10167132799628414, + "nauc_map_at_100_std": -0.1873170321174228, + "nauc_map_at_10_diff1": 0.11532445899451405, + "nauc_map_at_10_max": -0.0989270727790863, + "nauc_map_at_10_std": -0.18555919763501583, + "nauc_map_at_1_diff1": 0.1344411865499415, + "nauc_map_at_1_max": -0.13253906649555908, + "nauc_map_at_1_std": -0.18486458506533457, + "nauc_map_at_20_diff1": 0.11474920852828616, + "nauc_map_at_20_max": -0.1014092067380479, + "nauc_map_at_20_std": -0.18706605406540486, + "nauc_map_at_3_diff1": 0.10664133218203026, + "nauc_map_at_3_max": -0.10450512209746964, + "nauc_map_at_3_std": -0.19976618975788185, + "nauc_map_at_5_diff1": 0.11933285738048886, + "nauc_map_at_5_max": -0.0938378317396114, + "nauc_map_at_5_std": -0.1866538889762921, + "nauc_mrr_at_1000_diff1": 0.11519320922485428, + "nauc_mrr_at_1000_max": -0.10172486978320612, + "nauc_mrr_at_1000_std": -0.18735274249432798, + "nauc_mrr_at_100_diff1": 0.11518793608384066, + "nauc_mrr_at_100_max": -0.10167132799628414, + "nauc_mrr_at_100_std": -0.1873170321174228, + "nauc_mrr_at_10_diff1": 0.11532445899451405, + "nauc_mrr_at_10_max": -0.0989270727790863, + "nauc_mrr_at_10_std": -0.18555919763501583, + "nauc_mrr_at_1_diff1": 0.1344411865499415, + "nauc_mrr_at_1_max": -0.13253906649555908, + "nauc_mrr_at_1_std": -0.18486458506533457, + "nauc_mrr_at_20_diff1": 0.11474920852828616, + "nauc_mrr_at_20_max": -0.1014092067380479, + "nauc_mrr_at_20_std": -0.18706605406540486, + "nauc_mrr_at_3_diff1": 0.10664133218203026, + "nauc_mrr_at_3_max": -0.10450512209746964, + "nauc_mrr_at_3_std": -0.19976618975788185, + "nauc_mrr_at_5_diff1": 0.11933285738048886, + "nauc_mrr_at_5_max": -0.0938378317396114, + "nauc_mrr_at_5_std": -0.1866538889762921, + "nauc_ndcg_at_1000_diff1": 0.11330864927688393, + "nauc_ndcg_at_1000_max": -0.09436863195857823, + "nauc_ndcg_at_1000_std": -0.1831012742795625, + "nauc_ndcg_at_100_diff1": 0.11328468411110973, + "nauc_ndcg_at_100_max": -0.09288690174933568, + "nauc_ndcg_at_100_std": -0.1821920978303095, + "nauc_ndcg_at_10_diff1": 0.11232762798595314, + "nauc_ndcg_at_10_max": -0.08015480791447117, + "nauc_ndcg_at_10_std": -0.17334000764172103, + "nauc_ndcg_at_1_diff1": 0.1344411865499415, + "nauc_ndcg_at_1_max": -0.13253906649555908, + "nauc_ndcg_at_1_std": -0.18486458506533457, + "nauc_ndcg_at_20_diff1": 0.11008813518731168, + "nauc_ndcg_at_20_max": -0.09038013342909992, + "nauc_ndcg_at_20_std": -0.1802076935236352, + "nauc_ndcg_at_3_diff1": 0.09895322667549003, + "nauc_ndcg_at_3_max": -0.09125553984442554, + "nauc_ndcg_at_3_std": -0.20465040262832981, + "nauc_ndcg_at_5_diff1": 0.12298012909677512, + "nauc_ndcg_at_5_max": -0.06793074241820368, + "nauc_ndcg_at_5_std": -0.17773505029111197, + "nauc_precision_at_1000_diff1": -0.0313224102116296, + "nauc_precision_at_1000_max": 0.2085546049013785, + "nauc_precision_at_1000_std": 0.4095849260391426, + "nauc_precision_at_100_diff1": 0.024017721273557675, + "nauc_precision_at_100_max": 0.5053466280633614, + "nauc_precision_at_100_std": 0.433483801823953, + "nauc_precision_at_10_diff1": 0.07936979879202169, + "nauc_precision_at_10_max": 0.14225500208712952, + "nauc_precision_at_10_std": 0.0020625593472650768, + "nauc_precision_at_1_diff1": 0.1344411865499415, + "nauc_precision_at_1_max": -0.13253906649555908, + "nauc_precision_at_1_std": -0.18486458506533457, + "nauc_precision_at_20_diff1": -0.03530331268281447, + "nauc_precision_at_20_max": 0.15413207406447058, + "nauc_precision_at_20_std": 0.038469470998343976, + "nauc_precision_at_3_diff1": 0.0718506749796813, + "nauc_precision_at_3_max": -0.04164681090756132, + "nauc_precision_at_3_std": -0.22227120951623774, + "nauc_precision_at_5_diff1": 0.15000475771120342, + "nauc_precision_at_5_max": 0.07822940650572034, + "nauc_precision_at_5_std": -0.12111564857328351, + "nauc_recall_at_1000_diff1": -0.031322410211636634, + "nauc_recall_at_1000_max": 0.20855460490135933, + "nauc_recall_at_1000_std": 0.40958492603917024, + "nauc_recall_at_100_diff1": 0.024017721273597505, + "nauc_recall_at_100_max": 0.5053466280633541, + "nauc_recall_at_100_std": 0.43348380182397117, + "nauc_recall_at_10_diff1": 0.07936979879202402, + "nauc_recall_at_10_max": 0.14225500208712785, + "nauc_recall_at_10_std": 0.0020625593472650737, + "nauc_recall_at_1_diff1": 0.1344411865499415, + "nauc_recall_at_1_max": -0.13253906649555908, + "nauc_recall_at_1_std": -0.18486458506533457, + "nauc_recall_at_20_diff1": -0.035303312682816994, + "nauc_recall_at_20_max": 0.15413207406446455, + "nauc_recall_at_20_std": 0.038469470998334095, + "nauc_recall_at_3_diff1": 0.07185067497968155, + "nauc_recall_at_3_max": -0.04164681090756132, + "nauc_recall_at_3_std": -0.22227120951623786, + "nauc_recall_at_5_diff1": 0.15000475771120306, + "nauc_recall_at_5_max": 0.07822940650572234, + "nauc_recall_at_5_std": -0.12111564857328255, + "ndcg_at_1": 0.43741, + "ndcg_at_10": 0.68209, + "ndcg_at_100": 0.69661, + "ndcg_at_1000": 0.6969, + "ndcg_at_20": 0.6926, + "ndcg_at_3": 0.60288, + "ndcg_at_5": 0.64977, + "precision_at_1": 0.43741, + "precision_at_10": 0.09331, + "precision_at_100": 0.00994, + "precision_at_1000": 0.001, + "precision_at_20": 0.04868, + "precision_at_3": 0.24016, + "precision_at_5": 0.16686, + "recall_at_1": 0.43741, + "recall_at_10": 0.93314, + "recall_at_100": 0.99431, + "recall_at_1000": 0.99644, + "recall_at_20": 0.97368, + "recall_at_3": 0.72048, + "recall_at_5": 0.83428 } } }, @@ -1133,44 +974,44 @@ "resampled_items_per_trial": 16384 }, "scores": { - "nano": 0.44884717457494344, - "mini": 0.39564816247699763 + "nano": 0.5087260708286914, + "mini": 0.4854458980619449 }, "metrics": { "nano": { "v_measures": { "Level 0": [ - 0.48056098748371134, - 0.44085942767025554, - 0.43416151810540005, - 0.45299559752505, - 0.44116051339311235, - 0.4409873172673812, - 0.46155584917327236, - 0.444819328389149, - 0.44792019268133576, - 0.44345101406076604 + 0.4966639569479829, + 0.5210509442892017, + 0.5333989625396617, + 0.5094460445939109, + 0.5208429535416115, + 0.5012604404318527, + 0.5210119744629907, + 0.4617825059979706, + 0.5226704101107219, + 0.49913251537100806 ] }, - "v_measure": 0.44884717457494344, - "v_measure_std": 0.01277114663774588, + "v_measure": 0.5087260708286914, + "v_measure_std": 0.01936580527644773, "ami_scores": { "Level 0": [ - 0.4786019649327535, - 0.43870317373726253, - 0.43201951778515824, - 0.4509348225877084, - 0.43904842199622773, - 0.43887546309559994, - 0.45953222355170814, - 0.44270204450063555, - 0.44583346842612326, - 0.4413500450061514 + 0.49471495769938123, + 0.5192201304733433, + 0.5316138913144499, + 0.5075814211955173, + 0.5190252719774525, + 0.49937824878079495, + 0.5191850550712727, + 0.4597515065381579, + 0.5208311166380218, + 0.49721166070769024 ] }, - "ami": 0.4467601145619328, - "ami_std": 0.012825331450796505, - "main_score": 0.44884717457494344, + "ami": 0.5068513260396083, + "ami_std": 0.01943429917113934, + "main_score": 0.5087260708286914, "hf_subset": "default", "languages": [ "eng-Latn" @@ -1178,44 +1019,44 @@ "mteb_version": "2.21.0" }, "mini": { - "v_measures": { - "Level 0": [ - 0.39572980998506846, - 0.38917194299997865, - 0.39658108632781985, - 0.3882063003549651, - 0.42337354342291456, - 0.3720031649827461, - 0.3807394225447415, - 0.3992318667638805, - 0.41303222975761383, - 0.398412257630248 - ] - }, - "v_measure": 0.39564816247699763, - "v_measure_std": 0.014039023751318776, + "ami": 0.4834422068644281, "ami_scores": { "Level 0": [ - 0.3934199479124463, - 0.3868466462971748, - 0.3942803942094721, - 0.3858546965037562, - 0.42112187856391137, - 0.36963503354675786, - 0.3783969871192455, - 0.3969321307574592, - 0.4107603441547548, - 0.3961435066912112 + 0.5237779172758207, + 0.49322312218863495, + 0.49568367154165427, + 0.47534322399179285, + 0.49762104367383714, + 0.47939796463482504, + 0.44715009465468164, + 0.4684435195493317, + 0.48512897761213203, + 0.4686525335215719 ] }, - "ami": 0.3933391565756189, - "ami_std": 0.01407255280759407, - "main_score": 0.39564816247699763, + "ami_std": 0.019796807296708636, "hf_subset": "default", "languages": [ "eng-Latn" ], - "mteb_version": "2.21.0" + "main_score": 0.4854458980619449, + "mteb_version": "2.21.0", + "v_measure": 0.4854458980619449, + "v_measure_std": 0.019745525230436267, + "v_measures": { + "Level 0": [ + 0.5256726781796038, + 0.49522771197215554, + 0.4976030621216199, + 0.47734199601401234, + 0.49961575789338, + 0.4814390025295903, + 0.4492393337162524, + 0.4705195658859963, + 0.487123682558257, + 0.4706761897485818 + ] + } } } }, @@ -1239,133 +1080,133 @@ "probe_samples_per_label": 8 }, "scores": { - "nano": 0.8758836689038031, - "mini": 0.8593288590604026 + "nano": 0.9476510067114093, + "mini": 0.91248322147651 }, "metrics": { "nano": { "scores_per_experiment": [ { - "accuracy": 0.8527964205816555, - "f1": 0.8584702959865076, - "f1_weighted": 0.8520037076109549, - "precision": 0.8563814369311937, - "precision_weighted": 0.8604397214633738, - "recall": 0.8688240583945851, - "recall_weighted": 0.8527964205816555, + "accuracy": 0.9521252796420582, + "f1": 0.9509396968297774, + "f1_weighted": 0.9519302419648948, + "precision": 0.9493488952211522, + "precision_weighted": 0.9527659658117037, + "recall": 0.9535803496254757, + "recall_weighted": 0.9521252796420582, "ap": null, "ap_weighted": null }, { - "accuracy": 0.8675615212527964, - "f1": 0.8658660043655293, - "f1_weighted": 0.8661413651963479, - "precision": 0.8632122218181354, - "precision_weighted": 0.8687564991167056, - "recall": 0.8733845956286541, - "recall_weighted": 0.8675615212527964, + "accuracy": 0.9458612975391499, + "f1": 0.9439475266607382, + "f1_weighted": 0.945767354216612, + "precision": 0.9420005561621522, + "precision_weighted": 0.9462895866502771, + "recall": 0.946397940529448, + "recall_weighted": 0.9458612975391499, "ap": null, "ap_weighted": null }, { - "accuracy": 0.8514541387024609, - "f1": 0.8524202289070963, - "f1_weighted": 0.8491348169886809, - "precision": 0.8478156770106394, - "precision_weighted": 0.8512267923319679, - "recall": 0.8621133428009895, - "recall_weighted": 0.8514541387024609, + "accuracy": 0.9284116331096197, + "f1": 0.9284988704174789, + "f1_weighted": 0.9270840634554431, + "precision": 0.9271908775153325, + "precision_weighted": 0.9274634710035635, + "recall": 0.9317153602473662, + "recall_weighted": 0.9284116331096197, "ap": null, "ap_weighted": null }, { - "accuracy": 0.8876957494407158, - "f1": 0.8889547651830694, - "f1_weighted": 0.886393441219644, - "precision": 0.8855262149097257, - "precision_weighted": 0.8869170708224573, - "recall": 0.8941464114507656, - "recall_weighted": 0.8876957494407158, + "accuracy": 0.9525727069351231, + "f1": 0.950987470976036, + "f1_weighted": 0.9525185063610705, + "precision": 0.9504292297253194, + "precision_weighted": 0.9534501308715367, + "recall": 0.9526312322737922, + "recall_weighted": 0.9525727069351231, "ap": null, "ap_weighted": null }, { - "accuracy": 0.8684563758389262, - "f1": 0.8708980361567344, - "f1_weighted": 0.8668343457695604, - "precision": 0.8655644216656676, - "precision_weighted": 0.8685796219847712, - "recall": 0.8789277362563833, - "recall_weighted": 0.8684563758389262, + "accuracy": 0.9445190156599552, + "f1": 0.943383064264026, + "f1_weighted": 0.9442288513061988, + "precision": 0.9424858291605631, + "precision_weighted": 0.944823795147547, + "recall": 0.945181115724642, + "recall_weighted": 0.9445190156599552, "ap": null, "ap_weighted": null }, { - "accuracy": 0.8662192393736018, - "f1": 0.8725560012982636, - "f1_weighted": 0.8665613320034511, - "precision": 0.8737258252820442, - "precision_weighted": 0.8845738920578841, - "recall": 0.8863222925022911, - "recall_weighted": 0.8662192393736018, + "accuracy": 0.9525727069351231, + "f1": 0.949882476921466, + "f1_weighted": 0.9526817135384125, + "precision": 0.9498620235113855, + "precision_weighted": 0.9543297139412104, + "recall": 0.9514687067629995, + "recall_weighted": 0.9525727069351231, "ap": null, "ap_weighted": null }, { - "accuracy": 0.8953020134228188, - "f1": 0.8976332709955689, - "f1_weighted": 0.8945899154754817, - "precision": 0.8912735415957905, - "precision_weighted": 0.8979949591784757, - "recall": 0.908092721226657, - "recall_weighted": 0.8953020134228188, + "accuracy": 0.9498881431767338, + "f1": 0.9496336255013932, + "f1_weighted": 0.9495703695110613, + "precision": 0.9490015795096777, + "precision_weighted": 0.9511221301014349, + "recall": 0.9520470037532829, + "recall_weighted": 0.9498881431767338, "ap": null, "ap_weighted": null }, { - "accuracy": 0.9029082774049217, - "f1": 0.9022310464802398, - "f1_weighted": 0.9020525018623339, - "precision": 0.899983538025133, - "precision_weighted": 0.9036775461946326, - "recall": 0.9067592943176375, - "recall_weighted": 0.9029082774049217, + "accuracy": 0.9570469798657718, + "f1": 0.9557194953102311, + "f1_weighted": 0.9568992928300293, + "precision": 0.9544888722394763, + "precision_weighted": 0.9584505479016514, + "recall": 0.9586898382241177, + "recall_weighted": 0.9570469798657718, "ap": null, "ap_weighted": null }, { - "accuracy": 0.876510067114094, - "f1": 0.8776887066559357, - "f1_weighted": 0.8761842615927375, - "precision": 0.8722809982436143, - "precision_weighted": 0.8829904469624968, - "recall": 0.8888620916111349, - "recall_weighted": 0.876510067114094, + "accuracy": 0.9494407158836689, + "f1": 0.9460535182757994, + "f1_weighted": 0.9492959546761313, + "precision": 0.9454088954645755, + "precision_weighted": 0.9499145785798135, + "recall": 0.9472953341577202, + "recall_weighted": 0.9494407158836689, "ap": null, "ap_weighted": null }, { - "accuracy": 0.8899328859060402, - "f1": 0.8916131163462019, - "f1_weighted": 0.8885446252332447, - "precision": 0.8855482140891268, - "precision_weighted": 0.8923990752693544, - "recall": 0.9020276692969261, - "recall_weighted": 0.8899328859060402, + "accuracy": 0.9440715883668904, + "f1": 0.9431192151526329, + "f1_weighted": 0.9437397930273277, + "precision": 0.9435095731735328, + "precision_weighted": 0.9461253948734786, + "recall": 0.9455297617432863, + "recall_weighted": 0.9440715883668904, "ap": null, "ap_weighted": null } ], - "accuracy": 0.8758836689038031, - "f1": 0.8778331472375147, - "f1_weighted": 0.8748440312952436, - "precision": 0.8741312089571069, - "precision_weighted": 0.8797555625382119, - "recall": 0.8869460213486026, - "recall_weighted": 0.8758836689038031, + "accuracy": 0.9476510067114093, + "f1": 0.9462164960309579, + "f1_weighted": 0.9473716140887183, + "precision": 0.9453726331683165, + "precision_weighted": 0.9484735314882217, + "recall": 0.948453664304213, + "recall_weighted": 0.9476510067114093, "ap": null, "ap_weighted": null, - "main_score": 0.8758836689038031, + "main_score": 0.9476510067114093, "hf_subset": "en", "languages": [ "eng-Latn" @@ -1373,133 +1214,133 @@ "mteb_version": "2.21.0" }, "mini": { + "accuracy": 0.91248322147651, + "ap": null, + "ap_weighted": null, + "f1": 0.9129343084325063, + "f1_weighted": 0.9117403910816442, + "hf_subset": "en", + "languages": [ + "eng-Latn" + ], + "main_score": 0.91248322147651, + "mteb_version": "2.21.0", + "precision": 0.9118310485105534, + "precision_weighted": 0.9143866303064343, + "recall": 0.9170949806878589, + "recall_weighted": 0.91248322147651, "scores_per_experiment": [ { - "accuracy": 0.8456375838926175, - "f1": 0.8498379922231407, - "f1_weighted": 0.846014889074395, - "precision": 0.8475229479246422, - "precision_weighted": 0.8580587769509141, - "recall": 0.8634534008305287, - "recall_weighted": 0.8456375838926175, + "accuracy": 0.9230425055928412, "ap": null, - "ap_weighted": null + "ap_weighted": null, + "f1": 0.9240037259023892, + "f1_weighted": 0.9226579709690474, + "precision": 0.9212314333927655, + "precision_weighted": 0.9235041933092486, + "recall": 0.9277347390578957, + "recall_weighted": 0.9230425055928412 }, { - "accuracy": 0.8662192393736018, - "f1": 0.8653334878151532, - "f1_weighted": 0.8652530918602429, - "precision": 0.865517487298354, - "precision_weighted": 0.8669057620056321, - "recall": 0.8680466660853953, - "recall_weighted": 0.8662192393736018, + "accuracy": 0.9154362416107382, "ap": null, - "ap_weighted": null + "ap_weighted": null, + "f1": 0.9138415749868535, + "f1_weighted": 0.914467623792272, + "precision": 0.9160409174037327, + "precision_weighted": 0.9147331845626346, + "recall": 0.9128885078468488, + "recall_weighted": 0.9154362416107382 }, { - "accuracy": 0.8604026845637583, - "f1": 0.85698051564255, - "f1_weighted": 0.8586612924937915, - "precision": 0.8509073197609578, - "precision_weighted": 0.8631423169146264, - "recall": 0.870036912853717, - "recall_weighted": 0.8604026845637583, + "accuracy": 0.8948545861297539, "ap": null, - "ap_weighted": null + "ap_weighted": null, + "f1": 0.8947271740410357, + "f1_weighted": 0.892577650722491, + "precision": 0.8888703830898386, + "precision_weighted": 0.8928044790863988, + "recall": 0.9028068818349797, + "recall_weighted": 0.8948545861297539 }, { - "accuracy": 0.8823266219239374, - "f1": 0.8799397028406161, - "f1_weighted": 0.8812894825359326, - "precision": 0.8800910510215822, - "precision_weighted": 0.8824176757212396, - "recall": 0.882647940110217, - "recall_weighted": 0.8823266219239374, + "accuracy": 0.9270693512304251, "ap": null, - "ap_weighted": null + "ap_weighted": null, + "f1": 0.925007904505356, + "f1_weighted": 0.9267003644527942, + "precision": 0.9243835988741065, + "precision_weighted": 0.9271686939006436, + "recall": 0.9266180181665273, + "recall_weighted": 0.9270693512304251 }, { - "accuracy": 0.8519015659955257, - "f1": 0.8535652182405795, - "f1_weighted": 0.8489492261042354, - "precision": 0.8478481803733672, - "precision_weighted": 0.8490720379074748, - "recall": 0.8627734778179271, - "recall_weighted": 0.8519015659955257, + "accuracy": 0.9131991051454139, "ap": null, - "ap_weighted": null + "ap_weighted": null, + "f1": 0.9168577112961764, + "f1_weighted": 0.9128391765031965, + "precision": 0.9184688614471448, + "precision_weighted": 0.9202301200605457, + "recall": 0.9223148293767295, + "recall_weighted": 0.9131991051454139 }, { - "accuracy": 0.8313199105145413, - "f1": 0.8403817297701731, - "f1_weighted": 0.8318002711323348, - "precision": 0.852929679947227, - "precision_weighted": 0.8629062064418095, - "recall": 0.8531894669682702, - "recall_weighted": 0.8313199105145413, + "accuracy": 0.9078299776286354, "ap": null, - "ap_weighted": null + "ap_weighted": null, + "f1": 0.910184865271744, + "f1_weighted": 0.9076815104080443, + "precision": 0.9113815611852641, + "precision_weighted": 0.91137656479833, + "recall": 0.9127764467674854, + "recall_weighted": 0.9078299776286354 }, { - "accuracy": 0.8697986577181208, - "f1": 0.870139586833364, - "f1_weighted": 0.8694571527237246, - "precision": 0.8662624227250791, - "precision_weighted": 0.8748550430216528, - "recall": 0.8794021267832393, - "recall_weighted": 0.8697986577181208, + "accuracy": 0.9109619686800895, "ap": null, - "ap_weighted": null + "ap_weighted": null, + "f1": 0.9118764145119242, + "f1_weighted": 0.9103351364502124, + "precision": 0.9081939213675884, + "precision_weighted": 0.9127584524444332, + "recall": 0.9180295433371218, + "recall_weighted": 0.9109619686800895 }, { - "accuracy": 0.8782997762863535, - "f1": 0.8755645802192206, - "f1_weighted": 0.8764702278243536, - "precision": 0.8772774587011537, - "precision_weighted": 0.8775338681714182, - "recall": 0.8773353490597977, - "recall_weighted": 0.8782997762863535, + "accuracy": 0.9221476510067114, "ap": null, - "ap_weighted": null + "ap_weighted": null, + "f1": 0.9202838678380185, + "f1_weighted": 0.9213525647799862, + "precision": 0.9201107803026847, + "precision_weighted": 0.9224288496231989, + "recall": 0.9225820562196617, + "recall_weighted": 0.9221476510067114 }, { - "accuracy": 0.8380313199105145, - "f1": 0.8383353072252739, - "f1_weighted": 0.8371416573059621, - "precision": 0.8398183538600482, - "precision_weighted": 0.8594030837180388, - "recall": 0.8555408228705527, - "recall_weighted": 0.8380313199105145, + "accuracy": 0.9033557046979865, "ap": null, - "ap_weighted": null + "ap_weighted": null, + "f1": 0.9035080229721874, + "f1_weighted": 0.9025264866229921, + "precision": 0.9021301372500449, + "precision_weighted": 0.9084410845992621, + "recall": 0.9102364347556008, + "recall_weighted": 0.9033557046979865 }, { - "accuracy": 0.8693512304250559, - "f1": 0.8694540443089611, - "f1_weighted": 0.8675967912323099, - "precision": 0.8656366051632016, - "precision_weighted": 0.8691183431838075, - "recall": 0.8769362896633872, - "recall_weighted": 0.8693512304250559, + "accuracy": 0.9069351230425056, "ap": null, - "ap_weighted": null + "ap_weighted": null, + "f1": 0.9090518229993763, + "f1_weighted": 0.9062654261154061, + "precision": 0.9074988907923621, + "precision_weighted": 0.9104206806796459, + "recall": 0.9149623495157385, + "recall_weighted": 0.9069351230425056 } - ], - "accuracy": 0.8593288590604026, - "f1": 0.8599532165119032, - "f1_weighted": 0.8582634082287284, - "precision": 0.8593811506775613, - "precision_weighted": 0.8663413114036613, - "recall": 0.8689362453043031, - "recall_weighted": 0.8593288590604026, - "ap": null, - "ap_weighted": null, - "main_score": 0.8593288590604026, - "hf_subset": "en", - "languages": [ - "eng-Latn" - ], - "mteb_version": "2.21.0" + ] } } }, @@ -1523,133 +1364,133 @@ "probe_samples_per_label": 8 }, "scores": { - "nano": 0.8671910624715002, - "mini": 0.8600091199270405 + "nano": 0.9491792065663475, + "mini": 0.9140218878248975 }, "metrics": { "nano": { "scores_per_experiment": [ { - "accuracy": 0.8492932056543547, - "f1": 0.8506491551806685, - "f1_weighted": 0.8484338913731928, - "precision": 0.847171519646365, - "precision_weighted": 0.857621901965629, - "recall": 0.8634426315569471, - "recall_weighted": 0.8492932056543547, + "accuracy": 0.9507523939808481, + "f1": 0.9490862282573499, + "f1_weighted": 0.9508003256061616, + "precision": 0.9476949859831407, + "precision_weighted": 0.9520734016967529, + "recall": 0.9517099502990664, + "recall_weighted": 0.9507523939808481, "ap": null, "ap_weighted": null }, { - "accuracy": 0.8593251253989969, - "f1": 0.8525570927270549, - "f1_weighted": 0.8588269042231532, - "precision": 0.849564509883911, - "precision_weighted": 0.8659425848467468, - "recall": 0.8649266896842414, - "recall_weighted": 0.8593251253989969, + "accuracy": 0.9473324213406292, + "f1": 0.9425946320317333, + "f1_weighted": 0.9473133266787116, + "precision": 0.9392105665237355, + "precision_weighted": 0.9479986101450497, + "recall": 0.9468765008841218, + "recall_weighted": 0.9473324213406292, "ap": null, "ap_weighted": null }, { - "accuracy": 0.8508891928864569, - "f1": 0.8445077740960396, - "f1_weighted": 0.8500136039882616, - "precision": 0.83976876536009, - "precision_weighted": 0.8551546537983608, - "recall": 0.8573297788521569, - "recall_weighted": 0.8508891928864569, + "accuracy": 0.9306885544915641, + "f1": 0.9261678157710709, + "f1_weighted": 0.9296159823715927, + "precision": 0.923024327342814, + "precision_weighted": 0.9307965712480242, + "recall": 0.932201810783149, + "recall_weighted": 0.9306885544915641, "ap": null, "ap_weighted": null }, { - "accuracy": 0.8702690378476973, - "f1": 0.8680982220684261, - "f1_weighted": 0.8688890737378698, - "precision": 0.8651925230157509, - "precision_weighted": 0.8699395458498034, - "recall": 0.873034481111479, - "recall_weighted": 0.8702690378476973, + "accuracy": 0.958048335613315, + "f1": 0.9552869971485034, + "f1_weighted": 0.9579502405954128, + "precision": 0.954624514001619, + "precision_weighted": 0.9582682331625527, + "recall": 0.9564289859937923, + "recall_weighted": 0.958048335613315, "ap": null, "ap_weighted": null }, { - "accuracy": 0.8657090743274054, - "f1": 0.8639593934313017, - "f1_weighted": 0.8646291417790959, - "precision": 0.8570893217526883, - "precision_weighted": 0.8672508842970594, - "recall": 0.8739690035000478, - "recall_weighted": 0.8657090743274054, + "accuracy": 0.9464204286365709, + "f1": 0.94433525009326, + "f1_weighted": 0.9462665825311201, + "precision": 0.9422860464299823, + "precision_weighted": 0.9467645700971977, + "recall": 0.9469874285322217, + "recall_weighted": 0.9464204286365709, "ap": null, "ap_weighted": null }, { - "accuracy": 0.8520291837665299, - "f1": 0.8534418954962738, - "f1_weighted": 0.8548114072639492, - "precision": 0.85875145236006, - "precision_weighted": 0.8803115046026259, - "recall": 0.869567932536695, - "recall_weighted": 0.8520291837665299, + "accuracy": 0.9505243958048336, + "f1": 0.9447844972663006, + "f1_weighted": 0.9507789369812125, + "precision": 0.9429136617255864, + "precision_weighted": 0.9524852717381878, + "recall": 0.9483171314260196, + "recall_weighted": 0.9505243958048336, "ap": null, "ap_weighted": null }, { - "accuracy": 0.8869129046967624, - "f1": 0.8849494340089117, - "f1_weighted": 0.8871874824644057, - "precision": 0.8779834780597185, - "precision_weighted": 0.8955288700582439, - "recall": 0.9009013020211545, - "recall_weighted": 0.8869129046967624, + "accuracy": 0.9509803921568627, + "f1": 0.9480693879894481, + "f1_weighted": 0.9510087641402198, + "precision": 0.9447979434260977, + "precision_weighted": 0.9522624035738166, + "recall": 0.952376748039714, + "recall_weighted": 0.9509803921568627, "ap": null, "ap_weighted": null }, { - "accuracy": 0.875968992248062, - "f1": 0.8715420414043028, - "f1_weighted": 0.8753897998685723, - "precision": 0.86652024620862, - "precision_weighted": 0.8811614860860878, - "recall": 0.881802359118481, - "recall_weighted": 0.875968992248062, + "accuracy": 0.9573643410852714, + "f1": 0.9551153583299494, + "f1_weighted": 0.9573535659550753, + "precision": 0.952401865813031, + "precision_weighted": 0.9583040343256545, + "recall": 0.9586994420587002, + "recall_weighted": 0.9573643410852714, "ap": null, "ap_weighted": null }, { - "accuracy": 0.8755129958960328, - "f1": 0.8710791039688399, - "f1_weighted": 0.8760974709416987, - "precision": 0.8647173948263537, - "precision_weighted": 0.8831121050674345, - "recall": 0.8845140142042784, - "recall_weighted": 0.8755129958960328, + "accuracy": 0.9548563611491108, + "f1": 0.9495763118202308, + "f1_weighted": 0.9549499907192655, + "precision": 0.946851274794657, + "precision_weighted": 0.9555638233419986, + "recall": 0.9529512436069031, + "recall_weighted": 0.9548563611491108, "ap": null, "ap_weighted": null }, { - "accuracy": 0.886000911992704, - "f1": 0.8827257199119498, - "f1_weighted": 0.8855721338303499, - "precision": 0.8760471482914741, - "precision_weighted": 0.8912159468319844, - "recall": 0.8952867706565555, - "recall_weighted": 0.886000911992704, + "accuracy": 0.9448244414044688, + "f1": 0.9427993333346522, + "f1_weighted": 0.9447713522464498, + "precision": 0.9432072583992069, + "precision_weighted": 0.9468986783847072, + "recall": 0.9447479153319868, + "recall_weighted": 0.9448244414044688, "ap": null, "ap_weighted": null } ], - "accuracy": 0.8671910624715002, - "f1": 0.8643509832293768, - "f1_weighted": 0.8669850909470549, - "precision": 0.8602806359405031, - "precision_weighted": 0.8747239483403975, - "recall": 0.8764774963242037, - "recall_weighted": 0.8671910624715002, + "accuracy": 0.9491792065663475, + "f1": 0.9457815812042497, + "f1_weighted": 0.9490809067825221, + "precision": 0.9437012444439871, + "precision_weighted": 0.9501415597713943, + "recall": 0.9491297156955675, + "recall_weighted": 0.9491792065663475, "ap": null, "ap_weighted": null, - "main_score": 0.8671910624715002, + "main_score": 0.9491792065663475, "hf_subset": "en", "languages": [ "eng-Latn" @@ -1657,133 +1498,133 @@ "mteb_version": "2.21.0" }, "mini": { + "accuracy": 0.9140218878248975, + "ap": null, + "ap_weighted": null, + "f1": 0.9102311028772032, + "f1_weighted": 0.9135601666191583, + "hf_subset": "en", + "languages": [ + "eng-Latn" + ], + "main_score": 0.9140218878248975, + "mteb_version": "2.21.0", + "precision": 0.908374503175667, + "precision_weighted": 0.9159330806711703, + "recall": 0.9149431630249987, + "recall_weighted": 0.9140218878248975, "scores_per_experiment": [ { - "accuracy": 0.8513451892384861, - "f1": 0.8509247338652547, - "f1_weighted": 0.8521527005277307, - "precision": 0.8492202894031071, - "precision_weighted": 0.864855059096428, - "recall": 0.8648355427309645, - "recall_weighted": 0.8513451892384861, + "accuracy": 0.9293205654354765, "ap": null, - "ap_weighted": null + "ap_weighted": null, + "f1": 0.9259418606390145, + "f1_weighted": 0.9292254772834052, + "precision": 0.9236746300229985, + "precision_weighted": 0.9303279843296995, + "recall": 0.9293068230638082, + "recall_weighted": 0.9293205654354765 }, { - "accuracy": 0.8650250797993616, - "f1": 0.8586976403069525, - "f1_weighted": 0.8655217950862607, - "precision": 0.859417882373164, - "precision_weighted": 0.8709859478064751, - "recall": 0.8644600886823848, - "recall_weighted": 0.8650250797993616, + "accuracy": 0.9222526219790241, "ap": null, - "ap_weighted": null + "ap_weighted": null, + "f1": 0.9176218274169997, + "f1_weighted": 0.9215729146494458, + "precision": 0.9172049643942963, + "precision_weighted": 0.922112545996546, + "recall": 0.9194863278550411, + "recall_weighted": 0.9222526219790241 }, { - "accuracy": 0.8620611035111719, - "f1": 0.8552436164120518, - "f1_weighted": 0.8617173770687293, - "precision": 0.8502697390874757, - "precision_weighted": 0.86811806179067, - "recall": 0.8699849171497175, - "recall_weighted": 0.8620611035111719, + "accuracy": 0.9046967624259006, "ap": null, - "ap_weighted": null + "ap_weighted": null, + "f1": 0.9008301428673469, + "f1_weighted": 0.90324585696942, + "precision": 0.8956318560429776, + "precision_weighted": 0.9038104887210681, + "recall": 0.9081680378975933, + "recall_weighted": 0.9046967624259006 }, { - "accuracy": 0.8766529867761058, - "f1": 0.8705128792667853, - "f1_weighted": 0.8757214075056072, - "precision": 0.87316769782497, - "precision_weighted": 0.8767318725122171, - "recall": 0.8701675544281682, - "recall_weighted": 0.8766529867761058, + "accuracy": 0.9247606019151847, "ap": null, - "ap_weighted": null + "ap_weighted": null, + "f1": 0.9206828862018472, + "f1_weighted": 0.9244439447593147, + "precision": 0.9216061453974003, + "precision_weighted": 0.9252191059148221, + "recall": 0.9211403915714979, + "recall_weighted": 0.9247606019151847 }, { - "accuracy": 0.8543091655266758, - "f1": 0.8492230242084531, - "f1_weighted": 0.8515865514378859, - "precision": 0.8456776748794668, - "precision_weighted": 0.8525101783754875, - "recall": 0.8573596646596781, - "recall_weighted": 0.8543091655266758, + "accuracy": 0.9192886456908345, "ap": null, - "ap_weighted": null + "ap_weighted": null, + "f1": 0.9180511523699185, + "f1_weighted": 0.9185946035149317, + "precision": 0.9179036900043184, + "precision_weighted": 0.922774689921659, + "recall": 0.9230787101853277, + "recall_weighted": 0.9192886456908345 }, { - "accuracy": 0.8381212950296397, - "f1": 0.8423239061875581, - "f1_weighted": 0.8406760430631188, - "precision": 0.8570748104639369, - "precision_weighted": 0.8724745768326015, - "recall": 0.8552650856577653, - "recall_weighted": 0.8381212950296397, + "accuracy": 0.903328773369813, "ap": null, - "ap_weighted": null + "ap_weighted": null, + "f1": 0.8998428103070271, + "f1_weighted": 0.9032982960491259, + "precision": 0.8986478562698227, + "precision_weighted": 0.9066388154730338, + "recall": 0.9043507125206696, + "recall_weighted": 0.903328773369813 }, { - "accuracy": 0.8684450524395805, - "f1": 0.8644968735100075, - "f1_weighted": 0.8690876911265336, - "precision": 0.8593299356814467, - "precision_weighted": 0.8752537969844317, - "recall": 0.8755653527627548, - "recall_weighted": 0.8684450524395805, + "accuracy": 0.905608755129959, "ap": null, - "ap_weighted": null + "ap_weighted": null, + "f1": 0.9015987506730646, + "f1_weighted": 0.9049297748787466, + "precision": 0.8987728785349184, + "precision_weighted": 0.9070418385617854, + "recall": 0.9068414370623596, + "recall_weighted": 0.905608755129959 }, { - "accuracy": 0.8679890560875513, - "f1": 0.8624731791673977, - "f1_weighted": 0.8672975210406725, - "precision": 0.8621948292766003, - "precision_weighted": 0.8690363827942069, - "recall": 0.8653801474899807, - "recall_weighted": 0.8679890560875513, + "accuracy": 0.920656634746922, "ap": null, - "ap_weighted": null + "ap_weighted": null, + "f1": 0.9170483919986423, + "f1_weighted": 0.9204468928041081, + "precision": 0.9158737117986611, + "precision_weighted": 0.9221089996958914, + "recall": 0.9202483568175329, + "recall_weighted": 0.920656634746922 }, { - "accuracy": 0.8433652530779754, - "f1": 0.8369678685556871, - "f1_weighted": 0.8436804643935586, - "precision": 0.8374706560586799, - "precision_weighted": 0.8618829308349459, - "recall": 0.8524633984515572, - "recall_weighted": 0.8433652530779754, + "accuracy": 0.9031007751937985, "ap": null, - "ap_weighted": null + "ap_weighted": null, + "f1": 0.8973501099256722, + "f1_weighted": 0.9028712379103624, + "precision": 0.8937404468938929, + "precision_weighted": 0.9077436387795031, + "recall": 0.9058160195105333, + "recall_weighted": 0.9031007751937985 }, { - "accuracy": 0.8727770177838577, - "f1": 0.8676769115909195, - "f1_weighted": 0.8724576521970784, - "precision": 0.8630033237258433, - "precision_weighted": 0.875289367479208, - "recall": 0.8767720138319759, - "recall_weighted": 0.8727770177838577, + "accuracy": 0.9072047423620611, "ap": null, - "ap_weighted": null + "ap_weighted": null, + "f1": 0.903343096372501, + "f1_weighted": 0.9069726673727218, + "precision": 0.9006888523973842, + "precision_weighted": 0.9115526993176939, + "recall": 0.9109948137656244, + "recall_weighted": 0.9072047423620611 } - ], - "accuracy": 0.8600091199270405, - "f1": 0.8558540633071068, - "f1_weighted": 0.8599899203447177, - "precision": 0.8556826838774689, - "precision_weighted": 0.8687138174506671, - "recall": 0.8652253765844946, - "recall_weighted": 0.8600091199270405, - "ap": null, - "ap_weighted": null, - "main_score": 0.8600091199270405, - "hf_subset": "en", - "languages": [ - "eng-Latn" - ], - "mteb_version": "2.21.0" + ] } } }, @@ -1886,78 +1727,78 @@ "mteb_version": "2.21.0" }, "mini": { + "accuracy": 0.9226492232215863, + "ap": null, + "ap_weighted": null, + "f1": 0.9203716879784526, + "f1_weighted": 0.9208665388092886, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.9226492232215863, + "mteb_version": "2.21.0", + "precision": 0.9306201709007512, + "precision_weighted": 0.9317555157109533, + "recall": 0.92289418751056, + "recall_weighted": 0.9226492232215863, "scores_per_experiment": [ { "accuracy": 0.9116925592804579, + "ap": null, + "ap_weighted": null, "f1": 0.9093729214948953, "f1_weighted": 0.9097909562101416, "precision": 0.9202666715986694, "precision_weighted": 0.9214317847552188, "recall": 0.9121512634063311, - "recall_weighted": 0.9116925592804579, - "ap": null, - "ap_weighted": null + "recall_weighted": 0.9116925592804579 }, { "accuracy": 0.91632597437994, + "ap": null, + "ap_weighted": null, "f1": 0.913359872197621, "f1_weighted": 0.9138906410509396, "precision": 0.9263974728959862, "precision_weighted": 0.9277066224314515, "recall": 0.9166933943457583, - "recall_weighted": 0.91632597437994, - "ap": null, - "ap_weighted": null + "recall_weighted": 0.91632597437994 }, { "accuracy": 0.9296811120196239, + "ap": null, + "ap_weighted": null, "f1": 0.927185635992295, "f1_weighted": 0.9278041399803281, "precision": 0.938292438925259, "precision_weighted": 0.9396495102897695, "recall": 0.9299617386197123, - "recall_weighted": 0.9296811120196239, - "ap": null, - "ap_weighted": null + "recall_weighted": 0.9296811120196239 }, { "accuracy": 0.9242300354319978, + "ap": null, + "ap_weighted": null, "f1": 0.9210816317727427, "f1_weighted": 0.9216606459296687, "precision": 0.9322521793365194, "precision_weighted": 0.9334978229495479, "recall": 0.9244216090442365, - "recall_weighted": 0.9242300354319978, - "ap": null, - "ap_weighted": null + "recall_weighted": 0.9242300354319978 }, { "accuracy": 0.9313164349959117, + "ap": null, + "ap_weighted": null, "f1": 0.9308583784347093, "f1_weighted": 0.9311863108753653, "precision": 0.935892091747322, "precision_weighted": 0.9364918381287788, "recall": 0.9312429321367615, - "recall_weighted": 0.9313164349959117, - "ap": null, - "ap_weighted": null + "recall_weighted": 0.9313164349959117 } - ], - "accuracy": 0.9226492232215863, - "f1": 0.9203716879784526, - "f1_weighted": 0.9208665388092886, - "precision": 0.9306201709007512, - "precision_weighted": 0.9317555157109533, - "recall": 0.92289418751056, - "recall_weighted": 0.9226492232215863, - "ap": null, - "ap_weighted": null, - "main_score": 0.9226492232215863, - "hf_subset": "default", - "languages": [ - "eng-Latn" - ], - "mteb_version": "2.21.0" + ] } } }, @@ -1995,16 +1836,16 @@ "mteb_version": "2.21.0" }, "mini": { - "v_measure": 0.75554581177099, - "nmi": 0.7555458117709901, "ari": 0.3859506107046988, "cluster_accuracy": 0.5567, - "main_score": 0.7555458117709901, "hf_subset": "default", "languages": [ "eng-Latn" ], - "mteb_version": "2.21.0" + "main_score": 0.7555458117709901, + "mteb_version": "2.21.0", + "nmi": 0.7555458117709901, + "v_measure": 0.75554581177099 } } }, @@ -2027,21 +1868,21 @@ "fixed_text_labels": 37 }, "scores": { - "nano": 0.1163804851458163, - "mini": 0.09566639411283728 + "nano": 0.08013082583810302, + "mini": 0.1264649768329245 }, "metrics": { "nano": { - "accuracy": 0.1163804851458163, - "f1": 0.085822378073852, - "f1_weighted": 0.08530524305774022, - "precision": 0.08885688520073651, - "precision_weighted": 0.08856851170630324, - "recall": 0.11709192030781942, - "recall_weighted": 0.1163804851458163, + "accuracy": 0.08013082583810302, + "f1": 0.0611345635215895, + "f1_weighted": 0.06128470828202288, + "precision": 0.1359645526412158, + "precision_weighted": 0.13671480699577512, + "recall": 0.0798783003173134, + "recall_weighted": 0.08013082583810302, "ap": null, "ap_weighted": null, - "main_score": 0.1163804851458163, + "main_score": 0.08013082583810302, "hf_subset": "default", "languages": [ "eng-Latn" @@ -2049,21 +1890,21 @@ "mteb_version": "2.21.0" }, "mini": { - "accuracy": 0.09566639411283728, - "f1": 0.055047148656272936, - "f1_weighted": 0.05538822267324211, - "precision": 0.07153585831531258, - "precision_weighted": 0.07157245637321427, - "recall": 0.09493440324901, - "recall_weighted": 0.09566639411283728, + "accuracy": 0.1264649768329245, "ap": null, "ap_weighted": null, - "main_score": 0.09566639411283728, + "f1": 0.08819096015243695, + "f1_weighted": 0.08875300942790583, "hf_subset": "default", "languages": [ "eng-Latn" ], - "mteb_version": "2.21.0" + "main_score": 0.1264649768329245, + "mteb_version": "2.21.0", + "precision": 0.1343898903035731, + "precision_weighted": 0.13539679070300115, + "recall": 0.12572515314088348, + "recall_weighted": 0.1264649768329245 } } }, @@ -2087,7 +1928,7 @@ }, "scores": { "nano": 0.29763670140167686, - "mini": 0.21700995472172274 + "mini": 0.21795099547217225 }, "metrics": { "nano": { @@ -2165,78 +2006,78 @@ "mteb_version": "2.21.0" }, "mini": { + "accuracy": 0.21795099547217225, + "ap": null, + "ap_weighted": null, + "f1": 0.21411225654211394, + "f1_weighted": 0.21441037459344017, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.21795099547217225, + "mteb_version": "2.21.0", + "precision": 0.22482947419473298, + "precision_weighted": 0.2256504251933237, + "recall": 0.2184925649825324, + "recall_weighted": 0.21795099547217225, "scores_per_experiment": [ { "accuracy": 0.21893888515782403, - "f1": 0.2163810638240982, - "f1_weighted": 0.21643889141270933, - "precision": 0.23012618754096079, - "precision_weighted": 0.231091560350251, - "recall": 0.2194242335392654, - "recall_weighted": 0.21893888515782403, "ap": null, - "ap_weighted": null + "ap_weighted": null, + "f1": 0.21690967388030988, + "f1_weighted": 0.21712602671013928, + "precision": 0.2300670425006146, + "precision_weighted": 0.23061158584485825, + "recall": 0.21902691937636262, + "recall_weighted": 0.21893888515782403 }, { - "accuracy": 0.2269979852249832, - "f1": 0.22514611796351458, - "f1_weighted": 0.22499154241770708, - "precision": 0.23620907852470988, - "precision_weighted": 0.23781826575997397, - "recall": 0.22968978054909028, - "recall_weighted": 0.2269979852249832, + "accuracy": 0.2256548018804567, "ap": null, - "ap_weighted": null + "ap_weighted": null, + "f1": 0.22392783012000228, + "f1_weighted": 0.2239076962805371, + "precision": 0.23281424390975716, + "precision_weighted": 0.23463096121359348, + "recall": 0.22805866893508786, + "recall_weighted": 0.2256548018804567 }, { - "accuracy": 0.1989247311827957, - "f1": 0.19641740270113128, - "f1_weighted": 0.1971013872432, - "precision": 0.1990141667072113, - "precision_weighted": 0.1992048597710001, - "recall": 0.197960110151026, - "recall_weighted": 0.1989247311827957, + "accuracy": 0.20228494623655913, "ap": null, - "ap_weighted": null + "ap_weighted": null, + "f1": 0.19976499959743785, + "f1_weighted": 0.2005608365597875, + "precision": 0.20310978209068556, + "precision_weighted": 0.20340095458657095, + "recall": 0.20109924019064773, + "recall_weighted": 0.20228494623655913 }, { - "accuracy": 0.2103494623655914, - "f1": 0.20288276044167974, - "f1_weighted": 0.20260049471624383, - "precision": 0.2265501279266561, - "precision_weighted": 0.22499018704680598, - "recall": 0.21118914553810386, - "recall_weighted": 0.2103494623655914, + "accuracy": 0.21303763440860216, "ap": null, - "ap_weighted": null + "ap_weighted": null, + "f1": 0.2058164390039309, + "f1_weighted": 0.20561048978468363, + "precision": 0.22470030024844204, + "precision_weighted": 0.22322426630307032, + "recall": 0.21351405866510034, + "recall_weighted": 0.21303763440860216 }, { "accuracy": 0.22983870967741934, - "f1": 0.2249779898205239, - "f1_weighted": 0.22528396587445315, - "precision": 0.23399902471576128, - "precision_weighted": 0.2366606742233258, - "recall": 0.23136899419433946, - "recall_weighted": 0.22983870967741934, "ap": null, - "ap_weighted": null + "ap_weighted": null, + "f1": 0.22414234010888875, + "f1_weighted": 0.22484682363205344, + "precision": 0.23345600222416554, + "precision_weighted": 0.23638435801852556, + "recall": 0.23076393774546353, + "recall_weighted": 0.22983870967741934 } - ], - "accuracy": 0.21700995472172274, - "f1": 0.21316106695018955, - "f1_weighted": 0.2132832563328627, - "precision": 0.22517971708305984, - "precision_weighted": 0.22595310943027136, - "recall": 0.217926452794365, - "recall_weighted": 0.21700995472172274, - "ap": null, - "ap_weighted": null, - "main_score": 0.21700995472172274, - "hf_subset": "default", - "languages": [ - "eng-Latn" - ], - "mteb_version": "2.21.0" + ] } } }, @@ -2260,7 +2101,7 @@ }, "scores": { "nano": 0.023601715918443712, - "mini": 0.0056023574446678595 + "mini": 0.005355272531678118 }, "metrics": { "nano": { @@ -2304,44 +2145,44 @@ "mteb_version": "2.21.0" }, "mini": { - "v_measures": { - "Level 0": [ - 0.005289211932003536, - 0.006015061067516978, - 0.00585616389349341, - 0.0047416312293699515, - 0.006024239951412609, - 0.006419682693312149, - 0.005562929968573973, - 0.005527689308882548, - 0.005252015253790995, - 0.005334949148322441 - ] - }, - "v_measure": 0.0056023574446678595, - "v_measure_std": 0.0004605468314064617, + "ami": 0.004917751625959246, "ami_scores": { "Level 0": [ - 0.0048571187069303915, - 0.0055751027826630006, - 0.0054260622718215485, - 0.004294680111053968, - 0.005583369086893376, - 0.005970317949926442, - 0.005124865072190019, - 0.005097099789936521, - 0.004814508590607281, - 0.004882314064688779 + 0.006002163134283694, + 0.004301228007145657, + 0.004964278458743201, + 0.005602365904695249, + 0.005597019480654666, + 0.0063906895027782875, + 0.004817247420836509, + 0.0037430972644923056, + 0.003962836859700492, + 0.003796590226262398 ] }, - "ami": 0.005162543842671133, - "ami_std": 0.0004605509293362838, - "main_score": 0.0056023574446678595, + "ami_std": 0.0009064568944687993, "hf_subset": "default", "languages": [ "eng-Latn" ], - "mteb_version": "2.21.0" + "main_score": 0.005355272531678118, + "mteb_version": "2.21.0", + "v_measure": 0.005355272531678118, + "v_measure_std": 0.0009032459696080586, + "v_measures": { + "Level 0": [ + 0.006441320725518733, + 0.004731546591920122, + 0.005401001698436892, + 0.006032792790334663, + 0.00603480422262663, + 0.006823368977853802, + 0.005252141740252315, + 0.004193268290377624, + 0.00440019248252484, + 0.00424228779693555 + ] + } } } }, @@ -2364,21 +2205,21 @@ "fixed_text_labels": 10 }, "scores": { - "nano": 0.14972999509081983, + "nano": 0.1411389297987236, "mini": 0.41998036327933236 }, "metrics": { "nano": { - "accuracy": 0.14972999509081983, - "f1": 0.06986246632697812, - "f1_weighted": 0.07083899432793754, - "precision": 0.15715381051614374, - "precision_weighted": 0.1573685095178851, - "recall": 0.14663865718054886, - "recall_weighted": 0.14972999509081983, + "accuracy": 0.1411389297987236, + "f1": 0.06629702019800886, + "f1_weighted": 0.06559730133350446, + "precision": 0.21924492134747023, + "precision_weighted": 0.21974351147386872, + "recall": 0.14280319771568778, + "recall_weighted": 0.1411389297987236, "ap": null, "ap_weighted": null, - "main_score": 0.14972999509081983, + "main_score": 0.1411389297987236, "hf_subset": "default", "languages": [ "eng-Latn" @@ -2387,224 +2228,457 @@ }, "mini": { "accuracy": 0.41998036327933236, - "f1": 0.32093074584252623, - "f1_weighted": 0.32423854890642734, - "precision": 0.2982950429312295, - "precision_weighted": 0.30077500323366896, - "recall": 0.41602176893918197, - "recall_weighted": 0.41998036327933236, "ap": null, "ap_weighted": null, - "main_score": 0.41998036327933236, + "f1": 0.36673766539054486, + "f1_weighted": 0.3649764058710554, "hf_subset": "default", "languages": [ "eng-Latn" ], - "mteb_version": "2.21.0" + "main_score": 0.41998036327933236, + "mteb_version": "2.21.0", + "precision": 0.5160762176036948, + "precision_weighted": 0.5131299699947005, + "recall": 0.4204235240846076, + "recall_weighted": 0.41998036327933236 } } } ], - "text_truncation": { - "SICK-R/nano": { - "text_rows": 19854, - "text_rows_truncated": 0, - "image_rows": 0, - "audio_rows": 0 - }, - "SICK-R/mini": { - "text_rows": 19854, - "text_rows_truncated": 0, - "image_rows": 0, - "audio_rows": 0 - }, - "STSBenchmark/nano": { - "text_rows": 2758, - "text_rows_truncated": 0, - "image_rows": 0, - "audio_rows": 0 - }, - "STSBenchmark/mini": { - "text_rows": 2758, - "text_rows_truncated": 0, - "image_rows": 0, - "audio_rows": 0 - }, - "ArguAna/nano": { - "text_rows": 10080, - "text_rows_truncated": 222, - "image_rows": 0, - "audio_rows": 0 - }, - "ArguAna/mini": { - "text_rows": 10080, - "text_rows_truncated": 0, - "image_rows": 0, - "audio_rows": 0 - }, - "TwentyNewsgroupsClustering.v2/nano": { - "text_rows": 2381, - "text_rows_truncated": 0, - "image_rows": 0, - "audio_rows": 0 - }, - "TwentyNewsgroupsClustering.v2/mini": { - "text_rows": 2381, - "text_rows_truncated": 0, - "image_rows": 0, - "audio_rows": 0 - }, - "MTOPDomainClassification/nano": { - "text_rows": 8349, - "text_rows_truncated": 0, - "image_rows": 0, - "audio_rows": 0 - }, - "MTOPDomainClassification/mini": { - "text_rows": 8349, - "text_rows_truncated": 0, - "image_rows": 0, - "audio_rows": 0 - }, - "OxfordPets/nano": { - "text_rows": 0, - "text_rows_truncated": 0, - "image_rows": 5816, - "audio_rows": 0 - }, - "OxfordPets/mini": { - "text_rows": 0, - "text_rows_truncated": 0, - "image_rows": 5816, - "audio_rows": 0 - }, - "TinyImageNetClustering/nano": { - "text_rows": 0, - "text_rows_truncated": 0, - "image_rows": 10000, - "audio_rows": 0 - }, - "TinyImageNetClustering/mini": { - "text_rows": 0, - "text_rows_truncated": 0, - "image_rows": 10000, - "audio_rows": 0 - }, - "OxfordPetsZeroShot/nano": { - "text_rows": 37, - "text_rows_truncated": 0, - "image_rows": 3669, - "audio_rows": 0 - }, - "OxfordPetsZeroShot/mini": { - "text_rows": 37, - "text_rows_truncated": 0, - "image_rows": 3669, - "audio_rows": 0 - }, - "CREMA_D/nano": { - "text_rows": 0, - "text_rows_truncated": 0, - "image_rows": 0, - "audio_rows": 7442 - }, - "CREMA_D/mini": { - "text_rows": 0, - "text_rows_truncated": 0, - "image_rows": 0, - "audio_rows": 7442 - }, - "CREMA_DClustering/nano": { - "text_rows": 0, - "text_rows_truncated": 0, - "image_rows": 0, - "audio_rows": 2048 - }, - "CREMA_DClustering/mini": { - "text_rows": 0, - "text_rows_truncated": 0, - "image_rows": 0, - "audio_rows": 2048 - }, - "SpeechCommandsZeroshotv0.02/nano": { - "text_rows": 10, - "text_rows_truncated": 0, - "image_rows": 0, - "audio_rows": 4074 - }, - "SpeechCommandsZeroshotv0.02/mini": { - "text_rows": 10, - "text_rows_truncated": 0, - "image_rows": 0, - "audio_rows": 4074 - } - }, "comparison": { "baseline_model": "llm-semantic-router/Vela-1.0-Omni-Nano", - "baseline_revision": "70c3412bb0ddfb19386d673d4a102c753a574f1b", - "baseline_native_artifact_sha256": "b596f1dbab89369555e0a72a0d6854d780e305686750949b308c8f167993bdeb", - "card_bold": "In the standard task table, bold marks the higher unrounded score between current Omni Nano and Mini. Baseline release differences are separately identified." - }, - "additional_comparisons": { - "mini": { - "baseline_model": "llm-semantic-router/Vela-1.0-Omni-Mini", - "baseline_revision": "b5c1674f7630930f908687c62e6ca428205d9729", - "baseline_native_artifact_sha256": "3f7fbf0dc1055a31a03a733fbc8c31fecd88575b9b507fda995a70e011781748", - "card_bold": "Mini primary score strictly improves over the previous Mini release; unrounded values." - } - } - }, - "capabilities": { - "max_text_tokens": 512, - "tokens_include_special_tokens": true, - "public_text_overflow": "Raises ValueError; shorten or explicitly chunk longer inputs.", - "embedding_dimensions": 384, - "parameters": 133956480 - }, - "efficiency": { - "attention": "native SDPA", - "batch_size": 8, - "embedding_dtype": "float32", - "hardware": "AMD gfx942 GPU", - "latency_includes": "Native public encode_text: tokenization, host-to-device transfer and encoding; excludes model loading.", - "measured_batches": 20, - "measurements": { - "128": { - "batch_size": 8, - "p50_ms": 4.283615489839576, - "p95_ms": 4.314010989037342, - "peak_allocated_bytes": 653202944, - "peak_reserved_bytes": 673185792, - "tokens_each_including_special": 128 - }, - "512": { - "batch_size": 8, - "p50_ms": 12.445501997717656, - "p95_ms": 12.603181986196432, - "peak_allocated_bytes": 878682112, - "peak_reserved_bytes": 970981376, - "tokens_each_including_special": 512 - } + "baseline_revision": "3a1efc48995e2b71622d65c6cf71cfc00bbf3aa3", + "baseline_native_artifact_sha256": "fb9cd39c9980a085b75456c715c2295b997784fe733314be68c13635f6384edf", + "card_bold": "Nano gains are relative to the named previous Nano release; cross-model columns are separately identified." }, - "memory_scope": "Process CUDA allocator peak with the complete multimodal model loaded.", - "model_revision": "70c3412bb0ddfb19386d673d4a102c753a574f1b", - "native_artifact_sha256": "b596f1dbab89369555e0a72a0d6854d780e305686750949b308c8f167993bdeb", - "scope": "Synthetic repeated text-batch measurements on one GPU; not end-to-end service latency.", - "tf32": false, - "token_lengths_include_special_tokens": true, - "torch_version_family": "2.12", - "transformers_version": "4.57.6", - "warmup_batches": 2, - "status": "historical_previous_release_measurement", - "applicability": "Measured on the complete previous native artifact; not a measurement or efficiency claim for this image update." - }, - "previous_release": { - "model": "llm-semantic-router/Vela-1.0-Omni-Nano", - "revision": "70c3412bb0ddfb19386d673d4a102c753a574f1b", - "native_artifact_sha256": "b596f1dbab89369555e0a72a0d6854d780e305686750949b308c8f167993bdeb", - "metrics": { - "audio.audio_to_text.recall@1": { - "denominator": 2611, + "observation_scope": "Only actual current encoding observations are included; earlier tokenizer/truncation statistics are not inherited by the new text towers.", + "encoding_observations": { + "SICK-R": { + "nano": { + "before": { + "text_rows": 0, + "text_rows_truncated": 0, + "max_original_text_tokens": 0, + "image_rows": 0, + "audio_rows": 0 + }, + "after": { + "text_rows": 19854, + "text_rows_truncated": 0, + "max_original_text_tokens": 38, + "image_rows": 0, + "audio_rows": 0 + }, + "elapsed_seconds": 10.807591969991336, + "peak_allocated_gpu_bytes": 679632896, + "peak_reserved_gpu_bytes": 719323136 + }, + "mini": { + "cumulative_rows": { + "audio_rows": 0, + "image_rows": 0, + "max_original_text_tokens": 37, + "text_rows": 19854, + "text_rows_truncated": 0 + }, + "peak_allocated_bytes": 5528268288, + "peak_reserved_bytes": 5672796160, + "seconds": 48.09956211101962 + } + }, + "STSBenchmark": { + "nano": { + "before": { + "text_rows": 19854, + "text_rows_truncated": 0, + "max_original_text_tokens": 38, + "image_rows": 0, + "audio_rows": 0 + }, + "after": { + "text_rows": 22612, + "text_rows_truncated": 0, + "max_original_text_tokens": 46, + "image_rows": 0, + "audio_rows": 0 + }, + "elapsed_seconds": 1.294741225981852, + "peak_allocated_gpu_bytes": 681261056, + "peak_reserved_gpu_bytes": 723517440 + }, + "mini": { + "cumulative_rows": { + "audio_rows": 0, + "image_rows": 0, + "max_original_text_tokens": 55, + "text_rows": 22612, + "text_rows_truncated": 0 + }, + "peak_allocated_bytes": 5566614016, + "peak_reserved_bytes": 5714739200, + "seconds": 6.265706841979409 + } + }, + "ArguAna": { + "nano": { + "before": { + "text_rows": 22612, + "text_rows_truncated": 0, + "max_original_text_tokens": 46, + "image_rows": 0, + "audio_rows": 0 + }, + "after": { + "text_rows": 32692, + "text_rows_truncated": 222, + "max_original_text_tokens": 1401, + "image_rows": 0, + "audio_rows": 0 + }, + "elapsed_seconds": 15.392282173008425, + "peak_allocated_gpu_bytes": 809891328, + "peak_reserved_gpu_bytes": 887095296 + }, + "mini": { + "cumulative_rows": { + "audio_rows": 0, + "image_rows": 0, + "max_original_text_tokens": 1398, + "text_rows": 32692, + "text_rows_truncated": 0 + }, + "peak_allocated_bytes": 8686103552, + "peak_reserved_bytes": 10506731520, + "seconds": 73.0957911729929 + } + }, + "TwentyNewsgroupsClustering.v2": { + "nano": { + "before": { + "text_rows": 32692, + "text_rows_truncated": 222, + "max_original_text_tokens": 1401, + "image_rows": 0, + "audio_rows": 0 + }, + "after": { + "text_rows": 35073, + "text_rows_truncated": 222, + "max_original_text_tokens": 1401, + "image_rows": 0, + "audio_rows": 0 + }, + "elapsed_seconds": 48.54122133200872, + "peak_allocated_gpu_bytes": 684661248, + "peak_reserved_gpu_bytes": 727711744 + }, + "mini": { + "cumulative_rows": { + "audio_rows": 0, + "image_rows": 0, + "max_original_text_tokens": 1398, + "text_rows": 35073, + "text_rows_truncated": 0 + }, + "peak_allocated_bytes": 5532500992, + "peak_reserved_bytes": 5672796160, + "seconds": 9.794138230005046 + } + }, + "MTOPDomainClassification": { + "nano": { + "before": { + "text_rows": 35073, + "text_rows_truncated": 222, + "max_original_text_tokens": 1401, + "image_rows": 0, + "audio_rows": 0 + }, + "after": { + "text_rows": 43422, + "text_rows_truncated": 222, + "max_original_text_tokens": 1401, + "image_rows": 0, + "audio_rows": 0 + }, + "elapsed_seconds": 25.26324965900858, + "peak_allocated_gpu_bytes": 679308288, + "peak_reserved_gpu_bytes": 719323136 + }, + "mini": { + "cumulative_rows": { + "audio_rows": 0, + "image_rows": 0, + "max_original_text_tokens": 1398, + "text_rows": 43422, + "text_rows_truncated": 0 + }, + "peak_allocated_bytes": 5526888960, + "peak_reserved_bytes": 5672796160, + "seconds": 43.97947172098793 + } + }, + "OxfordPets": { + "nano": { + "before": { + "text_rows": 43422, + "text_rows_truncated": 222, + "max_original_text_tokens": 1401, + "image_rows": 0, + "audio_rows": 0 + }, + "after": { + "text_rows": 43422, + "text_rows_truncated": 222, + "max_original_text_tokens": 1401, + "image_rows": 5816, + "audio_rows": 0 + }, + "elapsed_seconds": 54.48569396699895, + "peak_allocated_gpu_bytes": 996472320, + "peak_reserved_gpu_bytes": 1207959552 + }, + "mini": { + "cumulative_rows": { + "audio_rows": 0, + "image_rows": 5816, + "max_original_text_tokens": 1398, + "text_rows": 43422, + "text_rows_truncated": 0 + }, + "peak_allocated_bytes": 5766737408, + "peak_reserved_bytes": 6014631936, + "seconds": 78.51040079200175 + } + }, + "TinyImageNetClustering": { + "nano": { + "before": { + "text_rows": 43422, + "text_rows_truncated": 222, + "max_original_text_tokens": 1401, + "image_rows": 5816, + "audio_rows": 0 + }, + "after": { + "text_rows": 43422, + "text_rows_truncated": 222, + "max_original_text_tokens": 1401, + "image_rows": 15816, + "audio_rows": 0 + }, + "elapsed_seconds": 72.07803665398387, + "peak_allocated_gpu_bytes": 996472320, + "peak_reserved_gpu_bytes": 1117782016 + }, + "mini": { + "cumulative_rows": { + "audio_rows": 0, + "image_rows": 15816, + "max_original_text_tokens": 1398, + "text_rows": 43422, + "text_rows_truncated": 0 + }, + "peak_allocated_bytes": 5766737408, + "peak_reserved_bytes": 5932843008, + "seconds": 110.89981973200338 + } + }, + "OxfordPetsZeroShot": { + "nano": { + "before": { + "text_rows": 43422, + "text_rows_truncated": 222, + "max_original_text_tokens": 1401, + "image_rows": 15816, + "audio_rows": 0 + }, + "after": { + "text_rows": 43459, + "text_rows_truncated": 222, + "max_original_text_tokens": 1401, + "image_rows": 19485, + "audio_rows": 0 + }, + "elapsed_seconds": 32.662016286019934, + "peak_allocated_gpu_bytes": 996472320, + "peak_reserved_gpu_bytes": 1119879168 + }, + "mini": { + "cumulative_rows": { + "audio_rows": 0, + "image_rows": 19485, + "max_original_text_tokens": 1398, + "text_rows": 43459, + "text_rows_truncated": 0 + }, + "peak_allocated_bytes": 5766737408, + "peak_reserved_bytes": 5968494592, + "seconds": 46.64819714499754 + } + }, + "CREMA_D": { + "nano": { + "before": { + "text_rows": 43459, + "text_rows_truncated": 222, + "max_original_text_tokens": 1401, + "image_rows": 19485, + "audio_rows": 0 + }, + "after": { + "text_rows": 43459, + "text_rows_truncated": 222, + "max_original_text_tokens": 1401, + "image_rows": 19485, + "audio_rows": 7442 + }, + "elapsed_seconds": 135.41051389899803, + "peak_allocated_gpu_bytes": 971823104, + "peak_reserved_gpu_bytes": 1224736768 + }, + "mini": { + "cumulative_rows": { + "audio_rows": 7442, + "image_rows": 19485, + "max_original_text_tokens": 1398, + "text_rows": 43459, + "text_rows_truncated": 0 + }, + "peak_allocated_bytes": 6040899072, + "peak_reserved_bytes": 6874464256, + "seconds": 239.3669580439746 + } + }, + "CREMA_DClustering": { + "nano": { + "before": { + "text_rows": 43459, + "text_rows_truncated": 222, + "max_original_text_tokens": 1401, + "image_rows": 19485, + "audio_rows": 7442 + }, + "after": { + "text_rows": 43459, + "text_rows_truncated": 222, + "max_original_text_tokens": 1401, + "image_rows": 19485, + "audio_rows": 9490 + }, + "elapsed_seconds": 73.78414583200356, + "peak_allocated_gpu_bytes": 971823104, + "peak_reserved_gpu_bytes": 1224736768 + }, + "mini": { + "cumulative_rows": { + "audio_rows": 9490, + "image_rows": 19485, + "max_original_text_tokens": 1398, + "text_rows": 43459, + "text_rows_truncated": 0 + }, + "peak_allocated_bytes": 6040899072, + "peak_reserved_bytes": 6874464256, + "seconds": 61.47769567198702 + } + }, + "SpeechCommandsZeroshotv0.02": { + "nano": { + "before": { + "text_rows": 43459, + "text_rows_truncated": 222, + "max_original_text_tokens": 1401, + "image_rows": 19485, + "audio_rows": 9490 + }, + "after": { + "text_rows": 43469, + "text_rows_truncated": 222, + "max_original_text_tokens": 1401, + "image_rows": 19485, + "audio_rows": 13564 + }, + "elapsed_seconds": 63.873327202978544, + "peak_allocated_gpu_bytes": 971823104, + "peak_reserved_gpu_bytes": 1224736768 + }, + "mini": { + "cumulative_rows": { + "audio_rows": 13564, + "image_rows": 19485, + "max_original_text_tokens": 1398, + "text_rows": 43469, + "text_rows_truncated": 0 + }, + "peak_allocated_bytes": 6040899072, + "peak_reserved_bytes": 6878658560, + "seconds": 118.83011079698917 + } + } + } + }, + "capabilities": { + "max_text_tokens": 512, + "tokens_include_special_tokens": true, + "public_text_overflow": "Raises ValueError; shorten or explicitly chunk longer inputs.", + "embedding_dimensions": 384, + "parameters": 144554112 + }, + "efficiency": { + "status": "not_measured_for_this_release", + "parameters": 144554112, + "embedding_dimensions": 384, + "dense_fp32_index_bytes_per_vector": 1536, + "weights_safetensors_bytes": 578289664, + "scope": "No new latency or peak-memory measurement is claimed. Vector storage excludes index structures and metadata.", + "historical_measurement": { + "attention": "native SDPA", + "batch_size": 8, + "embedding_dtype": "float32", + "hardware": "AMD gfx942 GPU", + "latency_includes": "Native public encode_text: tokenization, host-to-device transfer and encoding; excludes model loading.", + "measured_batches": 20, + "measurements": { + "128": { + "batch_size": 8, + "p50_ms": 4.283615489839576, + "p95_ms": 4.314010989037342, + "peak_allocated_bytes": 653202944, + "peak_reserved_bytes": 673185792, + "tokens_each_including_special": 128 + }, + "512": { + "batch_size": 8, + "p50_ms": 12.445501997717656, + "p95_ms": 12.603181986196432, + "peak_allocated_bytes": 878682112, + "peak_reserved_bytes": 970981376, + "tokens_each_including_special": 512 + } + }, + "memory_scope": "Process CUDA allocator peak with the complete multimodal model loaded.", + "model_revision": "70c3412bb0ddfb19386d673d4a102c753a574f1b", + "native_artifact_sha256": "b596f1dbab89369555e0a72a0d6854d780e305686750949b308c8f167993bdeb", + "scope": "Synthetic repeated text-batch measurements on one GPU; not end-to-end service latency.", + "tf32": false, + "token_lengths_include_special_tokens": true, + "torch_version_family": "2.12", + "transformers_version": "4.57.6", + "warmup_batches": 2, + "status": "historical_previous_release_measurement", + "applicability": "Historical measurement on its recorded native artifact; not applicable to this text-backbone update." + } + }, + "previous_release": { + "model": "llm-semantic-router/Vela-1.0-Omni-Nano", + "revision": "3a1efc48995e2b71622d65c6cf71cfc00bbf3aa3", + "native_artifact_sha256": "fb9cd39c9980a085b75456c715c2295b997784fe733314be68c13635f6384edf", + "parameters": 133956480, + "metrics": { + "audio.audio_to_text.recall@1": { + "denominator": 2611, "numerator": 235, "value": 0.09000382995021065 }, @@ -2635,33 +2709,33 @@ }, "image.image_to_text.recall@1": { "denominator": 823, - "numerator": 389, - "value": 0.4726609963547995 + "numerator": 540, + "value": 0.6561360874848117 }, "image.image_to_text.recall@10": { "denominator": 823, - "numerator": 707, - "value": 0.8590522478736331 + "numerator": 791, + "value": 0.9611178614823815 }, "image.image_to_text.recall@5": { "denominator": 823, - "numerator": 616, - "value": 0.7484811664641555 + "numerator": 731, + "value": 0.8882138517618469 }, "image.text_to_image.recall@1": { "denominator": 4115, - "numerator": 1558, - "value": 0.37861482381530986 + "numerator": 2184, + "value": 0.5307411907654921 }, "image.text_to_image.recall@10": { "denominator": 4115, - "numerator": 3420, - "value": 0.8311057108140948 + "numerator": 3847, + "value": 0.934872417982989 }, "image.text_to_image.recall@5": { "denominator": 4115, - "numerator": 2898, - "value": 0.7042527339003645 + "numerator": 3465, + "value": 0.8420413122721749 }, "text.banking77.accuracy": { "denominator": 3080, @@ -3220,75 +3294,75 @@ "split": "test", "hf_subset": "default", "main_metric": "accuracy", - "score": 0.44469882801853367, + "score": 0.9126737530662306, "metrics": { "scores_per_experiment": [ { - "accuracy": 0.4445352957209049, - "f1": 0.4309328513119121, - "f1_weighted": 0.4305399188756519, - "precision": 0.4391250185285425, - "precision_weighted": 0.43917490262148334, - "recall": 0.44540298862079075, - "recall_weighted": 0.4445352957209049, + "accuracy": 0.9062414826928318, + "f1": 0.9045301316021502, + "f1_weighted": 0.9050055409371512, + "precision": 0.9099213291951324, + "precision_weighted": 0.9109544817876485, + "recall": 0.9064295837897134, + "recall_weighted": 0.9062414826928318, "ap": null, "ap_weighted": null }, { - "accuracy": 0.45271191060234395, - "f1": 0.4374624127896425, - "f1_weighted": 0.4376012595127548, - "precision": 0.45586632178451736, - "precision_weighted": 0.4563107134922699, - "recall": 0.4526852227201386, - "recall_weighted": 0.45271191060234395, + "accuracy": 0.914418097574271, + "f1": 0.9126724638115935, + "f1_weighted": 0.9132186480967779, + "precision": 0.9192904646518116, + "precision_weighted": 0.9203471940294945, + "recall": 0.9144593228002407, + "recall_weighted": 0.914418097574271, "ap": null, "ap_weighted": null }, { - "accuracy": 0.4306350504224584, - "f1": 0.4170407359605212, - "f1_weighted": 0.41681774818448114, - "precision": 0.4342902708829311, - "precision_weighted": 0.43423856423244445, - "recall": 0.431028779615763, - "recall_weighted": 0.4306350504224584, + "accuracy": 0.9187789588443718, + "f1": 0.9177765846692281, + "f1_weighted": 0.9184111765200857, + "precision": 0.9220755461580387, + "precision_weighted": 0.9229278273528245, + "recall": 0.9183739103921627, + "recall_weighted": 0.9187789588443718, "ap": null, "ap_weighted": null }, { - "accuracy": 0.46061597165440177, - "f1": 0.44539537134563145, - "f1_weighted": 0.4451308646434834, - "precision": 0.4575017984531489, - "precision_weighted": 0.45771802232058695, - "recall": 0.46141694456896926, - "recall_weighted": 0.46061597165440177, + "accuracy": 0.9174161896974653, + "f1": 0.9147622354932347, + "f1_weighted": 0.9154663340384108, + "precision": 0.9209055798643112, + "precision_weighted": 0.9221124471295344, + "recall": 0.9173606715378323, + "recall_weighted": 0.9174161896974653, "ap": null, "ap_weighted": null }, { - "accuracy": 0.4349959116925593, - "f1": 0.4131091743752867, - "f1_weighted": 0.4126577885391281, - "precision": 0.4391459040033189, - "precision_weighted": 0.4391489734217057, - "recall": 0.4356331882435207, - "recall_weighted": 0.4349959116925593, + "accuracy": 0.9065140365222132, + "f1": 0.9051669100036687, + "f1_weighted": 0.9057728210054364, + "precision": 0.9098426973534027, + "precision_weighted": 0.9105143284060627, + "recall": 0.9059560197061645, + "recall_weighted": 0.9065140365222132, "ap": null, "ap_weighted": null } ], - "accuracy": 0.44469882801853367, - "f1": 0.4287881091565988, - "f1_weighted": 0.4285495159510998, - "precision": 0.4451858627304917, - "precision_weighted": 0.4453182352176981, - "recall": 0.44523342475383654, - "recall_weighted": 0.44469882801853367, + "accuracy": 0.9126737530662306, + "f1": 0.910981665115975, + "f1_weighted": 0.9115749041195725, + "precision": 0.9164071234445392, + "precision_weighted": 0.9173712557411129, + "recall": 0.9125159016452227, + "recall_weighted": 0.9126737530662306, "ap": null, "ap_weighted": null, - "main_score": 0.44469882801853367, + "main_score": 0.9126737530662306, "hf_subset": "default", "languages": [ "eng-Latn" @@ -3301,13 +3375,13 @@ "split": "valid", "hf_subset": "default", "main_metric": "nmi", - "score": 0.46033275813996855, + "score": 0.6169193395626786, "metrics": { - "v_measure": 0.4603327581399686, - "nmi": 0.46033275813996855, - "ari": 0.08461353562910791, - "cluster_accuracy": 0.2023, - "main_score": 0.46033275813996855, + "v_measure": 0.6169193395626787, + "nmi": 0.6169193395626786, + "ari": 0.21497891543638262, + "cluster_accuracy": 0.3638, + "main_score": 0.6169193395626786, "hf_subset": "default", "languages": [ "eng-Latn" @@ -3320,18 +3394,18 @@ "split": "test", "hf_subset": "default", "main_metric": "accuracy", - "score": 0.15263014445352957, + "score": 0.1163804851458163, "metrics": { - "accuracy": 0.15263014445352957, - "f1": 0.12154978453849745, - "f1_weighted": 0.12185768220651595, - "precision": 0.13972243713583027, - "precision_weighted": 0.14024267679350327, - "recall": 0.15235326485727774, - "recall_weighted": 0.15263014445352957, + "accuracy": 0.1163804851458163, + "f1": 0.085822378073852, + "f1_weighted": 0.08530524305774022, + "precision": 0.08885688520073651, + "precision_weighted": 0.08856851170630324, + "recall": 0.11709192030781942, + "recall_weighted": 0.1163804851458163, "ap": null, "ap_weighted": null, - "main_score": 0.15263014445352957, + "main_score": 0.1163804851458163, "hf_subset": "default", "languages": [ "eng-Latn" @@ -3491,6 +3565,105 @@ "mteb_version": "2.21.0" } } + ], + "full_benchmarks": [ + { + "benchmark": "MTEB(eng, v2)", + "framework_version": "2.21.0", + "coverage": { + "completed_tasks": 41, + "required_tasks": 41, + "task_types": 7 + }, + "model_revision": "70c3412bb0ddfb19386d673d4a102c753a574f1b", + "native_artifact_sha256": "b596f1dbab89369555e0a72a0d6854d780e305686750949b308c8f167993bdeb", + "aggregation": { + "mean_task": 0.5337130125278776, + "mean_task_type": 0.5197553164396411, + "definition": "First apply the official TaskResult aggregation within each task. Mean task gives each of the 41 tasks equal weight; mean task type first averages within each type, then gives all seven types equal weight.", + "score_scale": "0-1; multiply by 100 for display", + "by_task_type": { + "Classification": { + "score": 0.6552554795025091, + "tasks": 8 + }, + "Clustering": { + "score": 0.47072550487857034, + "tasks": 8 + }, + "PairClassification": { + "score": 0.7494593709186261, + "tasks": 3 + }, + "Reranking": { + "score": 0.455685, + "tasks": 2 + }, + "Retrieval": { + "score": 0.290152, + "tasks": 10 + }, + "STS": { + "score": 0.724513458257586, + "tasks": 9 + }, + "Summarization": { + "score": 0.29249640152019635, + "tasks": 1 + } + } + }, + "report": "benchmarks/mteb-eng-v2.json", + "text_component_applicability": { + "applies_to_revision": "self", + "applies_to_native_artifact_sha256": "fb9cd39c9980a085b75456c715c2295b997784fe733314be68c13635f6384edf", + "evaluated_revision": "70c3412bb0ddfb19386d673d4a102c753a574f1b", + "evaluated_native_artifact_sha256": "b596f1dbab89369555e0a72a0d6854d780e305686750949b308c8f167993bdeb", + "basis": "Exact component tensor and inference-file identity; original measured identity is retained.", + "report": "benchmarks/component-equivalence.json", + "new_full_panel_inference": false, + "component": "text" + } + }, + { + "benchmark": "MAEB(beta, audio-only)", + "framework_version": "2.21.0", + "coverage": { + "completed_tasks": 19, + "required_tasks": 19, + "subset_split_slots": 134, + "task_types": 5, + "selection": "All MAEB(beta) tasks whose modalities are exactly [audio].", + "full_MAEB": false, + "excluded": "Tasks that require text/audio or other joint-modality inputs." + }, + "model_revision": "70c3412bb0ddfb19386d673d4a102c753a574f1b", + "native_artifact_sha256": "b596f1dbab89369555e0a72a0d6854d780e305686750949b308c8f167993bdeb", + "aggregation": { + "official": { + "Any2AnyRetrieval": 0.6890050000000001, + "AudioClassification": 0.39566923071716426, + "AudioClustering": 0.054218008138417706, + "AudioPairClassification": 0.571886979184418, + "AudioReranking": 0.63794, + "Mean(Task)": 0.39776955262406916, + "Mean(TaskType)": 0.46974384360800003 + }, + "definition": "Official TaskResult aggregation within each task, followed by official mean_task, mean_task_type and task_types aggregation. The 134 subset/split slots are not averaged as independent tasks.", + "score_scale": "0-1; multiply by 100 for display" + }, + "audio_component_applicability": { + "applies_to_revision": "self", + "applies_to_native_artifact_sha256": "fb9cd39c9980a085b75456c715c2295b997784fe733314be68c13635f6384edf", + "evaluated_revision": "70c3412bb0ddfb19386d673d4a102c753a574f1b", + "evaluated_native_artifact_sha256": "b596f1dbab89369555e0a72a0d6854d780e305686750949b308c8f167993bdeb", + "basis": "Exact component tensor and inference-file identity; original measured identity is retained.", + "report": "benchmarks/component-equivalence.json", + "new_full_panel_inference": false, + "component": "audio" + }, + "report": "benchmarks/maeb-audio-only.json" + } ] }, "full_benchmarks": [ @@ -3502,54 +3675,56 @@ "required_tasks": 41, "task_types": 7 }, - "model_revision": "70c3412bb0ddfb19386d673d4a102c753a574f1b", - "native_artifact_sha256": "b596f1dbab89369555e0a72a0d6854d780e305686750949b308c8f167993bdeb", + "model_revision": "29994142677559ca657c27b657eda0dddbf583bd4cf42a3a31095ab3f99d1646", + "native_artifact_sha256": "29994142677559ca657c27b657eda0dddbf583bd4cf42a3a31095ab3f99d1646", "aggregation": { - "mean_task": 0.5337130125278776, - "mean_task_type": 0.5197553164396411, - "definition": "First apply the official TaskResult aggregation within each task. Mean task gives each of the 41 tasks equal weight; mean task type first averages within each type, then gives all seven types equal weight.", - "score_scale": "0-1; multiply by 100 for display", + "mean_task": 0.6488431514310018, + "mean_task_type": 0.6078184975654923, "by_task_type": { "Classification": { - "score": 0.6552554795025091, + "score": 0.7782970776285105, "tasks": 8 }, "Clustering": { - "score": 0.47072550487857034, + "score": 0.48962597555764586, "tasks": 8 }, "PairClassification": { - "score": 0.7494593709186261, + "score": 0.8468485561827029, "tasks": 3 }, "Reranking": { - "score": 0.455685, + "score": 0.473475, "tasks": 2 }, "Retrieval": { - "score": 0.290152, + "score": 0.520098, "tasks": 10 }, "STS": { - "score": 0.724513458257586, + "score": 0.8280405301305157, "tasks": 9 }, "Summarization": { - "score": 0.29249640152019635, + "score": 0.31834434345907126, "tasks": 1 } - } + }, + "definition": "Official within-task aggregation; equal task mean across 41 tasks, and equal mean of seven within-type means.", + "score_scale": "0-1; display multiplied by 100" }, "report": "benchmarks/mteb-eng-v2.json", "text_component_applicability": { + "component": "text", "applies_to_revision": "self", - "applies_to_native_artifact_sha256": "fb9cd39c9980a085b75456c715c2295b997784fe733314be68c13635f6384edf", - "evaluated_revision": "70c3412bb0ddfb19386d673d4a102c753a574f1b", - "evaluated_native_artifact_sha256": "b596f1dbab89369555e0a72a0d6854d780e305686750949b308c8f167993bdeb", - "basis": "Exact component tensor and inference-file identity; original measured identity is retained.", + "applies_to_native_artifact_sha256": "96d76f97478e114b816df3ab1b3c6947e4c5ef0babfb40ab865d7e3c838d34c4", + "evaluated_model": "local/Vela-Omni-Nano-GIST-Rotation", + "evaluated_revision": "29994142677559ca657c27b657eda0dddbf583bd4cf42a3a31095ab3f99d1646", + "evaluated_native_artifact_sha256": "29994142677559ca657c27b657eda0dddbf583bd4cf42a3a31095ab3f99d1646", + "revision_kind": "local native artifact fingerprint, not a Hugging Face commit", "report": "benchmarks/component-equivalence.json", "new_full_panel_inference": false, - "component": "text" + "basis": "199 exact text tensors and byte-identical inference/tokenizer/configuration under the same official evaluator protocol." } }, { @@ -3564,79 +3739,65 @@ "full_MAEB": false, "excluded": "Tasks that require text/audio or other joint-modality inputs." }, - "model_revision": "70c3412bb0ddfb19386d673d4a102c753a574f1b", - "native_artifact_sha256": "b596f1dbab89369555e0a72a0d6854d780e305686750949b308c8f167993bdeb", + "model_revision": "96d76f97478e114b816df3ab1b3c6947e4c5ef0babfb40ab865d7e3c838d34c4", + "native_artifact_sha256": "96d76f97478e114b816df3ab1b3c6947e4c5ef0babfb40ab865d7e3c838d34c4", "aggregation": { "official": { "Any2AnyRetrieval": 0.6890050000000001, - "AudioClassification": 0.39566923071716426, + "AudioClassification": 0.39553889510295764, "AudioClustering": 0.054218008138417706, - "AudioPairClassification": 0.571886979184418, + "AudioPairClassification": 0.5716888010864635, "AudioReranking": 0.63794, - "Mean(Task)": 0.39776955262406916, - "Mean(TaskType)": 0.46974384360800003 + "Mean(Task)": 0.3976628038845884, + "Mean(TaskType)": 0.4696781408655678 }, "definition": "Official TaskResult aggregation within each task, followed by official mean_task, mean_task_type and task_types aggregation. The 134 subset/split slots are not averaged as independent tasks.", "score_scale": "0-1; multiply by 100 for display" }, - "audio_component_applicability": { - "applies_to_revision": "self", - "applies_to_native_artifact_sha256": "fb9cd39c9980a085b75456c715c2295b997784fe733314be68c13635f6384edf", - "evaluated_revision": "70c3412bb0ddfb19386d673d4a102c753a574f1b", - "evaluated_native_artifact_sha256": "b596f1dbab89369555e0a72a0d6854d780e305686750949b308c8f167993bdeb", - "basis": "Exact component tensor and inference-file identity; original measured identity is retained.", - "report": "benchmarks/component-equivalence.json", - "new_full_panel_inference": false, - "component": "audio" - }, "report": "benchmarks/maeb-audio-only.json" } ], "net_benefit": { - "scope": "Product-weighted comparison against the previous release; not an official benchmark aggregate or SOTA claim.", + "scope": "Product-weighted progress comparison, not an official benchmark aggregate or SOTA claim.", "baseline_model": "llm-semantic-router/Vela-1.0-Omni-Nano", - "baseline_revision": "70c3412bb0ddfb19386d673d4a102c753a574f1b", - "baseline_native_artifact_sha256": "b596f1dbab89369555e0a72a0d6854d780e305686750949b308c8f167993bdeb", - "candidate_native_artifact_sha256": "fb9cd39c9980a085b75456c715c2295b997784fe733314be68c13635f6384edf", + "baseline_revision": "3a1efc48995e2b71622d65c6cf71cfc00bbf3aa3", + "baseline_native_artifact_sha256": "fb9cd39c9980a085b75456c715c2295b997784fe733314be68c13635f6384edf", + "candidate_native_artifact_sha256": "96d76f97478e114b816df3ab1b3c6947e4c5ef0babfb40ab865d7e3c838d34c4", "score_scale": "0-100; changes in percentage points", "components": [ { - "baseline_100": 54.82312157078607, - "candidate_100": 59.372871807049066, - "component": "common", - "delta_pp": 4.5497502362629945, - "weight": 0.3, - "evidence": "fresh_evaluation" + "baseline_100": 59.37287180704907, + "candidate_100": 67.57407300861536, + "component": "common14", + "delta_pp": 8.201201201566288, + "weight": 0.3 }, { "baseline_100": 51.97553164396411, - "candidate_100": 51.97553164396411, + "candidate_100": 60.781849756549235, "component": "English41_MeanTaskType", - "delta_pp": 0.0, - "weight": 0.3, - "evidence": "exact_component_applicability" + "delta_pp": 8.806318112585124, + "weight": 0.3 }, { - "baseline_100": 35.255391020401056, - "candidate_100": 54.865785925824184, + "baseline_100": 54.865785925824184, + "candidate_100": 53.65746394890041, "component": "image3", - "delta_pp": 19.610394905423124, - "weight": 0.2, - "evidence": "fresh_evaluation" + "delta_pp": -1.208321976923776, + "weight": 0.2 }, { "baseline_100": 46.9743843608, - "candidate_100": 46.9743843608, + "candidate_100": 46.96781408655678, "component": "audio19_MeanTaskType", - "delta_pp": 0.0, - "weight": 0.2, - "evidence": "exact_component_applicability" + "delta_pp": -0.00657027424322, + "weight": 0.2 } ], - "weighted_net_delta_pp": 5.287004051963523, - "common_definition": "Mean of three equally weighted families: the two text accuracies, all six image retrieval recalls and all six audio retrieval recalls.", - "image3_definition": "Equal mean of OxfordPets accuracy, TinyImageNetClustering NMI and OxfordPetsZeroShot accuracy.", - "text_audio_definition": "Complete English41 and audio19 mean-task-type scores, with explicit unchanged-component applicability.", + "weighted_net_delta_pp": 4.859277344012025, + "common_definition": "Equal mean of three families: two text accuracies, all six image retrieval recalls, all six audio retrieval recalls.", + "image3_definition": "Equal mean of OxfordPets classification accuracy, TinyImageNet clustering NMI and OxfordPets zero-shot accuracy.", + "text_audio_definition": "Full English41 and full audio19 mean task-type scores. English uses the explicit exact text-component bridge; audio is freshly evaluated.", "coverage": { "common_metrics": 14, "standard_tasks": 11, @@ -3646,131 +3807,495 @@ "audio_subset_split_slots": 134 }, "common_metric_delta_pp": { - "audio.audio_to_text.recall@1": 0.0, - "audio.audio_to_text.recall@10": 0.0, - "audio.audio_to_text.recall@5": 0.0, - "audio.text_to_audio.recall@1": 0.0, - "audio.text_to_audio.recall@10": 0.0, - "audio.text_to_audio.recall@5": 0.0, - "image.image_to_text.recall@1": 18.347509113001216, - "image.image_to_text.recall@10": 10.206561360874849, - "image.image_to_text.recall@5": 13.973268529769138, - "image.text_to_image.recall@1": 15.212636695018226, - "image.text_to_image.recall@10": 10.376670716889429, - "image.text_to_image.recall@5": 13.778857837181045, - "text.banking77.accuracy": 0.0, - "text.massive-en.accuracy": 0.0 + "audio.audio_to_text.recall@1": 8.157793948678668, + "audio.audio_to_text.recall@10": 15.51129835312141, + "audio.audio_to_text.recall@5": 14.362313289927231, + "audio.text_to_audio.recall@1": 11.417624521072797, + "audio.text_to_audio.recall@10": 15.478927203065133, + "audio.text_to_audio.recall@5": 14.444444444444445, + "image.image_to_text.recall@1": -4.738760631834751, + "image.image_to_text.recall@10": -1.2150668286755772, + "image.image_to_text.recall@5": -1.701093560145808, + "image.text_to_image.recall@1": 2.7460510328068044, + "image.text_to_image.recall@10": 0.7776427703523694, + "image.text_to_image.recall@5": 1.0206561360874848, + "text.banking77.accuracy": 11.168831168831169, + "text.massive-en.accuracy": 12.617765814266487 }, "standard_task_deltas": [ { - "task": "SICK-R", - "split": "test", + "baseline": 0.7169618925026362, + "candidate": 0.8053174565838819, "hf_subset": "default", "main_metric": "cosine_spearman", - "baseline": 0.7169618925026362, - "candidate": 0.7169618925026362, - "delta_pp": 0.0 + "split": "test", + "status": "complete", + "task": "SICK-R", + "delta_pp": 8.835556408124567 }, { - "task": "STSBenchmark", - "split": "test", + "baseline": 0.7792400226361769, + "candidate": 0.870781967302773, "hf_subset": "default", "main_metric": "cosine_spearman", - "baseline": 0.7792400226361769, - "candidate": 0.7792400226361769, - "delta_pp": 0.0 + "split": "test", + "status": "complete", + "task": "STSBenchmark", + "delta_pp": 9.154194466659614 }, { - "task": "ArguAna", - "split": "test", + "baseline": 0.37514, + "candidate": 0.59246, "hf_subset": "default", "main_metric": "ndcg_at_10", - "baseline": 0.37514, - "candidate": 0.37514, - "delta_pp": 0.0 + "split": "test", + "status": "complete", + "task": "ArguAna", + "delta_pp": 21.732000000000003 }, { - "task": "TwentyNewsgroupsClustering.v2", - "split": "test", + "baseline": 0.44884717457494344, + "candidate": 0.5087260708286914, "hf_subset": "default", "main_metric": "v_measure", - "baseline": 0.44884717457494344, - "candidate": 0.44884717457494344, - "delta_pp": 0.0 + "split": "test", + "status": "complete", + "task": "TwentyNewsgroupsClustering.v2", + "delta_pp": 5.987889625374793 }, { - "task": "MTOPDomainClassification", - "split": "validation", + "baseline": 0.8758836689038031, + "candidate": 0.9476510067114093, "hf_subset": "en", "main_metric": "accuracy", - "baseline": 0.8758836689038031, - "candidate": 0.8758836689038031, - "delta_pp": 0.0 + "split": "validation", + "status": "complete", + "task": "MTOPDomainClassification", + "delta_pp": 7.176733780760625 }, { - "task": "MTOPDomainClassification", - "split": "test", + "baseline": 0.8671910624715002, + "candidate": 0.9491792065663475, "hf_subset": "en", "main_metric": "accuracy", - "baseline": 0.8671910624715002, - "candidate": 0.8671910624715002, - "delta_pp": 0.0 + "split": "test", + "status": "complete", + "task": "MTOPDomainClassification", + "delta_pp": 8.198814409484722 }, { - "task": "OxfordPets", - "split": "test", + "baseline": 0.9126737530662306, + "candidate": 0.9126737530662306, "hf_subset": "default", "main_metric": "accuracy", - "baseline": 0.44469882801853367, - "candidate": 0.9126737530662306, - "delta_pp": 46.797492504769686 + "split": "test", + "status": "complete", + "task": "OxfordPets", + "delta_pp": 0.0 }, { - "task": "TinyImageNetClustering", - "split": "valid", + "baseline": 0.6169193395626786, + "candidate": 0.6169193395626786, "hf_subset": "default", "main_metric": "nmi", - "baseline": 0.46033275813996855, - "candidate": 0.6169193395626786, - "delta_pp": 15.658658142271008 + "split": "valid", + "status": "complete", + "task": "TinyImageNetClustering", + "delta_pp": 0.0 }, { - "task": "OxfordPetsZeroShot", - "split": "test", + "baseline": 0.1163804851458163, + "candidate": 0.08013082583810302, "hf_subset": "default", "main_metric": "accuracy", - "baseline": 0.15263014445352957, - "candidate": 0.1163804851458163, + "split": "test", + "status": "complete", + "task": "OxfordPetsZeroShot", "delta_pp": -3.6249659307713276 }, { - "task": "CREMA_D", - "split": "train", - "hf_subset": "default", - "main_metric": "accuracy", "baseline": 0.29763670140167686, "candidate": 0.29763670140167686, + "hf_subset": "default", + "main_metric": "accuracy", + "split": "train", + "status": "complete", + "task": "CREMA_D", "delta_pp": 0.0 }, { - "task": "CREMA_DClustering", - "split": "train", - "hf_subset": "default", - "main_metric": "v_measure", "baseline": 0.023601715918443712, "candidate": 0.023601715918443712, + "hf_subset": "default", + "main_metric": "v_measure", + "split": "train", + "status": "complete", + "task": "CREMA_DClustering", "delta_pp": 0.0 }, { - "task": "SpeechCommandsZeroshotv0.02", - "split": "test", + "baseline": 0.14972999509081983, + "candidate": 0.1411389297987236, "hf_subset": "default", "main_metric": "accuracy", - "baseline": 0.14972999509081983, - "candidate": 0.14972999509081983, - "delta_pp": 0.0 + "split": "test", + "status": "complete", + "task": "SpeechCommandsZeroshotv0.02", + "delta_pp": -0.8591065292096217 + } + ], + "English_task_deltas": [ + { + "candidate_100": 59.245999999999995, + "current_100": 37.513999999999996, + "delta_pp": 21.732, + "primary_metric": "ndcg_at_10", + "task": "ArguAna", + "task_type": "Retrieval" + }, + { + "candidate_100": 64.88847259326147, + "current_100": 65.25395409281045, + "delta_pp": -0.36548149954897724, + "primary_metric": "v_measure", + "task": "ArXivHierarchicalClusteringP2P", + "task_type": "Clustering" + }, + { + "candidate_100": 57.44315313869382, + "current_100": 55.989532291560415, + "delta_pp": 1.4536208471334078, + "primary_metric": "v_measure", + "task": "ArXivHierarchicalClusteringS2S", + "task_type": "Clustering" + }, + { + "candidate_100": 62.329, + "current_100": 60.514, + "delta_pp": 1.8149999999999977, + "primary_metric": "map_at_1000", + "task": "AskUbuntuDupQuestions", + "task_type": "Reranking" + }, + { + "candidate_100": 86.98752423842483, + "current_100": 62.24539420714732, + "delta_pp": 24.742130031277505, + "primary_metric": "cosine_spearman", + "task": "BIOSSES", + "task_type": "STS" + }, + { + "candidate_100": 82.14285714285714, + "current_100": 69.57467532467533, + "delta_pp": 12.568181818181813, + "primary_metric": "accuracy", + "task": "Banking77Classification", + "task_type": "Classification" + }, + { + "candidate_100": 41.313796273674654, + "current_100": 39.82923119776807, + "delta_pp": 1.4845650759065876, + "primary_metric": "v_measure", + "task": "BiorxivClusteringP2P.v2", + "task_type": "Clustering" + }, + { + "candidate_100": 56.979, + "current_100": 42.976, + "delta_pp": 14.003, + "primary_metric": "ndcg_at_10", + "task": "CQADupstackGamingRetrieval", + "task_type": "Retrieval" + }, + { + "candidate_100": 39.586, + "current_100": 30.362000000000002, + "delta_pp": 9.223999999999997, + "primary_metric": "ndcg_at_10", + "task": "CQADupstackUnixRetrieval", + "task_type": "Retrieval" + }, + { + "candidate_100": 31.846000000000004, + "current_100": 16.073999999999998, + "delta_pp": 15.772000000000006, + "primary_metric": "ndcg_at_10", + "task": "ClimateFEVERHardNegatives", + "task_type": "Retrieval" + }, + { + "candidate_100": 87.576, + "current_100": 24.055, + "delta_pp": 63.520999999999994, + "primary_metric": "ndcg_at_10", + "task": "FEVERHardNegatives", + "task_type": "Retrieval" + }, + { + "candidate_100": 39.143, + "current_100": 22.588, + "delta_pp": 16.555, + "primary_metric": "ndcg_at_10", + "task": "FiQA2018", + "task_type": "Retrieval" + }, + { + "candidate_100": 66.349, + "current_100": 26.987, + "delta_pp": 39.36200000000001, + "primary_metric": "ndcg_at_10", + "task": "HotpotQAHardNegatives", + "task_type": "Retrieval" + }, + { + "candidate_100": 91.94760000000001, + "current_100": 62.1364, + "delta_pp": 29.811200000000007, + "primary_metric": "accuracy", + "task": "ImdbClassification", + "task_type": "Classification" + }, + { + "candidate_100": 94.91792065663475, + "current_100": 86.71910624715002, + "delta_pp": 8.198814409484726, + "primary_metric": "accuracy", + "task": "MTOPDomainClassification", + "task_type": "Classification" + }, + { + "candidate_100": 70.96839273705447, + "current_100": 60.39340954942838, + "delta_pp": 10.574983187626096, + "primary_metric": "accuracy", + "task": "MassiveIntentClassification", + "task_type": "Classification" + }, + { + "candidate_100": 76.11297915265635, + "current_100": 73.37256220578345, + "delta_pp": 2.740416946872898, + "primary_metric": "accuracy", + "task": "MassiveScenarioClassification", + "task_type": "Classification" + }, + { + "candidate_100": 39.83023091223438, + "current_100": 37.76813963073551, + "delta_pp": 2.0620912814988657, + "primary_metric": "v_measure", + "task": "MedrxivClusteringP2P.v2", + "task_type": "Clustering" + }, + { + "candidate_100": 37.875084453707764, + "current_100": 36.13349306174702, + "delta_pp": 1.7415913919607462, + "primary_metric": "v_measure", + "task": "MedrxivClusteringS2S.v2", + "task_type": "Clustering" + }, + { + "candidate_100": 32.366, + "current_100": 30.623, + "delta_pp": 1.7429999999999986, + "primary_metric": "max_over_subqueries_map_at_1000", + "task": "MindSmallReranking", + "task_type": "Reranking" + }, + { + "candidate_100": 21.889, + "current_100": 16.827, + "delta_pp": 5.061999999999998, + "primary_metric": "ndcg_at_10", + "task": "SCIDOCS", + "task_type": "Retrieval" + }, + { + "candidate_100": 80.53174565838819, + "current_100": 71.69618925026361, + "delta_pp": 8.835556408124575, + "primary_metric": "cosine_spearman", + "task": "SICK-R", + "task_type": "STS" + }, + { + "candidate_100": 75.5658801516535, + "current_100": 68.16915974310523, + "delta_pp": 7.396720408548276, + "primary_metric": "cosine_spearman", + "task": "STS12", + "task_type": "STS" + }, + { + "candidate_100": 86.26346974038235, + "current_100": 74.40342742804673, + "delta_pp": 11.860042312335622, + "primary_metric": "cosine_spearman", + "task": "STS13", + "task_type": "STS" + }, + { + "candidate_100": 82.29882793900394, + "current_100": 68.02653891259911, + "delta_pp": 14.272289026404835, + "primary_metric": "cosine_spearman", + "task": "STS14", + "task_type": "STS" + }, + { + "candidate_100": 88.73648284326485, + "current_100": 77.86144933320074, + "delta_pp": 10.875033510064114, + "primary_metric": "cosine_spearman", + "task": "STS15", + "task_type": "STS" + }, + { + "candidate_100": 87.0781967302773, + "current_100": 77.92400226361768, + "delta_pp": 9.154194466659618, + "primary_metric": "cosine_spearman", + "task": "STSBenchmark", + "task_type": "STS" + }, + { + "candidate_100": 95.79964673462752, + "current_100": 89.79096125036985, + "delta_pp": 6.0086854842576685, + "primary_metric": "max_ap", + "task": "SprintDuplicateQuestions", + "task_type": "PairClassification" + }, + { + "candidate_100": 58.424418738430106, + "current_100": 57.34464990059502, + "delta_pp": 1.0797688378350827, + "primary_metric": "v_measure", + "task": "StackExchangeClustering.v2", + "task_type": "Clustering" + }, + { + "candidate_100": 41.053017253245386, + "current_100": 39.37668627014545, + "delta_pp": 1.6763309830999376, + "primary_metric": "v_measure", + "task": "StackExchangeClusteringP2P.v2", + "task_type": "Clustering" + }, + { + "candidate_100": 69.121, + "current_100": 41.292, + "delta_pp": 27.828999999999994, + "primary_metric": "ndcg_at_10", + "task": "TRECCOVID", + "task_type": "Retrieval" + }, + { + "candidate_100": 48.363, + "current_100": 31.477, + "delta_pp": 16.886, + "primary_metric": "ndcg_at_10", + "task": "Touche2020Retrieval.v3", + "task_type": "Retrieval" + }, + { + "candidate_100": 71.904296875, + "current_100": 64.4091796875, + "delta_pp": 7.4951171875, + "primary_metric": "accuracy", + "task": "ToxicConversationsClassification", + "task_type": "Classification" + }, + { + "candidate_100": 62.79286926994907, + "current_100": 50.77815506508207, + "delta_pp": 12.014714204867005, + "primary_metric": "accuracy", + "task": "TweetSentimentExtractionClassification", + "task_type": "Classification" + }, + { + "candidate_100": 50.87260708286914, + "current_100": 44.88471745749435, + "delta_pp": 5.987889625374791, + "primary_metric": "v_measure", + "task": "TwentyNewsgroupsClustering.v2", + "task_type": "Clustering" + }, + { + "candidate_100": 72.95151691526065, + "current_100": 56.34475441213181, + "delta_pp": 16.60676250312884, + "primary_metric": "max_ap", + "task": "TwitterSemEval2015", + "task_type": "PairClassification" + }, + { + "candidate_100": 85.30340320492272, + "current_100": 78.70209561308617, + "delta_pp": 6.6013075918365445, + "primary_metric": "max_ap", + "task": "TwitterURLCorpus", + "task_type": "PairClassification" + }, + { + "candidate_100": 31.834434345907127, + "current_100": 29.249640152019634, + "delta_pp": 2.584794193887493, + "primary_metric": "cosine_spearman", + "task": "SummEvalSummarization.v2", + "task_type": "Summarization" + }, + { + "candidate_100": 71.8507462686567, + "current_100": 56.82089552238806, + "delta_pp": 15.02985074626865, + "primary_metric": "accuracy", + "task": "AmazonCounterfactualClassification", + "task_type": "Classification" + }, + { + "candidate_100": 89.02101343562433, + "current_100": 84.16589279728525, + "delta_pp": 4.8551206383390735, + "primary_metric": "cosine_spearman", + "task": "STS17", + "task_type": "STS" + }, + { + "candidate_100": 68.75333638044488, + "current_100": 67.57005849656171, + "delta_pp": 1.1832778838831643, + "primary_metric": "cosine_spearman", + "task": "STS22.v2", + "task_type": "STS" } ], - "tradeoff": "OxfordPets zero-shot accuracy decreases while image linear-probe accuracy, clustering NMI and cross-modal retrieval improve. All task scores and regressions remain reported." - } + "audio_task_delta_pp": { + "BeijingOpera": 0.0, + "BirdCLEF": 0.0, + "CREMADPairClassification": 0.12104996267769952, + "CREMA_D": 0.0, + "CREMA_DClustering": 0.0, + "CommonLanguageAgeDetection": 0.0, + "GTZANAudioReranking": 0.0, + "GTZANGenre": 0.0, + "IEMOCAPGender": 0.0, + "JamAltArtistA2ARetrieval": 0.0, + "MInDS14": 0.0, + "MridinghamTonic": -0.14336917562723928, + "NMSQAPairClassification": -0.1520590751812989, + "SIBFLEURS": 0.0, + "VehicleSoundClustering": 0.0, + "VoxCelebSA": 0.0, + "VoxPopuliAccentPairClassification": -0.02844431688273641, + "VoxPopuliGenderClustering": 0.0, + "VoxPopuliLanguageID": 0.0 + }, + "paired_interval_scope": "Paired intervals cover the original common metrics, family and two-primary-per-family utility. They are not confidence intervals for the new all-14 common mean or the four-component product aggregate." + }, + "paired_uncertainty_scope": "Paired intervals cover the original common metrics, family and two-primary-per-family utility. They are not confidence intervals for the new all-14 common mean or the four-component product aggregate." }