diff --git "a/scores.json" "b/scores.json" --- "a/scores.json" +++ "b/scores.json" @@ -1,7 +1,7 @@ { "schema_version": 2, "model": "llm-semantic-router/Vela-1.0-Omni-Nano", - "revision_semantics": "self denotes this published inference package; evaluated_* identities retain original measurements. The exact public-inference equivalence report binds score applicability.", + "revision_semantics": "self denotes the released native package. Each benchmark retains its actual evaluated artifact; exact unchanged-component applicability is explicit.", "score_scale": "0-1; multiply by 100 for percentages", "matrix": { "small": { @@ -84,37 +84,39 @@ "candidate": { "model": "llm-semantic-router/Vela-1.0-Omni-Nano", "revision": "self", - "artifact_fingerprint_sha256": "50ce808197fb43a1b8913a66d4a47339ec66ecd10d52cbd9f9e752d02f8c43fa", + "artifact_fingerprint_sha256": "474ad3d52f6ee7f818a3dbb6112e3d66c2423542d08228e7996a8cd600bd99a2", + "evaluated_native_artifact_sha256": "474ad3d52f6ee7f818a3dbb6112e3d66c2423542d08228e7996a8cd600bd99a2", + "parameters": 163771288, "metrics": { "audio.audio_to_text.recall@1": { "denominator": 2611, - "numerator": 448, - "value": 0.17158176943699732 + "numerator": 421, + "value": 0.1612409038682497 }, "audio.audio_to_text.recall@10": { "denominator": 2611, - "numerator": 1227, - "value": 0.469934890846419 + "numerator": 1191, + "value": 0.45614707008808886 }, "audio.audio_to_text.recall@5": { "denominator": 2611, - "numerator": 957, - "value": 0.36652623515894295 + "numerator": 909, + "value": 0.34814247414783606 }, "audio.text_to_audio.recall@1": { "denominator": 2610, - "numerator": 664, - "value": 0.25440613026819925 + "numerator": 531, + "value": 0.20344827586206896 }, "audio.text_to_audio.recall@10": { "denominator": 2610, - "numerator": 1459, - "value": 0.5590038314176246 + "numerator": 1321, + "value": 0.5061302681992337 }, "audio.text_to_audio.recall@5": { "denominator": 2610, - "numerator": 1204, - "value": 0.46130268199233715 + "numerator": 1043, + "value": 0.3996168582375479 }, "image.image_to_text.recall@1": { "denominator": 823, @@ -156,12 +158,7 @@ "numerator": 2436, "value": 0.819650067294751 } - }, - "parameters": 135383808, - "evaluated_native_artifact_sha256": "96d76f97478e114b816df3ab1b3c6947e4c5ef0babfb40ab865d7e3c838d34c4", - "evaluated_revision": "de02c367bc9aa7a6f7ad3e59fb38962502e021c1", - "evaluated_parameters": 144554112, - "inference_equivalence_report": "benchmarks/inference-equivalence.json" + } } }, "datasets": { @@ -220,226 +217,133 @@ "ties": "lexicographic candidate ID", "retrieval": "complete candidate pools; all annotated positives", "classification": "Nearest normalized mean class prototype from labeled training examples; the original small comparator has task-adapted history, while current GIST is frozen.", - "aggregation": "Each family is the mean of its two primary metrics; utility is the mean of all six primaries. Retrieval primaries are R@1; R@5 and R@10 are reported separately.", + "aggregation": "Exact per-metric counts; the product common aggregate averages all fourteen metrics with equal text/image/audio family weights (2/6/6 metrics). Historical paired family/utility intervals retain their original definitions.", "evaluation_scope": "fixed test-split evaluation pools reused across releases", - "text_training_scope": "Current GIST text tower is frozen; supervised training prototypes are recomputed under the fixed common protocol. The earlier small comparator has task-adapted history. This is not the official MTEB classification protocol." + "text_training_scope": "Current GIST text tower is frozen; supervised training prototypes are recomputed under the fixed common protocol. The earlier small comparator has task-adapted history. This is not the official MTEB classification protocol.", + "audio_preprocessing": "Original PCM is separately resampled to 16 kHz for Whisper and 48 kHz for CLAP; official evaluation resamples before applying its 30-second cap. Public API rejects inputs longer than 30 seconds." }, "paired_uncertainty": { - "original": { - "intervals": { - "audio.audio_to_text.recall@1": { - "lower": 0.11352472177122043, - "point_delta": 0.12945231711987745, - "upper": 0.14665221762155492 - }, - "audio.audio_to_text.recall@10": { - "lower": 0.25338950715018704, - "point_delta": 0.2795863653772501, - "upper": 0.30833452722063037 - }, - "audio.audio_to_text.recall@5": { - "lower": 0.22198472459131105, - "point_delta": 0.24664879356568364, - "upper": 0.2736971153456769 - }, - "audio.text_to_audio.recall@1": { - "lower": 0.14202610349345837, - "point_delta": 0.15862068965517243, - "upper": 0.17613425616168824 - }, - "audio.text_to_audio.recall@10": { - "lower": 0.23515433278434666, - "point_delta": 0.25210727969348656, - "upper": 0.2707672328928673 - }, - "audio.text_to_audio.recall@5": { - "lower": 0.21796104940042046, - "point_delta": 0.23601532567049807, - "upper": 0.2547919081648684 - }, - "family.audio": { - "lower": 0.1322499685832721, - "point_delta": 0.14403650338752494, - "upper": 0.15644009735751394 - }, - "family.image": { - "lower": 0.20084750911300123, - "point_delta": 0.2284325637910085, - "upper": 0.25637910085054677 - }, - "family.text": { - "lower": 0.15747009669727144, - "point_delta": 0.16790542902588662, - "upper": 0.17869785413173905 - }, - "image.image_to_text.recall@1": { - "lower": 0.1591737545565006, - "point_delta": 0.20048602673147023, - "upper": 0.24301336573511542 - }, - "image.image_to_text.recall@10": { - "lower": 0.1336573511543135, - "point_delta": 0.16403402187120292, - "upper": 0.19441069258809235 - }, - "image.image_to_text.recall@5": { - "lower": 0.16160388821385177, - "point_delta": 0.19927095990279464, - "upper": 0.23572296476306198 - }, - "image.text_to_image.recall@1": { - "lower": 0.23013365735115432, - "point_delta": 0.25637910085054677, - "upper": 0.2831166464155528 - }, - "image.text_to_image.recall@10": { - "lower": 0.17618469015795868, - "point_delta": 0.1997569866342649, - "upper": 0.22235722964763063 - }, - "image.text_to_image.recall@5": { - "lower": 0.23207776427703525, - "point_delta": 0.2580801944106926, - "upper": 0.28360267314702303 - }, - "text.banking77.accuracy": { - "lower": 0.16066212268743915, - "point_delta": 0.17564935064935064, - "upper": 0.19058596303826084 - }, - "text.massive-en.accuracy": { - "lower": 0.14554621848739496, - "point_delta": 0.1601615074024226, - "upper": 0.17569841804106362 + "scope": "Fresh paired intervals compare this candidate with the preceding Vela Nano computation. They are not an interval for the all-14 aggregate or the weighted net. An original-small interval was not recomputed for this update.", + "previous_release": { + "published_fingerprint": "50ce808197fb43a1b8913a66d4a47339ec66ecd10d52cbd9f9e752d02f8c43fa", + "evaluated_fingerprint": "96d76f97478e114b816df3ab1b3c6947e4c5ef0babfb40ab865d7e3c838d34c4", + "result": { + "intervals": { + "audio.audio_to_text.recall@1": { + "lower": -0.018926721728713448, + "point_delta": -0.010340865568747606, + "upper": -0.001484096923424178 + }, + "audio.audio_to_text.recall@10": { + "lower": -0.02562171008540101, + "point_delta": -0.013787820758330141, + "upper": -0.001602915531368321 + }, + "audio.audio_to_text.recall@5": { + "lower": -0.02957690797530578, + "point_delta": -0.018383761011106857, + "upper": -0.007289100963896155 + }, + "audio.text_to_audio.recall@1": { + "lower": -0.06884900952272836, + "point_delta": -0.050957854406130265, + "upper": -0.03441425975371444 + }, + "audio.text_to_audio.recall@10": { + "lower": -0.07240471605447092, + "point_delta": -0.052873563218390804, + "upper": -0.03213863952299779 + }, + "audio.text_to_audio.recall@5": { + "lower": -0.07963438278344268, + "point_delta": -0.061685823754789273, + "upper": -0.044619605933086656 + }, + "family.audio": { + "lower": -0.03959510843694133, + "point_delta": -0.030649359987438937, + "upper": -0.02150226739148048 + }, + "family.image": { + "lower": 0.0, + "point_delta": 0.0, + "upper": 0.0 + }, + "family.text": { + "lower": 0.0, + "point_delta": 0.0, + "upper": 0.0 + }, + "image.image_to_text.recall@1": { + "lower": 0.0, + "point_delta": 0.0, + "upper": 0.0 + }, + "image.image_to_text.recall@10": { + "lower": 0.0, + "point_delta": 0.0, + "upper": 0.0 + }, + "image.image_to_text.recall@5": { + "lower": 0.0, + "point_delta": 0.0, + "upper": 0.0 + }, + "image.text_to_image.recall@1": { + "lower": 0.0, + "point_delta": 0.0, + "upper": 0.0 + }, + "image.text_to_image.recall@10": { + "lower": 0.0, + "point_delta": 0.0, + "upper": 0.0 + }, + "image.text_to_image.recall@5": { + "lower": 0.0, + "point_delta": 0.0, + "upper": 0.0 + }, + "text.banking77.accuracy": { + "lower": 0.0, + "point_delta": 0.0, + "upper": 0.0 + }, + "text.massive-en.accuracy": { + "lower": 0.0, + "point_delta": 0.0, + "upper": 0.0 + }, + "utility": { + "lower": -0.013198369478980442, + "point_delta": -0.010216453329146313, + "upper": -0.0071674224638268265 + } }, - "utility": { - "lower": 0.16995251624973068, - "point_delta": 0.18012483206814003, - "upper": 0.1910563694034248 - } - }, - "method": "paired stratified group percentile; PCG64; fixed candidate pools", - "replicates": 2000, - "seed": 20260918, - "comparison": { - "candidate": "llm-semantic-router/Vela-1.0-Omni-Nano", - "candidate_artifact_fingerprint_sha256": "96d76f97478e114b816df3ab1b3c6947e4c5ef0babfb40ab865d7e3c838d34c4", - "baseline": "llm-semantic-router/multi-modal-embed-small", - "baseline_revision": "fdf8e01b7b0f3a69ac1ac8e2a64dcb1ede177ba4", - "baseline_artifact_fingerprint_sha256": "684fe0c48a4ddd3944167f6e5dc75e97ada24e5b120bd9a2921f00dc03112eeb" + "method": "paired stratified group percentile; PCG64; fixed candidate pools", + "replicates": 2000, + "seed": 20260918 } }, - "previous": { - "intervals": { - "audio.audio_to_text.recall@1": { - "lower": 0.06612466882994839, - "point_delta": 0.08157793948678667, - "upper": 0.0967748551165683 - }, - "audio.audio_to_text.recall@10": { - "lower": 0.13256556125090607, - "point_delta": 0.15511298353121408, - "upper": 0.18059527375151307 - }, - "audio.audio_to_text.recall@5": { - "lower": 0.12086266595311207, - "point_delta": 0.1436231328992723, - "upper": 0.16909569992403942 - }, - "audio.text_to_audio.recall@1": { - "lower": 0.09860745201354912, - "point_delta": 0.11417624521072797, - "upper": 0.13014135201210755 - }, - "audio.text_to_audio.recall@10": { - "lower": 0.13604064880650404, - "point_delta": 0.15478927203065135, - "upper": 0.17574363317277317 - }, - "audio.text_to_audio.recall@5": { - "lower": 0.1272585913379362, - "point_delta": 0.14444444444444443, - "upper": 0.16333503134796237 - }, - "family.audio": { - "lower": 0.08555566687871614, - "point_delta": 0.09787709234875733, - "upper": 0.11042213832911257 - }, - "family.image": { - "lower": -0.028797083839611177, - "point_delta": -0.00996354799513973, - "upper": 0.008630012150668272 - }, - "family.text": { - "lower": 0.10934917350884273, - "point_delta": 0.11893298491548829, - "upper": 0.12872853420036084 - }, - "image.image_to_text.recall@1": { - "lower": -0.08140947752126367, - "point_delta": -0.04738760631834751, - "upper": -0.012150668286755772 - }, - "image.image_to_text.recall@10": { - "lower": -0.027946537059538274, - "point_delta": -0.012150668286755772, - "upper": 0.002430133657351154 - }, - "image.image_to_text.recall@5": { - "lower": -0.040097205346294046, - "point_delta": -0.01701093560145808, - "upper": 0.004860267314702308 - }, - "image.text_to_image.recall@1": { - "lower": 0.01117861482381531, - "point_delta": 0.027460510328068045, - "upper": 0.042770352369380316 - }, - "image.text_to_image.recall@10": { - "lower": 0.001215066828675577, - "point_delta": 0.007776427703523694, - "upper": 0.014580801944106925 - }, - "image.text_to_image.recall@5": { - "lower": -0.001701093560145808, - "point_delta": 0.010206561360874849, - "upper": 0.02187120291616039 - }, - "text.banking77.accuracy": { - "lower": 0.0990581357583631, - "point_delta": 0.11168831168831168, - "upper": 0.12439713161970098 - }, - "text.massive-en.accuracy": { - "lower": 0.1117468865701784, - "point_delta": 0.12617765814266488, - "upper": 0.14069455352356733 - }, - "utility": { - "lower": 0.060760442670994376, - "point_delta": 0.06894884308970196, - "upper": 0.07704865461098383 - } - }, - "method": "paired stratified group percentile; PCG64; fixed candidate pools", - "replicates": 2000, - "seed": 20260918, - "comparison": { - "candidate": "llm-semantic-router/Vela-1.0-Omni-Nano", - "candidate_artifact_fingerprint_sha256": "96d76f97478e114b816df3ab1b3c6947e4c5ef0babfb40ab865d7e3c838d34c4", - "baseline": "llm-semantic-router/Vela-1.0-Omni-Nano", - "baseline_revision": "3a1efc48995e2b71622d65c6cf71cfc00bbf3aa3", - "baseline_artifact_fingerprint_sha256": "fb9cd39c9980a085b75456c715c2295b997784fe733314be68c13635f6384edf" - } + "original_small": { + "status": "not_recomputed_for_this_update", + "value": null } }, "training_scope": { "text": "The complete text tower is the frozen official avsolatorio/GIST-small-Embedding-v0 checkpoint at 75e62fd210b9fde790430e0b2f040b0b00a021b1. Its model card discloses MTEB classification training data; no claim of independent pretraining data is made.", "image": "Frozen current vision encoder and initial image head; an orthogonal shared-space rotation is trained on 16,509 COCO training images, 81,998 unique caption texts and 82,522 complete positive associations, then folded into the image affine head.", - "audio": "Frozen current speech encoder and initial audio head; a separate orthogonal shared-space rotation is trained on 28,535 LibriSpeech train-clean-100 clips and all positive transcript associations, then folded into the audio affine head.", - "procedure": "One fixed 300-step run; only the final artifact is reported. Encoders remain frozen. The two rotations preserve within-modality inner products, up to floating-point rounding.", - "evaluation_scope": "Common text classification uses labeled training prototypes with the earlier task-adapted comparator. It is separate from the official frozen-encoder MTEB classification protocols." + "audio": "The existing Whisper Tiny encoder and affine head stay frozen. A frozen CLAP audio tower and a 384×512 residual projection are added; only that projection is trained for one fixed 600-step run. Speech alignment and a pointwise parent anchor use 28,535 LibriSpeech TRAIN clips; known-positive event labels and CLAP relational geometry use 3,299 FSD50K TRAIN clips. Center/scale and the geometry coefficient use TRAIN only.", + "procedure": "One fixed final-600-step residual artifact; no held-out benchmark examples fit the projection, statistics or coefficient. Full benchmark outcomes and regressions are retained.", + "evaluation_scope": "Common text classification uses labeled training prototypes with the earlier task-adapted comparator. It is separate from the official frozen-encoder MTEB classification protocols.", + "prior_alignment_history": { + "text": "The complete text tower is the frozen official avsolatorio/GIST-small-Embedding-v0 checkpoint at 75e62fd210b9fde790430e0b2f040b0b00a021b1. Its model card discloses MTEB classification training data; no claim of independent pretraining data is made.", + "image": "Frozen current vision encoder and initial image head; an orthogonal shared-space rotation is trained on 16,509 COCO training images, 81,998 unique caption texts and 82,522 complete positive associations, then folded into the image affine head.", + "audio": "Frozen current speech encoder and initial audio head; a separate orthogonal shared-space rotation is trained on 28,535 LibriSpeech train-clean-100 clips and all positive transcript associations, then folded into the audio affine head.", + "procedure": "One fixed 300-step run; only the final artifact is reported. Encoders remain frozen. The two rotations preserve within-modality inner products, up to floating-point rounding.", + "evaluation_scope": "Common text classification uses labeled training prototypes with the earlier task-adapted comparator. It is separate from the official frozen-encoder MTEB classification protocols." + }, + "event_labels": "149 labels with TRAIN support and 7,690 known-positive associations. Unannotated labels are not negatives; label names are weak class supervision, not observed clip captions." }, "descriptive_metrics": { "definition": "Macro-F1 is the unweighted mean of per-class F1 across all fixed prototype classes; zero_division=0.", @@ -469,6 +373,993 @@ "queries": 2972 } } + }, + "candidate_per_class": { + "banking77": { + "accuracy": { + "denominator": 3080, + "numerator": 2710, + "value": 0.8798701298701299 + }, + "evaluated_classes": 77, + "macro_f1": 0.8791681858119436, + "macro_f1_class_universe": "all fixed prototype classes; zero_division=0", + "per_class": { + "Refund_not_showing_up": { + "f1": 0.9512195121951219, + "fn": 1, + "fp": 3, + "support": 40, + "tp": 39 + }, + "activate_my_card": { + "f1": 0.935064935064935, + "fn": 4, + "fp": 1, + "support": 40, + "tp": 36 + }, + "age_limit": { + "f1": 1.0, + "fn": 0, + "fp": 0, + "support": 40, + "tp": 40 + }, + "apple_pay_or_google_pay": { + "f1": 1.0, + "fn": 0, + "fp": 0, + "support": 40, + "tp": 40 + }, + "atm_support": { + "f1": 0.9382716049382716, + "fn": 2, + "fp": 3, + "support": 40, + "tp": 38 + }, + "automatic_top_up": { + "f1": 0.8918918918918919, + "fn": 7, + "fp": 1, + "support": 40, + "tp": 33 + }, + "balance_not_updated_after_bank_transfer": { + "f1": 0.7407407407407407, + "fn": 10, + "fp": 11, + "support": 40, + "tp": 30 + }, + "balance_not_updated_after_cheque_or_cash_deposit": { + "f1": 0.9285714285714286, + "fn": 1, + "fp": 5, + "support": 40, + "tp": 39 + }, + "beneficiary_not_allowed": { + "f1": 0.7246376811594203, + "fn": 15, + "fp": 4, + "support": 40, + "tp": 25 + }, + "cancel_transfer": { + "f1": 0.95, + "fn": 2, + "fp": 2, + "support": 40, + "tp": 38 + }, + "card_about_to_expire": { + "f1": 0.9523809523809523, + "fn": 0, + "fp": 4, + "support": 40, + "tp": 40 + }, + "card_acceptance": { + "f1": 0.9069767441860465, + "fn": 1, + "fp": 7, + "support": 40, + "tp": 39 + }, + "card_arrival": { + "f1": 0.8421052631578947, + "fn": 8, + "fp": 4, + "support": 40, + "tp": 32 + }, + "card_delivery_estimate": { + "f1": 0.8536585365853658, + "fn": 5, + "fp": 7, + "support": 40, + "tp": 35 + }, + "card_linking": { + "f1": 0.8947368421052632, + "fn": 6, + "fp": 2, + "support": 40, + "tp": 34 + }, + "card_not_working": { + "f1": 0.8181818181818182, + "fn": 4, + "fp": 12, + "support": 40, + "tp": 36 + }, + "card_payment_fee_charged": { + "f1": 0.8536585365853658, + "fn": 5, + "fp": 7, + "support": 40, + "tp": 35 + }, + "card_payment_not_recognised": { + "f1": 0.8395061728395061, + "fn": 6, + "fp": 7, + "support": 40, + "tp": 34 + }, + "card_payment_wrong_exchange_rate": { + "f1": 0.95, + "fn": 2, + "fp": 2, + "support": 40, + "tp": 38 + }, + "card_swallowed": { + "f1": 0.9473684210526315, + "fn": 4, + "fp": 0, + "support": 40, + "tp": 36 + }, + "cash_withdrawal_charge": { + "f1": 0.8941176470588236, + "fn": 2, + "fp": 7, + "support": 40, + "tp": 38 + }, + "cash_withdrawal_not_recognised": { + "f1": 0.7951807228915663, + "fn": 7, + "fp": 10, + "support": 40, + "tp": 33 + }, + "change_pin": { + "f1": 0.9873417721518988, + "fn": 1, + "fp": 0, + "support": 40, + "tp": 39 + }, + "compromised_card": { + "f1": 0.8395061728395061, + "fn": 6, + "fp": 7, + "support": 40, + "tp": 34 + }, + "contactless_not_working": { + "f1": 0.9473684210526315, + "fn": 4, + "fp": 0, + "support": 40, + "tp": 36 + }, + "country_support": { + "f1": 0.9210526315789473, + "fn": 5, + "fp": 1, + "support": 40, + "tp": 35 + }, + "declined_card_payment": { + "f1": 0.8043478260869565, + "fn": 3, + "fp": 15, + "support": 40, + "tp": 37 + }, + "declined_cash_withdrawal": { + "f1": 0.8333333333333334, + "fn": 5, + "fp": 9, + "support": 40, + "tp": 35 + }, + "declined_transfer": { + "f1": 0.8115942028985508, + "fn": 12, + "fp": 1, + "support": 40, + "tp": 28 + }, + "direct_debit_payment_not_recognised": { + "f1": 0.8450704225352113, + "fn": 10, + "fp": 1, + "support": 40, + "tp": 30 + }, + "disposable_card_limits": { + "f1": 0.8395061728395061, + "fn": 6, + "fp": 7, + "support": 40, + "tp": 34 + }, + "edit_personal_details": { + "f1": 0.975, + "fn": 1, + "fp": 1, + "support": 40, + "tp": 39 + }, + "exchange_charge": { + "f1": 0.9367088607594937, + "fn": 3, + "fp": 2, + "support": 40, + "tp": 37 + }, + "exchange_rate": { + "f1": 0.9397590361445783, + "fn": 1, + "fp": 4, + "support": 40, + "tp": 39 + }, + "exchange_via_app": { + "f1": 0.7906976744186046, + "fn": 6, + "fp": 12, + "support": 40, + "tp": 34 + }, + "extra_charge_on_statement": { + "f1": 0.8409090909090909, + "fn": 3, + "fp": 11, + "support": 40, + "tp": 37 + }, + "failed_transfer": { + "f1": 0.7956989247311828, + "fn": 3, + "fp": 16, + "support": 40, + "tp": 37 + }, + "fiat_currency_support": { + "f1": 0.7804878048780488, + "fn": 8, + "fp": 10, + "support": 40, + "tp": 32 + }, + "get_disposable_virtual_card": { + "f1": 0.7887323943661971, + "fn": 12, + "fp": 3, + "support": 40, + "tp": 28 + }, + "get_physical_card": { + "f1": 0.963855421686747, + "fn": 0, + "fp": 3, + "support": 40, + "tp": 40 + }, + "getting_spare_card": { + "f1": 0.8941176470588236, + "fn": 2, + "fp": 7, + "support": 40, + "tp": 38 + }, + "getting_virtual_card": { + "f1": 0.8764044943820225, + "fn": 1, + "fp": 10, + "support": 40, + "tp": 39 + }, + "lost_or_stolen_card": { + "f1": 0.8666666666666667, + "fn": 1, + "fp": 11, + "support": 40, + "tp": 39 + }, + "lost_or_stolen_phone": { + "f1": 0.9873417721518988, + "fn": 1, + "fp": 0, + "support": 40, + "tp": 39 + }, + "order_physical_card": { + "f1": 0.8275862068965517, + "fn": 4, + "fp": 11, + "support": 40, + "tp": 36 + }, + "passcode_forgotten": { + "f1": 0.9876543209876543, + "fn": 0, + "fp": 1, + "support": 40, + "tp": 40 + }, + "pending_card_payment": { + "f1": 0.95, + "fn": 2, + "fp": 2, + "support": 40, + "tp": 38 + }, + "pending_cash_withdrawal": { + "f1": 0.8611111111111112, + "fn": 9, + "fp": 1, + "support": 40, + "tp": 31 + }, + "pending_top_up": { + "f1": 0.8831168831168831, + "fn": 6, + "fp": 3, + "support": 40, + "tp": 34 + }, + "pending_transfer": { + "f1": 0.7647058823529411, + "fn": 14, + "fp": 2, + "support": 40, + "tp": 26 + }, + "pin_blocked": { + "f1": 0.9230769230769231, + "fn": 4, + "fp": 2, + "support": 40, + "tp": 36 + }, + "receiving_money": { + "f1": 0.9090909090909091, + "fn": 5, + "fp": 2, + "support": 40, + "tp": 35 + }, + "request_refund": { + "f1": 0.9066666666666666, + "fn": 6, + "fp": 1, + "support": 40, + "tp": 34 + }, + "reverted_card_payment?": { + "f1": 0.825, + "fn": 7, + "fp": 7, + "support": 40, + "tp": 33 + }, + "supported_cards_and_currencies": { + "f1": 0.7777777777777778, + "fn": 12, + "fp": 4, + "support": 40, + "tp": 28 + }, + "terminate_account": { + "f1": 0.9411764705882353, + "fn": 0, + "fp": 5, + "support": 40, + "tp": 40 + }, + "top_up_by_bank_transfer_charge": { + "f1": 0.8888888888888888, + "fn": 4, + "fp": 5, + "support": 40, + "tp": 36 + }, + "top_up_by_card_charge": { + "f1": 0.95, + "fn": 2, + "fp": 2, + "support": 40, + "tp": 38 + }, + "top_up_by_cash_or_cheque": { + "f1": 0.9230769230769231, + "fn": 4, + "fp": 2, + "support": 40, + "tp": 36 + }, + "top_up_failed": { + "f1": 0.8705882352941177, + "fn": 3, + "fp": 8, + "support": 40, + "tp": 37 + }, + "top_up_limits": { + "f1": 0.9512195121951219, + "fn": 1, + "fp": 3, + "support": 40, + "tp": 39 + }, + "top_up_reverted": { + "f1": 0.875, + "fn": 5, + "fp": 5, + "support": 40, + "tp": 35 + }, + "topping_up_by_card": { + "f1": 0.5, + "fn": 23, + "fp": 11, + "support": 40, + "tp": 17 + }, + "transaction_charged_twice": { + "f1": 0.9629629629629629, + "fn": 1, + "fp": 2, + "support": 40, + "tp": 39 + }, + "transfer_fee_charged": { + "f1": 0.8333333333333334, + "fn": 10, + "fp": 2, + "support": 40, + "tp": 30 + }, + "transfer_into_account": { + "f1": 0.7659574468085106, + "fn": 4, + "fp": 18, + "support": 40, + "tp": 36 + }, + "transfer_not_received_by_recipient": { + "f1": 0.7088607594936709, + "fn": 12, + "fp": 11, + "support": 40, + "tp": 28 + }, + "transfer_timing": { + "f1": 0.7333333333333333, + "fn": 7, + "fp": 17, + "support": 40, + "tp": 33 + }, + "unable_to_verify_identity": { + "f1": 0.926829268292683, + "fn": 2, + "fp": 4, + "support": 40, + "tp": 38 + }, + "verify_my_identity": { + "f1": 0.9024390243902439, + "fn": 3, + "fp": 5, + "support": 40, + "tp": 37 + }, + "verify_source_of_funds": { + "f1": 1.0, + "fn": 0, + "fp": 0, + "support": 40, + "tp": 40 + }, + "verify_top_up": { + "f1": 1.0, + "fn": 0, + "fp": 0, + "support": 40, + "tp": 40 + }, + "virtual_card_not_working": { + "f1": 0.9333333333333333, + "fn": 5, + "fp": 0, + "support": 40, + "tp": 35 + }, + "visa_or_mastercard": { + "f1": 0.9333333333333333, + "fn": 5, + "fp": 0, + "support": 40, + "tp": 35 + }, + "why_verify_identity": { + "f1": 0.9090909090909091, + "fn": 5, + "fp": 2, + "support": 40, + "tp": 35 + }, + "wrong_amount_of_cash_received": { + "f1": 0.9090909090909091, + "fn": 5, + "fp": 2, + "support": 40, + "tp": 35 + }, + "wrong_exchange_rate_for_cash_withdrawal": { + "f1": 0.7878787878787878, + "fn": 14, + "fp": 0, + "support": 40, + "tp": 26 + } + }, + "prototype_classes": 77 + }, + "massive-en": { + "accuracy": { + "denominator": 2972, + "numerator": 2436, + "value": 0.819650067294751 + }, + "evaluated_classes": 59, + "macro_f1": 0.7923223418549947, + "macro_f1_class_universe": "all fixed prototype classes; zero_division=0", + "per_class": { + "alarm_query": { + "f1": 0.9523809523809523, + "fn": 3, + "fp": 0, + "support": 33, + "tp": 30 + }, + "alarm_remove": { + "f1": 0.9545454545454546, + "fn": 0, + "fp": 2, + "support": 21, + "tp": 21 + }, + "alarm_set": { + "f1": 0.9047619047619048, + "fn": 3, + "fp": 5, + "support": 41, + "tp": 38 + }, + "audio_volume_down": { + "f1": 0.9565217391304348, + "fn": 0, + "fp": 1, + "support": 11, + "tp": 11 + }, + "audio_volume_mute": { + "f1": 0.8787878787878788, + "fn": 3, + "fp": 5, + "support": 32, + "tp": 29 + }, + "audio_volume_other": { + "f1": 1.0, + "fn": 0, + "fp": 0, + "support": 6, + "tp": 6 + }, + "audio_volume_up": { + "f1": 0.782608695652174, + "fn": 4, + "fp": 1, + "support": 13, + "tp": 9 + }, + "calendar_query": { + "f1": 0.6864406779661016, + "fn": 45, + "fp": 29, + "support": 126, + "tp": 81 + }, + "calendar_remove": { + "f1": 0.9264705882352942, + "fn": 4, + "fp": 6, + "support": 67, + "tp": 63 + }, + "calendar_set": { + "f1": 0.8759124087591241, + "fn": 29, + "fp": 22, + "support": 209, + "tp": 180 + }, + "cooking_query": { + "f1": 0.0, + "fn": 0, + "fp": 2, + "support": 0, + "tp": 0 + }, + "cooking_recipe": { + "f1": 0.9523809523809523, + "fn": 2, + "fp": 5, + "support": 72, + "tp": 70 + }, + "datetime_convert": { + "f1": 0.5263157894736842, + "fn": 5, + "fp": 13, + "support": 15, + "tp": 10 + }, + "datetime_query": { + "f1": 0.7861271676300579, + "fn": 20, + "fp": 17, + "support": 88, + "tp": 68 + }, + "email_addcontact": { + "f1": 0.5714285714285714, + "fn": 2, + "fp": 13, + "support": 12, + "tp": 10 + }, + "email_query": { + "f1": 0.9385964912280702, + "fn": 12, + "fp": 2, + "support": 119, + "tp": 107 + }, + "email_querycontact": { + "f1": 0.7540983606557377, + "fn": 3, + "fp": 12, + "support": 26, + "tp": 23 + }, + "email_sendemail": { + "f1": 0.8959276018099548, + "fn": 15, + "fp": 8, + "support": 114, + "tp": 99 + }, + "general_greet": { + "f1": 0.18181818181818182, + "fn": 0, + "fp": 9, + "support": 1, + "tp": 1 + }, + "general_joke": { + "f1": 0.9230769230769231, + "fn": 1, + "fp": 2, + "support": 19, + "tp": 18 + }, + "general_quirky": { + "f1": 0.4728434504792332, + "fn": 94, + "fp": 71, + "support": 168, + "tp": 74 + }, + "iot_cleaning": { + "f1": 0.9433962264150944, + "fn": 1, + "fp": 2, + "support": 26, + "tp": 25 + }, + "iot_coffee": { + "f1": 0.9859154929577465, + "fn": 1, + "fp": 0, + "support": 36, + "tp": 35 + }, + "iot_hue_lightchange": { + "f1": 0.8181818181818182, + "fn": 9, + "fp": 3, + "support": 36, + "tp": 27 + }, + "iot_hue_lightdim": { + "f1": 0.8780487804878049, + "fn": 3, + "fp": 2, + "support": 21, + "tp": 18 + }, + "iot_hue_lightoff": { + "f1": 0.9176470588235294, + "fn": 4, + "fp": 3, + "support": 43, + "tp": 39 + }, + "iot_hue_lighton": { + "f1": 0.18181818181818182, + "fn": 2, + "fp": 7, + "support": 3, + "tp": 1 + }, + "iot_hue_lightup": { + "f1": 0.7857142857142857, + "fn": 5, + "fp": 7, + "support": 27, + "tp": 22 + }, + "iot_wemo_off": { + "f1": 0.9444444444444444, + "fn": 1, + "fp": 1, + "support": 18, + "tp": 17 + }, + "iot_wemo_on": { + "f1": 0.9, + "fn": 1, + "fp": 1, + "support": 10, + "tp": 9 + }, + "lists_createoradd": { + "f1": 0.7777777777777778, + "fn": 11, + "fp": 5, + "support": 39, + "tp": 28 + }, + "lists_query": { + "f1": 0.8333333333333334, + "fn": 11, + "fp": 5, + "support": 51, + "tp": 40 + }, + "lists_remove": { + "f1": 0.9215686274509803, + "fn": 5, + "fp": 3, + "support": 52, + "tp": 47 + }, + "music_dislikeness": { + "f1": 0.8888888888888888, + "fn": 0, + "fp": 1, + "support": 4, + "tp": 4 + }, + "music_likeness": { + "f1": 0.7878787878787878, + "fn": 10, + "fp": 4, + "support": 36, + "tp": 26 + }, + "music_query": { + "f1": 0.7945205479452054, + "fn": 6, + "fp": 9, + "support": 35, + "tp": 29 + }, + "music_settings": { + "f1": 0.36363636363636365, + "fn": 2, + "fp": 12, + "support": 6, + "tp": 4 + }, + "news_query": { + "f1": 0.8583333333333333, + "fn": 21, + "fp": 13, + "support": 124, + "tp": 103 + }, + "play_audiobook": { + "f1": 0.8311688311688312, + "fn": 9, + "fp": 4, + "support": 41, + "tp": 32 + }, + "play_game": { + "f1": 0.717391304347826, + "fn": 2, + "fp": 24, + "support": 35, + "tp": 33 + }, + "play_music": { + "f1": 0.8238805970149253, + "fn": 38, + "fp": 21, + "support": 176, + "tp": 138 + }, + "play_podcasts": { + "f1": 0.9043478260869565, + "fn": 11, + "fp": 0, + "support": 63, + "tp": 52 + }, + "play_radio": { + "f1": 0.8208955223880597, + "fn": 17, + "fp": 7, + "support": 72, + "tp": 55 + }, + "qa_currency": { + "f1": 0.9473684210526315, + "fn": 3, + "fp": 1, + "support": 39, + "tp": 36 + }, + "qa_definition": { + "f1": 0.8305084745762712, + "fn": 8, + "fp": 12, + "support": 57, + "tp": 49 + }, + "qa_factoid": { + "f1": 0.7205882352941176, + "fn": 43, + "fp": 33, + "support": 141, + "tp": 98 + }, + "qa_maths": { + "f1": 0.7868852459016393, + "fn": 1, + "fp": 12, + "support": 25, + "tp": 24 + }, + "qa_stock": { + "f1": 0.8727272727272727, + "fn": 2, + "fp": 5, + "support": 26, + "tp": 24 + }, + "recommendation_events": { + "f1": 0.7102803738317757, + "fn": 5, + "fp": 26, + "support": 43, + "tp": 38 + }, + "recommendation_locations": { + "f1": 0.7945205479452054, + "fn": 2, + "fp": 13, + "support": 31, + "tp": 29 + }, + "recommendation_movies": { + "f1": 0.7058823529411765, + "fn": 2, + "fp": 13, + "support": 20, + "tp": 18 + }, + "social_post": { + "f1": 0.8531468531468531, + "fn": 20, + "fp": 1, + "support": 81, + "tp": 61 + }, + "social_query": { + "f1": 0.7301587301587301, + "fn": 2, + "fp": 15, + "support": 25, + "tp": 23 + }, + "takeaway_order": { + "f1": 0.7659574468085106, + "fn": 4, + "fp": 7, + "support": 22, + "tp": 18 + }, + "takeaway_query": { + "f1": 0.8266666666666667, + "fn": 4, + "fp": 9, + "support": 35, + "tp": 31 + }, + "transport_query": { + "f1": 0.782608695652174, + "fn": 15, + "fp": 5, + "support": 51, + "tp": 36 + }, + "transport_taxi": { + "f1": 0.9787234042553191, + "fn": 0, + "fp": 1, + "support": 23, + "tp": 23 + }, + "transport_ticket": { + "f1": 0.88, + "fn": 2, + "fp": 7, + "support": 35, + "tp": 33 + }, + "transport_traffic": { + "f1": 0.8108108108108109, + "fn": 0, + "fp": 7, + "support": 15, + "tp": 15 + }, + "weather_query": { + "f1": 0.9426751592356688, + "fn": 8, + "fp": 10, + "support": 156, + "tp": 148 + } + }, + "prototype_classes": 60 + } } }, "standard_benchmarks": { @@ -479,15 +1370,11 @@ "nano": { "repo_id": "llm-semantic-router/Vela-1.0-Omni-Nano", "revision": "self", - "native_artifact_sha256": "50ce808197fb43a1b8913a66d4a47339ec66ecd10d52cbd9f9e752d02f8c43fa", - "parameters": 135383808, + "native_artifact_sha256": "474ad3d52f6ee7f818a3dbb6112e3d66c2423542d08228e7996a8cd600bd99a2", + "parameters": 163771288, "embedding_dimensions": 384, "max_text_tokens": 512, - "evaluated_native_artifact_sha256": "96d76f97478e114b816df3ab1b3c6947e4c5ef0babfb40ab865d7e3c838d34c4", - "evaluated_revision": "de02c367bc9aa7a6f7ad3e59fb38962502e021c1", - "evaluated_parameters": 144554112, - "inference_equivalence_report": "benchmarks/inference-equivalence.json", - "revision_kind": "HF_snapshot" + "applicability": "Each row states whether the full artifact was freshly evaluated or an exact unchanged component applies." }, "small": { "repo_id": "llm-semantic-router/multi-modal-embed-small", @@ -507,7 +1394,7 @@ "classification": "Official task-specific frozen-embedding probes or fixed-label zero-shot classification.", "sampling": "Official task subsampling and repeated-probe protocols are preserved.", "text": "Each encoder uses its native preprocessing and context limit without added task instructions. Original small: 128 tokens; current Nano: 512 tokens.", - "audio": "Mono 16 kHz, first 30 seconds, official MTEB audio preprocessing.", + "audio": "Current Nano: original PCM independently resampled to 16/48 kHz for speech/CLAP, then downmixed and capped at 30 seconds. Original small: official mono 16 kHz path. The official task algorithms, samples and seed are identical.", "score_scale": "Metrics retain their native MTEB scale; the model card multiplies primary scores by 100.", "undefined_metric": null, "missing_model_score": "Not measured in this standard evaluation.", @@ -574,8 +1461,16 @@ "original_checkpoint_revision": "fdf8e01b7b0f3a69ac1ac8e2a64dcb1ede177ba4", "raw_result_sha256": "e20f02a347841b3d5f61209f0b209c6314d006f8cc94f6e488e06239c862badf", "kind": "direct_frozen_encoder_evaluation" + }, + "nano": { + "evaluated_native_fingerprint": "96d76f97478e114b816df3ab1b3c6947e4c5ef0babfb40ab865d7e3c838d34c4", + "applicable_native_fingerprint": "474ad3d52f6ee7f818a3dbb6112e3d66c2423542d08228e7996a8cd600bd99a2", + "kind": "exact_component_applicability" } - } + }, + "evaluated_native_fingerprint": "96d76f97478e114b816df3ab1b3c6947e4c5ef0babfb40ab865d7e3c838d34c4", + "applicable_native_fingerprint": "474ad3d52f6ee7f818a3dbb6112e3d66c2423542d08228e7996a8cd600bd99a2", + "measurement_kind": "exact_component_applicability" }, { "task": "STSBenchmark", @@ -637,8 +1532,16 @@ "original_checkpoint_revision": "fdf8e01b7b0f3a69ac1ac8e2a64dcb1ede177ba4", "raw_result_sha256": "ea0772149930250452298e3525b3405ce4fd5caed4aaf676f3745d4555692a07", "kind": "direct_frozen_encoder_evaluation" + }, + "nano": { + "evaluated_native_fingerprint": "96d76f97478e114b816df3ab1b3c6947e4c5ef0babfb40ab865d7e3c838d34c4", + "applicable_native_fingerprint": "474ad3d52f6ee7f818a3dbb6112e3d66c2423542d08228e7996a8cd600bd99a2", + "kind": "exact_component_applicability" } - } + }, + "evaluated_native_fingerprint": "96d76f97478e114b816df3ab1b3c6947e4c5ef0babfb40ab865d7e3c838d34c4", + "applicable_native_fingerprint": "474ad3d52f6ee7f818a3dbb6112e3d66c2423542d08228e7996a8cd600bd99a2", + "measurement_kind": "exact_component_applicability" }, { "task": "ArguAna", @@ -981,8 +1884,16 @@ "original_checkpoint_revision": "fdf8e01b7b0f3a69ac1ac8e2a64dcb1ede177ba4", "raw_result_sha256": "92cbe96f6e84ebd0bc144001424ac46bb3a630ef64bffb433c2cdd63a10c43ec", "kind": "direct_frozen_encoder_evaluation" + }, + "nano": { + "evaluated_native_fingerprint": "96d76f97478e114b816df3ab1b3c6947e4c5ef0babfb40ab865d7e3c838d34c4", + "applicable_native_fingerprint": "474ad3d52f6ee7f818a3dbb6112e3d66c2423542d08228e7996a8cd600bd99a2", + "kind": "exact_component_applicability" } - } + }, + "evaluated_native_fingerprint": "96d76f97478e114b816df3ab1b3c6947e4c5ef0babfb40ab865d7e3c838d34c4", + "applicable_native_fingerprint": "474ad3d52f6ee7f818a3dbb6112e3d66c2423542d08228e7996a8cd600bd99a2", + "measurement_kind": "exact_component_applicability" }, { "task": "TwentyNewsgroupsClustering.v2", @@ -1096,8 +2007,16 @@ "original_checkpoint_revision": "fdf8e01b7b0f3a69ac1ac8e2a64dcb1ede177ba4", "raw_result_sha256": "62fe52980cb4445b9969d53135fa1663dd8dcd811743a711dde55f2a097ced19", "kind": "direct_frozen_encoder_evaluation" + }, + "nano": { + "evaluated_native_fingerprint": "96d76f97478e114b816df3ab1b3c6947e4c5ef0babfb40ab865d7e3c838d34c4", + "applicable_native_fingerprint": "474ad3d52f6ee7f818a3dbb6112e3d66c2423542d08228e7996a8cd600bd99a2", + "kind": "exact_component_applicability" } - } + }, + "evaluated_native_fingerprint": "96d76f97478e114b816df3ab1b3c6947e4c5ef0babfb40ab865d7e3c838d34c4", + "applicable_native_fingerprint": "474ad3d52f6ee7f818a3dbb6112e3d66c2423542d08228e7996a8cd600bd99a2", + "measurement_kind": "exact_component_applicability" }, { "task": "MTOPDomainClassification", @@ -1388,8 +2307,16 @@ "original_checkpoint_revision": "fdf8e01b7b0f3a69ac1ac8e2a64dcb1ede177ba4", "raw_result_sha256": "256702650b93bd559d9a2a9b70f32b996b51e166debc0c20d63ce24cc6f605f6", "kind": "direct_frozen_encoder_evaluation" + }, + "nano": { + "evaluated_native_fingerprint": "96d76f97478e114b816df3ab1b3c6947e4c5ef0babfb40ab865d7e3c838d34c4", + "applicable_native_fingerprint": "474ad3d52f6ee7f818a3dbb6112e3d66c2423542d08228e7996a8cd600bd99a2", + "kind": "exact_component_applicability" } - } + }, + "evaluated_native_fingerprint": "96d76f97478e114b816df3ab1b3c6947e4c5ef0babfb40ab865d7e3c838d34c4", + "applicable_native_fingerprint": "474ad3d52f6ee7f818a3dbb6112e3d66c2423542d08228e7996a8cd600bd99a2", + "measurement_kind": "exact_component_applicability" }, { "task": "MTOPDomainClassification", @@ -1680,8 +2607,16 @@ "original_checkpoint_revision": "fdf8e01b7b0f3a69ac1ac8e2a64dcb1ede177ba4", "raw_result_sha256": "256702650b93bd559d9a2a9b70f32b996b51e166debc0c20d63ce24cc6f605f6", "kind": "direct_frozen_encoder_evaluation" + }, + "nano": { + "evaluated_native_fingerprint": "96d76f97478e114b816df3ab1b3c6947e4c5ef0babfb40ab865d7e3c838d34c4", + "applicable_native_fingerprint": "474ad3d52f6ee7f818a3dbb6112e3d66c2423542d08228e7996a8cd600bd99a2", + "kind": "exact_component_applicability" } - } + }, + "evaluated_native_fingerprint": "96d76f97478e114b816df3ab1b3c6947e4c5ef0babfb40ab865d7e3c838d34c4", + "applicable_native_fingerprint": "474ad3d52f6ee7f818a3dbb6112e3d66c2423542d08228e7996a8cd600bd99a2", + "measurement_kind": "exact_component_applicability" }, { "task": "OxfordPets", @@ -1862,8 +2797,16 @@ "original_checkpoint_revision": "fdf8e01b7b0f3a69ac1ac8e2a64dcb1ede177ba4", "raw_result_sha256": "ef1f4928b9ae7d9da25a98b82a28fda3e098db9504ad9372cb5305ad32421a06", "kind": "direct_frozen_encoder_evaluation" + }, + "nano": { + "evaluated_native_fingerprint": "96d76f97478e114b816df3ab1b3c6947e4c5ef0babfb40ab865d7e3c838d34c4", + "applicable_native_fingerprint": "474ad3d52f6ee7f818a3dbb6112e3d66c2423542d08228e7996a8cd600bd99a2", + "kind": "exact_component_applicability" } - } + }, + "evaluated_native_fingerprint": "96d76f97478e114b816df3ab1b3c6947e4c5ef0babfb40ab865d7e3c838d34c4", + "applicable_native_fingerprint": "474ad3d52f6ee7f818a3dbb6112e3d66c2423542d08228e7996a8cd600bd99a2", + "measurement_kind": "exact_component_applicability" }, { "task": "TinyImageNetClustering", @@ -1917,8 +2860,16 @@ "original_checkpoint_revision": "fdf8e01b7b0f3a69ac1ac8e2a64dcb1ede177ba4", "raw_result_sha256": "eb344e20e7d498aadbbd67a57015cc2220cba42b9550f5822b5a72d0d6cf1514", "kind": "direct_frozen_encoder_evaluation" + }, + "nano": { + "evaluated_native_fingerprint": "96d76f97478e114b816df3ab1b3c6947e4c5ef0babfb40ab865d7e3c838d34c4", + "applicable_native_fingerprint": "474ad3d52f6ee7f818a3dbb6112e3d66c2423542d08228e7996a8cd600bd99a2", + "kind": "exact_component_applicability" } - } + }, + "evaluated_native_fingerprint": "96d76f97478e114b816df3ab1b3c6947e4c5ef0babfb40ab865d7e3c838d34c4", + "applicable_native_fingerprint": "474ad3d52f6ee7f818a3dbb6112e3d66c2423542d08228e7996a8cd600bd99a2", + "measurement_kind": "exact_component_applicability" }, { "task": "OxfordPetsZeroShot", @@ -1984,8 +2935,16 @@ "original_checkpoint_revision": "fdf8e01b7b0f3a69ac1ac8e2a64dcb1ede177ba4", "raw_result_sha256": "e52e42c4716e62710296df98a90b3a0b986875685748d043cfd7de913e5a6c42", "kind": "direct_frozen_encoder_evaluation" + }, + "nano": { + "evaluated_native_fingerprint": "96d76f97478e114b816df3ab1b3c6947e4c5ef0babfb40ab865d7e3c838d34c4", + "applicable_native_fingerprint": "474ad3d52f6ee7f818a3dbb6112e3d66c2423542d08228e7996a8cd600bd99a2", + "kind": "exact_component_applicability" } - } + }, + "evaluated_native_fingerprint": "96d76f97478e114b816df3ab1b3c6947e4c5ef0babfb40ab865d7e3c838d34c4", + "applicable_native_fingerprint": "474ad3d52f6ee7f818a3dbb6112e3d66c2423542d08228e7996a8cd600bd99a2", + "measurement_kind": "exact_component_applicability" }, { "task": "CREMA_D", @@ -2006,83 +2965,83 @@ "probe_samples_per_label": 8 }, "scores": { - "nano": 0.29763670140167686, + "nano": 0.32048146984697823, "small": 0.2995196855795547 }, "metrics": { "nano": { + "accuracy": 0.32048146984697823, + "ap": null, + "ap_weighted": null, + "f1": 0.303943978298811, + "f1_weighted": 0.30451603344534794, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.32048146984697823, + "mteb_version": "2.21.0", + "precision": 0.31510472165152337, + "precision_weighted": 0.3169398085592857, + "recall": 0.3215883809208585, + "recall_weighted": 0.32048146984697823, "scores_per_experiment": [ { - "accuracy": 0.2847548690396239, - "f1": 0.25980556646698677, - "f1_weighted": 0.2606862265023784, - "precision": 0.2685141966762838, - "precision_weighted": 0.26869432571348717, - "recall": 0.283361264324777, - "recall_weighted": 0.2847548690396239, + "accuracy": 0.30288784419073206, "ap": null, - "ap_weighted": null + "ap_weighted": null, + "f1": 0.27969233326985626, + "f1_weighted": 0.2799132430776581, + "precision": 0.29992207114753305, + "precision_weighted": 0.29961025331880187, + "recall": 0.3024507805686864, + "recall_weighted": 0.30288784419073206 }, { - "accuracy": 0.2981867024848892, - "f1": 0.2927999429509691, - "f1_weighted": 0.2952410831252662, - "precision": 0.29988591178720175, - "precision_weighted": 0.30336098208055257, - "recall": 0.29763527945216217, - "recall_weighted": 0.2981867024848892, + "accuracy": 0.3149764942914708, "ap": null, - "ap_weighted": null + "ap_weighted": null, + "f1": 0.30808145721796526, + "f1_weighted": 0.31085435981340914, + "precision": 0.336772371610985, + "precision_weighted": 0.3411130620509704, + "recall": 0.3160853641774642, + "recall_weighted": 0.3149764942914708 }, { - "accuracy": 0.28696236559139787, - "f1": 0.26582489649494206, - "f1_weighted": 0.26920117230368773, - "precision": 0.26489987169540286, - "precision_weighted": 0.2682170436584329, - "recall": 0.2834616565816636, - "recall_weighted": 0.28696236559139787, + "accuracy": 0.31586021505376344, "ap": null, - "ap_weighted": null + "ap_weighted": null, + "f1": 0.2931797954478092, + "f1_weighted": 0.2955391094563843, + "precision": 0.2916058030686394, + "precision_weighted": 0.29407622403946154, + "recall": 0.312973406205352, + "recall_weighted": 0.31586021505376344 }, { - "accuracy": 0.29973118279569894, - "f1": 0.291485315935673, - "f1_weighted": 0.28841683151437103, - "precision": 0.3068565359099591, - "precision_weighted": 0.3086441612810119, - "recall": 0.30801945566008065, - "recall_weighted": 0.29973118279569894, + "accuracy": 0.32123655913978494, "ap": null, - "ap_weighted": null + "ap_weighted": null, + "f1": 0.3066232673451424, + "f1_weighted": 0.3032561325852315, + "precision": 0.3141358238179796, + "precision_weighted": 0.31520429243540943, + "recall": 0.32961458988542325, + "recall_weighted": 0.32123655913978494 }, { - "accuracy": 0.3185483870967742, - "f1": 0.3051858414398825, - "f1_weighted": 0.3066161074219095, - "precision": 0.310702613862649, - "precision_weighted": 0.3133755862555043, - "recall": 0.31849511838471833, - "recall_weighted": 0.3185483870967742, + "accuracy": 0.3474462365591398, "ap": null, - "ap_weighted": null + "ap_weighted": null, + "f1": 0.3321430382132817, + "f1_weighted": 0.3330173222940568, + "precision": 0.3330875386124797, + "precision_weighted": 0.3346952109517855, + "recall": 0.3468177637673669, + "recall_weighted": 0.3474462365591398 } - ], - "accuracy": 0.29763670140167686, - "f1": 0.2830203126576907, - "f1_weighted": 0.2840322841735226, - "precision": 0.29017182598629926, - "precision_weighted": 0.2924584197977978, - "recall": 0.2981945548806803, - "recall_weighted": 0.29763670140167686, - "ap": null, - "ap_weighted": null, - "main_score": 0.29763670140167686, - "hf_subset": "default", - "languages": [ - "eng-Latn" - ], - "mteb_version": "2.21.0" + ] }, "small": { "scores_per_experiment": [ @@ -2165,8 +3124,18 @@ "original_checkpoint_revision": "fdf8e01b7b0f3a69ac1ac8e2a64dcb1ede177ba4", "raw_result_sha256": "d79b8d114cb86c460dad13d564c2705a41df5119f83a0f1f27d452f36b64af47", "kind": "direct_frozen_encoder_evaluation" + }, + "nano": { + "evaluated_native_fingerprint": "474ad3d52f6ee7f818a3dbb6112e3d66c2423542d08228e7996a8cd600bd99a2", + "applicable_native_fingerprint": "474ad3d52f6ee7f818a3dbb6112e3d66c2423542d08228e7996a8cd600bd99a2", + "kind": "fresh_native", + "raw_result_sha256": "62439b0ed4b3a8c26b1a8f122880a7bd3b3d86566841333a9060a4bbd0954baf" } - } + }, + "evaluated_native_fingerprint": "474ad3d52f6ee7f818a3dbb6112e3d66c2423542d08228e7996a8cd600bd99a2", + "applicable_native_fingerprint": "474ad3d52f6ee7f818a3dbb6112e3d66c2423542d08228e7996a8cd600bd99a2", + "measurement_kind": "fresh_native", + "raw_result_sha256": "62439b0ed4b3a8c26b1a8f122880a7bd3b3d86566841333a9060a4bbd0954baf" }, { "task": "CREMA_DClustering", @@ -2187,49 +3156,49 @@ "resampled_items_per_trial": 16384 }, "scores": { - "nano": 0.023601715918443712, + "nano": 0.05214346580804754, "small": 0.024058408888362488 }, "metrics": { "nano": { - "v_measures": { - "Level 0": [ - 0.014290784661453301, - 0.02755216925549553, - 0.017220745995259155, - 0.02907041540588703, - 0.017365020267924067, - 0.022161217359367304, - 0.02806069377818825, - 0.017815432451092263, - 0.03081951157077731, - 0.03166116843899292 - ] - }, - "v_measure": 0.023601715918443712, - "v_measure_std": 0.006199268435462002, + "ami": 0.05172990441350066, "ami_scores": { "Level 0": [ - 0.01385865484522556, - 0.027134003314929028, - 0.016785414361355624, - 0.028644278423611804, - 0.016937404067902156, - 0.02173522206231376, - 0.0276339699482267, - 0.0173943490723341, - 0.030397587303172737, - 0.031246814385090853 + 0.07691371568295576, + 0.05921735688360533, + 0.040011204150084696, + 0.04041243062073308, + 0.042015892825421196, + 0.06432477056741849, + 0.06285137443486444, + 0.05640769402749227, + 0.058836854203017874, + 0.01630775073941345 ] }, - "ami": 0.023176769778416234, - "ami_std": 0.0062032784119843, - "main_score": 0.023601715918443712, + "ami_std": 0.016307969325946296, "hf_subset": "default", "languages": [ "eng-Latn" ], - "mteb_version": "2.21.0" + "main_score": 0.05214346580804754, + "mteb_version": "2.21.0", + "v_measure": 0.05214346580804754, + "v_measure_std": 0.016298021302929667, + "v_measures": { + "Level 0": [ + 0.07731089214293434, + 0.05962833919011462, + 0.040438217966279694, + 0.04084038764100412, + 0.04243023115915348, + 0.06473220137035035, + 0.06325438263275382, + 0.05681796105920958, + 0.059243669688975126, + 0.016738375229700227 + ] + } }, "small": { "v_measures": { @@ -2278,8 +3247,18 @@ "original_checkpoint_revision": "fdf8e01b7b0f3a69ac1ac8e2a64dcb1ede177ba4", "raw_result_sha256": "b23d4c552b9fa242923fc0c21a898ee63eb32850d9d91e0682b2cb5e8bea0e76", "kind": "direct_frozen_encoder_evaluation" + }, + "nano": { + "evaluated_native_fingerprint": "474ad3d52f6ee7f818a3dbb6112e3d66c2423542d08228e7996a8cd600bd99a2", + "applicable_native_fingerprint": "474ad3d52f6ee7f818a3dbb6112e3d66c2423542d08228e7996a8cd600bd99a2", + "kind": "fresh_native", + "raw_result_sha256": "89db580a8e7039474d3bf150da208adbd98eabce358dad64a85f6c2027d4d62e" } - } + }, + "evaluated_native_fingerprint": "474ad3d52f6ee7f818a3dbb6112e3d66c2423542d08228e7996a8cd600bd99a2", + "applicable_native_fingerprint": "474ad3d52f6ee7f818a3dbb6112e3d66c2423542d08228e7996a8cd600bd99a2", + "measurement_kind": "fresh_native", + "raw_result_sha256": "89db580a8e7039474d3bf150da208adbd98eabce358dad64a85f6c2027d4d62e" }, { "task": "SpeechCommandsZeroshotv0.02", @@ -2300,26 +3279,26 @@ "fixed_text_labels": 10 }, "scores": { - "nano": 0.1411389297987236, + "nano": 0.15709376534118802, "small": 0.05522827687776141 }, "metrics": { "nano": { - "accuracy": 0.1411389297987236, - "f1": 0.06629702019800886, - "f1_weighted": 0.06559730133350446, - "precision": 0.21924492134747023, - "precision_weighted": 0.21974351147386872, - "recall": 0.14280319771568778, - "recall_weighted": 0.1411389297987236, + "accuracy": 0.15709376534118802, "ap": null, "ap_weighted": null, - "main_score": 0.1411389297987236, + "f1": 0.08449976339531617, + "f1_weighted": 0.0833837827950463, "hf_subset": "default", "languages": [ "eng-Latn" ], - "mteb_version": "2.21.0" + "main_score": 0.15709376534118802, + "mteb_version": "2.21.0", + "precision": 0.07985904570619766, + "precision_weighted": 0.078775844073094, + "recall": 0.15917885320870395, + "recall_weighted": 0.15709376534118802 }, "small": { "accuracy": 0.05522827687776141, @@ -2345,18 +3324,28 @@ "original_checkpoint_revision": "fdf8e01b7b0f3a69ac1ac8e2a64dcb1ede177ba4", "raw_result_sha256": "ca19c03bbe2fdcf3c2e1282f5e826af702c152287fde75d67b28ef877fae5335", "kind": "direct_frozen_encoder_evaluation" + }, + "nano": { + "evaluated_native_fingerprint": "474ad3d52f6ee7f818a3dbb6112e3d66c2423542d08228e7996a8cd600bd99a2", + "applicable_native_fingerprint": "474ad3d52f6ee7f818a3dbb6112e3d66c2423542d08228e7996a8cd600bd99a2", + "kind": "fresh_native", + "raw_result_sha256": "c5b151588a51abbf4ca2e9bc88ec91d235b0971d8035f02e97b7c58910f66538" } - } + }, + "evaluated_native_fingerprint": "474ad3d52f6ee7f818a3dbb6112e3d66c2423542d08228e7996a8cd600bd99a2", + "applicable_native_fingerprint": "474ad3d52f6ee7f818a3dbb6112e3d66c2423542d08228e7996a8cd600bd99a2", + "measurement_kind": "fresh_native", + "raw_result_sha256": "c5b151588a51abbf4ca2e9bc88ec91d235b0971d8035f02e97b7c58910f66538" } ], "comparison": { "baseline_model": "llm-semantic-router/multi-modal-embed-small", "baseline_revision": "fdf8e01b7b0f3a69ac1ac8e2a64dcb1ede177ba4", "baseline_native_artifact_sha256": "684fe0c48a4ddd3944167f6e5dc75e97ada24e5b120bd9a2921f00dc03112eeb", - "baseline_standard_status": "not_evaluated_in_this_standard_protocol", + "baseline_standard_status": "evaluated_all_11_selected_tasks", "card_bold": "Only unrounded gains against this original baseline are bold; missing baseline scores imply no comparison." }, - "observation_scope": "Only this model is reported; original comparator has no observations in the fixed11 protocol. Evaluated/applicable identities are preserved.", + "observation_scope": "Source-specific counters only: eight unchanged text/image tasks retain their actual earlier measurement; three audio tasks are fresh. Old timing is not a measurement of the new audio architecture.", "encoding_observations": { "SICK-R": { "nano": { @@ -2527,66 +3516,48 @@ } }, "CREMA_D": { - "nano": { - "before": { - "text_rows": 43459, - "text_rows_truncated": 222, - "max_original_text_tokens": 1401, - "image_rows": 19485, - "audio_rows": 0 - }, - "after": { - "text_rows": 43459, - "text_rows_truncated": 222, - "max_original_text_tokens": 1401, - "image_rows": 19485, - "audio_rows": 7442 + "evaluated_native_fingerprint": "474ad3d52f6ee7f818a3dbb6112e3d66c2423542d08228e7996a8cd600bd99a2", + "observations": { + "cumulative_rows": { + "audio_rows": 7442, + "image_rows": 0, + "max_original_text_tokens": 0, + "text_rows": 0, + "text_rows_truncated": 0 }, - "elapsed_seconds": 135.41051389899803, - "peak_allocated_gpu_bytes": 971823104, - "peak_reserved_gpu_bytes": 1224736768 + "peak_allocated_bytes": 966564864, + "peak_reserved_bytes": 1493172224, + "seconds": 228.799308784015 } }, "CREMA_DClustering": { - "nano": { - "before": { - "text_rows": 43459, - "text_rows_truncated": 222, - "max_original_text_tokens": 1401, - "image_rows": 19485, - "audio_rows": 7442 - }, - "after": { - "text_rows": 43459, - "text_rows_truncated": 222, - "max_original_text_tokens": 1401, - "image_rows": 19485, - "audio_rows": 9490 + "evaluated_native_fingerprint": "474ad3d52f6ee7f818a3dbb6112e3d66c2423542d08228e7996a8cd600bd99a2", + "observations": { + "cumulative_rows": { + "audio_rows": 9490, + "image_rows": 0, + "max_original_text_tokens": 0, + "text_rows": 0, + "text_rows_truncated": 0 }, - "elapsed_seconds": 73.78414583200356, - "peak_allocated_gpu_bytes": 971823104, - "peak_reserved_gpu_bytes": 1224736768 + "peak_allocated_bytes": 966577152, + "peak_reserved_bytes": 1321205760, + "seconds": 102.58190695600933 } }, "SpeechCommandsZeroshotv0.02": { - "nano": { - "before": { - "text_rows": 43459, - "text_rows_truncated": 222, - "max_original_text_tokens": 1401, - "image_rows": 19485, - "audio_rows": 9490 - }, - "after": { - "text_rows": 43469, - "text_rows_truncated": 222, - "max_original_text_tokens": 1401, - "image_rows": 19485, - "audio_rows": 13564 + "evaluated_native_fingerprint": "474ad3d52f6ee7f818a3dbb6112e3d66c2423542d08228e7996a8cd600bd99a2", + "observations": { + "cumulative_rows": { + "audio_rows": 13564, + "image_rows": 0, + "max_original_text_tokens": 3, + "text_rows": 10, + "text_rows_truncated": 0 }, - "elapsed_seconds": 63.873327202978544, - "peak_allocated_gpu_bytes": 971823104, - "peak_reserved_gpu_bytes": 1224736768 + "peak_allocated_bytes": 966577152, + "peak_reserved_bytes": 1329594368, + "seconds": 122.96637481401558 } } } @@ -2596,1055 +3567,85 @@ "tokens_include_special_tokens": true, "public_text_overflow": "Raises ValueError; shorten or explicitly chunk longer inputs.", "embedding_dimensions": 384, - "parameters": 135383808 + "parameters": 163771288, + "audio_sample_rates": [ + 16000, + 44100, + 48000 + ], + "audio_max_seconds": 30, + "audio_input": "Original PCM, mono or channels-first; each rate is independently resampled for the two audio branches." }, "efficiency": { - "status": "not_measured_for_this_release", - "parameters": 135383808, + "status": "no_current_latency_measurement", + "parameters": 163771288, + "parameter_increase_percent_vs_previous_release": 20.968150046422096, "embedding_dimensions": 384, "dense_fp32_index_bytes_per_vector": 1536, - "weights_safetensors_bytes": 541598024, - "scope": "No new latency or peak-memory measurement is claimed. Vector storage excludes index structures and metadata.", - "historical_measurement": { - "attention": "native SDPA", - "batch_size": 8, - "embedding_dtype": "float32", - "hardware": "AMD gfx942 GPU", - "latency_includes": "Native public encode_text: tokenization, host-to-device transfer and encoding; excludes model loading.", - "measured_batches": 20, - "measurements": { - "128": { - "batch_size": 8, - "p50_ms": 4.283615489839576, - "p95_ms": 4.314010989037342, - "peak_allocated_bytes": 653202944, - "peak_reserved_bytes": 673185792, - "tokens_each_including_special": 128 - }, - "512": { - "batch_size": 8, - "p50_ms": 12.445501997717656, - "p95_ms": 12.603181986196432, - "peak_allocated_bytes": 878682112, - "peak_reserved_bytes": 970981376, - "tokens_each_including_special": 512 - } - }, - "memory_scope": "Process CUDA allocator peak with the complete multimodal model loaded.", - "model_revision": "70c3412bb0ddfb19386d673d4a102c753a574f1b", - "native_artifact_sha256": "b596f1dbab89369555e0a72a0d6854d780e305686750949b308c8f167993bdeb", - "scope": "Synthetic repeated text-batch measurements on one GPU; not end-to-end service latency.", - "tf32": false, - "token_lengths_include_special_tokens": true, - "torch_version_family": "2.12", - "transformers_version": "4.57.6", - "warmup_batches": 2, - "status": "historical_previous_release_measurement", - "applicability": "Historical measurement on its recorded native artifact; not applicable to this text-backbone update." - }, - "parameter_storage_change": { - "old_safetensors_bytes": 578289664, - "new_safetensors_bytes": 541598024, - "reduction_bytes": 36691640, - "old_FP32_parameter_bytes": 578216448, - "new_FP32_parameter_bytes": 541535232, - "scope": "Parameter storage only; not measured process peak memory or latency. Vector and index dimensions are unchanged." - } - }, - "previous_release": { - "model": "llm-semantic-router/Vela-1.0-Omni-Nano", - "revision": "3a1efc48995e2b71622d65c6cf71cfc00bbf3aa3", - "native_artifact_sha256": "fb9cd39c9980a085b75456c715c2295b997784fe733314be68c13635f6384edf", - "parameters": 133956480, - "metrics": { - "audio.audio_to_text.recall@1": { - "denominator": 2611, - "numerator": 235, - "value": 0.09000382995021065 - }, - "audio.audio_to_text.recall@10": { - "denominator": 2611, - "numerator": 822, - "value": 0.31482190731520493 - }, - "audio.audio_to_text.recall@5": { - "denominator": 2611, - "numerator": 582, - "value": 0.22290310225967064 - }, - "audio.text_to_audio.recall@1": { - "denominator": 2610, - "numerator": 366, - "value": 0.14022988505747128 - }, - "audio.text_to_audio.recall@10": { - "denominator": 2610, - "numerator": 1055, - "value": 0.4042145593869732 - }, - "audio.text_to_audio.recall@5": { - "denominator": 2610, - "numerator": 827, - "value": 0.3168582375478927 - }, - "image.image_to_text.recall@1": { - "denominator": 823, - "numerator": 540, - "value": 0.6561360874848117 - }, - "image.image_to_text.recall@10": { - "denominator": 823, - "numerator": 791, - "value": 0.9611178614823815 - }, - "image.image_to_text.recall@5": { - "denominator": 823, - "numerator": 731, - "value": 0.8882138517618469 - }, - "image.text_to_image.recall@1": { - "denominator": 4115, - "numerator": 2184, - "value": 0.5307411907654921 - }, - "image.text_to_image.recall@10": { - "denominator": 4115, - "numerator": 3847, - "value": 0.934872417982989 - }, - "image.text_to_image.recall@5": { - "denominator": 4115, - "numerator": 3465, - "value": 0.8420413122721749 - }, - "text.banking77.accuracy": { - "denominator": 3080, - "numerator": 2366, - "value": 0.7681818181818182 - }, - "text.massive-en.accuracy": { - "denominator": 2972, - "numerator": 2061, - "value": 0.6934724091520862 - } - }, - "descriptive_metrics": { - "text.banking77.macro_f1": { - "value": 0.7658421887247635, - "classes": 77, - "queries": 3080 - }, - "text.massive-en.macro_f1": { - "value": 0.6524376423162815, - "classes": 60, - "queries": 2972 - } - }, - "standard_rows": [ - { - "task": "SICK-R", - "split": "test", - "hf_subset": "default", - "main_metric": "cosine_spearman", - "score": 0.7169618925026362, - "metrics": { - "pearson": 0.8194169233518556, - "spearman": 0.7169617707661863, - "cosine_pearson": 0.819416922085493, - "cosine_spearman": 0.7169618925026362, - "manhattan_pearson": 0.7935695063285152, - "manhattan_spearman": 0.7160084195066224, - "euclidean_pearson": 0.7944328514895539, - "euclidean_spearman": 0.7169617703551949, - "main_score": 0.7169618925026362, - "hf_subset": "default", - "languages": [ - "eng-Latn" - ], - "mteb_version": "2.21.0" - } - }, - { - "task": "STSBenchmark", - "split": "test", - "hf_subset": "default", - "main_metric": "cosine_spearman", - "score": 0.7792400226361769, - "metrics": { - "pearson": 0.7717062133633645, - "spearman": 0.7792400217447207, - "cosine_pearson": 0.7717062156025293, - "cosine_spearman": 0.7792400226361769, - "manhattan_pearson": 0.7928914389432686, - "manhattan_spearman": 0.7777756255094064, - "euclidean_pearson": 0.7949414830254274, - "euclidean_spearman": 0.779239966513648, - "main_score": 0.7792400226361769, - "hf_subset": "default", - "languages": [ - "eng-Latn" - ], - "mteb_version": "2.21.0" - } - }, - { - "task": "ArguAna", - "split": "test", - "hf_subset": "default", - "main_metric": "ndcg_at_10", - "score": 0.37514, - "metrics": { - "ndcg_at_1": 0.17425, - "ndcg_at_3": 0.27012, - "ndcg_at_5": 0.31718, - "ndcg_at_10": 0.37514, - "ndcg_at_20": 0.40855, - "ndcg_at_100": 0.43759, - "ndcg_at_1000": 0.44701, - "map_at_1": 0.17425, - "map_at_3": 0.24561, - "map_at_5": 0.2715, - "map_at_10": 0.29556, - "map_at_20": 0.30489, - "map_at_100": 0.3093, - "map_at_1000": 0.30967, - "recall_at_1": 0.17425, - "recall_at_3": 0.34139, - "recall_at_5": 0.45661, - "recall_at_10": 0.63514, - "recall_at_20": 0.766, - "recall_at_100": 0.91679, - "recall_at_1000": 0.99004, - "accuracy": 0.17425, - "precision_at_1": 0.17425, - "precision_at_3": 0.1138, - "precision_at_5": 0.09132, - "precision_at_10": 0.06351, - "precision_at_20": 0.0383, - "precision_at_100": 0.00917, - "precision_at_1000": 0.00099, - "mrr_at_1": 0.17425320056899005, - "mrr_at_3": 0.24561403508771928, - "mrr_at_5": 0.2715030820293978, - "mrr_at_10": 0.2955561764772291, - "mrr_at_20": 0.30489450370601584, - "mrr_at_100": 0.3092982324964941, - "mrr_at_1000": 0.30966870165383564, - "nauc_ndcg_at_1_max": -0.012854409302840068, - "nauc_ndcg_at_1_std": 0.08676381700185752, - "nauc_ndcg_at_1_diff1": 0.11334791281377288, - "nauc_ndcg_at_3_max": 0.0007934267314863777, - "nauc_ndcg_at_3_std": 0.05484641820257593, - "nauc_ndcg_at_3_diff1": 0.07348303062976835, - "nauc_ndcg_at_5_max": 0.020607224558072373, - "nauc_ndcg_at_5_std": 0.0741284488966682, - "nauc_ndcg_at_5_diff1": 0.08730888772052957, - "nauc_ndcg_at_10_max": 0.035048132350077, - "nauc_ndcg_at_10_std": 0.10523053259120095, - "nauc_ndcg_at_10_diff1": 0.08020955970997308, - "nauc_ndcg_at_20_max": 0.04122292301118309, - "nauc_ndcg_at_20_std": 0.12537385022454034, - "nauc_ndcg_at_20_diff1": 0.0842700247341082, - "nauc_ndcg_at_100_max": 0.04333008607419203, - "nauc_ndcg_at_100_std": 0.12294409372370381, - "nauc_ndcg_at_100_diff1": 0.08845423517529309, - "nauc_ndcg_at_1000_max": 0.030127261370811845, - "nauc_ndcg_at_1000_std": 0.10766365950024334, - "nauc_ndcg_at_1000_diff1": 0.0861546357811246, - "nauc_map_at_1_max": -0.012854409302840068, - "nauc_map_at_1_std": 0.08676381700185752, - "nauc_map_at_1_diff1": 0.11334791281377288, - "nauc_map_at_3_max": -0.0029405293797002733, - "nauc_map_at_3_std": 0.06167764879997172, - "nauc_map_at_3_diff1": 0.07950290715235987, - "nauc_map_at_5_max": 0.00830536326209409, - "nauc_map_at_5_std": 0.07256833511730693, - "nauc_map_at_5_diff1": 0.08747148302006987, - "nauc_map_at_10_max": 0.015140286093876175, - "nauc_map_at_10_std": 0.08580662522686537, - "nauc_map_at_10_diff1": 0.08508762518017199, - "nauc_map_at_20_max": 0.0163028127500422, - "nauc_map_at_20_std": 0.09050711291668995, - "nauc_map_at_20_diff1": 0.08629712373980958, - "nauc_map_at_100_max": 0.016183246425002705, - "nauc_map_at_100_std": 0.09000154689011092, - "nauc_map_at_100_diff1": 0.08699050038257111, - "nauc_map_at_1000_max": 0.015701662616224514, - "nauc_map_at_1000_std": 0.0894576065792463, - "nauc_map_at_1000_diff1": 0.08694453879028893, - "nauc_recall_at_1_max": -0.012854409302840068, - "nauc_recall_at_1_std": 0.08676381700185752, - "nauc_recall_at_1_diff1": 0.11334791281377288, - "nauc_recall_at_3_max": 0.010338756866368223, - "nauc_recall_at_3_std": 0.03789278134763109, - "nauc_recall_at_3_diff1": 0.059403567851730316, - "nauc_recall_at_5_max": 0.05398176066498534, - "nauc_recall_at_5_std": 0.08064232472596307, - "nauc_recall_at_5_diff1": 0.08949514479783123, - "nauc_recall_at_10_max": 0.09984307699040616, - "nauc_recall_at_10_std": 0.17703668336587267, - "nauc_recall_at_10_diff1": 0.06498907396830537, - "nauc_recall_at_20_max": 0.15799758987163673, - "nauc_recall_at_20_std": 0.31157412853790695, - "nauc_recall_at_20_diff1": 0.07941702046000772, - "nauc_recall_at_100_max": 0.37265902510843735, - "nauc_recall_at_100_std": 0.5476974790176777, - "nauc_recall_at_100_diff1": 0.12977852219557848, - "nauc_recall_at_1000_max": 0.3004922701567, - "nauc_recall_at_1000_std": 0.7475055455618029, - "nauc_recall_at_1000_diff1": -0.05500901218402981, - "nauc_precision_at_1_max": -0.012854409302840068, - "nauc_precision_at_1_std": 0.08676381700185752, - "nauc_precision_at_1_diff1": 0.11334791281377288, - "nauc_precision_at_3_max": 0.010338756866368036, - "nauc_precision_at_3_std": 0.037892781347630994, - "nauc_precision_at_3_diff1": 0.05940356785173012, - "nauc_precision_at_5_max": 0.05398176066498592, - "nauc_precision_at_5_std": 0.08064232472596324, - "nauc_precision_at_5_diff1": 0.08949514479783155, - "nauc_precision_at_10_max": 0.09984307699040575, - "nauc_precision_at_10_std": 0.17703668336587192, - "nauc_precision_at_10_diff1": 0.06498907396830521, - "nauc_precision_at_20_max": 0.1579975898716373, - "nauc_precision_at_20_std": 0.31157412853790756, - "nauc_precision_at_20_diff1": 0.0794170204600081, - "nauc_precision_at_100_max": 0.3726590251084365, - "nauc_precision_at_100_std": 0.5476974790176794, - "nauc_precision_at_100_diff1": 0.1297785221955785, - "nauc_precision_at_1000_max": 0.3004922701567069, - "nauc_precision_at_1000_std": 0.7475055455618104, - "nauc_precision_at_1000_diff1": -0.055009012184042826, - "nauc_mrr_at_1_max": -0.012854409302840068, - "nauc_mrr_at_1_std": 0.08676381700185752, - "nauc_mrr_at_1_diff1": 0.11334791281377288, - "nauc_mrr_at_3_max": -0.0029405293797002733, - "nauc_mrr_at_3_std": 0.06167764879997172, - "nauc_mrr_at_3_diff1": 0.07950290715235987, - "nauc_mrr_at_5_max": 0.00830536326209409, - "nauc_mrr_at_5_std": 0.07256833511730693, - "nauc_mrr_at_5_diff1": 0.08747148302006987, - "nauc_mrr_at_10_max": 0.015140286093876175, - "nauc_mrr_at_10_std": 0.08580662522686537, - "nauc_mrr_at_10_diff1": 0.08508762518017199, - "nauc_mrr_at_20_max": 0.0163028127500422, - "nauc_mrr_at_20_std": 0.09050711291668995, - "nauc_mrr_at_20_diff1": 0.08629712373980958, - "nauc_mrr_at_100_max": 0.016183246425002705, - "nauc_mrr_at_100_std": 0.09000154689011092, - "nauc_mrr_at_100_diff1": 0.08699050038257111, - "nauc_mrr_at_1000_max": 0.015701662616224514, - "nauc_mrr_at_1000_std": 0.0894576065792463, - "nauc_mrr_at_1000_diff1": 0.08694453879028893, - "hit_rate_at_1": 0.17425, - "hit_rate_at_3": 0.34139, - "hit_rate_at_5": 0.45661, - "hit_rate_at_10": 0.63514, - "hit_rate_at_20": 0.766, - "hit_rate_at_100": 0.91679, - "hit_rate_at_1000": 0.99004, - "main_score": 0.37514, - "hf_subset": "default", - "languages": [ - "eng-Latn" - ], - "mteb_version": "2.21.0" - } - }, - { - "task": "TwentyNewsgroupsClustering.v2", - "split": "test", - "hf_subset": "default", - "main_metric": "v_measure", - "score": 0.44884717457494344, - "metrics": { - "v_measures": { - "Level 0": [ - 0.48056098748371134, - 0.44085942767025554, - 0.43416151810540005, - 0.45299559752505, - 0.44116051339311235, - 0.4409873172673812, - 0.46155584917327236, - 0.444819328389149, - 0.44792019268133576, - 0.44345101406076604 - ] - }, - "v_measure": 0.44884717457494344, - "v_measure_std": 0.01277114663774588, - "ami_scores": { - "Level 0": [ - 0.4786019649327535, - 0.43870317373726253, - 0.43201951778515824, - 0.4509348225877084, - 0.43904842199622773, - 0.43887546309559994, - 0.45953222355170814, - 0.44270204450063555, - 0.44583346842612326, - 0.4413500450061514 - ] - }, - "ami": 0.4467601145619328, - "ami_std": 0.012825331450796505, - "main_score": 0.44884717457494344, - "hf_subset": "default", - "languages": [ - "eng-Latn" - ], - "mteb_version": "2.21.0" - } - }, - { - "task": "MTOPDomainClassification", - "split": "validation", - "hf_subset": "en", - "main_metric": "accuracy", - "score": 0.8758836689038031, - "metrics": { - "scores_per_experiment": [ - { - "accuracy": 0.8527964205816555, - "f1": 0.8584702959865076, - "f1_weighted": 0.8520037076109549, - "precision": 0.8563814369311937, - "precision_weighted": 0.8604397214633738, - "recall": 0.8688240583945851, - "recall_weighted": 0.8527964205816555, - "ap": null, - "ap_weighted": null - }, - { - "accuracy": 0.8675615212527964, - "f1": 0.8658660043655293, - "f1_weighted": 0.8661413651963479, - "precision": 0.8632122218181354, - "precision_weighted": 0.8687564991167056, - "recall": 0.8733845956286541, - "recall_weighted": 0.8675615212527964, - "ap": null, - "ap_weighted": null - }, - { - "accuracy": 0.8514541387024609, - "f1": 0.8524202289070963, - "f1_weighted": 0.8491348169886809, - "precision": 0.8478156770106394, - "precision_weighted": 0.8512267923319679, - "recall": 0.8621133428009895, - "recall_weighted": 0.8514541387024609, - "ap": null, - "ap_weighted": null - }, - { - "accuracy": 0.8876957494407158, - "f1": 0.8889547651830694, - "f1_weighted": 0.886393441219644, - "precision": 0.8855262149097257, - "precision_weighted": 0.8869170708224573, - "recall": 0.8941464114507656, - "recall_weighted": 0.8876957494407158, - "ap": null, - "ap_weighted": null - }, - { - "accuracy": 0.8684563758389262, - "f1": 0.8708980361567344, - "f1_weighted": 0.8668343457695604, - "precision": 0.8655644216656676, - "precision_weighted": 0.8685796219847712, - "recall": 0.8789277362563833, - "recall_weighted": 0.8684563758389262, - "ap": null, - "ap_weighted": null - }, - { - "accuracy": 0.8662192393736018, - "f1": 0.8725560012982636, - "f1_weighted": 0.8665613320034511, - "precision": 0.8737258252820442, - "precision_weighted": 0.8845738920578841, - "recall": 0.8863222925022911, - "recall_weighted": 0.8662192393736018, - "ap": null, - "ap_weighted": null - }, - { - "accuracy": 0.8953020134228188, - "f1": 0.8976332709955689, - "f1_weighted": 0.8945899154754817, - "precision": 0.8912735415957905, - "precision_weighted": 0.8979949591784757, - "recall": 0.908092721226657, - "recall_weighted": 0.8953020134228188, - "ap": null, - "ap_weighted": null - }, - { - "accuracy": 0.9029082774049217, - "f1": 0.9022310464802398, - "f1_weighted": 0.9020525018623339, - "precision": 0.899983538025133, - "precision_weighted": 0.9036775461946326, - "recall": 0.9067592943176375, - "recall_weighted": 0.9029082774049217, - "ap": null, - "ap_weighted": null - }, - { - "accuracy": 0.876510067114094, - "f1": 0.8776887066559357, - "f1_weighted": 0.8761842615927375, - "precision": 0.8722809982436143, - "precision_weighted": 0.8829904469624968, - "recall": 0.8888620916111349, - "recall_weighted": 0.876510067114094, - "ap": null, - "ap_weighted": null - }, - { - "accuracy": 0.8899328859060402, - "f1": 0.8916131163462019, - "f1_weighted": 0.8885446252332447, - "precision": 0.8855482140891268, - "precision_weighted": 0.8923990752693544, - "recall": 0.9020276692969261, - "recall_weighted": 0.8899328859060402, - "ap": null, - "ap_weighted": null - } - ], - "accuracy": 0.8758836689038031, - "f1": 0.8778331472375147, - "f1_weighted": 0.8748440312952436, - "precision": 0.8741312089571069, - "precision_weighted": 0.8797555625382119, - "recall": 0.8869460213486026, - "recall_weighted": 0.8758836689038031, - "ap": null, - "ap_weighted": null, - "main_score": 0.8758836689038031, - "hf_subset": "en", - "languages": [ - "eng-Latn" - ], - "mteb_version": "2.21.0" - } - }, - { - "task": "MTOPDomainClassification", - "split": "test", - "hf_subset": "en", - "main_metric": "accuracy", - "score": 0.8671910624715002, - "metrics": { - "scores_per_experiment": [ - { - "accuracy": 0.8492932056543547, - "f1": 0.8506491551806685, - "f1_weighted": 0.8484338913731928, - "precision": 0.847171519646365, - "precision_weighted": 0.857621901965629, - "recall": 0.8634426315569471, - "recall_weighted": 0.8492932056543547, - "ap": null, - "ap_weighted": null - }, - { - "accuracy": 0.8593251253989969, - "f1": 0.8525570927270549, - "f1_weighted": 0.8588269042231532, - "precision": 0.849564509883911, - "precision_weighted": 0.8659425848467468, - "recall": 0.8649266896842414, - "recall_weighted": 0.8593251253989969, - "ap": null, - "ap_weighted": null - }, - { - "accuracy": 0.8508891928864569, - "f1": 0.8445077740960396, - "f1_weighted": 0.8500136039882616, - "precision": 0.83976876536009, - "precision_weighted": 0.8551546537983608, - "recall": 0.8573297788521569, - "recall_weighted": 0.8508891928864569, - "ap": null, - "ap_weighted": null - }, - { - "accuracy": 0.8702690378476973, - "f1": 0.8680982220684261, - "f1_weighted": 0.8688890737378698, - "precision": 0.8651925230157509, - "precision_weighted": 0.8699395458498034, - "recall": 0.873034481111479, - "recall_weighted": 0.8702690378476973, - "ap": null, - "ap_weighted": null - }, - { - "accuracy": 0.8657090743274054, - "f1": 0.8639593934313017, - "f1_weighted": 0.8646291417790959, - "precision": 0.8570893217526883, - "precision_weighted": 0.8672508842970594, - "recall": 0.8739690035000478, - "recall_weighted": 0.8657090743274054, - "ap": null, - "ap_weighted": null - }, - { - "accuracy": 0.8520291837665299, - "f1": 0.8534418954962738, - "f1_weighted": 0.8548114072639492, - "precision": 0.85875145236006, - "precision_weighted": 0.8803115046026259, - "recall": 0.869567932536695, - "recall_weighted": 0.8520291837665299, - "ap": null, - "ap_weighted": null - }, - { - "accuracy": 0.8869129046967624, - "f1": 0.8849494340089117, - "f1_weighted": 0.8871874824644057, - "precision": 0.8779834780597185, - "precision_weighted": 0.8955288700582439, - "recall": 0.9009013020211545, - "recall_weighted": 0.8869129046967624, - "ap": null, - "ap_weighted": null - }, - { - "accuracy": 0.875968992248062, - "f1": 0.8715420414043028, - "f1_weighted": 0.8753897998685723, - "precision": 0.86652024620862, - "precision_weighted": 0.8811614860860878, - "recall": 0.881802359118481, - "recall_weighted": 0.875968992248062, - "ap": null, - "ap_weighted": null - }, - { - "accuracy": 0.8755129958960328, - "f1": 0.8710791039688399, - "f1_weighted": 0.8760974709416987, - "precision": 0.8647173948263537, - "precision_weighted": 0.8831121050674345, - "recall": 0.8845140142042784, - "recall_weighted": 0.8755129958960328, - "ap": null, - "ap_weighted": null - }, - { - "accuracy": 0.886000911992704, - "f1": 0.8827257199119498, - "f1_weighted": 0.8855721338303499, - "precision": 0.8760471482914741, - "precision_weighted": 0.8912159468319844, - "recall": 0.8952867706565555, - "recall_weighted": 0.886000911992704, - "ap": null, - "ap_weighted": null - } - ], - "accuracy": 0.8671910624715002, - "f1": 0.8643509832293768, - "f1_weighted": 0.8669850909470549, - "precision": 0.8602806359405031, - "precision_weighted": 0.8747239483403975, - "recall": 0.8764774963242037, - "recall_weighted": 0.8671910624715002, - "ap": null, - "ap_weighted": null, - "main_score": 0.8671910624715002, - "hf_subset": "en", - "languages": [ - "eng-Latn" - ], - "mteb_version": "2.21.0" - } - }, - { - "task": "OxfordPets", - "split": "test", - "hf_subset": "default", - "main_metric": "accuracy", - "score": 0.9126737530662306, - "metrics": { - "scores_per_experiment": [ - { - "accuracy": 0.9062414826928318, - "f1": 0.9045301316021502, - "f1_weighted": 0.9050055409371512, - "precision": 0.9099213291951324, - "precision_weighted": 0.9109544817876485, - "recall": 0.9064295837897134, - "recall_weighted": 0.9062414826928318, - "ap": null, - "ap_weighted": null - }, - { - "accuracy": 0.914418097574271, - "f1": 0.9126724638115935, - "f1_weighted": 0.9132186480967779, - "precision": 0.9192904646518116, - "precision_weighted": 0.9203471940294945, - "recall": 0.9144593228002407, - "recall_weighted": 0.914418097574271, - "ap": null, - "ap_weighted": null - }, - { - "accuracy": 0.9187789588443718, - "f1": 0.9177765846692281, - "f1_weighted": 0.9184111765200857, - "precision": 0.9220755461580387, - "precision_weighted": 0.9229278273528245, - "recall": 0.9183739103921627, - "recall_weighted": 0.9187789588443718, - "ap": null, - "ap_weighted": null - }, - { - "accuracy": 0.9174161896974653, - "f1": 0.9147622354932347, - "f1_weighted": 0.9154663340384108, - "precision": 0.9209055798643112, - "precision_weighted": 0.9221124471295344, - "recall": 0.9173606715378323, - "recall_weighted": 0.9174161896974653, - "ap": null, - "ap_weighted": null - }, - { - "accuracy": 0.9065140365222132, - "f1": 0.9051669100036687, - "f1_weighted": 0.9057728210054364, - "precision": 0.9098426973534027, - "precision_weighted": 0.9105143284060627, - "recall": 0.9059560197061645, - "recall_weighted": 0.9065140365222132, - "ap": null, - "ap_weighted": null - } - ], - "accuracy": 0.9126737530662306, - "f1": 0.910981665115975, - "f1_weighted": 0.9115749041195725, - "precision": 0.9164071234445392, - "precision_weighted": 0.9173712557411129, - "recall": 0.9125159016452227, - "recall_weighted": 0.9126737530662306, - "ap": null, - "ap_weighted": null, - "main_score": 0.9126737530662306, - "hf_subset": "default", - "languages": [ - "eng-Latn" - ], - "mteb_version": "2.21.0" - } - }, - { - "task": "TinyImageNetClustering", - "split": "valid", - "hf_subset": "default", - "main_metric": "nmi", - "score": 0.6169193395626786, - "metrics": { - "v_measure": 0.6169193395626787, - "nmi": 0.6169193395626786, - "ari": 0.21497891543638262, - "cluster_accuracy": 0.3638, - "main_score": 0.6169193395626786, - "hf_subset": "default", - "languages": [ - "eng-Latn" - ], - "mteb_version": "2.21.0" - } - }, - { - "task": "OxfordPetsZeroShot", - "split": "test", - "hf_subset": "default", - "main_metric": "accuracy", - "score": 0.1163804851458163, - "metrics": { - "accuracy": 0.1163804851458163, - "f1": 0.085822378073852, - "f1_weighted": 0.08530524305774022, - "precision": 0.08885688520073651, - "precision_weighted": 0.08856851170630324, - "recall": 0.11709192030781942, - "recall_weighted": 0.1163804851458163, - "ap": null, - "ap_weighted": null, - "main_score": 0.1163804851458163, - "hf_subset": "default", - "languages": [ - "eng-Latn" - ], - "mteb_version": "2.21.0" - } - }, - { - "task": "CREMA_D", - "split": "train", - "hf_subset": "default", - "main_metric": "accuracy", - "score": 0.29763670140167686, - "metrics": { - "scores_per_experiment": [ - { - "accuracy": 0.2847548690396239, - "f1": 0.25980556646698677, - "f1_weighted": 0.2606862265023784, - "precision": 0.2685141966762838, - "precision_weighted": 0.26869432571348717, - "recall": 0.283361264324777, - "recall_weighted": 0.2847548690396239, - "ap": null, - "ap_weighted": null - }, - { - "accuracy": 0.2981867024848892, - "f1": 0.2927999429509691, - "f1_weighted": 0.2952410831252662, - "precision": 0.29988591178720175, - "precision_weighted": 0.30336098208055257, - "recall": 0.29763527945216217, - "recall_weighted": 0.2981867024848892, - "ap": null, - "ap_weighted": null - }, - { - "accuracy": 0.28696236559139787, - "f1": 0.26582489649494206, - "f1_weighted": 0.26920117230368773, - "precision": 0.26489987169540286, - "precision_weighted": 0.2682170436584329, - "recall": 0.2834616565816636, - "recall_weighted": 0.28696236559139787, - "ap": null, - "ap_weighted": null - }, - { - "accuracy": 0.29973118279569894, - "f1": 0.291485315935673, - "f1_weighted": 0.28841683151437103, - "precision": 0.3068565359099591, - "precision_weighted": 0.3086441612810119, - "recall": 0.30801945566008065, - "recall_weighted": 0.29973118279569894, - "ap": null, - "ap_weighted": null - }, - { - "accuracy": 0.3185483870967742, - "f1": 0.3051858414398825, - "f1_weighted": 0.3066161074219095, - "precision": 0.310702613862649, - "precision_weighted": 0.3133755862555043, - "recall": 0.31849511838471833, - "recall_weighted": 0.3185483870967742, - "ap": null, - "ap_weighted": null - } - ], - "accuracy": 0.29763670140167686, - "f1": 0.2830203126576907, - "f1_weighted": 0.2840322841735226, - "precision": 0.29017182598629926, - "precision_weighted": 0.2924584197977978, - "recall": 0.2981945548806803, - "recall_weighted": 0.29763670140167686, - "ap": null, - "ap_weighted": null, - "main_score": 0.29763670140167686, - "hf_subset": "default", - "languages": [ - "eng-Latn" - ], - "mteb_version": "2.21.0" - } - }, - { - "task": "CREMA_DClustering", - "split": "train", - "hf_subset": "default", - "main_metric": "v_measure", - "score": 0.023601715918443712, - "metrics": { - "v_measures": { - "Level 0": [ - 0.014290784661453301, - 0.02755216925549553, - 0.017220745995259155, - 0.02907041540588703, - 0.017365020267924067, - 0.022161217359367304, - 0.02806069377818825, - 0.017815432451092263, - 0.03081951157077731, - 0.03166116843899292 - ] - }, - "v_measure": 0.023601715918443712, - "v_measure_std": 0.006199268435462002, - "ami_scores": { - "Level 0": [ - 0.01385865484522556, - 0.027134003314929028, - 0.016785414361355624, - 0.028644278423611804, - 0.016937404067902156, - 0.02173522206231376, - 0.0276339699482267, - 0.0173943490723341, - 0.030397587303172737, - 0.031246814385090853 - ] + "weights_safetensors_bytes": 655583800, + "weights_safetensors_sha256": "d5aa7f00e217bd4fd5314140a3603b9a873636320c8d09732a14f79f4cb0667e", + "scope": "The CLAP branch increases parameters and audio computation. No isolated latency or peak-memory improvement is claimed; previous measurements belong to their recorded artifacts.", + "previous_release_measurement": { + "status": "not_measured_for_this_release", + "parameters": 135383808, + "embedding_dimensions": 384, + "dense_fp32_index_bytes_per_vector": 1536, + "weights_safetensors_bytes": 541598024, + "scope": "No new latency or peak-memory measurement is claimed. Vector storage excludes index structures and metadata.", + "historical_measurement": { + "attention": "native SDPA", + "batch_size": 8, + "embedding_dtype": "float32", + "hardware": "AMD gfx942 GPU", + "latency_includes": "Native public encode_text: tokenization, host-to-device transfer and encoding; excludes model loading.", + "measured_batches": 20, + "measurements": { + "128": { + "batch_size": 8, + "p50_ms": 4.283615489839576, + "p95_ms": 4.314010989037342, + "peak_allocated_bytes": 653202944, + "peak_reserved_bytes": 673185792, + "tokens_each_including_special": 128 }, - "ami": 0.023176769778416234, - "ami_std": 0.0062032784119843, - "main_score": 0.023601715918443712, - "hf_subset": "default", - "languages": [ - "eng-Latn" - ], - "mteb_version": "2.21.0" - } - }, - { - "task": "SpeechCommandsZeroshotv0.02", - "split": "test", - "hf_subset": "default", - "main_metric": "accuracy", - "score": 0.14972999509081983, - "metrics": { - "accuracy": 0.14972999509081983, - "f1": 0.06986246632697812, - "f1_weighted": 0.07083899432793754, - "precision": 0.15715381051614374, - "precision_weighted": 0.1573685095178851, - "recall": 0.14663865718054886, - "recall_weighted": 0.14972999509081983, - "ap": null, - "ap_weighted": null, - "main_score": 0.14972999509081983, - "hf_subset": "default", - "languages": [ - "eng-Latn" - ], - "mteb_version": "2.21.0" - } - } - ], - "full_benchmarks": [ - { - "benchmark": "MTEB(eng, v2)", - "framework_version": "2.21.0", - "coverage": { - "completed_tasks": 41, - "required_tasks": 41, - "task_types": 7 - }, - "model_revision": "70c3412bb0ddfb19386d673d4a102c753a574f1b", - "native_artifact_sha256": "b596f1dbab89369555e0a72a0d6854d780e305686750949b308c8f167993bdeb", - "aggregation": { - "mean_task": 0.5337130125278776, - "mean_task_type": 0.5197553164396411, - "definition": "First apply the official TaskResult aggregation within each task. Mean task gives each of the 41 tasks equal weight; mean task type first averages within each type, then gives all seven types equal weight.", - "score_scale": "0-1; multiply by 100 for display", - "by_task_type": { - "Classification": { - "score": 0.6552554795025091, - "tasks": 8 - }, - "Clustering": { - "score": 0.47072550487857034, - "tasks": 8 - }, - "PairClassification": { - "score": 0.7494593709186261, - "tasks": 3 - }, - "Reranking": { - "score": 0.455685, - "tasks": 2 - }, - "Retrieval": { - "score": 0.290152, - "tasks": 10 - }, - "STS": { - "score": 0.724513458257586, - "tasks": 9 - }, - "Summarization": { - "score": 0.29249640152019635, - "tasks": 1 - } + "512": { + "batch_size": 8, + "p50_ms": 12.445501997717656, + "p95_ms": 12.603181986196432, + "peak_allocated_bytes": 878682112, + "peak_reserved_bytes": 970981376, + "tokens_each_including_special": 512 } }, - "report": "benchmarks/mteb-eng-v2.json", - "text_component_applicability": { - "applies_to_revision": "self", - "applies_to_native_artifact_sha256": "fb9cd39c9980a085b75456c715c2295b997784fe733314be68c13635f6384edf", - "evaluated_revision": "70c3412bb0ddfb19386d673d4a102c753a574f1b", - "evaluated_native_artifact_sha256": "b596f1dbab89369555e0a72a0d6854d780e305686750949b308c8f167993bdeb", - "basis": "Exact component tensor and inference-file identity; original measured identity is retained.", - "report": "benchmarks/component-equivalence.json", - "new_full_panel_inference": false, - "component": "text" - } - }, - { - "benchmark": "MAEB(beta, audio-only)", - "framework_version": "2.21.0", - "coverage": { - "completed_tasks": 19, - "required_tasks": 19, - "subset_split_slots": 134, - "task_types": 5, - "selection": "All MAEB(beta) tasks whose modalities are exactly [audio].", - "full_MAEB": false, - "excluded": "Tasks that require text/audio or other joint-modality inputs." - }, + "memory_scope": "Process CUDA allocator peak with the complete multimodal model loaded.", "model_revision": "70c3412bb0ddfb19386d673d4a102c753a574f1b", "native_artifact_sha256": "b596f1dbab89369555e0a72a0d6854d780e305686750949b308c8f167993bdeb", - "aggregation": { - "official": { - "Any2AnyRetrieval": 0.6890050000000001, - "AudioClassification": 0.39566923071716426, - "AudioClustering": 0.054218008138417706, - "AudioPairClassification": 0.571886979184418, - "AudioReranking": 0.63794, - "Mean(Task)": 0.39776955262406916, - "Mean(TaskType)": 0.46974384360800003 - }, - "definition": "Official TaskResult aggregation within each task, followed by official mean_task, mean_task_type and task_types aggregation. The 134 subset/split slots are not averaged as independent tasks.", - "score_scale": "0-1; multiply by 100 for display" - }, - "audio_component_applicability": { - "applies_to_revision": "self", - "applies_to_native_artifact_sha256": "fb9cd39c9980a085b75456c715c2295b997784fe733314be68c13635f6384edf", - "evaluated_revision": "70c3412bb0ddfb19386d673d4a102c753a574f1b", - "evaluated_native_artifact_sha256": "b596f1dbab89369555e0a72a0d6854d780e305686750949b308c8f167993bdeb", - "basis": "Exact component tensor and inference-file identity; original measured identity is retained.", - "report": "benchmarks/component-equivalence.json", - "new_full_panel_inference": false, - "component": "audio" - }, - "report": "benchmarks/maeb-audio-only.json" + "scope": "Synthetic repeated text-batch measurements on one GPU; not end-to-end service latency.", + "tf32": false, + "token_lengths_include_special_tokens": true, + "torch_version_family": "2.12", + "transformers_version": "4.57.6", + "warmup_batches": 2, + "status": "historical_previous_release_measurement", + "applicability": "Historical measurement on its recorded native artifact; not applicable to this text-backbone update." + }, + "parameter_storage_change": { + "old_safetensors_bytes": 578289664, + "new_safetensors_bytes": 541598024, + "reduction_bytes": 36691640, + "old_FP32_parameter_bytes": 578216448, + "new_FP32_parameter_bytes": 541535232, + "scope": "Parameter storage only; not measured process peak memory or latency. Vector and index dimensions are unchanged." } - ] + } + }, + "previous_release": { + "model": "llm-semantic-router/Vela-1.0-Omni-Nano", + "revision": "9e747d2e1d4b97d3081ff4ac2ac79986c7a8da95", + "native_artifact_sha256": "50ce808197fb43a1b8913a66d4a47339ec66ecd10d52cbd9f9e752d02f8c43fa", + "evaluated_artifact_fingerprint_sha256": "96d76f97478e114b816df3ab1b3c6947e4c5ef0babfb40ab865d7e3c838d34c4", + "parameters": 135383808, + "scores_report": "benchmarks/previous-release-scores.json" }, "full_benchmarks": [ { @@ -3697,21 +3698,15 @@ "text_component_applicability": { "component": "text", "applies_to_revision": "self", - "applies_to_native_artifact_sha256": "96d76f97478e114b816df3ab1b3c6947e4c5ef0babfb40ab865d7e3c838d34c4", + "applies_to_native_artifact_sha256": "474ad3d52f6ee7f818a3dbb6112e3d66c2423542d08228e7996a8cd600bd99a2", "evaluated_model": "local/Vela-Omni-Nano-GIST-Rotation", "evaluated_revision": "29994142677559ca657c27b657eda0dddbf583bd4cf42a3a31095ab3f99d1646", "evaluated_native_artifact_sha256": "29994142677559ca657c27b657eda0dddbf583bd4cf42a3a31095ab3f99d1646", "revision_kind": "local native artifact fingerprint, not a Hugging Face commit", "report": "benchmarks/component-equivalence.json", "new_full_panel_inference": false, - "basis": "199 exact text tensors and byte-identical inference/tokenizer/configuration under the same official evaluator protocol." - }, - "inference_profile_applicability": { - "revision": "self", - "native_artifact_sha256": "50ce808197fb43a1b8913a66d4a47339ec66ecd10d52cbd9f9e752d02f8c43fa", - "parameters": 135383808, - "report": "benchmarks/inference-equivalence.json", - "new_benchmark_run": false + "basis": "The evaluated text computation is retained through the published Nano and its inference-only trim. All 197 retained text tensors, effective FP32 values, tokenizer/config and text operation path match; the two removed pooler tensors are unused.", + "bridge_sha256": "42d802c7279a1c20728926c079f5a4f952b16629fc49a8ed121da4c51f3d77aa" } }, { @@ -3726,586 +3721,118 @@ "full_MAEB": false, "excluded": "Tasks that require text/audio or other joint-modality inputs." }, - "model_revision": "96d76f97478e114b816df3ab1b3c6947e4c5ef0babfb40ab865d7e3c838d34c4", - "native_artifact_sha256": "96d76f97478e114b816df3ab1b3c6947e4c5ef0babfb40ab865d7e3c838d34c4", + "model_revision": "474ad3d52f6ee7f818a3dbb6112e3d66c2423542d08228e7996a8cd600bd99a2", + "native_artifact_sha256": "474ad3d52f6ee7f818a3dbb6112e3d66c2423542d08228e7996a8cd600bd99a2", "aggregation": { "official": { - "Any2AnyRetrieval": 0.6890050000000001, - "AudioClassification": 0.39553889510295764, - "AudioClustering": 0.054218008138417706, - "AudioPairClassification": 0.5716888010864635, - "AudioReranking": 0.63794, - "Mean(Task)": 0.3976628038845884, - "Mean(TaskType)": 0.4696781408655678 + "Any2AnyRetrieval": 0.8580149999999999, + "AudioClassification": 0.44475865771818446, + "AudioClustering": 0.03504342925419187, + "AudioPairClassification": 0.5741932198702043, + "AudioReranking": 0.70476, + "Mean(Task)": 0.4359384306459587, + "Mean(TaskType)": 0.5233540613685161 }, "definition": "Official TaskResult aggregation within each task, followed by official mean_task, mean_task_type and task_types aggregation. The 134 subset/split slots are not averaged as independent tasks.", "score_scale": "0-1; multiply by 100 for display" }, "report": "benchmarks/maeb-audio-only.json", - "inference_profile_applicability": { - "revision": "self", - "native_artifact_sha256": "50ce808197fb43a1b8913a66d4a47339ec66ecd10d52cbd9f9e752d02f8c43fa", - "parameters": 135383808, - "report": "benchmarks/inference-equivalence.json", - "new_benchmark_run": false - } + "evaluated_native_artifact_sha256": "474ad3d52f6ee7f818a3dbb6112e3d66c2423542d08228e7996a8cd600bd99a2", + "release_native_artifact_sha256": "474ad3d52f6ee7f818a3dbb6112e3d66c2423542d08228e7996a8cd600bd99a2" } ], "net_benefit": { - "scope": "Product-weighted progress comparison, not an official benchmark aggregate or SOTA claim.", - "baseline_model": "llm-semantic-router/Vela-1.0-Omni-Nano", - "baseline_revision": "3a1efc48995e2b71622d65c6cf71cfc00bbf3aa3", - "baseline_native_artifact_sha256": "fb9cd39c9980a085b75456c715c2295b997784fe733314be68c13635f6384edf", - "candidate_native_artifact_sha256": "96d76f97478e114b816df3ab1b3c6947e4c5ef0babfb40ab865d7e3c838d34c4", - "score_scale": "0-100; changes in percentage points", + "scope": "Fixed product progress metric; not an official benchmark aggregate, significance claim, or proof of overall SOTA.", + "baseline_revision": "9e747d2e1d4b97d3081ff4ac2ac79986c7a8da95", + "baseline_native_fingerprint": "50ce808197fb43a1b8913a66d4a47339ec66ecd10d52cbd9f9e752d02f8c43fa", + "baseline_evaluated_fingerprint": "96d76f97478e114b816df3ab1b3c6947e4c5ef0babfb40ab865d7e3c838d34c4", + "candidate_native_fingerprint": "474ad3d52f6ee7f818a3dbb6112e3d66c2423542d08228e7996a8cd600bd99a2", "components": [ { - "baseline_100": 59.37287180704907, - "candidate_100": 67.57407300861536, + "candidate_100": 66.41835251574038, "component": "common14", - "delta_pp": 8.201201201566288, + "delta_pp": -1.1557204928749676, + "parent_100": 67.57407300861536, "weight": 0.3 }, { - "baseline_100": 51.97553164396411, "candidate_100": 60.781849756549235, "component": "English41_MeanTaskType", - "delta_pp": 8.806318112585124, + "delta_pp": 0.0, + "parent_100": 60.781849756549235, "weight": 0.3 }, { - "baseline_100": 54.865785925824184, "candidate_100": 53.65746394890041, "component": "image3", - "delta_pp": -1.208321976923776, + "delta_pp": 0.0, + "parent_100": 53.65746394890041, "weight": 0.2 }, { - "baseline_100": 46.9743843608, - "candidate_100": 46.96781408655678, + "candidate_100": 52.33540613685162, "component": "audio19_MeanTaskType", - "delta_pp": -0.00657027424322, + "delta_pp": 5.367592050294833, + "parent_100": 46.96781408655678, "weight": 0.2 } ], - "weighted_net_delta_pp": 4.859277344012025, - "common_definition": "Equal mean of three families: two text accuracies, all six image retrieval recalls, all six audio retrieval recalls.", - "image3_definition": "Equal mean of OxfordPets classification accuracy, TinyImageNet clustering NMI and OxfordPets zero-shot accuracy.", - "text_audio_definition": "Full English41 and full audio19 mean task-type scores. English uses the explicit exact text-component bridge; audio is freshly evaluated.", - "coverage": { - "common_metrics": 14, - "standard_tasks": 11, - "standard_split_rows": 12, - "English_tasks": 41, - "audio_tasks": 19, - "audio_subset_split_slots": 134 - }, + "weighted_net_delta_pp": 0.7268022621964765, + "score_scale": "0–100; deltas in percentage points", + "parameter_increase_percent": 20.968150046422096, + "common_definition": "Mean of text accuracy (2), image recall (6), and audio recall (6), with equal one-third family weights.", + "image3_definition": "Mean of OxfordPets accuracy, TinyImageNetClustering NMI, and OxfordPetsZeroShot accuracy.", "common_metric_delta_pp": { - "audio.audio_to_text.recall@1": 8.157793948678668, - "audio.audio_to_text.recall@10": 15.51129835312141, - "audio.audio_to_text.recall@5": 14.362313289927231, - "audio.text_to_audio.recall@1": 11.417624521072797, - "audio.text_to_audio.recall@10": 15.478927203065133, - "audio.text_to_audio.recall@5": 14.444444444444445, - "image.image_to_text.recall@1": -4.738760631834751, - "image.image_to_text.recall@10": -1.2150668286755772, - "image.image_to_text.recall@5": -1.701093560145808, - "image.text_to_image.recall@1": 2.7460510328068044, - "image.text_to_image.recall@10": 0.7776427703523694, - "image.text_to_image.recall@5": 1.0206561360874848, - "text.banking77.accuracy": 11.168831168831169, - "text.massive-en.accuracy": 12.617765814266487 + "audio.audio_to_text.recall@1": -1.0340865568747608, + "audio.audio_to_text.recall@10": -1.3787820758330127, + "audio.audio_to_text.recall@5": -1.8383761011106892, + "audio.text_to_audio.recall@1": -5.095785440613029, + "audio.text_to_audio.recall@10": -5.287356321839088, + "audio.text_to_audio.recall@5": -6.168582375478926, + "image.image_to_text.recall@1": 0.0, + "image.image_to_text.recall@10": 0.0, + "image.image_to_text.recall@5": 0.0, + "image.text_to_image.recall@1": 0.0, + "image.text_to_image.recall@10": 0.0, + "image.text_to_image.recall@5": 0.0, + "text.banking77.accuracy": 0.0, + "text.massive-en.accuracy": 0.0 }, - "standard_task_deltas": [ - { - "baseline": 0.7169618925026362, - "candidate": 0.8053174565838819, - "hf_subset": "default", - "main_metric": "cosine_spearman", - "split": "test", - "status": "complete", - "task": "SICK-R", - "delta_pp": 8.835556408124567 - }, - { - "baseline": 0.7792400226361769, - "candidate": 0.870781967302773, - "hf_subset": "default", - "main_metric": "cosine_spearman", - "split": "test", - "status": "complete", - "task": "STSBenchmark", - "delta_pp": 9.154194466659614 - }, - { - "baseline": 0.37514, - "candidate": 0.59246, - "hf_subset": "default", - "main_metric": "ndcg_at_10", - "split": "test", - "status": "complete", - "task": "ArguAna", - "delta_pp": 21.732000000000003 - }, - { - "baseline": 0.44884717457494344, - "candidate": 0.5087260708286914, - "hf_subset": "default", - "main_metric": "v_measure", - "split": "test", - "status": "complete", - "task": "TwentyNewsgroupsClustering.v2", - "delta_pp": 5.987889625374793 - }, - { - "baseline": 0.8758836689038031, - "candidate": 0.9476510067114093, - "hf_subset": "en", - "main_metric": "accuracy", - "split": "validation", - "status": "complete", - "task": "MTOPDomainClassification", - "delta_pp": 7.176733780760625 - }, - { - "baseline": 0.8671910624715002, - "candidate": 0.9491792065663475, - "hf_subset": "en", - "main_metric": "accuracy", - "split": "test", - "status": "complete", - "task": "MTOPDomainClassification", - "delta_pp": 8.198814409484722 - }, - { - "baseline": 0.9126737530662306, - "candidate": 0.9126737530662306, - "hf_subset": "default", - "main_metric": "accuracy", - "split": "test", - "status": "complete", - "task": "OxfordPets", - "delta_pp": 0.0 - }, - { - "baseline": 0.6169193395626786, - "candidate": 0.6169193395626786, - "hf_subset": "default", - "main_metric": "nmi", - "split": "valid", - "status": "complete", - "task": "TinyImageNetClustering", - "delta_pp": 0.0 - }, - { - "baseline": 0.1163804851458163, - "candidate": 0.08013082583810302, - "hf_subset": "default", - "main_metric": "accuracy", - "split": "test", - "status": "complete", - "task": "OxfordPetsZeroShot", - "delta_pp": -3.6249659307713276 - }, - { - "baseline": 0.29763670140167686, - "candidate": 0.29763670140167686, - "hf_subset": "default", - "main_metric": "accuracy", - "split": "train", - "status": "complete", - "task": "CREMA_D", - "delta_pp": 0.0 - }, - { - "baseline": 0.023601715918443712, - "candidate": 0.023601715918443712, - "hf_subset": "default", - "main_metric": "v_measure", - "split": "train", - "status": "complete", - "task": "CREMA_DClustering", - "delta_pp": 0.0 - }, - { - "baseline": 0.14972999509081983, - "candidate": 0.1411389297987236, - "hf_subset": "default", - "main_metric": "accuracy", - "split": "test", - "status": "complete", - "task": "SpeechCommandsZeroshotv0.02", - "delta_pp": -0.8591065292096217 - } - ], - "English_task_deltas": [ - { - "candidate_100": 59.245999999999995, - "current_100": 37.513999999999996, - "delta_pp": 21.732, - "primary_metric": "ndcg_at_10", - "task": "ArguAna", - "task_type": "Retrieval" - }, - { - "candidate_100": 64.88847259326147, - "current_100": 65.25395409281045, - "delta_pp": -0.36548149954897724, - "primary_metric": "v_measure", - "task": "ArXivHierarchicalClusteringP2P", - "task_type": "Clustering" - }, - { - "candidate_100": 57.44315313869382, - "current_100": 55.989532291560415, - "delta_pp": 1.4536208471334078, - "primary_metric": "v_measure", - "task": "ArXivHierarchicalClusteringS2S", - "task_type": "Clustering" - }, - { - "candidate_100": 62.329, - "current_100": 60.514, - "delta_pp": 1.8149999999999977, - "primary_metric": "map_at_1000", - "task": "AskUbuntuDupQuestions", - "task_type": "Reranking" - }, - { - "candidate_100": 86.98752423842483, - "current_100": 62.24539420714732, - "delta_pp": 24.742130031277505, - "primary_metric": "cosine_spearman", - "task": "BIOSSES", - "task_type": "STS" - }, - { - "candidate_100": 82.14285714285714, - "current_100": 69.57467532467533, - "delta_pp": 12.568181818181813, - "primary_metric": "accuracy", - "task": "Banking77Classification", - "task_type": "Classification" - }, - { - "candidate_100": 41.313796273674654, - "current_100": 39.82923119776807, - "delta_pp": 1.4845650759065876, - "primary_metric": "v_measure", - "task": "BiorxivClusteringP2P.v2", - "task_type": "Clustering" - }, - { - "candidate_100": 56.979, - "current_100": 42.976, - "delta_pp": 14.003, - "primary_metric": "ndcg_at_10", - "task": "CQADupstackGamingRetrieval", - "task_type": "Retrieval" - }, - { - "candidate_100": 39.586, - "current_100": 30.362000000000002, - "delta_pp": 9.223999999999997, - "primary_metric": "ndcg_at_10", - "task": "CQADupstackUnixRetrieval", - "task_type": "Retrieval" - }, - { - "candidate_100": 31.846000000000004, - "current_100": 16.073999999999998, - "delta_pp": 15.772000000000006, - "primary_metric": "ndcg_at_10", - "task": "ClimateFEVERHardNegatives", - "task_type": "Retrieval" - }, - { - "candidate_100": 87.576, - "current_100": 24.055, - "delta_pp": 63.520999999999994, - "primary_metric": "ndcg_at_10", - "task": "FEVERHardNegatives", - "task_type": "Retrieval" - }, - { - "candidate_100": 39.143, - "current_100": 22.588, - "delta_pp": 16.555, - "primary_metric": "ndcg_at_10", - "task": "FiQA2018", - "task_type": "Retrieval" - }, - { - "candidate_100": 66.349, - "current_100": 26.987, - "delta_pp": 39.36200000000001, - "primary_metric": "ndcg_at_10", - "task": "HotpotQAHardNegatives", - "task_type": "Retrieval" - }, - { - "candidate_100": 91.94760000000001, - "current_100": 62.1364, - "delta_pp": 29.811200000000007, - "primary_metric": "accuracy", - "task": "ImdbClassification", - "task_type": "Classification" - }, - { - "candidate_100": 94.91792065663475, - "current_100": 86.71910624715002, - "delta_pp": 8.198814409484726, - "primary_metric": "accuracy", - "task": "MTOPDomainClassification", - "task_type": "Classification" - }, - { - "candidate_100": 70.96839273705447, - "current_100": 60.39340954942838, - "delta_pp": 10.574983187626096, - "primary_metric": "accuracy", - "task": "MassiveIntentClassification", - "task_type": "Classification" - }, - { - "candidate_100": 76.11297915265635, - "current_100": 73.37256220578345, - "delta_pp": 2.740416946872898, - "primary_metric": "accuracy", - "task": "MassiveScenarioClassification", - "task_type": "Classification" - }, - { - "candidate_100": 39.83023091223438, - "current_100": 37.76813963073551, - "delta_pp": 2.0620912814988657, - "primary_metric": "v_measure", - "task": "MedrxivClusteringP2P.v2", - "task_type": "Clustering" - }, - { - "candidate_100": 37.875084453707764, - "current_100": 36.13349306174702, - "delta_pp": 1.7415913919607462, - "primary_metric": "v_measure", - "task": "MedrxivClusteringS2S.v2", - "task_type": "Clustering" - }, - { - "candidate_100": 32.366, - "current_100": 30.623, - "delta_pp": 1.7429999999999986, - "primary_metric": "max_over_subqueries_map_at_1000", - "task": "MindSmallReranking", - "task_type": "Reranking" - }, - { - "candidate_100": 21.889, - "current_100": 16.827, - "delta_pp": 5.061999999999998, - "primary_metric": "ndcg_at_10", - "task": "SCIDOCS", - "task_type": "Retrieval" - }, - { - "candidate_100": 80.53174565838819, - "current_100": 71.69618925026361, - "delta_pp": 8.835556408124575, - "primary_metric": "cosine_spearman", - "task": "SICK-R", - "task_type": "STS" - }, - { - "candidate_100": 75.5658801516535, - "current_100": 68.16915974310523, - "delta_pp": 7.396720408548276, - "primary_metric": "cosine_spearman", - "task": "STS12", - "task_type": "STS" - }, - { - "candidate_100": 86.26346974038235, - "current_100": 74.40342742804673, - "delta_pp": 11.860042312335622, - "primary_metric": "cosine_spearman", - "task": "STS13", - "task_type": "STS" - }, - { - "candidate_100": 82.29882793900394, - "current_100": 68.02653891259911, - "delta_pp": 14.272289026404835, - "primary_metric": "cosine_spearman", - "task": "STS14", - "task_type": "STS" - }, - { - "candidate_100": 88.73648284326485, - "current_100": 77.86144933320074, - "delta_pp": 10.875033510064114, - "primary_metric": "cosine_spearman", - "task": "STS15", - "task_type": "STS" - }, - { - "candidate_100": 87.0781967302773, - "current_100": 77.92400226361768, - "delta_pp": 9.154194466659618, - "primary_metric": "cosine_spearman", - "task": "STSBenchmark", - "task_type": "STS" - }, - { - "candidate_100": 95.79964673462752, - "current_100": 89.79096125036985, - "delta_pp": 6.0086854842576685, - "primary_metric": "max_ap", - "task": "SprintDuplicateQuestions", - "task_type": "PairClassification" - }, - { - "candidate_100": 58.424418738430106, - "current_100": 57.34464990059502, - "delta_pp": 1.0797688378350827, - "primary_metric": "v_measure", - "task": "StackExchangeClustering.v2", - "task_type": "Clustering" - }, - { - "candidate_100": 41.053017253245386, - "current_100": 39.37668627014545, - "delta_pp": 1.6763309830999376, - "primary_metric": "v_measure", - "task": "StackExchangeClusteringP2P.v2", - "task_type": "Clustering" - }, - { - "candidate_100": 69.121, - "current_100": 41.292, - "delta_pp": 27.828999999999994, - "primary_metric": "ndcg_at_10", - "task": "TRECCOVID", - "task_type": "Retrieval" - }, - { - "candidate_100": 48.363, - "current_100": 31.477, - "delta_pp": 16.886, - "primary_metric": "ndcg_at_10", - "task": "Touche2020Retrieval.v3", - "task_type": "Retrieval" - }, - { - "candidate_100": 71.904296875, - "current_100": 64.4091796875, - "delta_pp": 7.4951171875, - "primary_metric": "accuracy", - "task": "ToxicConversationsClassification", - "task_type": "Classification" - }, - { - "candidate_100": 62.79286926994907, - "current_100": 50.77815506508207, - "delta_pp": 12.014714204867005, - "primary_metric": "accuracy", - "task": "TweetSentimentExtractionClassification", - "task_type": "Classification" - }, - { - "candidate_100": 50.87260708286914, - "current_100": 44.88471745749435, - "delta_pp": 5.987889625374791, - "primary_metric": "v_measure", - "task": "TwentyNewsgroupsClustering.v2", - "task_type": "Clustering" - }, - { - "candidate_100": 72.95151691526065, - "current_100": 56.34475441213181, - "delta_pp": 16.60676250312884, - "primary_metric": "max_ap", - "task": "TwitterSemEval2015", - "task_type": "PairClassification" - }, - { - "candidate_100": 85.30340320492272, - "current_100": 78.70209561308617, - "delta_pp": 6.6013075918365445, - "primary_metric": "max_ap", - "task": "TwitterURLCorpus", - "task_type": "PairClassification" - }, - { - "candidate_100": 31.834434345907127, - "current_100": 29.249640152019634, - "delta_pp": 2.584794193887493, - "primary_metric": "cosine_spearman", - "task": "SummEvalSummarization.v2", - "task_type": "Summarization" - }, - { - "candidate_100": 71.8507462686567, - "current_100": 56.82089552238806, - "delta_pp": 15.02985074626865, - "primary_metric": "accuracy", - "task": "AmazonCounterfactualClassification", - "task_type": "Classification" - }, - { - "candidate_100": 89.02101343562433, - "current_100": 84.16589279728525, - "delta_pp": 4.8551206383390735, - "primary_metric": "cosine_spearman", - "task": "STS17", - "task_type": "STS" - }, - { - "candidate_100": 68.75333638044488, - "current_100": 67.57005849656171, - "delta_pp": 1.1832778838831643, - "primary_metric": "cosine_spearman", - "task": "STS22.v2", - "task_type": "STS" - } - ], "audio_task_delta_pp": { - "BeijingOpera": 0.0, - "BirdCLEF": 0.0, - "CREMADPairClassification": 0.12104996267769952, - "CREMA_D": 0.0, - "CREMA_DClustering": 0.0, - "CommonLanguageAgeDetection": 0.0, - "GTZANAudioReranking": 0.0, - "GTZANGenre": 0.0, - "IEMOCAPGender": 0.0, - "JamAltArtistA2ARetrieval": 0.0, - "MInDS14": 0.0, - "MridinghamTonic": -0.14336917562723928, - "NMSQAPairClassification": -0.1520590751812989, - "SIBFLEURS": 0.0, - "VehicleSoundClustering": 0.0, - "VoxCelebSA": 0.0, - "VoxPopuliAccentPairClassification": -0.02844431688273641, - "VoxPopuliGenderClustering": 0.0, - "VoxPopuliLanguageID": 0.0 + "BeijingOpera": 10.549645390070905, + "BirdCLEF": 2.2999999999999963, + "CREMADPairClassification": 1.0060252268561554, + "CREMA_D": 2.2844768445301367, + "CREMA_DClustering": 2.8541749889603825, + "CommonLanguageAgeDetection": 0.6150000000000017, + "GTZANAudioReranking": 6.68200000000001, + "GTZANGenre": 11.399999999999999, + "IEMOCAPGender": 3.237875602721152, + "JamAltArtistA2ARetrieval": 16.90099999999998, + "MInDS14": -1.4691308536005643, + "MridinghamTonic": 26.05657742038183, + "NMSQAPairClassification": -0.3840625955228716, + "SIBFLEURS": -0.23211656203982745, + "VehicleSoundClustering": -8.525275870377525, + "VoxCelebSA": -0.00058896531414665, + "VoxPopuliAccentPairClassification": 0.1293630037889515, + "VoxPopuliGenderClustering": -0.08127278385060711, + "VoxPopuliLanguageID": -0.5999999999999783 }, - "paired_interval_scope": "Paired intervals cover the original common metrics, family and two-primary-per-family utility. They are not confidence intervals for the new all-14 common mean or the four-component product aggregate.", - "inference_profile_applicability": { - "revision": "self", - "native_artifact_sha256": "50ce808197fb43a1b8913a66d4a47339ec66ecd10d52cbd9f9e752d02f8c43fa", - "parameters": 135383808, - "report": "benchmarks/inference-equivalence.json", - "new_benchmark_run": false - } - }, - "paired_uncertainty_scope": "Paired intervals cover the original common metrics, family and two-primary-per-family utility. They are not confidence intervals for the new all-14 common mean or the four-component product aggregate.", - "inference_profile": { - "revision": "self", - "native_artifact_sha256": "50ce808197fb43a1b8913a66d4a47339ec66ecd10d52cbd9f9e752d02f8c43fa", - "parameters": 135383808, - "report": "benchmarks/inference-equivalence.json", - "new_benchmark_run": false + "audio_aggregate_delta_pp": { + "Any2AnyRetrieval": 16.90099999999998, + "AudioClassification": 4.921976261522682, + "AudioClustering": -1.9174578884225837, + "AudioPairClassification": 0.2504418783740858, + "AudioReranking": 6.68200000000001, + "Mean(Task)": 3.8275626761370307, + "Mean(TaskType)": 5.367592050294833 + }, + "English_and_image_unchanged": true, + "event840_in_net": false, + "paired_interval_scope": "Fresh paired intervals compare this candidate with the preceding Vela Nano computation. They are not an interval for the all-14 aggregate or the weighted net. An original-small interval was not recomputed for this update." }, + "paired_uncertainty_scope": "Fresh paired intervals compare this candidate with the preceding Vela Nano computation. They are not an interval for the all-14 aggregate or the weighted net. An original-small interval was not recomputed for this update.", "scorecard_comparison": { "primary_comparator": "llm-semantic-router/multi-modal-embed-small", "revision": "fdf8e01b7b0f3a69ac1ac8e2a64dcb1ede177ba4",