{ "schema_version": 2, "model": "vllm-sr/Vela-1.0-Omni-Nano", "revision_semantics": "self denotes the released native package. Each benchmark retains its actual evaluated artifact; exact unchanged-component applicability is explicit.", "score_scale": "0-1; multiply by 100 for percentages", "matrix": { "small": { "model": "vllm-sr/multi-modal-embed-small", "revision": "fdf8e01b7b0f3a69ac1ac8e2a64dcb1ede177ba4", "artifact_fingerprint_sha256": "684fe0c48a4ddd3944167f6e5dc75e97ada24e5b120bd9a2921f00dc03112eeb", "metrics": { "text.banking77.accuracy": { "numerator": 2169, "denominator": 3080, "value": 0.7042207792207792 }, "text.massive-en.accuracy": { "numerator": 1960, "denominator": 2972, "value": 0.6594885598923284 }, "image.image_to_text.recall@1": { "numerator": 336, "denominator": 823, "value": 0.4082624544349939 }, "image.image_to_text.recall@5": { "numerator": 553, "denominator": 823, "value": 0.6719319562575942 }, "image.image_to_text.recall@10": { "numerator": 646, "denominator": 823, "value": 0.7849331713244229 }, "image.text_to_image.recall@1": { "numerator": 1242, "denominator": 4115, "value": 0.3018226002430134 }, "image.text_to_image.recall@5": { "numerator": 2445, "denominator": 4115, "value": 0.5941676792223572 }, "image.text_to_image.recall@10": { "numerator": 3057, "denominator": 4115, "value": 0.7428918590522479 }, "audio.audio_to_text.recall@1": { "numerator": 110, "denominator": 2611, "value": 0.04212945231711988 }, "audio.audio_to_text.recall@5": { "numerator": 313, "denominator": 2611, "value": 0.11987744159325929 }, "audio.audio_to_text.recall@10": { "numerator": 497, "denominator": 2611, "value": 0.1903485254691689 }, "audio.text_to_audio.recall@1": { "numerator": 250, "denominator": 2610, "value": 0.09578544061302682 }, "audio.text_to_audio.recall@5": { "numerator": 588, "denominator": 2610, "value": 0.22528735632183908 }, "audio.text_to_audio.recall@10": { "numerator": 801, "denominator": 2610, "value": 0.30689655172413793 } } }, "candidate": { "model": "vllm-sr/Vela-1.0-Omni-Nano", "revision": "self", "artifact_fingerprint_sha256": "474ad3d52f6ee7f818a3dbb6112e3d66c2423542d08228e7996a8cd600bd99a2", "evaluated_native_artifact_sha256": "474ad3d52f6ee7f818a3dbb6112e3d66c2423542d08228e7996a8cd600bd99a2", "parameters": 163771288, "metrics": { "audio.audio_to_text.recall@1": { "denominator": 2611, "numerator": 421, "value": 0.1612409038682497 }, "audio.audio_to_text.recall@10": { "denominator": 2611, "numerator": 1191, "value": 0.45614707008808886 }, "audio.audio_to_text.recall@5": { "denominator": 2611, "numerator": 909, "value": 0.34814247414783606 }, "audio.text_to_audio.recall@1": { "denominator": 2610, "numerator": 531, "value": 0.20344827586206896 }, "audio.text_to_audio.recall@10": { "denominator": 2610, "numerator": 1321, "value": 0.5061302681992337 }, "audio.text_to_audio.recall@5": { "denominator": 2610, "numerator": 1043, "value": 0.3996168582375479 }, "image.image_to_text.recall@1": { "denominator": 823, "numerator": 501, "value": 0.6087484811664642 }, "image.image_to_text.recall@10": { "denominator": 823, "numerator": 781, "value": 0.9489671931956257 }, "image.image_to_text.recall@5": { "denominator": 823, "numerator": 717, "value": 0.8712029161603888 }, "image.text_to_image.recall@1": { "denominator": 4115, "numerator": 2297, "value": 0.5582017010935602 }, "image.text_to_image.recall@10": { "denominator": 4115, "numerator": 3879, "value": 0.9426488456865127 }, "image.text_to_image.recall@5": { "denominator": 4115, "numerator": 3507, "value": 0.8522478736330498 }, "text.banking77.accuracy": { "denominator": 3080, "numerator": 2710, "value": 0.8798701298701299 }, "text.massive-en.accuracy": { "denominator": 2972, "numerator": 2436, "value": 0.819650067294751 } } } }, "datasets": { "banking77": { "source": "https://github.com/PolyAI-LDN/task-specific-datasets", "revision": "57ec275d8078af65b7731c2a98be812d844a6d6b", "license": "CC-BY-4.0", "split": "test", "queries": 3080, "classes": 77 }, "massive-en": { "source": "https://huggingface.co/datasets/AmazonScience/massive", "revision": "ff6bd8e4b27c3543e4f8fe2108f32bb95a6f8740", "license": "CC-BY-4.0", "configuration": "en-US", "split": "test", "queries": 2972, "prototype_classes": 60 }, "image": { "source": "https://cocodataset.org/", "caption_split_source": "https://cs.stanford.edu/people/karpathy/deepimagesent/", "split": "Karpathy test, CC-BY 2.0 image subset", "images": 823, "captions": 4115, "image_license": "CC-BY-2.0", "caption_license": "CC-BY-4.0" }, "audio": { "source": "https://www.openslr.org/12", "split": "test-clean", "archive_sha256": "39fde525e59672dc6d1551919b1478f724438a95aa55f874b576be21967e6c23", "license": "CC-BY-4.0", "clips": 2611, "unique_transcripts": 2610, "maximum_seconds": 30 } }, "protocol": { "embedding_dtype": "float32", "attention": "SDPA", "autocast": false, "tf32": false, "transformers_version": "4.57.6", "torch_version_family": "2.12", "text_max_tokens": 128, "truncation": "right, native tokenizer including special tokens", "batch_sizes": { "text": 64, "image": 8, "audio": 8 }, "pooling": "native full-dimension model pooling", "similarity": "FP32 cosine of L2-normalized embeddings", "ties": "lexicographic candidate ID", "retrieval": "complete candidate pools; all annotated positives", "classification": "Nearest normalized mean class prototype from labeled training examples; the original small comparator has task-adapted history, while current GIST is frozen.", "aggregation": "Exact per-metric counts; the product common aggregate averages all fourteen metrics with equal text/image/audio family weights (2/6/6 metrics). Historical paired family/utility intervals retain their original definitions.", "evaluation_scope": "fixed test-split evaluation pools reused across releases", "text_training_scope": "Current GIST text tower is frozen; supervised training prototypes are recomputed under the fixed common protocol. The earlier small comparator has task-adapted history. This is not the official MTEB classification protocol.", "audio_preprocessing": "Original PCM is separately resampled to 16 kHz for Whisper and 48 kHz for CLAP; official evaluation resamples before applying its 30-second cap. Public API rejects inputs longer than 30 seconds." }, "paired_uncertainty": { "scope": "Fresh paired intervals compare this candidate with the preceding Vela Nano computation. They are not an interval for the all-14 aggregate or the weighted net. An original-small interval was not recomputed for this update.", "previous_release": { "published_fingerprint": "50ce808197fb43a1b8913a66d4a47339ec66ecd10d52cbd9f9e752d02f8c43fa", "evaluated_fingerprint": "96d76f97478e114b816df3ab1b3c6947e4c5ef0babfb40ab865d7e3c838d34c4", "result": { "intervals": { "audio.audio_to_text.recall@1": { "lower": -0.018926721728713448, "point_delta": -0.010340865568747606, "upper": -0.001484096923424178 }, "audio.audio_to_text.recall@10": { "lower": -0.02562171008540101, "point_delta": -0.013787820758330141, "upper": -0.001602915531368321 }, "audio.audio_to_text.recall@5": { "lower": -0.02957690797530578, "point_delta": -0.018383761011106857, "upper": -0.007289100963896155 }, "audio.text_to_audio.recall@1": { "lower": -0.06884900952272836, "point_delta": -0.050957854406130265, "upper": -0.03441425975371444 }, "audio.text_to_audio.recall@10": { "lower": -0.07240471605447092, "point_delta": -0.052873563218390804, "upper": -0.03213863952299779 }, "audio.text_to_audio.recall@5": { "lower": -0.07963438278344268, "point_delta": -0.061685823754789273, "upper": -0.044619605933086656 }, "family.audio": { "lower": -0.03959510843694133, "point_delta": -0.030649359987438937, "upper": -0.02150226739148048 }, "family.image": { "lower": 0.0, "point_delta": 0.0, "upper": 0.0 }, "family.text": { "lower": 0.0, "point_delta": 0.0, "upper": 0.0 }, "image.image_to_text.recall@1": { "lower": 0.0, "point_delta": 0.0, "upper": 0.0 }, "image.image_to_text.recall@10": { "lower": 0.0, "point_delta": 0.0, "upper": 0.0 }, "image.image_to_text.recall@5": { "lower": 0.0, "point_delta": 0.0, "upper": 0.0 }, "image.text_to_image.recall@1": { "lower": 0.0, "point_delta": 0.0, "upper": 0.0 }, "image.text_to_image.recall@10": { "lower": 0.0, "point_delta": 0.0, "upper": 0.0 }, "image.text_to_image.recall@5": { "lower": 0.0, "point_delta": 0.0, "upper": 0.0 }, "text.banking77.accuracy": { "lower": 0.0, "point_delta": 0.0, "upper": 0.0 }, "text.massive-en.accuracy": { "lower": 0.0, "point_delta": 0.0, "upper": 0.0 }, "utility": { "lower": -0.013198369478980442, "point_delta": -0.010216453329146313, "upper": -0.0071674224638268265 } }, "method": "paired stratified group percentile; PCG64; fixed candidate pools", "replicates": 2000, "seed": 20260918 } }, "original_small": { "status": "not_recomputed_for_this_update", "value": null } }, "training_scope": { "text": "The complete text tower is the frozen official avsolatorio/GIST-small-Embedding-v0 checkpoint at 75e62fd210b9fde790430e0b2f040b0b00a021b1. Its model card discloses MTEB classification training data; no claim of independent pretraining data is made.", "image": "Frozen current vision encoder and initial image head; an orthogonal shared-space rotation is trained on 16,509 COCO training images, 81,998 unique caption texts and 82,522 complete positive associations, then folded into the image affine head.", "audio": "The existing Whisper Tiny encoder and affine head stay frozen. A frozen CLAP audio tower and a 384×512 residual projection are added; only that projection is trained for one fixed 600-step run. Speech alignment and a pointwise parent anchor use 28,535 LibriSpeech TRAIN clips; known-positive event labels and CLAP relational geometry use 3,299 FSD50K TRAIN clips. Center/scale and the geometry coefficient use TRAIN only.", "procedure": "One fixed final-600-step residual artifact; no held-out benchmark examples fit the projection, statistics or coefficient. Full benchmark outcomes and regressions are retained.", "evaluation_scope": "Common text classification uses labeled training prototypes with the earlier task-adapted comparator. It is separate from the official frozen-encoder MTEB classification protocols.", "prior_alignment_history": { "text": "The complete text tower is the frozen official avsolatorio/GIST-small-Embedding-v0 checkpoint at 75e62fd210b9fde790430e0b2f040b0b00a021b1. Its model card discloses MTEB classification training data; no claim of independent pretraining data is made.", "image": "Frozen current vision encoder and initial image head; an orthogonal shared-space rotation is trained on 16,509 COCO training images, 81,998 unique caption texts and 82,522 complete positive associations, then folded into the image affine head.", "audio": "Frozen current speech encoder and initial audio head; a separate orthogonal shared-space rotation is trained on 28,535 LibriSpeech train-clean-100 clips and all positive transcript associations, then folded into the audio affine head.", "procedure": "One fixed 300-step run; only the final artifact is reported. Encoders remain frozen. The two rotations preserve within-modality inner products, up to floating-point rounding.", "evaluation_scope": "Common text classification uses labeled training prototypes with the earlier task-adapted comparator. It is separate from the official frozen-encoder MTEB classification protocols." }, "event_labels": "149 labels with TRAIN support and 7,690 known-positive associations. Unannotated labels are not negatives; label names are weak class supervision, not observed clip captions." }, "descriptive_metrics": { "definition": "Macro-F1 is the unweighted mean of per-class F1 across all fixed prototype classes; zero_division=0.", "scope": "Same models, examples, class prototypes and predictions as the accuracy matrix.", "metrics": { "text.banking77.macro_f1": { "small": { "value": 0.7020781051899517, "classes": 77, "queries": 3080 }, "candidate": { "value": 0.8791681858119436, "classes": 77, "queries": 3080 } }, "text.massive-en.macro_f1": { "small": { "value": 0.6171775818092488, "classes": 60, "queries": 2972 }, "candidate": { "value": 0.7923223418549947, "classes": 60, "queries": 2972 } } }, "candidate_per_class": { "banking77": { "accuracy": { "denominator": 3080, "numerator": 2710, "value": 0.8798701298701299 }, "evaluated_classes": 77, "macro_f1": 0.8791681858119436, "macro_f1_class_universe": "all fixed prototype classes; zero_division=0", "per_class": { "Refund_not_showing_up": { "f1": 0.9512195121951219, "fn": 1, "fp": 3, "support": 40, "tp": 39 }, "activate_my_card": { "f1": 0.935064935064935, "fn": 4, "fp": 1, "support": 40, "tp": 36 }, "age_limit": { "f1": 1.0, "fn": 0, "fp": 0, "support": 40, "tp": 40 }, "apple_pay_or_google_pay": { "f1": 1.0, "fn": 0, "fp": 0, "support": 40, "tp": 40 }, "atm_support": { "f1": 0.9382716049382716, "fn": 2, "fp": 3, "support": 40, "tp": 38 }, "automatic_top_up": { "f1": 0.8918918918918919, "fn": 7, "fp": 1, "support": 40, "tp": 33 }, "balance_not_updated_after_bank_transfer": { "f1": 0.7407407407407407, "fn": 10, "fp": 11, "support": 40, "tp": 30 }, "balance_not_updated_after_cheque_or_cash_deposit": { "f1": 0.9285714285714286, "fn": 1, "fp": 5, "support": 40, "tp": 39 }, "beneficiary_not_allowed": { "f1": 0.7246376811594203, "fn": 15, "fp": 4, "support": 40, "tp": 25 }, "cancel_transfer": { "f1": 0.95, "fn": 2, "fp": 2, "support": 40, "tp": 38 }, "card_about_to_expire": { "f1": 0.9523809523809523, "fn": 0, "fp": 4, "support": 40, "tp": 40 }, "card_acceptance": { "f1": 0.9069767441860465, "fn": 1, "fp": 7, "support": 40, "tp": 39 }, "card_arrival": { "f1": 0.8421052631578947, "fn": 8, "fp": 4, "support": 40, "tp": 32 }, "card_delivery_estimate": { "f1": 0.8536585365853658, "fn": 5, "fp": 7, "support": 40, "tp": 35 }, "card_linking": { "f1": 0.8947368421052632, "fn": 6, "fp": 2, "support": 40, "tp": 34 }, "card_not_working": { "f1": 0.8181818181818182, "fn": 4, "fp": 12, "support": 40, "tp": 36 }, "card_payment_fee_charged": { "f1": 0.8536585365853658, "fn": 5, "fp": 7, "support": 40, "tp": 35 }, "card_payment_not_recognised": { "f1": 0.8395061728395061, "fn": 6, "fp": 7, "support": 40, "tp": 34 }, "card_payment_wrong_exchange_rate": { "f1": 0.95, "fn": 2, "fp": 2, "support": 40, "tp": 38 }, "card_swallowed": { "f1": 0.9473684210526315, "fn": 4, "fp": 0, "support": 40, "tp": 36 }, "cash_withdrawal_charge": { "f1": 0.8941176470588236, "fn": 2, "fp": 7, "support": 40, "tp": 38 }, "cash_withdrawal_not_recognised": { "f1": 0.7951807228915663, "fn": 7, "fp": 10, "support": 40, "tp": 33 }, "change_pin": { "f1": 0.9873417721518988, "fn": 1, "fp": 0, "support": 40, "tp": 39 }, "compromised_card": { "f1": 0.8395061728395061, "fn": 6, "fp": 7, "support": 40, "tp": 34 }, "contactless_not_working": { "f1": 0.9473684210526315, "fn": 4, "fp": 0, "support": 40, "tp": 36 }, "country_support": { "f1": 0.9210526315789473, "fn": 5, "fp": 1, "support": 40, "tp": 35 }, "declined_card_payment": { "f1": 0.8043478260869565, "fn": 3, "fp": 15, "support": 40, "tp": 37 }, "declined_cash_withdrawal": { "f1": 0.8333333333333334, "fn": 5, "fp": 9, "support": 40, "tp": 35 }, "declined_transfer": { "f1": 0.8115942028985508, "fn": 12, "fp": 1, "support": 40, "tp": 28 }, "direct_debit_payment_not_recognised": { "f1": 0.8450704225352113, "fn": 10, "fp": 1, "support": 40, "tp": 30 }, "disposable_card_limits": { "f1": 0.8395061728395061, "fn": 6, "fp": 7, "support": 40, "tp": 34 }, "edit_personal_details": { "f1": 0.975, "fn": 1, "fp": 1, "support": 40, "tp": 39 }, "exchange_charge": { "f1": 0.9367088607594937, "fn": 3, "fp": 2, "support": 40, "tp": 37 }, "exchange_rate": { "f1": 0.9397590361445783, "fn": 1, "fp": 4, "support": 40, "tp": 39 }, "exchange_via_app": { "f1": 0.7906976744186046, "fn": 6, "fp": 12, "support": 40, "tp": 34 }, "extra_charge_on_statement": { "f1": 0.8409090909090909, "fn": 3, "fp": 11, "support": 40, "tp": 37 }, "failed_transfer": { "f1": 0.7956989247311828, "fn": 3, "fp": 16, "support": 40, "tp": 37 }, "fiat_currency_support": { "f1": 0.7804878048780488, "fn": 8, "fp": 10, "support": 40, "tp": 32 }, "get_disposable_virtual_card": { "f1": 0.7887323943661971, "fn": 12, "fp": 3, "support": 40, "tp": 28 }, "get_physical_card": { "f1": 0.963855421686747, "fn": 0, "fp": 3, "support": 40, "tp": 40 }, "getting_spare_card": { "f1": 0.8941176470588236, "fn": 2, "fp": 7, "support": 40, "tp": 38 }, "getting_virtual_card": { "f1": 0.8764044943820225, "fn": 1, "fp": 10, "support": 40, "tp": 39 }, "lost_or_stolen_card": { "f1": 0.8666666666666667, "fn": 1, "fp": 11, "support": 40, "tp": 39 }, "lost_or_stolen_phone": { "f1": 0.9873417721518988, "fn": 1, "fp": 0, "support": 40, "tp": 39 }, "order_physical_card": { "f1": 0.8275862068965517, "fn": 4, "fp": 11, "support": 40, "tp": 36 }, "passcode_forgotten": { "f1": 0.9876543209876543, "fn": 0, "fp": 1, "support": 40, "tp": 40 }, "pending_card_payment": { "f1": 0.95, "fn": 2, "fp": 2, "support": 40, "tp": 38 }, "pending_cash_withdrawal": { "f1": 0.8611111111111112, "fn": 9, "fp": 1, "support": 40, "tp": 31 }, "pending_top_up": { "f1": 0.8831168831168831, "fn": 6, "fp": 3, "support": 40, "tp": 34 }, "pending_transfer": { "f1": 0.7647058823529411, "fn": 14, "fp": 2, "support": 40, "tp": 26 }, "pin_blocked": { "f1": 0.9230769230769231, "fn": 4, "fp": 2, "support": 40, "tp": 36 }, "receiving_money": { "f1": 0.9090909090909091, "fn": 5, "fp": 2, "support": 40, "tp": 35 }, "request_refund": { "f1": 0.9066666666666666, "fn": 6, "fp": 1, "support": 40, "tp": 34 }, "reverted_card_payment?": { "f1": 0.825, "fn": 7, "fp": 7, "support": 40, "tp": 33 }, "supported_cards_and_currencies": { "f1": 0.7777777777777778, "fn": 12, "fp": 4, "support": 40, "tp": 28 }, "terminate_account": { "f1": 0.9411764705882353, "fn": 0, "fp": 5, "support": 40, "tp": 40 }, "top_up_by_bank_transfer_charge": { "f1": 0.8888888888888888, "fn": 4, "fp": 5, "support": 40, "tp": 36 }, "top_up_by_card_charge": { "f1": 0.95, "fn": 2, "fp": 2, "support": 40, "tp": 38 }, "top_up_by_cash_or_cheque": { "f1": 0.9230769230769231, "fn": 4, "fp": 2, "support": 40, "tp": 36 }, "top_up_failed": { "f1": 0.8705882352941177, "fn": 3, "fp": 8, "support": 40, "tp": 37 }, "top_up_limits": { "f1": 0.9512195121951219, "fn": 1, "fp": 3, "support": 40, "tp": 39 }, "top_up_reverted": { "f1": 0.875, "fn": 5, "fp": 5, "support": 40, "tp": 35 }, "topping_up_by_card": { "f1": 0.5, "fn": 23, "fp": 11, "support": 40, "tp": 17 }, "transaction_charged_twice": { "f1": 0.9629629629629629, "fn": 1, "fp": 2, "support": 40, "tp": 39 }, "transfer_fee_charged": { "f1": 0.8333333333333334, "fn": 10, "fp": 2, "support": 40, "tp": 30 }, "transfer_into_account": { "f1": 0.7659574468085106, "fn": 4, "fp": 18, "support": 40, "tp": 36 }, "transfer_not_received_by_recipient": { "f1": 0.7088607594936709, "fn": 12, "fp": 11, "support": 40, "tp": 28 }, "transfer_timing": { "f1": 0.7333333333333333, "fn": 7, "fp": 17, "support": 40, "tp": 33 }, "unable_to_verify_identity": { "f1": 0.926829268292683, "fn": 2, "fp": 4, "support": 40, "tp": 38 }, "verify_my_identity": { "f1": 0.9024390243902439, "fn": 3, "fp": 5, "support": 40, "tp": 37 }, "verify_source_of_funds": { "f1": 1.0, "fn": 0, "fp": 0, "support": 40, "tp": 40 }, "verify_top_up": { "f1": 1.0, "fn": 0, "fp": 0, "support": 40, "tp": 40 }, "virtual_card_not_working": { "f1": 0.9333333333333333, "fn": 5, "fp": 0, "support": 40, "tp": 35 }, "visa_or_mastercard": { "f1": 0.9333333333333333, "fn": 5, "fp": 0, "support": 40, "tp": 35 }, "why_verify_identity": { "f1": 0.9090909090909091, "fn": 5, "fp": 2, "support": 40, "tp": 35 }, "wrong_amount_of_cash_received": { "f1": 0.9090909090909091, "fn": 5, "fp": 2, "support": 40, "tp": 35 }, "wrong_exchange_rate_for_cash_withdrawal": { "f1": 0.7878787878787878, "fn": 14, "fp": 0, "support": 40, "tp": 26 } }, "prototype_classes": 77 }, "massive-en": { "accuracy": { "denominator": 2972, "numerator": 2436, "value": 0.819650067294751 }, "evaluated_classes": 59, "macro_f1": 0.7923223418549947, "macro_f1_class_universe": "all fixed prototype classes; zero_division=0", "per_class": { "alarm_query": { "f1": 0.9523809523809523, "fn": 3, "fp": 0, "support": 33, "tp": 30 }, "alarm_remove": { "f1": 0.9545454545454546, "fn": 0, "fp": 2, "support": 21, "tp": 21 }, "alarm_set": { "f1": 0.9047619047619048, "fn": 3, "fp": 5, "support": 41, "tp": 38 }, "audio_volume_down": { "f1": 0.9565217391304348, "fn": 0, "fp": 1, "support": 11, "tp": 11 }, "audio_volume_mute": { "f1": 0.8787878787878788, "fn": 3, "fp": 5, "support": 32, "tp": 29 }, "audio_volume_other": { "f1": 1.0, "fn": 0, "fp": 0, "support": 6, "tp": 6 }, "audio_volume_up": { "f1": 0.782608695652174, "fn": 4, "fp": 1, "support": 13, "tp": 9 }, "calendar_query": { "f1": 0.6864406779661016, "fn": 45, "fp": 29, "support": 126, "tp": 81 }, "calendar_remove": { "f1": 0.9264705882352942, "fn": 4, "fp": 6, "support": 67, "tp": 63 }, "calendar_set": { "f1": 0.8759124087591241, "fn": 29, "fp": 22, "support": 209, "tp": 180 }, "cooking_query": { "f1": 0.0, "fn": 0, "fp": 2, "support": 0, "tp": 0 }, "cooking_recipe": { "f1": 0.9523809523809523, "fn": 2, "fp": 5, "support": 72, "tp": 70 }, "datetime_convert": { "f1": 0.5263157894736842, "fn": 5, "fp": 13, "support": 15, "tp": 10 }, "datetime_query": { "f1": 0.7861271676300579, "fn": 20, "fp": 17, "support": 88, "tp": 68 }, "email_addcontact": { "f1": 0.5714285714285714, "fn": 2, "fp": 13, "support": 12, "tp": 10 }, "email_query": { "f1": 0.9385964912280702, "fn": 12, "fp": 2, "support": 119, "tp": 107 }, "email_querycontact": { "f1": 0.7540983606557377, "fn": 3, "fp": 12, "support": 26, "tp": 23 }, "email_sendemail": { "f1": 0.8959276018099548, "fn": 15, "fp": 8, "support": 114, "tp": 99 }, "general_greet": { "f1": 0.18181818181818182, "fn": 0, "fp": 9, "support": 1, "tp": 1 }, "general_joke": { "f1": 0.9230769230769231, "fn": 1, "fp": 2, "support": 19, "tp": 18 }, "general_quirky": { "f1": 0.4728434504792332, "fn": 94, "fp": 71, "support": 168, "tp": 74 }, "iot_cleaning": { "f1": 0.9433962264150944, "fn": 1, "fp": 2, "support": 26, "tp": 25 }, "iot_coffee": { "f1": 0.9859154929577465, "fn": 1, "fp": 0, "support": 36, "tp": 35 }, "iot_hue_lightchange": { "f1": 0.8181818181818182, "fn": 9, "fp": 3, "support": 36, "tp": 27 }, "iot_hue_lightdim": { "f1": 0.8780487804878049, "fn": 3, "fp": 2, "support": 21, "tp": 18 }, "iot_hue_lightoff": { "f1": 0.9176470588235294, "fn": 4, "fp": 3, "support": 43, "tp": 39 }, "iot_hue_lighton": { "f1": 0.18181818181818182, "fn": 2, "fp": 7, "support": 3, "tp": 1 }, "iot_hue_lightup": { "f1": 0.7857142857142857, "fn": 5, "fp": 7, "support": 27, "tp": 22 }, "iot_wemo_off": { "f1": 0.9444444444444444, "fn": 1, "fp": 1, "support": 18, "tp": 17 }, "iot_wemo_on": { "f1": 0.9, "fn": 1, "fp": 1, "support": 10, "tp": 9 }, "lists_createoradd": { "f1": 0.7777777777777778, "fn": 11, "fp": 5, "support": 39, "tp": 28 }, "lists_query": { "f1": 0.8333333333333334, "fn": 11, "fp": 5, "support": 51, "tp": 40 }, "lists_remove": { "f1": 0.9215686274509803, "fn": 5, "fp": 3, "support": 52, "tp": 47 }, "music_dislikeness": { "f1": 0.8888888888888888, "fn": 0, "fp": 1, "support": 4, "tp": 4 }, "music_likeness": { "f1": 0.7878787878787878, "fn": 10, "fp": 4, "support": 36, "tp": 26 }, "music_query": { "f1": 0.7945205479452054, "fn": 6, "fp": 9, "support": 35, "tp": 29 }, "music_settings": { "f1": 0.36363636363636365, "fn": 2, "fp": 12, "support": 6, "tp": 4 }, "news_query": { "f1": 0.8583333333333333, "fn": 21, "fp": 13, "support": 124, "tp": 103 }, "play_audiobook": { "f1": 0.8311688311688312, "fn": 9, "fp": 4, "support": 41, "tp": 32 }, "play_game": { "f1": 0.717391304347826, "fn": 2, "fp": 24, "support": 35, "tp": 33 }, "play_music": { "f1": 0.8238805970149253, "fn": 38, "fp": 21, "support": 176, "tp": 138 }, "play_podcasts": { "f1": 0.9043478260869565, "fn": 11, "fp": 0, "support": 63, "tp": 52 }, "play_radio": { "f1": 0.8208955223880597, "fn": 17, "fp": 7, "support": 72, "tp": 55 }, "qa_currency": { "f1": 0.9473684210526315, "fn": 3, "fp": 1, "support": 39, "tp": 36 }, "qa_definition": { "f1": 0.8305084745762712, "fn": 8, "fp": 12, "support": 57, "tp": 49 }, "qa_factoid": { "f1": 0.7205882352941176, "fn": 43, "fp": 33, "support": 141, "tp": 98 }, "qa_maths": { "f1": 0.7868852459016393, "fn": 1, "fp": 12, "support": 25, "tp": 24 }, "qa_stock": { "f1": 0.8727272727272727, "fn": 2, "fp": 5, "support": 26, "tp": 24 }, "recommendation_events": { "f1": 0.7102803738317757, "fn": 5, "fp": 26, "support": 43, "tp": 38 }, "recommendation_locations": { "f1": 0.7945205479452054, "fn": 2, "fp": 13, "support": 31, "tp": 29 }, "recommendation_movies": { "f1": 0.7058823529411765, "fn": 2, "fp": 13, "support": 20, "tp": 18 }, "social_post": { "f1": 0.8531468531468531, "fn": 20, "fp": 1, "support": 81, "tp": 61 }, "social_query": { "f1": 0.7301587301587301, "fn": 2, "fp": 15, "support": 25, "tp": 23 }, "takeaway_order": { "f1": 0.7659574468085106, "fn": 4, "fp": 7, "support": 22, "tp": 18 }, "takeaway_query": { "f1": 0.8266666666666667, "fn": 4, "fp": 9, "support": 35, "tp": 31 }, "transport_query": { "f1": 0.782608695652174, "fn": 15, "fp": 5, "support": 51, "tp": 36 }, "transport_taxi": { "f1": 0.9787234042553191, "fn": 0, "fp": 1, "support": 23, "tp": 23 }, "transport_ticket": { "f1": 0.88, "fn": 2, "fp": 7, "support": 35, "tp": 33 }, "transport_traffic": { "f1": 0.8108108108108109, "fn": 0, "fp": 7, "support": 15, "tp": 15 }, "weather_query": { "f1": 0.9426751592356688, "fn": 8, "fp": 10, "support": 156, "tp": 148 } }, "prototype_classes": 60 } } }, "standard_benchmarks": { "scope": "Eleven selected standard tasks; not a complete MTEB, MMTEB, MIEB or MAEB benchmark or leaderboard submission.", "library": "mteb", "library_version": "2.21.0", "model_revisions": { "nano": { "repo_id": "vllm-sr/Vela-1.0-Omni-Nano", "revision": "self", "native_artifact_sha256": "474ad3d52f6ee7f818a3dbb6112e3d66c2423542d08228e7996a8cd600bd99a2", "parameters": 163771288, "embedding_dimensions": 384, "max_text_tokens": 512, "applicability": "Each row states whether the full artifact was freshly evaluated or an exact unchanged component applies." }, "small": { "repo_id": "vllm-sr/multi-modal-embed-small", "revision": "fdf8e01b7b0f3a69ac1ac8e2a64dcb1ede177ba4", "native_artifact_sha256": "684fe0c48a4ddd3944167f6e5dc75e97ada24e5b120bd9a2921f00dc03112eeb", "status": "evaluated_all_11_selected_tasks", "max_text_tokens": 128, "inference_parameters": 133759488 } }, "protocol": { "precision": "float32", "batch_size": 8, "seed": 20260918, "similarity": "Cosine for primary STS/retrieval scores and normalized media-label matching; classification probes and clustering use each official task's algorithms.", "encoder_parameters_frozen": true, "classification": "Official task-specific frozen-embedding probes or fixed-label zero-shot classification.", "sampling": "Official task subsampling and repeated-probe protocols are preserved.", "text": "Each encoder uses its native preprocessing and context limit without added task instructions. Original small: 128 tokens; current Nano: 512 tokens.", "audio": "Current Nano: original PCM independently resampled to 16/48 kHz for speech/CLAP, then downmixed and capped at 30 seconds. Original small: official mono 16 kHz path. The official task algorithms, samples and seed are identical.", "score_scale": "Metrics retain their native MTEB scale; the model card multiplies primary scores by 100.", "undefined_metric": null, "missing_model_score": "Not measured in this standard evaluation.", "overlap": "No claim of independence from component pretraining data is made." }, "rows": [ { "task": "SICK-R", "type": "STS", "modalities": [ "text" ], "split": "test", "hf_subset": "default", "main_metric": "cosine_spearman", "dataset": { "path": "mteb/sickr-sts", "revision": "20a6d6f312dd54037fe07a32d58e5e168867909d" }, "counts": { "evaluation_pairs": 9927 }, "scores": { "nano": 0.8053174565838819, "small": 0.7139020320965759 }, "metrics": { "nano": { "pearson": 0.8603345909155973, "spearman": 0.8053174398238511, "cosine_pearson": 0.8603345867277692, "cosine_spearman": 0.8053174565838819, "manhattan_pearson": 0.8332190477069294, "manhattan_spearman": 0.8044409124391882, "euclidean_pearson": 0.834049309726515, "euclidean_spearman": 0.8053172111347874, "main_score": 0.8053174565838819, "hf_subset": "default", "languages": [ "eng-Latn" ], "mteb_version": "2.21.0" }, "small": { "pearson": 0.8179714611300387, "spearman": 0.713902378578463, "cosine_pearson": 0.81797146109438, "cosine_spearman": 0.7139020320965759, "manhattan_pearson": 0.7919193141236369, "manhattan_spearman": 0.7128438640347968, "euclidean_pearson": 0.792891259170334, "euclidean_spearman": 0.7139020311053554, "main_score": 0.7139020320965759, "hf_subset": "default", "languages": [ "eng-Latn" ], "mteb_version": "2.21.0" } }, "measurement_by_model": { "small": { "evaluated_native_fingerprint": "684fe0c48a4ddd3944167f6e5dc75e97ada24e5b120bd9a2921f00dc03112eeb", "original_checkpoint_revision": "fdf8e01b7b0f3a69ac1ac8e2a64dcb1ede177ba4", "raw_result_sha256": "e20f02a347841b3d5f61209f0b209c6314d006f8cc94f6e488e06239c862badf", "kind": "direct_frozen_encoder_evaluation" }, "nano": { "evaluated_native_fingerprint": "96d76f97478e114b816df3ab1b3c6947e4c5ef0babfb40ab865d7e3c838d34c4", "applicable_native_fingerprint": "474ad3d52f6ee7f818a3dbb6112e3d66c2423542d08228e7996a8cd600bd99a2", "kind": "exact_component_applicability" } }, "evaluated_native_fingerprint": "96d76f97478e114b816df3ab1b3c6947e4c5ef0babfb40ab865d7e3c838d34c4", "applicable_native_fingerprint": "474ad3d52f6ee7f818a3dbb6112e3d66c2423542d08228e7996a8cd600bd99a2", "measurement_kind": "exact_component_applicability" }, { "task": "STSBenchmark", "type": "STS", "modalities": [ "text" ], "split": "test", "hf_subset": "default", "main_metric": "cosine_spearman", "dataset": { "path": "mteb/stsbenchmark-sts", "revision": "b0fddb56ed78048fa8b90373c8a3cfc37b684831" }, "counts": { "evaluation_pairs": 1379 }, "scores": { "nano": 0.870781967302773, "small": 0.7729871863268857 }, "metrics": { "nano": { "pearson": 0.8541596951761798, "spearman": 0.8707819111802441, "cosine_pearson": 0.854159691434247, "cosine_spearman": 0.870781967302773, "manhattan_pearson": 0.867700912246326, "manhattan_spearman": 0.8690795304788012, "euclidean_pearson": 0.8699231026532712, "euclidean_spearman": 0.8707819663065923, "main_score": 0.870781967302773, "hf_subset": "default", "languages": [ "eng-Latn" ], "mteb_version": "2.21.0" }, "small": { "pearson": 0.765316203785143, "spearman": 0.7729871863268857, "cosine_pearson": 0.7653161983330018, "cosine_spearman": 0.7729871863268857, "manhattan_pearson": 0.7881395913225815, "manhattan_spearman": 0.7716275619454571, "euclidean_pearson": 0.789916662830987, "euclidean_spearman": 0.7729857841479683, "main_score": 0.7729871863268857, "hf_subset": "default", "languages": [ "eng-Latn" ], "mteb_version": "2.21.0" } }, "measurement_by_model": { "small": { "evaluated_native_fingerprint": "684fe0c48a4ddd3944167f6e5dc75e97ada24e5b120bd9a2921f00dc03112eeb", "original_checkpoint_revision": "fdf8e01b7b0f3a69ac1ac8e2a64dcb1ede177ba4", "raw_result_sha256": "ea0772149930250452298e3525b3405ce4fd5caed4aaf676f3745d4555692a07", "kind": "direct_frozen_encoder_evaluation" }, "nano": { "evaluated_native_fingerprint": "96d76f97478e114b816df3ab1b3c6947e4c5ef0babfb40ab865d7e3c838d34c4", "applicable_native_fingerprint": "474ad3d52f6ee7f818a3dbb6112e3d66c2423542d08228e7996a8cd600bd99a2", "kind": "exact_component_applicability" } }, "evaluated_native_fingerprint": "96d76f97478e114b816df3ab1b3c6947e4c5ef0babfb40ab865d7e3c838d34c4", "applicable_native_fingerprint": "474ad3d52f6ee7f818a3dbb6112e3d66c2423542d08228e7996a8cd600bd99a2", "measurement_kind": "exact_component_applicability" }, { "task": "ArguAna", "type": "Retrieval", "modalities": [ "text" ], "split": "test", "hf_subset": "default", "main_metric": "ndcg_at_10", "dataset": { "path": "mteb/arguana", "revision": "c22ab2a51041ffd869aaddef7af8d8215647e41a" }, "counts": { "queries": 1406, "documents": 8674 }, "scores": { "nano": 0.59246, "small": 0.32889 }, "metrics": { "nano": { "ndcg_at_1": 0.35349, "ndcg_at_3": 0.49665, "ndcg_at_5": 0.54581, "ndcg_at_10": 0.59246, "ndcg_at_20": 0.6132, "ndcg_at_100": 0.62189, "ndcg_at_1000": 0.62242, "map_at_1": 0.35349, "map_at_3": 0.46005, "map_at_5": 0.48729, "map_at_10": 0.50679, "map_at_20": 0.51261, "map_at_100": 0.51395, "map_at_1000": 0.51397, "recall_at_1": 0.35349, "recall_at_3": 0.60313, "recall_at_5": 0.72262, "recall_at_10": 0.86558, "recall_at_20": 0.94666, "recall_at_100": 0.99147, "recall_at_1000": 0.99573, "accuracy": 0.35349, "precision_at_1": 0.35349, "precision_at_3": 0.20104, "precision_at_5": 0.14452, "precision_at_10": 0.08656, "precision_at_20": 0.04733, "precision_at_100": 0.00991, "precision_at_1000": 0.001, "mrr_at_1": 0.3534850640113798, "mrr_at_3": 0.4600521574205785, "mrr_at_5": 0.48729255571360836, "mrr_at_10": 0.5067903429745535, "mrr_at_20": 0.5126066994732466, "mrr_at_100": 0.5139513666778618, "mrr_at_1000": 0.5139717342260931, "nauc_ndcg_at_1_max": -0.11747299439249903, "nauc_ndcg_at_1_std": -0.11436695908554317, "nauc_ndcg_at_1_diff1": 0.12406213904460131, "nauc_ndcg_at_3_max": -0.0810374905617702, "nauc_ndcg_at_3_std": -0.10902178800246173, "nauc_ndcg_at_3_diff1": 0.09457199601062195, "nauc_ndcg_at_5_max": -0.08485395980147033, "nauc_ndcg_at_5_std": -0.10485663186916257, "nauc_ndcg_at_5_diff1": 0.08582902365914505, "nauc_ndcg_at_10_max": -0.08309974111344137, "nauc_ndcg_at_10_std": -0.11506227102720018, "nauc_ndcg_at_10_diff1": 0.08990562536626041, "nauc_ndcg_at_20_max": -0.0791935521931642, "nauc_ndcg_at_20_std": -0.10319594721919796, "nauc_ndcg_at_20_diff1": 0.09971433495514617, "nauc_ndcg_at_100_max": -0.08635224865444148, "nauc_ndcg_at_100_std": -0.10462787774486179, "nauc_ndcg_at_100_diff1": 0.09791984963152253, "nauc_ndcg_at_1000_max": -0.0879373318368675, "nauc_ndcg_at_1000_std": -0.10553141137460995, "nauc_ndcg_at_1000_diff1": 0.09704987719570578, "nauc_map_at_1_max": -0.11747299439249903, "nauc_map_at_1_std": -0.11436695908554317, "nauc_map_at_1_diff1": 0.12406213904460131, "nauc_map_at_3_max": -0.09064151313996319, "nauc_map_at_3_std": -0.1105170515941861, "nauc_map_at_3_diff1": 0.09940664377727629, "nauc_map_at_5_max": -0.09342459992783216, "nauc_map_at_5_std": -0.10864465771524179, "nauc_map_at_5_diff1": 0.09517709876339578, "nauc_map_at_10_max": -0.09304942751697344, "nauc_map_at_10_std": -0.11264010939720397, "nauc_map_at_10_diff1": 0.09707068152210184, "nauc_map_at_20_max": -0.09221792922323616, "nauc_map_at_20_std": -0.10961211427408772, "nauc_map_at_20_diff1": 0.09943255521491648, "nauc_map_at_100_max": -0.09316799067959038, "nauc_map_at_100_std": -0.10968589627543163, "nauc_map_at_100_diff1": 0.09910011165040408, "nauc_map_at_1000_max": -0.09321385399598632, "nauc_map_at_1000_std": -0.10970513219025511, "nauc_map_at_1000_diff1": 0.09907801272427635, "nauc_recall_at_1_max": -0.11747299439249903, "nauc_recall_at_1_std": -0.11436695908554317, "nauc_recall_at_1_diff1": 0.12406213904460131, "nauc_recall_at_3_max": -0.05070404620274185, "nauc_recall_at_3_std": -0.10426214370217692, "nauc_recall_at_3_diff1": 0.0805612409902579, "nauc_recall_at_5_max": -0.051784236042595515, "nauc_recall_at_5_std": -0.0889570674134141, "nauc_recall_at_5_diff1": 0.049390111709494505, "nauc_recall_at_10_max": -0.013727385294040483, "nauc_recall_at_10_std": -0.13798296178002195, "nauc_recall_at_10_diff1": 0.044023090224588696, "nauc_recall_at_20_max": 0.15251226818625876, "nauc_recall_at_20_std": 0.03363980242022382, "nauc_recall_at_20_diff1": 0.15796490326794294, "nauc_recall_at_100_max": 0.3311354440025815, "nauc_recall_at_100_std": 0.4782526238266158, "nauc_recall_at_100_diff1": 0.25005005298352057, "nauc_recall_at_1000_max": 0.02172236447995406, "nauc_recall_at_1000_std": 0.609819560470231, "nauc_recall_at_1000_diff1": -0.005495147382062338, "nauc_precision_at_1_max": -0.11747299439249903, "nauc_precision_at_1_std": -0.11436695908554317, "nauc_precision_at_1_diff1": 0.12406213904460131, "nauc_precision_at_3_max": -0.05070404620274239, "nauc_precision_at_3_std": -0.10426214370217736, "nauc_precision_at_3_diff1": 0.08056124099025751, "nauc_precision_at_5_max": -0.05178423604259566, "nauc_precision_at_5_std": -0.08895706741341412, "nauc_precision_at_5_diff1": 0.04939011170949452, "nauc_precision_at_10_max": -0.013727385294041244, "nauc_precision_at_10_std": -0.13798296178002226, "nauc_precision_at_10_diff1": 0.044023090224589216, "nauc_precision_at_20_max": 0.15251226818625896, "nauc_precision_at_20_std": 0.03363980242022203, "nauc_precision_at_20_diff1": 0.15796490326794188, "nauc_precision_at_100_max": 0.3311354440025917, "nauc_precision_at_100_std": 0.4782526238266313, "nauc_precision_at_100_diff1": 0.25005005298354305, "nauc_precision_at_1000_max": 0.0217223644799946, "nauc_precision_at_1000_std": 0.6098195604702127, "nauc_precision_at_1000_diff1": -0.0054951473820258456, "nauc_mrr_at_1_max": -0.11747299439249903, "nauc_mrr_at_1_std": -0.11436695908554317, "nauc_mrr_at_1_diff1": 0.12406213904460131, "nauc_mrr_at_3_max": -0.09064151313996319, "nauc_mrr_at_3_std": -0.1105170515941861, "nauc_mrr_at_3_diff1": 0.09940664377727629, "nauc_mrr_at_5_max": -0.09342459992783216, "nauc_mrr_at_5_std": -0.10864465771524179, "nauc_mrr_at_5_diff1": 0.09517709876339578, "nauc_mrr_at_10_max": -0.09304942751697344, "nauc_mrr_at_10_std": -0.11264010939720397, "nauc_mrr_at_10_diff1": 0.09707068152210184, "nauc_mrr_at_20_max": -0.09221792922323616, "nauc_mrr_at_20_std": -0.10961211427408772, "nauc_mrr_at_20_diff1": 0.09943255521491648, "nauc_mrr_at_100_max": -0.09316799067959038, "nauc_mrr_at_100_std": -0.10968589627543163, "nauc_mrr_at_100_diff1": 0.09910011165040408, "nauc_mrr_at_1000_max": -0.09321385399598632, "nauc_mrr_at_1000_std": -0.10970513219025511, "nauc_mrr_at_1000_diff1": 0.09907801272427635, "hit_rate_at_1": 0.35349, "hit_rate_at_3": 0.60313, "hit_rate_at_5": 0.72262, "hit_rate_at_10": 0.86558, "hit_rate_at_20": 0.94666, "hit_rate_at_100": 0.99147, "hit_rate_at_1000": 0.99573, "main_score": 0.59246, "hf_subset": "default", "languages": [ "eng-Latn" ], "mteb_version": "2.21.0" }, "small": { "ndcg_at_1": 0.1458, "ndcg_at_3": 0.23456, "ndcg_at_5": 0.27835, "ndcg_at_10": 0.32889, "ndcg_at_20": 0.36967, "ndcg_at_100": 0.4045, "ndcg_at_1000": 0.41446, "map_at_1": 0.1458, "map_at_3": 0.21136, "map_at_5": 0.2354, "map_at_10": 0.25622, "map_at_20": 0.26767, "map_at_100": 0.27279, "map_at_1000": 0.27315, "recall_at_1": 0.1458, "recall_at_3": 0.30228, "recall_at_5": 0.40967, "recall_at_10": 0.56615, "recall_at_20": 0.72546, "recall_at_100": 0.90896, "recall_at_1000": 0.98791, "accuracy": 0.1458, "precision_at_1": 0.1458, "precision_at_3": 0.10076, "precision_at_5": 0.08193, "precision_at_10": 0.05661, "precision_at_20": 0.03627, "precision_at_100": 0.00909, "precision_at_1000": 0.00099, "mrr_at_1": 0.14580369843527738, "mrr_at_3": 0.21135609293504032, "mrr_at_5": 0.2353959222380275, "mrr_at_10": 0.25621881279776015, "mrr_at_20": 0.26766780266556217, "mrr_at_100": 0.27278502616211503, "mrr_at_1000": 0.2731483695142967, "nauc_ndcg_at_1_max": 0.07124074977242485, "nauc_ndcg_at_1_std": 0.07043132234894327, "nauc_ndcg_at_1_diff1": 0.17867294841183828, "nauc_ndcg_at_3_max": 0.05391218816481269, "nauc_ndcg_at_3_std": 0.08300022308422525, "nauc_ndcg_at_3_diff1": 0.13721153170379125, "nauc_ndcg_at_5_max": 0.04761392610075392, "nauc_ndcg_at_5_std": 0.09330318818026477, "nauc_ndcg_at_5_diff1": 0.129721108359703, "nauc_ndcg_at_10_max": 0.06398717148477524, "nauc_ndcg_at_10_std": 0.11739949823329376, "nauc_ndcg_at_10_diff1": 0.1353789207379315, "nauc_ndcg_at_20_max": 0.06583701467786816, "nauc_ndcg_at_20_std": 0.11760807459667257, "nauc_ndcg_at_20_diff1": 0.1464058778549573, "nauc_ndcg_at_100_max": 0.07271569743212124, "nauc_ndcg_at_100_std": 0.12767735668965335, "nauc_ndcg_at_100_diff1": 0.14440559200255987, "nauc_ndcg_at_1000_max": 0.06760834890113898, "nauc_ndcg_at_1000_std": 0.11565058305226664, "nauc_ndcg_at_1000_diff1": 0.14440414116396014, "nauc_map_at_1_max": 0.07124074977242485, "nauc_map_at_1_std": 0.07043132234894327, "nauc_map_at_1_diff1": 0.17867294841183828, "nauc_map_at_3_max": 0.05536834016676611, "nauc_map_at_3_std": 0.07918800842578326, "nauc_map_at_3_diff1": 0.14582401585695304, "nauc_map_at_5_max": 0.05153209428882134, "nauc_map_at_5_std": 0.08553839884319275, "nauc_map_at_5_diff1": 0.1412262351733516, "nauc_map_at_10_max": 0.05764323340595916, "nauc_map_at_10_std": 0.09567768210275003, "nauc_map_at_10_diff1": 0.14384793290679312, "nauc_map_at_20_max": 0.058797567381990594, "nauc_map_at_20_std": 0.09556268067232435, "nauc_map_at_20_diff1": 0.14719660481591418, "nauc_map_at_100_max": 0.060341050948804, "nauc_map_at_100_std": 0.09715443446030105, "nauc_map_at_100_diff1": 0.14729508200613933, "nauc_map_at_1000_max": 0.060216285099770145, "nauc_map_at_1000_std": 0.09674391170833818, "nauc_map_at_1000_diff1": 0.14729630480806993, "nauc_recall_at_1_max": 0.07124074977242485, "nauc_recall_at_1_std": 0.07043132234894327, "nauc_recall_at_1_diff1": 0.17867294841183828, "nauc_recall_at_3_max": 0.05116463248307958, "nauc_recall_at_3_std": 0.09233493207170033, "nauc_recall_at_3_diff1": 0.11697591005218377, "nauc_recall_at_5_max": 0.0387185231377629, "nauc_recall_at_5_std": 0.11230251236242163, "nauc_recall_at_5_diff1": 0.10273687046821164, "nauc_recall_at_10_max": 0.08768343797110026, "nauc_recall_at_10_std": 0.18174312026830694, "nauc_recall_at_10_diff1": 0.11544199093498575, "nauc_recall_at_20_max": 0.0976177919930251, "nauc_recall_at_20_std": 0.20309410782757367, "nauc_recall_at_20_diff1": 0.15596172000438252, "nauc_recall_at_100_max": 0.21399140718760357, "nauc_recall_at_100_std": 0.45432867879288047, "nauc_recall_at_100_diff1": 0.13074733185800497, "nauc_recall_at_1000_max": 0.29184801184590964, "nauc_recall_at_1000_std": 0.7268160753847613, "nauc_recall_at_1000_diff1": 0.04649951766999351, "nauc_precision_at_1_max": 0.07124074977242485, "nauc_precision_at_1_std": 0.07043132234894327, "nauc_precision_at_1_diff1": 0.17867294841183828, "nauc_precision_at_3_max": 0.051164632483079536, "nauc_precision_at_3_std": 0.09233493207170006, "nauc_precision_at_3_diff1": 0.11697591005218336, "nauc_precision_at_5_max": 0.03871852313776277, "nauc_precision_at_5_std": 0.11230251236242186, "nauc_precision_at_5_diff1": 0.10273687046821152, "nauc_precision_at_10_max": 0.08768343797110059, "nauc_precision_at_10_std": 0.18174312026830697, "nauc_precision_at_10_diff1": 0.11544199093498574, "nauc_precision_at_20_max": 0.09761779199302502, "nauc_precision_at_20_std": 0.20309410782757284, "nauc_precision_at_20_diff1": 0.15596172000438274, "nauc_precision_at_100_max": 0.2139914071876045, "nauc_precision_at_100_std": 0.45432867879288213, "nauc_precision_at_100_diff1": 0.13074733185800294, "nauc_precision_at_1000_max": 0.29184801184591147, "nauc_precision_at_1000_std": 0.7268160753847517, "nauc_precision_at_1000_diff1": 0.04649951766998672, "nauc_mrr_at_1_max": 0.07124074977242485, "nauc_mrr_at_1_std": 0.07043132234894327, "nauc_mrr_at_1_diff1": 0.17867294841183828, "nauc_mrr_at_3_max": 0.05536834016676611, "nauc_mrr_at_3_std": 0.07918800842578326, "nauc_mrr_at_3_diff1": 0.14582401585695304, "nauc_mrr_at_5_max": 0.05153209428882134, "nauc_mrr_at_5_std": 0.08553839884319275, "nauc_mrr_at_5_diff1": 0.1412262351733516, "nauc_mrr_at_10_max": 0.05764323340595916, "nauc_mrr_at_10_std": 0.09567768210275003, "nauc_mrr_at_10_diff1": 0.14384793290679312, "nauc_mrr_at_20_max": 0.058797567381990594, "nauc_mrr_at_20_std": 0.09556268067232435, "nauc_mrr_at_20_diff1": 0.14719660481591418, "nauc_mrr_at_100_max": 0.060341050948804, "nauc_mrr_at_100_std": 0.09715443446030105, "nauc_mrr_at_100_diff1": 0.14729508200613933, "nauc_mrr_at_1000_max": 0.060216285099770145, "nauc_mrr_at_1000_std": 0.09674391170833818, "nauc_mrr_at_1000_diff1": 0.14729630480806993, "hit_rate_at_1": 0.1458, "hit_rate_at_3": 0.30228, "hit_rate_at_5": 0.40967, "hit_rate_at_10": 0.56615, "hit_rate_at_20": 0.72546, "hit_rate_at_100": 0.90896, "hit_rate_at_1000": 0.98791, "main_score": 0.32889, "hf_subset": "default", "languages": [ "eng-Latn" ], "mteb_version": "2.21.0" } }, "measurement_by_model": { "small": { "evaluated_native_fingerprint": "684fe0c48a4ddd3944167f6e5dc75e97ada24e5b120bd9a2921f00dc03112eeb", "original_checkpoint_revision": "fdf8e01b7b0f3a69ac1ac8e2a64dcb1ede177ba4", "raw_result_sha256": "92cbe96f6e84ebd0bc144001424ac46bb3a630ef64bffb433c2cdd63a10c43ec", "kind": "direct_frozen_encoder_evaluation" }, "nano": { "evaluated_native_fingerprint": "96d76f97478e114b816df3ab1b3c6947e4c5ef0babfb40ab865d7e3c838d34c4", "applicable_native_fingerprint": "474ad3d52f6ee7f818a3dbb6112e3d66c2423542d08228e7996a8cd600bd99a2", "kind": "exact_component_applicability" } }, "evaluated_native_fingerprint": "96d76f97478e114b816df3ab1b3c6947e4c5ef0babfb40ab865d7e3c838d34c4", "applicable_native_fingerprint": "474ad3d52f6ee7f818a3dbb6112e3d66c2423542d08228e7996a8cd600bd99a2", "measurement_kind": "exact_component_applicability" }, { "task": "TwentyNewsgroupsClustering.v2", "type": "Clustering", "modalities": [ "text" ], "split": "test", "hf_subset": "default", "main_metric": "v_measure", "dataset": { "path": "mteb/twentynewsgroups-clustering", "revision": "6125ec4e24fa026cec8a478383ee943acfbd5449" }, "counts": { "population_rows": 59545, "official_encoded_rows": 2381, "official_encoding_fraction": 0.04, "bootstrap_clustering_trials": 10, "resampled_items_per_trial": 16384 }, "scores": { "nano": 0.5087260708286914, "small": 0.44840076500124326 }, "metrics": { "nano": { "v_measures": { "Level 0": [ 0.4966639569479829, 0.5210509442892017, 0.5333989625396617, 0.5094460445939109, 0.5208429535416115, 0.5012604404318527, 0.5210119744629907, 0.4617825059979706, 0.5226704101107219, 0.49913251537100806 ] }, "v_measure": 0.5087260708286914, "v_measure_std": 0.01936580527644773, "ami_scores": { "Level 0": [ 0.49471495769938123, 0.5192201304733433, 0.5316138913144499, 0.5075814211955173, 0.5190252719774525, 0.49937824878079495, 0.5191850550712727, 0.4597515065381579, 0.5208311166380218, 0.49721166070769024 ] }, "ami": 0.5068513260396083, "ami_std": 0.01943429917113934, "main_score": 0.5087260708286914, "hf_subset": "default", "languages": [ "eng-Latn" ], "mteb_version": "2.21.0" }, "small": { "v_measures": { "Level 0": [ 0.46653683369731075, 0.46464237680676646, 0.4425518472920712, 0.4288571327957099, 0.4336826969564481, 0.43371621267306154, 0.43092596596465876, 0.4536864467956253, 0.4741214282291591, 0.45528670880162087 ] }, "v_measure": 0.44840076500124326, "v_measure_std": 0.01575881885263055, "ami_scores": { "Level 0": [ 0.46451147281180905, 0.4625982506803001, 0.44043666823974115, 0.42670551677458946, 0.43154706196129733, 0.4315549515134722, 0.4287870602853061, 0.45163080092170965, 0.47212403002394715, 0.4532359410185627 ] }, "ami": 0.44631317542307347, "ami_std": 0.01581387159916619, "main_score": 0.44840076500124326, "hf_subset": "default", "languages": [ "eng-Latn" ], "mteb_version": "2.21.0" } }, "measurement_by_model": { "small": { "evaluated_native_fingerprint": "684fe0c48a4ddd3944167f6e5dc75e97ada24e5b120bd9a2921f00dc03112eeb", "original_checkpoint_revision": "fdf8e01b7b0f3a69ac1ac8e2a64dcb1ede177ba4", "raw_result_sha256": "62fe52980cb4445b9969d53135fa1663dd8dcd811743a711dde55f2a097ced19", "kind": "direct_frozen_encoder_evaluation" }, "nano": { "evaluated_native_fingerprint": "96d76f97478e114b816df3ab1b3c6947e4c5ef0babfb40ab865d7e3c838d34c4", "applicable_native_fingerprint": "474ad3d52f6ee7f818a3dbb6112e3d66c2423542d08228e7996a8cd600bd99a2", "kind": "exact_component_applicability" } }, "evaluated_native_fingerprint": "96d76f97478e114b816df3ab1b3c6947e4c5ef0babfb40ab865d7e3c838d34c4", "applicable_native_fingerprint": "474ad3d52f6ee7f818a3dbb6112e3d66c2423542d08228e7996a8cd600bd99a2", "measurement_kind": "exact_component_applicability" }, { "task": "MTOPDomainClassification", "type": "Classification", "modalities": [ "text" ], "split": "validation", "hf_subset": "en", "main_metric": "accuracy", "dataset": { "path": "mteb/MTOPDomainClassification", "revision": "a76d16fae880597b9c73047b50159220a441cb54" }, "counts": { "evaluation_rows": 2235, "available_train_rows": 15667, "official_probe_experiments": 10, "probe_samples_per_label": 8 }, "scores": { "nano": 0.9476510067114093, "small": 0.8543624161073826 }, "metrics": { "nano": { "scores_per_experiment": [ { "accuracy": 0.9521252796420582, "f1": 0.9509396968297774, "f1_weighted": 0.9519302419648948, "precision": 0.9493488952211522, "precision_weighted": 0.9527659658117037, "recall": 0.9535803496254757, "recall_weighted": 0.9521252796420582, "ap": null, "ap_weighted": null }, { "accuracy": 0.9458612975391499, "f1": 0.9439475266607382, "f1_weighted": 0.945767354216612, "precision": 0.9420005561621522, "precision_weighted": 0.9462895866502771, "recall": 0.946397940529448, "recall_weighted": 0.9458612975391499, "ap": null, "ap_weighted": null }, { "accuracy": 0.9284116331096197, "f1": 0.9284988704174789, "f1_weighted": 0.9270840634554431, "precision": 0.9271908775153325, "precision_weighted": 0.9274634710035635, "recall": 0.9317153602473662, "recall_weighted": 0.9284116331096197, "ap": null, "ap_weighted": null }, { "accuracy": 0.9525727069351231, "f1": 0.950987470976036, "f1_weighted": 0.9525185063610705, "precision": 0.9504292297253194, "precision_weighted": 0.9534501308715367, "recall": 0.9526312322737922, "recall_weighted": 0.9525727069351231, "ap": null, "ap_weighted": null }, { "accuracy": 0.9445190156599552, "f1": 0.943383064264026, "f1_weighted": 0.9442288513061988, "precision": 0.9424858291605631, "precision_weighted": 0.944823795147547, "recall": 0.945181115724642, "recall_weighted": 0.9445190156599552, "ap": null, "ap_weighted": null }, { "accuracy": 0.9525727069351231, "f1": 0.949882476921466, "f1_weighted": 0.9526817135384125, "precision": 0.9498620235113855, "precision_weighted": 0.9543297139412104, "recall": 0.9514687067629995, "recall_weighted": 0.9525727069351231, "ap": null, "ap_weighted": null }, { "accuracy": 0.9498881431767338, "f1": 0.9496336255013932, "f1_weighted": 0.9495703695110613, "precision": 0.9490015795096777, "precision_weighted": 0.9511221301014349, "recall": 0.9520470037532829, "recall_weighted": 0.9498881431767338, "ap": null, "ap_weighted": null }, { "accuracy": 0.9570469798657718, "f1": 0.9557194953102311, "f1_weighted": 0.9568992928300293, "precision": 0.9544888722394763, "precision_weighted": 0.9584505479016514, "recall": 0.9586898382241177, "recall_weighted": 0.9570469798657718, "ap": null, "ap_weighted": null }, { "accuracy": 0.9494407158836689, "f1": 0.9460535182757994, "f1_weighted": 0.9492959546761313, "precision": 0.9454088954645755, "precision_weighted": 0.9499145785798135, "recall": 0.9472953341577202, "recall_weighted": 0.9494407158836689, "ap": null, "ap_weighted": null }, { "accuracy": 0.9440715883668904, "f1": 0.9431192151526329, "f1_weighted": 0.9437397930273277, "precision": 0.9435095731735328, "precision_weighted": 0.9461253948734786, "recall": 0.9455297617432863, "recall_weighted": 0.9440715883668904, "ap": null, "ap_weighted": null } ], "accuracy": 0.9476510067114093, "f1": 0.9462164960309579, "f1_weighted": 0.9473716140887183, "precision": 0.9453726331683165, "precision_weighted": 0.9484735314882217, "recall": 0.948453664304213, "recall_weighted": 0.9476510067114093, "ap": null, "ap_weighted": null, "main_score": 0.9476510067114093, "hf_subset": "en", "languages": [ "eng-Latn" ], "mteb_version": "2.21.0" }, "small": { "scores_per_experiment": [ { "accuracy": 0.8250559284116331, "f1": 0.8322814729997016, "f1_weighted": 0.8240316200136756, "precision": 0.8318540374193987, "precision_weighted": 0.8346068877064119, "recall": 0.8430207439526936, "recall_weighted": 0.8250559284116331, "ap": null, "ap_weighted": null }, { "accuracy": 0.8559284116331096, "f1": 0.8552447478537527, "f1_weighted": 0.8547401242258992, "precision": 0.8515715568445064, "precision_weighted": 0.8581578601876096, "recall": 0.8640896175706192, "recall_weighted": 0.8559284116331096, "ap": null, "ap_weighted": null }, { "accuracy": 0.8304250559284116, "f1": 0.8317375249321166, "f1_weighted": 0.8274798893470849, "precision": 0.8268066926411431, "precision_weighted": 0.8285364690282124, "recall": 0.8409132018240374, "recall_weighted": 0.8304250559284116, "ap": null, "ap_weighted": null }, { "accuracy": 0.8648769574944072, "f1": 0.8678594298716482, "f1_weighted": 0.8632422395451538, "precision": 0.865870318819693, "precision_weighted": 0.8646460721917335, "recall": 0.8726494617104266, "recall_weighted": 0.8648769574944072, "ap": null, "ap_weighted": null }, { "accuracy": 0.8532438478747204, "f1": 0.8558930928177336, "f1_weighted": 0.8512711153646526, "precision": 0.8501961024805756, "precision_weighted": 0.8547737986953718, "recall": 0.8657455326712605, "recall_weighted": 0.8532438478747204, "ap": null, "ap_weighted": null }, { "accuracy": 0.8402684563758389, "f1": 0.8478991888671523, "f1_weighted": 0.8396067623115454, "precision": 0.8508550254092774, "precision_weighted": 0.8598285030005444, "recall": 0.862051388925805, "recall_weighted": 0.8402684563758389, "ap": null, "ap_weighted": null }, { "accuracy": 0.8747203579418344, "f1": 0.8781373847485636, "f1_weighted": 0.8731828255054377, "precision": 0.8729737218664414, "precision_weighted": 0.8758466535281065, "recall": 0.8873478774616341, "recall_weighted": 0.8747203579418344, "ap": null, "ap_weighted": null }, { "accuracy": 0.876062639821029, "f1": 0.8754580934866012, "f1_weighted": 0.8749795409187, "precision": 0.8730455332008387, "precision_weighted": 0.8787904116312295, "recall": 0.8820295010368394, "recall_weighted": 0.876062639821029, "ap": null, "ap_weighted": null }, { "accuracy": 0.858165548098434, "f1": 0.8610701024195923, "f1_weighted": 0.8573656937247496, "precision": 0.8561531713932808, "precision_weighted": 0.8662234353266065, "recall": 0.8738853045389997, "recall_weighted": 0.858165548098434, "ap": null, "ap_weighted": null }, { "accuracy": 0.8648769574944072, "f1": 0.8671414562971872, "f1_weighted": 0.8628546885742888, "precision": 0.8607074291115159, "precision_weighted": 0.8655691598347531, "recall": 0.8776421302043499, "recall_weighted": 0.8648769574944072, "ap": null, "ap_weighted": null } ], "accuracy": 0.8543624161073826, "f1": 0.8572722494294049, "f1_weighted": 0.8528754499531187, "precision": 0.8540033589186671, "precision_weighted": 0.8586979251130579, "recall": 0.8669374759896666, "recall_weighted": 0.8543624161073826, "ap": null, "ap_weighted": null, "main_score": 0.8543624161073826, "hf_subset": "en", "languages": [ "eng-Latn" ], "mteb_version": "2.21.0" } }, "measurement_by_model": { "small": { "evaluated_native_fingerprint": "684fe0c48a4ddd3944167f6e5dc75e97ada24e5b120bd9a2921f00dc03112eeb", "original_checkpoint_revision": "fdf8e01b7b0f3a69ac1ac8e2a64dcb1ede177ba4", "raw_result_sha256": "256702650b93bd559d9a2a9b70f32b996b51e166debc0c20d63ce24cc6f605f6", "kind": "direct_frozen_encoder_evaluation" }, "nano": { "evaluated_native_fingerprint": "96d76f97478e114b816df3ab1b3c6947e4c5ef0babfb40ab865d7e3c838d34c4", "applicable_native_fingerprint": "474ad3d52f6ee7f818a3dbb6112e3d66c2423542d08228e7996a8cd600bd99a2", "kind": "exact_component_applicability" } }, "evaluated_native_fingerprint": "96d76f97478e114b816df3ab1b3c6947e4c5ef0babfb40ab865d7e3c838d34c4", "applicable_native_fingerprint": "474ad3d52f6ee7f818a3dbb6112e3d66c2423542d08228e7996a8cd600bd99a2", "measurement_kind": "exact_component_applicability" }, { "task": "MTOPDomainClassification", "type": "Classification", "modalities": [ "text" ], "split": "test", "hf_subset": "en", "main_metric": "accuracy", "dataset": { "path": "mteb/MTOPDomainClassification", "revision": "a76d16fae880597b9c73047b50159220a441cb54" }, "counts": { "evaluation_rows": 4386, "available_train_rows": 15667, "official_probe_experiments": 10, "probe_samples_per_label": 8 }, "scores": { "nano": 0.9491792065663475, "small": 0.8456452348381213 }, "metrics": { "nano": { "scores_per_experiment": [ { "accuracy": 0.9507523939808481, "f1": 0.9490862282573499, "f1_weighted": 0.9508003256061616, "precision": 0.9476949859831407, "precision_weighted": 0.9520734016967529, "recall": 0.9517099502990664, "recall_weighted": 0.9507523939808481, "ap": null, "ap_weighted": null }, { "accuracy": 0.9473324213406292, "f1": 0.9425946320317333, "f1_weighted": 0.9473133266787116, "precision": 0.9392105665237355, "precision_weighted": 0.9479986101450497, "recall": 0.9468765008841218, "recall_weighted": 0.9473324213406292, "ap": null, "ap_weighted": null }, { "accuracy": 0.9306885544915641, "f1": 0.9261678157710709, "f1_weighted": 0.9296159823715927, "precision": 0.923024327342814, "precision_weighted": 0.9307965712480242, "recall": 0.932201810783149, "recall_weighted": 0.9306885544915641, "ap": null, "ap_weighted": null }, { "accuracy": 0.958048335613315, "f1": 0.9552869971485034, "f1_weighted": 0.9579502405954128, "precision": 0.954624514001619, "precision_weighted": 0.9582682331625527, "recall": 0.9564289859937923, "recall_weighted": 0.958048335613315, "ap": null, "ap_weighted": null }, { "accuracy": 0.9464204286365709, "f1": 0.94433525009326, "f1_weighted": 0.9462665825311201, "precision": 0.9422860464299823, "precision_weighted": 0.9467645700971977, "recall": 0.9469874285322217, "recall_weighted": 0.9464204286365709, "ap": null, "ap_weighted": null }, { "accuracy": 0.9505243958048336, "f1": 0.9447844972663006, "f1_weighted": 0.9507789369812125, "precision": 0.9429136617255864, "precision_weighted": 0.9524852717381878, "recall": 0.9483171314260196, "recall_weighted": 0.9505243958048336, "ap": null, "ap_weighted": null }, { "accuracy": 0.9509803921568627, "f1": 0.9480693879894481, "f1_weighted": 0.9510087641402198, "precision": 0.9447979434260977, "precision_weighted": 0.9522624035738166, "recall": 0.952376748039714, "recall_weighted": 0.9509803921568627, "ap": null, "ap_weighted": null }, { "accuracy": 0.9573643410852714, "f1": 0.9551153583299494, "f1_weighted": 0.9573535659550753, "precision": 0.952401865813031, "precision_weighted": 0.9583040343256545, "recall": 0.9586994420587002, "recall_weighted": 0.9573643410852714, "ap": null, "ap_weighted": null }, { "accuracy": 0.9548563611491108, "f1": 0.9495763118202308, "f1_weighted": 0.9549499907192655, "precision": 0.946851274794657, "precision_weighted": 0.9555638233419986, "recall": 0.9529512436069031, "recall_weighted": 0.9548563611491108, "ap": null, "ap_weighted": null }, { "accuracy": 0.9448244414044688, "f1": 0.9427993333346522, "f1_weighted": 0.9447713522464498, "precision": 0.9432072583992069, "precision_weighted": 0.9468986783847072, "recall": 0.9447479153319868, "recall_weighted": 0.9448244414044688, "ap": null, "ap_weighted": null } ], "accuracy": 0.9491792065663475, "f1": 0.9457815812042497, "f1_weighted": 0.9490809067825221, "precision": 0.9437012444439871, "precision_weighted": 0.9501415597713943, "recall": 0.9491297156955675, "recall_weighted": 0.9491792065663475, "ap": null, "ap_weighted": null, "main_score": 0.9491792065663475, "hf_subset": "en", "languages": [ "eng-Latn" ], "mteb_version": "2.21.0" }, "small": { "scores_per_experiment": [ { "accuracy": 0.8180574555403557, "f1": 0.8207347394289347, "f1_weighted": 0.8170857264694432, "precision": 0.820363266003151, "precision_weighted": 0.8301897328408723, "recall": 0.8338753807988755, "recall_weighted": 0.8180574555403557, "ap": null, "ap_weighted": null }, { "accuracy": 0.8449612403100775, "f1": 0.839516667111528, "f1_weighted": 0.8449410274873432, "precision": 0.8369796042511596, "precision_weighted": 0.8536873756717578, "recall": 0.8523590480814156, "recall_weighted": 0.8449612403100775, "ap": null, "ap_weighted": null }, { "accuracy": 0.8258093935248518, "f1": 0.8185934728273051, "f1_weighted": 0.8251774946806023, "precision": 0.8147370918656317, "precision_weighted": 0.8330066052370535, "recall": 0.8324642506433119, "recall_weighted": 0.8258093935248518, "ap": null, "ap_weighted": null }, { "accuracy": 0.8497492020063839, "f1": 0.8486953297726213, "f1_weighted": 0.848088172519004, "precision": 0.8467111465364261, "precision_weighted": 0.8501008589343039, "recall": 0.853578648077031, "recall_weighted": 0.8497492020063839, "ap": null, "ap_weighted": null }, { "accuracy": 0.8481532147742818, "f1": 0.8467468733444872, "f1_weighted": 0.8467448330993773, "precision": 0.8393119909268087, "precision_weighted": 0.8506182695179204, "recall": 0.858498685671118, "recall_weighted": 0.8481532147742818, "ap": null, "ap_weighted": null }, { "accuracy": 0.8326493388052896, "f1": 0.8352387297727578, "f1_weighted": 0.8347572800634613, "precision": 0.8420129591399498, "precision_weighted": 0.8638673888380122, "recall": 0.8534348314007274, "recall_weighted": 0.8326493388052896, "ap": null, "ap_weighted": null }, { "accuracy": 0.8682170542635659, "f1": 0.8648438659784241, "f1_weighted": 0.8680876089933638, "precision": 0.8585183480506728, "precision_weighted": 0.876371854401442, "recall": 0.8802432046324339, "recall_weighted": 0.8682170542635659, "ap": null, "ap_weighted": null }, { "accuracy": 0.8518011855905153, "f1": 0.8484079910361157, "f1_weighted": 0.8509906563529788, "precision": 0.8443896101449812, "precision_weighted": 0.860550178646213, "recall": 0.8607698617029648, "recall_weighted": 0.8518011855905153, "ap": null, "ap_weighted": null }, { "accuracy": 0.8556771545827634, "f1": 0.8515433915603542, "f1_weighted": 0.856278856298033, "precision": 0.84610847827307, "precision_weighted": 0.8661492710958929, "recall": 0.8658189317867632, "recall_weighted": 0.8556771545827634, "ap": null, "ap_weighted": null }, { "accuracy": 0.8613771089831281, "f1": 0.8586787981057005, "f1_weighted": 0.8603956735839339, "precision": 0.8518135650230533, "precision_weighted": 0.8659927155990171, "recall": 0.8717255305513517, "recall_weighted": 0.8613771089831281, "ap": null, "ap_weighted": null } ], "accuracy": 0.8456452348381213, "f1": 0.8432999858938229, "f1_weighted": 0.8452547329547541, "precision": 0.8400946060214904, "precision_weighted": 0.8550534250782486, "recall": 0.8562768373345993, "recall_weighted": 0.8456452348381213, "ap": null, "ap_weighted": null, "main_score": 0.8456452348381213, "hf_subset": "en", "languages": [ "eng-Latn" ], "mteb_version": "2.21.0" } }, "measurement_by_model": { "small": { "evaluated_native_fingerprint": "684fe0c48a4ddd3944167f6e5dc75e97ada24e5b120bd9a2921f00dc03112eeb", "original_checkpoint_revision": "fdf8e01b7b0f3a69ac1ac8e2a64dcb1ede177ba4", "raw_result_sha256": "256702650b93bd559d9a2a9b70f32b996b51e166debc0c20d63ce24cc6f605f6", "kind": "direct_frozen_encoder_evaluation" }, "nano": { "evaluated_native_fingerprint": "96d76f97478e114b816df3ab1b3c6947e4c5ef0babfb40ab865d7e3c838d34c4", "applicable_native_fingerprint": "474ad3d52f6ee7f818a3dbb6112e3d66c2423542d08228e7996a8cd600bd99a2", "kind": "exact_component_applicability" } }, "evaluated_native_fingerprint": "96d76f97478e114b816df3ab1b3c6947e4c5ef0babfb40ab865d7e3c838d34c4", "applicable_native_fingerprint": "474ad3d52f6ee7f818a3dbb6112e3d66c2423542d08228e7996a8cd600bd99a2", "measurement_kind": "exact_component_applicability" }, { "task": "OxfordPets", "type": "ImageClassification", "modalities": [ "image" ], "split": "test", "hf_subset": "default", "main_metric": "accuracy", "dataset": { "path": "mteb/OxfordPets", "revision": "557b480fae8d69247be74d9503b378a09425096f" }, "counts": { "evaluation_rows": 3669, "available_train_rows": 3680, "official_probe_experiments": 5, "probe_samples_per_label": 16 }, "scores": { "nano": 0.9126737530662306, "small": 0.4184791496320523 }, "metrics": { "nano": { "scores_per_experiment": [ { "accuracy": 0.9062414826928318, "f1": 0.9045301316021502, "f1_weighted": 0.9050055409371512, "precision": 0.9099213291951324, "precision_weighted": 0.9109544817876485, "recall": 0.9064295837897134, "recall_weighted": 0.9062414826928318, "ap": null, "ap_weighted": null }, { "accuracy": 0.914418097574271, "f1": 0.9126724638115935, "f1_weighted": 0.9132186480967779, "precision": 0.9192904646518116, "precision_weighted": 0.9203471940294945, "recall": 0.9144593228002407, "recall_weighted": 0.914418097574271, "ap": null, "ap_weighted": null }, { "accuracy": 0.9187789588443718, "f1": 0.9177765846692281, "f1_weighted": 0.9184111765200857, "precision": 0.9220755461580387, "precision_weighted": 0.9229278273528245, "recall": 0.9183739103921627, "recall_weighted": 0.9187789588443718, "ap": null, "ap_weighted": null }, { "accuracy": 0.9174161896974653, "f1": 0.9147622354932347, "f1_weighted": 0.9154663340384108, "precision": 0.9209055798643112, "precision_weighted": 0.9221124471295344, "recall": 0.9173606715378323, "recall_weighted": 0.9174161896974653, "ap": null, "ap_weighted": null }, { "accuracy": 0.9065140365222132, "f1": 0.9051669100036687, "f1_weighted": 0.9057728210054364, "precision": 0.9098426973534027, "precision_weighted": 0.9105143284060627, "recall": 0.9059560197061645, "recall_weighted": 0.9065140365222132, "ap": null, "ap_weighted": null } ], "accuracy": 0.9126737530662306, "f1": 0.910981665115975, "f1_weighted": 0.9115749041195725, "precision": 0.9164071234445392, "precision_weighted": 0.9173712557411129, "recall": 0.9125159016452227, "recall_weighted": 0.9126737530662306, "ap": null, "ap_weighted": null, "main_score": 0.9126737530662306, "hf_subset": "default", "languages": [ "eng-Latn" ], "mteb_version": "2.21.0" }, "small": { "scores_per_experiment": [ { "accuracy": 0.4148269283183429, "f1": 0.40253654878032713, "f1_weighted": 0.4024461757957877, "precision": 0.4116431114014033, "precision_weighted": 0.4119318458992588, "recall": 0.4153030284766479, "recall_weighted": 0.4148269283183429, "ap": null, "ap_weighted": null }, { "accuracy": 0.4265467429817389, "f1": 0.4174251045251405, "f1_weighted": 0.4176529394767131, "precision": 0.43921297726376096, "precision_weighted": 0.4396338060561412, "recall": 0.42629825783951975, "recall_weighted": 0.4265467429817389, "ap": null, "ap_weighted": null }, { "accuracy": 0.4088307440719542, "f1": 0.3954869417563968, "f1_weighted": 0.39542431005304346, "precision": 0.40981373804908927, "precision_weighted": 0.40980533913060346, "recall": 0.40880378402552453, "recall_weighted": 0.4088307440719542, "ap": null, "ap_weighted": null }, { "accuracy": 0.4377214499863723, "f1": 0.4251494451023654, "f1_weighted": 0.4253356439185362, "precision": 0.438981154791822, "precision_weighted": 0.439719429760714, "recall": 0.4381658372912861, "recall_weighted": 0.4377214499863723, "ap": null, "ap_weighted": null }, { "accuracy": 0.40446988280185336, "f1": 0.3806382057679881, "f1_weighted": 0.380430443085735, "precision": 0.4088810121814978, "precision_weighted": 0.40903302714302686, "recall": 0.4046956272962792, "recall_weighted": 0.40446988280185336, "ap": null, "ap_weighted": null } ], "accuracy": 0.4184791496320523, "f1": 0.4042472491864436, "f1_weighted": 0.40425790246596305, "precision": 0.42170639873751464, "precision_weighted": 0.4220246895979488, "recall": 0.4186533069858515, "recall_weighted": 0.4184791496320523, "ap": null, "ap_weighted": null, "main_score": 0.4184791496320523, "hf_subset": "default", "languages": [ "eng-Latn" ], "mteb_version": "2.21.0" } }, "measurement_by_model": { "small": { "evaluated_native_fingerprint": "684fe0c48a4ddd3944167f6e5dc75e97ada24e5b120bd9a2921f00dc03112eeb", "original_checkpoint_revision": "fdf8e01b7b0f3a69ac1ac8e2a64dcb1ede177ba4", "raw_result_sha256": "ef1f4928b9ae7d9da25a98b82a28fda3e098db9504ad9372cb5305ad32421a06", "kind": "direct_frozen_encoder_evaluation" }, "nano": { "evaluated_native_fingerprint": "96d76f97478e114b816df3ab1b3c6947e4c5ef0babfb40ab865d7e3c838d34c4", "applicable_native_fingerprint": "474ad3d52f6ee7f818a3dbb6112e3d66c2423542d08228e7996a8cd600bd99a2", "kind": "exact_component_applicability" } }, "evaluated_native_fingerprint": "96d76f97478e114b816df3ab1b3c6947e4c5ef0babfb40ab865d7e3c838d34c4", "applicable_native_fingerprint": "474ad3d52f6ee7f818a3dbb6112e3d66c2423542d08228e7996a8cd600bd99a2", "measurement_kind": "exact_component_applicability" }, { "task": "TinyImageNetClustering", "type": "ImageClustering", "modalities": [ "image" ], "split": "valid", "hf_subset": "default", "main_metric": "nmi", "dataset": { "path": "mteb/tiny-imagenet", "revision": "e04e259f646f750bd0bdb2a56c2a4577d89bee3d" }, "counts": { "evaluation_rows": 10000 }, "scores": { "nano": 0.6169193395626786, "small": 0.4628490605491802 }, "metrics": { "nano": { "v_measure": 0.6169193395626787, "nmi": 0.6169193395626786, "ari": 0.21497891543638262, "cluster_accuracy": 0.3638, "main_score": 0.6169193395626786, "hf_subset": "default", "languages": [ "eng-Latn" ], "mteb_version": "2.21.0" }, "small": { "v_measure": 0.4628490605491802, "nmi": 0.4628490605491802, "ari": 0.08498222977932614, "cluster_accuracy": 0.2035, "main_score": 0.4628490605491802, "hf_subset": "default", "languages": [ "eng-Latn" ], "mteb_version": "2.21.0" } }, "measurement_by_model": { "small": { "evaluated_native_fingerprint": "684fe0c48a4ddd3944167f6e5dc75e97ada24e5b120bd9a2921f00dc03112eeb", "original_checkpoint_revision": "fdf8e01b7b0f3a69ac1ac8e2a64dcb1ede177ba4", "raw_result_sha256": "eb344e20e7d498aadbbd67a57015cc2220cba42b9550f5822b5a72d0d6cf1514", "kind": "direct_frozen_encoder_evaluation" }, "nano": { "evaluated_native_fingerprint": "96d76f97478e114b816df3ab1b3c6947e4c5ef0babfb40ab865d7e3c838d34c4", "applicable_native_fingerprint": "474ad3d52f6ee7f818a3dbb6112e3d66c2423542d08228e7996a8cd600bd99a2", "kind": "exact_component_applicability" } }, "evaluated_native_fingerprint": "96d76f97478e114b816df3ab1b3c6947e4c5ef0babfb40ab865d7e3c838d34c4", "applicable_native_fingerprint": "474ad3d52f6ee7f818a3dbb6112e3d66c2423542d08228e7996a8cd600bd99a2", "measurement_kind": "exact_component_applicability" }, { "task": "OxfordPetsZeroShot", "type": "ZeroShotClassification", "modalities": [ "text", "image" ], "split": "test", "hf_subset": "default", "main_metric": "accuracy", "dataset": { "path": "mteb/OxfordPets", "revision": "557b480fae8d69247be74d9503b378a09425096f" }, "counts": { "evaluation_rows": 3669, "fixed_text_labels": 37 }, "scores": { "nano": 0.08013082583810302, "small": 0.1640774052875443 }, "metrics": { "nano": { "accuracy": 0.08013082583810302, "f1": 0.0611345635215895, "f1_weighted": 0.06128470828202288, "precision": 0.1359645526412158, "precision_weighted": 0.13671480699577512, "recall": 0.0798783003173134, "recall_weighted": 0.08013082583810302, "ap": null, "ap_weighted": null, "main_score": 0.08013082583810302, "hf_subset": "default", "languages": [ "eng-Latn" ], "mteb_version": "2.21.0" }, "small": { "accuracy": 0.1640774052875443, "f1": 0.12868294474251654, "f1_weighted": 0.12888736955731406, "precision": 0.15179208674748698, "precision_weighted": 0.15241279079348374, "recall": 0.16458376660623852, "recall_weighted": 0.1640774052875443, "ap": null, "ap_weighted": null, "main_score": 0.1640774052875443, "hf_subset": "default", "languages": [ "eng-Latn" ], "mteb_version": "2.21.0" } }, "measurement_by_model": { "small": { "evaluated_native_fingerprint": "684fe0c48a4ddd3944167f6e5dc75e97ada24e5b120bd9a2921f00dc03112eeb", "original_checkpoint_revision": "fdf8e01b7b0f3a69ac1ac8e2a64dcb1ede177ba4", "raw_result_sha256": "e52e42c4716e62710296df98a90b3a0b986875685748d043cfd7de913e5a6c42", "kind": "direct_frozen_encoder_evaluation" }, "nano": { "evaluated_native_fingerprint": "96d76f97478e114b816df3ab1b3c6947e4c5ef0babfb40ab865d7e3c838d34c4", "applicable_native_fingerprint": "474ad3d52f6ee7f818a3dbb6112e3d66c2423542d08228e7996a8cd600bd99a2", "kind": "exact_component_applicability" } }, "evaluated_native_fingerprint": "96d76f97478e114b816df3ab1b3c6947e4c5ef0babfb40ab865d7e3c838d34c4", "applicable_native_fingerprint": "474ad3d52f6ee7f818a3dbb6112e3d66c2423542d08228e7996a8cd600bd99a2", "measurement_kind": "exact_component_applicability" }, { "task": "CREMA_D", "type": "AudioClassification", "modalities": [ "audio" ], "split": "train", "hf_subset": "default", "main_metric": "accuracy", "dataset": { "path": "mteb/crema-d", "revision": "a9d0821955c418752248b8310438854e56a1cf34" }, "counts": { "evaluation_rows": 7442, "official_cross_validation_folds": 5, "probe_samples_per_label": 8 }, "scores": { "nano": 0.32048146984697823, "small": 0.2995196855795547 }, "metrics": { "nano": { "accuracy": 0.32048146984697823, "ap": null, "ap_weighted": null, "f1": 0.303943978298811, "f1_weighted": 0.30451603344534794, "hf_subset": "default", "languages": [ "eng-Latn" ], "main_score": 0.32048146984697823, "mteb_version": "2.21.0", "precision": 0.31510472165152337, "precision_weighted": 0.3169398085592857, "recall": 0.3215883809208585, "recall_weighted": 0.32048146984697823, "scores_per_experiment": [ { "accuracy": 0.30288784419073206, "ap": null, "ap_weighted": null, "f1": 0.27969233326985626, "f1_weighted": 0.2799132430776581, "precision": 0.29992207114753305, "precision_weighted": 0.29961025331880187, "recall": 0.3024507805686864, "recall_weighted": 0.30288784419073206 }, { "accuracy": 0.3149764942914708, "ap": null, "ap_weighted": null, "f1": 0.30808145721796526, "f1_weighted": 0.31085435981340914, "precision": 0.336772371610985, "precision_weighted": 0.3411130620509704, "recall": 0.3160853641774642, "recall_weighted": 0.3149764942914708 }, { "accuracy": 0.31586021505376344, "ap": null, "ap_weighted": null, "f1": 0.2931797954478092, "f1_weighted": 0.2955391094563843, "precision": 0.2916058030686394, "precision_weighted": 0.29407622403946154, "recall": 0.312973406205352, "recall_weighted": 0.31586021505376344 }, { "accuracy": 0.32123655913978494, "ap": null, "ap_weighted": null, "f1": 0.3066232673451424, "f1_weighted": 0.3032561325852315, "precision": 0.3141358238179796, "precision_weighted": 0.31520429243540943, "recall": 0.32961458988542325, "recall_weighted": 0.32123655913978494 }, { "accuracy": 0.3474462365591398, "ap": null, "ap_weighted": null, "f1": 0.3321430382132817, "f1_weighted": 0.3330173222940568, "precision": 0.3330875386124797, "precision_weighted": 0.3346952109517855, "recall": 0.3468177637673669, "recall_weighted": 0.3474462365591398 } ] }, "small": { "scores_per_experiment": [ { "accuracy": 0.2699798522498321, "f1": 0.24239504058809014, "f1_weighted": 0.2423525571760308, "precision": 0.25470036430672977, "precision_weighted": 0.2542205264446883, "recall": 0.26975123145374097, "recall_weighted": 0.2699798522498321, "ap": null, "ap_weighted": null }, { "accuracy": 0.3035594358629953, "f1": 0.29613784795561465, "f1_weighted": 0.29906255840921786, "precision": 0.30407267120079884, "precision_weighted": 0.30800706496852154, "recall": 0.3024788327196796, "recall_weighted": 0.3035594358629953, "ap": null, "ap_weighted": null }, { "accuracy": 0.28225806451612906, "f1": 0.2593664352832718, "f1_weighted": 0.2636113556497887, "precision": 0.2609748445531306, "precision_weighted": 0.26468416511118426, "recall": 0.2775030429767242, "recall_weighted": 0.28225806451612906, "ap": null, "ap_weighted": null }, { "accuracy": 0.3111559139784946, "f1": 0.29454980492169786, "f1_weighted": 0.29125309340152467, "precision": 0.3224943453400395, "precision_weighted": 0.3242765511011472, "recall": 0.31997721819075986, "recall_weighted": 0.3111559139784946, "ap": null, "ap_weighted": null }, { "accuracy": 0.33064516129032256, "f1": 0.3185711713472314, "f1_weighted": 0.3198970407995879, "precision": 0.32320745672952395, "precision_weighted": 0.32608239411677886, "recall": 0.3309376196524952, "recall_weighted": 0.33064516129032256, "ap": null, "ap_weighted": null } ], "accuracy": 0.2995196855795547, "f1": 0.2822040600191812, "f1_weighted": 0.28323532108723004, "precision": 0.29308993642604453, "precision_weighted": 0.295454140348464, "recall": 0.30012958899868, "recall_weighted": 0.2995196855795547, "ap": null, "ap_weighted": null, "main_score": 0.2995196855795547, "hf_subset": "default", "languages": [ "eng-Latn" ], "mteb_version": "2.21.0" } }, "measurement_by_model": { "small": { "evaluated_native_fingerprint": "684fe0c48a4ddd3944167f6e5dc75e97ada24e5b120bd9a2921f00dc03112eeb", "original_checkpoint_revision": "fdf8e01b7b0f3a69ac1ac8e2a64dcb1ede177ba4", "raw_result_sha256": "d79b8d114cb86c460dad13d564c2705a41df5119f83a0f1f27d452f36b64af47", "kind": "direct_frozen_encoder_evaluation" }, "nano": { "evaluated_native_fingerprint": "474ad3d52f6ee7f818a3dbb6112e3d66c2423542d08228e7996a8cd600bd99a2", "applicable_native_fingerprint": "474ad3d52f6ee7f818a3dbb6112e3d66c2423542d08228e7996a8cd600bd99a2", "kind": "fresh_native", "raw_result_sha256": "62439b0ed4b3a8c26b1a8f122880a7bd3b3d86566841333a9060a4bbd0954baf" } }, "evaluated_native_fingerprint": "474ad3d52f6ee7f818a3dbb6112e3d66c2423542d08228e7996a8cd600bd99a2", "applicable_native_fingerprint": "474ad3d52f6ee7f818a3dbb6112e3d66c2423542d08228e7996a8cd600bd99a2", "measurement_kind": "fresh_native", "raw_result_sha256": "62439b0ed4b3a8c26b1a8f122880a7bd3b3d86566841333a9060a4bbd0954baf" }, { "task": "CREMA_DClustering", "type": "AudioClustering", "modalities": [ "audio" ], "split": "train", "hf_subset": "default", "main_metric": "v_measure", "dataset": { "path": "mteb/crema-d", "revision": "a9d0821955c418752248b8310438854e56a1cf34" }, "counts": { "official_stratified_subsample_rows": 2048, "bootstrap_clustering_trials": 10, "resampled_items_per_trial": 16384 }, "scores": { "nano": 0.05214346580804754, "small": 0.024058408888362488 }, "metrics": { "nano": { "ami": 0.05172990441350066, "ami_scores": { "Level 0": [ 0.07691371568295576, 0.05921735688360533, 0.040011204150084696, 0.04041243062073308, 0.042015892825421196, 0.06432477056741849, 0.06285137443486444, 0.05640769402749227, 0.058836854203017874, 0.01630775073941345 ] }, "ami_std": 0.016307969325946296, "hf_subset": "default", "languages": [ "eng-Latn" ], "main_score": 0.05214346580804754, "mteb_version": "2.21.0", "v_measure": 0.05214346580804754, "v_measure_std": 0.016298021302929667, "v_measures": { "Level 0": [ 0.07731089214293434, 0.05962833919011462, 0.040438217966279694, 0.04084038764100412, 0.04243023115915348, 0.06473220137035035, 0.06325438263275382, 0.05681796105920958, 0.059243669688975126, 0.016738375229700227 ] } }, "small": { "v_measures": { "Level 0": [ 0.041011040847839235, 0.03386543153714212, 0.029099626845588235, 0.013685755785810546, 0.02873132169365058, 0.01881126494637877, 0.011416660820064256, 0.02673877787517492, 0.00569343536524053, 0.031530773166735686 ] }, "v_measure": 0.024058408888362488, "v_measure_std": 0.010611552764041994, "ami_scores": { "Level 0": [ 0.04059499123714819, 0.03344615727359256, 0.02867541136577102, 0.01325228658253946, 0.028312443838566275, 0.01838242676310465, 0.01099044807520909, 0.026321334415061327, 0.005259713019363485, 0.031113076527253092 ] }, "ami": 0.023634828909760913, "ami_std": 0.010617195895208265, "main_score": 0.024058408888362488, "hf_subset": "default", "languages": [ "eng-Latn" ], "mteb_version": "2.21.0" } }, "measurement_by_model": { "small": { "evaluated_native_fingerprint": "684fe0c48a4ddd3944167f6e5dc75e97ada24e5b120bd9a2921f00dc03112eeb", "original_checkpoint_revision": "fdf8e01b7b0f3a69ac1ac8e2a64dcb1ede177ba4", "raw_result_sha256": "b23d4c552b9fa242923fc0c21a898ee63eb32850d9d91e0682b2cb5e8bea0e76", "kind": "direct_frozen_encoder_evaluation" }, "nano": { "evaluated_native_fingerprint": "474ad3d52f6ee7f818a3dbb6112e3d66c2423542d08228e7996a8cd600bd99a2", "applicable_native_fingerprint": "474ad3d52f6ee7f818a3dbb6112e3d66c2423542d08228e7996a8cd600bd99a2", "kind": "fresh_native", "raw_result_sha256": "89db580a8e7039474d3bf150da208adbd98eabce358dad64a85f6c2027d4d62e" } }, "evaluated_native_fingerprint": "474ad3d52f6ee7f818a3dbb6112e3d66c2423542d08228e7996a8cd600bd99a2", "applicable_native_fingerprint": "474ad3d52f6ee7f818a3dbb6112e3d66c2423542d08228e7996a8cd600bd99a2", "measurement_kind": "fresh_native", "raw_result_sha256": "89db580a8e7039474d3bf150da208adbd98eabce358dad64a85f6c2027d4d62e" }, { "task": "SpeechCommandsZeroshotv0.02", "type": "AudioZeroshotClassification", "modalities": [ "audio", "text" ], "split": "test", "hf_subset": "default", "main_metric": "accuracy", "dataset": { "path": "mteb/SpeechCommandsZeroshotv0.02", "revision": "d5491b288ab76356a5f3cbc634f1dc38e23da2b8" }, "counts": { "evaluation_rows": 4074, "fixed_text_labels": 10 }, "scores": { "nano": 0.15709376534118802, "small": 0.05522827687776141 }, "metrics": { "nano": { "accuracy": 0.15709376534118802, "ap": null, "ap_weighted": null, "f1": 0.08449976339531617, "f1_weighted": 0.0833837827950463, "hf_subset": "default", "languages": [ "eng-Latn" ], "main_score": 0.15709376534118802, "mteb_version": "2.21.0", "precision": 0.07985904570619766, "precision_weighted": 0.078775844073094, "recall": 0.15917885320870395, "recall_weighted": 0.15709376534118802 }, "small": { "accuracy": 0.05522827687776141, "f1": 0.037875265984290724, "f1_weighted": 0.03803483258988871, "precision": 0.14410977979833245, "precision_weighted": 0.143575354561912, "recall": 0.05521141290348569, "recall_weighted": 0.05522827687776141, "ap": null, "ap_weighted": null, "main_score": 0.05522827687776141, "hf_subset": "default", "languages": [ "eng-Latn" ], "mteb_version": "2.21.0" } }, "measurement_by_model": { "small": { "evaluated_native_fingerprint": "684fe0c48a4ddd3944167f6e5dc75e97ada24e5b120bd9a2921f00dc03112eeb", "original_checkpoint_revision": "fdf8e01b7b0f3a69ac1ac8e2a64dcb1ede177ba4", "raw_result_sha256": "ca19c03bbe2fdcf3c2e1282f5e826af702c152287fde75d67b28ef877fae5335", "kind": "direct_frozen_encoder_evaluation" }, "nano": { "evaluated_native_fingerprint": "474ad3d52f6ee7f818a3dbb6112e3d66c2423542d08228e7996a8cd600bd99a2", "applicable_native_fingerprint": "474ad3d52f6ee7f818a3dbb6112e3d66c2423542d08228e7996a8cd600bd99a2", "kind": "fresh_native", "raw_result_sha256": "c5b151588a51abbf4ca2e9bc88ec91d235b0971d8035f02e97b7c58910f66538" } }, "evaluated_native_fingerprint": "474ad3d52f6ee7f818a3dbb6112e3d66c2423542d08228e7996a8cd600bd99a2", "applicable_native_fingerprint": "474ad3d52f6ee7f818a3dbb6112e3d66c2423542d08228e7996a8cd600bd99a2", "measurement_kind": "fresh_native", "raw_result_sha256": "c5b151588a51abbf4ca2e9bc88ec91d235b0971d8035f02e97b7c58910f66538" } ], "comparison": { "baseline_model": "vllm-sr/multi-modal-embed-small", "baseline_revision": "fdf8e01b7b0f3a69ac1ac8e2a64dcb1ede177ba4", "baseline_native_artifact_sha256": "684fe0c48a4ddd3944167f6e5dc75e97ada24e5b120bd9a2921f00dc03112eeb", "baseline_standard_status": "evaluated_all_11_selected_tasks", "card_bold": "Only unrounded gains against this original baseline are bold; missing baseline scores imply no comparison." }, "observation_scope": "Source-specific counters only: eight unchanged text/image tasks retain their actual earlier measurement; three audio tasks are fresh. Old timing is not a measurement of the new audio architecture.", "encoding_observations": { "SICK-R": { "nano": { "before": { "text_rows": 0, "text_rows_truncated": 0, "max_original_text_tokens": 0, "image_rows": 0, "audio_rows": 0 }, "after": { "text_rows": 19854, "text_rows_truncated": 0, "max_original_text_tokens": 38, "image_rows": 0, "audio_rows": 0 }, "elapsed_seconds": 10.807591969991336, "peak_allocated_gpu_bytes": 679632896, "peak_reserved_gpu_bytes": 719323136 } }, "STSBenchmark": { "nano": { "before": { "text_rows": 19854, "text_rows_truncated": 0, "max_original_text_tokens": 38, "image_rows": 0, "audio_rows": 0 }, "after": { "text_rows": 22612, "text_rows_truncated": 0, "max_original_text_tokens": 46, "image_rows": 0, "audio_rows": 0 }, "elapsed_seconds": 1.294741225981852, "peak_allocated_gpu_bytes": 681261056, "peak_reserved_gpu_bytes": 723517440 } }, "ArguAna": { "nano": { "before": { "text_rows": 22612, "text_rows_truncated": 0, "max_original_text_tokens": 46, "image_rows": 0, "audio_rows": 0 }, "after": { "text_rows": 32692, "text_rows_truncated": 222, "max_original_text_tokens": 1401, "image_rows": 0, "audio_rows": 0 }, "elapsed_seconds": 15.392282173008425, "peak_allocated_gpu_bytes": 809891328, "peak_reserved_gpu_bytes": 887095296 } }, "TwentyNewsgroupsClustering.v2": { "nano": { "before": { "text_rows": 32692, "text_rows_truncated": 222, "max_original_text_tokens": 1401, "image_rows": 0, "audio_rows": 0 }, "after": { "text_rows": 35073, "text_rows_truncated": 222, "max_original_text_tokens": 1401, "image_rows": 0, "audio_rows": 0 }, "elapsed_seconds": 48.54122133200872, "peak_allocated_gpu_bytes": 684661248, "peak_reserved_gpu_bytes": 727711744 } }, "MTOPDomainClassification": { "nano": { "before": { "text_rows": 35073, "text_rows_truncated": 222, "max_original_text_tokens": 1401, "image_rows": 0, "audio_rows": 0 }, "after": { "text_rows": 43422, "text_rows_truncated": 222, "max_original_text_tokens": 1401, "image_rows": 0, "audio_rows": 0 }, "elapsed_seconds": 25.26324965900858, "peak_allocated_gpu_bytes": 679308288, "peak_reserved_gpu_bytes": 719323136 } }, "OxfordPets": { "nano": { "before": { "text_rows": 43422, "text_rows_truncated": 222, "max_original_text_tokens": 1401, "image_rows": 0, "audio_rows": 0 }, "after": { "text_rows": 43422, "text_rows_truncated": 222, "max_original_text_tokens": 1401, "image_rows": 5816, "audio_rows": 0 }, "elapsed_seconds": 54.48569396699895, "peak_allocated_gpu_bytes": 996472320, "peak_reserved_gpu_bytes": 1207959552 } }, "TinyImageNetClustering": { "nano": { "before": { "text_rows": 43422, "text_rows_truncated": 222, "max_original_text_tokens": 1401, "image_rows": 5816, "audio_rows": 0 }, "after": { "text_rows": 43422, "text_rows_truncated": 222, "max_original_text_tokens": 1401, "image_rows": 15816, "audio_rows": 0 }, "elapsed_seconds": 72.07803665398387, "peak_allocated_gpu_bytes": 996472320, "peak_reserved_gpu_bytes": 1117782016 } }, "OxfordPetsZeroShot": { "nano": { "before": { "text_rows": 43422, "text_rows_truncated": 222, "max_original_text_tokens": 1401, "image_rows": 15816, "audio_rows": 0 }, "after": { "text_rows": 43459, "text_rows_truncated": 222, "max_original_text_tokens": 1401, "image_rows": 19485, "audio_rows": 0 }, "elapsed_seconds": 32.662016286019934, "peak_allocated_gpu_bytes": 996472320, "peak_reserved_gpu_bytes": 1119879168 } }, "CREMA_D": { "evaluated_native_fingerprint": "474ad3d52f6ee7f818a3dbb6112e3d66c2423542d08228e7996a8cd600bd99a2", "observations": { "cumulative_rows": { "audio_rows": 7442, "image_rows": 0, "max_original_text_tokens": 0, "text_rows": 0, "text_rows_truncated": 0 }, "peak_allocated_bytes": 966564864, "peak_reserved_bytes": 1493172224, "seconds": 228.799308784015 } }, "CREMA_DClustering": { "evaluated_native_fingerprint": "474ad3d52f6ee7f818a3dbb6112e3d66c2423542d08228e7996a8cd600bd99a2", "observations": { "cumulative_rows": { "audio_rows": 9490, "image_rows": 0, "max_original_text_tokens": 0, "text_rows": 0, "text_rows_truncated": 0 }, "peak_allocated_bytes": 966577152, "peak_reserved_bytes": 1321205760, "seconds": 102.58190695600933 } }, "SpeechCommandsZeroshotv0.02": { "evaluated_native_fingerprint": "474ad3d52f6ee7f818a3dbb6112e3d66c2423542d08228e7996a8cd600bd99a2", "observations": { "cumulative_rows": { "audio_rows": 13564, "image_rows": 0, "max_original_text_tokens": 3, "text_rows": 10, "text_rows_truncated": 0 }, "peak_allocated_bytes": 966577152, "peak_reserved_bytes": 1329594368, "seconds": 122.96637481401558 } } } }, "capabilities": { "max_text_tokens": 512, "tokens_include_special_tokens": true, "public_text_overflow": "Raises ValueError; shorten or explicitly chunk longer inputs.", "embedding_dimensions": 384, "parameters": 163771288, "audio_sample_rates": [ 16000, 44100, 48000 ], "audio_max_seconds": 30, "audio_input": "Original PCM, mono or channels-first; each rate is independently resampled for the two audio branches." }, "efficiency": { "status": "no_current_latency_measurement", "parameters": 163771288, "parameter_increase_percent_vs_previous_release": 20.968150046422096, "embedding_dimensions": 384, "dense_fp32_index_bytes_per_vector": 1536, "weights_safetensors_bytes": 655583800, "weights_safetensors_sha256": "d5aa7f00e217bd4fd5314140a3603b9a873636320c8d09732a14f79f4cb0667e", "scope": "The CLAP branch increases parameters and audio computation. No isolated latency or peak-memory improvement is claimed; previous measurements belong to their recorded artifacts.", "previous_release_measurement": { "status": "not_measured_for_this_release", "parameters": 135383808, "embedding_dimensions": 384, "dense_fp32_index_bytes_per_vector": 1536, "weights_safetensors_bytes": 541598024, "scope": "No new latency or peak-memory measurement is claimed. Vector storage excludes index structures and metadata.", "historical_measurement": { "attention": "native SDPA", "batch_size": 8, "embedding_dtype": "float32", "hardware": "AMD gfx942 GPU", "latency_includes": "Native public encode_text: tokenization, host-to-device transfer and encoding; excludes model loading.", "measured_batches": 20, "measurements": { "128": { "batch_size": 8, "p50_ms": 4.283615489839576, "p95_ms": 4.314010989037342, "peak_allocated_bytes": 653202944, "peak_reserved_bytes": 673185792, "tokens_each_including_special": 128 }, "512": { "batch_size": 8, "p50_ms": 12.445501997717656, "p95_ms": 12.603181986196432, "peak_allocated_bytes": 878682112, "peak_reserved_bytes": 970981376, "tokens_each_including_special": 512 } }, "memory_scope": "Process CUDA allocator peak with the complete multimodal model loaded.", "model_revision": "70c3412bb0ddfb19386d673d4a102c753a574f1b", "native_artifact_sha256": "b596f1dbab89369555e0a72a0d6854d780e305686750949b308c8f167993bdeb", "scope": "Synthetic repeated text-batch measurements on one GPU; not end-to-end service latency.", "tf32": false, "token_lengths_include_special_tokens": true, "torch_version_family": "2.12", "transformers_version": "4.57.6", "warmup_batches": 2, "status": "historical_previous_release_measurement", "applicability": "Historical measurement on its recorded native artifact; not applicable to this text-backbone update." }, "parameter_storage_change": { "old_safetensors_bytes": 578289664, "new_safetensors_bytes": 541598024, "reduction_bytes": 36691640, "old_FP32_parameter_bytes": 578216448, "new_FP32_parameter_bytes": 541535232, "scope": "Parameter storage only; not measured process peak memory or latency. Vector and index dimensions are unchanged." } } }, "previous_release": { "model": "vllm-sr/Vela-1.0-Omni-Nano", "revision": "9e747d2e1d4b97d3081ff4ac2ac79986c7a8da95", "native_artifact_sha256": "50ce808197fb43a1b8913a66d4a47339ec66ecd10d52cbd9f9e752d02f8c43fa", "evaluated_artifact_fingerprint_sha256": "96d76f97478e114b816df3ab1b3c6947e4c5ef0babfb40ab865d7e3c838d34c4", "parameters": 135383808, "scores_report": "benchmarks/previous-release-scores.json" }, "full_benchmarks": [ { "benchmark": "MTEB(eng, v2)", "framework_version": "2.21.0", "coverage": { "completed_tasks": 41, "required_tasks": 41, "task_types": 7 }, "model_revision": "29994142677559ca657c27b657eda0dddbf583bd4cf42a3a31095ab3f99d1646", "native_artifact_sha256": "29994142677559ca657c27b657eda0dddbf583bd4cf42a3a31095ab3f99d1646", "aggregation": { "mean_task": 0.6488431514310018, "mean_task_type": 0.6078184975654923, "by_task_type": { "Classification": { "score": 0.7782970776285105, "tasks": 8 }, "Clustering": { "score": 0.48962597555764586, "tasks": 8 }, "PairClassification": { "score": 0.8468485561827029, "tasks": 3 }, "Reranking": { "score": 0.473475, "tasks": 2 }, "Retrieval": { "score": 0.520098, "tasks": 10 }, "STS": { "score": 0.8280405301305157, "tasks": 9 }, "Summarization": { "score": 0.31834434345907126, "tasks": 1 } }, "definition": "Official within-task aggregation; equal task mean across 41 tasks, and equal mean of seven within-type means.", "score_scale": "0-1; display multiplied by 100" }, "report": "benchmarks/mteb-eng-v2.json", "text_component_applicability": { "component": "text", "applies_to_revision": "self", "applies_to_native_artifact_sha256": "474ad3d52f6ee7f818a3dbb6112e3d66c2423542d08228e7996a8cd600bd99a2", "evaluated_model": "local/Vela-Omni-Nano-GIST-Rotation", "evaluated_revision": "29994142677559ca657c27b657eda0dddbf583bd4cf42a3a31095ab3f99d1646", "evaluated_native_artifact_sha256": "29994142677559ca657c27b657eda0dddbf583bd4cf42a3a31095ab3f99d1646", "revision_kind": "local native artifact fingerprint, not a Hugging Face commit", "report": "benchmarks/component-equivalence.json", "new_full_panel_inference": false, "basis": "The evaluated text computation is retained through the published Nano and its inference-only trim. All 197 retained text tensors, effective FP32 values, tokenizer/config and text operation path match; the two removed pooler tensors are unused.", "bridge_sha256": "42d802c7279a1c20728926c079f5a4f952b16629fc49a8ed121da4c51f3d77aa" } }, { "benchmark": "MAEB(beta, audio-only)", "framework_version": "2.21.0", "coverage": { "completed_tasks": 19, "required_tasks": 19, "subset_split_slots": 134, "task_types": 5, "selection": "All MAEB(beta) tasks whose modalities are exactly [audio].", "full_MAEB": false, "excluded": "Tasks that require text/audio or other joint-modality inputs." }, "model_revision": "474ad3d52f6ee7f818a3dbb6112e3d66c2423542d08228e7996a8cd600bd99a2", "native_artifact_sha256": "474ad3d52f6ee7f818a3dbb6112e3d66c2423542d08228e7996a8cd600bd99a2", "aggregation": { "official": { "Any2AnyRetrieval": 0.8580149999999999, "AudioClassification": 0.44475865771818446, "AudioClustering": 0.03504342925419187, "AudioPairClassification": 0.5741932198702043, "AudioReranking": 0.70476, "Mean(Task)": 0.4359384306459587, "Mean(TaskType)": 0.5233540613685161 }, "definition": "Official TaskResult aggregation within each task, followed by official mean_task, mean_task_type and task_types aggregation. The 134 subset/split slots are not averaged as independent tasks.", "score_scale": "0-1; multiply by 100 for display" }, "report": "benchmarks/maeb-audio-only.json", "evaluated_native_artifact_sha256": "474ad3d52f6ee7f818a3dbb6112e3d66c2423542d08228e7996a8cd600bd99a2", "release_native_artifact_sha256": "474ad3d52f6ee7f818a3dbb6112e3d66c2423542d08228e7996a8cd600bd99a2" } ], "net_benefit": { "scope": "Fixed product progress metric; not an official benchmark aggregate, significance claim, or proof of overall SOTA.", "baseline_revision": "9e747d2e1d4b97d3081ff4ac2ac79986c7a8da95", "baseline_native_fingerprint": "50ce808197fb43a1b8913a66d4a47339ec66ecd10d52cbd9f9e752d02f8c43fa", "baseline_evaluated_fingerprint": "96d76f97478e114b816df3ab1b3c6947e4c5ef0babfb40ab865d7e3c838d34c4", "candidate_native_fingerprint": "474ad3d52f6ee7f818a3dbb6112e3d66c2423542d08228e7996a8cd600bd99a2", "components": [ { "candidate_100": 66.41835251574038, "component": "common14", "delta_pp": -1.1557204928749676, "parent_100": 67.57407300861536, "weight": 0.3 }, { "candidate_100": 60.781849756549235, "component": "English41_MeanTaskType", "delta_pp": 0.0, "parent_100": 60.781849756549235, "weight": 0.3 }, { "candidate_100": 53.65746394890041, "component": "image3", "delta_pp": 0.0, "parent_100": 53.65746394890041, "weight": 0.2 }, { "candidate_100": 52.33540613685162, "component": "audio19_MeanTaskType", "delta_pp": 5.367592050294833, "parent_100": 46.96781408655678, "weight": 0.2 } ], "weighted_net_delta_pp": 0.7268022621964765, "score_scale": "0–100; deltas in percentage points", "parameter_increase_percent": 20.968150046422096, "common_definition": "Mean of text accuracy (2), image recall (6), and audio recall (6), with equal one-third family weights.", "image3_definition": "Mean of OxfordPets accuracy, TinyImageNetClustering NMI, and OxfordPetsZeroShot accuracy.", "common_metric_delta_pp": { "audio.audio_to_text.recall@1": -1.0340865568747608, "audio.audio_to_text.recall@10": -1.3787820758330127, "audio.audio_to_text.recall@5": -1.8383761011106892, "audio.text_to_audio.recall@1": -5.095785440613029, "audio.text_to_audio.recall@10": -5.287356321839088, "audio.text_to_audio.recall@5": -6.168582375478926, "image.image_to_text.recall@1": 0.0, "image.image_to_text.recall@10": 0.0, "image.image_to_text.recall@5": 0.0, "image.text_to_image.recall@1": 0.0, "image.text_to_image.recall@10": 0.0, "image.text_to_image.recall@5": 0.0, "text.banking77.accuracy": 0.0, "text.massive-en.accuracy": 0.0 }, "audio_task_delta_pp": { "BeijingOpera": 10.549645390070905, "BirdCLEF": 2.2999999999999963, "CREMADPairClassification": 1.0060252268561554, "CREMA_D": 2.2844768445301367, "CREMA_DClustering": 2.8541749889603825, "CommonLanguageAgeDetection": 0.6150000000000017, "GTZANAudioReranking": 6.68200000000001, "GTZANGenre": 11.399999999999999, "IEMOCAPGender": 3.237875602721152, "JamAltArtistA2ARetrieval": 16.90099999999998, "MInDS14": -1.4691308536005643, "MridinghamTonic": 26.05657742038183, "NMSQAPairClassification": -0.3840625955228716, "SIBFLEURS": -0.23211656203982745, "VehicleSoundClustering": -8.525275870377525, "VoxCelebSA": -0.00058896531414665, "VoxPopuliAccentPairClassification": 0.1293630037889515, "VoxPopuliGenderClustering": -0.08127278385060711, "VoxPopuliLanguageID": -0.5999999999999783 }, "audio_aggregate_delta_pp": { "Any2AnyRetrieval": 16.90099999999998, "AudioClassification": 4.921976261522682, "AudioClustering": -1.9174578884225837, "AudioPairClassification": 0.2504418783740858, "AudioReranking": 6.68200000000001, "Mean(Task)": 3.8275626761370307, "Mean(TaskType)": 5.367592050294833 }, "English_and_image_unchanged": true, "event840_in_net": false, "paired_interval_scope": "Fresh paired intervals compare this candidate with the preceding Vela Nano computation. They are not an interval for the all-14 aggregate or the weighted net. An original-small interval was not recomputed for this update." }, "paired_uncertainty_scope": "Fresh paired intervals compare this candidate with the preceding Vela Nano computation. They are not an interval for the all-14 aggregate or the weighted net. An original-small interval was not recomputed for this update.", "scorecard_comparison": { "primary_comparator": "vllm-sr/multi-modal-embed-small", "revision": "fdf8e01b7b0f3a69ac1ac8e2a64dcb1ede177ba4", "common_protocol_measured": true, "original_reference_evidence": "benchmarks/original-baseline.json", "previous_Vela_not_the_primary_comparator": true, "historical_release_progress": "benchmarks/release-history.md" } }