| { |
| "schema_version": 2, |
| "model": "vllm-sr/Vela-1.0-Omni-Nano", |
| "revision_semantics": "self denotes the released native package. Each benchmark retains its actual evaluated artifact; exact unchanged-component applicability is explicit.", |
| "score_scale": "0-1; multiply by 100 for percentages", |
| "matrix": { |
| "small": { |
| "model": "vllm-sr/multi-modal-embed-small", |
| "revision": "fdf8e01b7b0f3a69ac1ac8e2a64dcb1ede177ba4", |
| "artifact_fingerprint_sha256": "684fe0c48a4ddd3944167f6e5dc75e97ada24e5b120bd9a2921f00dc03112eeb", |
| "metrics": { |
| "text.banking77.accuracy": { |
| "numerator": 2169, |
| "denominator": 3080, |
| "value": 0.7042207792207792 |
| }, |
| "text.massive-en.accuracy": { |
| "numerator": 1960, |
| "denominator": 2972, |
| "value": 0.6594885598923284 |
| }, |
| "image.image_to_text.recall@1": { |
| "numerator": 336, |
| "denominator": 823, |
| "value": 0.4082624544349939 |
| }, |
| "image.image_to_text.recall@5": { |
| "numerator": 553, |
| "denominator": 823, |
| "value": 0.6719319562575942 |
| }, |
| "image.image_to_text.recall@10": { |
| "numerator": 646, |
| "denominator": 823, |
| "value": 0.7849331713244229 |
| }, |
| "image.text_to_image.recall@1": { |
| "numerator": 1242, |
| "denominator": 4115, |
| "value": 0.3018226002430134 |
| }, |
| "image.text_to_image.recall@5": { |
| "numerator": 2445, |
| "denominator": 4115, |
| "value": 0.5941676792223572 |
| }, |
| "image.text_to_image.recall@10": { |
| "numerator": 3057, |
| "denominator": 4115, |
| "value": 0.7428918590522479 |
| }, |
| "audio.audio_to_text.recall@1": { |
| "numerator": 110, |
| "denominator": 2611, |
| "value": 0.04212945231711988 |
| }, |
| "audio.audio_to_text.recall@5": { |
| "numerator": 313, |
| "denominator": 2611, |
| "value": 0.11987744159325929 |
| }, |
| "audio.audio_to_text.recall@10": { |
| "numerator": 497, |
| "denominator": 2611, |
| "value": 0.1903485254691689 |
| }, |
| "audio.text_to_audio.recall@1": { |
| "numerator": 250, |
| "denominator": 2610, |
| "value": 0.09578544061302682 |
| }, |
| "audio.text_to_audio.recall@5": { |
| "numerator": 588, |
| "denominator": 2610, |
| "value": 0.22528735632183908 |
| }, |
| "audio.text_to_audio.recall@10": { |
| "numerator": 801, |
| "denominator": 2610, |
| "value": 0.30689655172413793 |
| } |
| } |
| }, |
| "candidate": { |
| "model": "vllm-sr/Vela-1.0-Omni-Nano", |
| "revision": "self", |
| "artifact_fingerprint_sha256": "474ad3d52f6ee7f818a3dbb6112e3d66c2423542d08228e7996a8cd600bd99a2", |
| "evaluated_native_artifact_sha256": "474ad3d52f6ee7f818a3dbb6112e3d66c2423542d08228e7996a8cd600bd99a2", |
| "parameters": 163771288, |
| "metrics": { |
| "audio.audio_to_text.recall@1": { |
| "denominator": 2611, |
| "numerator": 421, |
| "value": 0.1612409038682497 |
| }, |
| "audio.audio_to_text.recall@10": { |
| "denominator": 2611, |
| "numerator": 1191, |
| "value": 0.45614707008808886 |
| }, |
| "audio.audio_to_text.recall@5": { |
| "denominator": 2611, |
| "numerator": 909, |
| "value": 0.34814247414783606 |
| }, |
| "audio.text_to_audio.recall@1": { |
| "denominator": 2610, |
| "numerator": 531, |
| "value": 0.20344827586206896 |
| }, |
| "audio.text_to_audio.recall@10": { |
| "denominator": 2610, |
| "numerator": 1321, |
| "value": 0.5061302681992337 |
| }, |
| "audio.text_to_audio.recall@5": { |
| "denominator": 2610, |
| "numerator": 1043, |
| "value": 0.3996168582375479 |
| }, |
| "image.image_to_text.recall@1": { |
| "denominator": 823, |
| "numerator": 501, |
| "value": 0.6087484811664642 |
| }, |
| "image.image_to_text.recall@10": { |
| "denominator": 823, |
| "numerator": 781, |
| "value": 0.9489671931956257 |
| }, |
| "image.image_to_text.recall@5": { |
| "denominator": 823, |
| "numerator": 717, |
| "value": 0.8712029161603888 |
| }, |
| "image.text_to_image.recall@1": { |
| "denominator": 4115, |
| "numerator": 2297, |
| "value": 0.5582017010935602 |
| }, |
| "image.text_to_image.recall@10": { |
| "denominator": 4115, |
| "numerator": 3879, |
| "value": 0.9426488456865127 |
| }, |
| "image.text_to_image.recall@5": { |
| "denominator": 4115, |
| "numerator": 3507, |
| "value": 0.8522478736330498 |
| }, |
| "text.banking77.accuracy": { |
| "denominator": 3080, |
| "numerator": 2710, |
| "value": 0.8798701298701299 |
| }, |
| "text.massive-en.accuracy": { |
| "denominator": 2972, |
| "numerator": 2436, |
| "value": 0.819650067294751 |
| } |
| } |
| } |
| }, |
| "datasets": { |
| "banking77": { |
| "source": "https://github.com/PolyAI-LDN/task-specific-datasets", |
| "revision": "57ec275d8078af65b7731c2a98be812d844a6d6b", |
| "license": "CC-BY-4.0", |
| "split": "test", |
| "queries": 3080, |
| "classes": 77 |
| }, |
| "massive-en": { |
| "source": "https://huggingface.co/datasets/AmazonScience/massive", |
| "revision": "ff6bd8e4b27c3543e4f8fe2108f32bb95a6f8740", |
| "license": "CC-BY-4.0", |
| "configuration": "en-US", |
| "split": "test", |
| "queries": 2972, |
| "prototype_classes": 60 |
| }, |
| "image": { |
| "source": "https://cocodataset.org/", |
| "caption_split_source": "https://cs.stanford.edu/people/karpathy/deepimagesent/", |
| "split": "Karpathy test, CC-BY 2.0 image subset", |
| "images": 823, |
| "captions": 4115, |
| "image_license": "CC-BY-2.0", |
| "caption_license": "CC-BY-4.0" |
| }, |
| "audio": { |
| "source": "https://www.openslr.org/12", |
| "split": "test-clean", |
| "archive_sha256": "39fde525e59672dc6d1551919b1478f724438a95aa55f874b576be21967e6c23", |
| "license": "CC-BY-4.0", |
| "clips": 2611, |
| "unique_transcripts": 2610, |
| "maximum_seconds": 30 |
| } |
| }, |
| "protocol": { |
| "embedding_dtype": "float32", |
| "attention": "SDPA", |
| "autocast": false, |
| "tf32": false, |
| "transformers_version": "4.57.6", |
| "torch_version_family": "2.12", |
| "text_max_tokens": 128, |
| "truncation": "right, native tokenizer including special tokens", |
| "batch_sizes": { |
| "text": 64, |
| "image": 8, |
| "audio": 8 |
| }, |
| "pooling": "native full-dimension model pooling", |
| "similarity": "FP32 cosine of L2-normalized embeddings", |
| "ties": "lexicographic candidate ID", |
| "retrieval": "complete candidate pools; all annotated positives", |
| "classification": "Nearest normalized mean class prototype from labeled training examples; the original small comparator has task-adapted history, while current GIST is frozen.", |
| "aggregation": "Exact per-metric counts; the product common aggregate averages all fourteen metrics with equal text/image/audio family weights (2/6/6 metrics). Historical paired family/utility intervals retain their original definitions.", |
| "evaluation_scope": "fixed test-split evaluation pools reused across releases", |
| "text_training_scope": "Current GIST text tower is frozen; supervised training prototypes are recomputed under the fixed common protocol. The earlier small comparator has task-adapted history. This is not the official MTEB classification protocol.", |
| "audio_preprocessing": "Original PCM is separately resampled to 16 kHz for Whisper and 48 kHz for CLAP; official evaluation resamples before applying its 30-second cap. Public API rejects inputs longer than 30 seconds." |
| }, |
| "paired_uncertainty": { |
| "scope": "Fresh paired intervals compare this candidate with the preceding Vela Nano computation. They are not an interval for the all-14 aggregate or the weighted net. An original-small interval was not recomputed for this update.", |
| "previous_release": { |
| "published_fingerprint": "50ce808197fb43a1b8913a66d4a47339ec66ecd10d52cbd9f9e752d02f8c43fa", |
| "evaluated_fingerprint": "96d76f97478e114b816df3ab1b3c6947e4c5ef0babfb40ab865d7e3c838d34c4", |
| "result": { |
| "intervals": { |
| "audio.audio_to_text.recall@1": { |
| "lower": -0.018926721728713448, |
| "point_delta": -0.010340865568747606, |
| "upper": -0.001484096923424178 |
| }, |
| "audio.audio_to_text.recall@10": { |
| "lower": -0.02562171008540101, |
| "point_delta": -0.013787820758330141, |
| "upper": -0.001602915531368321 |
| }, |
| "audio.audio_to_text.recall@5": { |
| "lower": -0.02957690797530578, |
| "point_delta": -0.018383761011106857, |
| "upper": -0.007289100963896155 |
| }, |
| "audio.text_to_audio.recall@1": { |
| "lower": -0.06884900952272836, |
| "point_delta": -0.050957854406130265, |
| "upper": -0.03441425975371444 |
| }, |
| "audio.text_to_audio.recall@10": { |
| "lower": -0.07240471605447092, |
| "point_delta": -0.052873563218390804, |
| "upper": -0.03213863952299779 |
| }, |
| "audio.text_to_audio.recall@5": { |
| "lower": -0.07963438278344268, |
| "point_delta": -0.061685823754789273, |
| "upper": -0.044619605933086656 |
| }, |
| "family.audio": { |
| "lower": -0.03959510843694133, |
| "point_delta": -0.030649359987438937, |
| "upper": -0.02150226739148048 |
| }, |
| "family.image": { |
| "lower": 0.0, |
| "point_delta": 0.0, |
| "upper": 0.0 |
| }, |
| "family.text": { |
| "lower": 0.0, |
| "point_delta": 0.0, |
| "upper": 0.0 |
| }, |
| "image.image_to_text.recall@1": { |
| "lower": 0.0, |
| "point_delta": 0.0, |
| "upper": 0.0 |
| }, |
| "image.image_to_text.recall@10": { |
| "lower": 0.0, |
| "point_delta": 0.0, |
| "upper": 0.0 |
| }, |
| "image.image_to_text.recall@5": { |
| "lower": 0.0, |
| "point_delta": 0.0, |
| "upper": 0.0 |
| }, |
| "image.text_to_image.recall@1": { |
| "lower": 0.0, |
| "point_delta": 0.0, |
| "upper": 0.0 |
| }, |
| "image.text_to_image.recall@10": { |
| "lower": 0.0, |
| "point_delta": 0.0, |
| "upper": 0.0 |
| }, |
| "image.text_to_image.recall@5": { |
| "lower": 0.0, |
| "point_delta": 0.0, |
| "upper": 0.0 |
| }, |
| "text.banking77.accuracy": { |
| "lower": 0.0, |
| "point_delta": 0.0, |
| "upper": 0.0 |
| }, |
| "text.massive-en.accuracy": { |
| "lower": 0.0, |
| "point_delta": 0.0, |
| "upper": 0.0 |
| }, |
| "utility": { |
| "lower": -0.013198369478980442, |
| "point_delta": -0.010216453329146313, |
| "upper": -0.0071674224638268265 |
| } |
| }, |
| "method": "paired stratified group percentile; PCG64; fixed candidate pools", |
| "replicates": 2000, |
| "seed": 20260918 |
| } |
| }, |
| "original_small": { |
| "status": "not_recomputed_for_this_update", |
| "value": null |
| } |
| }, |
| "training_scope": { |
| "text": "The complete text tower is the frozen official avsolatorio/GIST-small-Embedding-v0 checkpoint at 75e62fd210b9fde790430e0b2f040b0b00a021b1. Its model card discloses MTEB classification training data; no claim of independent pretraining data is made.", |
| "image": "Frozen current vision encoder and initial image head; an orthogonal shared-space rotation is trained on 16,509 COCO training images, 81,998 unique caption texts and 82,522 complete positive associations, then folded into the image affine head.", |
| "audio": "The existing Whisper Tiny encoder and affine head stay frozen. A frozen CLAP audio tower and a 384×512 residual projection are added; only that projection is trained for one fixed 600-step run. Speech alignment and a pointwise parent anchor use 28,535 LibriSpeech TRAIN clips; known-positive event labels and CLAP relational geometry use 3,299 FSD50K TRAIN clips. Center/scale and the geometry coefficient use TRAIN only.", |
| "procedure": "One fixed final-600-step residual artifact; no held-out benchmark examples fit the projection, statistics or coefficient. Full benchmark outcomes and regressions are retained.", |
| "evaluation_scope": "Common text classification uses labeled training prototypes with the earlier task-adapted comparator. It is separate from the official frozen-encoder MTEB classification protocols.", |
| "prior_alignment_history": { |
| "text": "The complete text tower is the frozen official avsolatorio/GIST-small-Embedding-v0 checkpoint at 75e62fd210b9fde790430e0b2f040b0b00a021b1. Its model card discloses MTEB classification training data; no claim of independent pretraining data is made.", |
| "image": "Frozen current vision encoder and initial image head; an orthogonal shared-space rotation is trained on 16,509 COCO training images, 81,998 unique caption texts and 82,522 complete positive associations, then folded into the image affine head.", |
| "audio": "Frozen current speech encoder and initial audio head; a separate orthogonal shared-space rotation is trained on 28,535 LibriSpeech train-clean-100 clips and all positive transcript associations, then folded into the audio affine head.", |
| "procedure": "One fixed 300-step run; only the final artifact is reported. Encoders remain frozen. The two rotations preserve within-modality inner products, up to floating-point rounding.", |
| "evaluation_scope": "Common text classification uses labeled training prototypes with the earlier task-adapted comparator. It is separate from the official frozen-encoder MTEB classification protocols." |
| }, |
| "event_labels": "149 labels with TRAIN support and 7,690 known-positive associations. Unannotated labels are not negatives; label names are weak class supervision, not observed clip captions." |
| }, |
| "descriptive_metrics": { |
| "definition": "Macro-F1 is the unweighted mean of per-class F1 across all fixed prototype classes; zero_division=0.", |
| "scope": "Same models, examples, class prototypes and predictions as the accuracy matrix.", |
| "metrics": { |
| "text.banking77.macro_f1": { |
| "small": { |
| "value": 0.7020781051899517, |
| "classes": 77, |
| "queries": 3080 |
| }, |
| "candidate": { |
| "value": 0.8791681858119436, |
| "classes": 77, |
| "queries": 3080 |
| } |
| }, |
| "text.massive-en.macro_f1": { |
| "small": { |
| "value": 0.6171775818092488, |
| "classes": 60, |
| "queries": 2972 |
| }, |
| "candidate": { |
| "value": 0.7923223418549947, |
| "classes": 60, |
| "queries": 2972 |
| } |
| } |
| }, |
| "candidate_per_class": { |
| "banking77": { |
| "accuracy": { |
| "denominator": 3080, |
| "numerator": 2710, |
| "value": 0.8798701298701299 |
| }, |
| "evaluated_classes": 77, |
| "macro_f1": 0.8791681858119436, |
| "macro_f1_class_universe": "all fixed prototype classes; zero_division=0", |
| "per_class": { |
| "Refund_not_showing_up": { |
| "f1": 0.9512195121951219, |
| "fn": 1, |
| "fp": 3, |
| "support": 40, |
| "tp": 39 |
| }, |
| "activate_my_card": { |
| "f1": 0.935064935064935, |
| "fn": 4, |
| "fp": 1, |
| "support": 40, |
| "tp": 36 |
| }, |
| "age_limit": { |
| "f1": 1.0, |
| "fn": 0, |
| "fp": 0, |
| "support": 40, |
| "tp": 40 |
| }, |
| "apple_pay_or_google_pay": { |
| "f1": 1.0, |
| "fn": 0, |
| "fp": 0, |
| "support": 40, |
| "tp": 40 |
| }, |
| "atm_support": { |
| "f1": 0.9382716049382716, |
| "fn": 2, |
| "fp": 3, |
| "support": 40, |
| "tp": 38 |
| }, |
| "automatic_top_up": { |
| "f1": 0.8918918918918919, |
| "fn": 7, |
| "fp": 1, |
| "support": 40, |
| "tp": 33 |
| }, |
| "balance_not_updated_after_bank_transfer": { |
| "f1": 0.7407407407407407, |
| "fn": 10, |
| "fp": 11, |
| "support": 40, |
| "tp": 30 |
| }, |
| "balance_not_updated_after_cheque_or_cash_deposit": { |
| "f1": 0.9285714285714286, |
| "fn": 1, |
| "fp": 5, |
| "support": 40, |
| "tp": 39 |
| }, |
| "beneficiary_not_allowed": { |
| "f1": 0.7246376811594203, |
| "fn": 15, |
| "fp": 4, |
| "support": 40, |
| "tp": 25 |
| }, |
| "cancel_transfer": { |
| "f1": 0.95, |
| "fn": 2, |
| "fp": 2, |
| "support": 40, |
| "tp": 38 |
| }, |
| "card_about_to_expire": { |
| "f1": 0.9523809523809523, |
| "fn": 0, |
| "fp": 4, |
| "support": 40, |
| "tp": 40 |
| }, |
| "card_acceptance": { |
| "f1": 0.9069767441860465, |
| "fn": 1, |
| "fp": 7, |
| "support": 40, |
| "tp": 39 |
| }, |
| "card_arrival": { |
| "f1": 0.8421052631578947, |
| "fn": 8, |
| "fp": 4, |
| "support": 40, |
| "tp": 32 |
| }, |
| "card_delivery_estimate": { |
| "f1": 0.8536585365853658, |
| "fn": 5, |
| "fp": 7, |
| "support": 40, |
| "tp": 35 |
| }, |
| "card_linking": { |
| "f1": 0.8947368421052632, |
| "fn": 6, |
| "fp": 2, |
| "support": 40, |
| "tp": 34 |
| }, |
| "card_not_working": { |
| "f1": 0.8181818181818182, |
| "fn": 4, |
| "fp": 12, |
| "support": 40, |
| "tp": 36 |
| }, |
| "card_payment_fee_charged": { |
| "f1": 0.8536585365853658, |
| "fn": 5, |
| "fp": 7, |
| "support": 40, |
| "tp": 35 |
| }, |
| "card_payment_not_recognised": { |
| "f1": 0.8395061728395061, |
| "fn": 6, |
| "fp": 7, |
| "support": 40, |
| "tp": 34 |
| }, |
| "card_payment_wrong_exchange_rate": { |
| "f1": 0.95, |
| "fn": 2, |
| "fp": 2, |
| "support": 40, |
| "tp": 38 |
| }, |
| "card_swallowed": { |
| "f1": 0.9473684210526315, |
| "fn": 4, |
| "fp": 0, |
| "support": 40, |
| "tp": 36 |
| }, |
| "cash_withdrawal_charge": { |
| "f1": 0.8941176470588236, |
| "fn": 2, |
| "fp": 7, |
| "support": 40, |
| "tp": 38 |
| }, |
| "cash_withdrawal_not_recognised": { |
| "f1": 0.7951807228915663, |
| "fn": 7, |
| "fp": 10, |
| "support": 40, |
| "tp": 33 |
| }, |
| "change_pin": { |
| "f1": 0.9873417721518988, |
| "fn": 1, |
| "fp": 0, |
| "support": 40, |
| "tp": 39 |
| }, |
| "compromised_card": { |
| "f1": 0.8395061728395061, |
| "fn": 6, |
| "fp": 7, |
| "support": 40, |
| "tp": 34 |
| }, |
| "contactless_not_working": { |
| "f1": 0.9473684210526315, |
| "fn": 4, |
| "fp": 0, |
| "support": 40, |
| "tp": 36 |
| }, |
| "country_support": { |
| "f1": 0.9210526315789473, |
| "fn": 5, |
| "fp": 1, |
| "support": 40, |
| "tp": 35 |
| }, |
| "declined_card_payment": { |
| "f1": 0.8043478260869565, |
| "fn": 3, |
| "fp": 15, |
| "support": 40, |
| "tp": 37 |
| }, |
| "declined_cash_withdrawal": { |
| "f1": 0.8333333333333334, |
| "fn": 5, |
| "fp": 9, |
| "support": 40, |
| "tp": 35 |
| }, |
| "declined_transfer": { |
| "f1": 0.8115942028985508, |
| "fn": 12, |
| "fp": 1, |
| "support": 40, |
| "tp": 28 |
| }, |
| "direct_debit_payment_not_recognised": { |
| "f1": 0.8450704225352113, |
| "fn": 10, |
| "fp": 1, |
| "support": 40, |
| "tp": 30 |
| }, |
| "disposable_card_limits": { |
| "f1": 0.8395061728395061, |
| "fn": 6, |
| "fp": 7, |
| "support": 40, |
| "tp": 34 |
| }, |
| "edit_personal_details": { |
| "f1": 0.975, |
| "fn": 1, |
| "fp": 1, |
| "support": 40, |
| "tp": 39 |
| }, |
| "exchange_charge": { |
| "f1": 0.9367088607594937, |
| "fn": 3, |
| "fp": 2, |
| "support": 40, |
| "tp": 37 |
| }, |
| "exchange_rate": { |
| "f1": 0.9397590361445783, |
| "fn": 1, |
| "fp": 4, |
| "support": 40, |
| "tp": 39 |
| }, |
| "exchange_via_app": { |
| "f1": 0.7906976744186046, |
| "fn": 6, |
| "fp": 12, |
| "support": 40, |
| "tp": 34 |
| }, |
| "extra_charge_on_statement": { |
| "f1": 0.8409090909090909, |
| "fn": 3, |
| "fp": 11, |
| "support": 40, |
| "tp": 37 |
| }, |
| "failed_transfer": { |
| "f1": 0.7956989247311828, |
| "fn": 3, |
| "fp": 16, |
| "support": 40, |
| "tp": 37 |
| }, |
| "fiat_currency_support": { |
| "f1": 0.7804878048780488, |
| "fn": 8, |
| "fp": 10, |
| "support": 40, |
| "tp": 32 |
| }, |
| "get_disposable_virtual_card": { |
| "f1": 0.7887323943661971, |
| "fn": 12, |
| "fp": 3, |
| "support": 40, |
| "tp": 28 |
| }, |
| "get_physical_card": { |
| "f1": 0.963855421686747, |
| "fn": 0, |
| "fp": 3, |
| "support": 40, |
| "tp": 40 |
| }, |
| "getting_spare_card": { |
| "f1": 0.8941176470588236, |
| "fn": 2, |
| "fp": 7, |
| "support": 40, |
| "tp": 38 |
| }, |
| "getting_virtual_card": { |
| "f1": 0.8764044943820225, |
| "fn": 1, |
| "fp": 10, |
| "support": 40, |
| "tp": 39 |
| }, |
| "lost_or_stolen_card": { |
| "f1": 0.8666666666666667, |
| "fn": 1, |
| "fp": 11, |
| "support": 40, |
| "tp": 39 |
| }, |
| "lost_or_stolen_phone": { |
| "f1": 0.9873417721518988, |
| "fn": 1, |
| "fp": 0, |
| "support": 40, |
| "tp": 39 |
| }, |
| "order_physical_card": { |
| "f1": 0.8275862068965517, |
| "fn": 4, |
| "fp": 11, |
| "support": 40, |
| "tp": 36 |
| }, |
| "passcode_forgotten": { |
| "f1": 0.9876543209876543, |
| "fn": 0, |
| "fp": 1, |
| "support": 40, |
| "tp": 40 |
| }, |
| "pending_card_payment": { |
| "f1": 0.95, |
| "fn": 2, |
| "fp": 2, |
| "support": 40, |
| "tp": 38 |
| }, |
| "pending_cash_withdrawal": { |
| "f1": 0.8611111111111112, |
| "fn": 9, |
| "fp": 1, |
| "support": 40, |
| "tp": 31 |
| }, |
| "pending_top_up": { |
| "f1": 0.8831168831168831, |
| "fn": 6, |
| "fp": 3, |
| "support": 40, |
| "tp": 34 |
| }, |
| "pending_transfer": { |
| "f1": 0.7647058823529411, |
| "fn": 14, |
| "fp": 2, |
| "support": 40, |
| "tp": 26 |
| }, |
| "pin_blocked": { |
| "f1": 0.9230769230769231, |
| "fn": 4, |
| "fp": 2, |
| "support": 40, |
| "tp": 36 |
| }, |
| "receiving_money": { |
| "f1": 0.9090909090909091, |
| "fn": 5, |
| "fp": 2, |
| "support": 40, |
| "tp": 35 |
| }, |
| "request_refund": { |
| "f1": 0.9066666666666666, |
| "fn": 6, |
| "fp": 1, |
| "support": 40, |
| "tp": 34 |
| }, |
| "reverted_card_payment?": { |
| "f1": 0.825, |
| "fn": 7, |
| "fp": 7, |
| "support": 40, |
| "tp": 33 |
| }, |
| "supported_cards_and_currencies": { |
| "f1": 0.7777777777777778, |
| "fn": 12, |
| "fp": 4, |
| "support": 40, |
| "tp": 28 |
| }, |
| "terminate_account": { |
| "f1": 0.9411764705882353, |
| "fn": 0, |
| "fp": 5, |
| "support": 40, |
| "tp": 40 |
| }, |
| "top_up_by_bank_transfer_charge": { |
| "f1": 0.8888888888888888, |
| "fn": 4, |
| "fp": 5, |
| "support": 40, |
| "tp": 36 |
| }, |
| "top_up_by_card_charge": { |
| "f1": 0.95, |
| "fn": 2, |
| "fp": 2, |
| "support": 40, |
| "tp": 38 |
| }, |
| "top_up_by_cash_or_cheque": { |
| "f1": 0.9230769230769231, |
| "fn": 4, |
| "fp": 2, |
| "support": 40, |
| "tp": 36 |
| }, |
| "top_up_failed": { |
| "f1": 0.8705882352941177, |
| "fn": 3, |
| "fp": 8, |
| "support": 40, |
| "tp": 37 |
| }, |
| "top_up_limits": { |
| "f1": 0.9512195121951219, |
| "fn": 1, |
| "fp": 3, |
| "support": 40, |
| "tp": 39 |
| }, |
| "top_up_reverted": { |
| "f1": 0.875, |
| "fn": 5, |
| "fp": 5, |
| "support": 40, |
| "tp": 35 |
| }, |
| "topping_up_by_card": { |
| "f1": 0.5, |
| "fn": 23, |
| "fp": 11, |
| "support": 40, |
| "tp": 17 |
| }, |
| "transaction_charged_twice": { |
| "f1": 0.9629629629629629, |
| "fn": 1, |
| "fp": 2, |
| "support": 40, |
| "tp": 39 |
| }, |
| "transfer_fee_charged": { |
| "f1": 0.8333333333333334, |
| "fn": 10, |
| "fp": 2, |
| "support": 40, |
| "tp": 30 |
| }, |
| "transfer_into_account": { |
| "f1": 0.7659574468085106, |
| "fn": 4, |
| "fp": 18, |
| "support": 40, |
| "tp": 36 |
| }, |
| "transfer_not_received_by_recipient": { |
| "f1": 0.7088607594936709, |
| "fn": 12, |
| "fp": 11, |
| "support": 40, |
| "tp": 28 |
| }, |
| "transfer_timing": { |
| "f1": 0.7333333333333333, |
| "fn": 7, |
| "fp": 17, |
| "support": 40, |
| "tp": 33 |
| }, |
| "unable_to_verify_identity": { |
| "f1": 0.926829268292683, |
| "fn": 2, |
| "fp": 4, |
| "support": 40, |
| "tp": 38 |
| }, |
| "verify_my_identity": { |
| "f1": 0.9024390243902439, |
| "fn": 3, |
| "fp": 5, |
| "support": 40, |
| "tp": 37 |
| }, |
| "verify_source_of_funds": { |
| "f1": 1.0, |
| "fn": 0, |
| "fp": 0, |
| "support": 40, |
| "tp": 40 |
| }, |
| "verify_top_up": { |
| "f1": 1.0, |
| "fn": 0, |
| "fp": 0, |
| "support": 40, |
| "tp": 40 |
| }, |
| "virtual_card_not_working": { |
| "f1": 0.9333333333333333, |
| "fn": 5, |
| "fp": 0, |
| "support": 40, |
| "tp": 35 |
| }, |
| "visa_or_mastercard": { |
| "f1": 0.9333333333333333, |
| "fn": 5, |
| "fp": 0, |
| "support": 40, |
| "tp": 35 |
| }, |
| "why_verify_identity": { |
| "f1": 0.9090909090909091, |
| "fn": 5, |
| "fp": 2, |
| "support": 40, |
| "tp": 35 |
| }, |
| "wrong_amount_of_cash_received": { |
| "f1": 0.9090909090909091, |
| "fn": 5, |
| "fp": 2, |
| "support": 40, |
| "tp": 35 |
| }, |
| "wrong_exchange_rate_for_cash_withdrawal": { |
| "f1": 0.7878787878787878, |
| "fn": 14, |
| "fp": 0, |
| "support": 40, |
| "tp": 26 |
| } |
| }, |
| "prototype_classes": 77 |
| }, |
| "massive-en": { |
| "accuracy": { |
| "denominator": 2972, |
| "numerator": 2436, |
| "value": 0.819650067294751 |
| }, |
| "evaluated_classes": 59, |
| "macro_f1": 0.7923223418549947, |
| "macro_f1_class_universe": "all fixed prototype classes; zero_division=0", |
| "per_class": { |
| "alarm_query": { |
| "f1": 0.9523809523809523, |
| "fn": 3, |
| "fp": 0, |
| "support": 33, |
| "tp": 30 |
| }, |
| "alarm_remove": { |
| "f1": 0.9545454545454546, |
| "fn": 0, |
| "fp": 2, |
| "support": 21, |
| "tp": 21 |
| }, |
| "alarm_set": { |
| "f1": 0.9047619047619048, |
| "fn": 3, |
| "fp": 5, |
| "support": 41, |
| "tp": 38 |
| }, |
| "audio_volume_down": { |
| "f1": 0.9565217391304348, |
| "fn": 0, |
| "fp": 1, |
| "support": 11, |
| "tp": 11 |
| }, |
| "audio_volume_mute": { |
| "f1": 0.8787878787878788, |
| "fn": 3, |
| "fp": 5, |
| "support": 32, |
| "tp": 29 |
| }, |
| "audio_volume_other": { |
| "f1": 1.0, |
| "fn": 0, |
| "fp": 0, |
| "support": 6, |
| "tp": 6 |
| }, |
| "audio_volume_up": { |
| "f1": 0.782608695652174, |
| "fn": 4, |
| "fp": 1, |
| "support": 13, |
| "tp": 9 |
| }, |
| "calendar_query": { |
| "f1": 0.6864406779661016, |
| "fn": 45, |
| "fp": 29, |
| "support": 126, |
| "tp": 81 |
| }, |
| "calendar_remove": { |
| "f1": 0.9264705882352942, |
| "fn": 4, |
| "fp": 6, |
| "support": 67, |
| "tp": 63 |
| }, |
| "calendar_set": { |
| "f1": 0.8759124087591241, |
| "fn": 29, |
| "fp": 22, |
| "support": 209, |
| "tp": 180 |
| }, |
| "cooking_query": { |
| "f1": 0.0, |
| "fn": 0, |
| "fp": 2, |
| "support": 0, |
| "tp": 0 |
| }, |
| "cooking_recipe": { |
| "f1": 0.9523809523809523, |
| "fn": 2, |
| "fp": 5, |
| "support": 72, |
| "tp": 70 |
| }, |
| "datetime_convert": { |
| "f1": 0.5263157894736842, |
| "fn": 5, |
| "fp": 13, |
| "support": 15, |
| "tp": 10 |
| }, |
| "datetime_query": { |
| "f1": 0.7861271676300579, |
| "fn": 20, |
| "fp": 17, |
| "support": 88, |
| "tp": 68 |
| }, |
| "email_addcontact": { |
| "f1": 0.5714285714285714, |
| "fn": 2, |
| "fp": 13, |
| "support": 12, |
| "tp": 10 |
| }, |
| "email_query": { |
| "f1": 0.9385964912280702, |
| "fn": 12, |
| "fp": 2, |
| "support": 119, |
| "tp": 107 |
| }, |
| "email_querycontact": { |
| "f1": 0.7540983606557377, |
| "fn": 3, |
| "fp": 12, |
| "support": 26, |
| "tp": 23 |
| }, |
| "email_sendemail": { |
| "f1": 0.8959276018099548, |
| "fn": 15, |
| "fp": 8, |
| "support": 114, |
| "tp": 99 |
| }, |
| "general_greet": { |
| "f1": 0.18181818181818182, |
| "fn": 0, |
| "fp": 9, |
| "support": 1, |
| "tp": 1 |
| }, |
| "general_joke": { |
| "f1": 0.9230769230769231, |
| "fn": 1, |
| "fp": 2, |
| "support": 19, |
| "tp": 18 |
| }, |
| "general_quirky": { |
| "f1": 0.4728434504792332, |
| "fn": 94, |
| "fp": 71, |
| "support": 168, |
| "tp": 74 |
| }, |
| "iot_cleaning": { |
| "f1": 0.9433962264150944, |
| "fn": 1, |
| "fp": 2, |
| "support": 26, |
| "tp": 25 |
| }, |
| "iot_coffee": { |
| "f1": 0.9859154929577465, |
| "fn": 1, |
| "fp": 0, |
| "support": 36, |
| "tp": 35 |
| }, |
| "iot_hue_lightchange": { |
| "f1": 0.8181818181818182, |
| "fn": 9, |
| "fp": 3, |
| "support": 36, |
| "tp": 27 |
| }, |
| "iot_hue_lightdim": { |
| "f1": 0.8780487804878049, |
| "fn": 3, |
| "fp": 2, |
| "support": 21, |
| "tp": 18 |
| }, |
| "iot_hue_lightoff": { |
| "f1": 0.9176470588235294, |
| "fn": 4, |
| "fp": 3, |
| "support": 43, |
| "tp": 39 |
| }, |
| "iot_hue_lighton": { |
| "f1": 0.18181818181818182, |
| "fn": 2, |
| "fp": 7, |
| "support": 3, |
| "tp": 1 |
| }, |
| "iot_hue_lightup": { |
| "f1": 0.7857142857142857, |
| "fn": 5, |
| "fp": 7, |
| "support": 27, |
| "tp": 22 |
| }, |
| "iot_wemo_off": { |
| "f1": 0.9444444444444444, |
| "fn": 1, |
| "fp": 1, |
| "support": 18, |
| "tp": 17 |
| }, |
| "iot_wemo_on": { |
| "f1": 0.9, |
| "fn": 1, |
| "fp": 1, |
| "support": 10, |
| "tp": 9 |
| }, |
| "lists_createoradd": { |
| "f1": 0.7777777777777778, |
| "fn": 11, |
| "fp": 5, |
| "support": 39, |
| "tp": 28 |
| }, |
| "lists_query": { |
| "f1": 0.8333333333333334, |
| "fn": 11, |
| "fp": 5, |
| "support": 51, |
| "tp": 40 |
| }, |
| "lists_remove": { |
| "f1": 0.9215686274509803, |
| "fn": 5, |
| "fp": 3, |
| "support": 52, |
| "tp": 47 |
| }, |
| "music_dislikeness": { |
| "f1": 0.8888888888888888, |
| "fn": 0, |
| "fp": 1, |
| "support": 4, |
| "tp": 4 |
| }, |
| "music_likeness": { |
| "f1": 0.7878787878787878, |
| "fn": 10, |
| "fp": 4, |
| "support": 36, |
| "tp": 26 |
| }, |
| "music_query": { |
| "f1": 0.7945205479452054, |
| "fn": 6, |
| "fp": 9, |
| "support": 35, |
| "tp": 29 |
| }, |
| "music_settings": { |
| "f1": 0.36363636363636365, |
| "fn": 2, |
| "fp": 12, |
| "support": 6, |
| "tp": 4 |
| }, |
| "news_query": { |
| "f1": 0.8583333333333333, |
| "fn": 21, |
| "fp": 13, |
| "support": 124, |
| "tp": 103 |
| }, |
| "play_audiobook": { |
| "f1": 0.8311688311688312, |
| "fn": 9, |
| "fp": 4, |
| "support": 41, |
| "tp": 32 |
| }, |
| "play_game": { |
| "f1": 0.717391304347826, |
| "fn": 2, |
| "fp": 24, |
| "support": 35, |
| "tp": 33 |
| }, |
| "play_music": { |
| "f1": 0.8238805970149253, |
| "fn": 38, |
| "fp": 21, |
| "support": 176, |
| "tp": 138 |
| }, |
| "play_podcasts": { |
| "f1": 0.9043478260869565, |
| "fn": 11, |
| "fp": 0, |
| "support": 63, |
| "tp": 52 |
| }, |
| "play_radio": { |
| "f1": 0.8208955223880597, |
| "fn": 17, |
| "fp": 7, |
| "support": 72, |
| "tp": 55 |
| }, |
| "qa_currency": { |
| "f1": 0.9473684210526315, |
| "fn": 3, |
| "fp": 1, |
| "support": 39, |
| "tp": 36 |
| }, |
| "qa_definition": { |
| "f1": 0.8305084745762712, |
| "fn": 8, |
| "fp": 12, |
| "support": 57, |
| "tp": 49 |
| }, |
| "qa_factoid": { |
| "f1": 0.7205882352941176, |
| "fn": 43, |
| "fp": 33, |
| "support": 141, |
| "tp": 98 |
| }, |
| "qa_maths": { |
| "f1": 0.7868852459016393, |
| "fn": 1, |
| "fp": 12, |
| "support": 25, |
| "tp": 24 |
| }, |
| "qa_stock": { |
| "f1": 0.8727272727272727, |
| "fn": 2, |
| "fp": 5, |
| "support": 26, |
| "tp": 24 |
| }, |
| "recommendation_events": { |
| "f1": 0.7102803738317757, |
| "fn": 5, |
| "fp": 26, |
| "support": 43, |
| "tp": 38 |
| }, |
| "recommendation_locations": { |
| "f1": 0.7945205479452054, |
| "fn": 2, |
| "fp": 13, |
| "support": 31, |
| "tp": 29 |
| }, |
| "recommendation_movies": { |
| "f1": 0.7058823529411765, |
| "fn": 2, |
| "fp": 13, |
| "support": 20, |
| "tp": 18 |
| }, |
| "social_post": { |
| "f1": 0.8531468531468531, |
| "fn": 20, |
| "fp": 1, |
| "support": 81, |
| "tp": 61 |
| }, |
| "social_query": { |
| "f1": 0.7301587301587301, |
| "fn": 2, |
| "fp": 15, |
| "support": 25, |
| "tp": 23 |
| }, |
| "takeaway_order": { |
| "f1": 0.7659574468085106, |
| "fn": 4, |
| "fp": 7, |
| "support": 22, |
| "tp": 18 |
| }, |
| "takeaway_query": { |
| "f1": 0.8266666666666667, |
| "fn": 4, |
| "fp": 9, |
| "support": 35, |
| "tp": 31 |
| }, |
| "transport_query": { |
| "f1": 0.782608695652174, |
| "fn": 15, |
| "fp": 5, |
| "support": 51, |
| "tp": 36 |
| }, |
| "transport_taxi": { |
| "f1": 0.9787234042553191, |
| "fn": 0, |
| "fp": 1, |
| "support": 23, |
| "tp": 23 |
| }, |
| "transport_ticket": { |
| "f1": 0.88, |
| "fn": 2, |
| "fp": 7, |
| "support": 35, |
| "tp": 33 |
| }, |
| "transport_traffic": { |
| "f1": 0.8108108108108109, |
| "fn": 0, |
| "fp": 7, |
| "support": 15, |
| "tp": 15 |
| }, |
| "weather_query": { |
| "f1": 0.9426751592356688, |
| "fn": 8, |
| "fp": 10, |
| "support": 156, |
| "tp": 148 |
| } |
| }, |
| "prototype_classes": 60 |
| } |
| } |
| }, |
| "standard_benchmarks": { |
| "scope": "Eleven selected standard tasks; not a complete MTEB, MMTEB, MIEB or MAEB benchmark or leaderboard submission.", |
| "library": "mteb", |
| "library_version": "2.21.0", |
| "model_revisions": { |
| "nano": { |
| "repo_id": "vllm-sr/Vela-1.0-Omni-Nano", |
| "revision": "self", |
| "native_artifact_sha256": "474ad3d52f6ee7f818a3dbb6112e3d66c2423542d08228e7996a8cd600bd99a2", |
| "parameters": 163771288, |
| "embedding_dimensions": 384, |
| "max_text_tokens": 512, |
| "applicability": "Each row states whether the full artifact was freshly evaluated or an exact unchanged component applies." |
| }, |
| "small": { |
| "repo_id": "vllm-sr/multi-modal-embed-small", |
| "revision": "fdf8e01b7b0f3a69ac1ac8e2a64dcb1ede177ba4", |
| "native_artifact_sha256": "684fe0c48a4ddd3944167f6e5dc75e97ada24e5b120bd9a2921f00dc03112eeb", |
| "status": "evaluated_all_11_selected_tasks", |
| "max_text_tokens": 128, |
| "inference_parameters": 133759488 |
| } |
| }, |
| "protocol": { |
| "precision": "float32", |
| "batch_size": 8, |
| "seed": 20260918, |
| "similarity": "Cosine for primary STS/retrieval scores and normalized media-label matching; classification probes and clustering use each official task's algorithms.", |
| "encoder_parameters_frozen": true, |
| "classification": "Official task-specific frozen-embedding probes or fixed-label zero-shot classification.", |
| "sampling": "Official task subsampling and repeated-probe protocols are preserved.", |
| "text": "Each encoder uses its native preprocessing and context limit without added task instructions. Original small: 128 tokens; current Nano: 512 tokens.", |
| "audio": "Current Nano: original PCM independently resampled to 16/48 kHz for speech/CLAP, then downmixed and capped at 30 seconds. Original small: official mono 16 kHz path. The official task algorithms, samples and seed are identical.", |
| "score_scale": "Metrics retain their native MTEB scale; the model card multiplies primary scores by 100.", |
| "undefined_metric": null, |
| "missing_model_score": "Not measured in this standard evaluation.", |
| "overlap": "No claim of independence from component pretraining data is made." |
| }, |
| "rows": [ |
| { |
| "task": "SICK-R", |
| "type": "STS", |
| "modalities": [ |
| "text" |
| ], |
| "split": "test", |
| "hf_subset": "default", |
| "main_metric": "cosine_spearman", |
| "dataset": { |
| "path": "mteb/sickr-sts", |
| "revision": "20a6d6f312dd54037fe07a32d58e5e168867909d" |
| }, |
| "counts": { |
| "evaluation_pairs": 9927 |
| }, |
| "scores": { |
| "nano": 0.8053174565838819, |
| "small": 0.7139020320965759 |
| }, |
| "metrics": { |
| "nano": { |
| "pearson": 0.8603345909155973, |
| "spearman": 0.8053174398238511, |
| "cosine_pearson": 0.8603345867277692, |
| "cosine_spearman": 0.8053174565838819, |
| "manhattan_pearson": 0.8332190477069294, |
| "manhattan_spearman": 0.8044409124391882, |
| "euclidean_pearson": 0.834049309726515, |
| "euclidean_spearman": 0.8053172111347874, |
| "main_score": 0.8053174565838819, |
| "hf_subset": "default", |
| "languages": [ |
| "eng-Latn" |
| ], |
| "mteb_version": "2.21.0" |
| }, |
| "small": { |
| "pearson": 0.8179714611300387, |
| "spearman": 0.713902378578463, |
| "cosine_pearson": 0.81797146109438, |
| "cosine_spearman": 0.7139020320965759, |
| "manhattan_pearson": 0.7919193141236369, |
| "manhattan_spearman": 0.7128438640347968, |
| "euclidean_pearson": 0.792891259170334, |
| "euclidean_spearman": 0.7139020311053554, |
| "main_score": 0.7139020320965759, |
| "hf_subset": "default", |
| "languages": [ |
| "eng-Latn" |
| ], |
| "mteb_version": "2.21.0" |
| } |
| }, |
| "measurement_by_model": { |
| "small": { |
| "evaluated_native_fingerprint": "684fe0c48a4ddd3944167f6e5dc75e97ada24e5b120bd9a2921f00dc03112eeb", |
| "original_checkpoint_revision": "fdf8e01b7b0f3a69ac1ac8e2a64dcb1ede177ba4", |
| "raw_result_sha256": "e20f02a347841b3d5f61209f0b209c6314d006f8cc94f6e488e06239c862badf", |
| "kind": "direct_frozen_encoder_evaluation" |
| }, |
| "nano": { |
| "evaluated_native_fingerprint": "96d76f97478e114b816df3ab1b3c6947e4c5ef0babfb40ab865d7e3c838d34c4", |
| "applicable_native_fingerprint": "474ad3d52f6ee7f818a3dbb6112e3d66c2423542d08228e7996a8cd600bd99a2", |
| "kind": "exact_component_applicability" |
| } |
| }, |
| "evaluated_native_fingerprint": "96d76f97478e114b816df3ab1b3c6947e4c5ef0babfb40ab865d7e3c838d34c4", |
| "applicable_native_fingerprint": "474ad3d52f6ee7f818a3dbb6112e3d66c2423542d08228e7996a8cd600bd99a2", |
| "measurement_kind": "exact_component_applicability" |
| }, |
| { |
| "task": "STSBenchmark", |
| "type": "STS", |
| "modalities": [ |
| "text" |
| ], |
| "split": "test", |
| "hf_subset": "default", |
| "main_metric": "cosine_spearman", |
| "dataset": { |
| "path": "mteb/stsbenchmark-sts", |
| "revision": "b0fddb56ed78048fa8b90373c8a3cfc37b684831" |
| }, |
| "counts": { |
| "evaluation_pairs": 1379 |
| }, |
| "scores": { |
| "nano": 0.870781967302773, |
| "small": 0.7729871863268857 |
| }, |
| "metrics": { |
| "nano": { |
| "pearson": 0.8541596951761798, |
| "spearman": 0.8707819111802441, |
| "cosine_pearson": 0.854159691434247, |
| "cosine_spearman": 0.870781967302773, |
| "manhattan_pearson": 0.867700912246326, |
| "manhattan_spearman": 0.8690795304788012, |
| "euclidean_pearson": 0.8699231026532712, |
| "euclidean_spearman": 0.8707819663065923, |
| "main_score": 0.870781967302773, |
| "hf_subset": "default", |
| "languages": [ |
| "eng-Latn" |
| ], |
| "mteb_version": "2.21.0" |
| }, |
| "small": { |
| "pearson": 0.765316203785143, |
| "spearman": 0.7729871863268857, |
| "cosine_pearson": 0.7653161983330018, |
| "cosine_spearman": 0.7729871863268857, |
| "manhattan_pearson": 0.7881395913225815, |
| "manhattan_spearman": 0.7716275619454571, |
| "euclidean_pearson": 0.789916662830987, |
| "euclidean_spearman": 0.7729857841479683, |
| "main_score": 0.7729871863268857, |
| "hf_subset": "default", |
| "languages": [ |
| "eng-Latn" |
| ], |
| "mteb_version": "2.21.0" |
| } |
| }, |
| "measurement_by_model": { |
| "small": { |
| "evaluated_native_fingerprint": "684fe0c48a4ddd3944167f6e5dc75e97ada24e5b120bd9a2921f00dc03112eeb", |
| "original_checkpoint_revision": "fdf8e01b7b0f3a69ac1ac8e2a64dcb1ede177ba4", |
| "raw_result_sha256": "ea0772149930250452298e3525b3405ce4fd5caed4aaf676f3745d4555692a07", |
| "kind": "direct_frozen_encoder_evaluation" |
| }, |
| "nano": { |
| "evaluated_native_fingerprint": "96d76f97478e114b816df3ab1b3c6947e4c5ef0babfb40ab865d7e3c838d34c4", |
| "applicable_native_fingerprint": "474ad3d52f6ee7f818a3dbb6112e3d66c2423542d08228e7996a8cd600bd99a2", |
| "kind": "exact_component_applicability" |
| } |
| }, |
| "evaluated_native_fingerprint": "96d76f97478e114b816df3ab1b3c6947e4c5ef0babfb40ab865d7e3c838d34c4", |
| "applicable_native_fingerprint": "474ad3d52f6ee7f818a3dbb6112e3d66c2423542d08228e7996a8cd600bd99a2", |
| "measurement_kind": "exact_component_applicability" |
| }, |
| { |
| "task": "ArguAna", |
| "type": "Retrieval", |
| "modalities": [ |
| "text" |
| ], |
| "split": "test", |
| "hf_subset": "default", |
| "main_metric": "ndcg_at_10", |
| "dataset": { |
| "path": "mteb/arguana", |
| "revision": "c22ab2a51041ffd869aaddef7af8d8215647e41a" |
| }, |
| "counts": { |
| "queries": 1406, |
| "documents": 8674 |
| }, |
| "scores": { |
| "nano": 0.59246, |
| "small": 0.32889 |
| }, |
| "metrics": { |
| "nano": { |
| "ndcg_at_1": 0.35349, |
| "ndcg_at_3": 0.49665, |
| "ndcg_at_5": 0.54581, |
| "ndcg_at_10": 0.59246, |
| "ndcg_at_20": 0.6132, |
| "ndcg_at_100": 0.62189, |
| "ndcg_at_1000": 0.62242, |
| "map_at_1": 0.35349, |
| "map_at_3": 0.46005, |
| "map_at_5": 0.48729, |
| "map_at_10": 0.50679, |
| "map_at_20": 0.51261, |
| "map_at_100": 0.51395, |
| "map_at_1000": 0.51397, |
| "recall_at_1": 0.35349, |
| "recall_at_3": 0.60313, |
| "recall_at_5": 0.72262, |
| "recall_at_10": 0.86558, |
| "recall_at_20": 0.94666, |
| "recall_at_100": 0.99147, |
| "recall_at_1000": 0.99573, |
| "accuracy": 0.35349, |
| "precision_at_1": 0.35349, |
| "precision_at_3": 0.20104, |
| "precision_at_5": 0.14452, |
| "precision_at_10": 0.08656, |
| "precision_at_20": 0.04733, |
| "precision_at_100": 0.00991, |
| "precision_at_1000": 0.001, |
| "mrr_at_1": 0.3534850640113798, |
| "mrr_at_3": 0.4600521574205785, |
| "mrr_at_5": 0.48729255571360836, |
| "mrr_at_10": 0.5067903429745535, |
| "mrr_at_20": 0.5126066994732466, |
| "mrr_at_100": 0.5139513666778618, |
| "mrr_at_1000": 0.5139717342260931, |
| "nauc_ndcg_at_1_max": -0.11747299439249903, |
| "nauc_ndcg_at_1_std": -0.11436695908554317, |
| "nauc_ndcg_at_1_diff1": 0.12406213904460131, |
| "nauc_ndcg_at_3_max": -0.0810374905617702, |
| "nauc_ndcg_at_3_std": -0.10902178800246173, |
| "nauc_ndcg_at_3_diff1": 0.09457199601062195, |
| "nauc_ndcg_at_5_max": -0.08485395980147033, |
| "nauc_ndcg_at_5_std": -0.10485663186916257, |
| "nauc_ndcg_at_5_diff1": 0.08582902365914505, |
| "nauc_ndcg_at_10_max": -0.08309974111344137, |
| "nauc_ndcg_at_10_std": -0.11506227102720018, |
| "nauc_ndcg_at_10_diff1": 0.08990562536626041, |
| "nauc_ndcg_at_20_max": -0.0791935521931642, |
| "nauc_ndcg_at_20_std": -0.10319594721919796, |
| "nauc_ndcg_at_20_diff1": 0.09971433495514617, |
| "nauc_ndcg_at_100_max": -0.08635224865444148, |
| "nauc_ndcg_at_100_std": -0.10462787774486179, |
| "nauc_ndcg_at_100_diff1": 0.09791984963152253, |
| "nauc_ndcg_at_1000_max": -0.0879373318368675, |
| "nauc_ndcg_at_1000_std": -0.10553141137460995, |
| "nauc_ndcg_at_1000_diff1": 0.09704987719570578, |
| "nauc_map_at_1_max": -0.11747299439249903, |
| "nauc_map_at_1_std": -0.11436695908554317, |
| "nauc_map_at_1_diff1": 0.12406213904460131, |
| "nauc_map_at_3_max": -0.09064151313996319, |
| "nauc_map_at_3_std": -0.1105170515941861, |
| "nauc_map_at_3_diff1": 0.09940664377727629, |
| "nauc_map_at_5_max": -0.09342459992783216, |
| "nauc_map_at_5_std": -0.10864465771524179, |
| "nauc_map_at_5_diff1": 0.09517709876339578, |
| "nauc_map_at_10_max": -0.09304942751697344, |
| "nauc_map_at_10_std": -0.11264010939720397, |
| "nauc_map_at_10_diff1": 0.09707068152210184, |
| "nauc_map_at_20_max": -0.09221792922323616, |
| "nauc_map_at_20_std": -0.10961211427408772, |
| "nauc_map_at_20_diff1": 0.09943255521491648, |
| "nauc_map_at_100_max": -0.09316799067959038, |
| "nauc_map_at_100_std": -0.10968589627543163, |
| "nauc_map_at_100_diff1": 0.09910011165040408, |
| "nauc_map_at_1000_max": -0.09321385399598632, |
| "nauc_map_at_1000_std": -0.10970513219025511, |
| "nauc_map_at_1000_diff1": 0.09907801272427635, |
| "nauc_recall_at_1_max": -0.11747299439249903, |
| "nauc_recall_at_1_std": -0.11436695908554317, |
| "nauc_recall_at_1_diff1": 0.12406213904460131, |
| "nauc_recall_at_3_max": -0.05070404620274185, |
| "nauc_recall_at_3_std": -0.10426214370217692, |
| "nauc_recall_at_3_diff1": 0.0805612409902579, |
| "nauc_recall_at_5_max": -0.051784236042595515, |
| "nauc_recall_at_5_std": -0.0889570674134141, |
| "nauc_recall_at_5_diff1": 0.049390111709494505, |
| "nauc_recall_at_10_max": -0.013727385294040483, |
| "nauc_recall_at_10_std": -0.13798296178002195, |
| "nauc_recall_at_10_diff1": 0.044023090224588696, |
| "nauc_recall_at_20_max": 0.15251226818625876, |
| "nauc_recall_at_20_std": 0.03363980242022382, |
| "nauc_recall_at_20_diff1": 0.15796490326794294, |
| "nauc_recall_at_100_max": 0.3311354440025815, |
| "nauc_recall_at_100_std": 0.4782526238266158, |
| "nauc_recall_at_100_diff1": 0.25005005298352057, |
| "nauc_recall_at_1000_max": 0.02172236447995406, |
| "nauc_recall_at_1000_std": 0.609819560470231, |
| "nauc_recall_at_1000_diff1": -0.005495147382062338, |
| "nauc_precision_at_1_max": -0.11747299439249903, |
| "nauc_precision_at_1_std": -0.11436695908554317, |
| "nauc_precision_at_1_diff1": 0.12406213904460131, |
| "nauc_precision_at_3_max": -0.05070404620274239, |
| "nauc_precision_at_3_std": -0.10426214370217736, |
| "nauc_precision_at_3_diff1": 0.08056124099025751, |
| "nauc_precision_at_5_max": -0.05178423604259566, |
| "nauc_precision_at_5_std": -0.08895706741341412, |
| "nauc_precision_at_5_diff1": 0.04939011170949452, |
| "nauc_precision_at_10_max": -0.013727385294041244, |
| "nauc_precision_at_10_std": -0.13798296178002226, |
| "nauc_precision_at_10_diff1": 0.044023090224589216, |
| "nauc_precision_at_20_max": 0.15251226818625896, |
| "nauc_precision_at_20_std": 0.03363980242022203, |
| "nauc_precision_at_20_diff1": 0.15796490326794188, |
| "nauc_precision_at_100_max": 0.3311354440025917, |
| "nauc_precision_at_100_std": 0.4782526238266313, |
| "nauc_precision_at_100_diff1": 0.25005005298354305, |
| "nauc_precision_at_1000_max": 0.0217223644799946, |
| "nauc_precision_at_1000_std": 0.6098195604702127, |
| "nauc_precision_at_1000_diff1": -0.0054951473820258456, |
| "nauc_mrr_at_1_max": -0.11747299439249903, |
| "nauc_mrr_at_1_std": -0.11436695908554317, |
| "nauc_mrr_at_1_diff1": 0.12406213904460131, |
| "nauc_mrr_at_3_max": -0.09064151313996319, |
| "nauc_mrr_at_3_std": -0.1105170515941861, |
| "nauc_mrr_at_3_diff1": 0.09940664377727629, |
| "nauc_mrr_at_5_max": -0.09342459992783216, |
| "nauc_mrr_at_5_std": -0.10864465771524179, |
| "nauc_mrr_at_5_diff1": 0.09517709876339578, |
| "nauc_mrr_at_10_max": -0.09304942751697344, |
| "nauc_mrr_at_10_std": -0.11264010939720397, |
| "nauc_mrr_at_10_diff1": 0.09707068152210184, |
| "nauc_mrr_at_20_max": -0.09221792922323616, |
| "nauc_mrr_at_20_std": -0.10961211427408772, |
| "nauc_mrr_at_20_diff1": 0.09943255521491648, |
| "nauc_mrr_at_100_max": -0.09316799067959038, |
| "nauc_mrr_at_100_std": -0.10968589627543163, |
| "nauc_mrr_at_100_diff1": 0.09910011165040408, |
| "nauc_mrr_at_1000_max": -0.09321385399598632, |
| "nauc_mrr_at_1000_std": -0.10970513219025511, |
| "nauc_mrr_at_1000_diff1": 0.09907801272427635, |
| "hit_rate_at_1": 0.35349, |
| "hit_rate_at_3": 0.60313, |
| "hit_rate_at_5": 0.72262, |
| "hit_rate_at_10": 0.86558, |
| "hit_rate_at_20": 0.94666, |
| "hit_rate_at_100": 0.99147, |
| "hit_rate_at_1000": 0.99573, |
| "main_score": 0.59246, |
| "hf_subset": "default", |
| "languages": [ |
| "eng-Latn" |
| ], |
| "mteb_version": "2.21.0" |
| }, |
| "small": { |
| "ndcg_at_1": 0.1458, |
| "ndcg_at_3": 0.23456, |
| "ndcg_at_5": 0.27835, |
| "ndcg_at_10": 0.32889, |
| "ndcg_at_20": 0.36967, |
| "ndcg_at_100": 0.4045, |
| "ndcg_at_1000": 0.41446, |
| "map_at_1": 0.1458, |
| "map_at_3": 0.21136, |
| "map_at_5": 0.2354, |
| "map_at_10": 0.25622, |
| "map_at_20": 0.26767, |
| "map_at_100": 0.27279, |
| "map_at_1000": 0.27315, |
| "recall_at_1": 0.1458, |
| "recall_at_3": 0.30228, |
| "recall_at_5": 0.40967, |
| "recall_at_10": 0.56615, |
| "recall_at_20": 0.72546, |
| "recall_at_100": 0.90896, |
| "recall_at_1000": 0.98791, |
| "accuracy": 0.1458, |
| "precision_at_1": 0.1458, |
| "precision_at_3": 0.10076, |
| "precision_at_5": 0.08193, |
| "precision_at_10": 0.05661, |
| "precision_at_20": 0.03627, |
| "precision_at_100": 0.00909, |
| "precision_at_1000": 0.00099, |
| "mrr_at_1": 0.14580369843527738, |
| "mrr_at_3": 0.21135609293504032, |
| "mrr_at_5": 0.2353959222380275, |
| "mrr_at_10": 0.25621881279776015, |
| "mrr_at_20": 0.26766780266556217, |
| "mrr_at_100": 0.27278502616211503, |
| "mrr_at_1000": 0.2731483695142967, |
| "nauc_ndcg_at_1_max": 0.07124074977242485, |
| "nauc_ndcg_at_1_std": 0.07043132234894327, |
| "nauc_ndcg_at_1_diff1": 0.17867294841183828, |
| "nauc_ndcg_at_3_max": 0.05391218816481269, |
| "nauc_ndcg_at_3_std": 0.08300022308422525, |
| "nauc_ndcg_at_3_diff1": 0.13721153170379125, |
| "nauc_ndcg_at_5_max": 0.04761392610075392, |
| "nauc_ndcg_at_5_std": 0.09330318818026477, |
| "nauc_ndcg_at_5_diff1": 0.129721108359703, |
| "nauc_ndcg_at_10_max": 0.06398717148477524, |
| "nauc_ndcg_at_10_std": 0.11739949823329376, |
| "nauc_ndcg_at_10_diff1": 0.1353789207379315, |
| "nauc_ndcg_at_20_max": 0.06583701467786816, |
| "nauc_ndcg_at_20_std": 0.11760807459667257, |
| "nauc_ndcg_at_20_diff1": 0.1464058778549573, |
| "nauc_ndcg_at_100_max": 0.07271569743212124, |
| "nauc_ndcg_at_100_std": 0.12767735668965335, |
| "nauc_ndcg_at_100_diff1": 0.14440559200255987, |
| "nauc_ndcg_at_1000_max": 0.06760834890113898, |
| "nauc_ndcg_at_1000_std": 0.11565058305226664, |
| "nauc_ndcg_at_1000_diff1": 0.14440414116396014, |
| "nauc_map_at_1_max": 0.07124074977242485, |
| "nauc_map_at_1_std": 0.07043132234894327, |
| "nauc_map_at_1_diff1": 0.17867294841183828, |
| "nauc_map_at_3_max": 0.05536834016676611, |
| "nauc_map_at_3_std": 0.07918800842578326, |
| "nauc_map_at_3_diff1": 0.14582401585695304, |
| "nauc_map_at_5_max": 0.05153209428882134, |
| "nauc_map_at_5_std": 0.08553839884319275, |
| "nauc_map_at_5_diff1": 0.1412262351733516, |
| "nauc_map_at_10_max": 0.05764323340595916, |
| "nauc_map_at_10_std": 0.09567768210275003, |
| "nauc_map_at_10_diff1": 0.14384793290679312, |
| "nauc_map_at_20_max": 0.058797567381990594, |
| "nauc_map_at_20_std": 0.09556268067232435, |
| "nauc_map_at_20_diff1": 0.14719660481591418, |
| "nauc_map_at_100_max": 0.060341050948804, |
| "nauc_map_at_100_std": 0.09715443446030105, |
| "nauc_map_at_100_diff1": 0.14729508200613933, |
| "nauc_map_at_1000_max": 0.060216285099770145, |
| "nauc_map_at_1000_std": 0.09674391170833818, |
| "nauc_map_at_1000_diff1": 0.14729630480806993, |
| "nauc_recall_at_1_max": 0.07124074977242485, |
| "nauc_recall_at_1_std": 0.07043132234894327, |
| "nauc_recall_at_1_diff1": 0.17867294841183828, |
| "nauc_recall_at_3_max": 0.05116463248307958, |
| "nauc_recall_at_3_std": 0.09233493207170033, |
| "nauc_recall_at_3_diff1": 0.11697591005218377, |
| "nauc_recall_at_5_max": 0.0387185231377629, |
| "nauc_recall_at_5_std": 0.11230251236242163, |
| "nauc_recall_at_5_diff1": 0.10273687046821164, |
| "nauc_recall_at_10_max": 0.08768343797110026, |
| "nauc_recall_at_10_std": 0.18174312026830694, |
| "nauc_recall_at_10_diff1": 0.11544199093498575, |
| "nauc_recall_at_20_max": 0.0976177919930251, |
| "nauc_recall_at_20_std": 0.20309410782757367, |
| "nauc_recall_at_20_diff1": 0.15596172000438252, |
| "nauc_recall_at_100_max": 0.21399140718760357, |
| "nauc_recall_at_100_std": 0.45432867879288047, |
| "nauc_recall_at_100_diff1": 0.13074733185800497, |
| "nauc_recall_at_1000_max": 0.29184801184590964, |
| "nauc_recall_at_1000_std": 0.7268160753847613, |
| "nauc_recall_at_1000_diff1": 0.04649951766999351, |
| "nauc_precision_at_1_max": 0.07124074977242485, |
| "nauc_precision_at_1_std": 0.07043132234894327, |
| "nauc_precision_at_1_diff1": 0.17867294841183828, |
| "nauc_precision_at_3_max": 0.051164632483079536, |
| "nauc_precision_at_3_std": 0.09233493207170006, |
| "nauc_precision_at_3_diff1": 0.11697591005218336, |
| "nauc_precision_at_5_max": 0.03871852313776277, |
| "nauc_precision_at_5_std": 0.11230251236242186, |
| "nauc_precision_at_5_diff1": 0.10273687046821152, |
| "nauc_precision_at_10_max": 0.08768343797110059, |
| "nauc_precision_at_10_std": 0.18174312026830697, |
| "nauc_precision_at_10_diff1": 0.11544199093498574, |
| "nauc_precision_at_20_max": 0.09761779199302502, |
| "nauc_precision_at_20_std": 0.20309410782757284, |
| "nauc_precision_at_20_diff1": 0.15596172000438274, |
| "nauc_precision_at_100_max": 0.2139914071876045, |
| "nauc_precision_at_100_std": 0.45432867879288213, |
| "nauc_precision_at_100_diff1": 0.13074733185800294, |
| "nauc_precision_at_1000_max": 0.29184801184591147, |
| "nauc_precision_at_1000_std": 0.7268160753847517, |
| "nauc_precision_at_1000_diff1": 0.04649951766998672, |
| "nauc_mrr_at_1_max": 0.07124074977242485, |
| "nauc_mrr_at_1_std": 0.07043132234894327, |
| "nauc_mrr_at_1_diff1": 0.17867294841183828, |
| "nauc_mrr_at_3_max": 0.05536834016676611, |
| "nauc_mrr_at_3_std": 0.07918800842578326, |
| "nauc_mrr_at_3_diff1": 0.14582401585695304, |
| "nauc_mrr_at_5_max": 0.05153209428882134, |
| "nauc_mrr_at_5_std": 0.08553839884319275, |
| "nauc_mrr_at_5_diff1": 0.1412262351733516, |
| "nauc_mrr_at_10_max": 0.05764323340595916, |
| "nauc_mrr_at_10_std": 0.09567768210275003, |
| "nauc_mrr_at_10_diff1": 0.14384793290679312, |
| "nauc_mrr_at_20_max": 0.058797567381990594, |
| "nauc_mrr_at_20_std": 0.09556268067232435, |
| "nauc_mrr_at_20_diff1": 0.14719660481591418, |
| "nauc_mrr_at_100_max": 0.060341050948804, |
| "nauc_mrr_at_100_std": 0.09715443446030105, |
| "nauc_mrr_at_100_diff1": 0.14729508200613933, |
| "nauc_mrr_at_1000_max": 0.060216285099770145, |
| "nauc_mrr_at_1000_std": 0.09674391170833818, |
| "nauc_mrr_at_1000_diff1": 0.14729630480806993, |
| "hit_rate_at_1": 0.1458, |
| "hit_rate_at_3": 0.30228, |
| "hit_rate_at_5": 0.40967, |
| "hit_rate_at_10": 0.56615, |
| "hit_rate_at_20": 0.72546, |
| "hit_rate_at_100": 0.90896, |
| "hit_rate_at_1000": 0.98791, |
| "main_score": 0.32889, |
| "hf_subset": "default", |
| "languages": [ |
| "eng-Latn" |
| ], |
| "mteb_version": "2.21.0" |
| } |
| }, |
| "measurement_by_model": { |
| "small": { |
| "evaluated_native_fingerprint": "684fe0c48a4ddd3944167f6e5dc75e97ada24e5b120bd9a2921f00dc03112eeb", |
| "original_checkpoint_revision": "fdf8e01b7b0f3a69ac1ac8e2a64dcb1ede177ba4", |
| "raw_result_sha256": "92cbe96f6e84ebd0bc144001424ac46bb3a630ef64bffb433c2cdd63a10c43ec", |
| "kind": "direct_frozen_encoder_evaluation" |
| }, |
| "nano": { |
| "evaluated_native_fingerprint": "96d76f97478e114b816df3ab1b3c6947e4c5ef0babfb40ab865d7e3c838d34c4", |
| "applicable_native_fingerprint": "474ad3d52f6ee7f818a3dbb6112e3d66c2423542d08228e7996a8cd600bd99a2", |
| "kind": "exact_component_applicability" |
| } |
| }, |
| "evaluated_native_fingerprint": "96d76f97478e114b816df3ab1b3c6947e4c5ef0babfb40ab865d7e3c838d34c4", |
| "applicable_native_fingerprint": "474ad3d52f6ee7f818a3dbb6112e3d66c2423542d08228e7996a8cd600bd99a2", |
| "measurement_kind": "exact_component_applicability" |
| }, |
| { |
| "task": "TwentyNewsgroupsClustering.v2", |
| "type": "Clustering", |
| "modalities": [ |
| "text" |
| ], |
| "split": "test", |
| "hf_subset": "default", |
| "main_metric": "v_measure", |
| "dataset": { |
| "path": "mteb/twentynewsgroups-clustering", |
| "revision": "6125ec4e24fa026cec8a478383ee943acfbd5449" |
| }, |
| "counts": { |
| "population_rows": 59545, |
| "official_encoded_rows": 2381, |
| "official_encoding_fraction": 0.04, |
| "bootstrap_clustering_trials": 10, |
| "resampled_items_per_trial": 16384 |
| }, |
| "scores": { |
| "nano": 0.5087260708286914, |
| "small": 0.44840076500124326 |
| }, |
| "metrics": { |
| "nano": { |
| "v_measures": { |
| "Level 0": [ |
| 0.4966639569479829, |
| 0.5210509442892017, |
| 0.5333989625396617, |
| 0.5094460445939109, |
| 0.5208429535416115, |
| 0.5012604404318527, |
| 0.5210119744629907, |
| 0.4617825059979706, |
| 0.5226704101107219, |
| 0.49913251537100806 |
| ] |
| }, |
| "v_measure": 0.5087260708286914, |
| "v_measure_std": 0.01936580527644773, |
| "ami_scores": { |
| "Level 0": [ |
| 0.49471495769938123, |
| 0.5192201304733433, |
| 0.5316138913144499, |
| 0.5075814211955173, |
| 0.5190252719774525, |
| 0.49937824878079495, |
| 0.5191850550712727, |
| 0.4597515065381579, |
| 0.5208311166380218, |
| 0.49721166070769024 |
| ] |
| }, |
| "ami": 0.5068513260396083, |
| "ami_std": 0.01943429917113934, |
| "main_score": 0.5087260708286914, |
| "hf_subset": "default", |
| "languages": [ |
| "eng-Latn" |
| ], |
| "mteb_version": "2.21.0" |
| }, |
| "small": { |
| "v_measures": { |
| "Level 0": [ |
| 0.46653683369731075, |
| 0.46464237680676646, |
| 0.4425518472920712, |
| 0.4288571327957099, |
| 0.4336826969564481, |
| 0.43371621267306154, |
| 0.43092596596465876, |
| 0.4536864467956253, |
| 0.4741214282291591, |
| 0.45528670880162087 |
| ] |
| }, |
| "v_measure": 0.44840076500124326, |
| "v_measure_std": 0.01575881885263055, |
| "ami_scores": { |
| "Level 0": [ |
| 0.46451147281180905, |
| 0.4625982506803001, |
| 0.44043666823974115, |
| 0.42670551677458946, |
| 0.43154706196129733, |
| 0.4315549515134722, |
| 0.4287870602853061, |
| 0.45163080092170965, |
| 0.47212403002394715, |
| 0.4532359410185627 |
| ] |
| }, |
| "ami": 0.44631317542307347, |
| "ami_std": 0.01581387159916619, |
| "main_score": 0.44840076500124326, |
| "hf_subset": "default", |
| "languages": [ |
| "eng-Latn" |
| ], |
| "mteb_version": "2.21.0" |
| } |
| }, |
| "measurement_by_model": { |
| "small": { |
| "evaluated_native_fingerprint": "684fe0c48a4ddd3944167f6e5dc75e97ada24e5b120bd9a2921f00dc03112eeb", |
| "original_checkpoint_revision": "fdf8e01b7b0f3a69ac1ac8e2a64dcb1ede177ba4", |
| "raw_result_sha256": "62fe52980cb4445b9969d53135fa1663dd8dcd811743a711dde55f2a097ced19", |
| "kind": "direct_frozen_encoder_evaluation" |
| }, |
| "nano": { |
| "evaluated_native_fingerprint": "96d76f97478e114b816df3ab1b3c6947e4c5ef0babfb40ab865d7e3c838d34c4", |
| "applicable_native_fingerprint": "474ad3d52f6ee7f818a3dbb6112e3d66c2423542d08228e7996a8cd600bd99a2", |
| "kind": "exact_component_applicability" |
| } |
| }, |
| "evaluated_native_fingerprint": "96d76f97478e114b816df3ab1b3c6947e4c5ef0babfb40ab865d7e3c838d34c4", |
| "applicable_native_fingerprint": "474ad3d52f6ee7f818a3dbb6112e3d66c2423542d08228e7996a8cd600bd99a2", |
| "measurement_kind": "exact_component_applicability" |
| }, |
| { |
| "task": "MTOPDomainClassification", |
| "type": "Classification", |
| "modalities": [ |
| "text" |
| ], |
| "split": "validation", |
| "hf_subset": "en", |
| "main_metric": "accuracy", |
| "dataset": { |
| "path": "mteb/MTOPDomainClassification", |
| "revision": "a76d16fae880597b9c73047b50159220a441cb54" |
| }, |
| "counts": { |
| "evaluation_rows": 2235, |
| "available_train_rows": 15667, |
| "official_probe_experiments": 10, |
| "probe_samples_per_label": 8 |
| }, |
| "scores": { |
| "nano": 0.9476510067114093, |
| "small": 0.8543624161073826 |
| }, |
| "metrics": { |
| "nano": { |
| "scores_per_experiment": [ |
| { |
| "accuracy": 0.9521252796420582, |
| "f1": 0.9509396968297774, |
| "f1_weighted": 0.9519302419648948, |
| "precision": 0.9493488952211522, |
| "precision_weighted": 0.9527659658117037, |
| "recall": 0.9535803496254757, |
| "recall_weighted": 0.9521252796420582, |
| "ap": null, |
| "ap_weighted": null |
| }, |
| { |
| "accuracy": 0.9458612975391499, |
| "f1": 0.9439475266607382, |
| "f1_weighted": 0.945767354216612, |
| "precision": 0.9420005561621522, |
| "precision_weighted": 0.9462895866502771, |
| "recall": 0.946397940529448, |
| "recall_weighted": 0.9458612975391499, |
| "ap": null, |
| "ap_weighted": null |
| }, |
| { |
| "accuracy": 0.9284116331096197, |
| "f1": 0.9284988704174789, |
| "f1_weighted": 0.9270840634554431, |
| "precision": 0.9271908775153325, |
| "precision_weighted": 0.9274634710035635, |
| "recall": 0.9317153602473662, |
| "recall_weighted": 0.9284116331096197, |
| "ap": null, |
| "ap_weighted": null |
| }, |
| { |
| "accuracy": 0.9525727069351231, |
| "f1": 0.950987470976036, |
| "f1_weighted": 0.9525185063610705, |
| "precision": 0.9504292297253194, |
| "precision_weighted": 0.9534501308715367, |
| "recall": 0.9526312322737922, |
| "recall_weighted": 0.9525727069351231, |
| "ap": null, |
| "ap_weighted": null |
| }, |
| { |
| "accuracy": 0.9445190156599552, |
| "f1": 0.943383064264026, |
| "f1_weighted": 0.9442288513061988, |
| "precision": 0.9424858291605631, |
| "precision_weighted": 0.944823795147547, |
| "recall": 0.945181115724642, |
| "recall_weighted": 0.9445190156599552, |
| "ap": null, |
| "ap_weighted": null |
| }, |
| { |
| "accuracy": 0.9525727069351231, |
| "f1": 0.949882476921466, |
| "f1_weighted": 0.9526817135384125, |
| "precision": 0.9498620235113855, |
| "precision_weighted": 0.9543297139412104, |
| "recall": 0.9514687067629995, |
| "recall_weighted": 0.9525727069351231, |
| "ap": null, |
| "ap_weighted": null |
| }, |
| { |
| "accuracy": 0.9498881431767338, |
| "f1": 0.9496336255013932, |
| "f1_weighted": 0.9495703695110613, |
| "precision": 0.9490015795096777, |
| "precision_weighted": 0.9511221301014349, |
| "recall": 0.9520470037532829, |
| "recall_weighted": 0.9498881431767338, |
| "ap": null, |
| "ap_weighted": null |
| }, |
| { |
| "accuracy": 0.9570469798657718, |
| "f1": 0.9557194953102311, |
| "f1_weighted": 0.9568992928300293, |
| "precision": 0.9544888722394763, |
| "precision_weighted": 0.9584505479016514, |
| "recall": 0.9586898382241177, |
| "recall_weighted": 0.9570469798657718, |
| "ap": null, |
| "ap_weighted": null |
| }, |
| { |
| "accuracy": 0.9494407158836689, |
| "f1": 0.9460535182757994, |
| "f1_weighted": 0.9492959546761313, |
| "precision": 0.9454088954645755, |
| "precision_weighted": 0.9499145785798135, |
| "recall": 0.9472953341577202, |
| "recall_weighted": 0.9494407158836689, |
| "ap": null, |
| "ap_weighted": null |
| }, |
| { |
| "accuracy": 0.9440715883668904, |
| "f1": 0.9431192151526329, |
| "f1_weighted": 0.9437397930273277, |
| "precision": 0.9435095731735328, |
| "precision_weighted": 0.9461253948734786, |
| "recall": 0.9455297617432863, |
| "recall_weighted": 0.9440715883668904, |
| "ap": null, |
| "ap_weighted": null |
| } |
| ], |
| "accuracy": 0.9476510067114093, |
| "f1": 0.9462164960309579, |
| "f1_weighted": 0.9473716140887183, |
| "precision": 0.9453726331683165, |
| "precision_weighted": 0.9484735314882217, |
| "recall": 0.948453664304213, |
| "recall_weighted": 0.9476510067114093, |
| "ap": null, |
| "ap_weighted": null, |
| "main_score": 0.9476510067114093, |
| "hf_subset": "en", |
| "languages": [ |
| "eng-Latn" |
| ], |
| "mteb_version": "2.21.0" |
| }, |
| "small": { |
| "scores_per_experiment": [ |
| { |
| "accuracy": 0.8250559284116331, |
| "f1": 0.8322814729997016, |
| "f1_weighted": 0.8240316200136756, |
| "precision": 0.8318540374193987, |
| "precision_weighted": 0.8346068877064119, |
| "recall": 0.8430207439526936, |
| "recall_weighted": 0.8250559284116331, |
| "ap": null, |
| "ap_weighted": null |
| }, |
| { |
| "accuracy": 0.8559284116331096, |
| "f1": 0.8552447478537527, |
| "f1_weighted": 0.8547401242258992, |
| "precision": 0.8515715568445064, |
| "precision_weighted": 0.8581578601876096, |
| "recall": 0.8640896175706192, |
| "recall_weighted": 0.8559284116331096, |
| "ap": null, |
| "ap_weighted": null |
| }, |
| { |
| "accuracy": 0.8304250559284116, |
| "f1": 0.8317375249321166, |
| "f1_weighted": 0.8274798893470849, |
| "precision": 0.8268066926411431, |
| "precision_weighted": 0.8285364690282124, |
| "recall": 0.8409132018240374, |
| "recall_weighted": 0.8304250559284116, |
| "ap": null, |
| "ap_weighted": null |
| }, |
| { |
| "accuracy": 0.8648769574944072, |
| "f1": 0.8678594298716482, |
| "f1_weighted": 0.8632422395451538, |
| "precision": 0.865870318819693, |
| "precision_weighted": 0.8646460721917335, |
| "recall": 0.8726494617104266, |
| "recall_weighted": 0.8648769574944072, |
| "ap": null, |
| "ap_weighted": null |
| }, |
| { |
| "accuracy": 0.8532438478747204, |
| "f1": 0.8558930928177336, |
| "f1_weighted": 0.8512711153646526, |
| "precision": 0.8501961024805756, |
| "precision_weighted": 0.8547737986953718, |
| "recall": 0.8657455326712605, |
| "recall_weighted": 0.8532438478747204, |
| "ap": null, |
| "ap_weighted": null |
| }, |
| { |
| "accuracy": 0.8402684563758389, |
| "f1": 0.8478991888671523, |
| "f1_weighted": 0.8396067623115454, |
| "precision": 0.8508550254092774, |
| "precision_weighted": 0.8598285030005444, |
| "recall": 0.862051388925805, |
| "recall_weighted": 0.8402684563758389, |
| "ap": null, |
| "ap_weighted": null |
| }, |
| { |
| "accuracy": 0.8747203579418344, |
| "f1": 0.8781373847485636, |
| "f1_weighted": 0.8731828255054377, |
| "precision": 0.8729737218664414, |
| "precision_weighted": 0.8758466535281065, |
| "recall": 0.8873478774616341, |
| "recall_weighted": 0.8747203579418344, |
| "ap": null, |
| "ap_weighted": null |
| }, |
| { |
| "accuracy": 0.876062639821029, |
| "f1": 0.8754580934866012, |
| "f1_weighted": 0.8749795409187, |
| "precision": 0.8730455332008387, |
| "precision_weighted": 0.8787904116312295, |
| "recall": 0.8820295010368394, |
| "recall_weighted": 0.876062639821029, |
| "ap": null, |
| "ap_weighted": null |
| }, |
| { |
| "accuracy": 0.858165548098434, |
| "f1": 0.8610701024195923, |
| "f1_weighted": 0.8573656937247496, |
| "precision": 0.8561531713932808, |
| "precision_weighted": 0.8662234353266065, |
| "recall": 0.8738853045389997, |
| "recall_weighted": 0.858165548098434, |
| "ap": null, |
| "ap_weighted": null |
| }, |
| { |
| "accuracy": 0.8648769574944072, |
| "f1": 0.8671414562971872, |
| "f1_weighted": 0.8628546885742888, |
| "precision": 0.8607074291115159, |
| "precision_weighted": 0.8655691598347531, |
| "recall": 0.8776421302043499, |
| "recall_weighted": 0.8648769574944072, |
| "ap": null, |
| "ap_weighted": null |
| } |
| ], |
| "accuracy": 0.8543624161073826, |
| "f1": 0.8572722494294049, |
| "f1_weighted": 0.8528754499531187, |
| "precision": 0.8540033589186671, |
| "precision_weighted": 0.8586979251130579, |
| "recall": 0.8669374759896666, |
| "recall_weighted": 0.8543624161073826, |
| "ap": null, |
| "ap_weighted": null, |
| "main_score": 0.8543624161073826, |
| "hf_subset": "en", |
| "languages": [ |
| "eng-Latn" |
| ], |
| "mteb_version": "2.21.0" |
| } |
| }, |
| "measurement_by_model": { |
| "small": { |
| "evaluated_native_fingerprint": "684fe0c48a4ddd3944167f6e5dc75e97ada24e5b120bd9a2921f00dc03112eeb", |
| "original_checkpoint_revision": "fdf8e01b7b0f3a69ac1ac8e2a64dcb1ede177ba4", |
| "raw_result_sha256": "256702650b93bd559d9a2a9b70f32b996b51e166debc0c20d63ce24cc6f605f6", |
| "kind": "direct_frozen_encoder_evaluation" |
| }, |
| "nano": { |
| "evaluated_native_fingerprint": "96d76f97478e114b816df3ab1b3c6947e4c5ef0babfb40ab865d7e3c838d34c4", |
| "applicable_native_fingerprint": "474ad3d52f6ee7f818a3dbb6112e3d66c2423542d08228e7996a8cd600bd99a2", |
| "kind": "exact_component_applicability" |
| } |
| }, |
| "evaluated_native_fingerprint": "96d76f97478e114b816df3ab1b3c6947e4c5ef0babfb40ab865d7e3c838d34c4", |
| "applicable_native_fingerprint": "474ad3d52f6ee7f818a3dbb6112e3d66c2423542d08228e7996a8cd600bd99a2", |
| "measurement_kind": "exact_component_applicability" |
| }, |
| { |
| "task": "MTOPDomainClassification", |
| "type": "Classification", |
| "modalities": [ |
| "text" |
| ], |
| "split": "test", |
| "hf_subset": "en", |
| "main_metric": "accuracy", |
| "dataset": { |
| "path": "mteb/MTOPDomainClassification", |
| "revision": "a76d16fae880597b9c73047b50159220a441cb54" |
| }, |
| "counts": { |
| "evaluation_rows": 4386, |
| "available_train_rows": 15667, |
| "official_probe_experiments": 10, |
| "probe_samples_per_label": 8 |
| }, |
| "scores": { |
| "nano": 0.9491792065663475, |
| "small": 0.8456452348381213 |
| }, |
| "metrics": { |
| "nano": { |
| "scores_per_experiment": [ |
| { |
| "accuracy": 0.9507523939808481, |
| "f1": 0.9490862282573499, |
| "f1_weighted": 0.9508003256061616, |
| "precision": 0.9476949859831407, |
| "precision_weighted": 0.9520734016967529, |
| "recall": 0.9517099502990664, |
| "recall_weighted": 0.9507523939808481, |
| "ap": null, |
| "ap_weighted": null |
| }, |
| { |
| "accuracy": 0.9473324213406292, |
| "f1": 0.9425946320317333, |
| "f1_weighted": 0.9473133266787116, |
| "precision": 0.9392105665237355, |
| "precision_weighted": 0.9479986101450497, |
| "recall": 0.9468765008841218, |
| "recall_weighted": 0.9473324213406292, |
| "ap": null, |
| "ap_weighted": null |
| }, |
| { |
| "accuracy": 0.9306885544915641, |
| "f1": 0.9261678157710709, |
| "f1_weighted": 0.9296159823715927, |
| "precision": 0.923024327342814, |
| "precision_weighted": 0.9307965712480242, |
| "recall": 0.932201810783149, |
| "recall_weighted": 0.9306885544915641, |
| "ap": null, |
| "ap_weighted": null |
| }, |
| { |
| "accuracy": 0.958048335613315, |
| "f1": 0.9552869971485034, |
| "f1_weighted": 0.9579502405954128, |
| "precision": 0.954624514001619, |
| "precision_weighted": 0.9582682331625527, |
| "recall": 0.9564289859937923, |
| "recall_weighted": 0.958048335613315, |
| "ap": null, |
| "ap_weighted": null |
| }, |
| { |
| "accuracy": 0.9464204286365709, |
| "f1": 0.94433525009326, |
| "f1_weighted": 0.9462665825311201, |
| "precision": 0.9422860464299823, |
| "precision_weighted": 0.9467645700971977, |
| "recall": 0.9469874285322217, |
| "recall_weighted": 0.9464204286365709, |
| "ap": null, |
| "ap_weighted": null |
| }, |
| { |
| "accuracy": 0.9505243958048336, |
| "f1": 0.9447844972663006, |
| "f1_weighted": 0.9507789369812125, |
| "precision": 0.9429136617255864, |
| "precision_weighted": 0.9524852717381878, |
| "recall": 0.9483171314260196, |
| "recall_weighted": 0.9505243958048336, |
| "ap": null, |
| "ap_weighted": null |
| }, |
| { |
| "accuracy": 0.9509803921568627, |
| "f1": 0.9480693879894481, |
| "f1_weighted": 0.9510087641402198, |
| "precision": 0.9447979434260977, |
| "precision_weighted": 0.9522624035738166, |
| "recall": 0.952376748039714, |
| "recall_weighted": 0.9509803921568627, |
| "ap": null, |
| "ap_weighted": null |
| }, |
| { |
| "accuracy": 0.9573643410852714, |
| "f1": 0.9551153583299494, |
| "f1_weighted": 0.9573535659550753, |
| "precision": 0.952401865813031, |
| "precision_weighted": 0.9583040343256545, |
| "recall": 0.9586994420587002, |
| "recall_weighted": 0.9573643410852714, |
| "ap": null, |
| "ap_weighted": null |
| }, |
| { |
| "accuracy": 0.9548563611491108, |
| "f1": 0.9495763118202308, |
| "f1_weighted": 0.9549499907192655, |
| "precision": 0.946851274794657, |
| "precision_weighted": 0.9555638233419986, |
| "recall": 0.9529512436069031, |
| "recall_weighted": 0.9548563611491108, |
| "ap": null, |
| "ap_weighted": null |
| }, |
| { |
| "accuracy": 0.9448244414044688, |
| "f1": 0.9427993333346522, |
| "f1_weighted": 0.9447713522464498, |
| "precision": 0.9432072583992069, |
| "precision_weighted": 0.9468986783847072, |
| "recall": 0.9447479153319868, |
| "recall_weighted": 0.9448244414044688, |
| "ap": null, |
| "ap_weighted": null |
| } |
| ], |
| "accuracy": 0.9491792065663475, |
| "f1": 0.9457815812042497, |
| "f1_weighted": 0.9490809067825221, |
| "precision": 0.9437012444439871, |
| "precision_weighted": 0.9501415597713943, |
| "recall": 0.9491297156955675, |
| "recall_weighted": 0.9491792065663475, |
| "ap": null, |
| "ap_weighted": null, |
| "main_score": 0.9491792065663475, |
| "hf_subset": "en", |
| "languages": [ |
| "eng-Latn" |
| ], |
| "mteb_version": "2.21.0" |
| }, |
| "small": { |
| "scores_per_experiment": [ |
| { |
| "accuracy": 0.8180574555403557, |
| "f1": 0.8207347394289347, |
| "f1_weighted": 0.8170857264694432, |
| "precision": 0.820363266003151, |
| "precision_weighted": 0.8301897328408723, |
| "recall": 0.8338753807988755, |
| "recall_weighted": 0.8180574555403557, |
| "ap": null, |
| "ap_weighted": null |
| }, |
| { |
| "accuracy": 0.8449612403100775, |
| "f1": 0.839516667111528, |
| "f1_weighted": 0.8449410274873432, |
| "precision": 0.8369796042511596, |
| "precision_weighted": 0.8536873756717578, |
| "recall": 0.8523590480814156, |
| "recall_weighted": 0.8449612403100775, |
| "ap": null, |
| "ap_weighted": null |
| }, |
| { |
| "accuracy": 0.8258093935248518, |
| "f1": 0.8185934728273051, |
| "f1_weighted": 0.8251774946806023, |
| "precision": 0.8147370918656317, |
| "precision_weighted": 0.8330066052370535, |
| "recall": 0.8324642506433119, |
| "recall_weighted": 0.8258093935248518, |
| "ap": null, |
| "ap_weighted": null |
| }, |
| { |
| "accuracy": 0.8497492020063839, |
| "f1": 0.8486953297726213, |
| "f1_weighted": 0.848088172519004, |
| "precision": 0.8467111465364261, |
| "precision_weighted": 0.8501008589343039, |
| "recall": 0.853578648077031, |
| "recall_weighted": 0.8497492020063839, |
| "ap": null, |
| "ap_weighted": null |
| }, |
| { |
| "accuracy": 0.8481532147742818, |
| "f1": 0.8467468733444872, |
| "f1_weighted": 0.8467448330993773, |
| "precision": 0.8393119909268087, |
| "precision_weighted": 0.8506182695179204, |
| "recall": 0.858498685671118, |
| "recall_weighted": 0.8481532147742818, |
| "ap": null, |
| "ap_weighted": null |
| }, |
| { |
| "accuracy": 0.8326493388052896, |
| "f1": 0.8352387297727578, |
| "f1_weighted": 0.8347572800634613, |
| "precision": 0.8420129591399498, |
| "precision_weighted": 0.8638673888380122, |
| "recall": 0.8534348314007274, |
| "recall_weighted": 0.8326493388052896, |
| "ap": null, |
| "ap_weighted": null |
| }, |
| { |
| "accuracy": 0.8682170542635659, |
| "f1": 0.8648438659784241, |
| "f1_weighted": 0.8680876089933638, |
| "precision": 0.8585183480506728, |
| "precision_weighted": 0.876371854401442, |
| "recall": 0.8802432046324339, |
| "recall_weighted": 0.8682170542635659, |
| "ap": null, |
| "ap_weighted": null |
| }, |
| { |
| "accuracy": 0.8518011855905153, |
| "f1": 0.8484079910361157, |
| "f1_weighted": 0.8509906563529788, |
| "precision": 0.8443896101449812, |
| "precision_weighted": 0.860550178646213, |
| "recall": 0.8607698617029648, |
| "recall_weighted": 0.8518011855905153, |
| "ap": null, |
| "ap_weighted": null |
| }, |
| { |
| "accuracy": 0.8556771545827634, |
| "f1": 0.8515433915603542, |
| "f1_weighted": 0.856278856298033, |
| "precision": 0.84610847827307, |
| "precision_weighted": 0.8661492710958929, |
| "recall": 0.8658189317867632, |
| "recall_weighted": 0.8556771545827634, |
| "ap": null, |
| "ap_weighted": null |
| }, |
| { |
| "accuracy": 0.8613771089831281, |
| "f1": 0.8586787981057005, |
| "f1_weighted": 0.8603956735839339, |
| "precision": 0.8518135650230533, |
| "precision_weighted": 0.8659927155990171, |
| "recall": 0.8717255305513517, |
| "recall_weighted": 0.8613771089831281, |
| "ap": null, |
| "ap_weighted": null |
| } |
| ], |
| "accuracy": 0.8456452348381213, |
| "f1": 0.8432999858938229, |
| "f1_weighted": 0.8452547329547541, |
| "precision": 0.8400946060214904, |
| "precision_weighted": 0.8550534250782486, |
| "recall": 0.8562768373345993, |
| "recall_weighted": 0.8456452348381213, |
| "ap": null, |
| "ap_weighted": null, |
| "main_score": 0.8456452348381213, |
| "hf_subset": "en", |
| "languages": [ |
| "eng-Latn" |
| ], |
| "mteb_version": "2.21.0" |
| } |
| }, |
| "measurement_by_model": { |
| "small": { |
| "evaluated_native_fingerprint": "684fe0c48a4ddd3944167f6e5dc75e97ada24e5b120bd9a2921f00dc03112eeb", |
| "original_checkpoint_revision": "fdf8e01b7b0f3a69ac1ac8e2a64dcb1ede177ba4", |
| "raw_result_sha256": "256702650b93bd559d9a2a9b70f32b996b51e166debc0c20d63ce24cc6f605f6", |
| "kind": "direct_frozen_encoder_evaluation" |
| }, |
| "nano": { |
| "evaluated_native_fingerprint": "96d76f97478e114b816df3ab1b3c6947e4c5ef0babfb40ab865d7e3c838d34c4", |
| "applicable_native_fingerprint": "474ad3d52f6ee7f818a3dbb6112e3d66c2423542d08228e7996a8cd600bd99a2", |
| "kind": "exact_component_applicability" |
| } |
| }, |
| "evaluated_native_fingerprint": "96d76f97478e114b816df3ab1b3c6947e4c5ef0babfb40ab865d7e3c838d34c4", |
| "applicable_native_fingerprint": "474ad3d52f6ee7f818a3dbb6112e3d66c2423542d08228e7996a8cd600bd99a2", |
| "measurement_kind": "exact_component_applicability" |
| }, |
| { |
| "task": "OxfordPets", |
| "type": "ImageClassification", |
| "modalities": [ |
| "image" |
| ], |
| "split": "test", |
| "hf_subset": "default", |
| "main_metric": "accuracy", |
| "dataset": { |
| "path": "mteb/OxfordPets", |
| "revision": "557b480fae8d69247be74d9503b378a09425096f" |
| }, |
| "counts": { |
| "evaluation_rows": 3669, |
| "available_train_rows": 3680, |
| "official_probe_experiments": 5, |
| "probe_samples_per_label": 16 |
| }, |
| "scores": { |
| "nano": 0.9126737530662306, |
| "small": 0.4184791496320523 |
| }, |
| "metrics": { |
| "nano": { |
| "scores_per_experiment": [ |
| { |
| "accuracy": 0.9062414826928318, |
| "f1": 0.9045301316021502, |
| "f1_weighted": 0.9050055409371512, |
| "precision": 0.9099213291951324, |
| "precision_weighted": 0.9109544817876485, |
| "recall": 0.9064295837897134, |
| "recall_weighted": 0.9062414826928318, |
| "ap": null, |
| "ap_weighted": null |
| }, |
| { |
| "accuracy": 0.914418097574271, |
| "f1": 0.9126724638115935, |
| "f1_weighted": 0.9132186480967779, |
| "precision": 0.9192904646518116, |
| "precision_weighted": 0.9203471940294945, |
| "recall": 0.9144593228002407, |
| "recall_weighted": 0.914418097574271, |
| "ap": null, |
| "ap_weighted": null |
| }, |
| { |
| "accuracy": 0.9187789588443718, |
| "f1": 0.9177765846692281, |
| "f1_weighted": 0.9184111765200857, |
| "precision": 0.9220755461580387, |
| "precision_weighted": 0.9229278273528245, |
| "recall": 0.9183739103921627, |
| "recall_weighted": 0.9187789588443718, |
| "ap": null, |
| "ap_weighted": null |
| }, |
| { |
| "accuracy": 0.9174161896974653, |
| "f1": 0.9147622354932347, |
| "f1_weighted": 0.9154663340384108, |
| "precision": 0.9209055798643112, |
| "precision_weighted": 0.9221124471295344, |
| "recall": 0.9173606715378323, |
| "recall_weighted": 0.9174161896974653, |
| "ap": null, |
| "ap_weighted": null |
| }, |
| { |
| "accuracy": 0.9065140365222132, |
| "f1": 0.9051669100036687, |
| "f1_weighted": 0.9057728210054364, |
| "precision": 0.9098426973534027, |
| "precision_weighted": 0.9105143284060627, |
| "recall": 0.9059560197061645, |
| "recall_weighted": 0.9065140365222132, |
| "ap": null, |
| "ap_weighted": null |
| } |
| ], |
| "accuracy": 0.9126737530662306, |
| "f1": 0.910981665115975, |
| "f1_weighted": 0.9115749041195725, |
| "precision": 0.9164071234445392, |
| "precision_weighted": 0.9173712557411129, |
| "recall": 0.9125159016452227, |
| "recall_weighted": 0.9126737530662306, |
| "ap": null, |
| "ap_weighted": null, |
| "main_score": 0.9126737530662306, |
| "hf_subset": "default", |
| "languages": [ |
| "eng-Latn" |
| ], |
| "mteb_version": "2.21.0" |
| }, |
| "small": { |
| "scores_per_experiment": [ |
| { |
| "accuracy": 0.4148269283183429, |
| "f1": 0.40253654878032713, |
| "f1_weighted": 0.4024461757957877, |
| "precision": 0.4116431114014033, |
| "precision_weighted": 0.4119318458992588, |
| "recall": 0.4153030284766479, |
| "recall_weighted": 0.4148269283183429, |
| "ap": null, |
| "ap_weighted": null |
| }, |
| { |
| "accuracy": 0.4265467429817389, |
| "f1": 0.4174251045251405, |
| "f1_weighted": 0.4176529394767131, |
| "precision": 0.43921297726376096, |
| "precision_weighted": 0.4396338060561412, |
| "recall": 0.42629825783951975, |
| "recall_weighted": 0.4265467429817389, |
| "ap": null, |
| "ap_weighted": null |
| }, |
| { |
| "accuracy": 0.4088307440719542, |
| "f1": 0.3954869417563968, |
| "f1_weighted": 0.39542431005304346, |
| "precision": 0.40981373804908927, |
| "precision_weighted": 0.40980533913060346, |
| "recall": 0.40880378402552453, |
| "recall_weighted": 0.4088307440719542, |
| "ap": null, |
| "ap_weighted": null |
| }, |
| { |
| "accuracy": 0.4377214499863723, |
| "f1": 0.4251494451023654, |
| "f1_weighted": 0.4253356439185362, |
| "precision": 0.438981154791822, |
| "precision_weighted": 0.439719429760714, |
| "recall": 0.4381658372912861, |
| "recall_weighted": 0.4377214499863723, |
| "ap": null, |
| "ap_weighted": null |
| }, |
| { |
| "accuracy": 0.40446988280185336, |
| "f1": 0.3806382057679881, |
| "f1_weighted": 0.380430443085735, |
| "precision": 0.4088810121814978, |
| "precision_weighted": 0.40903302714302686, |
| "recall": 0.4046956272962792, |
| "recall_weighted": 0.40446988280185336, |
| "ap": null, |
| "ap_weighted": null |
| } |
| ], |
| "accuracy": 0.4184791496320523, |
| "f1": 0.4042472491864436, |
| "f1_weighted": 0.40425790246596305, |
| "precision": 0.42170639873751464, |
| "precision_weighted": 0.4220246895979488, |
| "recall": 0.4186533069858515, |
| "recall_weighted": 0.4184791496320523, |
| "ap": null, |
| "ap_weighted": null, |
| "main_score": 0.4184791496320523, |
| "hf_subset": "default", |
| "languages": [ |
| "eng-Latn" |
| ], |
| "mteb_version": "2.21.0" |
| } |
| }, |
| "measurement_by_model": { |
| "small": { |
| "evaluated_native_fingerprint": "684fe0c48a4ddd3944167f6e5dc75e97ada24e5b120bd9a2921f00dc03112eeb", |
| "original_checkpoint_revision": "fdf8e01b7b0f3a69ac1ac8e2a64dcb1ede177ba4", |
| "raw_result_sha256": "ef1f4928b9ae7d9da25a98b82a28fda3e098db9504ad9372cb5305ad32421a06", |
| "kind": "direct_frozen_encoder_evaluation" |
| }, |
| "nano": { |
| "evaluated_native_fingerprint": "96d76f97478e114b816df3ab1b3c6947e4c5ef0babfb40ab865d7e3c838d34c4", |
| "applicable_native_fingerprint": "474ad3d52f6ee7f818a3dbb6112e3d66c2423542d08228e7996a8cd600bd99a2", |
| "kind": "exact_component_applicability" |
| } |
| }, |
| "evaluated_native_fingerprint": "96d76f97478e114b816df3ab1b3c6947e4c5ef0babfb40ab865d7e3c838d34c4", |
| "applicable_native_fingerprint": "474ad3d52f6ee7f818a3dbb6112e3d66c2423542d08228e7996a8cd600bd99a2", |
| "measurement_kind": "exact_component_applicability" |
| }, |
| { |
| "task": "TinyImageNetClustering", |
| "type": "ImageClustering", |
| "modalities": [ |
| "image" |
| ], |
| "split": "valid", |
| "hf_subset": "default", |
| "main_metric": "nmi", |
| "dataset": { |
| "path": "mteb/tiny-imagenet", |
| "revision": "e04e259f646f750bd0bdb2a56c2a4577d89bee3d" |
| }, |
| "counts": { |
| "evaluation_rows": 10000 |
| }, |
| "scores": { |
| "nano": 0.6169193395626786, |
| "small": 0.4628490605491802 |
| }, |
| "metrics": { |
| "nano": { |
| "v_measure": 0.6169193395626787, |
| "nmi": 0.6169193395626786, |
| "ari": 0.21497891543638262, |
| "cluster_accuracy": 0.3638, |
| "main_score": 0.6169193395626786, |
| "hf_subset": "default", |
| "languages": [ |
| "eng-Latn" |
| ], |
| "mteb_version": "2.21.0" |
| }, |
| "small": { |
| "v_measure": 0.4628490605491802, |
| "nmi": 0.4628490605491802, |
| "ari": 0.08498222977932614, |
| "cluster_accuracy": 0.2035, |
| "main_score": 0.4628490605491802, |
| "hf_subset": "default", |
| "languages": [ |
| "eng-Latn" |
| ], |
| "mteb_version": "2.21.0" |
| } |
| }, |
| "measurement_by_model": { |
| "small": { |
| "evaluated_native_fingerprint": "684fe0c48a4ddd3944167f6e5dc75e97ada24e5b120bd9a2921f00dc03112eeb", |
| "original_checkpoint_revision": "fdf8e01b7b0f3a69ac1ac8e2a64dcb1ede177ba4", |
| "raw_result_sha256": "eb344e20e7d498aadbbd67a57015cc2220cba42b9550f5822b5a72d0d6cf1514", |
| "kind": "direct_frozen_encoder_evaluation" |
| }, |
| "nano": { |
| "evaluated_native_fingerprint": "96d76f97478e114b816df3ab1b3c6947e4c5ef0babfb40ab865d7e3c838d34c4", |
| "applicable_native_fingerprint": "474ad3d52f6ee7f818a3dbb6112e3d66c2423542d08228e7996a8cd600bd99a2", |
| "kind": "exact_component_applicability" |
| } |
| }, |
| "evaluated_native_fingerprint": "96d76f97478e114b816df3ab1b3c6947e4c5ef0babfb40ab865d7e3c838d34c4", |
| "applicable_native_fingerprint": "474ad3d52f6ee7f818a3dbb6112e3d66c2423542d08228e7996a8cd600bd99a2", |
| "measurement_kind": "exact_component_applicability" |
| }, |
| { |
| "task": "OxfordPetsZeroShot", |
| "type": "ZeroShotClassification", |
| "modalities": [ |
| "text", |
| "image" |
| ], |
| "split": "test", |
| "hf_subset": "default", |
| "main_metric": "accuracy", |
| "dataset": { |
| "path": "mteb/OxfordPets", |
| "revision": "557b480fae8d69247be74d9503b378a09425096f" |
| }, |
| "counts": { |
| "evaluation_rows": 3669, |
| "fixed_text_labels": 37 |
| }, |
| "scores": { |
| "nano": 0.08013082583810302, |
| "small": 0.1640774052875443 |
| }, |
| "metrics": { |
| "nano": { |
| "accuracy": 0.08013082583810302, |
| "f1": 0.0611345635215895, |
| "f1_weighted": 0.06128470828202288, |
| "precision": 0.1359645526412158, |
| "precision_weighted": 0.13671480699577512, |
| "recall": 0.0798783003173134, |
| "recall_weighted": 0.08013082583810302, |
| "ap": null, |
| "ap_weighted": null, |
| "main_score": 0.08013082583810302, |
| "hf_subset": "default", |
| "languages": [ |
| "eng-Latn" |
| ], |
| "mteb_version": "2.21.0" |
| }, |
| "small": { |
| "accuracy": 0.1640774052875443, |
| "f1": 0.12868294474251654, |
| "f1_weighted": 0.12888736955731406, |
| "precision": 0.15179208674748698, |
| "precision_weighted": 0.15241279079348374, |
| "recall": 0.16458376660623852, |
| "recall_weighted": 0.1640774052875443, |
| "ap": null, |
| "ap_weighted": null, |
| "main_score": 0.1640774052875443, |
| "hf_subset": "default", |
| "languages": [ |
| "eng-Latn" |
| ], |
| "mteb_version": "2.21.0" |
| } |
| }, |
| "measurement_by_model": { |
| "small": { |
| "evaluated_native_fingerprint": "684fe0c48a4ddd3944167f6e5dc75e97ada24e5b120bd9a2921f00dc03112eeb", |
| "original_checkpoint_revision": "fdf8e01b7b0f3a69ac1ac8e2a64dcb1ede177ba4", |
| "raw_result_sha256": "e52e42c4716e62710296df98a90b3a0b986875685748d043cfd7de913e5a6c42", |
| "kind": "direct_frozen_encoder_evaluation" |
| }, |
| "nano": { |
| "evaluated_native_fingerprint": "96d76f97478e114b816df3ab1b3c6947e4c5ef0babfb40ab865d7e3c838d34c4", |
| "applicable_native_fingerprint": "474ad3d52f6ee7f818a3dbb6112e3d66c2423542d08228e7996a8cd600bd99a2", |
| "kind": "exact_component_applicability" |
| } |
| }, |
| "evaluated_native_fingerprint": "96d76f97478e114b816df3ab1b3c6947e4c5ef0babfb40ab865d7e3c838d34c4", |
| "applicable_native_fingerprint": "474ad3d52f6ee7f818a3dbb6112e3d66c2423542d08228e7996a8cd600bd99a2", |
| "measurement_kind": "exact_component_applicability" |
| }, |
| { |
| "task": "CREMA_D", |
| "type": "AudioClassification", |
| "modalities": [ |
| "audio" |
| ], |
| "split": "train", |
| "hf_subset": "default", |
| "main_metric": "accuracy", |
| "dataset": { |
| "path": "mteb/crema-d", |
| "revision": "a9d0821955c418752248b8310438854e56a1cf34" |
| }, |
| "counts": { |
| "evaluation_rows": 7442, |
| "official_cross_validation_folds": 5, |
| "probe_samples_per_label": 8 |
| }, |
| "scores": { |
| "nano": 0.32048146984697823, |
| "small": 0.2995196855795547 |
| }, |
| "metrics": { |
| "nano": { |
| "accuracy": 0.32048146984697823, |
| "ap": null, |
| "ap_weighted": null, |
| "f1": 0.303943978298811, |
| "f1_weighted": 0.30451603344534794, |
| "hf_subset": "default", |
| "languages": [ |
| "eng-Latn" |
| ], |
| "main_score": 0.32048146984697823, |
| "mteb_version": "2.21.0", |
| "precision": 0.31510472165152337, |
| "precision_weighted": 0.3169398085592857, |
| "recall": 0.3215883809208585, |
| "recall_weighted": 0.32048146984697823, |
| "scores_per_experiment": [ |
| { |
| "accuracy": 0.30288784419073206, |
| "ap": null, |
| "ap_weighted": null, |
| "f1": 0.27969233326985626, |
| "f1_weighted": 0.2799132430776581, |
| "precision": 0.29992207114753305, |
| "precision_weighted": 0.29961025331880187, |
| "recall": 0.3024507805686864, |
| "recall_weighted": 0.30288784419073206 |
| }, |
| { |
| "accuracy": 0.3149764942914708, |
| "ap": null, |
| "ap_weighted": null, |
| "f1": 0.30808145721796526, |
| "f1_weighted": 0.31085435981340914, |
| "precision": 0.336772371610985, |
| "precision_weighted": 0.3411130620509704, |
| "recall": 0.3160853641774642, |
| "recall_weighted": 0.3149764942914708 |
| }, |
| { |
| "accuracy": 0.31586021505376344, |
| "ap": null, |
| "ap_weighted": null, |
| "f1": 0.2931797954478092, |
| "f1_weighted": 0.2955391094563843, |
| "precision": 0.2916058030686394, |
| "precision_weighted": 0.29407622403946154, |
| "recall": 0.312973406205352, |
| "recall_weighted": 0.31586021505376344 |
| }, |
| { |
| "accuracy": 0.32123655913978494, |
| "ap": null, |
| "ap_weighted": null, |
| "f1": 0.3066232673451424, |
| "f1_weighted": 0.3032561325852315, |
| "precision": 0.3141358238179796, |
| "precision_weighted": 0.31520429243540943, |
| "recall": 0.32961458988542325, |
| "recall_weighted": 0.32123655913978494 |
| }, |
| { |
| "accuracy": 0.3474462365591398, |
| "ap": null, |
| "ap_weighted": null, |
| "f1": 0.3321430382132817, |
| "f1_weighted": 0.3330173222940568, |
| "precision": 0.3330875386124797, |
| "precision_weighted": 0.3346952109517855, |
| "recall": 0.3468177637673669, |
| "recall_weighted": 0.3474462365591398 |
| } |
| ] |
| }, |
| "small": { |
| "scores_per_experiment": [ |
| { |
| "accuracy": 0.2699798522498321, |
| "f1": 0.24239504058809014, |
| "f1_weighted": 0.2423525571760308, |
| "precision": 0.25470036430672977, |
| "precision_weighted": 0.2542205264446883, |
| "recall": 0.26975123145374097, |
| "recall_weighted": 0.2699798522498321, |
| "ap": null, |
| "ap_weighted": null |
| }, |
| { |
| "accuracy": 0.3035594358629953, |
| "f1": 0.29613784795561465, |
| "f1_weighted": 0.29906255840921786, |
| "precision": 0.30407267120079884, |
| "precision_weighted": 0.30800706496852154, |
| "recall": 0.3024788327196796, |
| "recall_weighted": 0.3035594358629953, |
| "ap": null, |
| "ap_weighted": null |
| }, |
| { |
| "accuracy": 0.28225806451612906, |
| "f1": 0.2593664352832718, |
| "f1_weighted": 0.2636113556497887, |
| "precision": 0.2609748445531306, |
| "precision_weighted": 0.26468416511118426, |
| "recall": 0.2775030429767242, |
| "recall_weighted": 0.28225806451612906, |
| "ap": null, |
| "ap_weighted": null |
| }, |
| { |
| "accuracy": 0.3111559139784946, |
| "f1": 0.29454980492169786, |
| "f1_weighted": 0.29125309340152467, |
| "precision": 0.3224943453400395, |
| "precision_weighted": 0.3242765511011472, |
| "recall": 0.31997721819075986, |
| "recall_weighted": 0.3111559139784946, |
| "ap": null, |
| "ap_weighted": null |
| }, |
| { |
| "accuracy": 0.33064516129032256, |
| "f1": 0.3185711713472314, |
| "f1_weighted": 0.3198970407995879, |
| "precision": 0.32320745672952395, |
| "precision_weighted": 0.32608239411677886, |
| "recall": 0.3309376196524952, |
| "recall_weighted": 0.33064516129032256, |
| "ap": null, |
| "ap_weighted": null |
| } |
| ], |
| "accuracy": 0.2995196855795547, |
| "f1": 0.2822040600191812, |
| "f1_weighted": 0.28323532108723004, |
| "precision": 0.29308993642604453, |
| "precision_weighted": 0.295454140348464, |
| "recall": 0.30012958899868, |
| "recall_weighted": 0.2995196855795547, |
| "ap": null, |
| "ap_weighted": null, |
| "main_score": 0.2995196855795547, |
| "hf_subset": "default", |
| "languages": [ |
| "eng-Latn" |
| ], |
| "mteb_version": "2.21.0" |
| } |
| }, |
| "measurement_by_model": { |
| "small": { |
| "evaluated_native_fingerprint": "684fe0c48a4ddd3944167f6e5dc75e97ada24e5b120bd9a2921f00dc03112eeb", |
| "original_checkpoint_revision": "fdf8e01b7b0f3a69ac1ac8e2a64dcb1ede177ba4", |
| "raw_result_sha256": "d79b8d114cb86c460dad13d564c2705a41df5119f83a0f1f27d452f36b64af47", |
| "kind": "direct_frozen_encoder_evaluation" |
| }, |
| "nano": { |
| "evaluated_native_fingerprint": "474ad3d52f6ee7f818a3dbb6112e3d66c2423542d08228e7996a8cd600bd99a2", |
| "applicable_native_fingerprint": "474ad3d52f6ee7f818a3dbb6112e3d66c2423542d08228e7996a8cd600bd99a2", |
| "kind": "fresh_native", |
| "raw_result_sha256": "62439b0ed4b3a8c26b1a8f122880a7bd3b3d86566841333a9060a4bbd0954baf" |
| } |
| }, |
| "evaluated_native_fingerprint": "474ad3d52f6ee7f818a3dbb6112e3d66c2423542d08228e7996a8cd600bd99a2", |
| "applicable_native_fingerprint": "474ad3d52f6ee7f818a3dbb6112e3d66c2423542d08228e7996a8cd600bd99a2", |
| "measurement_kind": "fresh_native", |
| "raw_result_sha256": "62439b0ed4b3a8c26b1a8f122880a7bd3b3d86566841333a9060a4bbd0954baf" |
| }, |
| { |
| "task": "CREMA_DClustering", |
| "type": "AudioClustering", |
| "modalities": [ |
| "audio" |
| ], |
| "split": "train", |
| "hf_subset": "default", |
| "main_metric": "v_measure", |
| "dataset": { |
| "path": "mteb/crema-d", |
| "revision": "a9d0821955c418752248b8310438854e56a1cf34" |
| }, |
| "counts": { |
| "official_stratified_subsample_rows": 2048, |
| "bootstrap_clustering_trials": 10, |
| "resampled_items_per_trial": 16384 |
| }, |
| "scores": { |
| "nano": 0.05214346580804754, |
| "small": 0.024058408888362488 |
| }, |
| "metrics": { |
| "nano": { |
| "ami": 0.05172990441350066, |
| "ami_scores": { |
| "Level 0": [ |
| 0.07691371568295576, |
| 0.05921735688360533, |
| 0.040011204150084696, |
| 0.04041243062073308, |
| 0.042015892825421196, |
| 0.06432477056741849, |
| 0.06285137443486444, |
| 0.05640769402749227, |
| 0.058836854203017874, |
| 0.01630775073941345 |
| ] |
| }, |
| "ami_std": 0.016307969325946296, |
| "hf_subset": "default", |
| "languages": [ |
| "eng-Latn" |
| ], |
| "main_score": 0.05214346580804754, |
| "mteb_version": "2.21.0", |
| "v_measure": 0.05214346580804754, |
| "v_measure_std": 0.016298021302929667, |
| "v_measures": { |
| "Level 0": [ |
| 0.07731089214293434, |
| 0.05962833919011462, |
| 0.040438217966279694, |
| 0.04084038764100412, |
| 0.04243023115915348, |
| 0.06473220137035035, |
| 0.06325438263275382, |
| 0.05681796105920958, |
| 0.059243669688975126, |
| 0.016738375229700227 |
| ] |
| } |
| }, |
| "small": { |
| "v_measures": { |
| "Level 0": [ |
| 0.041011040847839235, |
| 0.03386543153714212, |
| 0.029099626845588235, |
| 0.013685755785810546, |
| 0.02873132169365058, |
| 0.01881126494637877, |
| 0.011416660820064256, |
| 0.02673877787517492, |
| 0.00569343536524053, |
| 0.031530773166735686 |
| ] |
| }, |
| "v_measure": 0.024058408888362488, |
| "v_measure_std": 0.010611552764041994, |
| "ami_scores": { |
| "Level 0": [ |
| 0.04059499123714819, |
| 0.03344615727359256, |
| 0.02867541136577102, |
| 0.01325228658253946, |
| 0.028312443838566275, |
| 0.01838242676310465, |
| 0.01099044807520909, |
| 0.026321334415061327, |
| 0.005259713019363485, |
| 0.031113076527253092 |
| ] |
| }, |
| "ami": 0.023634828909760913, |
| "ami_std": 0.010617195895208265, |
| "main_score": 0.024058408888362488, |
| "hf_subset": "default", |
| "languages": [ |
| "eng-Latn" |
| ], |
| "mteb_version": "2.21.0" |
| } |
| }, |
| "measurement_by_model": { |
| "small": { |
| "evaluated_native_fingerprint": "684fe0c48a4ddd3944167f6e5dc75e97ada24e5b120bd9a2921f00dc03112eeb", |
| "original_checkpoint_revision": "fdf8e01b7b0f3a69ac1ac8e2a64dcb1ede177ba4", |
| "raw_result_sha256": "b23d4c552b9fa242923fc0c21a898ee63eb32850d9d91e0682b2cb5e8bea0e76", |
| "kind": "direct_frozen_encoder_evaluation" |
| }, |
| "nano": { |
| "evaluated_native_fingerprint": "474ad3d52f6ee7f818a3dbb6112e3d66c2423542d08228e7996a8cd600bd99a2", |
| "applicable_native_fingerprint": "474ad3d52f6ee7f818a3dbb6112e3d66c2423542d08228e7996a8cd600bd99a2", |
| "kind": "fresh_native", |
| "raw_result_sha256": "89db580a8e7039474d3bf150da208adbd98eabce358dad64a85f6c2027d4d62e" |
| } |
| }, |
| "evaluated_native_fingerprint": "474ad3d52f6ee7f818a3dbb6112e3d66c2423542d08228e7996a8cd600bd99a2", |
| "applicable_native_fingerprint": "474ad3d52f6ee7f818a3dbb6112e3d66c2423542d08228e7996a8cd600bd99a2", |
| "measurement_kind": "fresh_native", |
| "raw_result_sha256": "89db580a8e7039474d3bf150da208adbd98eabce358dad64a85f6c2027d4d62e" |
| }, |
| { |
| "task": "SpeechCommandsZeroshotv0.02", |
| "type": "AudioZeroshotClassification", |
| "modalities": [ |
| "audio", |
| "text" |
| ], |
| "split": "test", |
| "hf_subset": "default", |
| "main_metric": "accuracy", |
| "dataset": { |
| "path": "mteb/SpeechCommandsZeroshotv0.02", |
| "revision": "d5491b288ab76356a5f3cbc634f1dc38e23da2b8" |
| }, |
| "counts": { |
| "evaluation_rows": 4074, |
| "fixed_text_labels": 10 |
| }, |
| "scores": { |
| "nano": 0.15709376534118802, |
| "small": 0.05522827687776141 |
| }, |
| "metrics": { |
| "nano": { |
| "accuracy": 0.15709376534118802, |
| "ap": null, |
| "ap_weighted": null, |
| "f1": 0.08449976339531617, |
| "f1_weighted": 0.0833837827950463, |
| "hf_subset": "default", |
| "languages": [ |
| "eng-Latn" |
| ], |
| "main_score": 0.15709376534118802, |
| "mteb_version": "2.21.0", |
| "precision": 0.07985904570619766, |
| "precision_weighted": 0.078775844073094, |
| "recall": 0.15917885320870395, |
| "recall_weighted": 0.15709376534118802 |
| }, |
| "small": { |
| "accuracy": 0.05522827687776141, |
| "f1": 0.037875265984290724, |
| "f1_weighted": 0.03803483258988871, |
| "precision": 0.14410977979833245, |
| "precision_weighted": 0.143575354561912, |
| "recall": 0.05521141290348569, |
| "recall_weighted": 0.05522827687776141, |
| "ap": null, |
| "ap_weighted": null, |
| "main_score": 0.05522827687776141, |
| "hf_subset": "default", |
| "languages": [ |
| "eng-Latn" |
| ], |
| "mteb_version": "2.21.0" |
| } |
| }, |
| "measurement_by_model": { |
| "small": { |
| "evaluated_native_fingerprint": "684fe0c48a4ddd3944167f6e5dc75e97ada24e5b120bd9a2921f00dc03112eeb", |
| "original_checkpoint_revision": "fdf8e01b7b0f3a69ac1ac8e2a64dcb1ede177ba4", |
| "raw_result_sha256": "ca19c03bbe2fdcf3c2e1282f5e826af702c152287fde75d67b28ef877fae5335", |
| "kind": "direct_frozen_encoder_evaluation" |
| }, |
| "nano": { |
| "evaluated_native_fingerprint": "474ad3d52f6ee7f818a3dbb6112e3d66c2423542d08228e7996a8cd600bd99a2", |
| "applicable_native_fingerprint": "474ad3d52f6ee7f818a3dbb6112e3d66c2423542d08228e7996a8cd600bd99a2", |
| "kind": "fresh_native", |
| "raw_result_sha256": "c5b151588a51abbf4ca2e9bc88ec91d235b0971d8035f02e97b7c58910f66538" |
| } |
| }, |
| "evaluated_native_fingerprint": "474ad3d52f6ee7f818a3dbb6112e3d66c2423542d08228e7996a8cd600bd99a2", |
| "applicable_native_fingerprint": "474ad3d52f6ee7f818a3dbb6112e3d66c2423542d08228e7996a8cd600bd99a2", |
| "measurement_kind": "fresh_native", |
| "raw_result_sha256": "c5b151588a51abbf4ca2e9bc88ec91d235b0971d8035f02e97b7c58910f66538" |
| } |
| ], |
| "comparison": { |
| "baseline_model": "vllm-sr/multi-modal-embed-small", |
| "baseline_revision": "fdf8e01b7b0f3a69ac1ac8e2a64dcb1ede177ba4", |
| "baseline_native_artifact_sha256": "684fe0c48a4ddd3944167f6e5dc75e97ada24e5b120bd9a2921f00dc03112eeb", |
| "baseline_standard_status": "evaluated_all_11_selected_tasks", |
| "card_bold": "Only unrounded gains against this original baseline are bold; missing baseline scores imply no comparison." |
| }, |
| "observation_scope": "Source-specific counters only: eight unchanged text/image tasks retain their actual earlier measurement; three audio tasks are fresh. Old timing is not a measurement of the new audio architecture.", |
| "encoding_observations": { |
| "SICK-R": { |
| "nano": { |
| "before": { |
| "text_rows": 0, |
| "text_rows_truncated": 0, |
| "max_original_text_tokens": 0, |
| "image_rows": 0, |
| "audio_rows": 0 |
| }, |
| "after": { |
| "text_rows": 19854, |
| "text_rows_truncated": 0, |
| "max_original_text_tokens": 38, |
| "image_rows": 0, |
| "audio_rows": 0 |
| }, |
| "elapsed_seconds": 10.807591969991336, |
| "peak_allocated_gpu_bytes": 679632896, |
| "peak_reserved_gpu_bytes": 719323136 |
| } |
| }, |
| "STSBenchmark": { |
| "nano": { |
| "before": { |
| "text_rows": 19854, |
| "text_rows_truncated": 0, |
| "max_original_text_tokens": 38, |
| "image_rows": 0, |
| "audio_rows": 0 |
| }, |
| "after": { |
| "text_rows": 22612, |
| "text_rows_truncated": 0, |
| "max_original_text_tokens": 46, |
| "image_rows": 0, |
| "audio_rows": 0 |
| }, |
| "elapsed_seconds": 1.294741225981852, |
| "peak_allocated_gpu_bytes": 681261056, |
| "peak_reserved_gpu_bytes": 723517440 |
| } |
| }, |
| "ArguAna": { |
| "nano": { |
| "before": { |
| "text_rows": 22612, |
| "text_rows_truncated": 0, |
| "max_original_text_tokens": 46, |
| "image_rows": 0, |
| "audio_rows": 0 |
| }, |
| "after": { |
| "text_rows": 32692, |
| "text_rows_truncated": 222, |
| "max_original_text_tokens": 1401, |
| "image_rows": 0, |
| "audio_rows": 0 |
| }, |
| "elapsed_seconds": 15.392282173008425, |
| "peak_allocated_gpu_bytes": 809891328, |
| "peak_reserved_gpu_bytes": 887095296 |
| } |
| }, |
| "TwentyNewsgroupsClustering.v2": { |
| "nano": { |
| "before": { |
| "text_rows": 32692, |
| "text_rows_truncated": 222, |
| "max_original_text_tokens": 1401, |
| "image_rows": 0, |
| "audio_rows": 0 |
| }, |
| "after": { |
| "text_rows": 35073, |
| "text_rows_truncated": 222, |
| "max_original_text_tokens": 1401, |
| "image_rows": 0, |
| "audio_rows": 0 |
| }, |
| "elapsed_seconds": 48.54122133200872, |
| "peak_allocated_gpu_bytes": 684661248, |
| "peak_reserved_gpu_bytes": 727711744 |
| } |
| }, |
| "MTOPDomainClassification": { |
| "nano": { |
| "before": { |
| "text_rows": 35073, |
| "text_rows_truncated": 222, |
| "max_original_text_tokens": 1401, |
| "image_rows": 0, |
| "audio_rows": 0 |
| }, |
| "after": { |
| "text_rows": 43422, |
| "text_rows_truncated": 222, |
| "max_original_text_tokens": 1401, |
| "image_rows": 0, |
| "audio_rows": 0 |
| }, |
| "elapsed_seconds": 25.26324965900858, |
| "peak_allocated_gpu_bytes": 679308288, |
| "peak_reserved_gpu_bytes": 719323136 |
| } |
| }, |
| "OxfordPets": { |
| "nano": { |
| "before": { |
| "text_rows": 43422, |
| "text_rows_truncated": 222, |
| "max_original_text_tokens": 1401, |
| "image_rows": 0, |
| "audio_rows": 0 |
| }, |
| "after": { |
| "text_rows": 43422, |
| "text_rows_truncated": 222, |
| "max_original_text_tokens": 1401, |
| "image_rows": 5816, |
| "audio_rows": 0 |
| }, |
| "elapsed_seconds": 54.48569396699895, |
| "peak_allocated_gpu_bytes": 996472320, |
| "peak_reserved_gpu_bytes": 1207959552 |
| } |
| }, |
| "TinyImageNetClustering": { |
| "nano": { |
| "before": { |
| "text_rows": 43422, |
| "text_rows_truncated": 222, |
| "max_original_text_tokens": 1401, |
| "image_rows": 5816, |
| "audio_rows": 0 |
| }, |
| "after": { |
| "text_rows": 43422, |
| "text_rows_truncated": 222, |
| "max_original_text_tokens": 1401, |
| "image_rows": 15816, |
| "audio_rows": 0 |
| }, |
| "elapsed_seconds": 72.07803665398387, |
| "peak_allocated_gpu_bytes": 996472320, |
| "peak_reserved_gpu_bytes": 1117782016 |
| } |
| }, |
| "OxfordPetsZeroShot": { |
| "nano": { |
| "before": { |
| "text_rows": 43422, |
| "text_rows_truncated": 222, |
| "max_original_text_tokens": 1401, |
| "image_rows": 15816, |
| "audio_rows": 0 |
| }, |
| "after": { |
| "text_rows": 43459, |
| "text_rows_truncated": 222, |
| "max_original_text_tokens": 1401, |
| "image_rows": 19485, |
| "audio_rows": 0 |
| }, |
| "elapsed_seconds": 32.662016286019934, |
| "peak_allocated_gpu_bytes": 996472320, |
| "peak_reserved_gpu_bytes": 1119879168 |
| } |
| }, |
| "CREMA_D": { |
| "evaluated_native_fingerprint": "474ad3d52f6ee7f818a3dbb6112e3d66c2423542d08228e7996a8cd600bd99a2", |
| "observations": { |
| "cumulative_rows": { |
| "audio_rows": 7442, |
| "image_rows": 0, |
| "max_original_text_tokens": 0, |
| "text_rows": 0, |
| "text_rows_truncated": 0 |
| }, |
| "peak_allocated_bytes": 966564864, |
| "peak_reserved_bytes": 1493172224, |
| "seconds": 228.799308784015 |
| } |
| }, |
| "CREMA_DClustering": { |
| "evaluated_native_fingerprint": "474ad3d52f6ee7f818a3dbb6112e3d66c2423542d08228e7996a8cd600bd99a2", |
| "observations": { |
| "cumulative_rows": { |
| "audio_rows": 9490, |
| "image_rows": 0, |
| "max_original_text_tokens": 0, |
| "text_rows": 0, |
| "text_rows_truncated": 0 |
| }, |
| "peak_allocated_bytes": 966577152, |
| "peak_reserved_bytes": 1321205760, |
| "seconds": 102.58190695600933 |
| } |
| }, |
| "SpeechCommandsZeroshotv0.02": { |
| "evaluated_native_fingerprint": "474ad3d52f6ee7f818a3dbb6112e3d66c2423542d08228e7996a8cd600bd99a2", |
| "observations": { |
| "cumulative_rows": { |
| "audio_rows": 13564, |
| "image_rows": 0, |
| "max_original_text_tokens": 3, |
| "text_rows": 10, |
| "text_rows_truncated": 0 |
| }, |
| "peak_allocated_bytes": 966577152, |
| "peak_reserved_bytes": 1329594368, |
| "seconds": 122.96637481401558 |
| } |
| } |
| } |
| }, |
| "capabilities": { |
| "max_text_tokens": 512, |
| "tokens_include_special_tokens": true, |
| "public_text_overflow": "Raises ValueError; shorten or explicitly chunk longer inputs.", |
| "embedding_dimensions": 384, |
| "parameters": 163771288, |
| "audio_sample_rates": [ |
| 16000, |
| 44100, |
| 48000 |
| ], |
| "audio_max_seconds": 30, |
| "audio_input": "Original PCM, mono or channels-first; each rate is independently resampled for the two audio branches." |
| }, |
| "efficiency": { |
| "status": "no_current_latency_measurement", |
| "parameters": 163771288, |
| "parameter_increase_percent_vs_previous_release": 20.968150046422096, |
| "embedding_dimensions": 384, |
| "dense_fp32_index_bytes_per_vector": 1536, |
| "weights_safetensors_bytes": 655583800, |
| "weights_safetensors_sha256": "d5aa7f00e217bd4fd5314140a3603b9a873636320c8d09732a14f79f4cb0667e", |
| "scope": "The CLAP branch increases parameters and audio computation. No isolated latency or peak-memory improvement is claimed; previous measurements belong to their recorded artifacts.", |
| "previous_release_measurement": { |
| "status": "not_measured_for_this_release", |
| "parameters": 135383808, |
| "embedding_dimensions": 384, |
| "dense_fp32_index_bytes_per_vector": 1536, |
| "weights_safetensors_bytes": 541598024, |
| "scope": "No new latency or peak-memory measurement is claimed. Vector storage excludes index structures and metadata.", |
| "historical_measurement": { |
| "attention": "native SDPA", |
| "batch_size": 8, |
| "embedding_dtype": "float32", |
| "hardware": "AMD gfx942 GPU", |
| "latency_includes": "Native public encode_text: tokenization, host-to-device transfer and encoding; excludes model loading.", |
| "measured_batches": 20, |
| "measurements": { |
| "128": { |
| "batch_size": 8, |
| "p50_ms": 4.283615489839576, |
| "p95_ms": 4.314010989037342, |
| "peak_allocated_bytes": 653202944, |
| "peak_reserved_bytes": 673185792, |
| "tokens_each_including_special": 128 |
| }, |
| "512": { |
| "batch_size": 8, |
| "p50_ms": 12.445501997717656, |
| "p95_ms": 12.603181986196432, |
| "peak_allocated_bytes": 878682112, |
| "peak_reserved_bytes": 970981376, |
| "tokens_each_including_special": 512 |
| } |
| }, |
| "memory_scope": "Process CUDA allocator peak with the complete multimodal model loaded.", |
| "model_revision": "70c3412bb0ddfb19386d673d4a102c753a574f1b", |
| "native_artifact_sha256": "b596f1dbab89369555e0a72a0d6854d780e305686750949b308c8f167993bdeb", |
| "scope": "Synthetic repeated text-batch measurements on one GPU; not end-to-end service latency.", |
| "tf32": false, |
| "token_lengths_include_special_tokens": true, |
| "torch_version_family": "2.12", |
| "transformers_version": "4.57.6", |
| "warmup_batches": 2, |
| "status": "historical_previous_release_measurement", |
| "applicability": "Historical measurement on its recorded native artifact; not applicable to this text-backbone update." |
| }, |
| "parameter_storage_change": { |
| "old_safetensors_bytes": 578289664, |
| "new_safetensors_bytes": 541598024, |
| "reduction_bytes": 36691640, |
| "old_FP32_parameter_bytes": 578216448, |
| "new_FP32_parameter_bytes": 541535232, |
| "scope": "Parameter storage only; not measured process peak memory or latency. Vector and index dimensions are unchanged." |
| } |
| } |
| }, |
| "previous_release": { |
| "model": "vllm-sr/Vela-1.0-Omni-Nano", |
| "revision": "9e747d2e1d4b97d3081ff4ac2ac79986c7a8da95", |
| "native_artifact_sha256": "50ce808197fb43a1b8913a66d4a47339ec66ecd10d52cbd9f9e752d02f8c43fa", |
| "evaluated_artifact_fingerprint_sha256": "96d76f97478e114b816df3ab1b3c6947e4c5ef0babfb40ab865d7e3c838d34c4", |
| "parameters": 135383808, |
| "scores_report": "benchmarks/previous-release-scores.json" |
| }, |
| "full_benchmarks": [ |
| { |
| "benchmark": "MTEB(eng, v2)", |
| "framework_version": "2.21.0", |
| "coverage": { |
| "completed_tasks": 41, |
| "required_tasks": 41, |
| "task_types": 7 |
| }, |
| "model_revision": "29994142677559ca657c27b657eda0dddbf583bd4cf42a3a31095ab3f99d1646", |
| "native_artifact_sha256": "29994142677559ca657c27b657eda0dddbf583bd4cf42a3a31095ab3f99d1646", |
| "aggregation": { |
| "mean_task": 0.6488431514310018, |
| "mean_task_type": 0.6078184975654923, |
| "by_task_type": { |
| "Classification": { |
| "score": 0.7782970776285105, |
| "tasks": 8 |
| }, |
| "Clustering": { |
| "score": 0.48962597555764586, |
| "tasks": 8 |
| }, |
| "PairClassification": { |
| "score": 0.8468485561827029, |
| "tasks": 3 |
| }, |
| "Reranking": { |
| "score": 0.473475, |
| "tasks": 2 |
| }, |
| "Retrieval": { |
| "score": 0.520098, |
| "tasks": 10 |
| }, |
| "STS": { |
| "score": 0.8280405301305157, |
| "tasks": 9 |
| }, |
| "Summarization": { |
| "score": 0.31834434345907126, |
| "tasks": 1 |
| } |
| }, |
| "definition": "Official within-task aggregation; equal task mean across 41 tasks, and equal mean of seven within-type means.", |
| "score_scale": "0-1; display multiplied by 100" |
| }, |
| "report": "benchmarks/mteb-eng-v2.json", |
| "text_component_applicability": { |
| "component": "text", |
| "applies_to_revision": "self", |
| "applies_to_native_artifact_sha256": "474ad3d52f6ee7f818a3dbb6112e3d66c2423542d08228e7996a8cd600bd99a2", |
| "evaluated_model": "local/Vela-Omni-Nano-GIST-Rotation", |
| "evaluated_revision": "29994142677559ca657c27b657eda0dddbf583bd4cf42a3a31095ab3f99d1646", |
| "evaluated_native_artifact_sha256": "29994142677559ca657c27b657eda0dddbf583bd4cf42a3a31095ab3f99d1646", |
| "revision_kind": "local native artifact fingerprint, not a Hugging Face commit", |
| "report": "benchmarks/component-equivalence.json", |
| "new_full_panel_inference": false, |
| "basis": "The evaluated text computation is retained through the published Nano and its inference-only trim. All 197 retained text tensors, effective FP32 values, tokenizer/config and text operation path match; the two removed pooler tensors are unused.", |
| "bridge_sha256": "42d802c7279a1c20728926c079f5a4f952b16629fc49a8ed121da4c51f3d77aa" |
| } |
| }, |
| { |
| "benchmark": "MAEB(beta, audio-only)", |
| "framework_version": "2.21.0", |
| "coverage": { |
| "completed_tasks": 19, |
| "required_tasks": 19, |
| "subset_split_slots": 134, |
| "task_types": 5, |
| "selection": "All MAEB(beta) tasks whose modalities are exactly [audio].", |
| "full_MAEB": false, |
| "excluded": "Tasks that require text/audio or other joint-modality inputs." |
| }, |
| "model_revision": "474ad3d52f6ee7f818a3dbb6112e3d66c2423542d08228e7996a8cd600bd99a2", |
| "native_artifact_sha256": "474ad3d52f6ee7f818a3dbb6112e3d66c2423542d08228e7996a8cd600bd99a2", |
| "aggregation": { |
| "official": { |
| "Any2AnyRetrieval": 0.8580149999999999, |
| "AudioClassification": 0.44475865771818446, |
| "AudioClustering": 0.03504342925419187, |
| "AudioPairClassification": 0.5741932198702043, |
| "AudioReranking": 0.70476, |
| "Mean(Task)": 0.4359384306459587, |
| "Mean(TaskType)": 0.5233540613685161 |
| }, |
| "definition": "Official TaskResult aggregation within each task, followed by official mean_task, mean_task_type and task_types aggregation. The 134 subset/split slots are not averaged as independent tasks.", |
| "score_scale": "0-1; multiply by 100 for display" |
| }, |
| "report": "benchmarks/maeb-audio-only.json", |
| "evaluated_native_artifact_sha256": "474ad3d52f6ee7f818a3dbb6112e3d66c2423542d08228e7996a8cd600bd99a2", |
| "release_native_artifact_sha256": "474ad3d52f6ee7f818a3dbb6112e3d66c2423542d08228e7996a8cd600bd99a2" |
| } |
| ], |
| "net_benefit": { |
| "scope": "Fixed product progress metric; not an official benchmark aggregate, significance claim, or proof of overall SOTA.", |
| "baseline_revision": "9e747d2e1d4b97d3081ff4ac2ac79986c7a8da95", |
| "baseline_native_fingerprint": "50ce808197fb43a1b8913a66d4a47339ec66ecd10d52cbd9f9e752d02f8c43fa", |
| "baseline_evaluated_fingerprint": "96d76f97478e114b816df3ab1b3c6947e4c5ef0babfb40ab865d7e3c838d34c4", |
| "candidate_native_fingerprint": "474ad3d52f6ee7f818a3dbb6112e3d66c2423542d08228e7996a8cd600bd99a2", |
| "components": [ |
| { |
| "candidate_100": 66.41835251574038, |
| "component": "common14", |
| "delta_pp": -1.1557204928749676, |
| "parent_100": 67.57407300861536, |
| "weight": 0.3 |
| }, |
| { |
| "candidate_100": 60.781849756549235, |
| "component": "English41_MeanTaskType", |
| "delta_pp": 0.0, |
| "parent_100": 60.781849756549235, |
| "weight": 0.3 |
| }, |
| { |
| "candidate_100": 53.65746394890041, |
| "component": "image3", |
| "delta_pp": 0.0, |
| "parent_100": 53.65746394890041, |
| "weight": 0.2 |
| }, |
| { |
| "candidate_100": 52.33540613685162, |
| "component": "audio19_MeanTaskType", |
| "delta_pp": 5.367592050294833, |
| "parent_100": 46.96781408655678, |
| "weight": 0.2 |
| } |
| ], |
| "weighted_net_delta_pp": 0.7268022621964765, |
| "score_scale": "0–100; deltas in percentage points", |
| "parameter_increase_percent": 20.968150046422096, |
| "common_definition": "Mean of text accuracy (2), image recall (6), and audio recall (6), with equal one-third family weights.", |
| "image3_definition": "Mean of OxfordPets accuracy, TinyImageNetClustering NMI, and OxfordPetsZeroShot accuracy.", |
| "common_metric_delta_pp": { |
| "audio.audio_to_text.recall@1": -1.0340865568747608, |
| "audio.audio_to_text.recall@10": -1.3787820758330127, |
| "audio.audio_to_text.recall@5": -1.8383761011106892, |
| "audio.text_to_audio.recall@1": -5.095785440613029, |
| "audio.text_to_audio.recall@10": -5.287356321839088, |
| "audio.text_to_audio.recall@5": -6.168582375478926, |
| "image.image_to_text.recall@1": 0.0, |
| "image.image_to_text.recall@10": 0.0, |
| "image.image_to_text.recall@5": 0.0, |
| "image.text_to_image.recall@1": 0.0, |
| "image.text_to_image.recall@10": 0.0, |
| "image.text_to_image.recall@5": 0.0, |
| "text.banking77.accuracy": 0.0, |
| "text.massive-en.accuracy": 0.0 |
| }, |
| "audio_task_delta_pp": { |
| "BeijingOpera": 10.549645390070905, |
| "BirdCLEF": 2.2999999999999963, |
| "CREMADPairClassification": 1.0060252268561554, |
| "CREMA_D": 2.2844768445301367, |
| "CREMA_DClustering": 2.8541749889603825, |
| "CommonLanguageAgeDetection": 0.6150000000000017, |
| "GTZANAudioReranking": 6.68200000000001, |
| "GTZANGenre": 11.399999999999999, |
| "IEMOCAPGender": 3.237875602721152, |
| "JamAltArtistA2ARetrieval": 16.90099999999998, |
| "MInDS14": -1.4691308536005643, |
| "MridinghamTonic": 26.05657742038183, |
| "NMSQAPairClassification": -0.3840625955228716, |
| "SIBFLEURS": -0.23211656203982745, |
| "VehicleSoundClustering": -8.525275870377525, |
| "VoxCelebSA": -0.00058896531414665, |
| "VoxPopuliAccentPairClassification": 0.1293630037889515, |
| "VoxPopuliGenderClustering": -0.08127278385060711, |
| "VoxPopuliLanguageID": -0.5999999999999783 |
| }, |
| "audio_aggregate_delta_pp": { |
| "Any2AnyRetrieval": 16.90099999999998, |
| "AudioClassification": 4.921976261522682, |
| "AudioClustering": -1.9174578884225837, |
| "AudioPairClassification": 0.2504418783740858, |
| "AudioReranking": 6.68200000000001, |
| "Mean(Task)": 3.8275626761370307, |
| "Mean(TaskType)": 5.367592050294833 |
| }, |
| "English_and_image_unchanged": true, |
| "event840_in_net": false, |
| "paired_interval_scope": "Fresh paired intervals compare this candidate with the preceding Vela Nano computation. They are not an interval for the all-14 aggregate or the weighted net. An original-small interval was not recomputed for this update." |
| }, |
| "paired_uncertainty_scope": "Fresh paired intervals compare this candidate with the preceding Vela Nano computation. They are not an interval for the all-14 aggregate or the weighted net. An original-small interval was not recomputed for this update.", |
| "scorecard_comparison": { |
| "primary_comparator": "vllm-sr/multi-modal-embed-small", |
| "revision": "fdf8e01b7b0f3a69ac1ac8e2a64dcb1ede177ba4", |
| "common_protocol_measured": true, |
| "original_reference_evidence": "benchmarks/original-baseline.json", |
| "previous_Vela_not_the_primary_comparator": true, |
| "historical_release_progress": "benchmarks/release-history.md" |
| } |
| } |
|
|
