{ "schema_version": 2, "model": "vllm-sr/Vela-1.0-Omni-Nano", "revision_semantics": "self denotes this published inference package; evaluated_* identities retain original measurements. The exact public-inference equivalence report binds score applicability.", "score_scale": "0-1; multiply by 100 for percentages", "matrix": { "small": { "model": "vllm-sr/multi-modal-embed-small", "revision": "fdf8e01b7b0f3a69ac1ac8e2a64dcb1ede177ba4", "artifact_fingerprint_sha256": "684fe0c48a4ddd3944167f6e5dc75e97ada24e5b120bd9a2921f00dc03112eeb", "metrics": { "text.banking77.accuracy": { "numerator": 2169, "denominator": 3080, "value": 0.7042207792207792 }, "text.massive-en.accuracy": { "numerator": 1960, "denominator": 2972, "value": 0.6594885598923284 }, "image.image_to_text.recall@1": { "numerator": 336, "denominator": 823, "value": 0.4082624544349939 }, "image.image_to_text.recall@5": { "numerator": 553, "denominator": 823, "value": 0.6719319562575942 }, "image.image_to_text.recall@10": { "numerator": 646, "denominator": 823, "value": 0.7849331713244229 }, "image.text_to_image.recall@1": { "numerator": 1242, "denominator": 4115, "value": 0.3018226002430134 }, "image.text_to_image.recall@5": { "numerator": 2445, "denominator": 4115, "value": 0.5941676792223572 }, "image.text_to_image.recall@10": { "numerator": 3057, "denominator": 4115, "value": 0.7428918590522479 }, "audio.audio_to_text.recall@1": { "numerator": 110, "denominator": 2611, "value": 0.04212945231711988 }, "audio.audio_to_text.recall@5": { "numerator": 313, "denominator": 2611, "value": 0.11987744159325929 }, "audio.audio_to_text.recall@10": { "numerator": 497, "denominator": 2611, "value": 0.1903485254691689 }, "audio.text_to_audio.recall@1": { "numerator": 250, "denominator": 2610, "value": 0.09578544061302682 }, "audio.text_to_audio.recall@5": { "numerator": 588, "denominator": 2610, "value": 0.22528735632183908 }, "audio.text_to_audio.recall@10": { "numerator": 801, "denominator": 2610, "value": 0.30689655172413793 } } }, "candidate": { "model": "vllm-sr/Vela-1.0-Omni-Nano", "revision": "self", "artifact_fingerprint_sha256": "50ce808197fb43a1b8913a66d4a47339ec66ecd10d52cbd9f9e752d02f8c43fa", "metrics": { "audio.audio_to_text.recall@1": { "denominator": 2611, "numerator": 448, "value": 0.17158176943699732 }, "audio.audio_to_text.recall@10": { "denominator": 2611, "numerator": 1227, "value": 0.469934890846419 }, "audio.audio_to_text.recall@5": { "denominator": 2611, "numerator": 957, "value": 0.36652623515894295 }, "audio.text_to_audio.recall@1": { "denominator": 2610, "numerator": 664, "value": 0.25440613026819925 }, "audio.text_to_audio.recall@10": { "denominator": 2610, "numerator": 1459, "value": 0.5590038314176246 }, "audio.text_to_audio.recall@5": { "denominator": 2610, "numerator": 1204, "value": 0.46130268199233715 }, "image.image_to_text.recall@1": { "denominator": 823, "numerator": 501, "value": 0.6087484811664642 }, "image.image_to_text.recall@10": { "denominator": 823, "numerator": 781, "value": 0.9489671931956257 }, "image.image_to_text.recall@5": { "denominator": 823, "numerator": 717, "value": 0.8712029161603888 }, "image.text_to_image.recall@1": { "denominator": 4115, "numerator": 2297, "value": 0.5582017010935602 }, "image.text_to_image.recall@10": { "denominator": 4115, "numerator": 3879, "value": 0.9426488456865127 }, "image.text_to_image.recall@5": { "denominator": 4115, "numerator": 3507, "value": 0.8522478736330498 }, "text.banking77.accuracy": { "denominator": 3080, "numerator": 2710, "value": 0.8798701298701299 }, "text.massive-en.accuracy": { "denominator": 2972, "numerator": 2436, "value": 0.819650067294751 } }, "parameters": 135383808, "evaluated_native_artifact_sha256": "96d76f97478e114b816df3ab1b3c6947e4c5ef0babfb40ab865d7e3c838d34c4", "evaluated_revision": "de02c367bc9aa7a6f7ad3e59fb38962502e021c1", "evaluated_parameters": 144554112, "inference_equivalence_report": "benchmarks/inference-equivalence.json" } }, "datasets": { "banking77": { "source": "https://github.com/PolyAI-LDN/task-specific-datasets", "revision": "57ec275d8078af65b7731c2a98be812d844a6d6b", "license": "CC-BY-4.0", "split": "test", "queries": 3080, "classes": 77 }, "massive-en": { "source": "https://huggingface.co/datasets/AmazonScience/massive", "revision": "ff6bd8e4b27c3543e4f8fe2108f32bb95a6f8740", "license": "CC-BY-4.0", "configuration": "en-US", "split": "test", "queries": 2972, "prototype_classes": 60 }, "image": { "source": "https://cocodataset.org/", "caption_split_source": "https://cs.stanford.edu/people/karpathy/deepimagesent/", "split": "Karpathy test, CC-BY 2.0 image subset", "images": 823, "captions": 4115, "image_license": "CC-BY-2.0", "caption_license": "CC-BY-4.0" }, "audio": { "source": "https://www.openslr.org/12", "split": "test-clean", "archive_sha256": "39fde525e59672dc6d1551919b1478f724438a95aa55f874b576be21967e6c23", "license": "CC-BY-4.0", "clips": 2611, "unique_transcripts": 2610, "maximum_seconds": 30 } }, "protocol": { "embedding_dtype": "float32", "attention": "SDPA", "autocast": false, "tf32": false, "transformers_version": "4.57.6", "torch_version_family": "2.12", "text_max_tokens": 128, "truncation": "right, native tokenizer including special tokens", "batch_sizes": { "text": 64, "image": 8, "audio": 8 }, "pooling": "native full-dimension model pooling", "similarity": "FP32 cosine of L2-normalized embeddings", "ties": "lexicographic candidate ID", "retrieval": "complete candidate pools; all annotated positives", "classification": "Nearest normalized mean class prototype from labeled training examples; the original small comparator has task-adapted history, while current GIST is frozen.", "aggregation": "Each family is the mean of its two primary metrics; utility is the mean of all six primaries. Retrieval primaries are R@1; R@5 and R@10 are reported separately.", "evaluation_scope": "fixed test-split evaluation pools reused across releases", "text_training_scope": "Current GIST text tower is frozen; supervised training prototypes are recomputed under the fixed common protocol. The earlier small comparator has task-adapted history. This is not the official MTEB classification protocol." }, "paired_uncertainty": { "original": { "intervals": { "audio.audio_to_text.recall@1": { "lower": 0.11352472177122043, "point_delta": 0.12945231711987745, "upper": 0.14665221762155492 }, "audio.audio_to_text.recall@10": { "lower": 0.25338950715018704, "point_delta": 0.2795863653772501, "upper": 0.30833452722063037 }, "audio.audio_to_text.recall@5": { "lower": 0.22198472459131105, "point_delta": 0.24664879356568364, "upper": 0.2736971153456769 }, "audio.text_to_audio.recall@1": { "lower": 0.14202610349345837, "point_delta": 0.15862068965517243, "upper": 0.17613425616168824 }, "audio.text_to_audio.recall@10": { "lower": 0.23515433278434666, "point_delta": 0.25210727969348656, "upper": 0.2707672328928673 }, "audio.text_to_audio.recall@5": { "lower": 0.21796104940042046, "point_delta": 0.23601532567049807, "upper": 0.2547919081648684 }, "family.audio": { "lower": 0.1322499685832721, "point_delta": 0.14403650338752494, "upper": 0.15644009735751394 }, "family.image": { "lower": 0.20084750911300123, "point_delta": 0.2284325637910085, "upper": 0.25637910085054677 }, "family.text": { "lower": 0.15747009669727144, "point_delta": 0.16790542902588662, "upper": 0.17869785413173905 }, "image.image_to_text.recall@1": { "lower": 0.1591737545565006, "point_delta": 0.20048602673147023, "upper": 0.24301336573511542 }, "image.image_to_text.recall@10": { "lower": 0.1336573511543135, "point_delta": 0.16403402187120292, "upper": 0.19441069258809235 }, "image.image_to_text.recall@5": { "lower": 0.16160388821385177, "point_delta": 0.19927095990279464, "upper": 0.23572296476306198 }, "image.text_to_image.recall@1": { "lower": 0.23013365735115432, "point_delta": 0.25637910085054677, "upper": 0.2831166464155528 }, "image.text_to_image.recall@10": { "lower": 0.17618469015795868, "point_delta": 0.1997569866342649, "upper": 0.22235722964763063 }, "image.text_to_image.recall@5": { "lower": 0.23207776427703525, "point_delta": 0.2580801944106926, "upper": 0.28360267314702303 }, "text.banking77.accuracy": { "lower": 0.16066212268743915, "point_delta": 0.17564935064935064, "upper": 0.19058596303826084 }, "text.massive-en.accuracy": { "lower": 0.14554621848739496, "point_delta": 0.1601615074024226, "upper": 0.17569841804106362 }, "utility": { "lower": 0.16995251624973068, "point_delta": 0.18012483206814003, "upper": 0.1910563694034248 } }, "method": "paired stratified group percentile; PCG64; fixed candidate pools", "replicates": 2000, "seed": 20260918, "comparison": { "candidate": "vllm-sr/Vela-1.0-Omni-Nano", "candidate_artifact_fingerprint_sha256": "96d76f97478e114b816df3ab1b3c6947e4c5ef0babfb40ab865d7e3c838d34c4", "baseline": "vllm-sr/multi-modal-embed-small", "baseline_revision": "fdf8e01b7b0f3a69ac1ac8e2a64dcb1ede177ba4", "baseline_artifact_fingerprint_sha256": "684fe0c48a4ddd3944167f6e5dc75e97ada24e5b120bd9a2921f00dc03112eeb" } }, "previous": { "intervals": { "audio.audio_to_text.recall@1": { "lower": 0.06612466882994839, "point_delta": 0.08157793948678667, "upper": 0.0967748551165683 }, "audio.audio_to_text.recall@10": { "lower": 0.13256556125090607, "point_delta": 0.15511298353121408, "upper": 0.18059527375151307 }, "audio.audio_to_text.recall@5": { "lower": 0.12086266595311207, "point_delta": 0.1436231328992723, "upper": 0.16909569992403942 }, "audio.text_to_audio.recall@1": { "lower": 0.09860745201354912, "point_delta": 0.11417624521072797, "upper": 0.13014135201210755 }, "audio.text_to_audio.recall@10": { "lower": 0.13604064880650404, "point_delta": 0.15478927203065135, "upper": 0.17574363317277317 }, "audio.text_to_audio.recall@5": { "lower": 0.1272585913379362, "point_delta": 0.14444444444444443, "upper": 0.16333503134796237 }, "family.audio": { "lower": 0.08555566687871614, "point_delta": 0.09787709234875733, "upper": 0.11042213832911257 }, "family.image": { "lower": -0.028797083839611177, "point_delta": -0.00996354799513973, "upper": 0.008630012150668272 }, "family.text": { "lower": 0.10934917350884273, "point_delta": 0.11893298491548829, "upper": 0.12872853420036084 }, "image.image_to_text.recall@1": { "lower": -0.08140947752126367, "point_delta": -0.04738760631834751, "upper": -0.012150668286755772 }, "image.image_to_text.recall@10": { "lower": -0.027946537059538274, "point_delta": -0.012150668286755772, "upper": 0.002430133657351154 }, "image.image_to_text.recall@5": { "lower": -0.040097205346294046, "point_delta": -0.01701093560145808, "upper": 0.004860267314702308 }, "image.text_to_image.recall@1": { "lower": 0.01117861482381531, "point_delta": 0.027460510328068045, "upper": 0.042770352369380316 }, "image.text_to_image.recall@10": { "lower": 0.001215066828675577, "point_delta": 0.007776427703523694, "upper": 0.014580801944106925 }, "image.text_to_image.recall@5": { "lower": -0.001701093560145808, "point_delta": 0.010206561360874849, "upper": 0.02187120291616039 }, "text.banking77.accuracy": { "lower": 0.0990581357583631, "point_delta": 0.11168831168831168, "upper": 0.12439713161970098 }, "text.massive-en.accuracy": { "lower": 0.1117468865701784, "point_delta": 0.12617765814266488, "upper": 0.14069455352356733 }, "utility": { "lower": 0.060760442670994376, "point_delta": 0.06894884308970196, "upper": 0.07704865461098383 } }, "method": "paired stratified group percentile; PCG64; fixed candidate pools", "replicates": 2000, "seed": 20260918, "comparison": { "candidate": "vllm-sr/Vela-1.0-Omni-Nano", "candidate_artifact_fingerprint_sha256": "96d76f97478e114b816df3ab1b3c6947e4c5ef0babfb40ab865d7e3c838d34c4", "baseline": "vllm-sr/Vela-1.0-Omni-Nano", "baseline_revision": "3a1efc48995e2b71622d65c6cf71cfc00bbf3aa3", "baseline_artifact_fingerprint_sha256": "fb9cd39c9980a085b75456c715c2295b997784fe733314be68c13635f6384edf" } } }, "training_scope": { "text": "The complete text tower is the frozen official avsolatorio/GIST-small-Embedding-v0 checkpoint at 75e62fd210b9fde790430e0b2f040b0b00a021b1. Its model card discloses MTEB classification training data; no claim of independent pretraining data is made.", "image": "Frozen current vision encoder and initial image head; an orthogonal shared-space rotation is trained on 16,509 COCO training images, 81,998 unique caption texts and 82,522 complete positive associations, then folded into the image affine head.", "audio": "Frozen current speech encoder and initial audio head; a separate orthogonal shared-space rotation is trained on 28,535 LibriSpeech train-clean-100 clips and all positive transcript associations, then folded into the audio affine head.", "procedure": "One fixed 300-step run; only the final artifact is reported. Encoders remain frozen. The two rotations preserve within-modality inner products, up to floating-point rounding.", "evaluation_scope": "Common text classification uses labeled training prototypes with the earlier task-adapted comparator. It is separate from the official frozen-encoder MTEB classification protocols." }, "descriptive_metrics": { "definition": "Macro-F1 is the unweighted mean of per-class F1 across all fixed prototype classes; zero_division=0.", "scope": "Same models, examples, class prototypes and predictions as the accuracy matrix.", "metrics": { "text.banking77.macro_f1": { "small": { "value": 0.7020781051899517, "classes": 77, "queries": 3080 }, "candidate": { "value": 0.8791681858119436, "classes": 77, "queries": 3080 } }, "text.massive-en.macro_f1": { "small": { "value": 0.6171775818092488, "classes": 60, "queries": 2972 }, "candidate": { "value": 0.7923223418549947, "classes": 60, "queries": 2972 } } } }, "standard_benchmarks": { "scope": "Eleven selected standard tasks; not a complete MTEB, MMTEB, MIEB or MAEB benchmark or leaderboard submission.", "library": "mteb", "library_version": "2.21.0", "model_revisions": { "nano": { "repo_id": "vllm-sr/Vela-1.0-Omni-Nano", "revision": "self", "native_artifact_sha256": "50ce808197fb43a1b8913a66d4a47339ec66ecd10d52cbd9f9e752d02f8c43fa", "parameters": 135383808, "embedding_dimensions": 384, "max_text_tokens": 512, "evaluated_native_artifact_sha256": "96d76f97478e114b816df3ab1b3c6947e4c5ef0babfb40ab865d7e3c838d34c4", "evaluated_revision": "de02c367bc9aa7a6f7ad3e59fb38962502e021c1", "evaluated_parameters": 144554112, "inference_equivalence_report": "benchmarks/inference-equivalence.json", "revision_kind": "HF_snapshot" }, "small": { "repo_id": "vllm-sr/multi-modal-embed-small", "revision": "fdf8e01b7b0f3a69ac1ac8e2a64dcb1ede177ba4", "native_artifact_sha256": "684fe0c48a4ddd3944167f6e5dc75e97ada24e5b120bd9a2921f00dc03112eeb", "status": "evaluated_all_11_selected_tasks", "max_text_tokens": 128, "inference_parameters": 133759488 } }, "protocol": { "precision": "float32", "batch_size": 8, "seed": 20260918, "similarity": "Cosine for primary STS/retrieval scores and normalized media-label matching; classification probes and clustering use each official task's algorithms.", "encoder_parameters_frozen": true, "classification": "Official task-specific frozen-embedding probes or fixed-label zero-shot classification.", "sampling": "Official task subsampling and repeated-probe protocols are preserved.", "text": "Each encoder uses its native preprocessing and context limit without added task instructions. Original small: 128 tokens; current Nano: 512 tokens.", "audio": "Mono 16 kHz, first 30 seconds, official MTEB audio preprocessing.", "score_scale": "Metrics retain their native MTEB scale; the model card multiplies primary scores by 100.", "undefined_metric": null, "missing_model_score": "Not measured in this standard evaluation.", "overlap": "No claim of independence from component pretraining data is made." }, "rows": [ { "task": "SICK-R", "type": "STS", "modalities": [ "text" ], "split": "test", "hf_subset": "default", "main_metric": "cosine_spearman", "dataset": { "path": "mteb/sickr-sts", "revision": "20a6d6f312dd54037fe07a32d58e5e168867909d" }, "counts": { "evaluation_pairs": 9927 }, "scores": { "nano": 0.8053174565838819, "small": 0.7139020320965759 }, "metrics": { "nano": { "pearson": 0.8603345909155973, "spearman": 0.8053174398238511, "cosine_pearson": 0.8603345867277692, "cosine_spearman": 0.8053174565838819, "manhattan_pearson": 0.8332190477069294, "manhattan_spearman": 0.8044409124391882, "euclidean_pearson": 0.834049309726515, "euclidean_spearman": 0.8053172111347874, "main_score": 0.8053174565838819, "hf_subset": "default", "languages": [ "eng-Latn" ], "mteb_version": "2.21.0" }, "small": { "pearson": 0.8179714611300387, "spearman": 0.713902378578463, "cosine_pearson": 0.81797146109438, "cosine_spearman": 0.7139020320965759, "manhattan_pearson": 0.7919193141236369, "manhattan_spearman": 0.7128438640347968, "euclidean_pearson": 0.792891259170334, "euclidean_spearman": 0.7139020311053554, "main_score": 0.7139020320965759, "hf_subset": "default", "languages": [ "eng-Latn" ], "mteb_version": "2.21.0" } }, "measurement_by_model": { "small": { "evaluated_native_fingerprint": "684fe0c48a4ddd3944167f6e5dc75e97ada24e5b120bd9a2921f00dc03112eeb", "original_checkpoint_revision": "fdf8e01b7b0f3a69ac1ac8e2a64dcb1ede177ba4", "raw_result_sha256": "e20f02a347841b3d5f61209f0b209c6314d006f8cc94f6e488e06239c862badf", "kind": "direct_frozen_encoder_evaluation" } } }, { "task": "STSBenchmark", "type": "STS", "modalities": [ "text" ], "split": "test", "hf_subset": "default", "main_metric": "cosine_spearman", "dataset": { "path": "mteb/stsbenchmark-sts", "revision": "b0fddb56ed78048fa8b90373c8a3cfc37b684831" }, "counts": { "evaluation_pairs": 1379 }, "scores": { "nano": 0.870781967302773, "small": 0.7729871863268857 }, "metrics": { "nano": { "pearson": 0.8541596951761798, "spearman": 0.8707819111802441, "cosine_pearson": 0.854159691434247, "cosine_spearman": 0.870781967302773, "manhattan_pearson": 0.867700912246326, "manhattan_spearman": 0.8690795304788012, "euclidean_pearson": 0.8699231026532712, "euclidean_spearman": 0.8707819663065923, "main_score": 0.870781967302773, "hf_subset": "default", "languages": [ "eng-Latn" ], "mteb_version": "2.21.0" }, "small": { "pearson": 0.765316203785143, "spearman": 0.7729871863268857, "cosine_pearson": 0.7653161983330018, "cosine_spearman": 0.7729871863268857, "manhattan_pearson": 0.7881395913225815, "manhattan_spearman": 0.7716275619454571, "euclidean_pearson": 0.789916662830987, "euclidean_spearman": 0.7729857841479683, "main_score": 0.7729871863268857, "hf_subset": "default", "languages": [ "eng-Latn" ], "mteb_version": "2.21.0" } }, "measurement_by_model": { "small": { "evaluated_native_fingerprint": "684fe0c48a4ddd3944167f6e5dc75e97ada24e5b120bd9a2921f00dc03112eeb", "original_checkpoint_revision": "fdf8e01b7b0f3a69ac1ac8e2a64dcb1ede177ba4", "raw_result_sha256": "ea0772149930250452298e3525b3405ce4fd5caed4aaf676f3745d4555692a07", "kind": "direct_frozen_encoder_evaluation" } } }, { "task": "ArguAna", "type": "Retrieval", "modalities": [ "text" ], "split": "test", "hf_subset": "default", "main_metric": "ndcg_at_10", "dataset": { "path": "mteb/arguana", "revision": "c22ab2a51041ffd869aaddef7af8d8215647e41a" }, "counts": { "queries": 1406, "documents": 8674 }, "scores": { "nano": 0.59246, "small": 0.32889 }, "metrics": { "nano": { "ndcg_at_1": 0.35349, "ndcg_at_3": 0.49665, "ndcg_at_5": 0.54581, "ndcg_at_10": 0.59246, "ndcg_at_20": 0.6132, "ndcg_at_100": 0.62189, "ndcg_at_1000": 0.62242, "map_at_1": 0.35349, "map_at_3": 0.46005, "map_at_5": 0.48729, "map_at_10": 0.50679, "map_at_20": 0.51261, "map_at_100": 0.51395, "map_at_1000": 0.51397, "recall_at_1": 0.35349, "recall_at_3": 0.60313, "recall_at_5": 0.72262, "recall_at_10": 0.86558, "recall_at_20": 0.94666, "recall_at_100": 0.99147, "recall_at_1000": 0.99573, "accuracy": 0.35349, "precision_at_1": 0.35349, "precision_at_3": 0.20104, "precision_at_5": 0.14452, "precision_at_10": 0.08656, "precision_at_20": 0.04733, "precision_at_100": 0.00991, "precision_at_1000": 0.001, "mrr_at_1": 0.3534850640113798, "mrr_at_3": 0.4600521574205785, "mrr_at_5": 0.48729255571360836, "mrr_at_10": 0.5067903429745535, "mrr_at_20": 0.5126066994732466, "mrr_at_100": 0.5139513666778618, "mrr_at_1000": 0.5139717342260931, "nauc_ndcg_at_1_max": -0.11747299439249903, "nauc_ndcg_at_1_std": -0.11436695908554317, "nauc_ndcg_at_1_diff1": 0.12406213904460131, "nauc_ndcg_at_3_max": -0.0810374905617702, "nauc_ndcg_at_3_std": -0.10902178800246173, "nauc_ndcg_at_3_diff1": 0.09457199601062195, "nauc_ndcg_at_5_max": -0.08485395980147033, "nauc_ndcg_at_5_std": -0.10485663186916257, "nauc_ndcg_at_5_diff1": 0.08582902365914505, "nauc_ndcg_at_10_max": -0.08309974111344137, "nauc_ndcg_at_10_std": -0.11506227102720018, "nauc_ndcg_at_10_diff1": 0.08990562536626041, "nauc_ndcg_at_20_max": -0.0791935521931642, "nauc_ndcg_at_20_std": -0.10319594721919796, "nauc_ndcg_at_20_diff1": 0.09971433495514617, "nauc_ndcg_at_100_max": -0.08635224865444148, "nauc_ndcg_at_100_std": -0.10462787774486179, "nauc_ndcg_at_100_diff1": 0.09791984963152253, "nauc_ndcg_at_1000_max": -0.0879373318368675, "nauc_ndcg_at_1000_std": -0.10553141137460995, "nauc_ndcg_at_1000_diff1": 0.09704987719570578, "nauc_map_at_1_max": -0.11747299439249903, "nauc_map_at_1_std": -0.11436695908554317, "nauc_map_at_1_diff1": 0.12406213904460131, "nauc_map_at_3_max": -0.09064151313996319, "nauc_map_at_3_std": -0.1105170515941861, "nauc_map_at_3_diff1": 0.09940664377727629, "nauc_map_at_5_max": -0.09342459992783216, "nauc_map_at_5_std": -0.10864465771524179, "nauc_map_at_5_diff1": 0.09517709876339578, "nauc_map_at_10_max": -0.09304942751697344, "nauc_map_at_10_std": -0.11264010939720397, "nauc_map_at_10_diff1": 0.09707068152210184, "nauc_map_at_20_max": -0.09221792922323616, "nauc_map_at_20_std": -0.10961211427408772, "nauc_map_at_20_diff1": 0.09943255521491648, "nauc_map_at_100_max": -0.09316799067959038, "nauc_map_at_100_std": -0.10968589627543163, "nauc_map_at_100_diff1": 0.09910011165040408, "nauc_map_at_1000_max": -0.09321385399598632, "nauc_map_at_1000_std": -0.10970513219025511, "nauc_map_at_1000_diff1": 0.09907801272427635, "nauc_recall_at_1_max": -0.11747299439249903, "nauc_recall_at_1_std": -0.11436695908554317, "nauc_recall_at_1_diff1": 0.12406213904460131, "nauc_recall_at_3_max": -0.05070404620274185, "nauc_recall_at_3_std": -0.10426214370217692, "nauc_recall_at_3_diff1": 0.0805612409902579, "nauc_recall_at_5_max": -0.051784236042595515, "nauc_recall_at_5_std": -0.0889570674134141, "nauc_recall_at_5_diff1": 0.049390111709494505, "nauc_recall_at_10_max": -0.013727385294040483, "nauc_recall_at_10_std": -0.13798296178002195, "nauc_recall_at_10_diff1": 0.044023090224588696, "nauc_recall_at_20_max": 0.15251226818625876, "nauc_recall_at_20_std": 0.03363980242022382, "nauc_recall_at_20_diff1": 0.15796490326794294, "nauc_recall_at_100_max": 0.3311354440025815, "nauc_recall_at_100_std": 0.4782526238266158, "nauc_recall_at_100_diff1": 0.25005005298352057, "nauc_recall_at_1000_max": 0.02172236447995406, "nauc_recall_at_1000_std": 0.609819560470231, "nauc_recall_at_1000_diff1": -0.005495147382062338, "nauc_precision_at_1_max": -0.11747299439249903, "nauc_precision_at_1_std": -0.11436695908554317, "nauc_precision_at_1_diff1": 0.12406213904460131, "nauc_precision_at_3_max": -0.05070404620274239, "nauc_precision_at_3_std": -0.10426214370217736, "nauc_precision_at_3_diff1": 0.08056124099025751, "nauc_precision_at_5_max": -0.05178423604259566, "nauc_precision_at_5_std": -0.08895706741341412, "nauc_precision_at_5_diff1": 0.04939011170949452, "nauc_precision_at_10_max": -0.013727385294041244, "nauc_precision_at_10_std": -0.13798296178002226, "nauc_precision_at_10_diff1": 0.044023090224589216, "nauc_precision_at_20_max": 0.15251226818625896, "nauc_precision_at_20_std": 0.03363980242022203, "nauc_precision_at_20_diff1": 0.15796490326794188, "nauc_precision_at_100_max": 0.3311354440025917, "nauc_precision_at_100_std": 0.4782526238266313, "nauc_precision_at_100_diff1": 0.25005005298354305, "nauc_precision_at_1000_max": 0.0217223644799946, "nauc_precision_at_1000_std": 0.6098195604702127, "nauc_precision_at_1000_diff1": -0.0054951473820258456, "nauc_mrr_at_1_max": -0.11747299439249903, "nauc_mrr_at_1_std": -0.11436695908554317, "nauc_mrr_at_1_diff1": 0.12406213904460131, "nauc_mrr_at_3_max": -0.09064151313996319, "nauc_mrr_at_3_std": -0.1105170515941861, "nauc_mrr_at_3_diff1": 0.09940664377727629, "nauc_mrr_at_5_max": -0.09342459992783216, "nauc_mrr_at_5_std": -0.10864465771524179, "nauc_mrr_at_5_diff1": 0.09517709876339578, "nauc_mrr_at_10_max": -0.09304942751697344, "nauc_mrr_at_10_std": -0.11264010939720397, "nauc_mrr_at_10_diff1": 0.09707068152210184, "nauc_mrr_at_20_max": -0.09221792922323616, "nauc_mrr_at_20_std": -0.10961211427408772, "nauc_mrr_at_20_diff1": 0.09943255521491648, "nauc_mrr_at_100_max": -0.09316799067959038, "nauc_mrr_at_100_std": -0.10968589627543163, "nauc_mrr_at_100_diff1": 0.09910011165040408, "nauc_mrr_at_1000_max": -0.09321385399598632, "nauc_mrr_at_1000_std": -0.10970513219025511, "nauc_mrr_at_1000_diff1": 0.09907801272427635, "hit_rate_at_1": 0.35349, "hit_rate_at_3": 0.60313, "hit_rate_at_5": 0.72262, "hit_rate_at_10": 0.86558, "hit_rate_at_20": 0.94666, "hit_rate_at_100": 0.99147, "hit_rate_at_1000": 0.99573, "main_score": 0.59246, "hf_subset": "default", "languages": [ "eng-Latn" ], "mteb_version": "2.21.0" }, "small": { "ndcg_at_1": 0.1458, "ndcg_at_3": 0.23456, "ndcg_at_5": 0.27835, "ndcg_at_10": 0.32889, "ndcg_at_20": 0.36967, "ndcg_at_100": 0.4045, "ndcg_at_1000": 0.41446, "map_at_1": 0.1458, "map_at_3": 0.21136, "map_at_5": 0.2354, "map_at_10": 0.25622, "map_at_20": 0.26767, "map_at_100": 0.27279, "map_at_1000": 0.27315, "recall_at_1": 0.1458, "recall_at_3": 0.30228, "recall_at_5": 0.40967, "recall_at_10": 0.56615, "recall_at_20": 0.72546, "recall_at_100": 0.90896, "recall_at_1000": 0.98791, "accuracy": 0.1458, "precision_at_1": 0.1458, "precision_at_3": 0.10076, "precision_at_5": 0.08193, "precision_at_10": 0.05661, "precision_at_20": 0.03627, "precision_at_100": 0.00909, "precision_at_1000": 0.00099, "mrr_at_1": 0.14580369843527738, "mrr_at_3": 0.21135609293504032, "mrr_at_5": 0.2353959222380275, "mrr_at_10": 0.25621881279776015, "mrr_at_20": 0.26766780266556217, "mrr_at_100": 0.27278502616211503, "mrr_at_1000": 0.2731483695142967, "nauc_ndcg_at_1_max": 0.07124074977242485, "nauc_ndcg_at_1_std": 0.07043132234894327, "nauc_ndcg_at_1_diff1": 0.17867294841183828, "nauc_ndcg_at_3_max": 0.05391218816481269, "nauc_ndcg_at_3_std": 0.08300022308422525, "nauc_ndcg_at_3_diff1": 0.13721153170379125, "nauc_ndcg_at_5_max": 0.04761392610075392, "nauc_ndcg_at_5_std": 0.09330318818026477, "nauc_ndcg_at_5_diff1": 0.129721108359703, "nauc_ndcg_at_10_max": 0.06398717148477524, "nauc_ndcg_at_10_std": 0.11739949823329376, "nauc_ndcg_at_10_diff1": 0.1353789207379315, "nauc_ndcg_at_20_max": 0.06583701467786816, "nauc_ndcg_at_20_std": 0.11760807459667257, "nauc_ndcg_at_20_diff1": 0.1464058778549573, "nauc_ndcg_at_100_max": 0.07271569743212124, "nauc_ndcg_at_100_std": 0.12767735668965335, "nauc_ndcg_at_100_diff1": 0.14440559200255987, "nauc_ndcg_at_1000_max": 0.06760834890113898, "nauc_ndcg_at_1000_std": 0.11565058305226664, "nauc_ndcg_at_1000_diff1": 0.14440414116396014, "nauc_map_at_1_max": 0.07124074977242485, "nauc_map_at_1_std": 0.07043132234894327, "nauc_map_at_1_diff1": 0.17867294841183828, "nauc_map_at_3_max": 0.05536834016676611, "nauc_map_at_3_std": 0.07918800842578326, "nauc_map_at_3_diff1": 0.14582401585695304, "nauc_map_at_5_max": 0.05153209428882134, "nauc_map_at_5_std": 0.08553839884319275, "nauc_map_at_5_diff1": 0.1412262351733516, "nauc_map_at_10_max": 0.05764323340595916, "nauc_map_at_10_std": 0.09567768210275003, "nauc_map_at_10_diff1": 0.14384793290679312, "nauc_map_at_20_max": 0.058797567381990594, "nauc_map_at_20_std": 0.09556268067232435, "nauc_map_at_20_diff1": 0.14719660481591418, "nauc_map_at_100_max": 0.060341050948804, "nauc_map_at_100_std": 0.09715443446030105, "nauc_map_at_100_diff1": 0.14729508200613933, "nauc_map_at_1000_max": 0.060216285099770145, "nauc_map_at_1000_std": 0.09674391170833818, "nauc_map_at_1000_diff1": 0.14729630480806993, "nauc_recall_at_1_max": 0.07124074977242485, "nauc_recall_at_1_std": 0.07043132234894327, "nauc_recall_at_1_diff1": 0.17867294841183828, "nauc_recall_at_3_max": 0.05116463248307958, "nauc_recall_at_3_std": 0.09233493207170033, "nauc_recall_at_3_diff1": 0.11697591005218377, "nauc_recall_at_5_max": 0.0387185231377629, "nauc_recall_at_5_std": 0.11230251236242163, "nauc_recall_at_5_diff1": 0.10273687046821164, "nauc_recall_at_10_max": 0.08768343797110026, "nauc_recall_at_10_std": 0.18174312026830694, "nauc_recall_at_10_diff1": 0.11544199093498575, "nauc_recall_at_20_max": 0.0976177919930251, "nauc_recall_at_20_std": 0.20309410782757367, "nauc_recall_at_20_diff1": 0.15596172000438252, "nauc_recall_at_100_max": 0.21399140718760357, "nauc_recall_at_100_std": 0.45432867879288047, "nauc_recall_at_100_diff1": 0.13074733185800497, "nauc_recall_at_1000_max": 0.29184801184590964, "nauc_recall_at_1000_std": 0.7268160753847613, "nauc_recall_at_1000_diff1": 0.04649951766999351, "nauc_precision_at_1_max": 0.07124074977242485, "nauc_precision_at_1_std": 0.07043132234894327, "nauc_precision_at_1_diff1": 0.17867294841183828, "nauc_precision_at_3_max": 0.051164632483079536, "nauc_precision_at_3_std": 0.09233493207170006, "nauc_precision_at_3_diff1": 0.11697591005218336, "nauc_precision_at_5_max": 0.03871852313776277, "nauc_precision_at_5_std": 0.11230251236242186, "nauc_precision_at_5_diff1": 0.10273687046821152, "nauc_precision_at_10_max": 0.08768343797110059, "nauc_precision_at_10_std": 0.18174312026830697, "nauc_precision_at_10_diff1": 0.11544199093498574, "nauc_precision_at_20_max": 0.09761779199302502, "nauc_precision_at_20_std": 0.20309410782757284, "nauc_precision_at_20_diff1": 0.15596172000438274, "nauc_precision_at_100_max": 0.2139914071876045, "nauc_precision_at_100_std": 0.45432867879288213, "nauc_precision_at_100_diff1": 0.13074733185800294, "nauc_precision_at_1000_max": 0.29184801184591147, "nauc_precision_at_1000_std": 0.7268160753847517, "nauc_precision_at_1000_diff1": 0.04649951766998672, "nauc_mrr_at_1_max": 0.07124074977242485, "nauc_mrr_at_1_std": 0.07043132234894327, "nauc_mrr_at_1_diff1": 0.17867294841183828, "nauc_mrr_at_3_max": 0.05536834016676611, "nauc_mrr_at_3_std": 0.07918800842578326, "nauc_mrr_at_3_diff1": 0.14582401585695304, "nauc_mrr_at_5_max": 0.05153209428882134, "nauc_mrr_at_5_std": 0.08553839884319275, "nauc_mrr_at_5_diff1": 0.1412262351733516, "nauc_mrr_at_10_max": 0.05764323340595916, "nauc_mrr_at_10_std": 0.09567768210275003, "nauc_mrr_at_10_diff1": 0.14384793290679312, "nauc_mrr_at_20_max": 0.058797567381990594, "nauc_mrr_at_20_std": 0.09556268067232435, "nauc_mrr_at_20_diff1": 0.14719660481591418, "nauc_mrr_at_100_max": 0.060341050948804, "nauc_mrr_at_100_std": 0.09715443446030105, "nauc_mrr_at_100_diff1": 0.14729508200613933, "nauc_mrr_at_1000_max": 0.060216285099770145, "nauc_mrr_at_1000_std": 0.09674391170833818, "nauc_mrr_at_1000_diff1": 0.14729630480806993, "hit_rate_at_1": 0.1458, "hit_rate_at_3": 0.30228, "hit_rate_at_5": 0.40967, "hit_rate_at_10": 0.56615, "hit_rate_at_20": 0.72546, "hit_rate_at_100": 0.90896, "hit_rate_at_1000": 0.98791, "main_score": 0.32889, "hf_subset": "default", "languages": [ "eng-Latn" ], "mteb_version": "2.21.0" } }, "measurement_by_model": { "small": { "evaluated_native_fingerprint": "684fe0c48a4ddd3944167f6e5dc75e97ada24e5b120bd9a2921f00dc03112eeb", "original_checkpoint_revision": "fdf8e01b7b0f3a69ac1ac8e2a64dcb1ede177ba4", "raw_result_sha256": "92cbe96f6e84ebd0bc144001424ac46bb3a630ef64bffb433c2cdd63a10c43ec", "kind": "direct_frozen_encoder_evaluation" } } }, { "task": "TwentyNewsgroupsClustering.v2", "type": "Clustering", "modalities": [ "text" ], "split": "test", "hf_subset": "default", "main_metric": "v_measure", "dataset": { "path": "mteb/twentynewsgroups-clustering", "revision": "6125ec4e24fa026cec8a478383ee943acfbd5449" }, "counts": { "population_rows": 59545, "official_encoded_rows": 2381, "official_encoding_fraction": 0.04, "bootstrap_clustering_trials": 10, "resampled_items_per_trial": 16384 }, "scores": { "nano": 0.5087260708286914, "small": 0.44840076500124326 }, "metrics": { "nano": { "v_measures": { "Level 0": [ 0.4966639569479829, 0.5210509442892017, 0.5333989625396617, 0.5094460445939109, 0.5208429535416115, 0.5012604404318527, 0.5210119744629907, 0.4617825059979706, 0.5226704101107219, 0.49913251537100806 ] }, "v_measure": 0.5087260708286914, "v_measure_std": 0.01936580527644773, "ami_scores": { "Level 0": [ 0.49471495769938123, 0.5192201304733433, 0.5316138913144499, 0.5075814211955173, 0.5190252719774525, 0.49937824878079495, 0.5191850550712727, 0.4597515065381579, 0.5208311166380218, 0.49721166070769024 ] }, "ami": 0.5068513260396083, "ami_std": 0.01943429917113934, "main_score": 0.5087260708286914, "hf_subset": "default", "languages": [ "eng-Latn" ], "mteb_version": "2.21.0" }, "small": { "v_measures": { "Level 0": [ 0.46653683369731075, 0.46464237680676646, 0.4425518472920712, 0.4288571327957099, 0.4336826969564481, 0.43371621267306154, 0.43092596596465876, 0.4536864467956253, 0.4741214282291591, 0.45528670880162087 ] }, "v_measure": 0.44840076500124326, "v_measure_std": 0.01575881885263055, "ami_scores": { "Level 0": [ 0.46451147281180905, 0.4625982506803001, 0.44043666823974115, 0.42670551677458946, 0.43154706196129733, 0.4315549515134722, 0.4287870602853061, 0.45163080092170965, 0.47212403002394715, 0.4532359410185627 ] }, "ami": 0.44631317542307347, "ami_std": 0.01581387159916619, "main_score": 0.44840076500124326, "hf_subset": "default", "languages": [ "eng-Latn" ], "mteb_version": "2.21.0" } }, "measurement_by_model": { "small": { "evaluated_native_fingerprint": "684fe0c48a4ddd3944167f6e5dc75e97ada24e5b120bd9a2921f00dc03112eeb", "original_checkpoint_revision": "fdf8e01b7b0f3a69ac1ac8e2a64dcb1ede177ba4", "raw_result_sha256": "62fe52980cb4445b9969d53135fa1663dd8dcd811743a711dde55f2a097ced19", "kind": "direct_frozen_encoder_evaluation" } } }, { "task": "MTOPDomainClassification", "type": "Classification", "modalities": [ "text" ], "split": "validation", "hf_subset": "en", "main_metric": "accuracy", "dataset": { "path": "mteb/MTOPDomainClassification", "revision": "a76d16fae880597b9c73047b50159220a441cb54" }, "counts": { "evaluation_rows": 2235, "available_train_rows": 15667, "official_probe_experiments": 10, "probe_samples_per_label": 8 }, "scores": { "nano": 0.9476510067114093, "small": 0.8543624161073826 }, "metrics": { "nano": { "scores_per_experiment": [ { "accuracy": 0.9521252796420582, "f1": 0.9509396968297774, "f1_weighted": 0.9519302419648948, "precision": 0.9493488952211522, "precision_weighted": 0.9527659658117037, "recall": 0.9535803496254757, "recall_weighted": 0.9521252796420582, "ap": null, "ap_weighted": null }, { "accuracy": 0.9458612975391499, "f1": 0.9439475266607382, "f1_weighted": 0.945767354216612, "precision": 0.9420005561621522, "precision_weighted": 0.9462895866502771, "recall": 0.946397940529448, "recall_weighted": 0.9458612975391499, "ap": null, "ap_weighted": null }, { "accuracy": 0.9284116331096197, "f1": 0.9284988704174789, "f1_weighted": 0.9270840634554431, "precision": 0.9271908775153325, "precision_weighted": 0.9274634710035635, "recall": 0.9317153602473662, "recall_weighted": 0.9284116331096197, "ap": null, "ap_weighted": null }, { "accuracy": 0.9525727069351231, "f1": 0.950987470976036, "f1_weighted": 0.9525185063610705, "precision": 0.9504292297253194, "precision_weighted": 0.9534501308715367, "recall": 0.9526312322737922, "recall_weighted": 0.9525727069351231, "ap": null, "ap_weighted": null }, { "accuracy": 0.9445190156599552, "f1": 0.943383064264026, "f1_weighted": 0.9442288513061988, "precision": 0.9424858291605631, "precision_weighted": 0.944823795147547, "recall": 0.945181115724642, "recall_weighted": 0.9445190156599552, "ap": null, "ap_weighted": null }, { "accuracy": 0.9525727069351231, "f1": 0.949882476921466, "f1_weighted": 0.9526817135384125, "precision": 0.9498620235113855, "precision_weighted": 0.9543297139412104, "recall": 0.9514687067629995, "recall_weighted": 0.9525727069351231, "ap": null, "ap_weighted": null }, { "accuracy": 0.9498881431767338, "f1": 0.9496336255013932, "f1_weighted": 0.9495703695110613, "precision": 0.9490015795096777, "precision_weighted": 0.9511221301014349, "recall": 0.9520470037532829, "recall_weighted": 0.9498881431767338, "ap": null, "ap_weighted": null }, { "accuracy": 0.9570469798657718, "f1": 0.9557194953102311, "f1_weighted": 0.9568992928300293, "precision": 0.9544888722394763, "precision_weighted": 0.9584505479016514, "recall": 0.9586898382241177, "recall_weighted": 0.9570469798657718, "ap": null, "ap_weighted": null }, { "accuracy": 0.9494407158836689, "f1": 0.9460535182757994, "f1_weighted": 0.9492959546761313, "precision": 0.9454088954645755, "precision_weighted": 0.9499145785798135, "recall": 0.9472953341577202, "recall_weighted": 0.9494407158836689, "ap": null, "ap_weighted": null }, { "accuracy": 0.9440715883668904, "f1": 0.9431192151526329, "f1_weighted": 0.9437397930273277, "precision": 0.9435095731735328, "precision_weighted": 0.9461253948734786, "recall": 0.9455297617432863, "recall_weighted": 0.9440715883668904, "ap": null, "ap_weighted": null } ], "accuracy": 0.9476510067114093, "f1": 0.9462164960309579, "f1_weighted": 0.9473716140887183, "precision": 0.9453726331683165, "precision_weighted": 0.9484735314882217, "recall": 0.948453664304213, "recall_weighted": 0.9476510067114093, "ap": null, "ap_weighted": null, "main_score": 0.9476510067114093, "hf_subset": "en", "languages": [ "eng-Latn" ], "mteb_version": "2.21.0" }, "small": { "scores_per_experiment": [ { "accuracy": 0.8250559284116331, "f1": 0.8322814729997016, "f1_weighted": 0.8240316200136756, "precision": 0.8318540374193987, "precision_weighted": 0.8346068877064119, "recall": 0.8430207439526936, "recall_weighted": 0.8250559284116331, "ap": null, "ap_weighted": null }, { "accuracy": 0.8559284116331096, "f1": 0.8552447478537527, "f1_weighted": 0.8547401242258992, "precision": 0.8515715568445064, "precision_weighted": 0.8581578601876096, "recall": 0.8640896175706192, "recall_weighted": 0.8559284116331096, "ap": null, "ap_weighted": null }, { "accuracy": 0.8304250559284116, "f1": 0.8317375249321166, "f1_weighted": 0.8274798893470849, "precision": 0.8268066926411431, "precision_weighted": 0.8285364690282124, "recall": 0.8409132018240374, "recall_weighted": 0.8304250559284116, "ap": null, "ap_weighted": null }, { "accuracy": 0.8648769574944072, "f1": 0.8678594298716482, "f1_weighted": 0.8632422395451538, "precision": 0.865870318819693, "precision_weighted": 0.8646460721917335, "recall": 0.8726494617104266, "recall_weighted": 0.8648769574944072, "ap": null, "ap_weighted": null }, { "accuracy": 0.8532438478747204, "f1": 0.8558930928177336, "f1_weighted": 0.8512711153646526, "precision": 0.8501961024805756, "precision_weighted": 0.8547737986953718, "recall": 0.8657455326712605, "recall_weighted": 0.8532438478747204, "ap": null, "ap_weighted": null }, { "accuracy": 0.8402684563758389, "f1": 0.8478991888671523, "f1_weighted": 0.8396067623115454, "precision": 0.8508550254092774, "precision_weighted": 0.8598285030005444, "recall": 0.862051388925805, "recall_weighted": 0.8402684563758389, "ap": null, "ap_weighted": null }, { "accuracy": 0.8747203579418344, "f1": 0.8781373847485636, "f1_weighted": 0.8731828255054377, "precision": 0.8729737218664414, "precision_weighted": 0.8758466535281065, "recall": 0.8873478774616341, "recall_weighted": 0.8747203579418344, "ap": null, "ap_weighted": null }, { "accuracy": 0.876062639821029, "f1": 0.8754580934866012, "f1_weighted": 0.8749795409187, "precision": 0.8730455332008387, "precision_weighted": 0.8787904116312295, "recall": 0.8820295010368394, "recall_weighted": 0.876062639821029, "ap": null, "ap_weighted": null }, { "accuracy": 0.858165548098434, "f1": 0.8610701024195923, "f1_weighted": 0.8573656937247496, "precision": 0.8561531713932808, "precision_weighted": 0.8662234353266065, "recall": 0.8738853045389997, "recall_weighted": 0.858165548098434, "ap": null, "ap_weighted": null }, { "accuracy": 0.8648769574944072, "f1": 0.8671414562971872, "f1_weighted": 0.8628546885742888, "precision": 0.8607074291115159, "precision_weighted": 0.8655691598347531, "recall": 0.8776421302043499, "recall_weighted": 0.8648769574944072, "ap": null, "ap_weighted": null } ], "accuracy": 0.8543624161073826, "f1": 0.8572722494294049, "f1_weighted": 0.8528754499531187, "precision": 0.8540033589186671, "precision_weighted": 0.8586979251130579, "recall": 0.8669374759896666, "recall_weighted": 0.8543624161073826, "ap": null, "ap_weighted": null, "main_score": 0.8543624161073826, "hf_subset": "en", "languages": [ "eng-Latn" ], "mteb_version": "2.21.0" } }, "measurement_by_model": { "small": { "evaluated_native_fingerprint": "684fe0c48a4ddd3944167f6e5dc75e97ada24e5b120bd9a2921f00dc03112eeb", "original_checkpoint_revision": "fdf8e01b7b0f3a69ac1ac8e2a64dcb1ede177ba4", "raw_result_sha256": "256702650b93bd559d9a2a9b70f32b996b51e166debc0c20d63ce24cc6f605f6", "kind": "direct_frozen_encoder_evaluation" } } }, { "task": "MTOPDomainClassification", "type": "Classification", "modalities": [ "text" ], "split": "test", "hf_subset": "en", "main_metric": "accuracy", "dataset": { "path": "mteb/MTOPDomainClassification", "revision": "a76d16fae880597b9c73047b50159220a441cb54" }, "counts": { "evaluation_rows": 4386, "available_train_rows": 15667, "official_probe_experiments": 10, "probe_samples_per_label": 8 }, "scores": { "nano": 0.9491792065663475, "small": 0.8456452348381213 }, "metrics": { "nano": { "scores_per_experiment": [ { "accuracy": 0.9507523939808481, "f1": 0.9490862282573499, "f1_weighted": 0.9508003256061616, "precision": 0.9476949859831407, "precision_weighted": 0.9520734016967529, "recall": 0.9517099502990664, "recall_weighted": 0.9507523939808481, "ap": null, "ap_weighted": null }, { "accuracy": 0.9473324213406292, "f1": 0.9425946320317333, "f1_weighted": 0.9473133266787116, "precision": 0.9392105665237355, "precision_weighted": 0.9479986101450497, "recall": 0.9468765008841218, "recall_weighted": 0.9473324213406292, "ap": null, "ap_weighted": null }, { "accuracy": 0.9306885544915641, "f1": 0.9261678157710709, "f1_weighted": 0.9296159823715927, "precision": 0.923024327342814, "precision_weighted": 0.9307965712480242, "recall": 0.932201810783149, "recall_weighted": 0.9306885544915641, "ap": null, "ap_weighted": null }, { "accuracy": 0.958048335613315, "f1": 0.9552869971485034, "f1_weighted": 0.9579502405954128, "precision": 0.954624514001619, "precision_weighted": 0.9582682331625527, "recall": 0.9564289859937923, "recall_weighted": 0.958048335613315, "ap": null, "ap_weighted": null }, { "accuracy": 0.9464204286365709, "f1": 0.94433525009326, "f1_weighted": 0.9462665825311201, "precision": 0.9422860464299823, "precision_weighted": 0.9467645700971977, "recall": 0.9469874285322217, "recall_weighted": 0.9464204286365709, "ap": null, "ap_weighted": null }, { "accuracy": 0.9505243958048336, "f1": 0.9447844972663006, "f1_weighted": 0.9507789369812125, "precision": 0.9429136617255864, "precision_weighted": 0.9524852717381878, "recall": 0.9483171314260196, "recall_weighted": 0.9505243958048336, "ap": null, "ap_weighted": null }, { "accuracy": 0.9509803921568627, "f1": 0.9480693879894481, "f1_weighted": 0.9510087641402198, "precision": 0.9447979434260977, "precision_weighted": 0.9522624035738166, "recall": 0.952376748039714, "recall_weighted": 0.9509803921568627, "ap": null, "ap_weighted": null }, { "accuracy": 0.9573643410852714, "f1": 0.9551153583299494, "f1_weighted": 0.9573535659550753, "precision": 0.952401865813031, "precision_weighted": 0.9583040343256545, "recall": 0.9586994420587002, "recall_weighted": 0.9573643410852714, "ap": null, "ap_weighted": null }, { "accuracy": 0.9548563611491108, "f1": 0.9495763118202308, "f1_weighted": 0.9549499907192655, "precision": 0.946851274794657, "precision_weighted": 0.9555638233419986, "recall": 0.9529512436069031, "recall_weighted": 0.9548563611491108, "ap": null, "ap_weighted": null }, { "accuracy": 0.9448244414044688, "f1": 0.9427993333346522, "f1_weighted": 0.9447713522464498, "precision": 0.9432072583992069, "precision_weighted": 0.9468986783847072, "recall": 0.9447479153319868, "recall_weighted": 0.9448244414044688, "ap": null, "ap_weighted": null } ], "accuracy": 0.9491792065663475, "f1": 0.9457815812042497, "f1_weighted": 0.9490809067825221, "precision": 0.9437012444439871, "precision_weighted": 0.9501415597713943, "recall": 0.9491297156955675, "recall_weighted": 0.9491792065663475, "ap": null, "ap_weighted": null, "main_score": 0.9491792065663475, "hf_subset": "en", "languages": [ "eng-Latn" ], "mteb_version": "2.21.0" }, "small": { "scores_per_experiment": [ { "accuracy": 0.8180574555403557, "f1": 0.8207347394289347, "f1_weighted": 0.8170857264694432, "precision": 0.820363266003151, "precision_weighted": 0.8301897328408723, "recall": 0.8338753807988755, "recall_weighted": 0.8180574555403557, "ap": null, "ap_weighted": null }, { "accuracy": 0.8449612403100775, "f1": 0.839516667111528, "f1_weighted": 0.8449410274873432, "precision": 0.8369796042511596, "precision_weighted": 0.8536873756717578, "recall": 0.8523590480814156, "recall_weighted": 0.8449612403100775, "ap": null, "ap_weighted": null }, { "accuracy": 0.8258093935248518, "f1": 0.8185934728273051, "f1_weighted": 0.8251774946806023, "precision": 0.8147370918656317, "precision_weighted": 0.8330066052370535, "recall": 0.8324642506433119, "recall_weighted": 0.8258093935248518, "ap": null, "ap_weighted": null }, { "accuracy": 0.8497492020063839, "f1": 0.8486953297726213, "f1_weighted": 0.848088172519004, "precision": 0.8467111465364261, "precision_weighted": 0.8501008589343039, "recall": 0.853578648077031, "recall_weighted": 0.8497492020063839, "ap": null, "ap_weighted": null }, { "accuracy": 0.8481532147742818, "f1": 0.8467468733444872, "f1_weighted": 0.8467448330993773, "precision": 0.8393119909268087, "precision_weighted": 0.8506182695179204, "recall": 0.858498685671118, "recall_weighted": 0.8481532147742818, "ap": null, "ap_weighted": null }, { "accuracy": 0.8326493388052896, "f1": 0.8352387297727578, "f1_weighted": 0.8347572800634613, "precision": 0.8420129591399498, "precision_weighted": 0.8638673888380122, "recall": 0.8534348314007274, "recall_weighted": 0.8326493388052896, "ap": null, "ap_weighted": null }, { "accuracy": 0.8682170542635659, "f1": 0.8648438659784241, "f1_weighted": 0.8680876089933638, "precision": 0.8585183480506728, "precision_weighted": 0.876371854401442, "recall": 0.8802432046324339, "recall_weighted": 0.8682170542635659, "ap": null, "ap_weighted": null }, { "accuracy": 0.8518011855905153, "f1": 0.8484079910361157, "f1_weighted": 0.8509906563529788, "precision": 0.8443896101449812, "precision_weighted": 0.860550178646213, "recall": 0.8607698617029648, "recall_weighted": 0.8518011855905153, "ap": null, "ap_weighted": null }, { "accuracy": 0.8556771545827634, "f1": 0.8515433915603542, "f1_weighted": 0.856278856298033, "precision": 0.84610847827307, "precision_weighted": 0.8661492710958929, "recall": 0.8658189317867632, "recall_weighted": 0.8556771545827634, "ap": null, "ap_weighted": null }, { "accuracy": 0.8613771089831281, "f1": 0.8586787981057005, "f1_weighted": 0.8603956735839339, "precision": 0.8518135650230533, "precision_weighted": 0.8659927155990171, "recall": 0.8717255305513517, "recall_weighted": 0.8613771089831281, "ap": null, "ap_weighted": null } ], "accuracy": 0.8456452348381213, "f1": 0.8432999858938229, "f1_weighted": 0.8452547329547541, "precision": 0.8400946060214904, "precision_weighted": 0.8550534250782486, "recall": 0.8562768373345993, "recall_weighted": 0.8456452348381213, "ap": null, "ap_weighted": null, "main_score": 0.8456452348381213, "hf_subset": "en", "languages": [ "eng-Latn" ], "mteb_version": "2.21.0" } }, "measurement_by_model": { "small": { "evaluated_native_fingerprint": "684fe0c48a4ddd3944167f6e5dc75e97ada24e5b120bd9a2921f00dc03112eeb", "original_checkpoint_revision": "fdf8e01b7b0f3a69ac1ac8e2a64dcb1ede177ba4", "raw_result_sha256": "256702650b93bd559d9a2a9b70f32b996b51e166debc0c20d63ce24cc6f605f6", "kind": "direct_frozen_encoder_evaluation" } } }, { "task": "OxfordPets", "type": "ImageClassification", "modalities": [ "image" ], "split": "test", "hf_subset": "default", "main_metric": "accuracy", "dataset": { "path": "mteb/OxfordPets", "revision": "557b480fae8d69247be74d9503b378a09425096f" }, "counts": { "evaluation_rows": 3669, "available_train_rows": 3680, "official_probe_experiments": 5, "probe_samples_per_label": 16 }, "scores": { "nano": 0.9126737530662306, "small": 0.4184791496320523 }, "metrics": { "nano": { "scores_per_experiment": [ { "accuracy": 0.9062414826928318, "f1": 0.9045301316021502, "f1_weighted": 0.9050055409371512, "precision": 0.9099213291951324, "precision_weighted": 0.9109544817876485, "recall": 0.9064295837897134, "recall_weighted": 0.9062414826928318, "ap": null, "ap_weighted": null }, { "accuracy": 0.914418097574271, "f1": 0.9126724638115935, "f1_weighted": 0.9132186480967779, "precision": 0.9192904646518116, "precision_weighted": 0.9203471940294945, "recall": 0.9144593228002407, "recall_weighted": 0.914418097574271, "ap": null, "ap_weighted": null }, { "accuracy": 0.9187789588443718, "f1": 0.9177765846692281, "f1_weighted": 0.9184111765200857, "precision": 0.9220755461580387, "precision_weighted": 0.9229278273528245, "recall": 0.9183739103921627, "recall_weighted": 0.9187789588443718, "ap": null, "ap_weighted": null }, { "accuracy": 0.9174161896974653, "f1": 0.9147622354932347, "f1_weighted": 0.9154663340384108, "precision": 0.9209055798643112, "precision_weighted": 0.9221124471295344, "recall": 0.9173606715378323, "recall_weighted": 0.9174161896974653, "ap": null, "ap_weighted": null }, { "accuracy": 0.9065140365222132, "f1": 0.9051669100036687, "f1_weighted": 0.9057728210054364, "precision": 0.9098426973534027, "precision_weighted": 0.9105143284060627, "recall": 0.9059560197061645, "recall_weighted": 0.9065140365222132, "ap": null, "ap_weighted": null } ], "accuracy": 0.9126737530662306, "f1": 0.910981665115975, "f1_weighted": 0.9115749041195725, "precision": 0.9164071234445392, "precision_weighted": 0.9173712557411129, "recall": 0.9125159016452227, "recall_weighted": 0.9126737530662306, "ap": null, "ap_weighted": null, "main_score": 0.9126737530662306, "hf_subset": "default", "languages": [ "eng-Latn" ], "mteb_version": "2.21.0" }, "small": { "scores_per_experiment": [ { "accuracy": 0.4148269283183429, "f1": 0.40253654878032713, "f1_weighted": 0.4024461757957877, "precision": 0.4116431114014033, "precision_weighted": 0.4119318458992588, "recall": 0.4153030284766479, "recall_weighted": 0.4148269283183429, "ap": null, "ap_weighted": null }, { "accuracy": 0.4265467429817389, "f1": 0.4174251045251405, "f1_weighted": 0.4176529394767131, "precision": 0.43921297726376096, "precision_weighted": 0.4396338060561412, "recall": 0.42629825783951975, "recall_weighted": 0.4265467429817389, "ap": null, "ap_weighted": null }, { "accuracy": 0.4088307440719542, "f1": 0.3954869417563968, "f1_weighted": 0.39542431005304346, "precision": 0.40981373804908927, "precision_weighted": 0.40980533913060346, "recall": 0.40880378402552453, "recall_weighted": 0.4088307440719542, "ap": null, "ap_weighted": null }, { "accuracy": 0.4377214499863723, "f1": 0.4251494451023654, "f1_weighted": 0.4253356439185362, "precision": 0.438981154791822, "precision_weighted": 0.439719429760714, "recall": 0.4381658372912861, "recall_weighted": 0.4377214499863723, "ap": null, "ap_weighted": null }, { "accuracy": 0.40446988280185336, "f1": 0.3806382057679881, "f1_weighted": 0.380430443085735, "precision": 0.4088810121814978, "precision_weighted": 0.40903302714302686, "recall": 0.4046956272962792, "recall_weighted": 0.40446988280185336, "ap": null, "ap_weighted": null } ], "accuracy": 0.4184791496320523, "f1": 0.4042472491864436, "f1_weighted": 0.40425790246596305, "precision": 0.42170639873751464, "precision_weighted": 0.4220246895979488, "recall": 0.4186533069858515, "recall_weighted": 0.4184791496320523, "ap": null, "ap_weighted": null, "main_score": 0.4184791496320523, "hf_subset": "default", "languages": [ "eng-Latn" ], "mteb_version": "2.21.0" } }, "measurement_by_model": { "small": { "evaluated_native_fingerprint": "684fe0c48a4ddd3944167f6e5dc75e97ada24e5b120bd9a2921f00dc03112eeb", "original_checkpoint_revision": "fdf8e01b7b0f3a69ac1ac8e2a64dcb1ede177ba4", "raw_result_sha256": "ef1f4928b9ae7d9da25a98b82a28fda3e098db9504ad9372cb5305ad32421a06", "kind": "direct_frozen_encoder_evaluation" } } }, { "task": "TinyImageNetClustering", "type": "ImageClustering", "modalities": [ "image" ], "split": "valid", "hf_subset": "default", "main_metric": "nmi", "dataset": { "path": "mteb/tiny-imagenet", "revision": "e04e259f646f750bd0bdb2a56c2a4577d89bee3d" }, "counts": { "evaluation_rows": 10000 }, "scores": { "nano": 0.6169193395626786, "small": 0.4628490605491802 }, "metrics": { "nano": { "v_measure": 0.6169193395626787, "nmi": 0.6169193395626786, "ari": 0.21497891543638262, "cluster_accuracy": 0.3638, "main_score": 0.6169193395626786, "hf_subset": "default", "languages": [ "eng-Latn" ], "mteb_version": "2.21.0" }, "small": { "v_measure": 0.4628490605491802, "nmi": 0.4628490605491802, "ari": 0.08498222977932614, "cluster_accuracy": 0.2035, "main_score": 0.4628490605491802, "hf_subset": "default", "languages": [ "eng-Latn" ], "mteb_version": "2.21.0" } }, "measurement_by_model": { "small": { "evaluated_native_fingerprint": "684fe0c48a4ddd3944167f6e5dc75e97ada24e5b120bd9a2921f00dc03112eeb", "original_checkpoint_revision": "fdf8e01b7b0f3a69ac1ac8e2a64dcb1ede177ba4", "raw_result_sha256": "eb344e20e7d498aadbbd67a57015cc2220cba42b9550f5822b5a72d0d6cf1514", "kind": "direct_frozen_encoder_evaluation" } } }, { "task": "OxfordPetsZeroShot", "type": "ZeroShotClassification", "modalities": [ "text", "image" ], "split": "test", "hf_subset": "default", "main_metric": "accuracy", "dataset": { "path": "mteb/OxfordPets", "revision": "557b480fae8d69247be74d9503b378a09425096f" }, "counts": { "evaluation_rows": 3669, "fixed_text_labels": 37 }, "scores": { "nano": 0.08013082583810302, "small": 0.1640774052875443 }, "metrics": { "nano": { "accuracy": 0.08013082583810302, "f1": 0.0611345635215895, "f1_weighted": 0.06128470828202288, "precision": 0.1359645526412158, "precision_weighted": 0.13671480699577512, "recall": 0.0798783003173134, "recall_weighted": 0.08013082583810302, "ap": null, "ap_weighted": null, "main_score": 0.08013082583810302, "hf_subset": "default", "languages": [ "eng-Latn" ], "mteb_version": "2.21.0" }, "small": { "accuracy": 0.1640774052875443, "f1": 0.12868294474251654, "f1_weighted": 0.12888736955731406, "precision": 0.15179208674748698, "precision_weighted": 0.15241279079348374, "recall": 0.16458376660623852, "recall_weighted": 0.1640774052875443, "ap": null, "ap_weighted": null, "main_score": 0.1640774052875443, "hf_subset": "default", "languages": [ "eng-Latn" ], "mteb_version": "2.21.0" } }, "measurement_by_model": { "small": { "evaluated_native_fingerprint": "684fe0c48a4ddd3944167f6e5dc75e97ada24e5b120bd9a2921f00dc03112eeb", "original_checkpoint_revision": "fdf8e01b7b0f3a69ac1ac8e2a64dcb1ede177ba4", "raw_result_sha256": "e52e42c4716e62710296df98a90b3a0b986875685748d043cfd7de913e5a6c42", "kind": "direct_frozen_encoder_evaluation" } } }, { "task": "CREMA_D", "type": "AudioClassification", "modalities": [ "audio" ], "split": "train", "hf_subset": "default", "main_metric": "accuracy", "dataset": { "path": "mteb/crema-d", "revision": "a9d0821955c418752248b8310438854e56a1cf34" }, "counts": { "evaluation_rows": 7442, "official_cross_validation_folds": 5, "probe_samples_per_label": 8 }, "scores": { "nano": 0.29763670140167686, "small": 0.2995196855795547 }, "metrics": { "nano": { "scores_per_experiment": [ { "accuracy": 0.2847548690396239, "f1": 0.25980556646698677, "f1_weighted": 0.2606862265023784, "precision": 0.2685141966762838, "precision_weighted": 0.26869432571348717, "recall": 0.283361264324777, "recall_weighted": 0.2847548690396239, "ap": null, "ap_weighted": null }, { "accuracy": 0.2981867024848892, "f1": 0.2927999429509691, "f1_weighted": 0.2952410831252662, "precision": 0.29988591178720175, "precision_weighted": 0.30336098208055257, "recall": 0.29763527945216217, "recall_weighted": 0.2981867024848892, "ap": null, "ap_weighted": null }, { "accuracy": 0.28696236559139787, "f1": 0.26582489649494206, "f1_weighted": 0.26920117230368773, "precision": 0.26489987169540286, "precision_weighted": 0.2682170436584329, "recall": 0.2834616565816636, "recall_weighted": 0.28696236559139787, "ap": null, "ap_weighted": null }, { "accuracy": 0.29973118279569894, "f1": 0.291485315935673, "f1_weighted": 0.28841683151437103, "precision": 0.3068565359099591, "precision_weighted": 0.3086441612810119, "recall": 0.30801945566008065, "recall_weighted": 0.29973118279569894, "ap": null, "ap_weighted": null }, { "accuracy": 0.3185483870967742, "f1": 0.3051858414398825, "f1_weighted": 0.3066161074219095, "precision": 0.310702613862649, "precision_weighted": 0.3133755862555043, "recall": 0.31849511838471833, "recall_weighted": 0.3185483870967742, "ap": null, "ap_weighted": null } ], "accuracy": 0.29763670140167686, "f1": 0.2830203126576907, "f1_weighted": 0.2840322841735226, "precision": 0.29017182598629926, "precision_weighted": 0.2924584197977978, "recall": 0.2981945548806803, "recall_weighted": 0.29763670140167686, "ap": null, "ap_weighted": null, "main_score": 0.29763670140167686, "hf_subset": "default", "languages": [ "eng-Latn" ], "mteb_version": "2.21.0" }, "small": { "scores_per_experiment": [ { "accuracy": 0.2699798522498321, "f1": 0.24239504058809014, "f1_weighted": 0.2423525571760308, "precision": 0.25470036430672977, "precision_weighted": 0.2542205264446883, "recall": 0.26975123145374097, "recall_weighted": 0.2699798522498321, "ap": null, "ap_weighted": null }, { "accuracy": 0.3035594358629953, "f1": 0.29613784795561465, "f1_weighted": 0.29906255840921786, "precision": 0.30407267120079884, "precision_weighted": 0.30800706496852154, "recall": 0.3024788327196796, "recall_weighted": 0.3035594358629953, "ap": null, "ap_weighted": null }, { "accuracy": 0.28225806451612906, "f1": 0.2593664352832718, "f1_weighted": 0.2636113556497887, "precision": 0.2609748445531306, "precision_weighted": 0.26468416511118426, "recall": 0.2775030429767242, "recall_weighted": 0.28225806451612906, "ap": null, "ap_weighted": null }, { "accuracy": 0.3111559139784946, "f1": 0.29454980492169786, "f1_weighted": 0.29125309340152467, "precision": 0.3224943453400395, "precision_weighted": 0.3242765511011472, "recall": 0.31997721819075986, "recall_weighted": 0.3111559139784946, "ap": null, "ap_weighted": null }, { "accuracy": 0.33064516129032256, "f1": 0.3185711713472314, "f1_weighted": 0.3198970407995879, "precision": 0.32320745672952395, "precision_weighted": 0.32608239411677886, "recall": 0.3309376196524952, "recall_weighted": 0.33064516129032256, "ap": null, "ap_weighted": null } ], "accuracy": 0.2995196855795547, "f1": 0.2822040600191812, "f1_weighted": 0.28323532108723004, "precision": 0.29308993642604453, "precision_weighted": 0.295454140348464, "recall": 0.30012958899868, "recall_weighted": 0.2995196855795547, "ap": null, "ap_weighted": null, "main_score": 0.2995196855795547, "hf_subset": "default", "languages": [ "eng-Latn" ], "mteb_version": "2.21.0" } }, "measurement_by_model": { "small": { "evaluated_native_fingerprint": "684fe0c48a4ddd3944167f6e5dc75e97ada24e5b120bd9a2921f00dc03112eeb", "original_checkpoint_revision": "fdf8e01b7b0f3a69ac1ac8e2a64dcb1ede177ba4", "raw_result_sha256": "d79b8d114cb86c460dad13d564c2705a41df5119f83a0f1f27d452f36b64af47", "kind": "direct_frozen_encoder_evaluation" } } }, { "task": "CREMA_DClustering", "type": "AudioClustering", "modalities": [ "audio" ], "split": "train", "hf_subset": "default", "main_metric": "v_measure", "dataset": { "path": "mteb/crema-d", "revision": "a9d0821955c418752248b8310438854e56a1cf34" }, "counts": { "official_stratified_subsample_rows": 2048, "bootstrap_clustering_trials": 10, "resampled_items_per_trial": 16384 }, "scores": { "nano": 0.023601715918443712, "small": 0.024058408888362488 }, "metrics": { "nano": { "v_measures": { "Level 0": [ 0.014290784661453301, 0.02755216925549553, 0.017220745995259155, 0.02907041540588703, 0.017365020267924067, 0.022161217359367304, 0.02806069377818825, 0.017815432451092263, 0.03081951157077731, 0.03166116843899292 ] }, "v_measure": 0.023601715918443712, "v_measure_std": 0.006199268435462002, "ami_scores": { "Level 0": [ 0.01385865484522556, 0.027134003314929028, 0.016785414361355624, 0.028644278423611804, 0.016937404067902156, 0.02173522206231376, 0.0276339699482267, 0.0173943490723341, 0.030397587303172737, 0.031246814385090853 ] }, "ami": 0.023176769778416234, "ami_std": 0.0062032784119843, "main_score": 0.023601715918443712, "hf_subset": "default", "languages": [ "eng-Latn" ], "mteb_version": "2.21.0" }, "small": { "v_measures": { "Level 0": [ 0.041011040847839235, 0.03386543153714212, 0.029099626845588235, 0.013685755785810546, 0.02873132169365058, 0.01881126494637877, 0.011416660820064256, 0.02673877787517492, 0.00569343536524053, 0.031530773166735686 ] }, "v_measure": 0.024058408888362488, "v_measure_std": 0.010611552764041994, "ami_scores": { "Level 0": [ 0.04059499123714819, 0.03344615727359256, 0.02867541136577102, 0.01325228658253946, 0.028312443838566275, 0.01838242676310465, 0.01099044807520909, 0.026321334415061327, 0.005259713019363485, 0.031113076527253092 ] }, "ami": 0.023634828909760913, "ami_std": 0.010617195895208265, "main_score": 0.024058408888362488, "hf_subset": "default", "languages": [ "eng-Latn" ], "mteb_version": "2.21.0" } }, "measurement_by_model": { "small": { "evaluated_native_fingerprint": "684fe0c48a4ddd3944167f6e5dc75e97ada24e5b120bd9a2921f00dc03112eeb", "original_checkpoint_revision": "fdf8e01b7b0f3a69ac1ac8e2a64dcb1ede177ba4", "raw_result_sha256": "b23d4c552b9fa242923fc0c21a898ee63eb32850d9d91e0682b2cb5e8bea0e76", "kind": "direct_frozen_encoder_evaluation" } } }, { "task": "SpeechCommandsZeroshotv0.02", "type": "AudioZeroshotClassification", "modalities": [ "audio", "text" ], "split": "test", "hf_subset": "default", "main_metric": "accuracy", "dataset": { "path": "mteb/SpeechCommandsZeroshotv0.02", "revision": "d5491b288ab76356a5f3cbc634f1dc38e23da2b8" }, "counts": { "evaluation_rows": 4074, "fixed_text_labels": 10 }, "scores": { "nano": 0.1411389297987236, "small": 0.05522827687776141 }, "metrics": { "nano": { "accuracy": 0.1411389297987236, "f1": 0.06629702019800886, "f1_weighted": 0.06559730133350446, "precision": 0.21924492134747023, "precision_weighted": 0.21974351147386872, "recall": 0.14280319771568778, "recall_weighted": 0.1411389297987236, "ap": null, "ap_weighted": null, "main_score": 0.1411389297987236, "hf_subset": "default", "languages": [ "eng-Latn" ], "mteb_version": "2.21.0" }, "small": { "accuracy": 0.05522827687776141, "f1": 0.037875265984290724, "f1_weighted": 0.03803483258988871, "precision": 0.14410977979833245, "precision_weighted": 0.143575354561912, "recall": 0.05521141290348569, "recall_weighted": 0.05522827687776141, "ap": null, "ap_weighted": null, "main_score": 0.05522827687776141, "hf_subset": "default", "languages": [ "eng-Latn" ], "mteb_version": "2.21.0" } }, "measurement_by_model": { "small": { "evaluated_native_fingerprint": "684fe0c48a4ddd3944167f6e5dc75e97ada24e5b120bd9a2921f00dc03112eeb", "original_checkpoint_revision": "fdf8e01b7b0f3a69ac1ac8e2a64dcb1ede177ba4", "raw_result_sha256": "ca19c03bbe2fdcf3c2e1282f5e826af702c152287fde75d67b28ef877fae5335", "kind": "direct_frozen_encoder_evaluation" } } } ], "comparison": { "baseline_model": "vllm-sr/multi-modal-embed-small", "baseline_revision": "fdf8e01b7b0f3a69ac1ac8e2a64dcb1ede177ba4", "baseline_native_artifact_sha256": "684fe0c48a4ddd3944167f6e5dc75e97ada24e5b120bd9a2921f00dc03112eeb", "baseline_standard_status": "not_evaluated_in_this_standard_protocol", "card_bold": "Only unrounded gains against this original baseline are bold; missing baseline scores imply no comparison." }, "observation_scope": "Only this model is reported; original comparator has no observations in the fixed11 protocol. Evaluated/applicable identities are preserved.", "encoding_observations": { "SICK-R": { "nano": { "before": { "text_rows": 0, "text_rows_truncated": 0, "max_original_text_tokens": 0, "image_rows": 0, "audio_rows": 0 }, "after": { "text_rows": 19854, "text_rows_truncated": 0, "max_original_text_tokens": 38, "image_rows": 0, "audio_rows": 0 }, "elapsed_seconds": 10.807591969991336, "peak_allocated_gpu_bytes": 679632896, "peak_reserved_gpu_bytes": 719323136 } }, "STSBenchmark": { "nano": { "before": { "text_rows": 19854, "text_rows_truncated": 0, "max_original_text_tokens": 38, "image_rows": 0, "audio_rows": 0 }, "after": { "text_rows": 22612, "text_rows_truncated": 0, "max_original_text_tokens": 46, "image_rows": 0, "audio_rows": 0 }, "elapsed_seconds": 1.294741225981852, "peak_allocated_gpu_bytes": 681261056, "peak_reserved_gpu_bytes": 723517440 } }, "ArguAna": { "nano": { "before": { "text_rows": 22612, "text_rows_truncated": 0, "max_original_text_tokens": 46, "image_rows": 0, "audio_rows": 0 }, "after": { "text_rows": 32692, "text_rows_truncated": 222, "max_original_text_tokens": 1401, "image_rows": 0, "audio_rows": 0 }, "elapsed_seconds": 15.392282173008425, "peak_allocated_gpu_bytes": 809891328, "peak_reserved_gpu_bytes": 887095296 } }, "TwentyNewsgroupsClustering.v2": { "nano": { "before": { "text_rows": 32692, "text_rows_truncated": 222, "max_original_text_tokens": 1401, "image_rows": 0, "audio_rows": 0 }, "after": { "text_rows": 35073, "text_rows_truncated": 222, "max_original_text_tokens": 1401, "image_rows": 0, "audio_rows": 0 }, "elapsed_seconds": 48.54122133200872, "peak_allocated_gpu_bytes": 684661248, "peak_reserved_gpu_bytes": 727711744 } }, "MTOPDomainClassification": { "nano": { "before": { "text_rows": 35073, "text_rows_truncated": 222, "max_original_text_tokens": 1401, "image_rows": 0, "audio_rows": 0 }, "after": { "text_rows": 43422, "text_rows_truncated": 222, "max_original_text_tokens": 1401, "image_rows": 0, "audio_rows": 0 }, "elapsed_seconds": 25.26324965900858, "peak_allocated_gpu_bytes": 679308288, "peak_reserved_gpu_bytes": 719323136 } }, "OxfordPets": { "nano": { "before": { "text_rows": 43422, "text_rows_truncated": 222, "max_original_text_tokens": 1401, "image_rows": 0, "audio_rows": 0 }, "after": { "text_rows": 43422, "text_rows_truncated": 222, "max_original_text_tokens": 1401, "image_rows": 5816, "audio_rows": 0 }, "elapsed_seconds": 54.48569396699895, "peak_allocated_gpu_bytes": 996472320, "peak_reserved_gpu_bytes": 1207959552 } }, "TinyImageNetClustering": { "nano": { "before": { "text_rows": 43422, "text_rows_truncated": 222, "max_original_text_tokens": 1401, "image_rows": 5816, "audio_rows": 0 }, "after": { "text_rows": 43422, "text_rows_truncated": 222, "max_original_text_tokens": 1401, "image_rows": 15816, "audio_rows": 0 }, "elapsed_seconds": 72.07803665398387, "peak_allocated_gpu_bytes": 996472320, "peak_reserved_gpu_bytes": 1117782016 } }, "OxfordPetsZeroShot": { "nano": { "before": { "text_rows": 43422, "text_rows_truncated": 222, "max_original_text_tokens": 1401, "image_rows": 15816, "audio_rows": 0 }, "after": { "text_rows": 43459, "text_rows_truncated": 222, "max_original_text_tokens": 1401, "image_rows": 19485, "audio_rows": 0 }, "elapsed_seconds": 32.662016286019934, "peak_allocated_gpu_bytes": 996472320, "peak_reserved_gpu_bytes": 1119879168 } }, "CREMA_D": { "nano": { "before": { "text_rows": 43459, "text_rows_truncated": 222, "max_original_text_tokens": 1401, "image_rows": 19485, "audio_rows": 0 }, "after": { "text_rows": 43459, "text_rows_truncated": 222, "max_original_text_tokens": 1401, "image_rows": 19485, "audio_rows": 7442 }, "elapsed_seconds": 135.41051389899803, "peak_allocated_gpu_bytes": 971823104, "peak_reserved_gpu_bytes": 1224736768 } }, "CREMA_DClustering": { "nano": { "before": { "text_rows": 43459, "text_rows_truncated": 222, "max_original_text_tokens": 1401, "image_rows": 19485, "audio_rows": 7442 }, "after": { "text_rows": 43459, "text_rows_truncated": 222, "max_original_text_tokens": 1401, "image_rows": 19485, "audio_rows": 9490 }, "elapsed_seconds": 73.78414583200356, "peak_allocated_gpu_bytes": 971823104, "peak_reserved_gpu_bytes": 1224736768 } }, "SpeechCommandsZeroshotv0.02": { "nano": { "before": { "text_rows": 43459, "text_rows_truncated": 222, "max_original_text_tokens": 1401, "image_rows": 19485, "audio_rows": 9490 }, "after": { "text_rows": 43469, "text_rows_truncated": 222, "max_original_text_tokens": 1401, "image_rows": 19485, "audio_rows": 13564 }, "elapsed_seconds": 63.873327202978544, "peak_allocated_gpu_bytes": 971823104, "peak_reserved_gpu_bytes": 1224736768 } } } }, "capabilities": { "max_text_tokens": 512, "tokens_include_special_tokens": true, "public_text_overflow": "Raises ValueError; shorten or explicitly chunk longer inputs.", "embedding_dimensions": 384, "parameters": 135383808 }, "efficiency": { "status": "not_measured_for_this_release", "parameters": 135383808, "embedding_dimensions": 384, "dense_fp32_index_bytes_per_vector": 1536, "weights_safetensors_bytes": 541598024, "scope": "No new latency or peak-memory measurement is claimed. Vector storage excludes index structures and metadata.", "historical_measurement": { "attention": "native SDPA", "batch_size": 8, "embedding_dtype": "float32", "hardware": "AMD gfx942 GPU", "latency_includes": "Native public encode_text: tokenization, host-to-device transfer and encoding; excludes model loading.", "measured_batches": 20, "measurements": { "128": { "batch_size": 8, "p50_ms": 4.283615489839576, "p95_ms": 4.314010989037342, "peak_allocated_bytes": 653202944, "peak_reserved_bytes": 673185792, "tokens_each_including_special": 128 }, "512": { "batch_size": 8, "p50_ms": 12.445501997717656, "p95_ms": 12.603181986196432, "peak_allocated_bytes": 878682112, "peak_reserved_bytes": 970981376, "tokens_each_including_special": 512 } }, "memory_scope": "Process CUDA allocator peak with the complete multimodal model loaded.", "model_revision": "70c3412bb0ddfb19386d673d4a102c753a574f1b", "native_artifact_sha256": "b596f1dbab89369555e0a72a0d6854d780e305686750949b308c8f167993bdeb", "scope": "Synthetic repeated text-batch measurements on one GPU; not end-to-end service latency.", "tf32": false, "token_lengths_include_special_tokens": true, "torch_version_family": "2.12", "transformers_version": "4.57.6", "warmup_batches": 2, "status": "historical_previous_release_measurement", "applicability": "Historical measurement on its recorded native artifact; not applicable to this text-backbone update." }, "parameter_storage_change": { "old_safetensors_bytes": 578289664, "new_safetensors_bytes": 541598024, "reduction_bytes": 36691640, "old_FP32_parameter_bytes": 578216448, "new_FP32_parameter_bytes": 541535232, "scope": "Parameter storage only; not measured process peak memory or latency. Vector and index dimensions are unchanged." } }, "previous_release": { "model": "vllm-sr/Vela-1.0-Omni-Nano", "revision": "3a1efc48995e2b71622d65c6cf71cfc00bbf3aa3", "native_artifact_sha256": "fb9cd39c9980a085b75456c715c2295b997784fe733314be68c13635f6384edf", "parameters": 133956480, "metrics": { "audio.audio_to_text.recall@1": { "denominator": 2611, "numerator": 235, "value": 0.09000382995021065 }, "audio.audio_to_text.recall@10": { "denominator": 2611, "numerator": 822, "value": 0.31482190731520493 }, "audio.audio_to_text.recall@5": { "denominator": 2611, "numerator": 582, "value": 0.22290310225967064 }, "audio.text_to_audio.recall@1": { "denominator": 2610, "numerator": 366, "value": 0.14022988505747128 }, "audio.text_to_audio.recall@10": { "denominator": 2610, "numerator": 1055, "value": 0.4042145593869732 }, "audio.text_to_audio.recall@5": { "denominator": 2610, "numerator": 827, "value": 0.3168582375478927 }, "image.image_to_text.recall@1": { "denominator": 823, "numerator": 540, "value": 0.6561360874848117 }, "image.image_to_text.recall@10": { "denominator": 823, "numerator": 791, "value": 0.9611178614823815 }, "image.image_to_text.recall@5": { "denominator": 823, "numerator": 731, "value": 0.8882138517618469 }, "image.text_to_image.recall@1": { "denominator": 4115, "numerator": 2184, "value": 0.5307411907654921 }, "image.text_to_image.recall@10": { "denominator": 4115, "numerator": 3847, "value": 0.934872417982989 }, "image.text_to_image.recall@5": { "denominator": 4115, "numerator": 3465, "value": 0.8420413122721749 }, "text.banking77.accuracy": { "denominator": 3080, "numerator": 2366, "value": 0.7681818181818182 }, "text.massive-en.accuracy": { "denominator": 2972, "numerator": 2061, "value": 0.6934724091520862 } }, "descriptive_metrics": { "text.banking77.macro_f1": { "value": 0.7658421887247635, "classes": 77, "queries": 3080 }, "text.massive-en.macro_f1": { "value": 0.6524376423162815, "classes": 60, "queries": 2972 } }, "standard_rows": [ { "task": "SICK-R", "split": "test", "hf_subset": "default", "main_metric": "cosine_spearman", "score": 0.7169618925026362, "metrics": { "pearson": 0.8194169233518556, "spearman": 0.7169617707661863, "cosine_pearson": 0.819416922085493, "cosine_spearman": 0.7169618925026362, "manhattan_pearson": 0.7935695063285152, "manhattan_spearman": 0.7160084195066224, "euclidean_pearson": 0.7944328514895539, "euclidean_spearman": 0.7169617703551949, "main_score": 0.7169618925026362, "hf_subset": "default", "languages": [ "eng-Latn" ], "mteb_version": "2.21.0" } }, { "task": "STSBenchmark", "split": "test", "hf_subset": "default", "main_metric": "cosine_spearman", "score": 0.7792400226361769, "metrics": { "pearson": 0.7717062133633645, "spearman": 0.7792400217447207, "cosine_pearson": 0.7717062156025293, "cosine_spearman": 0.7792400226361769, "manhattan_pearson": 0.7928914389432686, "manhattan_spearman": 0.7777756255094064, "euclidean_pearson": 0.7949414830254274, "euclidean_spearman": 0.779239966513648, "main_score": 0.7792400226361769, "hf_subset": "default", "languages": [ "eng-Latn" ], "mteb_version": "2.21.0" } }, { "task": "ArguAna", "split": "test", "hf_subset": "default", "main_metric": "ndcg_at_10", "score": 0.37514, "metrics": { "ndcg_at_1": 0.17425, "ndcg_at_3": 0.27012, "ndcg_at_5": 0.31718, "ndcg_at_10": 0.37514, "ndcg_at_20": 0.40855, "ndcg_at_100": 0.43759, "ndcg_at_1000": 0.44701, "map_at_1": 0.17425, "map_at_3": 0.24561, "map_at_5": 0.2715, "map_at_10": 0.29556, "map_at_20": 0.30489, "map_at_100": 0.3093, "map_at_1000": 0.30967, "recall_at_1": 0.17425, "recall_at_3": 0.34139, "recall_at_5": 0.45661, "recall_at_10": 0.63514, "recall_at_20": 0.766, "recall_at_100": 0.91679, "recall_at_1000": 0.99004, "accuracy": 0.17425, "precision_at_1": 0.17425, "precision_at_3": 0.1138, "precision_at_5": 0.09132, "precision_at_10": 0.06351, "precision_at_20": 0.0383, "precision_at_100": 0.00917, "precision_at_1000": 0.00099, "mrr_at_1": 0.17425320056899005, "mrr_at_3": 0.24561403508771928, "mrr_at_5": 0.2715030820293978, "mrr_at_10": 0.2955561764772291, "mrr_at_20": 0.30489450370601584, "mrr_at_100": 0.3092982324964941, "mrr_at_1000": 0.30966870165383564, "nauc_ndcg_at_1_max": -0.012854409302840068, "nauc_ndcg_at_1_std": 0.08676381700185752, "nauc_ndcg_at_1_diff1": 0.11334791281377288, "nauc_ndcg_at_3_max": 0.0007934267314863777, "nauc_ndcg_at_3_std": 0.05484641820257593, "nauc_ndcg_at_3_diff1": 0.07348303062976835, "nauc_ndcg_at_5_max": 0.020607224558072373, "nauc_ndcg_at_5_std": 0.0741284488966682, "nauc_ndcg_at_5_diff1": 0.08730888772052957, "nauc_ndcg_at_10_max": 0.035048132350077, "nauc_ndcg_at_10_std": 0.10523053259120095, "nauc_ndcg_at_10_diff1": 0.08020955970997308, "nauc_ndcg_at_20_max": 0.04122292301118309, "nauc_ndcg_at_20_std": 0.12537385022454034, "nauc_ndcg_at_20_diff1": 0.0842700247341082, "nauc_ndcg_at_100_max": 0.04333008607419203, "nauc_ndcg_at_100_std": 0.12294409372370381, "nauc_ndcg_at_100_diff1": 0.08845423517529309, "nauc_ndcg_at_1000_max": 0.030127261370811845, "nauc_ndcg_at_1000_std": 0.10766365950024334, "nauc_ndcg_at_1000_diff1": 0.0861546357811246, "nauc_map_at_1_max": -0.012854409302840068, "nauc_map_at_1_std": 0.08676381700185752, "nauc_map_at_1_diff1": 0.11334791281377288, "nauc_map_at_3_max": -0.0029405293797002733, "nauc_map_at_3_std": 0.06167764879997172, "nauc_map_at_3_diff1": 0.07950290715235987, "nauc_map_at_5_max": 0.00830536326209409, "nauc_map_at_5_std": 0.07256833511730693, "nauc_map_at_5_diff1": 0.08747148302006987, "nauc_map_at_10_max": 0.015140286093876175, "nauc_map_at_10_std": 0.08580662522686537, "nauc_map_at_10_diff1": 0.08508762518017199, "nauc_map_at_20_max": 0.0163028127500422, "nauc_map_at_20_std": 0.09050711291668995, "nauc_map_at_20_diff1": 0.08629712373980958, "nauc_map_at_100_max": 0.016183246425002705, "nauc_map_at_100_std": 0.09000154689011092, "nauc_map_at_100_diff1": 0.08699050038257111, "nauc_map_at_1000_max": 0.015701662616224514, "nauc_map_at_1000_std": 0.0894576065792463, "nauc_map_at_1000_diff1": 0.08694453879028893, "nauc_recall_at_1_max": -0.012854409302840068, "nauc_recall_at_1_std": 0.08676381700185752, "nauc_recall_at_1_diff1": 0.11334791281377288, "nauc_recall_at_3_max": 0.010338756866368223, "nauc_recall_at_3_std": 0.03789278134763109, "nauc_recall_at_3_diff1": 0.059403567851730316, "nauc_recall_at_5_max": 0.05398176066498534, "nauc_recall_at_5_std": 0.08064232472596307, "nauc_recall_at_5_diff1": 0.08949514479783123, "nauc_recall_at_10_max": 0.09984307699040616, "nauc_recall_at_10_std": 0.17703668336587267, "nauc_recall_at_10_diff1": 0.06498907396830537, "nauc_recall_at_20_max": 0.15799758987163673, "nauc_recall_at_20_std": 0.31157412853790695, "nauc_recall_at_20_diff1": 0.07941702046000772, "nauc_recall_at_100_max": 0.37265902510843735, "nauc_recall_at_100_std": 0.5476974790176777, "nauc_recall_at_100_diff1": 0.12977852219557848, "nauc_recall_at_1000_max": 0.3004922701567, "nauc_recall_at_1000_std": 0.7475055455618029, "nauc_recall_at_1000_diff1": -0.05500901218402981, "nauc_precision_at_1_max": -0.012854409302840068, "nauc_precision_at_1_std": 0.08676381700185752, "nauc_precision_at_1_diff1": 0.11334791281377288, "nauc_precision_at_3_max": 0.010338756866368036, "nauc_precision_at_3_std": 0.037892781347630994, "nauc_precision_at_3_diff1": 0.05940356785173012, "nauc_precision_at_5_max": 0.05398176066498592, "nauc_precision_at_5_std": 0.08064232472596324, "nauc_precision_at_5_diff1": 0.08949514479783155, "nauc_precision_at_10_max": 0.09984307699040575, "nauc_precision_at_10_std": 0.17703668336587192, "nauc_precision_at_10_diff1": 0.06498907396830521, "nauc_precision_at_20_max": 0.1579975898716373, "nauc_precision_at_20_std": 0.31157412853790756, "nauc_precision_at_20_diff1": 0.0794170204600081, "nauc_precision_at_100_max": 0.3726590251084365, "nauc_precision_at_100_std": 0.5476974790176794, "nauc_precision_at_100_diff1": 0.1297785221955785, "nauc_precision_at_1000_max": 0.3004922701567069, "nauc_precision_at_1000_std": 0.7475055455618104, "nauc_precision_at_1000_diff1": -0.055009012184042826, "nauc_mrr_at_1_max": -0.012854409302840068, "nauc_mrr_at_1_std": 0.08676381700185752, "nauc_mrr_at_1_diff1": 0.11334791281377288, "nauc_mrr_at_3_max": -0.0029405293797002733, "nauc_mrr_at_3_std": 0.06167764879997172, "nauc_mrr_at_3_diff1": 0.07950290715235987, "nauc_mrr_at_5_max": 0.00830536326209409, "nauc_mrr_at_5_std": 0.07256833511730693, "nauc_mrr_at_5_diff1": 0.08747148302006987, "nauc_mrr_at_10_max": 0.015140286093876175, "nauc_mrr_at_10_std": 0.08580662522686537, "nauc_mrr_at_10_diff1": 0.08508762518017199, "nauc_mrr_at_20_max": 0.0163028127500422, "nauc_mrr_at_20_std": 0.09050711291668995, "nauc_mrr_at_20_diff1": 0.08629712373980958, "nauc_mrr_at_100_max": 0.016183246425002705, "nauc_mrr_at_100_std": 0.09000154689011092, "nauc_mrr_at_100_diff1": 0.08699050038257111, "nauc_mrr_at_1000_max": 0.015701662616224514, "nauc_mrr_at_1000_std": 0.0894576065792463, "nauc_mrr_at_1000_diff1": 0.08694453879028893, "hit_rate_at_1": 0.17425, "hit_rate_at_3": 0.34139, "hit_rate_at_5": 0.45661, "hit_rate_at_10": 0.63514, "hit_rate_at_20": 0.766, "hit_rate_at_100": 0.91679, "hit_rate_at_1000": 0.99004, "main_score": 0.37514, "hf_subset": "default", "languages": [ "eng-Latn" ], "mteb_version": "2.21.0" } }, { "task": "TwentyNewsgroupsClustering.v2", "split": "test", "hf_subset": "default", "main_metric": "v_measure", "score": 0.44884717457494344, "metrics": { "v_measures": { "Level 0": [ 0.48056098748371134, 0.44085942767025554, 0.43416151810540005, 0.45299559752505, 0.44116051339311235, 0.4409873172673812, 0.46155584917327236, 0.444819328389149, 0.44792019268133576, 0.44345101406076604 ] }, "v_measure": 0.44884717457494344, "v_measure_std": 0.01277114663774588, "ami_scores": { "Level 0": [ 0.4786019649327535, 0.43870317373726253, 0.43201951778515824, 0.4509348225877084, 0.43904842199622773, 0.43887546309559994, 0.45953222355170814, 0.44270204450063555, 0.44583346842612326, 0.4413500450061514 ] }, "ami": 0.4467601145619328, "ami_std": 0.012825331450796505, "main_score": 0.44884717457494344, "hf_subset": "default", "languages": [ "eng-Latn" ], "mteb_version": "2.21.0" } }, { "task": "MTOPDomainClassification", "split": "validation", "hf_subset": "en", "main_metric": "accuracy", "score": 0.8758836689038031, "metrics": { "scores_per_experiment": [ { "accuracy": 0.8527964205816555, "f1": 0.8584702959865076, "f1_weighted": 0.8520037076109549, "precision": 0.8563814369311937, "precision_weighted": 0.8604397214633738, "recall": 0.8688240583945851, "recall_weighted": 0.8527964205816555, "ap": null, "ap_weighted": null }, { "accuracy": 0.8675615212527964, "f1": 0.8658660043655293, "f1_weighted": 0.8661413651963479, "precision": 0.8632122218181354, "precision_weighted": 0.8687564991167056, "recall": 0.8733845956286541, "recall_weighted": 0.8675615212527964, "ap": null, "ap_weighted": null }, { "accuracy": 0.8514541387024609, "f1": 0.8524202289070963, "f1_weighted": 0.8491348169886809, "precision": 0.8478156770106394, "precision_weighted": 0.8512267923319679, "recall": 0.8621133428009895, "recall_weighted": 0.8514541387024609, "ap": null, "ap_weighted": null }, { "accuracy": 0.8876957494407158, "f1": 0.8889547651830694, "f1_weighted": 0.886393441219644, "precision": 0.8855262149097257, "precision_weighted": 0.8869170708224573, "recall": 0.8941464114507656, "recall_weighted": 0.8876957494407158, "ap": null, "ap_weighted": null }, { "accuracy": 0.8684563758389262, "f1": 0.8708980361567344, "f1_weighted": 0.8668343457695604, "precision": 0.8655644216656676, "precision_weighted": 0.8685796219847712, "recall": 0.8789277362563833, "recall_weighted": 0.8684563758389262, "ap": null, "ap_weighted": null }, { "accuracy": 0.8662192393736018, "f1": 0.8725560012982636, "f1_weighted": 0.8665613320034511, "precision": 0.8737258252820442, "precision_weighted": 0.8845738920578841, "recall": 0.8863222925022911, "recall_weighted": 0.8662192393736018, "ap": null, "ap_weighted": null }, { "accuracy": 0.8953020134228188, "f1": 0.8976332709955689, "f1_weighted": 0.8945899154754817, "precision": 0.8912735415957905, "precision_weighted": 0.8979949591784757, "recall": 0.908092721226657, "recall_weighted": 0.8953020134228188, "ap": null, "ap_weighted": null }, { "accuracy": 0.9029082774049217, "f1": 0.9022310464802398, "f1_weighted": 0.9020525018623339, "precision": 0.899983538025133, "precision_weighted": 0.9036775461946326, "recall": 0.9067592943176375, "recall_weighted": 0.9029082774049217, "ap": null, "ap_weighted": null }, { "accuracy": 0.876510067114094, "f1": 0.8776887066559357, "f1_weighted": 0.8761842615927375, "precision": 0.8722809982436143, "precision_weighted": 0.8829904469624968, "recall": 0.8888620916111349, "recall_weighted": 0.876510067114094, "ap": null, "ap_weighted": null }, { "accuracy": 0.8899328859060402, "f1": 0.8916131163462019, "f1_weighted": 0.8885446252332447, "precision": 0.8855482140891268, "precision_weighted": 0.8923990752693544, "recall": 0.9020276692969261, "recall_weighted": 0.8899328859060402, "ap": null, "ap_weighted": null } ], "accuracy": 0.8758836689038031, "f1": 0.8778331472375147, "f1_weighted": 0.8748440312952436, "precision": 0.8741312089571069, "precision_weighted": 0.8797555625382119, "recall": 0.8869460213486026, "recall_weighted": 0.8758836689038031, "ap": null, "ap_weighted": null, "main_score": 0.8758836689038031, "hf_subset": "en", "languages": [ "eng-Latn" ], "mteb_version": "2.21.0" } }, { "task": "MTOPDomainClassification", "split": "test", "hf_subset": "en", "main_metric": "accuracy", "score": 0.8671910624715002, "metrics": { "scores_per_experiment": [ { "accuracy": 0.8492932056543547, "f1": 0.8506491551806685, "f1_weighted": 0.8484338913731928, "precision": 0.847171519646365, "precision_weighted": 0.857621901965629, "recall": 0.8634426315569471, "recall_weighted": 0.8492932056543547, "ap": null, "ap_weighted": null }, { "accuracy": 0.8593251253989969, "f1": 0.8525570927270549, "f1_weighted": 0.8588269042231532, "precision": 0.849564509883911, "precision_weighted": 0.8659425848467468, "recall": 0.8649266896842414, "recall_weighted": 0.8593251253989969, "ap": null, "ap_weighted": null }, { "accuracy": 0.8508891928864569, "f1": 0.8445077740960396, "f1_weighted": 0.8500136039882616, "precision": 0.83976876536009, "precision_weighted": 0.8551546537983608, "recall": 0.8573297788521569, "recall_weighted": 0.8508891928864569, "ap": null, "ap_weighted": null }, { "accuracy": 0.8702690378476973, "f1": 0.8680982220684261, "f1_weighted": 0.8688890737378698, "precision": 0.8651925230157509, "precision_weighted": 0.8699395458498034, "recall": 0.873034481111479, "recall_weighted": 0.8702690378476973, "ap": null, "ap_weighted": null }, { "accuracy": 0.8657090743274054, "f1": 0.8639593934313017, "f1_weighted": 0.8646291417790959, "precision": 0.8570893217526883, "precision_weighted": 0.8672508842970594, "recall": 0.8739690035000478, "recall_weighted": 0.8657090743274054, "ap": null, "ap_weighted": null }, { "accuracy": 0.8520291837665299, "f1": 0.8534418954962738, "f1_weighted": 0.8548114072639492, "precision": 0.85875145236006, "precision_weighted": 0.8803115046026259, "recall": 0.869567932536695, "recall_weighted": 0.8520291837665299, "ap": null, "ap_weighted": null }, { "accuracy": 0.8869129046967624, "f1": 0.8849494340089117, "f1_weighted": 0.8871874824644057, "precision": 0.8779834780597185, "precision_weighted": 0.8955288700582439, "recall": 0.9009013020211545, "recall_weighted": 0.8869129046967624, "ap": null, "ap_weighted": null }, { "accuracy": 0.875968992248062, "f1": 0.8715420414043028, "f1_weighted": 0.8753897998685723, "precision": 0.86652024620862, "precision_weighted": 0.8811614860860878, "recall": 0.881802359118481, "recall_weighted": 0.875968992248062, "ap": null, "ap_weighted": null }, { "accuracy": 0.8755129958960328, "f1": 0.8710791039688399, "f1_weighted": 0.8760974709416987, "precision": 0.8647173948263537, "precision_weighted": 0.8831121050674345, "recall": 0.8845140142042784, "recall_weighted": 0.8755129958960328, "ap": null, "ap_weighted": null }, { "accuracy": 0.886000911992704, "f1": 0.8827257199119498, "f1_weighted": 0.8855721338303499, "precision": 0.8760471482914741, "precision_weighted": 0.8912159468319844, "recall": 0.8952867706565555, "recall_weighted": 0.886000911992704, "ap": null, "ap_weighted": null } ], "accuracy": 0.8671910624715002, "f1": 0.8643509832293768, "f1_weighted": 0.8669850909470549, "precision": 0.8602806359405031, "precision_weighted": 0.8747239483403975, "recall": 0.8764774963242037, "recall_weighted": 0.8671910624715002, "ap": null, "ap_weighted": null, "main_score": 0.8671910624715002, "hf_subset": "en", "languages": [ "eng-Latn" ], "mteb_version": "2.21.0" } }, { "task": "OxfordPets", "split": "test", "hf_subset": "default", "main_metric": "accuracy", "score": 0.9126737530662306, "metrics": { "scores_per_experiment": [ { "accuracy": 0.9062414826928318, "f1": 0.9045301316021502, "f1_weighted": 0.9050055409371512, "precision": 0.9099213291951324, "precision_weighted": 0.9109544817876485, "recall": 0.9064295837897134, "recall_weighted": 0.9062414826928318, "ap": null, "ap_weighted": null }, { "accuracy": 0.914418097574271, "f1": 0.9126724638115935, "f1_weighted": 0.9132186480967779, "precision": 0.9192904646518116, "precision_weighted": 0.9203471940294945, "recall": 0.9144593228002407, "recall_weighted": 0.914418097574271, "ap": null, "ap_weighted": null }, { "accuracy": 0.9187789588443718, "f1": 0.9177765846692281, "f1_weighted": 0.9184111765200857, "precision": 0.9220755461580387, "precision_weighted": 0.9229278273528245, "recall": 0.9183739103921627, "recall_weighted": 0.9187789588443718, "ap": null, "ap_weighted": null }, { "accuracy": 0.9174161896974653, "f1": 0.9147622354932347, "f1_weighted": 0.9154663340384108, "precision": 0.9209055798643112, "precision_weighted": 0.9221124471295344, "recall": 0.9173606715378323, "recall_weighted": 0.9174161896974653, "ap": null, "ap_weighted": null }, { "accuracy": 0.9065140365222132, "f1": 0.9051669100036687, "f1_weighted": 0.9057728210054364, "precision": 0.9098426973534027, "precision_weighted": 0.9105143284060627, "recall": 0.9059560197061645, "recall_weighted": 0.9065140365222132, "ap": null, "ap_weighted": null } ], "accuracy": 0.9126737530662306, "f1": 0.910981665115975, "f1_weighted": 0.9115749041195725, "precision": 0.9164071234445392, "precision_weighted": 0.9173712557411129, "recall": 0.9125159016452227, "recall_weighted": 0.9126737530662306, "ap": null, "ap_weighted": null, "main_score": 0.9126737530662306, "hf_subset": "default", "languages": [ "eng-Latn" ], "mteb_version": "2.21.0" } }, { "task": "TinyImageNetClustering", "split": "valid", "hf_subset": "default", "main_metric": "nmi", "score": 0.6169193395626786, "metrics": { "v_measure": 0.6169193395626787, "nmi": 0.6169193395626786, "ari": 0.21497891543638262, "cluster_accuracy": 0.3638, "main_score": 0.6169193395626786, "hf_subset": "default", "languages": [ "eng-Latn" ], "mteb_version": "2.21.0" } }, { "task": "OxfordPetsZeroShot", "split": "test", "hf_subset": "default", "main_metric": "accuracy", "score": 0.1163804851458163, "metrics": { "accuracy": 0.1163804851458163, "f1": 0.085822378073852, "f1_weighted": 0.08530524305774022, "precision": 0.08885688520073651, "precision_weighted": 0.08856851170630324, "recall": 0.11709192030781942, "recall_weighted": 0.1163804851458163, "ap": null, "ap_weighted": null, "main_score": 0.1163804851458163, "hf_subset": "default", "languages": [ "eng-Latn" ], "mteb_version": "2.21.0" } }, { "task": "CREMA_D", "split": "train", "hf_subset": "default", "main_metric": "accuracy", "score": 0.29763670140167686, "metrics": { "scores_per_experiment": [ { "accuracy": 0.2847548690396239, "f1": 0.25980556646698677, "f1_weighted": 0.2606862265023784, "precision": 0.2685141966762838, "precision_weighted": 0.26869432571348717, "recall": 0.283361264324777, "recall_weighted": 0.2847548690396239, "ap": null, "ap_weighted": null }, { "accuracy": 0.2981867024848892, "f1": 0.2927999429509691, "f1_weighted": 0.2952410831252662, "precision": 0.29988591178720175, "precision_weighted": 0.30336098208055257, "recall": 0.29763527945216217, "recall_weighted": 0.2981867024848892, "ap": null, "ap_weighted": null }, { "accuracy": 0.28696236559139787, "f1": 0.26582489649494206, "f1_weighted": 0.26920117230368773, "precision": 0.26489987169540286, "precision_weighted": 0.2682170436584329, "recall": 0.2834616565816636, "recall_weighted": 0.28696236559139787, "ap": null, "ap_weighted": null }, { "accuracy": 0.29973118279569894, "f1": 0.291485315935673, "f1_weighted": 0.28841683151437103, "precision": 0.3068565359099591, "precision_weighted": 0.3086441612810119, "recall": 0.30801945566008065, "recall_weighted": 0.29973118279569894, "ap": null, "ap_weighted": null }, { "accuracy": 0.3185483870967742, "f1": 0.3051858414398825, "f1_weighted": 0.3066161074219095, "precision": 0.310702613862649, "precision_weighted": 0.3133755862555043, "recall": 0.31849511838471833, "recall_weighted": 0.3185483870967742, "ap": null, "ap_weighted": null } ], "accuracy": 0.29763670140167686, "f1": 0.2830203126576907, "f1_weighted": 0.2840322841735226, "precision": 0.29017182598629926, "precision_weighted": 0.2924584197977978, "recall": 0.2981945548806803, "recall_weighted": 0.29763670140167686, "ap": null, "ap_weighted": null, "main_score": 0.29763670140167686, "hf_subset": "default", "languages": [ "eng-Latn" ], "mteb_version": "2.21.0" } }, { "task": "CREMA_DClustering", "split": "train", "hf_subset": "default", "main_metric": "v_measure", "score": 0.023601715918443712, "metrics": { "v_measures": { "Level 0": [ 0.014290784661453301, 0.02755216925549553, 0.017220745995259155, 0.02907041540588703, 0.017365020267924067, 0.022161217359367304, 0.02806069377818825, 0.017815432451092263, 0.03081951157077731, 0.03166116843899292 ] }, "v_measure": 0.023601715918443712, "v_measure_std": 0.006199268435462002, "ami_scores": { "Level 0": [ 0.01385865484522556, 0.027134003314929028, 0.016785414361355624, 0.028644278423611804, 0.016937404067902156, 0.02173522206231376, 0.0276339699482267, 0.0173943490723341, 0.030397587303172737, 0.031246814385090853 ] }, "ami": 0.023176769778416234, "ami_std": 0.0062032784119843, "main_score": 0.023601715918443712, "hf_subset": "default", "languages": [ "eng-Latn" ], "mteb_version": "2.21.0" } }, { "task": "SpeechCommandsZeroshotv0.02", "split": "test", "hf_subset": "default", "main_metric": "accuracy", "score": 0.14972999509081983, "metrics": { "accuracy": 0.14972999509081983, "f1": 0.06986246632697812, "f1_weighted": 0.07083899432793754, "precision": 0.15715381051614374, "precision_weighted": 0.1573685095178851, "recall": 0.14663865718054886, "recall_weighted": 0.14972999509081983, "ap": null, "ap_weighted": null, "main_score": 0.14972999509081983, "hf_subset": "default", "languages": [ "eng-Latn" ], "mteb_version": "2.21.0" } } ], "full_benchmarks": [ { "benchmark": "MTEB(eng, v2)", "framework_version": "2.21.0", "coverage": { "completed_tasks": 41, "required_tasks": 41, "task_types": 7 }, "model_revision": "70c3412bb0ddfb19386d673d4a102c753a574f1b", "native_artifact_sha256": "b596f1dbab89369555e0a72a0d6854d780e305686750949b308c8f167993bdeb", "aggregation": { "mean_task": 0.5337130125278776, "mean_task_type": 0.5197553164396411, "definition": "First apply the official TaskResult aggregation within each task. Mean task gives each of the 41 tasks equal weight; mean task type first averages within each type, then gives all seven types equal weight.", "score_scale": "0-1; multiply by 100 for display", "by_task_type": { "Classification": { "score": 0.6552554795025091, "tasks": 8 }, "Clustering": { "score": 0.47072550487857034, "tasks": 8 }, "PairClassification": { "score": 0.7494593709186261, "tasks": 3 }, "Reranking": { "score": 0.455685, "tasks": 2 }, "Retrieval": { "score": 0.290152, "tasks": 10 }, "STS": { "score": 0.724513458257586, "tasks": 9 }, "Summarization": { "score": 0.29249640152019635, "tasks": 1 } } }, "report": "benchmarks/mteb-eng-v2.json", "text_component_applicability": { "applies_to_revision": "self", "applies_to_native_artifact_sha256": "fb9cd39c9980a085b75456c715c2295b997784fe733314be68c13635f6384edf", "evaluated_revision": "70c3412bb0ddfb19386d673d4a102c753a574f1b", "evaluated_native_artifact_sha256": "b596f1dbab89369555e0a72a0d6854d780e305686750949b308c8f167993bdeb", "basis": "Exact component tensor and inference-file identity; original measured identity is retained.", "report": "benchmarks/component-equivalence.json", "new_full_panel_inference": false, "component": "text" } }, { "benchmark": "MAEB(beta, audio-only)", "framework_version": "2.21.0", "coverage": { "completed_tasks": 19, "required_tasks": 19, "subset_split_slots": 134, "task_types": 5, "selection": "All MAEB(beta) tasks whose modalities are exactly [audio].", "full_MAEB": false, "excluded": "Tasks that require text/audio or other joint-modality inputs." }, "model_revision": "70c3412bb0ddfb19386d673d4a102c753a574f1b", "native_artifact_sha256": "b596f1dbab89369555e0a72a0d6854d780e305686750949b308c8f167993bdeb", "aggregation": { "official": { "Any2AnyRetrieval": 0.6890050000000001, "AudioClassification": 0.39566923071716426, "AudioClustering": 0.054218008138417706, "AudioPairClassification": 0.571886979184418, "AudioReranking": 0.63794, "Mean(Task)": 0.39776955262406916, "Mean(TaskType)": 0.46974384360800003 }, "definition": "Official TaskResult aggregation within each task, followed by official mean_task, mean_task_type and task_types aggregation. The 134 subset/split slots are not averaged as independent tasks.", "score_scale": "0-1; multiply by 100 for display" }, "audio_component_applicability": { "applies_to_revision": "self", "applies_to_native_artifact_sha256": "fb9cd39c9980a085b75456c715c2295b997784fe733314be68c13635f6384edf", "evaluated_revision": "70c3412bb0ddfb19386d673d4a102c753a574f1b", "evaluated_native_artifact_sha256": "b596f1dbab89369555e0a72a0d6854d780e305686750949b308c8f167993bdeb", "basis": "Exact component tensor and inference-file identity; original measured identity is retained.", "report": "benchmarks/component-equivalence.json", "new_full_panel_inference": false, "component": "audio" }, "report": "benchmarks/maeb-audio-only.json" } ] }, "full_benchmarks": [ { "benchmark": "MTEB(eng, v2)", "framework_version": "2.21.0", "coverage": { "completed_tasks": 41, "required_tasks": 41, "task_types": 7 }, "model_revision": "29994142677559ca657c27b657eda0dddbf583bd4cf42a3a31095ab3f99d1646", "native_artifact_sha256": "29994142677559ca657c27b657eda0dddbf583bd4cf42a3a31095ab3f99d1646", "aggregation": { "mean_task": 0.6488431514310018, "mean_task_type": 0.6078184975654923, "by_task_type": { "Classification": { "score": 0.7782970776285105, "tasks": 8 }, "Clustering": { "score": 0.48962597555764586, "tasks": 8 }, "PairClassification": { "score": 0.8468485561827029, "tasks": 3 }, "Reranking": { "score": 0.473475, "tasks": 2 }, "Retrieval": { "score": 0.520098, "tasks": 10 }, "STS": { "score": 0.8280405301305157, "tasks": 9 }, "Summarization": { "score": 0.31834434345907126, "tasks": 1 } }, "definition": "Official within-task aggregation; equal task mean across 41 tasks, and equal mean of seven within-type means.", "score_scale": "0-1; display multiplied by 100" }, "report": "benchmarks/mteb-eng-v2.json", "text_component_applicability": { "component": "text", "applies_to_revision": "self", "applies_to_native_artifact_sha256": "96d76f97478e114b816df3ab1b3c6947e4c5ef0babfb40ab865d7e3c838d34c4", "evaluated_model": "local/Vela-Omni-Nano-GIST-Rotation", "evaluated_revision": "29994142677559ca657c27b657eda0dddbf583bd4cf42a3a31095ab3f99d1646", "evaluated_native_artifact_sha256": "29994142677559ca657c27b657eda0dddbf583bd4cf42a3a31095ab3f99d1646", "revision_kind": "local native artifact fingerprint, not a Hugging Face commit", "report": "benchmarks/component-equivalence.json", "new_full_panel_inference": false, "basis": "199 exact text tensors and byte-identical inference/tokenizer/configuration under the same official evaluator protocol." }, "inference_profile_applicability": { "revision": "self", "native_artifact_sha256": "50ce808197fb43a1b8913a66d4a47339ec66ecd10d52cbd9f9e752d02f8c43fa", "parameters": 135383808, "report": "benchmarks/inference-equivalence.json", "new_benchmark_run": false } }, { "benchmark": "MAEB(beta, audio-only)", "framework_version": "2.21.0", "coverage": { "completed_tasks": 19, "required_tasks": 19, "subset_split_slots": 134, "task_types": 5, "selection": "All MAEB(beta) tasks whose modalities are exactly [audio].", "full_MAEB": false, "excluded": "Tasks that require text/audio or other joint-modality inputs." }, "model_revision": "96d76f97478e114b816df3ab1b3c6947e4c5ef0babfb40ab865d7e3c838d34c4", "native_artifact_sha256": "96d76f97478e114b816df3ab1b3c6947e4c5ef0babfb40ab865d7e3c838d34c4", "aggregation": { "official": { "Any2AnyRetrieval": 0.6890050000000001, "AudioClassification": 0.39553889510295764, "AudioClustering": 0.054218008138417706, "AudioPairClassification": 0.5716888010864635, "AudioReranking": 0.63794, "Mean(Task)": 0.3976628038845884, "Mean(TaskType)": 0.4696781408655678 }, "definition": "Official TaskResult aggregation within each task, followed by official mean_task, mean_task_type and task_types aggregation. The 134 subset/split slots are not averaged as independent tasks.", "score_scale": "0-1; multiply by 100 for display" }, "report": "benchmarks/maeb-audio-only.json", "inference_profile_applicability": { "revision": "self", "native_artifact_sha256": "50ce808197fb43a1b8913a66d4a47339ec66ecd10d52cbd9f9e752d02f8c43fa", "parameters": 135383808, "report": "benchmarks/inference-equivalence.json", "new_benchmark_run": false } } ], "net_benefit": { "scope": "Product-weighted progress comparison, not an official benchmark aggregate or SOTA claim.", "baseline_model": "vllm-sr/Vela-1.0-Omni-Nano", "baseline_revision": "3a1efc48995e2b71622d65c6cf71cfc00bbf3aa3", "baseline_native_artifact_sha256": "fb9cd39c9980a085b75456c715c2295b997784fe733314be68c13635f6384edf", "candidate_native_artifact_sha256": "96d76f97478e114b816df3ab1b3c6947e4c5ef0babfb40ab865d7e3c838d34c4", "score_scale": "0-100; changes in percentage points", "components": [ { "baseline_100": 59.37287180704907, "candidate_100": 67.57407300861536, "component": "common14", "delta_pp": 8.201201201566288, "weight": 0.3 }, { "baseline_100": 51.97553164396411, "candidate_100": 60.781849756549235, "component": "English41_MeanTaskType", "delta_pp": 8.806318112585124, "weight": 0.3 }, { "baseline_100": 54.865785925824184, "candidate_100": 53.65746394890041, "component": "image3", "delta_pp": -1.208321976923776, "weight": 0.2 }, { "baseline_100": 46.9743843608, "candidate_100": 46.96781408655678, "component": "audio19_MeanTaskType", "delta_pp": -0.00657027424322, "weight": 0.2 } ], "weighted_net_delta_pp": 4.859277344012025, "common_definition": "Equal mean of three families: two text accuracies, all six image retrieval recalls, all six audio retrieval recalls.", "image3_definition": "Equal mean of OxfordPets classification accuracy, TinyImageNet clustering NMI and OxfordPets zero-shot accuracy.", "text_audio_definition": "Full English41 and full audio19 mean task-type scores. English uses the explicit exact text-component bridge; audio is freshly evaluated.", "coverage": { "common_metrics": 14, "standard_tasks": 11, "standard_split_rows": 12, "English_tasks": 41, "audio_tasks": 19, "audio_subset_split_slots": 134 }, "common_metric_delta_pp": { "audio.audio_to_text.recall@1": 8.157793948678668, "audio.audio_to_text.recall@10": 15.51129835312141, "audio.audio_to_text.recall@5": 14.362313289927231, "audio.text_to_audio.recall@1": 11.417624521072797, "audio.text_to_audio.recall@10": 15.478927203065133, "audio.text_to_audio.recall@5": 14.444444444444445, "image.image_to_text.recall@1": -4.738760631834751, "image.image_to_text.recall@10": -1.2150668286755772, "image.image_to_text.recall@5": -1.701093560145808, "image.text_to_image.recall@1": 2.7460510328068044, "image.text_to_image.recall@10": 0.7776427703523694, "image.text_to_image.recall@5": 1.0206561360874848, "text.banking77.accuracy": 11.168831168831169, "text.massive-en.accuracy": 12.617765814266487 }, "standard_task_deltas": [ { "baseline": 0.7169618925026362, "candidate": 0.8053174565838819, "hf_subset": "default", "main_metric": "cosine_spearman", "split": "test", "status": "complete", "task": "SICK-R", "delta_pp": 8.835556408124567 }, { "baseline": 0.7792400226361769, "candidate": 0.870781967302773, "hf_subset": "default", "main_metric": "cosine_spearman", "split": "test", "status": "complete", "task": "STSBenchmark", "delta_pp": 9.154194466659614 }, { "baseline": 0.37514, "candidate": 0.59246, "hf_subset": "default", "main_metric": "ndcg_at_10", "split": "test", "status": "complete", "task": "ArguAna", "delta_pp": 21.732000000000003 }, { "baseline": 0.44884717457494344, "candidate": 0.5087260708286914, "hf_subset": "default", "main_metric": "v_measure", "split": "test", "status": "complete", "task": "TwentyNewsgroupsClustering.v2", "delta_pp": 5.987889625374793 }, { "baseline": 0.8758836689038031, "candidate": 0.9476510067114093, "hf_subset": "en", "main_metric": "accuracy", "split": "validation", "status": "complete", "task": "MTOPDomainClassification", "delta_pp": 7.176733780760625 }, { "baseline": 0.8671910624715002, "candidate": 0.9491792065663475, "hf_subset": "en", "main_metric": "accuracy", "split": "test", "status": "complete", "task": "MTOPDomainClassification", "delta_pp": 8.198814409484722 }, { "baseline": 0.9126737530662306, "candidate": 0.9126737530662306, "hf_subset": "default", "main_metric": "accuracy", "split": "test", "status": "complete", "task": "OxfordPets", "delta_pp": 0.0 }, { "baseline": 0.6169193395626786, "candidate": 0.6169193395626786, "hf_subset": "default", "main_metric": "nmi", "split": "valid", "status": "complete", "task": "TinyImageNetClustering", "delta_pp": 0.0 }, { "baseline": 0.1163804851458163, "candidate": 0.08013082583810302, "hf_subset": "default", "main_metric": "accuracy", "split": "test", "status": "complete", "task": "OxfordPetsZeroShot", "delta_pp": -3.6249659307713276 }, { "baseline": 0.29763670140167686, "candidate": 0.29763670140167686, "hf_subset": "default", "main_metric": "accuracy", "split": "train", "status": "complete", "task": "CREMA_D", "delta_pp": 0.0 }, { "baseline": 0.023601715918443712, "candidate": 0.023601715918443712, "hf_subset": "default", "main_metric": "v_measure", "split": "train", "status": "complete", "task": "CREMA_DClustering", "delta_pp": 0.0 }, { "baseline": 0.14972999509081983, "candidate": 0.1411389297987236, "hf_subset": "default", "main_metric": "accuracy", "split": "test", "status": "complete", "task": "SpeechCommandsZeroshotv0.02", "delta_pp": -0.8591065292096217 } ], "English_task_deltas": [ { "candidate_100": 59.245999999999995, "current_100": 37.513999999999996, "delta_pp": 21.732, "primary_metric": "ndcg_at_10", "task": "ArguAna", "task_type": "Retrieval" }, { "candidate_100": 64.88847259326147, "current_100": 65.25395409281045, "delta_pp": -0.36548149954897724, "primary_metric": "v_measure", "task": "ArXivHierarchicalClusteringP2P", "task_type": "Clustering" }, { "candidate_100": 57.44315313869382, "current_100": 55.989532291560415, "delta_pp": 1.4536208471334078, "primary_metric": "v_measure", "task": "ArXivHierarchicalClusteringS2S", "task_type": "Clustering" }, { "candidate_100": 62.329, "current_100": 60.514, "delta_pp": 1.8149999999999977, "primary_metric": "map_at_1000", "task": "AskUbuntuDupQuestions", "task_type": "Reranking" }, { "candidate_100": 86.98752423842483, "current_100": 62.24539420714732, "delta_pp": 24.742130031277505, "primary_metric": "cosine_spearman", "task": "BIOSSES", "task_type": "STS" }, { "candidate_100": 82.14285714285714, "current_100": 69.57467532467533, "delta_pp": 12.568181818181813, "primary_metric": "accuracy", "task": "Banking77Classification", "task_type": "Classification" }, { "candidate_100": 41.313796273674654, "current_100": 39.82923119776807, "delta_pp": 1.4845650759065876, "primary_metric": "v_measure", "task": "BiorxivClusteringP2P.v2", "task_type": "Clustering" }, { "candidate_100": 56.979, "current_100": 42.976, "delta_pp": 14.003, "primary_metric": "ndcg_at_10", "task": "CQADupstackGamingRetrieval", "task_type": "Retrieval" }, { "candidate_100": 39.586, "current_100": 30.362000000000002, "delta_pp": 9.223999999999997, "primary_metric": "ndcg_at_10", "task": "CQADupstackUnixRetrieval", "task_type": "Retrieval" }, { "candidate_100": 31.846000000000004, "current_100": 16.073999999999998, "delta_pp": 15.772000000000006, "primary_metric": "ndcg_at_10", "task": "ClimateFEVERHardNegatives", "task_type": "Retrieval" }, { "candidate_100": 87.576, "current_100": 24.055, "delta_pp": 63.520999999999994, "primary_metric": "ndcg_at_10", "task": "FEVERHardNegatives", "task_type": "Retrieval" }, { "candidate_100": 39.143, "current_100": 22.588, "delta_pp": 16.555, "primary_metric": "ndcg_at_10", "task": "FiQA2018", "task_type": "Retrieval" }, { "candidate_100": 66.349, "current_100": 26.987, "delta_pp": 39.36200000000001, "primary_metric": "ndcg_at_10", "task": "HotpotQAHardNegatives", "task_type": "Retrieval" }, { "candidate_100": 91.94760000000001, "current_100": 62.1364, "delta_pp": 29.811200000000007, "primary_metric": "accuracy", "task": "ImdbClassification", "task_type": "Classification" }, { "candidate_100": 94.91792065663475, "current_100": 86.71910624715002, "delta_pp": 8.198814409484726, "primary_metric": "accuracy", "task": "MTOPDomainClassification", "task_type": "Classification" }, { "candidate_100": 70.96839273705447, "current_100": 60.39340954942838, "delta_pp": 10.574983187626096, "primary_metric": "accuracy", "task": "MassiveIntentClassification", "task_type": "Classification" }, { "candidate_100": 76.11297915265635, "current_100": 73.37256220578345, "delta_pp": 2.740416946872898, "primary_metric": "accuracy", "task": "MassiveScenarioClassification", "task_type": "Classification" }, { "candidate_100": 39.83023091223438, "current_100": 37.76813963073551, "delta_pp": 2.0620912814988657, "primary_metric": "v_measure", "task": "MedrxivClusteringP2P.v2", "task_type": "Clustering" }, { "candidate_100": 37.875084453707764, "current_100": 36.13349306174702, "delta_pp": 1.7415913919607462, "primary_metric": "v_measure", "task": "MedrxivClusteringS2S.v2", "task_type": "Clustering" }, { "candidate_100": 32.366, "current_100": 30.623, "delta_pp": 1.7429999999999986, "primary_metric": "max_over_subqueries_map_at_1000", "task": "MindSmallReranking", "task_type": "Reranking" }, { "candidate_100": 21.889, "current_100": 16.827, "delta_pp": 5.061999999999998, "primary_metric": "ndcg_at_10", "task": "SCIDOCS", "task_type": "Retrieval" }, { "candidate_100": 80.53174565838819, "current_100": 71.69618925026361, "delta_pp": 8.835556408124575, "primary_metric": "cosine_spearman", "task": "SICK-R", "task_type": "STS" }, { "candidate_100": 75.5658801516535, "current_100": 68.16915974310523, "delta_pp": 7.396720408548276, "primary_metric": "cosine_spearman", "task": "STS12", "task_type": "STS" }, { "candidate_100": 86.26346974038235, "current_100": 74.40342742804673, "delta_pp": 11.860042312335622, "primary_metric": "cosine_spearman", "task": "STS13", "task_type": "STS" }, { "candidate_100": 82.29882793900394, "current_100": 68.02653891259911, "delta_pp": 14.272289026404835, "primary_metric": "cosine_spearman", "task": "STS14", "task_type": "STS" }, { "candidate_100": 88.73648284326485, "current_100": 77.86144933320074, "delta_pp": 10.875033510064114, "primary_metric": "cosine_spearman", "task": "STS15", "task_type": "STS" }, { "candidate_100": 87.0781967302773, "current_100": 77.92400226361768, "delta_pp": 9.154194466659618, "primary_metric": "cosine_spearman", "task": "STSBenchmark", "task_type": "STS" }, { "candidate_100": 95.79964673462752, "current_100": 89.79096125036985, "delta_pp": 6.0086854842576685, "primary_metric": "max_ap", "task": "SprintDuplicateQuestions", "task_type": "PairClassification" }, { "candidate_100": 58.424418738430106, "current_100": 57.34464990059502, "delta_pp": 1.0797688378350827, "primary_metric": "v_measure", "task": "StackExchangeClustering.v2", "task_type": "Clustering" }, { "candidate_100": 41.053017253245386, "current_100": 39.37668627014545, "delta_pp": 1.6763309830999376, "primary_metric": "v_measure", "task": "StackExchangeClusteringP2P.v2", "task_type": "Clustering" }, { "candidate_100": 69.121, "current_100": 41.292, "delta_pp": 27.828999999999994, "primary_metric": "ndcg_at_10", "task": "TRECCOVID", "task_type": "Retrieval" }, { "candidate_100": 48.363, "current_100": 31.477, "delta_pp": 16.886, "primary_metric": "ndcg_at_10", "task": "Touche2020Retrieval.v3", "task_type": "Retrieval" }, { "candidate_100": 71.904296875, "current_100": 64.4091796875, "delta_pp": 7.4951171875, "primary_metric": "accuracy", "task": "ToxicConversationsClassification", "task_type": "Classification" }, { "candidate_100": 62.79286926994907, "current_100": 50.77815506508207, "delta_pp": 12.014714204867005, "primary_metric": "accuracy", "task": "TweetSentimentExtractionClassification", "task_type": "Classification" }, { "candidate_100": 50.87260708286914, "current_100": 44.88471745749435, "delta_pp": 5.987889625374791, "primary_metric": "v_measure", "task": "TwentyNewsgroupsClustering.v2", "task_type": "Clustering" }, { "candidate_100": 72.95151691526065, "current_100": 56.34475441213181, "delta_pp": 16.60676250312884, "primary_metric": "max_ap", "task": "TwitterSemEval2015", "task_type": "PairClassification" }, { "candidate_100": 85.30340320492272, "current_100": 78.70209561308617, "delta_pp": 6.6013075918365445, "primary_metric": "max_ap", "task": "TwitterURLCorpus", "task_type": "PairClassification" }, { "candidate_100": 31.834434345907127, "current_100": 29.249640152019634, "delta_pp": 2.584794193887493, "primary_metric": "cosine_spearman", "task": "SummEvalSummarization.v2", "task_type": "Summarization" }, { "candidate_100": 71.8507462686567, "current_100": 56.82089552238806, "delta_pp": 15.02985074626865, "primary_metric": "accuracy", "task": "AmazonCounterfactualClassification", "task_type": "Classification" }, { "candidate_100": 89.02101343562433, "current_100": 84.16589279728525, "delta_pp": 4.8551206383390735, "primary_metric": "cosine_spearman", "task": "STS17", "task_type": "STS" }, { "candidate_100": 68.75333638044488, "current_100": 67.57005849656171, "delta_pp": 1.1832778838831643, "primary_metric": "cosine_spearman", "task": "STS22.v2", "task_type": "STS" } ], "audio_task_delta_pp": { "BeijingOpera": 0.0, "BirdCLEF": 0.0, "CREMADPairClassification": 0.12104996267769952, "CREMA_D": 0.0, "CREMA_DClustering": 0.0, "CommonLanguageAgeDetection": 0.0, "GTZANAudioReranking": 0.0, "GTZANGenre": 0.0, "IEMOCAPGender": 0.0, "JamAltArtistA2ARetrieval": 0.0, "MInDS14": 0.0, "MridinghamTonic": -0.14336917562723928, "NMSQAPairClassification": -0.1520590751812989, "SIBFLEURS": 0.0, "VehicleSoundClustering": 0.0, "VoxCelebSA": 0.0, "VoxPopuliAccentPairClassification": -0.02844431688273641, "VoxPopuliGenderClustering": 0.0, "VoxPopuliLanguageID": 0.0 }, "paired_interval_scope": "Paired intervals cover the original common metrics, family and two-primary-per-family utility. They are not confidence intervals for the new all-14 common mean or the four-component product aggregate.", "inference_profile_applicability": { "revision": "self", "native_artifact_sha256": "50ce808197fb43a1b8913a66d4a47339ec66ecd10d52cbd9f9e752d02f8c43fa", "parameters": 135383808, "report": "benchmarks/inference-equivalence.json", "new_benchmark_run": false } }, "paired_uncertainty_scope": "Paired intervals cover the original common metrics, family and two-primary-per-family utility. They are not confidence intervals for the new all-14 common mean or the four-component product aggregate.", "inference_profile": { "revision": "self", "native_artifact_sha256": "50ce808197fb43a1b8913a66d4a47339ec66ecd10d52cbd9f9e752d02f8c43fa", "parameters": 135383808, "report": "benchmarks/inference-equivalence.json", "new_benchmark_run": false }, "scorecard_comparison": { "primary_comparator": "vllm-sr/multi-modal-embed-small", "revision": "fdf8e01b7b0f3a69ac1ac8e2a64dcb1ede177ba4", "common_protocol_measured": true, "original_reference_evidence": "benchmarks/original-baseline.json", "previous_Vela_not_the_primary_comparator": true, "historical_release_progress": "benchmarks/release-history.md" } }