diff --git "a/benchmarks/previous-release-scores.json" "b/benchmarks/previous-release-scores.json" new file mode 100644--- /dev/null +++ "b/benchmarks/previous-release-scores.json" @@ -0,0 +1,4317 @@ +{ + "schema_version": 2, + "model": "llm-semantic-router/Vela-1.0-Omni-Nano", + "revision_semantics": "self denotes this published inference package; evaluated_* identities retain original measurements. The exact public-inference equivalence report binds score applicability.", + "score_scale": "0-1; multiply by 100 for percentages", + "matrix": { + "small": { + "model": "llm-semantic-router/multi-modal-embed-small", + "revision": "fdf8e01b7b0f3a69ac1ac8e2a64dcb1ede177ba4", + "artifact_fingerprint_sha256": "684fe0c48a4ddd3944167f6e5dc75e97ada24e5b120bd9a2921f00dc03112eeb", + "metrics": { + "text.banking77.accuracy": { + "numerator": 2169, + "denominator": 3080, + "value": 0.7042207792207792 + }, + "text.massive-en.accuracy": { + "numerator": 1960, + "denominator": 2972, + "value": 0.6594885598923284 + }, + "image.image_to_text.recall@1": { + "numerator": 336, + "denominator": 823, + "value": 0.4082624544349939 + }, + "image.image_to_text.recall@5": { + "numerator": 553, + "denominator": 823, + "value": 0.6719319562575942 + }, + "image.image_to_text.recall@10": { + "numerator": 646, + "denominator": 823, + "value": 0.7849331713244229 + }, + "image.text_to_image.recall@1": { + "numerator": 1242, + "denominator": 4115, + "value": 0.3018226002430134 + }, + "image.text_to_image.recall@5": { + "numerator": 2445, + "denominator": 4115, + "value": 0.5941676792223572 + }, + "image.text_to_image.recall@10": { + "numerator": 3057, + "denominator": 4115, + "value": 0.7428918590522479 + }, + "audio.audio_to_text.recall@1": { + "numerator": 110, + "denominator": 2611, + "value": 0.04212945231711988 + }, + "audio.audio_to_text.recall@5": { + "numerator": 313, + "denominator": 2611, + "value": 0.11987744159325929 + }, + "audio.audio_to_text.recall@10": { + "numerator": 497, + "denominator": 2611, + "value": 0.1903485254691689 + }, + "audio.text_to_audio.recall@1": { + "numerator": 250, + "denominator": 2610, + "value": 0.09578544061302682 + }, + "audio.text_to_audio.recall@5": { + "numerator": 588, + "denominator": 2610, + "value": 0.22528735632183908 + }, + "audio.text_to_audio.recall@10": { + "numerator": 801, + "denominator": 2610, + "value": 0.30689655172413793 + } + } + }, + "candidate": { + "model": "llm-semantic-router/Vela-1.0-Omni-Nano", + "revision": "self", + "artifact_fingerprint_sha256": "50ce808197fb43a1b8913a66d4a47339ec66ecd10d52cbd9f9e752d02f8c43fa", + "metrics": { + "audio.audio_to_text.recall@1": { + "denominator": 2611, + "numerator": 448, + "value": 0.17158176943699732 + }, + "audio.audio_to_text.recall@10": { + "denominator": 2611, + "numerator": 1227, + "value": 0.469934890846419 + }, + "audio.audio_to_text.recall@5": { + "denominator": 2611, + "numerator": 957, + "value": 0.36652623515894295 + }, + "audio.text_to_audio.recall@1": { + "denominator": 2610, + "numerator": 664, + "value": 0.25440613026819925 + }, + "audio.text_to_audio.recall@10": { + "denominator": 2610, + "numerator": 1459, + "value": 0.5590038314176246 + }, + "audio.text_to_audio.recall@5": { + "denominator": 2610, + "numerator": 1204, + "value": 0.46130268199233715 + }, + "image.image_to_text.recall@1": { + "denominator": 823, + "numerator": 501, + "value": 0.6087484811664642 + }, + "image.image_to_text.recall@10": { + "denominator": 823, + "numerator": 781, + "value": 0.9489671931956257 + }, + "image.image_to_text.recall@5": { + "denominator": 823, + "numerator": 717, + "value": 0.8712029161603888 + }, + "image.text_to_image.recall@1": { + "denominator": 4115, + "numerator": 2297, + "value": 0.5582017010935602 + }, + "image.text_to_image.recall@10": { + "denominator": 4115, + "numerator": 3879, + "value": 0.9426488456865127 + }, + "image.text_to_image.recall@5": { + "denominator": 4115, + "numerator": 3507, + "value": 0.8522478736330498 + }, + "text.banking77.accuracy": { + "denominator": 3080, + "numerator": 2710, + "value": 0.8798701298701299 + }, + "text.massive-en.accuracy": { + "denominator": 2972, + "numerator": 2436, + "value": 0.819650067294751 + } + }, + "parameters": 135383808, + "evaluated_native_artifact_sha256": "96d76f97478e114b816df3ab1b3c6947e4c5ef0babfb40ab865d7e3c838d34c4", + "evaluated_revision": "de02c367bc9aa7a6f7ad3e59fb38962502e021c1", + "evaluated_parameters": 144554112, + "inference_equivalence_report": "benchmarks/inference-equivalence.json" + } + }, + "datasets": { + "banking77": { + "source": "https://github.com/PolyAI-LDN/task-specific-datasets", + "revision": "57ec275d8078af65b7731c2a98be812d844a6d6b", + "license": "CC-BY-4.0", + "split": "test", + "queries": 3080, + "classes": 77 + }, + "massive-en": { + "source": "https://huggingface.co/datasets/AmazonScience/massive", + "revision": "ff6bd8e4b27c3543e4f8fe2108f32bb95a6f8740", + "license": "CC-BY-4.0", + "configuration": "en-US", + "split": "test", + "queries": 2972, + "prototype_classes": 60 + }, + "image": { + "source": "https://cocodataset.org/", + "caption_split_source": "https://cs.stanford.edu/people/karpathy/deepimagesent/", + "split": "Karpathy test, CC-BY 2.0 image subset", + "images": 823, + "captions": 4115, + "image_license": "CC-BY-2.0", + "caption_license": "CC-BY-4.0" + }, + "audio": { + "source": "https://www.openslr.org/12", + "split": "test-clean", + "archive_sha256": "39fde525e59672dc6d1551919b1478f724438a95aa55f874b576be21967e6c23", + "license": "CC-BY-4.0", + "clips": 2611, + "unique_transcripts": 2610, + "maximum_seconds": 30 + } + }, + "protocol": { + "embedding_dtype": "float32", + "attention": "SDPA", + "autocast": false, + "tf32": false, + "transformers_version": "4.57.6", + "torch_version_family": "2.12", + "text_max_tokens": 128, + "truncation": "right, native tokenizer including special tokens", + "batch_sizes": { + "text": 64, + "image": 8, + "audio": 8 + }, + "pooling": "native full-dimension model pooling", + "similarity": "FP32 cosine of L2-normalized embeddings", + "ties": "lexicographic candidate ID", + "retrieval": "complete candidate pools; all annotated positives", + "classification": "Nearest normalized mean class prototype from labeled training examples; the original small comparator has task-adapted history, while current GIST is frozen.", + "aggregation": "Each family is the mean of its two primary metrics; utility is the mean of all six primaries. Retrieval primaries are R@1; R@5 and R@10 are reported separately.", + "evaluation_scope": "fixed test-split evaluation pools reused across releases", + "text_training_scope": "Current GIST text tower is frozen; supervised training prototypes are recomputed under the fixed common protocol. The earlier small comparator has task-adapted history. This is not the official MTEB classification protocol." + }, + "paired_uncertainty": { + "original": { + "intervals": { + "audio.audio_to_text.recall@1": { + "lower": 0.11352472177122043, + "point_delta": 0.12945231711987745, + "upper": 0.14665221762155492 + }, + "audio.audio_to_text.recall@10": { + "lower": 0.25338950715018704, + "point_delta": 0.2795863653772501, + "upper": 0.30833452722063037 + }, + "audio.audio_to_text.recall@5": { + "lower": 0.22198472459131105, + "point_delta": 0.24664879356568364, + "upper": 0.2736971153456769 + }, + "audio.text_to_audio.recall@1": { + "lower": 0.14202610349345837, + "point_delta": 0.15862068965517243, + "upper": 0.17613425616168824 + }, + "audio.text_to_audio.recall@10": { + "lower": 0.23515433278434666, + "point_delta": 0.25210727969348656, + "upper": 0.2707672328928673 + }, + "audio.text_to_audio.recall@5": { + "lower": 0.21796104940042046, + "point_delta": 0.23601532567049807, + "upper": 0.2547919081648684 + }, + "family.audio": { + "lower": 0.1322499685832721, + "point_delta": 0.14403650338752494, + "upper": 0.15644009735751394 + }, + "family.image": { + "lower": 0.20084750911300123, + "point_delta": 0.2284325637910085, + "upper": 0.25637910085054677 + }, + "family.text": { + "lower": 0.15747009669727144, + "point_delta": 0.16790542902588662, + "upper": 0.17869785413173905 + }, + "image.image_to_text.recall@1": { + "lower": 0.1591737545565006, + "point_delta": 0.20048602673147023, + "upper": 0.24301336573511542 + }, + "image.image_to_text.recall@10": { + "lower": 0.1336573511543135, + "point_delta": 0.16403402187120292, + "upper": 0.19441069258809235 + }, + "image.image_to_text.recall@5": { + "lower": 0.16160388821385177, + "point_delta": 0.19927095990279464, + "upper": 0.23572296476306198 + }, + "image.text_to_image.recall@1": { + "lower": 0.23013365735115432, + "point_delta": 0.25637910085054677, + "upper": 0.2831166464155528 + }, + "image.text_to_image.recall@10": { + "lower": 0.17618469015795868, + "point_delta": 0.1997569866342649, + "upper": 0.22235722964763063 + }, + "image.text_to_image.recall@5": { + "lower": 0.23207776427703525, + "point_delta": 0.2580801944106926, + "upper": 0.28360267314702303 + }, + "text.banking77.accuracy": { + "lower": 0.16066212268743915, + "point_delta": 0.17564935064935064, + "upper": 0.19058596303826084 + }, + "text.massive-en.accuracy": { + "lower": 0.14554621848739496, + "point_delta": 0.1601615074024226, + "upper": 0.17569841804106362 + }, + "utility": { + "lower": 0.16995251624973068, + "point_delta": 0.18012483206814003, + "upper": 0.1910563694034248 + } + }, + "method": "paired stratified group percentile; PCG64; fixed candidate pools", + "replicates": 2000, + "seed": 20260918, + "comparison": { + "candidate": "llm-semantic-router/Vela-1.0-Omni-Nano", + "candidate_artifact_fingerprint_sha256": "96d76f97478e114b816df3ab1b3c6947e4c5ef0babfb40ab865d7e3c838d34c4", + "baseline": "llm-semantic-router/multi-modal-embed-small", + "baseline_revision": "fdf8e01b7b0f3a69ac1ac8e2a64dcb1ede177ba4", + "baseline_artifact_fingerprint_sha256": "684fe0c48a4ddd3944167f6e5dc75e97ada24e5b120bd9a2921f00dc03112eeb" + } + }, + "previous": { + "intervals": { + "audio.audio_to_text.recall@1": { + "lower": 0.06612466882994839, + "point_delta": 0.08157793948678667, + "upper": 0.0967748551165683 + }, + "audio.audio_to_text.recall@10": { + "lower": 0.13256556125090607, + "point_delta": 0.15511298353121408, + "upper": 0.18059527375151307 + }, + "audio.audio_to_text.recall@5": { + "lower": 0.12086266595311207, + "point_delta": 0.1436231328992723, + "upper": 0.16909569992403942 + }, + "audio.text_to_audio.recall@1": { + "lower": 0.09860745201354912, + "point_delta": 0.11417624521072797, + "upper": 0.13014135201210755 + }, + "audio.text_to_audio.recall@10": { + "lower": 0.13604064880650404, + "point_delta": 0.15478927203065135, + "upper": 0.17574363317277317 + }, + "audio.text_to_audio.recall@5": { + "lower": 0.1272585913379362, + "point_delta": 0.14444444444444443, + "upper": 0.16333503134796237 + }, + "family.audio": { + "lower": 0.08555566687871614, + "point_delta": 0.09787709234875733, + "upper": 0.11042213832911257 + }, + "family.image": { + "lower": -0.028797083839611177, + "point_delta": -0.00996354799513973, + "upper": 0.008630012150668272 + }, + "family.text": { + "lower": 0.10934917350884273, + "point_delta": 0.11893298491548829, + "upper": 0.12872853420036084 + }, + "image.image_to_text.recall@1": { + "lower": -0.08140947752126367, + "point_delta": -0.04738760631834751, + "upper": -0.012150668286755772 + }, + "image.image_to_text.recall@10": { + "lower": -0.027946537059538274, + "point_delta": -0.012150668286755772, + "upper": 0.002430133657351154 + }, + "image.image_to_text.recall@5": { + "lower": -0.040097205346294046, + "point_delta": -0.01701093560145808, + "upper": 0.004860267314702308 + }, + "image.text_to_image.recall@1": { + "lower": 0.01117861482381531, + "point_delta": 0.027460510328068045, + "upper": 0.042770352369380316 + }, + "image.text_to_image.recall@10": { + "lower": 0.001215066828675577, + "point_delta": 0.007776427703523694, + "upper": 0.014580801944106925 + }, + "image.text_to_image.recall@5": { + "lower": -0.001701093560145808, + "point_delta": 0.010206561360874849, + "upper": 0.02187120291616039 + }, + "text.banking77.accuracy": { + "lower": 0.0990581357583631, + "point_delta": 0.11168831168831168, + "upper": 0.12439713161970098 + }, + "text.massive-en.accuracy": { + "lower": 0.1117468865701784, + "point_delta": 0.12617765814266488, + "upper": 0.14069455352356733 + }, + "utility": { + "lower": 0.060760442670994376, + "point_delta": 0.06894884308970196, + "upper": 0.07704865461098383 + } + }, + "method": "paired stratified group percentile; PCG64; fixed candidate pools", + "replicates": 2000, + "seed": 20260918, + "comparison": { + "candidate": "llm-semantic-router/Vela-1.0-Omni-Nano", + "candidate_artifact_fingerprint_sha256": "96d76f97478e114b816df3ab1b3c6947e4c5ef0babfb40ab865d7e3c838d34c4", + "baseline": "llm-semantic-router/Vela-1.0-Omni-Nano", + "baseline_revision": "3a1efc48995e2b71622d65c6cf71cfc00bbf3aa3", + "baseline_artifact_fingerprint_sha256": "fb9cd39c9980a085b75456c715c2295b997784fe733314be68c13635f6384edf" + } + } + }, + "training_scope": { + "text": "The complete text tower is the frozen official avsolatorio/GIST-small-Embedding-v0 checkpoint at 75e62fd210b9fde790430e0b2f040b0b00a021b1. Its model card discloses MTEB classification training data; no claim of independent pretraining data is made.", + "image": "Frozen current vision encoder and initial image head; an orthogonal shared-space rotation is trained on 16,509 COCO training images, 81,998 unique caption texts and 82,522 complete positive associations, then folded into the image affine head.", + "audio": "Frozen current speech encoder and initial audio head; a separate orthogonal shared-space rotation is trained on 28,535 LibriSpeech train-clean-100 clips and all positive transcript associations, then folded into the audio affine head.", + "procedure": "One fixed 300-step run; only the final artifact is reported. Encoders remain frozen. The two rotations preserve within-modality inner products, up to floating-point rounding.", + "evaluation_scope": "Common text classification uses labeled training prototypes with the earlier task-adapted comparator. It is separate from the official frozen-encoder MTEB classification protocols." + }, + "descriptive_metrics": { + "definition": "Macro-F1 is the unweighted mean of per-class F1 across all fixed prototype classes; zero_division=0.", + "scope": "Same models, examples, class prototypes and predictions as the accuracy matrix.", + "metrics": { + "text.banking77.macro_f1": { + "small": { + "value": 0.7020781051899517, + "classes": 77, + "queries": 3080 + }, + "candidate": { + "value": 0.8791681858119436, + "classes": 77, + "queries": 3080 + } + }, + "text.massive-en.macro_f1": { + "small": { + "value": 0.6171775818092488, + "classes": 60, + "queries": 2972 + }, + "candidate": { + "value": 0.7923223418549947, + "classes": 60, + "queries": 2972 + } + } + } + }, + "standard_benchmarks": { + "scope": "Eleven selected standard tasks; not a complete MTEB, MMTEB, MIEB or MAEB benchmark or leaderboard submission.", + "library": "mteb", + "library_version": "2.21.0", + "model_revisions": { + "nano": { + "repo_id": "llm-semantic-router/Vela-1.0-Omni-Nano", + "revision": "self", + "native_artifact_sha256": "50ce808197fb43a1b8913a66d4a47339ec66ecd10d52cbd9f9e752d02f8c43fa", + "parameters": 135383808, + "embedding_dimensions": 384, + "max_text_tokens": 512, + "evaluated_native_artifact_sha256": "96d76f97478e114b816df3ab1b3c6947e4c5ef0babfb40ab865d7e3c838d34c4", + "evaluated_revision": "de02c367bc9aa7a6f7ad3e59fb38962502e021c1", + "evaluated_parameters": 144554112, + "inference_equivalence_report": "benchmarks/inference-equivalence.json", + "revision_kind": "HF_snapshot" + }, + "small": { + "repo_id": "llm-semantic-router/multi-modal-embed-small", + "revision": "fdf8e01b7b0f3a69ac1ac8e2a64dcb1ede177ba4", + "native_artifact_sha256": "684fe0c48a4ddd3944167f6e5dc75e97ada24e5b120bd9a2921f00dc03112eeb", + "status": "evaluated_all_11_selected_tasks", + "max_text_tokens": 128, + "inference_parameters": 133759488 + } + }, + "protocol": { + "precision": "float32", + "batch_size": 8, + "seed": 20260918, + "similarity": "Cosine for primary STS/retrieval scores and normalized media-label matching; classification probes and clustering use each official task's algorithms.", + "encoder_parameters_frozen": true, + "classification": "Official task-specific frozen-embedding probes or fixed-label zero-shot classification.", + "sampling": "Official task subsampling and repeated-probe protocols are preserved.", + "text": "Each encoder uses its native preprocessing and context limit without added task instructions. Original small: 128 tokens; current Nano: 512 tokens.", + "audio": "Mono 16 kHz, first 30 seconds, official MTEB audio preprocessing.", + "score_scale": "Metrics retain their native MTEB scale; the model card multiplies primary scores by 100.", + "undefined_metric": null, + "missing_model_score": "Not measured in this standard evaluation.", + "overlap": "No claim of independence from component pretraining data is made." + }, + "rows": [ + { + "task": "SICK-R", + "type": "STS", + "modalities": [ + "text" + ], + "split": "test", + "hf_subset": "default", + "main_metric": "cosine_spearman", + "dataset": { + "path": "mteb/sickr-sts", + "revision": "20a6d6f312dd54037fe07a32d58e5e168867909d" + }, + "counts": { + "evaluation_pairs": 9927 + }, + "scores": { + "nano": 0.8053174565838819, + "small": 0.7139020320965759 + }, + "metrics": { + "nano": { + "pearson": 0.8603345909155973, + "spearman": 0.8053174398238511, + "cosine_pearson": 0.8603345867277692, + "cosine_spearman": 0.8053174565838819, + "manhattan_pearson": 0.8332190477069294, + "manhattan_spearman": 0.8044409124391882, + "euclidean_pearson": 0.834049309726515, + "euclidean_spearman": 0.8053172111347874, + "main_score": 0.8053174565838819, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "mteb_version": "2.21.0" + }, + "small": { + "pearson": 0.8179714611300387, + "spearman": 0.713902378578463, + "cosine_pearson": 0.81797146109438, + "cosine_spearman": 0.7139020320965759, + "manhattan_pearson": 0.7919193141236369, + "manhattan_spearman": 0.7128438640347968, + "euclidean_pearson": 0.792891259170334, + "euclidean_spearman": 0.7139020311053554, + "main_score": 0.7139020320965759, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "mteb_version": "2.21.0" + } + }, + "measurement_by_model": { + "small": { + "evaluated_native_fingerprint": "684fe0c48a4ddd3944167f6e5dc75e97ada24e5b120bd9a2921f00dc03112eeb", + "original_checkpoint_revision": "fdf8e01b7b0f3a69ac1ac8e2a64dcb1ede177ba4", + "raw_result_sha256": "e20f02a347841b3d5f61209f0b209c6314d006f8cc94f6e488e06239c862badf", + "kind": "direct_frozen_encoder_evaluation" + } + } + }, + { + "task": "STSBenchmark", + "type": "STS", + "modalities": [ + "text" + ], + "split": "test", + "hf_subset": "default", + "main_metric": "cosine_spearman", + "dataset": { + "path": "mteb/stsbenchmark-sts", + "revision": "b0fddb56ed78048fa8b90373c8a3cfc37b684831" + }, + "counts": { + "evaluation_pairs": 1379 + }, + "scores": { + "nano": 0.870781967302773, + "small": 0.7729871863268857 + }, + "metrics": { + "nano": { + "pearson": 0.8541596951761798, + "spearman": 0.8707819111802441, + "cosine_pearson": 0.854159691434247, + "cosine_spearman": 0.870781967302773, + "manhattan_pearson": 0.867700912246326, + "manhattan_spearman": 0.8690795304788012, + "euclidean_pearson": 0.8699231026532712, + "euclidean_spearman": 0.8707819663065923, + "main_score": 0.870781967302773, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "mteb_version": "2.21.0" + }, + "small": { + "pearson": 0.765316203785143, + "spearman": 0.7729871863268857, + "cosine_pearson": 0.7653161983330018, + "cosine_spearman": 0.7729871863268857, + "manhattan_pearson": 0.7881395913225815, + "manhattan_spearman": 0.7716275619454571, + "euclidean_pearson": 0.789916662830987, + "euclidean_spearman": 0.7729857841479683, + "main_score": 0.7729871863268857, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "mteb_version": "2.21.0" + } + }, + "measurement_by_model": { + "small": { + "evaluated_native_fingerprint": "684fe0c48a4ddd3944167f6e5dc75e97ada24e5b120bd9a2921f00dc03112eeb", + "original_checkpoint_revision": "fdf8e01b7b0f3a69ac1ac8e2a64dcb1ede177ba4", + "raw_result_sha256": "ea0772149930250452298e3525b3405ce4fd5caed4aaf676f3745d4555692a07", + "kind": "direct_frozen_encoder_evaluation" + } + } + }, + { + "task": "ArguAna", + "type": "Retrieval", + "modalities": [ + "text" + ], + "split": "test", + "hf_subset": "default", + "main_metric": "ndcg_at_10", + "dataset": { + "path": "mteb/arguana", + "revision": "c22ab2a51041ffd869aaddef7af8d8215647e41a" + }, + "counts": { + "queries": 1406, + "documents": 8674 + }, + "scores": { + "nano": 0.59246, + "small": 0.32889 + }, + "metrics": { + "nano": { + "ndcg_at_1": 0.35349, + "ndcg_at_3": 0.49665, + "ndcg_at_5": 0.54581, + "ndcg_at_10": 0.59246, + "ndcg_at_20": 0.6132, + "ndcg_at_100": 0.62189, + "ndcg_at_1000": 0.62242, + "map_at_1": 0.35349, + "map_at_3": 0.46005, + "map_at_5": 0.48729, + "map_at_10": 0.50679, + "map_at_20": 0.51261, + "map_at_100": 0.51395, + "map_at_1000": 0.51397, + "recall_at_1": 0.35349, + "recall_at_3": 0.60313, + "recall_at_5": 0.72262, + "recall_at_10": 0.86558, + "recall_at_20": 0.94666, + "recall_at_100": 0.99147, + "recall_at_1000": 0.99573, + "accuracy": 0.35349, + "precision_at_1": 0.35349, + "precision_at_3": 0.20104, + "precision_at_5": 0.14452, + "precision_at_10": 0.08656, + "precision_at_20": 0.04733, + "precision_at_100": 0.00991, + "precision_at_1000": 0.001, + "mrr_at_1": 0.3534850640113798, + "mrr_at_3": 0.4600521574205785, + "mrr_at_5": 0.48729255571360836, + "mrr_at_10": 0.5067903429745535, + "mrr_at_20": 0.5126066994732466, + "mrr_at_100": 0.5139513666778618, + "mrr_at_1000": 0.5139717342260931, + "nauc_ndcg_at_1_max": -0.11747299439249903, + "nauc_ndcg_at_1_std": -0.11436695908554317, + "nauc_ndcg_at_1_diff1": 0.12406213904460131, + "nauc_ndcg_at_3_max": -0.0810374905617702, + "nauc_ndcg_at_3_std": -0.10902178800246173, + "nauc_ndcg_at_3_diff1": 0.09457199601062195, + "nauc_ndcg_at_5_max": -0.08485395980147033, + "nauc_ndcg_at_5_std": -0.10485663186916257, + "nauc_ndcg_at_5_diff1": 0.08582902365914505, + "nauc_ndcg_at_10_max": -0.08309974111344137, + "nauc_ndcg_at_10_std": -0.11506227102720018, + "nauc_ndcg_at_10_diff1": 0.08990562536626041, + "nauc_ndcg_at_20_max": -0.0791935521931642, + "nauc_ndcg_at_20_std": -0.10319594721919796, + "nauc_ndcg_at_20_diff1": 0.09971433495514617, + "nauc_ndcg_at_100_max": -0.08635224865444148, + "nauc_ndcg_at_100_std": -0.10462787774486179, + "nauc_ndcg_at_100_diff1": 0.09791984963152253, + "nauc_ndcg_at_1000_max": -0.0879373318368675, + "nauc_ndcg_at_1000_std": -0.10553141137460995, + "nauc_ndcg_at_1000_diff1": 0.09704987719570578, + "nauc_map_at_1_max": -0.11747299439249903, + "nauc_map_at_1_std": -0.11436695908554317, + "nauc_map_at_1_diff1": 0.12406213904460131, + "nauc_map_at_3_max": -0.09064151313996319, + "nauc_map_at_3_std": -0.1105170515941861, + "nauc_map_at_3_diff1": 0.09940664377727629, + "nauc_map_at_5_max": -0.09342459992783216, + "nauc_map_at_5_std": -0.10864465771524179, + "nauc_map_at_5_diff1": 0.09517709876339578, + "nauc_map_at_10_max": -0.09304942751697344, + "nauc_map_at_10_std": -0.11264010939720397, + "nauc_map_at_10_diff1": 0.09707068152210184, + "nauc_map_at_20_max": -0.09221792922323616, + "nauc_map_at_20_std": -0.10961211427408772, + "nauc_map_at_20_diff1": 0.09943255521491648, + "nauc_map_at_100_max": -0.09316799067959038, + "nauc_map_at_100_std": -0.10968589627543163, + "nauc_map_at_100_diff1": 0.09910011165040408, + "nauc_map_at_1000_max": -0.09321385399598632, + "nauc_map_at_1000_std": -0.10970513219025511, + "nauc_map_at_1000_diff1": 0.09907801272427635, + "nauc_recall_at_1_max": -0.11747299439249903, + "nauc_recall_at_1_std": -0.11436695908554317, + "nauc_recall_at_1_diff1": 0.12406213904460131, + "nauc_recall_at_3_max": -0.05070404620274185, + "nauc_recall_at_3_std": -0.10426214370217692, + "nauc_recall_at_3_diff1": 0.0805612409902579, + "nauc_recall_at_5_max": -0.051784236042595515, + "nauc_recall_at_5_std": -0.0889570674134141, + "nauc_recall_at_5_diff1": 0.049390111709494505, + "nauc_recall_at_10_max": -0.013727385294040483, + "nauc_recall_at_10_std": -0.13798296178002195, + "nauc_recall_at_10_diff1": 0.044023090224588696, + "nauc_recall_at_20_max": 0.15251226818625876, + "nauc_recall_at_20_std": 0.03363980242022382, + "nauc_recall_at_20_diff1": 0.15796490326794294, + "nauc_recall_at_100_max": 0.3311354440025815, + "nauc_recall_at_100_std": 0.4782526238266158, + "nauc_recall_at_100_diff1": 0.25005005298352057, + "nauc_recall_at_1000_max": 0.02172236447995406, + "nauc_recall_at_1000_std": 0.609819560470231, + "nauc_recall_at_1000_diff1": -0.005495147382062338, + "nauc_precision_at_1_max": -0.11747299439249903, + "nauc_precision_at_1_std": -0.11436695908554317, + "nauc_precision_at_1_diff1": 0.12406213904460131, + "nauc_precision_at_3_max": -0.05070404620274239, + "nauc_precision_at_3_std": -0.10426214370217736, + "nauc_precision_at_3_diff1": 0.08056124099025751, + "nauc_precision_at_5_max": -0.05178423604259566, + "nauc_precision_at_5_std": -0.08895706741341412, + "nauc_precision_at_5_diff1": 0.04939011170949452, + "nauc_precision_at_10_max": -0.013727385294041244, + "nauc_precision_at_10_std": -0.13798296178002226, + "nauc_precision_at_10_diff1": 0.044023090224589216, + "nauc_precision_at_20_max": 0.15251226818625896, + "nauc_precision_at_20_std": 0.03363980242022203, + "nauc_precision_at_20_diff1": 0.15796490326794188, + "nauc_precision_at_100_max": 0.3311354440025917, + "nauc_precision_at_100_std": 0.4782526238266313, + "nauc_precision_at_100_diff1": 0.25005005298354305, + "nauc_precision_at_1000_max": 0.0217223644799946, + "nauc_precision_at_1000_std": 0.6098195604702127, + "nauc_precision_at_1000_diff1": -0.0054951473820258456, + "nauc_mrr_at_1_max": -0.11747299439249903, + "nauc_mrr_at_1_std": -0.11436695908554317, + "nauc_mrr_at_1_diff1": 0.12406213904460131, + "nauc_mrr_at_3_max": -0.09064151313996319, + "nauc_mrr_at_3_std": -0.1105170515941861, + "nauc_mrr_at_3_diff1": 0.09940664377727629, + "nauc_mrr_at_5_max": -0.09342459992783216, + "nauc_mrr_at_5_std": -0.10864465771524179, + "nauc_mrr_at_5_diff1": 0.09517709876339578, + "nauc_mrr_at_10_max": -0.09304942751697344, + "nauc_mrr_at_10_std": -0.11264010939720397, + "nauc_mrr_at_10_diff1": 0.09707068152210184, + "nauc_mrr_at_20_max": -0.09221792922323616, + "nauc_mrr_at_20_std": -0.10961211427408772, + "nauc_mrr_at_20_diff1": 0.09943255521491648, + "nauc_mrr_at_100_max": -0.09316799067959038, + "nauc_mrr_at_100_std": -0.10968589627543163, + "nauc_mrr_at_100_diff1": 0.09910011165040408, + "nauc_mrr_at_1000_max": -0.09321385399598632, + "nauc_mrr_at_1000_std": -0.10970513219025511, + "nauc_mrr_at_1000_diff1": 0.09907801272427635, + "hit_rate_at_1": 0.35349, + "hit_rate_at_3": 0.60313, + "hit_rate_at_5": 0.72262, + "hit_rate_at_10": 0.86558, + "hit_rate_at_20": 0.94666, + "hit_rate_at_100": 0.99147, + "hit_rate_at_1000": 0.99573, + "main_score": 0.59246, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "mteb_version": "2.21.0" + }, + "small": { + "ndcg_at_1": 0.1458, + "ndcg_at_3": 0.23456, + "ndcg_at_5": 0.27835, + "ndcg_at_10": 0.32889, + "ndcg_at_20": 0.36967, + "ndcg_at_100": 0.4045, + "ndcg_at_1000": 0.41446, + "map_at_1": 0.1458, + "map_at_3": 0.21136, + "map_at_5": 0.2354, + "map_at_10": 0.25622, + "map_at_20": 0.26767, + "map_at_100": 0.27279, + "map_at_1000": 0.27315, + "recall_at_1": 0.1458, + "recall_at_3": 0.30228, + "recall_at_5": 0.40967, + "recall_at_10": 0.56615, + "recall_at_20": 0.72546, + "recall_at_100": 0.90896, + "recall_at_1000": 0.98791, + "accuracy": 0.1458, + "precision_at_1": 0.1458, + "precision_at_3": 0.10076, + "precision_at_5": 0.08193, + "precision_at_10": 0.05661, + "precision_at_20": 0.03627, + "precision_at_100": 0.00909, + "precision_at_1000": 0.00099, + "mrr_at_1": 0.14580369843527738, + "mrr_at_3": 0.21135609293504032, + "mrr_at_5": 0.2353959222380275, + "mrr_at_10": 0.25621881279776015, + "mrr_at_20": 0.26766780266556217, + "mrr_at_100": 0.27278502616211503, + "mrr_at_1000": 0.2731483695142967, + "nauc_ndcg_at_1_max": 0.07124074977242485, + "nauc_ndcg_at_1_std": 0.07043132234894327, + "nauc_ndcg_at_1_diff1": 0.17867294841183828, + "nauc_ndcg_at_3_max": 0.05391218816481269, + "nauc_ndcg_at_3_std": 0.08300022308422525, + "nauc_ndcg_at_3_diff1": 0.13721153170379125, + "nauc_ndcg_at_5_max": 0.04761392610075392, + "nauc_ndcg_at_5_std": 0.09330318818026477, + "nauc_ndcg_at_5_diff1": 0.129721108359703, + "nauc_ndcg_at_10_max": 0.06398717148477524, + "nauc_ndcg_at_10_std": 0.11739949823329376, + "nauc_ndcg_at_10_diff1": 0.1353789207379315, + "nauc_ndcg_at_20_max": 0.06583701467786816, + "nauc_ndcg_at_20_std": 0.11760807459667257, + "nauc_ndcg_at_20_diff1": 0.1464058778549573, + "nauc_ndcg_at_100_max": 0.07271569743212124, + "nauc_ndcg_at_100_std": 0.12767735668965335, + "nauc_ndcg_at_100_diff1": 0.14440559200255987, + "nauc_ndcg_at_1000_max": 0.06760834890113898, + "nauc_ndcg_at_1000_std": 0.11565058305226664, + "nauc_ndcg_at_1000_diff1": 0.14440414116396014, + "nauc_map_at_1_max": 0.07124074977242485, + "nauc_map_at_1_std": 0.07043132234894327, + "nauc_map_at_1_diff1": 0.17867294841183828, + "nauc_map_at_3_max": 0.05536834016676611, + "nauc_map_at_3_std": 0.07918800842578326, + "nauc_map_at_3_diff1": 0.14582401585695304, + "nauc_map_at_5_max": 0.05153209428882134, + "nauc_map_at_5_std": 0.08553839884319275, + "nauc_map_at_5_diff1": 0.1412262351733516, + "nauc_map_at_10_max": 0.05764323340595916, + "nauc_map_at_10_std": 0.09567768210275003, + "nauc_map_at_10_diff1": 0.14384793290679312, + "nauc_map_at_20_max": 0.058797567381990594, + "nauc_map_at_20_std": 0.09556268067232435, + "nauc_map_at_20_diff1": 0.14719660481591418, + "nauc_map_at_100_max": 0.060341050948804, + "nauc_map_at_100_std": 0.09715443446030105, + "nauc_map_at_100_diff1": 0.14729508200613933, + "nauc_map_at_1000_max": 0.060216285099770145, + "nauc_map_at_1000_std": 0.09674391170833818, + "nauc_map_at_1000_diff1": 0.14729630480806993, + "nauc_recall_at_1_max": 0.07124074977242485, + "nauc_recall_at_1_std": 0.07043132234894327, + "nauc_recall_at_1_diff1": 0.17867294841183828, + "nauc_recall_at_3_max": 0.05116463248307958, + "nauc_recall_at_3_std": 0.09233493207170033, + "nauc_recall_at_3_diff1": 0.11697591005218377, + "nauc_recall_at_5_max": 0.0387185231377629, + "nauc_recall_at_5_std": 0.11230251236242163, + "nauc_recall_at_5_diff1": 0.10273687046821164, + "nauc_recall_at_10_max": 0.08768343797110026, + "nauc_recall_at_10_std": 0.18174312026830694, + "nauc_recall_at_10_diff1": 0.11544199093498575, + "nauc_recall_at_20_max": 0.0976177919930251, + "nauc_recall_at_20_std": 0.20309410782757367, + "nauc_recall_at_20_diff1": 0.15596172000438252, + "nauc_recall_at_100_max": 0.21399140718760357, + "nauc_recall_at_100_std": 0.45432867879288047, + "nauc_recall_at_100_diff1": 0.13074733185800497, + "nauc_recall_at_1000_max": 0.29184801184590964, + "nauc_recall_at_1000_std": 0.7268160753847613, + "nauc_recall_at_1000_diff1": 0.04649951766999351, + "nauc_precision_at_1_max": 0.07124074977242485, + "nauc_precision_at_1_std": 0.07043132234894327, + "nauc_precision_at_1_diff1": 0.17867294841183828, + "nauc_precision_at_3_max": 0.051164632483079536, + "nauc_precision_at_3_std": 0.09233493207170006, + "nauc_precision_at_3_diff1": 0.11697591005218336, + "nauc_precision_at_5_max": 0.03871852313776277, + "nauc_precision_at_5_std": 0.11230251236242186, + "nauc_precision_at_5_diff1": 0.10273687046821152, + "nauc_precision_at_10_max": 0.08768343797110059, + "nauc_precision_at_10_std": 0.18174312026830697, + "nauc_precision_at_10_diff1": 0.11544199093498574, + "nauc_precision_at_20_max": 0.09761779199302502, + "nauc_precision_at_20_std": 0.20309410782757284, + "nauc_precision_at_20_diff1": 0.15596172000438274, + "nauc_precision_at_100_max": 0.2139914071876045, + "nauc_precision_at_100_std": 0.45432867879288213, + "nauc_precision_at_100_diff1": 0.13074733185800294, + "nauc_precision_at_1000_max": 0.29184801184591147, + "nauc_precision_at_1000_std": 0.7268160753847517, + "nauc_precision_at_1000_diff1": 0.04649951766998672, + "nauc_mrr_at_1_max": 0.07124074977242485, + "nauc_mrr_at_1_std": 0.07043132234894327, + "nauc_mrr_at_1_diff1": 0.17867294841183828, + "nauc_mrr_at_3_max": 0.05536834016676611, + "nauc_mrr_at_3_std": 0.07918800842578326, + "nauc_mrr_at_3_diff1": 0.14582401585695304, + "nauc_mrr_at_5_max": 0.05153209428882134, + "nauc_mrr_at_5_std": 0.08553839884319275, + "nauc_mrr_at_5_diff1": 0.1412262351733516, + "nauc_mrr_at_10_max": 0.05764323340595916, + "nauc_mrr_at_10_std": 0.09567768210275003, + "nauc_mrr_at_10_diff1": 0.14384793290679312, + "nauc_mrr_at_20_max": 0.058797567381990594, + "nauc_mrr_at_20_std": 0.09556268067232435, + "nauc_mrr_at_20_diff1": 0.14719660481591418, + "nauc_mrr_at_100_max": 0.060341050948804, + "nauc_mrr_at_100_std": 0.09715443446030105, + "nauc_mrr_at_100_diff1": 0.14729508200613933, + "nauc_mrr_at_1000_max": 0.060216285099770145, + "nauc_mrr_at_1000_std": 0.09674391170833818, + "nauc_mrr_at_1000_diff1": 0.14729630480806993, + "hit_rate_at_1": 0.1458, + "hit_rate_at_3": 0.30228, + "hit_rate_at_5": 0.40967, + "hit_rate_at_10": 0.56615, + "hit_rate_at_20": 0.72546, + "hit_rate_at_100": 0.90896, + "hit_rate_at_1000": 0.98791, + "main_score": 0.32889, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "mteb_version": "2.21.0" + } + }, + "measurement_by_model": { + "small": { + "evaluated_native_fingerprint": "684fe0c48a4ddd3944167f6e5dc75e97ada24e5b120bd9a2921f00dc03112eeb", + "original_checkpoint_revision": "fdf8e01b7b0f3a69ac1ac8e2a64dcb1ede177ba4", + "raw_result_sha256": "92cbe96f6e84ebd0bc144001424ac46bb3a630ef64bffb433c2cdd63a10c43ec", + "kind": "direct_frozen_encoder_evaluation" + } + } + }, + { + "task": "TwentyNewsgroupsClustering.v2", + "type": "Clustering", + "modalities": [ + "text" + ], + "split": "test", + "hf_subset": "default", + "main_metric": "v_measure", + "dataset": { + "path": "mteb/twentynewsgroups-clustering", + "revision": "6125ec4e24fa026cec8a478383ee943acfbd5449" + }, + "counts": { + "population_rows": 59545, + "official_encoded_rows": 2381, + "official_encoding_fraction": 0.04, + "bootstrap_clustering_trials": 10, + "resampled_items_per_trial": 16384 + }, + "scores": { + "nano": 0.5087260708286914, + "small": 0.44840076500124326 + }, + "metrics": { + "nano": { + "v_measures": { + "Level 0": [ + 0.4966639569479829, + 0.5210509442892017, + 0.5333989625396617, + 0.5094460445939109, + 0.5208429535416115, + 0.5012604404318527, + 0.5210119744629907, + 0.4617825059979706, + 0.5226704101107219, + 0.49913251537100806 + ] + }, + "v_measure": 0.5087260708286914, + "v_measure_std": 0.01936580527644773, + "ami_scores": { + "Level 0": [ + 0.49471495769938123, + 0.5192201304733433, + 0.5316138913144499, + 0.5075814211955173, + 0.5190252719774525, + 0.49937824878079495, + 0.5191850550712727, + 0.4597515065381579, + 0.5208311166380218, + 0.49721166070769024 + ] + }, + "ami": 0.5068513260396083, + "ami_std": 0.01943429917113934, + "main_score": 0.5087260708286914, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "mteb_version": "2.21.0" + }, + "small": { + "v_measures": { + "Level 0": [ + 0.46653683369731075, + 0.46464237680676646, + 0.4425518472920712, + 0.4288571327957099, + 0.4336826969564481, + 0.43371621267306154, + 0.43092596596465876, + 0.4536864467956253, + 0.4741214282291591, + 0.45528670880162087 + ] + }, + "v_measure": 0.44840076500124326, + "v_measure_std": 0.01575881885263055, + "ami_scores": { + "Level 0": [ + 0.46451147281180905, + 0.4625982506803001, + 0.44043666823974115, + 0.42670551677458946, + 0.43154706196129733, + 0.4315549515134722, + 0.4287870602853061, + 0.45163080092170965, + 0.47212403002394715, + 0.4532359410185627 + ] + }, + "ami": 0.44631317542307347, + "ami_std": 0.01581387159916619, + "main_score": 0.44840076500124326, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "mteb_version": "2.21.0" + } + }, + "measurement_by_model": { + "small": { + "evaluated_native_fingerprint": "684fe0c48a4ddd3944167f6e5dc75e97ada24e5b120bd9a2921f00dc03112eeb", + "original_checkpoint_revision": "fdf8e01b7b0f3a69ac1ac8e2a64dcb1ede177ba4", + "raw_result_sha256": "62fe52980cb4445b9969d53135fa1663dd8dcd811743a711dde55f2a097ced19", + "kind": "direct_frozen_encoder_evaluation" + } + } + }, + { + "task": "MTOPDomainClassification", + "type": "Classification", + "modalities": [ + "text" + ], + "split": "validation", + "hf_subset": "en", + "main_metric": "accuracy", + "dataset": { + "path": "mteb/MTOPDomainClassification", + "revision": "a76d16fae880597b9c73047b50159220a441cb54" + }, + "counts": { + "evaluation_rows": 2235, + "available_train_rows": 15667, + "official_probe_experiments": 10, + "probe_samples_per_label": 8 + }, + "scores": { + "nano": 0.9476510067114093, + "small": 0.8543624161073826 + }, + "metrics": { + "nano": { + "scores_per_experiment": [ + { + "accuracy": 0.9521252796420582, + "f1": 0.9509396968297774, + "f1_weighted": 0.9519302419648948, + "precision": 0.9493488952211522, + "precision_weighted": 0.9527659658117037, + "recall": 0.9535803496254757, + "recall_weighted": 0.9521252796420582, + "ap": null, + "ap_weighted": null + }, + { + "accuracy": 0.9458612975391499, + "f1": 0.9439475266607382, + "f1_weighted": 0.945767354216612, + "precision": 0.9420005561621522, + "precision_weighted": 0.9462895866502771, + "recall": 0.946397940529448, + "recall_weighted": 0.9458612975391499, + "ap": null, + "ap_weighted": null + }, + { + "accuracy": 0.9284116331096197, + "f1": 0.9284988704174789, + "f1_weighted": 0.9270840634554431, + "precision": 0.9271908775153325, + "precision_weighted": 0.9274634710035635, + "recall": 0.9317153602473662, + "recall_weighted": 0.9284116331096197, + "ap": null, + "ap_weighted": null + }, + { + "accuracy": 0.9525727069351231, + "f1": 0.950987470976036, + "f1_weighted": 0.9525185063610705, + "precision": 0.9504292297253194, + "precision_weighted": 0.9534501308715367, + "recall": 0.9526312322737922, + "recall_weighted": 0.9525727069351231, + "ap": null, + "ap_weighted": null + }, + { + "accuracy": 0.9445190156599552, + "f1": 0.943383064264026, + "f1_weighted": 0.9442288513061988, + "precision": 0.9424858291605631, + "precision_weighted": 0.944823795147547, + "recall": 0.945181115724642, + "recall_weighted": 0.9445190156599552, + "ap": null, + "ap_weighted": null + }, + { + "accuracy": 0.9525727069351231, + "f1": 0.949882476921466, + "f1_weighted": 0.9526817135384125, + "precision": 0.9498620235113855, + "precision_weighted": 0.9543297139412104, + "recall": 0.9514687067629995, + "recall_weighted": 0.9525727069351231, + "ap": null, + "ap_weighted": null + }, + { + "accuracy": 0.9498881431767338, + "f1": 0.9496336255013932, + "f1_weighted": 0.9495703695110613, + "precision": 0.9490015795096777, + "precision_weighted": 0.9511221301014349, + "recall": 0.9520470037532829, + "recall_weighted": 0.9498881431767338, + "ap": null, + "ap_weighted": null + }, + { + "accuracy": 0.9570469798657718, + "f1": 0.9557194953102311, + "f1_weighted": 0.9568992928300293, + "precision": 0.9544888722394763, + "precision_weighted": 0.9584505479016514, + "recall": 0.9586898382241177, + "recall_weighted": 0.9570469798657718, + "ap": null, + "ap_weighted": null + }, + { + "accuracy": 0.9494407158836689, + "f1": 0.9460535182757994, + "f1_weighted": 0.9492959546761313, + "precision": 0.9454088954645755, + "precision_weighted": 0.9499145785798135, + "recall": 0.9472953341577202, + "recall_weighted": 0.9494407158836689, + "ap": null, + "ap_weighted": null + }, + { + "accuracy": 0.9440715883668904, + "f1": 0.9431192151526329, + "f1_weighted": 0.9437397930273277, + "precision": 0.9435095731735328, + "precision_weighted": 0.9461253948734786, + "recall": 0.9455297617432863, + "recall_weighted": 0.9440715883668904, + "ap": null, + "ap_weighted": null + } + ], + "accuracy": 0.9476510067114093, + "f1": 0.9462164960309579, + "f1_weighted": 0.9473716140887183, + "precision": 0.9453726331683165, + "precision_weighted": 0.9484735314882217, + "recall": 0.948453664304213, + "recall_weighted": 0.9476510067114093, + "ap": null, + "ap_weighted": null, + "main_score": 0.9476510067114093, + "hf_subset": "en", + "languages": [ + "eng-Latn" + ], + "mteb_version": "2.21.0" + }, + "small": { + "scores_per_experiment": [ + { + "accuracy": 0.8250559284116331, + "f1": 0.8322814729997016, + "f1_weighted": 0.8240316200136756, + "precision": 0.8318540374193987, + "precision_weighted": 0.8346068877064119, + "recall": 0.8430207439526936, + "recall_weighted": 0.8250559284116331, + "ap": null, + "ap_weighted": null + }, + { + "accuracy": 0.8559284116331096, + "f1": 0.8552447478537527, + "f1_weighted": 0.8547401242258992, + "precision": 0.8515715568445064, + "precision_weighted": 0.8581578601876096, + "recall": 0.8640896175706192, + "recall_weighted": 0.8559284116331096, + "ap": null, + "ap_weighted": null + }, + { + "accuracy": 0.8304250559284116, + "f1": 0.8317375249321166, + "f1_weighted": 0.8274798893470849, + "precision": 0.8268066926411431, + "precision_weighted": 0.8285364690282124, + "recall": 0.8409132018240374, + "recall_weighted": 0.8304250559284116, + "ap": null, + "ap_weighted": null + }, + { + "accuracy": 0.8648769574944072, + "f1": 0.8678594298716482, + "f1_weighted": 0.8632422395451538, + "precision": 0.865870318819693, + "precision_weighted": 0.8646460721917335, + "recall": 0.8726494617104266, + "recall_weighted": 0.8648769574944072, + "ap": null, + "ap_weighted": null + }, + { + "accuracy": 0.8532438478747204, + "f1": 0.8558930928177336, + "f1_weighted": 0.8512711153646526, + "precision": 0.8501961024805756, + "precision_weighted": 0.8547737986953718, + "recall": 0.8657455326712605, + "recall_weighted": 0.8532438478747204, + "ap": null, + "ap_weighted": null + }, + { + "accuracy": 0.8402684563758389, + "f1": 0.8478991888671523, + "f1_weighted": 0.8396067623115454, + "precision": 0.8508550254092774, + "precision_weighted": 0.8598285030005444, + "recall": 0.862051388925805, + "recall_weighted": 0.8402684563758389, + "ap": null, + "ap_weighted": null + }, + { + "accuracy": 0.8747203579418344, + "f1": 0.8781373847485636, + "f1_weighted": 0.8731828255054377, + "precision": 0.8729737218664414, + "precision_weighted": 0.8758466535281065, + "recall": 0.8873478774616341, + "recall_weighted": 0.8747203579418344, + "ap": null, + "ap_weighted": null + }, + { + "accuracy": 0.876062639821029, + "f1": 0.8754580934866012, + "f1_weighted": 0.8749795409187, + "precision": 0.8730455332008387, + "precision_weighted": 0.8787904116312295, + "recall": 0.8820295010368394, + "recall_weighted": 0.876062639821029, + "ap": null, + "ap_weighted": null + }, + { + "accuracy": 0.858165548098434, + "f1": 0.8610701024195923, + "f1_weighted": 0.8573656937247496, + "precision": 0.8561531713932808, + "precision_weighted": 0.8662234353266065, + "recall": 0.8738853045389997, + "recall_weighted": 0.858165548098434, + "ap": null, + "ap_weighted": null + }, + { + "accuracy": 0.8648769574944072, + "f1": 0.8671414562971872, + "f1_weighted": 0.8628546885742888, + "precision": 0.8607074291115159, + "precision_weighted": 0.8655691598347531, + "recall": 0.8776421302043499, + "recall_weighted": 0.8648769574944072, + "ap": null, + "ap_weighted": null + } + ], + "accuracy": 0.8543624161073826, + "f1": 0.8572722494294049, + "f1_weighted": 0.8528754499531187, + "precision": 0.8540033589186671, + "precision_weighted": 0.8586979251130579, + "recall": 0.8669374759896666, + "recall_weighted": 0.8543624161073826, + "ap": null, + "ap_weighted": null, + "main_score": 0.8543624161073826, + "hf_subset": "en", + "languages": [ + "eng-Latn" + ], + "mteb_version": "2.21.0" + } + }, + "measurement_by_model": { + "small": { + "evaluated_native_fingerprint": "684fe0c48a4ddd3944167f6e5dc75e97ada24e5b120bd9a2921f00dc03112eeb", + "original_checkpoint_revision": "fdf8e01b7b0f3a69ac1ac8e2a64dcb1ede177ba4", + "raw_result_sha256": "256702650b93bd559d9a2a9b70f32b996b51e166debc0c20d63ce24cc6f605f6", + "kind": "direct_frozen_encoder_evaluation" + } + } + }, + { + "task": "MTOPDomainClassification", + "type": "Classification", + "modalities": [ + "text" + ], + "split": "test", + "hf_subset": "en", + "main_metric": "accuracy", + "dataset": { + "path": "mteb/MTOPDomainClassification", + "revision": "a76d16fae880597b9c73047b50159220a441cb54" + }, + "counts": { + "evaluation_rows": 4386, + "available_train_rows": 15667, + "official_probe_experiments": 10, + "probe_samples_per_label": 8 + }, + "scores": { + "nano": 0.9491792065663475, + "small": 0.8456452348381213 + }, + "metrics": { + "nano": { + "scores_per_experiment": [ + { + "accuracy": 0.9507523939808481, + "f1": 0.9490862282573499, + "f1_weighted": 0.9508003256061616, + "precision": 0.9476949859831407, + "precision_weighted": 0.9520734016967529, + "recall": 0.9517099502990664, + "recall_weighted": 0.9507523939808481, + "ap": null, + "ap_weighted": null + }, + { + "accuracy": 0.9473324213406292, + "f1": 0.9425946320317333, + "f1_weighted": 0.9473133266787116, + "precision": 0.9392105665237355, + "precision_weighted": 0.9479986101450497, + "recall": 0.9468765008841218, + "recall_weighted": 0.9473324213406292, + "ap": null, + "ap_weighted": null + }, + { + "accuracy": 0.9306885544915641, + "f1": 0.9261678157710709, + "f1_weighted": 0.9296159823715927, + "precision": 0.923024327342814, + "precision_weighted": 0.9307965712480242, + "recall": 0.932201810783149, + "recall_weighted": 0.9306885544915641, + "ap": null, + "ap_weighted": null + }, + { + "accuracy": 0.958048335613315, + "f1": 0.9552869971485034, + "f1_weighted": 0.9579502405954128, + "precision": 0.954624514001619, + "precision_weighted": 0.9582682331625527, + "recall": 0.9564289859937923, + "recall_weighted": 0.958048335613315, + "ap": null, + "ap_weighted": null + }, + { + "accuracy": 0.9464204286365709, + "f1": 0.94433525009326, + "f1_weighted": 0.9462665825311201, + "precision": 0.9422860464299823, + "precision_weighted": 0.9467645700971977, + "recall": 0.9469874285322217, + "recall_weighted": 0.9464204286365709, + "ap": null, + "ap_weighted": null + }, + { + "accuracy": 0.9505243958048336, + "f1": 0.9447844972663006, + "f1_weighted": 0.9507789369812125, + "precision": 0.9429136617255864, + "precision_weighted": 0.9524852717381878, + "recall": 0.9483171314260196, + "recall_weighted": 0.9505243958048336, + "ap": null, + "ap_weighted": null + }, + { + "accuracy": 0.9509803921568627, + "f1": 0.9480693879894481, + "f1_weighted": 0.9510087641402198, + "precision": 0.9447979434260977, + "precision_weighted": 0.9522624035738166, + "recall": 0.952376748039714, + "recall_weighted": 0.9509803921568627, + "ap": null, + "ap_weighted": null + }, + { + "accuracy": 0.9573643410852714, + "f1": 0.9551153583299494, + "f1_weighted": 0.9573535659550753, + "precision": 0.952401865813031, + "precision_weighted": 0.9583040343256545, + "recall": 0.9586994420587002, + "recall_weighted": 0.9573643410852714, + "ap": null, + "ap_weighted": null + }, + { + "accuracy": 0.9548563611491108, + "f1": 0.9495763118202308, + "f1_weighted": 0.9549499907192655, + "precision": 0.946851274794657, + "precision_weighted": 0.9555638233419986, + "recall": 0.9529512436069031, + "recall_weighted": 0.9548563611491108, + "ap": null, + "ap_weighted": null + }, + { + "accuracy": 0.9448244414044688, + "f1": 0.9427993333346522, + "f1_weighted": 0.9447713522464498, + "precision": 0.9432072583992069, + "precision_weighted": 0.9468986783847072, + "recall": 0.9447479153319868, + "recall_weighted": 0.9448244414044688, + "ap": null, + "ap_weighted": null + } + ], + "accuracy": 0.9491792065663475, + "f1": 0.9457815812042497, + "f1_weighted": 0.9490809067825221, + "precision": 0.9437012444439871, + "precision_weighted": 0.9501415597713943, + "recall": 0.9491297156955675, + "recall_weighted": 0.9491792065663475, + "ap": null, + "ap_weighted": null, + "main_score": 0.9491792065663475, + "hf_subset": "en", + "languages": [ + "eng-Latn" + ], + "mteb_version": "2.21.0" + }, + "small": { + "scores_per_experiment": [ + { + "accuracy": 0.8180574555403557, + "f1": 0.8207347394289347, + "f1_weighted": 0.8170857264694432, + "precision": 0.820363266003151, + "precision_weighted": 0.8301897328408723, + "recall": 0.8338753807988755, + "recall_weighted": 0.8180574555403557, + "ap": null, + "ap_weighted": null + }, + { + "accuracy": 0.8449612403100775, + "f1": 0.839516667111528, + "f1_weighted": 0.8449410274873432, + "precision": 0.8369796042511596, + "precision_weighted": 0.8536873756717578, + "recall": 0.8523590480814156, + "recall_weighted": 0.8449612403100775, + "ap": null, + "ap_weighted": null + }, + { + "accuracy": 0.8258093935248518, + "f1": 0.8185934728273051, + "f1_weighted": 0.8251774946806023, + "precision": 0.8147370918656317, + "precision_weighted": 0.8330066052370535, + "recall": 0.8324642506433119, + "recall_weighted": 0.8258093935248518, + "ap": null, + "ap_weighted": null + }, + { + "accuracy": 0.8497492020063839, + "f1": 0.8486953297726213, + "f1_weighted": 0.848088172519004, + "precision": 0.8467111465364261, + "precision_weighted": 0.8501008589343039, + "recall": 0.853578648077031, + "recall_weighted": 0.8497492020063839, + "ap": null, + "ap_weighted": null + }, + { + "accuracy": 0.8481532147742818, + "f1": 0.8467468733444872, + "f1_weighted": 0.8467448330993773, + "precision": 0.8393119909268087, + "precision_weighted": 0.8506182695179204, + "recall": 0.858498685671118, + "recall_weighted": 0.8481532147742818, + "ap": null, + "ap_weighted": null + }, + { + "accuracy": 0.8326493388052896, + "f1": 0.8352387297727578, + "f1_weighted": 0.8347572800634613, + "precision": 0.8420129591399498, + "precision_weighted": 0.8638673888380122, + "recall": 0.8534348314007274, + "recall_weighted": 0.8326493388052896, + "ap": null, + "ap_weighted": null + }, + { + "accuracy": 0.8682170542635659, + "f1": 0.8648438659784241, + "f1_weighted": 0.8680876089933638, + "precision": 0.8585183480506728, + "precision_weighted": 0.876371854401442, + "recall": 0.8802432046324339, + "recall_weighted": 0.8682170542635659, + "ap": null, + "ap_weighted": null + }, + { + "accuracy": 0.8518011855905153, + "f1": 0.8484079910361157, + "f1_weighted": 0.8509906563529788, + "precision": 0.8443896101449812, + "precision_weighted": 0.860550178646213, + "recall": 0.8607698617029648, + "recall_weighted": 0.8518011855905153, + "ap": null, + "ap_weighted": null + }, + { + "accuracy": 0.8556771545827634, + "f1": 0.8515433915603542, + "f1_weighted": 0.856278856298033, + "precision": 0.84610847827307, + "precision_weighted": 0.8661492710958929, + "recall": 0.8658189317867632, + "recall_weighted": 0.8556771545827634, + "ap": null, + "ap_weighted": null + }, + { + "accuracy": 0.8613771089831281, + "f1": 0.8586787981057005, + "f1_weighted": 0.8603956735839339, + "precision": 0.8518135650230533, + "precision_weighted": 0.8659927155990171, + "recall": 0.8717255305513517, + "recall_weighted": 0.8613771089831281, + "ap": null, + "ap_weighted": null + } + ], + "accuracy": 0.8456452348381213, + "f1": 0.8432999858938229, + "f1_weighted": 0.8452547329547541, + "precision": 0.8400946060214904, + "precision_weighted": 0.8550534250782486, + "recall": 0.8562768373345993, + "recall_weighted": 0.8456452348381213, + "ap": null, + "ap_weighted": null, + "main_score": 0.8456452348381213, + "hf_subset": "en", + "languages": [ + "eng-Latn" + ], + "mteb_version": "2.21.0" + } + }, + "measurement_by_model": { + "small": { + "evaluated_native_fingerprint": "684fe0c48a4ddd3944167f6e5dc75e97ada24e5b120bd9a2921f00dc03112eeb", + "original_checkpoint_revision": "fdf8e01b7b0f3a69ac1ac8e2a64dcb1ede177ba4", + "raw_result_sha256": "256702650b93bd559d9a2a9b70f32b996b51e166debc0c20d63ce24cc6f605f6", + "kind": "direct_frozen_encoder_evaluation" + } + } + }, + { + "task": "OxfordPets", + "type": "ImageClassification", + "modalities": [ + "image" + ], + "split": "test", + "hf_subset": "default", + "main_metric": "accuracy", + "dataset": { + "path": "mteb/OxfordPets", + "revision": "557b480fae8d69247be74d9503b378a09425096f" + }, + "counts": { + "evaluation_rows": 3669, + "available_train_rows": 3680, + "official_probe_experiments": 5, + "probe_samples_per_label": 16 + }, + "scores": { + "nano": 0.9126737530662306, + "small": 0.4184791496320523 + }, + "metrics": { + "nano": { + "scores_per_experiment": [ + { + "accuracy": 0.9062414826928318, + "f1": 0.9045301316021502, + "f1_weighted": 0.9050055409371512, + "precision": 0.9099213291951324, + "precision_weighted": 0.9109544817876485, + "recall": 0.9064295837897134, + "recall_weighted": 0.9062414826928318, + "ap": null, + "ap_weighted": null + }, + { + "accuracy": 0.914418097574271, + "f1": 0.9126724638115935, + "f1_weighted": 0.9132186480967779, + "precision": 0.9192904646518116, + "precision_weighted": 0.9203471940294945, + "recall": 0.9144593228002407, + "recall_weighted": 0.914418097574271, + "ap": null, + "ap_weighted": null + }, + { + "accuracy": 0.9187789588443718, + "f1": 0.9177765846692281, + "f1_weighted": 0.9184111765200857, + "precision": 0.9220755461580387, + "precision_weighted": 0.9229278273528245, + "recall": 0.9183739103921627, + "recall_weighted": 0.9187789588443718, + "ap": null, + "ap_weighted": null + }, + { + "accuracy": 0.9174161896974653, + "f1": 0.9147622354932347, + "f1_weighted": 0.9154663340384108, + "precision": 0.9209055798643112, + "precision_weighted": 0.9221124471295344, + "recall": 0.9173606715378323, + "recall_weighted": 0.9174161896974653, + "ap": null, + "ap_weighted": null + }, + { + "accuracy": 0.9065140365222132, + "f1": 0.9051669100036687, + "f1_weighted": 0.9057728210054364, + "precision": 0.9098426973534027, + "precision_weighted": 0.9105143284060627, + "recall": 0.9059560197061645, + "recall_weighted": 0.9065140365222132, + "ap": null, + "ap_weighted": null + } + ], + "accuracy": 0.9126737530662306, + "f1": 0.910981665115975, + "f1_weighted": 0.9115749041195725, + "precision": 0.9164071234445392, + "precision_weighted": 0.9173712557411129, + "recall": 0.9125159016452227, + "recall_weighted": 0.9126737530662306, + "ap": null, + "ap_weighted": null, + "main_score": 0.9126737530662306, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "mteb_version": "2.21.0" + }, + "small": { + "scores_per_experiment": [ + { + "accuracy": 0.4148269283183429, + "f1": 0.40253654878032713, + "f1_weighted": 0.4024461757957877, + "precision": 0.4116431114014033, + "precision_weighted": 0.4119318458992588, + "recall": 0.4153030284766479, + "recall_weighted": 0.4148269283183429, + "ap": null, + "ap_weighted": null + }, + { + "accuracy": 0.4265467429817389, + "f1": 0.4174251045251405, + "f1_weighted": 0.4176529394767131, + "precision": 0.43921297726376096, + "precision_weighted": 0.4396338060561412, + "recall": 0.42629825783951975, + "recall_weighted": 0.4265467429817389, + "ap": null, + "ap_weighted": null + }, + { + "accuracy": 0.4088307440719542, + "f1": 0.3954869417563968, + "f1_weighted": 0.39542431005304346, + "precision": 0.40981373804908927, + "precision_weighted": 0.40980533913060346, + "recall": 0.40880378402552453, + "recall_weighted": 0.4088307440719542, + "ap": null, + "ap_weighted": null + }, + { + "accuracy": 0.4377214499863723, + "f1": 0.4251494451023654, + "f1_weighted": 0.4253356439185362, + "precision": 0.438981154791822, + "precision_weighted": 0.439719429760714, + "recall": 0.4381658372912861, + "recall_weighted": 0.4377214499863723, + "ap": null, + "ap_weighted": null + }, + { + "accuracy": 0.40446988280185336, + "f1": 0.3806382057679881, + "f1_weighted": 0.380430443085735, + "precision": 0.4088810121814978, + "precision_weighted": 0.40903302714302686, + "recall": 0.4046956272962792, + "recall_weighted": 0.40446988280185336, + "ap": null, + "ap_weighted": null + } + ], + "accuracy": 0.4184791496320523, + "f1": 0.4042472491864436, + "f1_weighted": 0.40425790246596305, + "precision": 0.42170639873751464, + "precision_weighted": 0.4220246895979488, + "recall": 0.4186533069858515, + "recall_weighted": 0.4184791496320523, + "ap": null, + "ap_weighted": null, + "main_score": 0.4184791496320523, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "mteb_version": "2.21.0" + } + }, + "measurement_by_model": { + "small": { + "evaluated_native_fingerprint": "684fe0c48a4ddd3944167f6e5dc75e97ada24e5b120bd9a2921f00dc03112eeb", + "original_checkpoint_revision": "fdf8e01b7b0f3a69ac1ac8e2a64dcb1ede177ba4", + "raw_result_sha256": "ef1f4928b9ae7d9da25a98b82a28fda3e098db9504ad9372cb5305ad32421a06", + "kind": "direct_frozen_encoder_evaluation" + } + } + }, + { + "task": "TinyImageNetClustering", + "type": "ImageClustering", + "modalities": [ + "image" + ], + "split": "valid", + "hf_subset": "default", + "main_metric": "nmi", + "dataset": { + "path": "mteb/tiny-imagenet", + "revision": "e04e259f646f750bd0bdb2a56c2a4577d89bee3d" + }, + "counts": { + "evaluation_rows": 10000 + }, + "scores": { + "nano": 0.6169193395626786, + "small": 0.4628490605491802 + }, + "metrics": { + "nano": { + "v_measure": 0.6169193395626787, + "nmi": 0.6169193395626786, + "ari": 0.21497891543638262, + "cluster_accuracy": 0.3638, + "main_score": 0.6169193395626786, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "mteb_version": "2.21.0" + }, + "small": { + "v_measure": 0.4628490605491802, + "nmi": 0.4628490605491802, + "ari": 0.08498222977932614, + "cluster_accuracy": 0.2035, + "main_score": 0.4628490605491802, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "mteb_version": "2.21.0" + } + }, + "measurement_by_model": { + "small": { + "evaluated_native_fingerprint": "684fe0c48a4ddd3944167f6e5dc75e97ada24e5b120bd9a2921f00dc03112eeb", + "original_checkpoint_revision": "fdf8e01b7b0f3a69ac1ac8e2a64dcb1ede177ba4", + "raw_result_sha256": "eb344e20e7d498aadbbd67a57015cc2220cba42b9550f5822b5a72d0d6cf1514", + "kind": "direct_frozen_encoder_evaluation" + } + } + }, + { + "task": "OxfordPetsZeroShot", + "type": "ZeroShotClassification", + "modalities": [ + "text", + "image" + ], + "split": "test", + "hf_subset": "default", + "main_metric": "accuracy", + "dataset": { + "path": "mteb/OxfordPets", + "revision": "557b480fae8d69247be74d9503b378a09425096f" + }, + "counts": { + "evaluation_rows": 3669, + "fixed_text_labels": 37 + }, + "scores": { + "nano": 0.08013082583810302, + "small": 0.1640774052875443 + }, + "metrics": { + "nano": { + "accuracy": 0.08013082583810302, + "f1": 0.0611345635215895, + "f1_weighted": 0.06128470828202288, + "precision": 0.1359645526412158, + "precision_weighted": 0.13671480699577512, + "recall": 0.0798783003173134, + "recall_weighted": 0.08013082583810302, + "ap": null, + "ap_weighted": null, + "main_score": 0.08013082583810302, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "mteb_version": "2.21.0" + }, + "small": { + "accuracy": 0.1640774052875443, + "f1": 0.12868294474251654, + "f1_weighted": 0.12888736955731406, + "precision": 0.15179208674748698, + "precision_weighted": 0.15241279079348374, + "recall": 0.16458376660623852, + "recall_weighted": 0.1640774052875443, + "ap": null, + "ap_weighted": null, + "main_score": 0.1640774052875443, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "mteb_version": "2.21.0" + } + }, + "measurement_by_model": { + "small": { + "evaluated_native_fingerprint": "684fe0c48a4ddd3944167f6e5dc75e97ada24e5b120bd9a2921f00dc03112eeb", + "original_checkpoint_revision": "fdf8e01b7b0f3a69ac1ac8e2a64dcb1ede177ba4", + "raw_result_sha256": "e52e42c4716e62710296df98a90b3a0b986875685748d043cfd7de913e5a6c42", + "kind": "direct_frozen_encoder_evaluation" + } + } + }, + { + "task": "CREMA_D", + "type": "AudioClassification", + "modalities": [ + "audio" + ], + "split": "train", + "hf_subset": "default", + "main_metric": "accuracy", + "dataset": { + "path": "mteb/crema-d", + "revision": "a9d0821955c418752248b8310438854e56a1cf34" + }, + "counts": { + "evaluation_rows": 7442, + "official_cross_validation_folds": 5, + "probe_samples_per_label": 8 + }, + "scores": { + "nano": 0.29763670140167686, + "small": 0.2995196855795547 + }, + "metrics": { + "nano": { + "scores_per_experiment": [ + { + "accuracy": 0.2847548690396239, + "f1": 0.25980556646698677, + "f1_weighted": 0.2606862265023784, + "precision": 0.2685141966762838, + "precision_weighted": 0.26869432571348717, + "recall": 0.283361264324777, + "recall_weighted": 0.2847548690396239, + "ap": null, + "ap_weighted": null + }, + { + "accuracy": 0.2981867024848892, + "f1": 0.2927999429509691, + "f1_weighted": 0.2952410831252662, + "precision": 0.29988591178720175, + "precision_weighted": 0.30336098208055257, + "recall": 0.29763527945216217, + "recall_weighted": 0.2981867024848892, + "ap": null, + "ap_weighted": null + }, + { + "accuracy": 0.28696236559139787, + "f1": 0.26582489649494206, + "f1_weighted": 0.26920117230368773, + "precision": 0.26489987169540286, + "precision_weighted": 0.2682170436584329, + "recall": 0.2834616565816636, + "recall_weighted": 0.28696236559139787, + "ap": null, + "ap_weighted": null + }, + { + "accuracy": 0.29973118279569894, + "f1": 0.291485315935673, + "f1_weighted": 0.28841683151437103, + "precision": 0.3068565359099591, + "precision_weighted": 0.3086441612810119, + "recall": 0.30801945566008065, + "recall_weighted": 0.29973118279569894, + "ap": null, + "ap_weighted": null + }, + { + "accuracy": 0.3185483870967742, + "f1": 0.3051858414398825, + "f1_weighted": 0.3066161074219095, + "precision": 0.310702613862649, + "precision_weighted": 0.3133755862555043, + "recall": 0.31849511838471833, + "recall_weighted": 0.3185483870967742, + "ap": null, + "ap_weighted": null + } + ], + "accuracy": 0.29763670140167686, + "f1": 0.2830203126576907, + "f1_weighted": 0.2840322841735226, + "precision": 0.29017182598629926, + "precision_weighted": 0.2924584197977978, + "recall": 0.2981945548806803, + "recall_weighted": 0.29763670140167686, + "ap": null, + "ap_weighted": null, + "main_score": 0.29763670140167686, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "mteb_version": "2.21.0" + }, + "small": { + "scores_per_experiment": [ + { + "accuracy": 0.2699798522498321, + "f1": 0.24239504058809014, + "f1_weighted": 0.2423525571760308, + "precision": 0.25470036430672977, + "precision_weighted": 0.2542205264446883, + "recall": 0.26975123145374097, + "recall_weighted": 0.2699798522498321, + "ap": null, + "ap_weighted": null + }, + { + "accuracy": 0.3035594358629953, + "f1": 0.29613784795561465, + "f1_weighted": 0.29906255840921786, + "precision": 0.30407267120079884, + "precision_weighted": 0.30800706496852154, + "recall": 0.3024788327196796, + "recall_weighted": 0.3035594358629953, + "ap": null, + "ap_weighted": null + }, + { + "accuracy": 0.28225806451612906, + "f1": 0.2593664352832718, + "f1_weighted": 0.2636113556497887, + "precision": 0.2609748445531306, + "precision_weighted": 0.26468416511118426, + "recall": 0.2775030429767242, + "recall_weighted": 0.28225806451612906, + "ap": null, + "ap_weighted": null + }, + { + "accuracy": 0.3111559139784946, + "f1": 0.29454980492169786, + "f1_weighted": 0.29125309340152467, + "precision": 0.3224943453400395, + "precision_weighted": 0.3242765511011472, + "recall": 0.31997721819075986, + "recall_weighted": 0.3111559139784946, + "ap": null, + "ap_weighted": null + }, + { + "accuracy": 0.33064516129032256, + "f1": 0.3185711713472314, + "f1_weighted": 0.3198970407995879, + "precision": 0.32320745672952395, + "precision_weighted": 0.32608239411677886, + "recall": 0.3309376196524952, + "recall_weighted": 0.33064516129032256, + "ap": null, + "ap_weighted": null + } + ], + "accuracy": 0.2995196855795547, + "f1": 0.2822040600191812, + "f1_weighted": 0.28323532108723004, + "precision": 0.29308993642604453, + "precision_weighted": 0.295454140348464, + "recall": 0.30012958899868, + "recall_weighted": 0.2995196855795547, + "ap": null, + "ap_weighted": null, + "main_score": 0.2995196855795547, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "mteb_version": "2.21.0" + } + }, + "measurement_by_model": { + "small": { + "evaluated_native_fingerprint": "684fe0c48a4ddd3944167f6e5dc75e97ada24e5b120bd9a2921f00dc03112eeb", + "original_checkpoint_revision": "fdf8e01b7b0f3a69ac1ac8e2a64dcb1ede177ba4", + "raw_result_sha256": "d79b8d114cb86c460dad13d564c2705a41df5119f83a0f1f27d452f36b64af47", + "kind": "direct_frozen_encoder_evaluation" + } + } + }, + { + "task": "CREMA_DClustering", + "type": "AudioClustering", + "modalities": [ + "audio" + ], + "split": "train", + "hf_subset": "default", + "main_metric": "v_measure", + "dataset": { + "path": "mteb/crema-d", + "revision": "a9d0821955c418752248b8310438854e56a1cf34" + }, + "counts": { + "official_stratified_subsample_rows": 2048, + "bootstrap_clustering_trials": 10, + "resampled_items_per_trial": 16384 + }, + "scores": { + "nano": 0.023601715918443712, + "small": 0.024058408888362488 + }, + "metrics": { + "nano": { + "v_measures": { + "Level 0": [ + 0.014290784661453301, + 0.02755216925549553, + 0.017220745995259155, + 0.02907041540588703, + 0.017365020267924067, + 0.022161217359367304, + 0.02806069377818825, + 0.017815432451092263, + 0.03081951157077731, + 0.03166116843899292 + ] + }, + "v_measure": 0.023601715918443712, + "v_measure_std": 0.006199268435462002, + "ami_scores": { + "Level 0": [ + 0.01385865484522556, + 0.027134003314929028, + 0.016785414361355624, + 0.028644278423611804, + 0.016937404067902156, + 0.02173522206231376, + 0.0276339699482267, + 0.0173943490723341, + 0.030397587303172737, + 0.031246814385090853 + ] + }, + "ami": 0.023176769778416234, + "ami_std": 0.0062032784119843, + "main_score": 0.023601715918443712, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "mteb_version": "2.21.0" + }, + "small": { + "v_measures": { + "Level 0": [ + 0.041011040847839235, + 0.03386543153714212, + 0.029099626845588235, + 0.013685755785810546, + 0.02873132169365058, + 0.01881126494637877, + 0.011416660820064256, + 0.02673877787517492, + 0.00569343536524053, + 0.031530773166735686 + ] + }, + "v_measure": 0.024058408888362488, + "v_measure_std": 0.010611552764041994, + "ami_scores": { + "Level 0": [ + 0.04059499123714819, + 0.03344615727359256, + 0.02867541136577102, + 0.01325228658253946, + 0.028312443838566275, + 0.01838242676310465, + 0.01099044807520909, + 0.026321334415061327, + 0.005259713019363485, + 0.031113076527253092 + ] + }, + "ami": 0.023634828909760913, + "ami_std": 0.010617195895208265, + "main_score": 0.024058408888362488, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "mteb_version": "2.21.0" + } + }, + "measurement_by_model": { + "small": { + "evaluated_native_fingerprint": "684fe0c48a4ddd3944167f6e5dc75e97ada24e5b120bd9a2921f00dc03112eeb", + "original_checkpoint_revision": "fdf8e01b7b0f3a69ac1ac8e2a64dcb1ede177ba4", + "raw_result_sha256": "b23d4c552b9fa242923fc0c21a898ee63eb32850d9d91e0682b2cb5e8bea0e76", + "kind": "direct_frozen_encoder_evaluation" + } + } + }, + { + "task": "SpeechCommandsZeroshotv0.02", + "type": "AudioZeroshotClassification", + "modalities": [ + "audio", + "text" + ], + "split": "test", + "hf_subset": "default", + "main_metric": "accuracy", + "dataset": { + "path": "mteb/SpeechCommandsZeroshotv0.02", + "revision": "d5491b288ab76356a5f3cbc634f1dc38e23da2b8" + }, + "counts": { + "evaluation_rows": 4074, + "fixed_text_labels": 10 + }, + "scores": { + "nano": 0.1411389297987236, + "small": 0.05522827687776141 + }, + "metrics": { + "nano": { + "accuracy": 0.1411389297987236, + "f1": 0.06629702019800886, + "f1_weighted": 0.06559730133350446, + "precision": 0.21924492134747023, + "precision_weighted": 0.21974351147386872, + "recall": 0.14280319771568778, + "recall_weighted": 0.1411389297987236, + "ap": null, + "ap_weighted": null, + "main_score": 0.1411389297987236, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "mteb_version": "2.21.0" + }, + "small": { + "accuracy": 0.05522827687776141, + "f1": 0.037875265984290724, + "f1_weighted": 0.03803483258988871, + "precision": 0.14410977979833245, + "precision_weighted": 0.143575354561912, + "recall": 0.05521141290348569, + "recall_weighted": 0.05522827687776141, + "ap": null, + "ap_weighted": null, + "main_score": 0.05522827687776141, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "mteb_version": "2.21.0" + } + }, + "measurement_by_model": { + "small": { + "evaluated_native_fingerprint": "684fe0c48a4ddd3944167f6e5dc75e97ada24e5b120bd9a2921f00dc03112eeb", + "original_checkpoint_revision": "fdf8e01b7b0f3a69ac1ac8e2a64dcb1ede177ba4", + "raw_result_sha256": "ca19c03bbe2fdcf3c2e1282f5e826af702c152287fde75d67b28ef877fae5335", + "kind": "direct_frozen_encoder_evaluation" + } + } + } + ], + "comparison": { + "baseline_model": "llm-semantic-router/multi-modal-embed-small", + "baseline_revision": "fdf8e01b7b0f3a69ac1ac8e2a64dcb1ede177ba4", + "baseline_native_artifact_sha256": "684fe0c48a4ddd3944167f6e5dc75e97ada24e5b120bd9a2921f00dc03112eeb", + "baseline_standard_status": "not_evaluated_in_this_standard_protocol", + "card_bold": "Only unrounded gains against this original baseline are bold; missing baseline scores imply no comparison." + }, + "observation_scope": "Only this model is reported; original comparator has no observations in the fixed11 protocol. Evaluated/applicable identities are preserved.", + "encoding_observations": { + "SICK-R": { + "nano": { + "before": { + "text_rows": 0, + "text_rows_truncated": 0, + "max_original_text_tokens": 0, + "image_rows": 0, + "audio_rows": 0 + }, + "after": { + "text_rows": 19854, + "text_rows_truncated": 0, + "max_original_text_tokens": 38, + "image_rows": 0, + "audio_rows": 0 + }, + "elapsed_seconds": 10.807591969991336, + "peak_allocated_gpu_bytes": 679632896, + "peak_reserved_gpu_bytes": 719323136 + } + }, + "STSBenchmark": { + "nano": { + "before": { + "text_rows": 19854, + "text_rows_truncated": 0, + "max_original_text_tokens": 38, + "image_rows": 0, + "audio_rows": 0 + }, + "after": { + "text_rows": 22612, + "text_rows_truncated": 0, + "max_original_text_tokens": 46, + "image_rows": 0, + "audio_rows": 0 + }, + "elapsed_seconds": 1.294741225981852, + "peak_allocated_gpu_bytes": 681261056, + "peak_reserved_gpu_bytes": 723517440 + } + }, + "ArguAna": { + "nano": { + "before": { + "text_rows": 22612, + "text_rows_truncated": 0, + "max_original_text_tokens": 46, + "image_rows": 0, + "audio_rows": 0 + }, + "after": { + "text_rows": 32692, + "text_rows_truncated": 222, + "max_original_text_tokens": 1401, + "image_rows": 0, + "audio_rows": 0 + }, + "elapsed_seconds": 15.392282173008425, + "peak_allocated_gpu_bytes": 809891328, + "peak_reserved_gpu_bytes": 887095296 + } + }, + "TwentyNewsgroupsClustering.v2": { + "nano": { + "before": { + "text_rows": 32692, + "text_rows_truncated": 222, + "max_original_text_tokens": 1401, + "image_rows": 0, + "audio_rows": 0 + }, + "after": { + "text_rows": 35073, + "text_rows_truncated": 222, + "max_original_text_tokens": 1401, + "image_rows": 0, + "audio_rows": 0 + }, + "elapsed_seconds": 48.54122133200872, + "peak_allocated_gpu_bytes": 684661248, + "peak_reserved_gpu_bytes": 727711744 + } + }, + "MTOPDomainClassification": { + "nano": { + "before": { + "text_rows": 35073, + "text_rows_truncated": 222, + "max_original_text_tokens": 1401, + "image_rows": 0, + "audio_rows": 0 + }, + "after": { + "text_rows": 43422, + "text_rows_truncated": 222, + "max_original_text_tokens": 1401, + "image_rows": 0, + "audio_rows": 0 + }, + "elapsed_seconds": 25.26324965900858, + "peak_allocated_gpu_bytes": 679308288, + "peak_reserved_gpu_bytes": 719323136 + } + }, + "OxfordPets": { + "nano": { + "before": { + "text_rows": 43422, + "text_rows_truncated": 222, + "max_original_text_tokens": 1401, + "image_rows": 0, + "audio_rows": 0 + }, + "after": { + "text_rows": 43422, + "text_rows_truncated": 222, + "max_original_text_tokens": 1401, + "image_rows": 5816, + "audio_rows": 0 + }, + "elapsed_seconds": 54.48569396699895, + "peak_allocated_gpu_bytes": 996472320, + "peak_reserved_gpu_bytes": 1207959552 + } + }, + "TinyImageNetClustering": { + "nano": { + "before": { + "text_rows": 43422, + "text_rows_truncated": 222, + "max_original_text_tokens": 1401, + "image_rows": 5816, + "audio_rows": 0 + }, + "after": { + "text_rows": 43422, + "text_rows_truncated": 222, + "max_original_text_tokens": 1401, + "image_rows": 15816, + "audio_rows": 0 + }, + "elapsed_seconds": 72.07803665398387, + "peak_allocated_gpu_bytes": 996472320, + "peak_reserved_gpu_bytes": 1117782016 + } + }, + "OxfordPetsZeroShot": { + "nano": { + "before": { + "text_rows": 43422, + "text_rows_truncated": 222, + "max_original_text_tokens": 1401, + "image_rows": 15816, + "audio_rows": 0 + }, + "after": { + "text_rows": 43459, + "text_rows_truncated": 222, + "max_original_text_tokens": 1401, + "image_rows": 19485, + "audio_rows": 0 + }, + "elapsed_seconds": 32.662016286019934, + "peak_allocated_gpu_bytes": 996472320, + "peak_reserved_gpu_bytes": 1119879168 + } + }, + "CREMA_D": { + "nano": { + "before": { + "text_rows": 43459, + "text_rows_truncated": 222, + "max_original_text_tokens": 1401, + "image_rows": 19485, + "audio_rows": 0 + }, + "after": { + "text_rows": 43459, + "text_rows_truncated": 222, + "max_original_text_tokens": 1401, + "image_rows": 19485, + "audio_rows": 7442 + }, + "elapsed_seconds": 135.41051389899803, + "peak_allocated_gpu_bytes": 971823104, + "peak_reserved_gpu_bytes": 1224736768 + } + }, + "CREMA_DClustering": { + "nano": { + "before": { + "text_rows": 43459, + "text_rows_truncated": 222, + "max_original_text_tokens": 1401, + "image_rows": 19485, + "audio_rows": 7442 + }, + "after": { + "text_rows": 43459, + "text_rows_truncated": 222, + "max_original_text_tokens": 1401, + "image_rows": 19485, + "audio_rows": 9490 + }, + "elapsed_seconds": 73.78414583200356, + "peak_allocated_gpu_bytes": 971823104, + "peak_reserved_gpu_bytes": 1224736768 + } + }, + "SpeechCommandsZeroshotv0.02": { + "nano": { + "before": { + "text_rows": 43459, + "text_rows_truncated": 222, + "max_original_text_tokens": 1401, + "image_rows": 19485, + "audio_rows": 9490 + }, + "after": { + "text_rows": 43469, + "text_rows_truncated": 222, + "max_original_text_tokens": 1401, + "image_rows": 19485, + "audio_rows": 13564 + }, + "elapsed_seconds": 63.873327202978544, + "peak_allocated_gpu_bytes": 971823104, + "peak_reserved_gpu_bytes": 1224736768 + } + } + } + }, + "capabilities": { + "max_text_tokens": 512, + "tokens_include_special_tokens": true, + "public_text_overflow": "Raises ValueError; shorten or explicitly chunk longer inputs.", + "embedding_dimensions": 384, + "parameters": 135383808 + }, + "efficiency": { + "status": "not_measured_for_this_release", + "parameters": 135383808, + "embedding_dimensions": 384, + "dense_fp32_index_bytes_per_vector": 1536, + "weights_safetensors_bytes": 541598024, + "scope": "No new latency or peak-memory measurement is claimed. Vector storage excludes index structures and metadata.", + "historical_measurement": { + "attention": "native SDPA", + "batch_size": 8, + "embedding_dtype": "float32", + "hardware": "AMD gfx942 GPU", + "latency_includes": "Native public encode_text: tokenization, host-to-device transfer and encoding; excludes model loading.", + "measured_batches": 20, + "measurements": { + "128": { + "batch_size": 8, + "p50_ms": 4.283615489839576, + "p95_ms": 4.314010989037342, + "peak_allocated_bytes": 653202944, + "peak_reserved_bytes": 673185792, + "tokens_each_including_special": 128 + }, + "512": { + "batch_size": 8, + "p50_ms": 12.445501997717656, + "p95_ms": 12.603181986196432, + "peak_allocated_bytes": 878682112, + "peak_reserved_bytes": 970981376, + "tokens_each_including_special": 512 + } + }, + "memory_scope": "Process CUDA allocator peak with the complete multimodal model loaded.", + "model_revision": "70c3412bb0ddfb19386d673d4a102c753a574f1b", + "native_artifact_sha256": "b596f1dbab89369555e0a72a0d6854d780e305686750949b308c8f167993bdeb", + "scope": "Synthetic repeated text-batch measurements on one GPU; not end-to-end service latency.", + "tf32": false, + "token_lengths_include_special_tokens": true, + "torch_version_family": "2.12", + "transformers_version": "4.57.6", + "warmup_batches": 2, + "status": "historical_previous_release_measurement", + "applicability": "Historical measurement on its recorded native artifact; not applicable to this text-backbone update." + }, + "parameter_storage_change": { + "old_safetensors_bytes": 578289664, + "new_safetensors_bytes": 541598024, + "reduction_bytes": 36691640, + "old_FP32_parameter_bytes": 578216448, + "new_FP32_parameter_bytes": 541535232, + "scope": "Parameter storage only; not measured process peak memory or latency. Vector and index dimensions are unchanged." + } + }, + "previous_release": { + "model": "llm-semantic-router/Vela-1.0-Omni-Nano", + "revision": "3a1efc48995e2b71622d65c6cf71cfc00bbf3aa3", + "native_artifact_sha256": "fb9cd39c9980a085b75456c715c2295b997784fe733314be68c13635f6384edf", + "parameters": 133956480, + "metrics": { + "audio.audio_to_text.recall@1": { + "denominator": 2611, + "numerator": 235, + "value": 0.09000382995021065 + }, + "audio.audio_to_text.recall@10": { + "denominator": 2611, + "numerator": 822, + "value": 0.31482190731520493 + }, + "audio.audio_to_text.recall@5": { + "denominator": 2611, + "numerator": 582, + "value": 0.22290310225967064 + }, + "audio.text_to_audio.recall@1": { + "denominator": 2610, + "numerator": 366, + "value": 0.14022988505747128 + }, + "audio.text_to_audio.recall@10": { + "denominator": 2610, + "numerator": 1055, + "value": 0.4042145593869732 + }, + "audio.text_to_audio.recall@5": { + "denominator": 2610, + "numerator": 827, + "value": 0.3168582375478927 + }, + "image.image_to_text.recall@1": { + "denominator": 823, + "numerator": 540, + "value": 0.6561360874848117 + }, + "image.image_to_text.recall@10": { + "denominator": 823, + "numerator": 791, + "value": 0.9611178614823815 + }, + "image.image_to_text.recall@5": { + "denominator": 823, + "numerator": 731, + "value": 0.8882138517618469 + }, + "image.text_to_image.recall@1": { + "denominator": 4115, + "numerator": 2184, + "value": 0.5307411907654921 + }, + "image.text_to_image.recall@10": { + "denominator": 4115, + "numerator": 3847, + "value": 0.934872417982989 + }, + "image.text_to_image.recall@5": { + "denominator": 4115, + "numerator": 3465, + "value": 0.8420413122721749 + }, + "text.banking77.accuracy": { + "denominator": 3080, + "numerator": 2366, + "value": 0.7681818181818182 + }, + "text.massive-en.accuracy": { + "denominator": 2972, + "numerator": 2061, + "value": 0.6934724091520862 + } + }, + "descriptive_metrics": { + "text.banking77.macro_f1": { + "value": 0.7658421887247635, + "classes": 77, + "queries": 3080 + }, + "text.massive-en.macro_f1": { + "value": 0.6524376423162815, + "classes": 60, + "queries": 2972 + } + }, + "standard_rows": [ + { + "task": "SICK-R", + "split": "test", + "hf_subset": "default", + "main_metric": "cosine_spearman", + "score": 0.7169618925026362, + "metrics": { + "pearson": 0.8194169233518556, + "spearman": 0.7169617707661863, + "cosine_pearson": 0.819416922085493, + "cosine_spearman": 0.7169618925026362, + "manhattan_pearson": 0.7935695063285152, + "manhattan_spearman": 0.7160084195066224, + "euclidean_pearson": 0.7944328514895539, + "euclidean_spearman": 0.7169617703551949, + "main_score": 0.7169618925026362, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "mteb_version": "2.21.0" + } + }, + { + "task": "STSBenchmark", + "split": "test", + "hf_subset": "default", + "main_metric": "cosine_spearman", + "score": 0.7792400226361769, + "metrics": { + "pearson": 0.7717062133633645, + "spearman": 0.7792400217447207, + "cosine_pearson": 0.7717062156025293, + "cosine_spearman": 0.7792400226361769, + "manhattan_pearson": 0.7928914389432686, + "manhattan_spearman": 0.7777756255094064, + "euclidean_pearson": 0.7949414830254274, + "euclidean_spearman": 0.779239966513648, + "main_score": 0.7792400226361769, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "mteb_version": "2.21.0" + } + }, + { + "task": "ArguAna", + "split": "test", + "hf_subset": "default", + "main_metric": "ndcg_at_10", + "score": 0.37514, + "metrics": { + "ndcg_at_1": 0.17425, + "ndcg_at_3": 0.27012, + "ndcg_at_5": 0.31718, + "ndcg_at_10": 0.37514, + "ndcg_at_20": 0.40855, + "ndcg_at_100": 0.43759, + "ndcg_at_1000": 0.44701, + "map_at_1": 0.17425, + "map_at_3": 0.24561, + "map_at_5": 0.2715, + "map_at_10": 0.29556, + "map_at_20": 0.30489, + "map_at_100": 0.3093, + "map_at_1000": 0.30967, + "recall_at_1": 0.17425, + "recall_at_3": 0.34139, + "recall_at_5": 0.45661, + "recall_at_10": 0.63514, + "recall_at_20": 0.766, + "recall_at_100": 0.91679, + "recall_at_1000": 0.99004, + "accuracy": 0.17425, + "precision_at_1": 0.17425, + "precision_at_3": 0.1138, + "precision_at_5": 0.09132, + "precision_at_10": 0.06351, + "precision_at_20": 0.0383, + "precision_at_100": 0.00917, + "precision_at_1000": 0.00099, + "mrr_at_1": 0.17425320056899005, + "mrr_at_3": 0.24561403508771928, + "mrr_at_5": 0.2715030820293978, + "mrr_at_10": 0.2955561764772291, + "mrr_at_20": 0.30489450370601584, + "mrr_at_100": 0.3092982324964941, + "mrr_at_1000": 0.30966870165383564, + "nauc_ndcg_at_1_max": -0.012854409302840068, + "nauc_ndcg_at_1_std": 0.08676381700185752, + "nauc_ndcg_at_1_diff1": 0.11334791281377288, + "nauc_ndcg_at_3_max": 0.0007934267314863777, + "nauc_ndcg_at_3_std": 0.05484641820257593, + "nauc_ndcg_at_3_diff1": 0.07348303062976835, + "nauc_ndcg_at_5_max": 0.020607224558072373, + "nauc_ndcg_at_5_std": 0.0741284488966682, + "nauc_ndcg_at_5_diff1": 0.08730888772052957, + "nauc_ndcg_at_10_max": 0.035048132350077, + "nauc_ndcg_at_10_std": 0.10523053259120095, + "nauc_ndcg_at_10_diff1": 0.08020955970997308, + "nauc_ndcg_at_20_max": 0.04122292301118309, + "nauc_ndcg_at_20_std": 0.12537385022454034, + "nauc_ndcg_at_20_diff1": 0.0842700247341082, + "nauc_ndcg_at_100_max": 0.04333008607419203, + "nauc_ndcg_at_100_std": 0.12294409372370381, + "nauc_ndcg_at_100_diff1": 0.08845423517529309, + "nauc_ndcg_at_1000_max": 0.030127261370811845, + "nauc_ndcg_at_1000_std": 0.10766365950024334, + "nauc_ndcg_at_1000_diff1": 0.0861546357811246, + "nauc_map_at_1_max": -0.012854409302840068, + "nauc_map_at_1_std": 0.08676381700185752, + "nauc_map_at_1_diff1": 0.11334791281377288, + "nauc_map_at_3_max": -0.0029405293797002733, + "nauc_map_at_3_std": 0.06167764879997172, + "nauc_map_at_3_diff1": 0.07950290715235987, + "nauc_map_at_5_max": 0.00830536326209409, + "nauc_map_at_5_std": 0.07256833511730693, + "nauc_map_at_5_diff1": 0.08747148302006987, + "nauc_map_at_10_max": 0.015140286093876175, + "nauc_map_at_10_std": 0.08580662522686537, + "nauc_map_at_10_diff1": 0.08508762518017199, + "nauc_map_at_20_max": 0.0163028127500422, + "nauc_map_at_20_std": 0.09050711291668995, + "nauc_map_at_20_diff1": 0.08629712373980958, + "nauc_map_at_100_max": 0.016183246425002705, + "nauc_map_at_100_std": 0.09000154689011092, + "nauc_map_at_100_diff1": 0.08699050038257111, + "nauc_map_at_1000_max": 0.015701662616224514, + "nauc_map_at_1000_std": 0.0894576065792463, + "nauc_map_at_1000_diff1": 0.08694453879028893, + "nauc_recall_at_1_max": -0.012854409302840068, + "nauc_recall_at_1_std": 0.08676381700185752, + "nauc_recall_at_1_diff1": 0.11334791281377288, + "nauc_recall_at_3_max": 0.010338756866368223, + "nauc_recall_at_3_std": 0.03789278134763109, + "nauc_recall_at_3_diff1": 0.059403567851730316, + "nauc_recall_at_5_max": 0.05398176066498534, + "nauc_recall_at_5_std": 0.08064232472596307, + "nauc_recall_at_5_diff1": 0.08949514479783123, + "nauc_recall_at_10_max": 0.09984307699040616, + "nauc_recall_at_10_std": 0.17703668336587267, + "nauc_recall_at_10_diff1": 0.06498907396830537, + "nauc_recall_at_20_max": 0.15799758987163673, + "nauc_recall_at_20_std": 0.31157412853790695, + "nauc_recall_at_20_diff1": 0.07941702046000772, + "nauc_recall_at_100_max": 0.37265902510843735, + "nauc_recall_at_100_std": 0.5476974790176777, + "nauc_recall_at_100_diff1": 0.12977852219557848, + "nauc_recall_at_1000_max": 0.3004922701567, + "nauc_recall_at_1000_std": 0.7475055455618029, + "nauc_recall_at_1000_diff1": -0.05500901218402981, + "nauc_precision_at_1_max": -0.012854409302840068, + "nauc_precision_at_1_std": 0.08676381700185752, + "nauc_precision_at_1_diff1": 0.11334791281377288, + "nauc_precision_at_3_max": 0.010338756866368036, + "nauc_precision_at_3_std": 0.037892781347630994, + "nauc_precision_at_3_diff1": 0.05940356785173012, + "nauc_precision_at_5_max": 0.05398176066498592, + "nauc_precision_at_5_std": 0.08064232472596324, + "nauc_precision_at_5_diff1": 0.08949514479783155, + "nauc_precision_at_10_max": 0.09984307699040575, + "nauc_precision_at_10_std": 0.17703668336587192, + "nauc_precision_at_10_diff1": 0.06498907396830521, + "nauc_precision_at_20_max": 0.1579975898716373, + "nauc_precision_at_20_std": 0.31157412853790756, + "nauc_precision_at_20_diff1": 0.0794170204600081, + "nauc_precision_at_100_max": 0.3726590251084365, + "nauc_precision_at_100_std": 0.5476974790176794, + "nauc_precision_at_100_diff1": 0.1297785221955785, + "nauc_precision_at_1000_max": 0.3004922701567069, + "nauc_precision_at_1000_std": 0.7475055455618104, + "nauc_precision_at_1000_diff1": -0.055009012184042826, + "nauc_mrr_at_1_max": -0.012854409302840068, + "nauc_mrr_at_1_std": 0.08676381700185752, + "nauc_mrr_at_1_diff1": 0.11334791281377288, + "nauc_mrr_at_3_max": -0.0029405293797002733, + "nauc_mrr_at_3_std": 0.06167764879997172, + "nauc_mrr_at_3_diff1": 0.07950290715235987, + "nauc_mrr_at_5_max": 0.00830536326209409, + "nauc_mrr_at_5_std": 0.07256833511730693, + "nauc_mrr_at_5_diff1": 0.08747148302006987, + "nauc_mrr_at_10_max": 0.015140286093876175, + "nauc_mrr_at_10_std": 0.08580662522686537, + "nauc_mrr_at_10_diff1": 0.08508762518017199, + "nauc_mrr_at_20_max": 0.0163028127500422, + "nauc_mrr_at_20_std": 0.09050711291668995, + "nauc_mrr_at_20_diff1": 0.08629712373980958, + "nauc_mrr_at_100_max": 0.016183246425002705, + "nauc_mrr_at_100_std": 0.09000154689011092, + "nauc_mrr_at_100_diff1": 0.08699050038257111, + "nauc_mrr_at_1000_max": 0.015701662616224514, + "nauc_mrr_at_1000_std": 0.0894576065792463, + "nauc_mrr_at_1000_diff1": 0.08694453879028893, + "hit_rate_at_1": 0.17425, + "hit_rate_at_3": 0.34139, + "hit_rate_at_5": 0.45661, + "hit_rate_at_10": 0.63514, + "hit_rate_at_20": 0.766, + "hit_rate_at_100": 0.91679, + "hit_rate_at_1000": 0.99004, + "main_score": 0.37514, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "mteb_version": "2.21.0" + } + }, + { + "task": "TwentyNewsgroupsClustering.v2", + "split": "test", + "hf_subset": "default", + "main_metric": "v_measure", + "score": 0.44884717457494344, + "metrics": { + "v_measures": { + "Level 0": [ + 0.48056098748371134, + 0.44085942767025554, + 0.43416151810540005, + 0.45299559752505, + 0.44116051339311235, + 0.4409873172673812, + 0.46155584917327236, + 0.444819328389149, + 0.44792019268133576, + 0.44345101406076604 + ] + }, + "v_measure": 0.44884717457494344, + "v_measure_std": 0.01277114663774588, + "ami_scores": { + "Level 0": [ + 0.4786019649327535, + 0.43870317373726253, + 0.43201951778515824, + 0.4509348225877084, + 0.43904842199622773, + 0.43887546309559994, + 0.45953222355170814, + 0.44270204450063555, + 0.44583346842612326, + 0.4413500450061514 + ] + }, + "ami": 0.4467601145619328, + "ami_std": 0.012825331450796505, + "main_score": 0.44884717457494344, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "mteb_version": "2.21.0" + } + }, + { + "task": "MTOPDomainClassification", + "split": "validation", + "hf_subset": "en", + "main_metric": "accuracy", + "score": 0.8758836689038031, + "metrics": { + "scores_per_experiment": [ + { + "accuracy": 0.8527964205816555, + "f1": 0.8584702959865076, + "f1_weighted": 0.8520037076109549, + "precision": 0.8563814369311937, + "precision_weighted": 0.8604397214633738, + "recall": 0.8688240583945851, + "recall_weighted": 0.8527964205816555, + "ap": null, + "ap_weighted": null + }, + { + "accuracy": 0.8675615212527964, + "f1": 0.8658660043655293, + "f1_weighted": 0.8661413651963479, + "precision": 0.8632122218181354, + "precision_weighted": 0.8687564991167056, + "recall": 0.8733845956286541, + "recall_weighted": 0.8675615212527964, + "ap": null, + "ap_weighted": null + }, + { + "accuracy": 0.8514541387024609, + "f1": 0.8524202289070963, + "f1_weighted": 0.8491348169886809, + "precision": 0.8478156770106394, + "precision_weighted": 0.8512267923319679, + "recall": 0.8621133428009895, + "recall_weighted": 0.8514541387024609, + "ap": null, + "ap_weighted": null + }, + { + "accuracy": 0.8876957494407158, + "f1": 0.8889547651830694, + "f1_weighted": 0.886393441219644, + "precision": 0.8855262149097257, + "precision_weighted": 0.8869170708224573, + "recall": 0.8941464114507656, + "recall_weighted": 0.8876957494407158, + "ap": null, + "ap_weighted": null + }, + { + "accuracy": 0.8684563758389262, + "f1": 0.8708980361567344, + "f1_weighted": 0.8668343457695604, + "precision": 0.8655644216656676, + "precision_weighted": 0.8685796219847712, + "recall": 0.8789277362563833, + "recall_weighted": 0.8684563758389262, + "ap": null, + "ap_weighted": null + }, + { + "accuracy": 0.8662192393736018, + "f1": 0.8725560012982636, + "f1_weighted": 0.8665613320034511, + "precision": 0.8737258252820442, + "precision_weighted": 0.8845738920578841, + "recall": 0.8863222925022911, + "recall_weighted": 0.8662192393736018, + "ap": null, + "ap_weighted": null + }, + { + "accuracy": 0.8953020134228188, + "f1": 0.8976332709955689, + "f1_weighted": 0.8945899154754817, + "precision": 0.8912735415957905, + "precision_weighted": 0.8979949591784757, + "recall": 0.908092721226657, + "recall_weighted": 0.8953020134228188, + "ap": null, + "ap_weighted": null + }, + { + "accuracy": 0.9029082774049217, + "f1": 0.9022310464802398, + "f1_weighted": 0.9020525018623339, + "precision": 0.899983538025133, + "precision_weighted": 0.9036775461946326, + "recall": 0.9067592943176375, + "recall_weighted": 0.9029082774049217, + "ap": null, + "ap_weighted": null + }, + { + "accuracy": 0.876510067114094, + "f1": 0.8776887066559357, + "f1_weighted": 0.8761842615927375, + "precision": 0.8722809982436143, + "precision_weighted": 0.8829904469624968, + "recall": 0.8888620916111349, + "recall_weighted": 0.876510067114094, + "ap": null, + "ap_weighted": null + }, + { + "accuracy": 0.8899328859060402, + "f1": 0.8916131163462019, + "f1_weighted": 0.8885446252332447, + "precision": 0.8855482140891268, + "precision_weighted": 0.8923990752693544, + "recall": 0.9020276692969261, + "recall_weighted": 0.8899328859060402, + "ap": null, + "ap_weighted": null + } + ], + "accuracy": 0.8758836689038031, + "f1": 0.8778331472375147, + "f1_weighted": 0.8748440312952436, + "precision": 0.8741312089571069, + "precision_weighted": 0.8797555625382119, + "recall": 0.8869460213486026, + "recall_weighted": 0.8758836689038031, + "ap": null, + "ap_weighted": null, + "main_score": 0.8758836689038031, + "hf_subset": "en", + "languages": [ + "eng-Latn" + ], + "mteb_version": "2.21.0" + } + }, + { + "task": "MTOPDomainClassification", + "split": "test", + "hf_subset": "en", + "main_metric": "accuracy", + "score": 0.8671910624715002, + "metrics": { + "scores_per_experiment": [ + { + "accuracy": 0.8492932056543547, + "f1": 0.8506491551806685, + "f1_weighted": 0.8484338913731928, + "precision": 0.847171519646365, + "precision_weighted": 0.857621901965629, + "recall": 0.8634426315569471, + "recall_weighted": 0.8492932056543547, + "ap": null, + "ap_weighted": null + }, + { + "accuracy": 0.8593251253989969, + "f1": 0.8525570927270549, + "f1_weighted": 0.8588269042231532, + "precision": 0.849564509883911, + "precision_weighted": 0.8659425848467468, + "recall": 0.8649266896842414, + "recall_weighted": 0.8593251253989969, + "ap": null, + "ap_weighted": null + }, + { + "accuracy": 0.8508891928864569, + "f1": 0.8445077740960396, + "f1_weighted": 0.8500136039882616, + "precision": 0.83976876536009, + "precision_weighted": 0.8551546537983608, + "recall": 0.8573297788521569, + "recall_weighted": 0.8508891928864569, + "ap": null, + "ap_weighted": null + }, + { + "accuracy": 0.8702690378476973, + "f1": 0.8680982220684261, + "f1_weighted": 0.8688890737378698, + "precision": 0.8651925230157509, + "precision_weighted": 0.8699395458498034, + "recall": 0.873034481111479, + "recall_weighted": 0.8702690378476973, + "ap": null, + "ap_weighted": null + }, + { + "accuracy": 0.8657090743274054, + "f1": 0.8639593934313017, + "f1_weighted": 0.8646291417790959, + "precision": 0.8570893217526883, + "precision_weighted": 0.8672508842970594, + "recall": 0.8739690035000478, + "recall_weighted": 0.8657090743274054, + "ap": null, + "ap_weighted": null + }, + { + "accuracy": 0.8520291837665299, + "f1": 0.8534418954962738, + "f1_weighted": 0.8548114072639492, + "precision": 0.85875145236006, + "precision_weighted": 0.8803115046026259, + "recall": 0.869567932536695, + "recall_weighted": 0.8520291837665299, + "ap": null, + "ap_weighted": null + }, + { + "accuracy": 0.8869129046967624, + "f1": 0.8849494340089117, + "f1_weighted": 0.8871874824644057, + "precision": 0.8779834780597185, + "precision_weighted": 0.8955288700582439, + "recall": 0.9009013020211545, + "recall_weighted": 0.8869129046967624, + "ap": null, + "ap_weighted": null + }, + { + "accuracy": 0.875968992248062, + "f1": 0.8715420414043028, + "f1_weighted": 0.8753897998685723, + "precision": 0.86652024620862, + "precision_weighted": 0.8811614860860878, + "recall": 0.881802359118481, + "recall_weighted": 0.875968992248062, + "ap": null, + "ap_weighted": null + }, + { + "accuracy": 0.8755129958960328, + "f1": 0.8710791039688399, + "f1_weighted": 0.8760974709416987, + "precision": 0.8647173948263537, + "precision_weighted": 0.8831121050674345, + "recall": 0.8845140142042784, + "recall_weighted": 0.8755129958960328, + "ap": null, + "ap_weighted": null + }, + { + "accuracy": 0.886000911992704, + "f1": 0.8827257199119498, + "f1_weighted": 0.8855721338303499, + "precision": 0.8760471482914741, + "precision_weighted": 0.8912159468319844, + "recall": 0.8952867706565555, + "recall_weighted": 0.886000911992704, + "ap": null, + "ap_weighted": null + } + ], + "accuracy": 0.8671910624715002, + "f1": 0.8643509832293768, + "f1_weighted": 0.8669850909470549, + "precision": 0.8602806359405031, + "precision_weighted": 0.8747239483403975, + "recall": 0.8764774963242037, + "recall_weighted": 0.8671910624715002, + "ap": null, + "ap_weighted": null, + "main_score": 0.8671910624715002, + "hf_subset": "en", + "languages": [ + "eng-Latn" + ], + "mteb_version": "2.21.0" + } + }, + { + "task": "OxfordPets", + "split": "test", + "hf_subset": "default", + "main_metric": "accuracy", + "score": 0.9126737530662306, + "metrics": { + "scores_per_experiment": [ + { + "accuracy": 0.9062414826928318, + "f1": 0.9045301316021502, + "f1_weighted": 0.9050055409371512, + "precision": 0.9099213291951324, + "precision_weighted": 0.9109544817876485, + "recall": 0.9064295837897134, + "recall_weighted": 0.9062414826928318, + "ap": null, + "ap_weighted": null + }, + { + "accuracy": 0.914418097574271, + "f1": 0.9126724638115935, + "f1_weighted": 0.9132186480967779, + "precision": 0.9192904646518116, + "precision_weighted": 0.9203471940294945, + "recall": 0.9144593228002407, + "recall_weighted": 0.914418097574271, + "ap": null, + "ap_weighted": null + }, + { + "accuracy": 0.9187789588443718, + "f1": 0.9177765846692281, + "f1_weighted": 0.9184111765200857, + "precision": 0.9220755461580387, + "precision_weighted": 0.9229278273528245, + "recall": 0.9183739103921627, + "recall_weighted": 0.9187789588443718, + "ap": null, + "ap_weighted": null + }, + { + "accuracy": 0.9174161896974653, + "f1": 0.9147622354932347, + "f1_weighted": 0.9154663340384108, + "precision": 0.9209055798643112, + "precision_weighted": 0.9221124471295344, + "recall": 0.9173606715378323, + "recall_weighted": 0.9174161896974653, + "ap": null, + "ap_weighted": null + }, + { + "accuracy": 0.9065140365222132, + "f1": 0.9051669100036687, + "f1_weighted": 0.9057728210054364, + "precision": 0.9098426973534027, + "precision_weighted": 0.9105143284060627, + "recall": 0.9059560197061645, + "recall_weighted": 0.9065140365222132, + "ap": null, + "ap_weighted": null + } + ], + "accuracy": 0.9126737530662306, + "f1": 0.910981665115975, + "f1_weighted": 0.9115749041195725, + "precision": 0.9164071234445392, + "precision_weighted": 0.9173712557411129, + "recall": 0.9125159016452227, + "recall_weighted": 0.9126737530662306, + "ap": null, + "ap_weighted": null, + "main_score": 0.9126737530662306, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "mteb_version": "2.21.0" + } + }, + { + "task": "TinyImageNetClustering", + "split": "valid", + "hf_subset": "default", + "main_metric": "nmi", + "score": 0.6169193395626786, + "metrics": { + "v_measure": 0.6169193395626787, + "nmi": 0.6169193395626786, + "ari": 0.21497891543638262, + "cluster_accuracy": 0.3638, + "main_score": 0.6169193395626786, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "mteb_version": "2.21.0" + } + }, + { + "task": "OxfordPetsZeroShot", + "split": "test", + "hf_subset": "default", + "main_metric": "accuracy", + "score": 0.1163804851458163, + "metrics": { + "accuracy": 0.1163804851458163, + "f1": 0.085822378073852, + "f1_weighted": 0.08530524305774022, + "precision": 0.08885688520073651, + "precision_weighted": 0.08856851170630324, + "recall": 0.11709192030781942, + "recall_weighted": 0.1163804851458163, + "ap": null, + "ap_weighted": null, + "main_score": 0.1163804851458163, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "mteb_version": "2.21.0" + } + }, + { + "task": "CREMA_D", + "split": "train", + "hf_subset": "default", + "main_metric": "accuracy", + "score": 0.29763670140167686, + "metrics": { + "scores_per_experiment": [ + { + "accuracy": 0.2847548690396239, + "f1": 0.25980556646698677, + "f1_weighted": 0.2606862265023784, + "precision": 0.2685141966762838, + "precision_weighted": 0.26869432571348717, + "recall": 0.283361264324777, + "recall_weighted": 0.2847548690396239, + "ap": null, + "ap_weighted": null + }, + { + "accuracy": 0.2981867024848892, + "f1": 0.2927999429509691, + "f1_weighted": 0.2952410831252662, + "precision": 0.29988591178720175, + "precision_weighted": 0.30336098208055257, + "recall": 0.29763527945216217, + "recall_weighted": 0.2981867024848892, + "ap": null, + "ap_weighted": null + }, + { + "accuracy": 0.28696236559139787, + "f1": 0.26582489649494206, + "f1_weighted": 0.26920117230368773, + "precision": 0.26489987169540286, + "precision_weighted": 0.2682170436584329, + "recall": 0.2834616565816636, + "recall_weighted": 0.28696236559139787, + "ap": null, + "ap_weighted": null + }, + { + "accuracy": 0.29973118279569894, + "f1": 0.291485315935673, + "f1_weighted": 0.28841683151437103, + "precision": 0.3068565359099591, + "precision_weighted": 0.3086441612810119, + "recall": 0.30801945566008065, + "recall_weighted": 0.29973118279569894, + "ap": null, + "ap_weighted": null + }, + { + "accuracy": 0.3185483870967742, + "f1": 0.3051858414398825, + "f1_weighted": 0.3066161074219095, + "precision": 0.310702613862649, + "precision_weighted": 0.3133755862555043, + "recall": 0.31849511838471833, + "recall_weighted": 0.3185483870967742, + "ap": null, + "ap_weighted": null + } + ], + "accuracy": 0.29763670140167686, + "f1": 0.2830203126576907, + "f1_weighted": 0.2840322841735226, + "precision": 0.29017182598629926, + "precision_weighted": 0.2924584197977978, + "recall": 0.2981945548806803, + "recall_weighted": 0.29763670140167686, + "ap": null, + "ap_weighted": null, + "main_score": 0.29763670140167686, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "mteb_version": "2.21.0" + } + }, + { + "task": "CREMA_DClustering", + "split": "train", + "hf_subset": "default", + "main_metric": "v_measure", + "score": 0.023601715918443712, + "metrics": { + "v_measures": { + "Level 0": [ + 0.014290784661453301, + 0.02755216925549553, + 0.017220745995259155, + 0.02907041540588703, + 0.017365020267924067, + 0.022161217359367304, + 0.02806069377818825, + 0.017815432451092263, + 0.03081951157077731, + 0.03166116843899292 + ] + }, + "v_measure": 0.023601715918443712, + "v_measure_std": 0.006199268435462002, + "ami_scores": { + "Level 0": [ + 0.01385865484522556, + 0.027134003314929028, + 0.016785414361355624, + 0.028644278423611804, + 0.016937404067902156, + 0.02173522206231376, + 0.0276339699482267, + 0.0173943490723341, + 0.030397587303172737, + 0.031246814385090853 + ] + }, + "ami": 0.023176769778416234, + "ami_std": 0.0062032784119843, + "main_score": 0.023601715918443712, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "mteb_version": "2.21.0" + } + }, + { + "task": "SpeechCommandsZeroshotv0.02", + "split": "test", + "hf_subset": "default", + "main_metric": "accuracy", + "score": 0.14972999509081983, + "metrics": { + "accuracy": 0.14972999509081983, + "f1": 0.06986246632697812, + "f1_weighted": 0.07083899432793754, + "precision": 0.15715381051614374, + "precision_weighted": 0.1573685095178851, + "recall": 0.14663865718054886, + "recall_weighted": 0.14972999509081983, + "ap": null, + "ap_weighted": null, + "main_score": 0.14972999509081983, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "mteb_version": "2.21.0" + } + } + ], + "full_benchmarks": [ + { + "benchmark": "MTEB(eng, v2)", + "framework_version": "2.21.0", + "coverage": { + "completed_tasks": 41, + "required_tasks": 41, + "task_types": 7 + }, + "model_revision": "70c3412bb0ddfb19386d673d4a102c753a574f1b", + "native_artifact_sha256": "b596f1dbab89369555e0a72a0d6854d780e305686750949b308c8f167993bdeb", + "aggregation": { + "mean_task": 0.5337130125278776, + "mean_task_type": 0.5197553164396411, + "definition": "First apply the official TaskResult aggregation within each task. Mean task gives each of the 41 tasks equal weight; mean task type first averages within each type, then gives all seven types equal weight.", + "score_scale": "0-1; multiply by 100 for display", + "by_task_type": { + "Classification": { + "score": 0.6552554795025091, + "tasks": 8 + }, + "Clustering": { + "score": 0.47072550487857034, + "tasks": 8 + }, + "PairClassification": { + "score": 0.7494593709186261, + "tasks": 3 + }, + "Reranking": { + "score": 0.455685, + "tasks": 2 + }, + "Retrieval": { + "score": 0.290152, + "tasks": 10 + }, + "STS": { + "score": 0.724513458257586, + "tasks": 9 + }, + "Summarization": { + "score": 0.29249640152019635, + "tasks": 1 + } + } + }, + "report": "benchmarks/mteb-eng-v2.json", + "text_component_applicability": { + "applies_to_revision": "self", + "applies_to_native_artifact_sha256": "fb9cd39c9980a085b75456c715c2295b997784fe733314be68c13635f6384edf", + "evaluated_revision": "70c3412bb0ddfb19386d673d4a102c753a574f1b", + "evaluated_native_artifact_sha256": "b596f1dbab89369555e0a72a0d6854d780e305686750949b308c8f167993bdeb", + "basis": "Exact component tensor and inference-file identity; original measured identity is retained.", + "report": "benchmarks/component-equivalence.json", + "new_full_panel_inference": false, + "component": "text" + } + }, + { + "benchmark": "MAEB(beta, audio-only)", + "framework_version": "2.21.0", + "coverage": { + "completed_tasks": 19, + "required_tasks": 19, + "subset_split_slots": 134, + "task_types": 5, + "selection": "All MAEB(beta) tasks whose modalities are exactly [audio].", + "full_MAEB": false, + "excluded": "Tasks that require text/audio or other joint-modality inputs." + }, + "model_revision": "70c3412bb0ddfb19386d673d4a102c753a574f1b", + "native_artifact_sha256": "b596f1dbab89369555e0a72a0d6854d780e305686750949b308c8f167993bdeb", + "aggregation": { + "official": { + "Any2AnyRetrieval": 0.6890050000000001, + "AudioClassification": 0.39566923071716426, + "AudioClustering": 0.054218008138417706, + "AudioPairClassification": 0.571886979184418, + "AudioReranking": 0.63794, + "Mean(Task)": 0.39776955262406916, + "Mean(TaskType)": 0.46974384360800003 + }, + "definition": "Official TaskResult aggregation within each task, followed by official mean_task, mean_task_type and task_types aggregation. The 134 subset/split slots are not averaged as independent tasks.", + "score_scale": "0-1; multiply by 100 for display" + }, + "audio_component_applicability": { + "applies_to_revision": "self", + "applies_to_native_artifact_sha256": "fb9cd39c9980a085b75456c715c2295b997784fe733314be68c13635f6384edf", + "evaluated_revision": "70c3412bb0ddfb19386d673d4a102c753a574f1b", + "evaluated_native_artifact_sha256": "b596f1dbab89369555e0a72a0d6854d780e305686750949b308c8f167993bdeb", + "basis": "Exact component tensor and inference-file identity; original measured identity is retained.", + "report": "benchmarks/component-equivalence.json", + "new_full_panel_inference": false, + "component": "audio" + }, + "report": "benchmarks/maeb-audio-only.json" + } + ] + }, + "full_benchmarks": [ + { + "benchmark": "MTEB(eng, v2)", + "framework_version": "2.21.0", + "coverage": { + "completed_tasks": 41, + "required_tasks": 41, + "task_types": 7 + }, + "model_revision": "29994142677559ca657c27b657eda0dddbf583bd4cf42a3a31095ab3f99d1646", + "native_artifact_sha256": "29994142677559ca657c27b657eda0dddbf583bd4cf42a3a31095ab3f99d1646", + "aggregation": { + "mean_task": 0.6488431514310018, + "mean_task_type": 0.6078184975654923, + "by_task_type": { + "Classification": { + "score": 0.7782970776285105, + "tasks": 8 + }, + "Clustering": { + "score": 0.48962597555764586, + "tasks": 8 + }, + "PairClassification": { + "score": 0.8468485561827029, + "tasks": 3 + }, + "Reranking": { + "score": 0.473475, + "tasks": 2 + }, + "Retrieval": { + "score": 0.520098, + "tasks": 10 + }, + "STS": { + "score": 0.8280405301305157, + "tasks": 9 + }, + "Summarization": { + "score": 0.31834434345907126, + "tasks": 1 + } + }, + "definition": "Official within-task aggregation; equal task mean across 41 tasks, and equal mean of seven within-type means.", + "score_scale": "0-1; display multiplied by 100" + }, + "report": "benchmarks/mteb-eng-v2.json", + "text_component_applicability": { + "component": "text", + "applies_to_revision": "self", + "applies_to_native_artifact_sha256": "96d76f97478e114b816df3ab1b3c6947e4c5ef0babfb40ab865d7e3c838d34c4", + "evaluated_model": "local/Vela-Omni-Nano-GIST-Rotation", + "evaluated_revision": "29994142677559ca657c27b657eda0dddbf583bd4cf42a3a31095ab3f99d1646", + "evaluated_native_artifact_sha256": "29994142677559ca657c27b657eda0dddbf583bd4cf42a3a31095ab3f99d1646", + "revision_kind": "local native artifact fingerprint, not a Hugging Face commit", + "report": "benchmarks/component-equivalence.json", + "new_full_panel_inference": false, + "basis": "199 exact text tensors and byte-identical inference/tokenizer/configuration under the same official evaluator protocol." + }, + "inference_profile_applicability": { + "revision": "self", + "native_artifact_sha256": "50ce808197fb43a1b8913a66d4a47339ec66ecd10d52cbd9f9e752d02f8c43fa", + "parameters": 135383808, + "report": "benchmarks/inference-equivalence.json", + "new_benchmark_run": false + } + }, + { + "benchmark": "MAEB(beta, audio-only)", + "framework_version": "2.21.0", + "coverage": { + "completed_tasks": 19, + "required_tasks": 19, + "subset_split_slots": 134, + "task_types": 5, + "selection": "All MAEB(beta) tasks whose modalities are exactly [audio].", + "full_MAEB": false, + "excluded": "Tasks that require text/audio or other joint-modality inputs." + }, + "model_revision": "96d76f97478e114b816df3ab1b3c6947e4c5ef0babfb40ab865d7e3c838d34c4", + "native_artifact_sha256": "96d76f97478e114b816df3ab1b3c6947e4c5ef0babfb40ab865d7e3c838d34c4", + "aggregation": { + "official": { + "Any2AnyRetrieval": 0.6890050000000001, + "AudioClassification": 0.39553889510295764, + "AudioClustering": 0.054218008138417706, + "AudioPairClassification": 0.5716888010864635, + "AudioReranking": 0.63794, + "Mean(Task)": 0.3976628038845884, + "Mean(TaskType)": 0.4696781408655678 + }, + "definition": "Official TaskResult aggregation within each task, followed by official mean_task, mean_task_type and task_types aggregation. The 134 subset/split slots are not averaged as independent tasks.", + "score_scale": "0-1; multiply by 100 for display" + }, + "report": "benchmarks/maeb-audio-only.json", + "inference_profile_applicability": { + "revision": "self", + "native_artifact_sha256": "50ce808197fb43a1b8913a66d4a47339ec66ecd10d52cbd9f9e752d02f8c43fa", + "parameters": 135383808, + "report": "benchmarks/inference-equivalence.json", + "new_benchmark_run": false + } + } + ], + "net_benefit": { + "scope": "Product-weighted progress comparison, not an official benchmark aggregate or SOTA claim.", + "baseline_model": "llm-semantic-router/Vela-1.0-Omni-Nano", + "baseline_revision": "3a1efc48995e2b71622d65c6cf71cfc00bbf3aa3", + "baseline_native_artifact_sha256": "fb9cd39c9980a085b75456c715c2295b997784fe733314be68c13635f6384edf", + "candidate_native_artifact_sha256": "96d76f97478e114b816df3ab1b3c6947e4c5ef0babfb40ab865d7e3c838d34c4", + "score_scale": "0-100; changes in percentage points", + "components": [ + { + "baseline_100": 59.37287180704907, + "candidate_100": 67.57407300861536, + "component": "common14", + "delta_pp": 8.201201201566288, + "weight": 0.3 + }, + { + "baseline_100": 51.97553164396411, + "candidate_100": 60.781849756549235, + "component": "English41_MeanTaskType", + "delta_pp": 8.806318112585124, + "weight": 0.3 + }, + { + "baseline_100": 54.865785925824184, + "candidate_100": 53.65746394890041, + "component": "image3", + "delta_pp": -1.208321976923776, + "weight": 0.2 + }, + { + "baseline_100": 46.9743843608, + "candidate_100": 46.96781408655678, + "component": "audio19_MeanTaskType", + "delta_pp": -0.00657027424322, + "weight": 0.2 + } + ], + "weighted_net_delta_pp": 4.859277344012025, + "common_definition": "Equal mean of three families: two text accuracies, all six image retrieval recalls, all six audio retrieval recalls.", + "image3_definition": "Equal mean of OxfordPets classification accuracy, TinyImageNet clustering NMI and OxfordPets zero-shot accuracy.", + "text_audio_definition": "Full English41 and full audio19 mean task-type scores. English uses the explicit exact text-component bridge; audio is freshly evaluated.", + "coverage": { + "common_metrics": 14, + "standard_tasks": 11, + "standard_split_rows": 12, + "English_tasks": 41, + "audio_tasks": 19, + "audio_subset_split_slots": 134 + }, + "common_metric_delta_pp": { + "audio.audio_to_text.recall@1": 8.157793948678668, + "audio.audio_to_text.recall@10": 15.51129835312141, + "audio.audio_to_text.recall@5": 14.362313289927231, + "audio.text_to_audio.recall@1": 11.417624521072797, + "audio.text_to_audio.recall@10": 15.478927203065133, + "audio.text_to_audio.recall@5": 14.444444444444445, + "image.image_to_text.recall@1": -4.738760631834751, + "image.image_to_text.recall@10": -1.2150668286755772, + "image.image_to_text.recall@5": -1.701093560145808, + "image.text_to_image.recall@1": 2.7460510328068044, + "image.text_to_image.recall@10": 0.7776427703523694, + "image.text_to_image.recall@5": 1.0206561360874848, + "text.banking77.accuracy": 11.168831168831169, + "text.massive-en.accuracy": 12.617765814266487 + }, + "standard_task_deltas": [ + { + "baseline": 0.7169618925026362, + "candidate": 0.8053174565838819, + "hf_subset": "default", + "main_metric": "cosine_spearman", + "split": "test", + "status": "complete", + "task": "SICK-R", + "delta_pp": 8.835556408124567 + }, + { + "baseline": 0.7792400226361769, + "candidate": 0.870781967302773, + "hf_subset": "default", + "main_metric": "cosine_spearman", + "split": "test", + "status": "complete", + "task": "STSBenchmark", + "delta_pp": 9.154194466659614 + }, + { + "baseline": 0.37514, + "candidate": 0.59246, + "hf_subset": "default", + "main_metric": "ndcg_at_10", + "split": "test", + "status": "complete", + "task": "ArguAna", + "delta_pp": 21.732000000000003 + }, + { + "baseline": 0.44884717457494344, + "candidate": 0.5087260708286914, + "hf_subset": "default", + "main_metric": "v_measure", + "split": "test", + "status": "complete", + "task": "TwentyNewsgroupsClustering.v2", + "delta_pp": 5.987889625374793 + }, + { + "baseline": 0.8758836689038031, + "candidate": 0.9476510067114093, + "hf_subset": "en", + "main_metric": "accuracy", + "split": "validation", + "status": "complete", + "task": "MTOPDomainClassification", + "delta_pp": 7.176733780760625 + }, + { + "baseline": 0.8671910624715002, + "candidate": 0.9491792065663475, + "hf_subset": "en", + "main_metric": "accuracy", + "split": "test", + "status": "complete", + "task": "MTOPDomainClassification", + "delta_pp": 8.198814409484722 + }, + { + "baseline": 0.9126737530662306, + "candidate": 0.9126737530662306, + "hf_subset": "default", + "main_metric": "accuracy", + "split": "test", + "status": "complete", + "task": "OxfordPets", + "delta_pp": 0.0 + }, + { + "baseline": 0.6169193395626786, + "candidate": 0.6169193395626786, + "hf_subset": "default", + "main_metric": "nmi", + "split": "valid", + "status": "complete", + "task": "TinyImageNetClustering", + "delta_pp": 0.0 + }, + { + "baseline": 0.1163804851458163, + "candidate": 0.08013082583810302, + "hf_subset": "default", + "main_metric": "accuracy", + "split": "test", + "status": "complete", + "task": "OxfordPetsZeroShot", + "delta_pp": -3.6249659307713276 + }, + { + "baseline": 0.29763670140167686, + "candidate": 0.29763670140167686, + "hf_subset": "default", + "main_metric": "accuracy", + "split": "train", + "status": "complete", + "task": "CREMA_D", + "delta_pp": 0.0 + }, + { + "baseline": 0.023601715918443712, + "candidate": 0.023601715918443712, + "hf_subset": "default", + "main_metric": "v_measure", + "split": "train", + "status": "complete", + "task": "CREMA_DClustering", + "delta_pp": 0.0 + }, + { + "baseline": 0.14972999509081983, + "candidate": 0.1411389297987236, + "hf_subset": "default", + "main_metric": "accuracy", + "split": "test", + "status": "complete", + "task": "SpeechCommandsZeroshotv0.02", + "delta_pp": -0.8591065292096217 + } + ], + "English_task_deltas": [ + { + "candidate_100": 59.245999999999995, + "current_100": 37.513999999999996, + "delta_pp": 21.732, + "primary_metric": "ndcg_at_10", + "task": "ArguAna", + "task_type": "Retrieval" + }, + { + "candidate_100": 64.88847259326147, + "current_100": 65.25395409281045, + "delta_pp": -0.36548149954897724, + "primary_metric": "v_measure", + "task": "ArXivHierarchicalClusteringP2P", + "task_type": "Clustering" + }, + { + "candidate_100": 57.44315313869382, + "current_100": 55.989532291560415, + "delta_pp": 1.4536208471334078, + "primary_metric": "v_measure", + "task": "ArXivHierarchicalClusteringS2S", + "task_type": "Clustering" + }, + { + "candidate_100": 62.329, + "current_100": 60.514, + "delta_pp": 1.8149999999999977, + "primary_metric": "map_at_1000", + "task": "AskUbuntuDupQuestions", + "task_type": "Reranking" + }, + { + "candidate_100": 86.98752423842483, + "current_100": 62.24539420714732, + "delta_pp": 24.742130031277505, + "primary_metric": "cosine_spearman", + "task": "BIOSSES", + "task_type": "STS" + }, + { + "candidate_100": 82.14285714285714, + "current_100": 69.57467532467533, + "delta_pp": 12.568181818181813, + "primary_metric": "accuracy", + "task": "Banking77Classification", + "task_type": "Classification" + }, + { + "candidate_100": 41.313796273674654, + "current_100": 39.82923119776807, + "delta_pp": 1.4845650759065876, + "primary_metric": "v_measure", + "task": "BiorxivClusteringP2P.v2", + "task_type": "Clustering" + }, + { + "candidate_100": 56.979, + "current_100": 42.976, + "delta_pp": 14.003, + "primary_metric": "ndcg_at_10", + "task": "CQADupstackGamingRetrieval", + "task_type": "Retrieval" + }, + { + "candidate_100": 39.586, + "current_100": 30.362000000000002, + "delta_pp": 9.223999999999997, + "primary_metric": "ndcg_at_10", + "task": "CQADupstackUnixRetrieval", + "task_type": "Retrieval" + }, + { + "candidate_100": 31.846000000000004, + "current_100": 16.073999999999998, + "delta_pp": 15.772000000000006, + "primary_metric": "ndcg_at_10", + "task": "ClimateFEVERHardNegatives", + "task_type": "Retrieval" + }, + { + "candidate_100": 87.576, + "current_100": 24.055, + "delta_pp": 63.520999999999994, + "primary_metric": "ndcg_at_10", + "task": "FEVERHardNegatives", + "task_type": "Retrieval" + }, + { + "candidate_100": 39.143, + "current_100": 22.588, + "delta_pp": 16.555, + "primary_metric": "ndcg_at_10", + "task": "FiQA2018", + "task_type": "Retrieval" + }, + { + "candidate_100": 66.349, + "current_100": 26.987, + "delta_pp": 39.36200000000001, + "primary_metric": "ndcg_at_10", + "task": "HotpotQAHardNegatives", + "task_type": "Retrieval" + }, + { + "candidate_100": 91.94760000000001, + "current_100": 62.1364, + "delta_pp": 29.811200000000007, + "primary_metric": "accuracy", + "task": "ImdbClassification", + "task_type": "Classification" + }, + { + "candidate_100": 94.91792065663475, + "current_100": 86.71910624715002, + "delta_pp": 8.198814409484726, + "primary_metric": "accuracy", + "task": "MTOPDomainClassification", + "task_type": "Classification" + }, + { + "candidate_100": 70.96839273705447, + "current_100": 60.39340954942838, + "delta_pp": 10.574983187626096, + "primary_metric": "accuracy", + "task": "MassiveIntentClassification", + "task_type": "Classification" + }, + { + "candidate_100": 76.11297915265635, + "current_100": 73.37256220578345, + "delta_pp": 2.740416946872898, + "primary_metric": "accuracy", + "task": "MassiveScenarioClassification", + "task_type": "Classification" + }, + { + "candidate_100": 39.83023091223438, + "current_100": 37.76813963073551, + "delta_pp": 2.0620912814988657, + "primary_metric": "v_measure", + "task": "MedrxivClusteringP2P.v2", + "task_type": "Clustering" + }, + { + "candidate_100": 37.875084453707764, + "current_100": 36.13349306174702, + "delta_pp": 1.7415913919607462, + "primary_metric": "v_measure", + "task": "MedrxivClusteringS2S.v2", + "task_type": "Clustering" + }, + { + "candidate_100": 32.366, + "current_100": 30.623, + "delta_pp": 1.7429999999999986, + "primary_metric": "max_over_subqueries_map_at_1000", + "task": "MindSmallReranking", + "task_type": "Reranking" + }, + { + "candidate_100": 21.889, + "current_100": 16.827, + "delta_pp": 5.061999999999998, + "primary_metric": "ndcg_at_10", + "task": "SCIDOCS", + "task_type": "Retrieval" + }, + { + "candidate_100": 80.53174565838819, + "current_100": 71.69618925026361, + "delta_pp": 8.835556408124575, + "primary_metric": "cosine_spearman", + "task": "SICK-R", + "task_type": "STS" + }, + { + "candidate_100": 75.5658801516535, + "current_100": 68.16915974310523, + "delta_pp": 7.396720408548276, + "primary_metric": "cosine_spearman", + "task": "STS12", + "task_type": "STS" + }, + { + "candidate_100": 86.26346974038235, + "current_100": 74.40342742804673, + "delta_pp": 11.860042312335622, + "primary_metric": "cosine_spearman", + "task": "STS13", + "task_type": "STS" + }, + { + "candidate_100": 82.29882793900394, + "current_100": 68.02653891259911, + "delta_pp": 14.272289026404835, + "primary_metric": "cosine_spearman", + "task": "STS14", + "task_type": "STS" + }, + { + "candidate_100": 88.73648284326485, + "current_100": 77.86144933320074, + "delta_pp": 10.875033510064114, + "primary_metric": "cosine_spearman", + "task": "STS15", + "task_type": "STS" + }, + { + "candidate_100": 87.0781967302773, + "current_100": 77.92400226361768, + "delta_pp": 9.154194466659618, + "primary_metric": "cosine_spearman", + "task": "STSBenchmark", + "task_type": "STS" + }, + { + "candidate_100": 95.79964673462752, + "current_100": 89.79096125036985, + "delta_pp": 6.0086854842576685, + "primary_metric": "max_ap", + "task": "SprintDuplicateQuestions", + "task_type": "PairClassification" + }, + { + "candidate_100": 58.424418738430106, + "current_100": 57.34464990059502, + "delta_pp": 1.0797688378350827, + "primary_metric": "v_measure", + "task": "StackExchangeClustering.v2", + "task_type": "Clustering" + }, + { + "candidate_100": 41.053017253245386, + "current_100": 39.37668627014545, + "delta_pp": 1.6763309830999376, + "primary_metric": "v_measure", + "task": "StackExchangeClusteringP2P.v2", + "task_type": "Clustering" + }, + { + "candidate_100": 69.121, + "current_100": 41.292, + "delta_pp": 27.828999999999994, + "primary_metric": "ndcg_at_10", + "task": "TRECCOVID", + "task_type": "Retrieval" + }, + { + "candidate_100": 48.363, + "current_100": 31.477, + "delta_pp": 16.886, + "primary_metric": "ndcg_at_10", + "task": "Touche2020Retrieval.v3", + "task_type": "Retrieval" + }, + { + "candidate_100": 71.904296875, + "current_100": 64.4091796875, + "delta_pp": 7.4951171875, + "primary_metric": "accuracy", + "task": "ToxicConversationsClassification", + "task_type": "Classification" + }, + { + "candidate_100": 62.79286926994907, + "current_100": 50.77815506508207, + "delta_pp": 12.014714204867005, + "primary_metric": "accuracy", + "task": "TweetSentimentExtractionClassification", + "task_type": "Classification" + }, + { + "candidate_100": 50.87260708286914, + "current_100": 44.88471745749435, + "delta_pp": 5.987889625374791, + "primary_metric": "v_measure", + "task": "TwentyNewsgroupsClustering.v2", + "task_type": "Clustering" + }, + { + "candidate_100": 72.95151691526065, + "current_100": 56.34475441213181, + "delta_pp": 16.60676250312884, + "primary_metric": "max_ap", + "task": "TwitterSemEval2015", + "task_type": "PairClassification" + }, + { + "candidate_100": 85.30340320492272, + "current_100": 78.70209561308617, + "delta_pp": 6.6013075918365445, + "primary_metric": "max_ap", + "task": "TwitterURLCorpus", + "task_type": "PairClassification" + }, + { + "candidate_100": 31.834434345907127, + "current_100": 29.249640152019634, + "delta_pp": 2.584794193887493, + "primary_metric": "cosine_spearman", + "task": "SummEvalSummarization.v2", + "task_type": "Summarization" + }, + { + "candidate_100": 71.8507462686567, + "current_100": 56.82089552238806, + "delta_pp": 15.02985074626865, + "primary_metric": "accuracy", + "task": "AmazonCounterfactualClassification", + "task_type": "Classification" + }, + { + "candidate_100": 89.02101343562433, + "current_100": 84.16589279728525, + "delta_pp": 4.8551206383390735, + "primary_metric": "cosine_spearman", + "task": "STS17", + "task_type": "STS" + }, + { + "candidate_100": 68.75333638044488, + "current_100": 67.57005849656171, + "delta_pp": 1.1832778838831643, + "primary_metric": "cosine_spearman", + "task": "STS22.v2", + "task_type": "STS" + } + ], + "audio_task_delta_pp": { + "BeijingOpera": 0.0, + "BirdCLEF": 0.0, + "CREMADPairClassification": 0.12104996267769952, + "CREMA_D": 0.0, + "CREMA_DClustering": 0.0, + "CommonLanguageAgeDetection": 0.0, + "GTZANAudioReranking": 0.0, + "GTZANGenre": 0.0, + "IEMOCAPGender": 0.0, + "JamAltArtistA2ARetrieval": 0.0, + "MInDS14": 0.0, + "MridinghamTonic": -0.14336917562723928, + "NMSQAPairClassification": -0.1520590751812989, + "SIBFLEURS": 0.0, + "VehicleSoundClustering": 0.0, + "VoxCelebSA": 0.0, + "VoxPopuliAccentPairClassification": -0.02844431688273641, + "VoxPopuliGenderClustering": 0.0, + "VoxPopuliLanguageID": 0.0 + }, + "paired_interval_scope": "Paired intervals cover the original common metrics, family and two-primary-per-family utility. They are not confidence intervals for the new all-14 common mean or the four-component product aggregate.", + "inference_profile_applicability": { + "revision": "self", + "native_artifact_sha256": "50ce808197fb43a1b8913a66d4a47339ec66ecd10d52cbd9f9e752d02f8c43fa", + "parameters": 135383808, + "report": "benchmarks/inference-equivalence.json", + "new_benchmark_run": false + } + }, + "paired_uncertainty_scope": "Paired intervals cover the original common metrics, family and two-primary-per-family utility. They are not confidence intervals for the new all-14 common mean or the four-component product aggregate.", + "inference_profile": { + "revision": "self", + "native_artifact_sha256": "50ce808197fb43a1b8913a66d4a47339ec66ecd10d52cbd9f9e752d02f8c43fa", + "parameters": 135383808, + "report": "benchmarks/inference-equivalence.json", + "new_benchmark_run": false + }, + "scorecard_comparison": { + "primary_comparator": "llm-semantic-router/multi-modal-embed-small", + "revision": "fdf8e01b7b0f3a69ac1ac8e2a64dcb1ede177ba4", + "common_protocol_measured": true, + "original_reference_evidence": "benchmarks/original-baseline.json", + "previous_Vela_not_the_primary_comparator": true, + "historical_release_progress": "benchmarks/release-history.md" + } +}