| { |
| "schema_version": 2, |
| "model": "vllm-sr/Vela-1.0-Omni-Nano", |
| "revision_semantics": "self denotes this published inference package; evaluated_* identities retain original measurements. The exact public-inference equivalence report binds score applicability.", |
| "score_scale": "0-1; multiply by 100 for percentages", |
| "matrix": { |
| "small": { |
| "model": "vllm-sr/multi-modal-embed-small", |
| "revision": "fdf8e01b7b0f3a69ac1ac8e2a64dcb1ede177ba4", |
| "artifact_fingerprint_sha256": "684fe0c48a4ddd3944167f6e5dc75e97ada24e5b120bd9a2921f00dc03112eeb", |
| "metrics": { |
| "text.banking77.accuracy": { |
| "numerator": 2169, |
| "denominator": 3080, |
| "value": 0.7042207792207792 |
| }, |
| "text.massive-en.accuracy": { |
| "numerator": 1960, |
| "denominator": 2972, |
| "value": 0.6594885598923284 |
| }, |
| "image.image_to_text.recall@1": { |
| "numerator": 336, |
| "denominator": 823, |
| "value": 0.4082624544349939 |
| }, |
| "image.image_to_text.recall@5": { |
| "numerator": 553, |
| "denominator": 823, |
| "value": 0.6719319562575942 |
| }, |
| "image.image_to_text.recall@10": { |
| "numerator": 646, |
| "denominator": 823, |
| "value": 0.7849331713244229 |
| }, |
| "image.text_to_image.recall@1": { |
| "numerator": 1242, |
| "denominator": 4115, |
| "value": 0.3018226002430134 |
| }, |
| "image.text_to_image.recall@5": { |
| "numerator": 2445, |
| "denominator": 4115, |
| "value": 0.5941676792223572 |
| }, |
| "image.text_to_image.recall@10": { |
| "numerator": 3057, |
| "denominator": 4115, |
| "value": 0.7428918590522479 |
| }, |
| "audio.audio_to_text.recall@1": { |
| "numerator": 110, |
| "denominator": 2611, |
| "value": 0.04212945231711988 |
| }, |
| "audio.audio_to_text.recall@5": { |
| "numerator": 313, |
| "denominator": 2611, |
| "value": 0.11987744159325929 |
| }, |
| "audio.audio_to_text.recall@10": { |
| "numerator": 497, |
| "denominator": 2611, |
| "value": 0.1903485254691689 |
| }, |
| "audio.text_to_audio.recall@1": { |
| "numerator": 250, |
| "denominator": 2610, |
| "value": 0.09578544061302682 |
| }, |
| "audio.text_to_audio.recall@5": { |
| "numerator": 588, |
| "denominator": 2610, |
| "value": 0.22528735632183908 |
| }, |
| "audio.text_to_audio.recall@10": { |
| "numerator": 801, |
| "denominator": 2610, |
| "value": 0.30689655172413793 |
| } |
| } |
| }, |
| "candidate": { |
| "model": "vllm-sr/Vela-1.0-Omni-Nano", |
| "revision": "self", |
| "artifact_fingerprint_sha256": "50ce808197fb43a1b8913a66d4a47339ec66ecd10d52cbd9f9e752d02f8c43fa", |
| "metrics": { |
| "audio.audio_to_text.recall@1": { |
| "denominator": 2611, |
| "numerator": 448, |
| "value": 0.17158176943699732 |
| }, |
| "audio.audio_to_text.recall@10": { |
| "denominator": 2611, |
| "numerator": 1227, |
| "value": 0.469934890846419 |
| }, |
| "audio.audio_to_text.recall@5": { |
| "denominator": 2611, |
| "numerator": 957, |
| "value": 0.36652623515894295 |
| }, |
| "audio.text_to_audio.recall@1": { |
| "denominator": 2610, |
| "numerator": 664, |
| "value": 0.25440613026819925 |
| }, |
| "audio.text_to_audio.recall@10": { |
| "denominator": 2610, |
| "numerator": 1459, |
| "value": 0.5590038314176246 |
| }, |
| "audio.text_to_audio.recall@5": { |
| "denominator": 2610, |
| "numerator": 1204, |
| "value": 0.46130268199233715 |
| }, |
| "image.image_to_text.recall@1": { |
| "denominator": 823, |
| "numerator": 501, |
| "value": 0.6087484811664642 |
| }, |
| "image.image_to_text.recall@10": { |
| "denominator": 823, |
| "numerator": 781, |
| "value": 0.9489671931956257 |
| }, |
| "image.image_to_text.recall@5": { |
| "denominator": 823, |
| "numerator": 717, |
| "value": 0.8712029161603888 |
| }, |
| "image.text_to_image.recall@1": { |
| "denominator": 4115, |
| "numerator": 2297, |
| "value": 0.5582017010935602 |
| }, |
| "image.text_to_image.recall@10": { |
| "denominator": 4115, |
| "numerator": 3879, |
| "value": 0.9426488456865127 |
| }, |
| "image.text_to_image.recall@5": { |
| "denominator": 4115, |
| "numerator": 3507, |
| "value": 0.8522478736330498 |
| }, |
| "text.banking77.accuracy": { |
| "denominator": 3080, |
| "numerator": 2710, |
| "value": 0.8798701298701299 |
| }, |
| "text.massive-en.accuracy": { |
| "denominator": 2972, |
| "numerator": 2436, |
| "value": 0.819650067294751 |
| } |
| }, |
| "parameters": 135383808, |
| "evaluated_native_artifact_sha256": "96d76f97478e114b816df3ab1b3c6947e4c5ef0babfb40ab865d7e3c838d34c4", |
| "evaluated_revision": "de02c367bc9aa7a6f7ad3e59fb38962502e021c1", |
| "evaluated_parameters": 144554112, |
| "inference_equivalence_report": "benchmarks/inference-equivalence.json" |
| } |
| }, |
| "datasets": { |
| "banking77": { |
| "source": "https://github.com/PolyAI-LDN/task-specific-datasets", |
| "revision": "57ec275d8078af65b7731c2a98be812d844a6d6b", |
| "license": "CC-BY-4.0", |
| "split": "test", |
| "queries": 3080, |
| "classes": 77 |
| }, |
| "massive-en": { |
| "source": "https://huggingface.co/datasets/AmazonScience/massive", |
| "revision": "ff6bd8e4b27c3543e4f8fe2108f32bb95a6f8740", |
| "license": "CC-BY-4.0", |
| "configuration": "en-US", |
| "split": "test", |
| "queries": 2972, |
| "prototype_classes": 60 |
| }, |
| "image": { |
| "source": "https://cocodataset.org/", |
| "caption_split_source": "https://cs.stanford.edu/people/karpathy/deepimagesent/", |
| "split": "Karpathy test, CC-BY 2.0 image subset", |
| "images": 823, |
| "captions": 4115, |
| "image_license": "CC-BY-2.0", |
| "caption_license": "CC-BY-4.0" |
| }, |
| "audio": { |
| "source": "https://www.openslr.org/12", |
| "split": "test-clean", |
| "archive_sha256": "39fde525e59672dc6d1551919b1478f724438a95aa55f874b576be21967e6c23", |
| "license": "CC-BY-4.0", |
| "clips": 2611, |
| "unique_transcripts": 2610, |
| "maximum_seconds": 30 |
| } |
| }, |
| "protocol": { |
| "embedding_dtype": "float32", |
| "attention": "SDPA", |
| "autocast": false, |
| "tf32": false, |
| "transformers_version": "4.57.6", |
| "torch_version_family": "2.12", |
| "text_max_tokens": 128, |
| "truncation": "right, native tokenizer including special tokens", |
| "batch_sizes": { |
| "text": 64, |
| "image": 8, |
| "audio": 8 |
| }, |
| "pooling": "native full-dimension model pooling", |
| "similarity": "FP32 cosine of L2-normalized embeddings", |
| "ties": "lexicographic candidate ID", |
| "retrieval": "complete candidate pools; all annotated positives", |
| "classification": "Nearest normalized mean class prototype from labeled training examples; the original small comparator has task-adapted history, while current GIST is frozen.", |
| "aggregation": "Each family is the mean of its two primary metrics; utility is the mean of all six primaries. Retrieval primaries are R@1; R@5 and R@10 are reported separately.", |
| "evaluation_scope": "fixed test-split evaluation pools reused across releases", |
| "text_training_scope": "Current GIST text tower is frozen; supervised training prototypes are recomputed under the fixed common protocol. The earlier small comparator has task-adapted history. This is not the official MTEB classification protocol." |
| }, |
| "paired_uncertainty": { |
| "original": { |
| "intervals": { |
| "audio.audio_to_text.recall@1": { |
| "lower": 0.11352472177122043, |
| "point_delta": 0.12945231711987745, |
| "upper": 0.14665221762155492 |
| }, |
| "audio.audio_to_text.recall@10": { |
| "lower": 0.25338950715018704, |
| "point_delta": 0.2795863653772501, |
| "upper": 0.30833452722063037 |
| }, |
| "audio.audio_to_text.recall@5": { |
| "lower": 0.22198472459131105, |
| "point_delta": 0.24664879356568364, |
| "upper": 0.2736971153456769 |
| }, |
| "audio.text_to_audio.recall@1": { |
| "lower": 0.14202610349345837, |
| "point_delta": 0.15862068965517243, |
| "upper": 0.17613425616168824 |
| }, |
| "audio.text_to_audio.recall@10": { |
| "lower": 0.23515433278434666, |
| "point_delta": 0.25210727969348656, |
| "upper": 0.2707672328928673 |
| }, |
| "audio.text_to_audio.recall@5": { |
| "lower": 0.21796104940042046, |
| "point_delta": 0.23601532567049807, |
| "upper": 0.2547919081648684 |
| }, |
| "family.audio": { |
| "lower": 0.1322499685832721, |
| "point_delta": 0.14403650338752494, |
| "upper": 0.15644009735751394 |
| }, |
| "family.image": { |
| "lower": 0.20084750911300123, |
| "point_delta": 0.2284325637910085, |
| "upper": 0.25637910085054677 |
| }, |
| "family.text": { |
| "lower": 0.15747009669727144, |
| "point_delta": 0.16790542902588662, |
| "upper": 0.17869785413173905 |
| }, |
| "image.image_to_text.recall@1": { |
| "lower": 0.1591737545565006, |
| "point_delta": 0.20048602673147023, |
| "upper": 0.24301336573511542 |
| }, |
| "image.image_to_text.recall@10": { |
| "lower": 0.1336573511543135, |
| "point_delta": 0.16403402187120292, |
| "upper": 0.19441069258809235 |
| }, |
| "image.image_to_text.recall@5": { |
| "lower": 0.16160388821385177, |
| "point_delta": 0.19927095990279464, |
| "upper": 0.23572296476306198 |
| }, |
| "image.text_to_image.recall@1": { |
| "lower": 0.23013365735115432, |
| "point_delta": 0.25637910085054677, |
| "upper": 0.2831166464155528 |
| }, |
| "image.text_to_image.recall@10": { |
| "lower": 0.17618469015795868, |
| "point_delta": 0.1997569866342649, |
| "upper": 0.22235722964763063 |
| }, |
| "image.text_to_image.recall@5": { |
| "lower": 0.23207776427703525, |
| "point_delta": 0.2580801944106926, |
| "upper": 0.28360267314702303 |
| }, |
| "text.banking77.accuracy": { |
| "lower": 0.16066212268743915, |
| "point_delta": 0.17564935064935064, |
| "upper": 0.19058596303826084 |
| }, |
| "text.massive-en.accuracy": { |
| "lower": 0.14554621848739496, |
| "point_delta": 0.1601615074024226, |
| "upper": 0.17569841804106362 |
| }, |
| "utility": { |
| "lower": 0.16995251624973068, |
| "point_delta": 0.18012483206814003, |
| "upper": 0.1910563694034248 |
| } |
| }, |
| "method": "paired stratified group percentile; PCG64; fixed candidate pools", |
| "replicates": 2000, |
| "seed": 20260918, |
| "comparison": { |
| "candidate": "vllm-sr/Vela-1.0-Omni-Nano", |
| "candidate_artifact_fingerprint_sha256": "96d76f97478e114b816df3ab1b3c6947e4c5ef0babfb40ab865d7e3c838d34c4", |
| "baseline": "vllm-sr/multi-modal-embed-small", |
| "baseline_revision": "fdf8e01b7b0f3a69ac1ac8e2a64dcb1ede177ba4", |
| "baseline_artifact_fingerprint_sha256": "684fe0c48a4ddd3944167f6e5dc75e97ada24e5b120bd9a2921f00dc03112eeb" |
| } |
| }, |
| "previous": { |
| "intervals": { |
| "audio.audio_to_text.recall@1": { |
| "lower": 0.06612466882994839, |
| "point_delta": 0.08157793948678667, |
| "upper": 0.0967748551165683 |
| }, |
| "audio.audio_to_text.recall@10": { |
| "lower": 0.13256556125090607, |
| "point_delta": 0.15511298353121408, |
| "upper": 0.18059527375151307 |
| }, |
| "audio.audio_to_text.recall@5": { |
| "lower": 0.12086266595311207, |
| "point_delta": 0.1436231328992723, |
| "upper": 0.16909569992403942 |
| }, |
| "audio.text_to_audio.recall@1": { |
| "lower": 0.09860745201354912, |
| "point_delta": 0.11417624521072797, |
| "upper": 0.13014135201210755 |
| }, |
| "audio.text_to_audio.recall@10": { |
| "lower": 0.13604064880650404, |
| "point_delta": 0.15478927203065135, |
| "upper": 0.17574363317277317 |
| }, |
| "audio.text_to_audio.recall@5": { |
| "lower": 0.1272585913379362, |
| "point_delta": 0.14444444444444443, |
| "upper": 0.16333503134796237 |
| }, |
| "family.audio": { |
| "lower": 0.08555566687871614, |
| "point_delta": 0.09787709234875733, |
| "upper": 0.11042213832911257 |
| }, |
| "family.image": { |
| "lower": -0.028797083839611177, |
| "point_delta": -0.00996354799513973, |
| "upper": 0.008630012150668272 |
| }, |
| "family.text": { |
| "lower": 0.10934917350884273, |
| "point_delta": 0.11893298491548829, |
| "upper": 0.12872853420036084 |
| }, |
| "image.image_to_text.recall@1": { |
| "lower": -0.08140947752126367, |
| "point_delta": -0.04738760631834751, |
| "upper": -0.012150668286755772 |
| }, |
| "image.image_to_text.recall@10": { |
| "lower": -0.027946537059538274, |
| "point_delta": -0.012150668286755772, |
| "upper": 0.002430133657351154 |
| }, |
| "image.image_to_text.recall@5": { |
| "lower": -0.040097205346294046, |
| "point_delta": -0.01701093560145808, |
| "upper": 0.004860267314702308 |
| }, |
| "image.text_to_image.recall@1": { |
| "lower": 0.01117861482381531, |
| "point_delta": 0.027460510328068045, |
| "upper": 0.042770352369380316 |
| }, |
| "image.text_to_image.recall@10": { |
| "lower": 0.001215066828675577, |
| "point_delta": 0.007776427703523694, |
| "upper": 0.014580801944106925 |
| }, |
| "image.text_to_image.recall@5": { |
| "lower": -0.001701093560145808, |
| "point_delta": 0.010206561360874849, |
| "upper": 0.02187120291616039 |
| }, |
| "text.banking77.accuracy": { |
| "lower": 0.0990581357583631, |
| "point_delta": 0.11168831168831168, |
| "upper": 0.12439713161970098 |
| }, |
| "text.massive-en.accuracy": { |
| "lower": 0.1117468865701784, |
| "point_delta": 0.12617765814266488, |
| "upper": 0.14069455352356733 |
| }, |
| "utility": { |
| "lower": 0.060760442670994376, |
| "point_delta": 0.06894884308970196, |
| "upper": 0.07704865461098383 |
| } |
| }, |
| "method": "paired stratified group percentile; PCG64; fixed candidate pools", |
| "replicates": 2000, |
| "seed": 20260918, |
| "comparison": { |
| "candidate": "vllm-sr/Vela-1.0-Omni-Nano", |
| "candidate_artifact_fingerprint_sha256": "96d76f97478e114b816df3ab1b3c6947e4c5ef0babfb40ab865d7e3c838d34c4", |
| "baseline": "vllm-sr/Vela-1.0-Omni-Nano", |
| "baseline_revision": "3a1efc48995e2b71622d65c6cf71cfc00bbf3aa3", |
| "baseline_artifact_fingerprint_sha256": "fb9cd39c9980a085b75456c715c2295b997784fe733314be68c13635f6384edf" |
| } |
| } |
| }, |
| "training_scope": { |
| "text": "The complete text tower is the frozen official avsolatorio/GIST-small-Embedding-v0 checkpoint at 75e62fd210b9fde790430e0b2f040b0b00a021b1. Its model card discloses MTEB classification training data; no claim of independent pretraining data is made.", |
| "image": "Frozen current vision encoder and initial image head; an orthogonal shared-space rotation is trained on 16,509 COCO training images, 81,998 unique caption texts and 82,522 complete positive associations, then folded into the image affine head.", |
| "audio": "Frozen current speech encoder and initial audio head; a separate orthogonal shared-space rotation is trained on 28,535 LibriSpeech train-clean-100 clips and all positive transcript associations, then folded into the audio affine head.", |
| "procedure": "One fixed 300-step run; only the final artifact is reported. Encoders remain frozen. The two rotations preserve within-modality inner products, up to floating-point rounding.", |
| "evaluation_scope": "Common text classification uses labeled training prototypes with the earlier task-adapted comparator. It is separate from the official frozen-encoder MTEB classification protocols." |
| }, |
| "descriptive_metrics": { |
| "definition": "Macro-F1 is the unweighted mean of per-class F1 across all fixed prototype classes; zero_division=0.", |
| "scope": "Same models, examples, class prototypes and predictions as the accuracy matrix.", |
| "metrics": { |
| "text.banking77.macro_f1": { |
| "small": { |
| "value": 0.7020781051899517, |
| "classes": 77, |
| "queries": 3080 |
| }, |
| "candidate": { |
| "value": 0.8791681858119436, |
| "classes": 77, |
| "queries": 3080 |
| } |
| }, |
| "text.massive-en.macro_f1": { |
| "small": { |
| "value": 0.6171775818092488, |
| "classes": 60, |
| "queries": 2972 |
| }, |
| "candidate": { |
| "value": 0.7923223418549947, |
| "classes": 60, |
| "queries": 2972 |
| } |
| } |
| } |
| }, |
| "standard_benchmarks": { |
| "scope": "Eleven selected standard tasks; not a complete MTEB, MMTEB, MIEB or MAEB benchmark or leaderboard submission.", |
| "library": "mteb", |
| "library_version": "2.21.0", |
| "model_revisions": { |
| "nano": { |
| "repo_id": "vllm-sr/Vela-1.0-Omni-Nano", |
| "revision": "self", |
| "native_artifact_sha256": "50ce808197fb43a1b8913a66d4a47339ec66ecd10d52cbd9f9e752d02f8c43fa", |
| "parameters": 135383808, |
| "embedding_dimensions": 384, |
| "max_text_tokens": 512, |
| "evaluated_native_artifact_sha256": "96d76f97478e114b816df3ab1b3c6947e4c5ef0babfb40ab865d7e3c838d34c4", |
| "evaluated_revision": "de02c367bc9aa7a6f7ad3e59fb38962502e021c1", |
| "evaluated_parameters": 144554112, |
| "inference_equivalence_report": "benchmarks/inference-equivalence.json", |
| "revision_kind": "HF_snapshot" |
| }, |
| "small": { |
| "repo_id": "vllm-sr/multi-modal-embed-small", |
| "revision": "fdf8e01b7b0f3a69ac1ac8e2a64dcb1ede177ba4", |
| "native_artifact_sha256": "684fe0c48a4ddd3944167f6e5dc75e97ada24e5b120bd9a2921f00dc03112eeb", |
| "status": "evaluated_all_11_selected_tasks", |
| "max_text_tokens": 128, |
| "inference_parameters": 133759488 |
| } |
| }, |
| "protocol": { |
| "precision": "float32", |
| "batch_size": 8, |
| "seed": 20260918, |
| "similarity": "Cosine for primary STS/retrieval scores and normalized media-label matching; classification probes and clustering use each official task's algorithms.", |
| "encoder_parameters_frozen": true, |
| "classification": "Official task-specific frozen-embedding probes or fixed-label zero-shot classification.", |
| "sampling": "Official task subsampling and repeated-probe protocols are preserved.", |
| "text": "Each encoder uses its native preprocessing and context limit without added task instructions. Original small: 128 tokens; current Nano: 512 tokens.", |
| "audio": "Mono 16 kHz, first 30 seconds, official MTEB audio preprocessing.", |
| "score_scale": "Metrics retain their native MTEB scale; the model card multiplies primary scores by 100.", |
| "undefined_metric": null, |
| "missing_model_score": "Not measured in this standard evaluation.", |
| "overlap": "No claim of independence from component pretraining data is made." |
| }, |
| "rows": [ |
| { |
| "task": "SICK-R", |
| "type": "STS", |
| "modalities": [ |
| "text" |
| ], |
| "split": "test", |
| "hf_subset": "default", |
| "main_metric": "cosine_spearman", |
| "dataset": { |
| "path": "mteb/sickr-sts", |
| "revision": "20a6d6f312dd54037fe07a32d58e5e168867909d" |
| }, |
| "counts": { |
| "evaluation_pairs": 9927 |
| }, |
| "scores": { |
| "nano": 0.8053174565838819, |
| "small": 0.7139020320965759 |
| }, |
| "metrics": { |
| "nano": { |
| "pearson": 0.8603345909155973, |
| "spearman": 0.8053174398238511, |
| "cosine_pearson": 0.8603345867277692, |
| "cosine_spearman": 0.8053174565838819, |
| "manhattan_pearson": 0.8332190477069294, |
| "manhattan_spearman": 0.8044409124391882, |
| "euclidean_pearson": 0.834049309726515, |
| "euclidean_spearman": 0.8053172111347874, |
| "main_score": 0.8053174565838819, |
| "hf_subset": "default", |
| "languages": [ |
| "eng-Latn" |
| ], |
| "mteb_version": "2.21.0" |
| }, |
| "small": { |
| "pearson": 0.8179714611300387, |
| "spearman": 0.713902378578463, |
| "cosine_pearson": 0.81797146109438, |
| "cosine_spearman": 0.7139020320965759, |
| "manhattan_pearson": 0.7919193141236369, |
| "manhattan_spearman": 0.7128438640347968, |
| "euclidean_pearson": 0.792891259170334, |
| "euclidean_spearman": 0.7139020311053554, |
| "main_score": 0.7139020320965759, |
| "hf_subset": "default", |
| "languages": [ |
| "eng-Latn" |
| ], |
| "mteb_version": "2.21.0" |
| } |
| }, |
| "measurement_by_model": { |
| "small": { |
| "evaluated_native_fingerprint": "684fe0c48a4ddd3944167f6e5dc75e97ada24e5b120bd9a2921f00dc03112eeb", |
| "original_checkpoint_revision": "fdf8e01b7b0f3a69ac1ac8e2a64dcb1ede177ba4", |
| "raw_result_sha256": "e20f02a347841b3d5f61209f0b209c6314d006f8cc94f6e488e06239c862badf", |
| "kind": "direct_frozen_encoder_evaluation" |
| } |
| } |
| }, |
| { |
| "task": "STSBenchmark", |
| "type": "STS", |
| "modalities": [ |
| "text" |
| ], |
| "split": "test", |
| "hf_subset": "default", |
| "main_metric": "cosine_spearman", |
| "dataset": { |
| "path": "mteb/stsbenchmark-sts", |
| "revision": "b0fddb56ed78048fa8b90373c8a3cfc37b684831" |
| }, |
| "counts": { |
| "evaluation_pairs": 1379 |
| }, |
| "scores": { |
| "nano": 0.870781967302773, |
| "small": 0.7729871863268857 |
| }, |
| "metrics": { |
| "nano": { |
| "pearson": 0.8541596951761798, |
| "spearman": 0.8707819111802441, |
| "cosine_pearson": 0.854159691434247, |
| "cosine_spearman": 0.870781967302773, |
| "manhattan_pearson": 0.867700912246326, |
| "manhattan_spearman": 0.8690795304788012, |
| "euclidean_pearson": 0.8699231026532712, |
| "euclidean_spearman": 0.8707819663065923, |
| "main_score": 0.870781967302773, |
| "hf_subset": "default", |
| "languages": [ |
| "eng-Latn" |
| ], |
| "mteb_version": "2.21.0" |
| }, |
| "small": { |
| "pearson": 0.765316203785143, |
| "spearman": 0.7729871863268857, |
| "cosine_pearson": 0.7653161983330018, |
| "cosine_spearman": 0.7729871863268857, |
| "manhattan_pearson": 0.7881395913225815, |
| "manhattan_spearman": 0.7716275619454571, |
| "euclidean_pearson": 0.789916662830987, |
| "euclidean_spearman": 0.7729857841479683, |
| "main_score": 0.7729871863268857, |
| "hf_subset": "default", |
| "languages": [ |
| "eng-Latn" |
| ], |
| "mteb_version": "2.21.0" |
| } |
| }, |
| "measurement_by_model": { |
| "small": { |
| "evaluated_native_fingerprint": "684fe0c48a4ddd3944167f6e5dc75e97ada24e5b120bd9a2921f00dc03112eeb", |
| "original_checkpoint_revision": "fdf8e01b7b0f3a69ac1ac8e2a64dcb1ede177ba4", |
| "raw_result_sha256": "ea0772149930250452298e3525b3405ce4fd5caed4aaf676f3745d4555692a07", |
| "kind": "direct_frozen_encoder_evaluation" |
| } |
| } |
| }, |
| { |
| "task": "ArguAna", |
| "type": "Retrieval", |
| "modalities": [ |
| "text" |
| ], |
| "split": "test", |
| "hf_subset": "default", |
| "main_metric": "ndcg_at_10", |
| "dataset": { |
| "path": "mteb/arguana", |
| "revision": "c22ab2a51041ffd869aaddef7af8d8215647e41a" |
| }, |
| "counts": { |
| "queries": 1406, |
| "documents": 8674 |
| }, |
| "scores": { |
| "nano": 0.59246, |
| "small": 0.32889 |
| }, |
| "metrics": { |
| "nano": { |
| "ndcg_at_1": 0.35349, |
| "ndcg_at_3": 0.49665, |
| "ndcg_at_5": 0.54581, |
| "ndcg_at_10": 0.59246, |
| "ndcg_at_20": 0.6132, |
| "ndcg_at_100": 0.62189, |
| "ndcg_at_1000": 0.62242, |
| "map_at_1": 0.35349, |
| "map_at_3": 0.46005, |
| "map_at_5": 0.48729, |
| "map_at_10": 0.50679, |
| "map_at_20": 0.51261, |
| "map_at_100": 0.51395, |
| "map_at_1000": 0.51397, |
| "recall_at_1": 0.35349, |
| "recall_at_3": 0.60313, |
| "recall_at_5": 0.72262, |
| "recall_at_10": 0.86558, |
| "recall_at_20": 0.94666, |
| "recall_at_100": 0.99147, |
| "recall_at_1000": 0.99573, |
| "accuracy": 0.35349, |
| "precision_at_1": 0.35349, |
| "precision_at_3": 0.20104, |
| "precision_at_5": 0.14452, |
| "precision_at_10": 0.08656, |
| "precision_at_20": 0.04733, |
| "precision_at_100": 0.00991, |
| "precision_at_1000": 0.001, |
| "mrr_at_1": 0.3534850640113798, |
| "mrr_at_3": 0.4600521574205785, |
| "mrr_at_5": 0.48729255571360836, |
| "mrr_at_10": 0.5067903429745535, |
| "mrr_at_20": 0.5126066994732466, |
| "mrr_at_100": 0.5139513666778618, |
| "mrr_at_1000": 0.5139717342260931, |
| "nauc_ndcg_at_1_max": -0.11747299439249903, |
| "nauc_ndcg_at_1_std": -0.11436695908554317, |
| "nauc_ndcg_at_1_diff1": 0.12406213904460131, |
| "nauc_ndcg_at_3_max": -0.0810374905617702, |
| "nauc_ndcg_at_3_std": -0.10902178800246173, |
| "nauc_ndcg_at_3_diff1": 0.09457199601062195, |
| "nauc_ndcg_at_5_max": -0.08485395980147033, |
| "nauc_ndcg_at_5_std": -0.10485663186916257, |
| "nauc_ndcg_at_5_diff1": 0.08582902365914505, |
| "nauc_ndcg_at_10_max": -0.08309974111344137, |
| "nauc_ndcg_at_10_std": -0.11506227102720018, |
| "nauc_ndcg_at_10_diff1": 0.08990562536626041, |
| "nauc_ndcg_at_20_max": -0.0791935521931642, |
| "nauc_ndcg_at_20_std": -0.10319594721919796, |
| "nauc_ndcg_at_20_diff1": 0.09971433495514617, |
| "nauc_ndcg_at_100_max": -0.08635224865444148, |
| "nauc_ndcg_at_100_std": -0.10462787774486179, |
| "nauc_ndcg_at_100_diff1": 0.09791984963152253, |
| "nauc_ndcg_at_1000_max": -0.0879373318368675, |
| "nauc_ndcg_at_1000_std": -0.10553141137460995, |
| "nauc_ndcg_at_1000_diff1": 0.09704987719570578, |
| "nauc_map_at_1_max": -0.11747299439249903, |
| "nauc_map_at_1_std": -0.11436695908554317, |
| "nauc_map_at_1_diff1": 0.12406213904460131, |
| "nauc_map_at_3_max": -0.09064151313996319, |
| "nauc_map_at_3_std": -0.1105170515941861, |
| "nauc_map_at_3_diff1": 0.09940664377727629, |
| "nauc_map_at_5_max": -0.09342459992783216, |
| "nauc_map_at_5_std": -0.10864465771524179, |
| "nauc_map_at_5_diff1": 0.09517709876339578, |
| "nauc_map_at_10_max": -0.09304942751697344, |
| "nauc_map_at_10_std": -0.11264010939720397, |
| "nauc_map_at_10_diff1": 0.09707068152210184, |
| "nauc_map_at_20_max": -0.09221792922323616, |
| "nauc_map_at_20_std": -0.10961211427408772, |
| "nauc_map_at_20_diff1": 0.09943255521491648, |
| "nauc_map_at_100_max": -0.09316799067959038, |
| "nauc_map_at_100_std": -0.10968589627543163, |
| "nauc_map_at_100_diff1": 0.09910011165040408, |
| "nauc_map_at_1000_max": -0.09321385399598632, |
| "nauc_map_at_1000_std": -0.10970513219025511, |
| "nauc_map_at_1000_diff1": 0.09907801272427635, |
| "nauc_recall_at_1_max": -0.11747299439249903, |
| "nauc_recall_at_1_std": -0.11436695908554317, |
| "nauc_recall_at_1_diff1": 0.12406213904460131, |
| "nauc_recall_at_3_max": -0.05070404620274185, |
| "nauc_recall_at_3_std": -0.10426214370217692, |
| "nauc_recall_at_3_diff1": 0.0805612409902579, |
| "nauc_recall_at_5_max": -0.051784236042595515, |
| "nauc_recall_at_5_std": -0.0889570674134141, |
| "nauc_recall_at_5_diff1": 0.049390111709494505, |
| "nauc_recall_at_10_max": -0.013727385294040483, |
| "nauc_recall_at_10_std": -0.13798296178002195, |
| "nauc_recall_at_10_diff1": 0.044023090224588696, |
| "nauc_recall_at_20_max": 0.15251226818625876, |
| "nauc_recall_at_20_std": 0.03363980242022382, |
| "nauc_recall_at_20_diff1": 0.15796490326794294, |
| "nauc_recall_at_100_max": 0.3311354440025815, |
| "nauc_recall_at_100_std": 0.4782526238266158, |
| "nauc_recall_at_100_diff1": 0.25005005298352057, |
| "nauc_recall_at_1000_max": 0.02172236447995406, |
| "nauc_recall_at_1000_std": 0.609819560470231, |
| "nauc_recall_at_1000_diff1": -0.005495147382062338, |
| "nauc_precision_at_1_max": -0.11747299439249903, |
| "nauc_precision_at_1_std": -0.11436695908554317, |
| "nauc_precision_at_1_diff1": 0.12406213904460131, |
| "nauc_precision_at_3_max": -0.05070404620274239, |
| "nauc_precision_at_3_std": -0.10426214370217736, |
| "nauc_precision_at_3_diff1": 0.08056124099025751, |
| "nauc_precision_at_5_max": -0.05178423604259566, |
| "nauc_precision_at_5_std": -0.08895706741341412, |
| "nauc_precision_at_5_diff1": 0.04939011170949452, |
| "nauc_precision_at_10_max": -0.013727385294041244, |
| "nauc_precision_at_10_std": -0.13798296178002226, |
| "nauc_precision_at_10_diff1": 0.044023090224589216, |
| "nauc_precision_at_20_max": 0.15251226818625896, |
| "nauc_precision_at_20_std": 0.03363980242022203, |
| "nauc_precision_at_20_diff1": 0.15796490326794188, |
| "nauc_precision_at_100_max": 0.3311354440025917, |
| "nauc_precision_at_100_std": 0.4782526238266313, |
| "nauc_precision_at_100_diff1": 0.25005005298354305, |
| "nauc_precision_at_1000_max": 0.0217223644799946, |
| "nauc_precision_at_1000_std": 0.6098195604702127, |
| "nauc_precision_at_1000_diff1": -0.0054951473820258456, |
| "nauc_mrr_at_1_max": -0.11747299439249903, |
| "nauc_mrr_at_1_std": -0.11436695908554317, |
| "nauc_mrr_at_1_diff1": 0.12406213904460131, |
| "nauc_mrr_at_3_max": -0.09064151313996319, |
| "nauc_mrr_at_3_std": -0.1105170515941861, |
| "nauc_mrr_at_3_diff1": 0.09940664377727629, |
| "nauc_mrr_at_5_max": -0.09342459992783216, |
| "nauc_mrr_at_5_std": -0.10864465771524179, |
| "nauc_mrr_at_5_diff1": 0.09517709876339578, |
| "nauc_mrr_at_10_max": -0.09304942751697344, |
| "nauc_mrr_at_10_std": -0.11264010939720397, |
| "nauc_mrr_at_10_diff1": 0.09707068152210184, |
| "nauc_mrr_at_20_max": -0.09221792922323616, |
| "nauc_mrr_at_20_std": -0.10961211427408772, |
| "nauc_mrr_at_20_diff1": 0.09943255521491648, |
| "nauc_mrr_at_100_max": -0.09316799067959038, |
| "nauc_mrr_at_100_std": -0.10968589627543163, |
| "nauc_mrr_at_100_diff1": 0.09910011165040408, |
| "nauc_mrr_at_1000_max": -0.09321385399598632, |
| "nauc_mrr_at_1000_std": -0.10970513219025511, |
| "nauc_mrr_at_1000_diff1": 0.09907801272427635, |
| "hit_rate_at_1": 0.35349, |
| "hit_rate_at_3": 0.60313, |
| "hit_rate_at_5": 0.72262, |
| "hit_rate_at_10": 0.86558, |
| "hit_rate_at_20": 0.94666, |
| "hit_rate_at_100": 0.99147, |
| "hit_rate_at_1000": 0.99573, |
| "main_score": 0.59246, |
| "hf_subset": "default", |
| "languages": [ |
| "eng-Latn" |
| ], |
| "mteb_version": "2.21.0" |
| }, |
| "small": { |
| "ndcg_at_1": 0.1458, |
| "ndcg_at_3": 0.23456, |
| "ndcg_at_5": 0.27835, |
| "ndcg_at_10": 0.32889, |
| "ndcg_at_20": 0.36967, |
| "ndcg_at_100": 0.4045, |
| "ndcg_at_1000": 0.41446, |
| "map_at_1": 0.1458, |
| "map_at_3": 0.21136, |
| "map_at_5": 0.2354, |
| "map_at_10": 0.25622, |
| "map_at_20": 0.26767, |
| "map_at_100": 0.27279, |
| "map_at_1000": 0.27315, |
| "recall_at_1": 0.1458, |
| "recall_at_3": 0.30228, |
| "recall_at_5": 0.40967, |
| "recall_at_10": 0.56615, |
| "recall_at_20": 0.72546, |
| "recall_at_100": 0.90896, |
| "recall_at_1000": 0.98791, |
| "accuracy": 0.1458, |
| "precision_at_1": 0.1458, |
| "precision_at_3": 0.10076, |
| "precision_at_5": 0.08193, |
| "precision_at_10": 0.05661, |
| "precision_at_20": 0.03627, |
| "precision_at_100": 0.00909, |
| "precision_at_1000": 0.00099, |
| "mrr_at_1": 0.14580369843527738, |
| "mrr_at_3": 0.21135609293504032, |
| "mrr_at_5": 0.2353959222380275, |
| "mrr_at_10": 0.25621881279776015, |
| "mrr_at_20": 0.26766780266556217, |
| "mrr_at_100": 0.27278502616211503, |
| "mrr_at_1000": 0.2731483695142967, |
| "nauc_ndcg_at_1_max": 0.07124074977242485, |
| "nauc_ndcg_at_1_std": 0.07043132234894327, |
| "nauc_ndcg_at_1_diff1": 0.17867294841183828, |
| "nauc_ndcg_at_3_max": 0.05391218816481269, |
| "nauc_ndcg_at_3_std": 0.08300022308422525, |
| "nauc_ndcg_at_3_diff1": 0.13721153170379125, |
| "nauc_ndcg_at_5_max": 0.04761392610075392, |
| "nauc_ndcg_at_5_std": 0.09330318818026477, |
| "nauc_ndcg_at_5_diff1": 0.129721108359703, |
| "nauc_ndcg_at_10_max": 0.06398717148477524, |
| "nauc_ndcg_at_10_std": 0.11739949823329376, |
| "nauc_ndcg_at_10_diff1": 0.1353789207379315, |
| "nauc_ndcg_at_20_max": 0.06583701467786816, |
| "nauc_ndcg_at_20_std": 0.11760807459667257, |
| "nauc_ndcg_at_20_diff1": 0.1464058778549573, |
| "nauc_ndcg_at_100_max": 0.07271569743212124, |
| "nauc_ndcg_at_100_std": 0.12767735668965335, |
| "nauc_ndcg_at_100_diff1": 0.14440559200255987, |
| "nauc_ndcg_at_1000_max": 0.06760834890113898, |
| "nauc_ndcg_at_1000_std": 0.11565058305226664, |
| "nauc_ndcg_at_1000_diff1": 0.14440414116396014, |
| "nauc_map_at_1_max": 0.07124074977242485, |
| "nauc_map_at_1_std": 0.07043132234894327, |
| "nauc_map_at_1_diff1": 0.17867294841183828, |
| "nauc_map_at_3_max": 0.05536834016676611, |
| "nauc_map_at_3_std": 0.07918800842578326, |
| "nauc_map_at_3_diff1": 0.14582401585695304, |
| "nauc_map_at_5_max": 0.05153209428882134, |
| "nauc_map_at_5_std": 0.08553839884319275, |
| "nauc_map_at_5_diff1": 0.1412262351733516, |
| "nauc_map_at_10_max": 0.05764323340595916, |
| "nauc_map_at_10_std": 0.09567768210275003, |
| "nauc_map_at_10_diff1": 0.14384793290679312, |
| "nauc_map_at_20_max": 0.058797567381990594, |
| "nauc_map_at_20_std": 0.09556268067232435, |
| "nauc_map_at_20_diff1": 0.14719660481591418, |
| "nauc_map_at_100_max": 0.060341050948804, |
| "nauc_map_at_100_std": 0.09715443446030105, |
| "nauc_map_at_100_diff1": 0.14729508200613933, |
| "nauc_map_at_1000_max": 0.060216285099770145, |
| "nauc_map_at_1000_std": 0.09674391170833818, |
| "nauc_map_at_1000_diff1": 0.14729630480806993, |
| "nauc_recall_at_1_max": 0.07124074977242485, |
| "nauc_recall_at_1_std": 0.07043132234894327, |
| "nauc_recall_at_1_diff1": 0.17867294841183828, |
| "nauc_recall_at_3_max": 0.05116463248307958, |
| "nauc_recall_at_3_std": 0.09233493207170033, |
| "nauc_recall_at_3_diff1": 0.11697591005218377, |
| "nauc_recall_at_5_max": 0.0387185231377629, |
| "nauc_recall_at_5_std": 0.11230251236242163, |
| "nauc_recall_at_5_diff1": 0.10273687046821164, |
| "nauc_recall_at_10_max": 0.08768343797110026, |
| "nauc_recall_at_10_std": 0.18174312026830694, |
| "nauc_recall_at_10_diff1": 0.11544199093498575, |
| "nauc_recall_at_20_max": 0.0976177919930251, |
| "nauc_recall_at_20_std": 0.20309410782757367, |
| "nauc_recall_at_20_diff1": 0.15596172000438252, |
| "nauc_recall_at_100_max": 0.21399140718760357, |
| "nauc_recall_at_100_std": 0.45432867879288047, |
| "nauc_recall_at_100_diff1": 0.13074733185800497, |
| "nauc_recall_at_1000_max": 0.29184801184590964, |
| "nauc_recall_at_1000_std": 0.7268160753847613, |
| "nauc_recall_at_1000_diff1": 0.04649951766999351, |
| "nauc_precision_at_1_max": 0.07124074977242485, |
| "nauc_precision_at_1_std": 0.07043132234894327, |
| "nauc_precision_at_1_diff1": 0.17867294841183828, |
| "nauc_precision_at_3_max": 0.051164632483079536, |
| "nauc_precision_at_3_std": 0.09233493207170006, |
| "nauc_precision_at_3_diff1": 0.11697591005218336, |
| "nauc_precision_at_5_max": 0.03871852313776277, |
| "nauc_precision_at_5_std": 0.11230251236242186, |
| "nauc_precision_at_5_diff1": 0.10273687046821152, |
| "nauc_precision_at_10_max": 0.08768343797110059, |
| "nauc_precision_at_10_std": 0.18174312026830697, |
| "nauc_precision_at_10_diff1": 0.11544199093498574, |
| "nauc_precision_at_20_max": 0.09761779199302502, |
| "nauc_precision_at_20_std": 0.20309410782757284, |
| "nauc_precision_at_20_diff1": 0.15596172000438274, |
| "nauc_precision_at_100_max": 0.2139914071876045, |
| "nauc_precision_at_100_std": 0.45432867879288213, |
| "nauc_precision_at_100_diff1": 0.13074733185800294, |
| "nauc_precision_at_1000_max": 0.29184801184591147, |
| "nauc_precision_at_1000_std": 0.7268160753847517, |
| "nauc_precision_at_1000_diff1": 0.04649951766998672, |
| "nauc_mrr_at_1_max": 0.07124074977242485, |
| "nauc_mrr_at_1_std": 0.07043132234894327, |
| "nauc_mrr_at_1_diff1": 0.17867294841183828, |
| "nauc_mrr_at_3_max": 0.05536834016676611, |
| "nauc_mrr_at_3_std": 0.07918800842578326, |
| "nauc_mrr_at_3_diff1": 0.14582401585695304, |
| "nauc_mrr_at_5_max": 0.05153209428882134, |
| "nauc_mrr_at_5_std": 0.08553839884319275, |
| "nauc_mrr_at_5_diff1": 0.1412262351733516, |
| "nauc_mrr_at_10_max": 0.05764323340595916, |
| "nauc_mrr_at_10_std": 0.09567768210275003, |
| "nauc_mrr_at_10_diff1": 0.14384793290679312, |
| "nauc_mrr_at_20_max": 0.058797567381990594, |
| "nauc_mrr_at_20_std": 0.09556268067232435, |
| "nauc_mrr_at_20_diff1": 0.14719660481591418, |
| "nauc_mrr_at_100_max": 0.060341050948804, |
| "nauc_mrr_at_100_std": 0.09715443446030105, |
| "nauc_mrr_at_100_diff1": 0.14729508200613933, |
| "nauc_mrr_at_1000_max": 0.060216285099770145, |
| "nauc_mrr_at_1000_std": 0.09674391170833818, |
| "nauc_mrr_at_1000_diff1": 0.14729630480806993, |
| "hit_rate_at_1": 0.1458, |
| "hit_rate_at_3": 0.30228, |
| "hit_rate_at_5": 0.40967, |
| "hit_rate_at_10": 0.56615, |
| "hit_rate_at_20": 0.72546, |
| "hit_rate_at_100": 0.90896, |
| "hit_rate_at_1000": 0.98791, |
| "main_score": 0.32889, |
| "hf_subset": "default", |
| "languages": [ |
| "eng-Latn" |
| ], |
| "mteb_version": "2.21.0" |
| } |
| }, |
| "measurement_by_model": { |
| "small": { |
| "evaluated_native_fingerprint": "684fe0c48a4ddd3944167f6e5dc75e97ada24e5b120bd9a2921f00dc03112eeb", |
| "original_checkpoint_revision": "fdf8e01b7b0f3a69ac1ac8e2a64dcb1ede177ba4", |
| "raw_result_sha256": "92cbe96f6e84ebd0bc144001424ac46bb3a630ef64bffb433c2cdd63a10c43ec", |
| "kind": "direct_frozen_encoder_evaluation" |
| } |
| } |
| }, |
| { |
| "task": "TwentyNewsgroupsClustering.v2", |
| "type": "Clustering", |
| "modalities": [ |
| "text" |
| ], |
| "split": "test", |
| "hf_subset": "default", |
| "main_metric": "v_measure", |
| "dataset": { |
| "path": "mteb/twentynewsgroups-clustering", |
| "revision": "6125ec4e24fa026cec8a478383ee943acfbd5449" |
| }, |
| "counts": { |
| "population_rows": 59545, |
| "official_encoded_rows": 2381, |
| "official_encoding_fraction": 0.04, |
| "bootstrap_clustering_trials": 10, |
| "resampled_items_per_trial": 16384 |
| }, |
| "scores": { |
| "nano": 0.5087260708286914, |
| "small": 0.44840076500124326 |
| }, |
| "metrics": { |
| "nano": { |
| "v_measures": { |
| "Level 0": [ |
| 0.4966639569479829, |
| 0.5210509442892017, |
| 0.5333989625396617, |
| 0.5094460445939109, |
| 0.5208429535416115, |
| 0.5012604404318527, |
| 0.5210119744629907, |
| 0.4617825059979706, |
| 0.5226704101107219, |
| 0.49913251537100806 |
| ] |
| }, |
| "v_measure": 0.5087260708286914, |
| "v_measure_std": 0.01936580527644773, |
| "ami_scores": { |
| "Level 0": [ |
| 0.49471495769938123, |
| 0.5192201304733433, |
| 0.5316138913144499, |
| 0.5075814211955173, |
| 0.5190252719774525, |
| 0.49937824878079495, |
| 0.5191850550712727, |
| 0.4597515065381579, |
| 0.5208311166380218, |
| 0.49721166070769024 |
| ] |
| }, |
| "ami": 0.5068513260396083, |
| "ami_std": 0.01943429917113934, |
| "main_score": 0.5087260708286914, |
| "hf_subset": "default", |
| "languages": [ |
| "eng-Latn" |
| ], |
| "mteb_version": "2.21.0" |
| }, |
| "small": { |
| "v_measures": { |
| "Level 0": [ |
| 0.46653683369731075, |
| 0.46464237680676646, |
| 0.4425518472920712, |
| 0.4288571327957099, |
| 0.4336826969564481, |
| 0.43371621267306154, |
| 0.43092596596465876, |
| 0.4536864467956253, |
| 0.4741214282291591, |
| 0.45528670880162087 |
| ] |
| }, |
| "v_measure": 0.44840076500124326, |
| "v_measure_std": 0.01575881885263055, |
| "ami_scores": { |
| "Level 0": [ |
| 0.46451147281180905, |
| 0.4625982506803001, |
| 0.44043666823974115, |
| 0.42670551677458946, |
| 0.43154706196129733, |
| 0.4315549515134722, |
| 0.4287870602853061, |
| 0.45163080092170965, |
| 0.47212403002394715, |
| 0.4532359410185627 |
| ] |
| }, |
| "ami": 0.44631317542307347, |
| "ami_std": 0.01581387159916619, |
| "main_score": 0.44840076500124326, |
| "hf_subset": "default", |
| "languages": [ |
| "eng-Latn" |
| ], |
| "mteb_version": "2.21.0" |
| } |
| }, |
| "measurement_by_model": { |
| "small": { |
| "evaluated_native_fingerprint": "684fe0c48a4ddd3944167f6e5dc75e97ada24e5b120bd9a2921f00dc03112eeb", |
| "original_checkpoint_revision": "fdf8e01b7b0f3a69ac1ac8e2a64dcb1ede177ba4", |
| "raw_result_sha256": "62fe52980cb4445b9969d53135fa1663dd8dcd811743a711dde55f2a097ced19", |
| "kind": "direct_frozen_encoder_evaluation" |
| } |
| } |
| }, |
| { |
| "task": "MTOPDomainClassification", |
| "type": "Classification", |
| "modalities": [ |
| "text" |
| ], |
| "split": "validation", |
| "hf_subset": "en", |
| "main_metric": "accuracy", |
| "dataset": { |
| "path": "mteb/MTOPDomainClassification", |
| "revision": "a76d16fae880597b9c73047b50159220a441cb54" |
| }, |
| "counts": { |
| "evaluation_rows": 2235, |
| "available_train_rows": 15667, |
| "official_probe_experiments": 10, |
| "probe_samples_per_label": 8 |
| }, |
| "scores": { |
| "nano": 0.9476510067114093, |
| "small": 0.8543624161073826 |
| }, |
| "metrics": { |
| "nano": { |
| "scores_per_experiment": [ |
| { |
| "accuracy": 0.9521252796420582, |
| "f1": 0.9509396968297774, |
| "f1_weighted": 0.9519302419648948, |
| "precision": 0.9493488952211522, |
| "precision_weighted": 0.9527659658117037, |
| "recall": 0.9535803496254757, |
| "recall_weighted": 0.9521252796420582, |
| "ap": null, |
| "ap_weighted": null |
| }, |
| { |
| "accuracy": 0.9458612975391499, |
| "f1": 0.9439475266607382, |
| "f1_weighted": 0.945767354216612, |
| "precision": 0.9420005561621522, |
| "precision_weighted": 0.9462895866502771, |
| "recall": 0.946397940529448, |
| "recall_weighted": 0.9458612975391499, |
| "ap": null, |
| "ap_weighted": null |
| }, |
| { |
| "accuracy": 0.9284116331096197, |
| "f1": 0.9284988704174789, |
| "f1_weighted": 0.9270840634554431, |
| "precision": 0.9271908775153325, |
| "precision_weighted": 0.9274634710035635, |
| "recall": 0.9317153602473662, |
| "recall_weighted": 0.9284116331096197, |
| "ap": null, |
| "ap_weighted": null |
| }, |
| { |
| "accuracy": 0.9525727069351231, |
| "f1": 0.950987470976036, |
| "f1_weighted": 0.9525185063610705, |
| "precision": 0.9504292297253194, |
| "precision_weighted": 0.9534501308715367, |
| "recall": 0.9526312322737922, |
| "recall_weighted": 0.9525727069351231, |
| "ap": null, |
| "ap_weighted": null |
| }, |
| { |
| "accuracy": 0.9445190156599552, |
| "f1": 0.943383064264026, |
| "f1_weighted": 0.9442288513061988, |
| "precision": 0.9424858291605631, |
| "precision_weighted": 0.944823795147547, |
| "recall": 0.945181115724642, |
| "recall_weighted": 0.9445190156599552, |
| "ap": null, |
| "ap_weighted": null |
| }, |
| { |
| "accuracy": 0.9525727069351231, |
| "f1": 0.949882476921466, |
| "f1_weighted": 0.9526817135384125, |
| "precision": 0.9498620235113855, |
| "precision_weighted": 0.9543297139412104, |
| "recall": 0.9514687067629995, |
| "recall_weighted": 0.9525727069351231, |
| "ap": null, |
| "ap_weighted": null |
| }, |
| { |
| "accuracy": 0.9498881431767338, |
| "f1": 0.9496336255013932, |
| "f1_weighted": 0.9495703695110613, |
| "precision": 0.9490015795096777, |
| "precision_weighted": 0.9511221301014349, |
| "recall": 0.9520470037532829, |
| "recall_weighted": 0.9498881431767338, |
| "ap": null, |
| "ap_weighted": null |
| }, |
| { |
| "accuracy": 0.9570469798657718, |
| "f1": 0.9557194953102311, |
| "f1_weighted": 0.9568992928300293, |
| "precision": 0.9544888722394763, |
| "precision_weighted": 0.9584505479016514, |
| "recall": 0.9586898382241177, |
| "recall_weighted": 0.9570469798657718, |
| "ap": null, |
| "ap_weighted": null |
| }, |
| { |
| "accuracy": 0.9494407158836689, |
| "f1": 0.9460535182757994, |
| "f1_weighted": 0.9492959546761313, |
| "precision": 0.9454088954645755, |
| "precision_weighted": 0.9499145785798135, |
| "recall": 0.9472953341577202, |
| "recall_weighted": 0.9494407158836689, |
| "ap": null, |
| "ap_weighted": null |
| }, |
| { |
| "accuracy": 0.9440715883668904, |
| "f1": 0.9431192151526329, |
| "f1_weighted": 0.9437397930273277, |
| "precision": 0.9435095731735328, |
| "precision_weighted": 0.9461253948734786, |
| "recall": 0.9455297617432863, |
| "recall_weighted": 0.9440715883668904, |
| "ap": null, |
| "ap_weighted": null |
| } |
| ], |
| "accuracy": 0.9476510067114093, |
| "f1": 0.9462164960309579, |
| "f1_weighted": 0.9473716140887183, |
| "precision": 0.9453726331683165, |
| "precision_weighted": 0.9484735314882217, |
| "recall": 0.948453664304213, |
| "recall_weighted": 0.9476510067114093, |
| "ap": null, |
| "ap_weighted": null, |
| "main_score": 0.9476510067114093, |
| "hf_subset": "en", |
| "languages": [ |
| "eng-Latn" |
| ], |
| "mteb_version": "2.21.0" |
| }, |
| "small": { |
| "scores_per_experiment": [ |
| { |
| "accuracy": 0.8250559284116331, |
| "f1": 0.8322814729997016, |
| "f1_weighted": 0.8240316200136756, |
| "precision": 0.8318540374193987, |
| "precision_weighted": 0.8346068877064119, |
| "recall": 0.8430207439526936, |
| "recall_weighted": 0.8250559284116331, |
| "ap": null, |
| "ap_weighted": null |
| }, |
| { |
| "accuracy": 0.8559284116331096, |
| "f1": 0.8552447478537527, |
| "f1_weighted": 0.8547401242258992, |
| "precision": 0.8515715568445064, |
| "precision_weighted": 0.8581578601876096, |
| "recall": 0.8640896175706192, |
| "recall_weighted": 0.8559284116331096, |
| "ap": null, |
| "ap_weighted": null |
| }, |
| { |
| "accuracy": 0.8304250559284116, |
| "f1": 0.8317375249321166, |
| "f1_weighted": 0.8274798893470849, |
| "precision": 0.8268066926411431, |
| "precision_weighted": 0.8285364690282124, |
| "recall": 0.8409132018240374, |
| "recall_weighted": 0.8304250559284116, |
| "ap": null, |
| "ap_weighted": null |
| }, |
| { |
| "accuracy": 0.8648769574944072, |
| "f1": 0.8678594298716482, |
| "f1_weighted": 0.8632422395451538, |
| "precision": 0.865870318819693, |
| "precision_weighted": 0.8646460721917335, |
| "recall": 0.8726494617104266, |
| "recall_weighted": 0.8648769574944072, |
| "ap": null, |
| "ap_weighted": null |
| }, |
| { |
| "accuracy": 0.8532438478747204, |
| "f1": 0.8558930928177336, |
| "f1_weighted": 0.8512711153646526, |
| "precision": 0.8501961024805756, |
| "precision_weighted": 0.8547737986953718, |
| "recall": 0.8657455326712605, |
| "recall_weighted": 0.8532438478747204, |
| "ap": null, |
| "ap_weighted": null |
| }, |
| { |
| "accuracy": 0.8402684563758389, |
| "f1": 0.8478991888671523, |
| "f1_weighted": 0.8396067623115454, |
| "precision": 0.8508550254092774, |
| "precision_weighted": 0.8598285030005444, |
| "recall": 0.862051388925805, |
| "recall_weighted": 0.8402684563758389, |
| "ap": null, |
| "ap_weighted": null |
| }, |
| { |
| "accuracy": 0.8747203579418344, |
| "f1": 0.8781373847485636, |
| "f1_weighted": 0.8731828255054377, |
| "precision": 0.8729737218664414, |
| "precision_weighted": 0.8758466535281065, |
| "recall": 0.8873478774616341, |
| "recall_weighted": 0.8747203579418344, |
| "ap": null, |
| "ap_weighted": null |
| }, |
| { |
| "accuracy": 0.876062639821029, |
| "f1": 0.8754580934866012, |
| "f1_weighted": 0.8749795409187, |
| "precision": 0.8730455332008387, |
| "precision_weighted": 0.8787904116312295, |
| "recall": 0.8820295010368394, |
| "recall_weighted": 0.876062639821029, |
| "ap": null, |
| "ap_weighted": null |
| }, |
| { |
| "accuracy": 0.858165548098434, |
| "f1": 0.8610701024195923, |
| "f1_weighted": 0.8573656937247496, |
| "precision": 0.8561531713932808, |
| "precision_weighted": 0.8662234353266065, |
| "recall": 0.8738853045389997, |
| "recall_weighted": 0.858165548098434, |
| "ap": null, |
| "ap_weighted": null |
| }, |
| { |
| "accuracy": 0.8648769574944072, |
| "f1": 0.8671414562971872, |
| "f1_weighted": 0.8628546885742888, |
| "precision": 0.8607074291115159, |
| "precision_weighted": 0.8655691598347531, |
| "recall": 0.8776421302043499, |
| "recall_weighted": 0.8648769574944072, |
| "ap": null, |
| "ap_weighted": null |
| } |
| ], |
| "accuracy": 0.8543624161073826, |
| "f1": 0.8572722494294049, |
| "f1_weighted": 0.8528754499531187, |
| "precision": 0.8540033589186671, |
| "precision_weighted": 0.8586979251130579, |
| "recall": 0.8669374759896666, |
| "recall_weighted": 0.8543624161073826, |
| "ap": null, |
| "ap_weighted": null, |
| "main_score": 0.8543624161073826, |
| "hf_subset": "en", |
| "languages": [ |
| "eng-Latn" |
| ], |
| "mteb_version": "2.21.0" |
| } |
| }, |
| "measurement_by_model": { |
| "small": { |
| "evaluated_native_fingerprint": "684fe0c48a4ddd3944167f6e5dc75e97ada24e5b120bd9a2921f00dc03112eeb", |
| "original_checkpoint_revision": "fdf8e01b7b0f3a69ac1ac8e2a64dcb1ede177ba4", |
| "raw_result_sha256": "256702650b93bd559d9a2a9b70f32b996b51e166debc0c20d63ce24cc6f605f6", |
| "kind": "direct_frozen_encoder_evaluation" |
| } |
| } |
| }, |
| { |
| "task": "MTOPDomainClassification", |
| "type": "Classification", |
| "modalities": [ |
| "text" |
| ], |
| "split": "test", |
| "hf_subset": "en", |
| "main_metric": "accuracy", |
| "dataset": { |
| "path": "mteb/MTOPDomainClassification", |
| "revision": "a76d16fae880597b9c73047b50159220a441cb54" |
| }, |
| "counts": { |
| "evaluation_rows": 4386, |
| "available_train_rows": 15667, |
| "official_probe_experiments": 10, |
| "probe_samples_per_label": 8 |
| }, |
| "scores": { |
| "nano": 0.9491792065663475, |
| "small": 0.8456452348381213 |
| }, |
| "metrics": { |
| "nano": { |
| "scores_per_experiment": [ |
| { |
| "accuracy": 0.9507523939808481, |
| "f1": 0.9490862282573499, |
| "f1_weighted": 0.9508003256061616, |
| "precision": 0.9476949859831407, |
| "precision_weighted": 0.9520734016967529, |
| "recall": 0.9517099502990664, |
| "recall_weighted": 0.9507523939808481, |
| "ap": null, |
| "ap_weighted": null |
| }, |
| { |
| "accuracy": 0.9473324213406292, |
| "f1": 0.9425946320317333, |
| "f1_weighted": 0.9473133266787116, |
| "precision": 0.9392105665237355, |
| "precision_weighted": 0.9479986101450497, |
| "recall": 0.9468765008841218, |
| "recall_weighted": 0.9473324213406292, |
| "ap": null, |
| "ap_weighted": null |
| }, |
| { |
| "accuracy": 0.9306885544915641, |
| "f1": 0.9261678157710709, |
| "f1_weighted": 0.9296159823715927, |
| "precision": 0.923024327342814, |
| "precision_weighted": 0.9307965712480242, |
| "recall": 0.932201810783149, |
| "recall_weighted": 0.9306885544915641, |
| "ap": null, |
| "ap_weighted": null |
| }, |
| { |
| "accuracy": 0.958048335613315, |
| "f1": 0.9552869971485034, |
| "f1_weighted": 0.9579502405954128, |
| "precision": 0.954624514001619, |
| "precision_weighted": 0.9582682331625527, |
| "recall": 0.9564289859937923, |
| "recall_weighted": 0.958048335613315, |
| "ap": null, |
| "ap_weighted": null |
| }, |
| { |
| "accuracy": 0.9464204286365709, |
| "f1": 0.94433525009326, |
| "f1_weighted": 0.9462665825311201, |
| "precision": 0.9422860464299823, |
| "precision_weighted": 0.9467645700971977, |
| "recall": 0.9469874285322217, |
| "recall_weighted": 0.9464204286365709, |
| "ap": null, |
| "ap_weighted": null |
| }, |
| { |
| "accuracy": 0.9505243958048336, |
| "f1": 0.9447844972663006, |
| "f1_weighted": 0.9507789369812125, |
| "precision": 0.9429136617255864, |
| "precision_weighted": 0.9524852717381878, |
| "recall": 0.9483171314260196, |
| "recall_weighted": 0.9505243958048336, |
| "ap": null, |
| "ap_weighted": null |
| }, |
| { |
| "accuracy": 0.9509803921568627, |
| "f1": 0.9480693879894481, |
| "f1_weighted": 0.9510087641402198, |
| "precision": 0.9447979434260977, |
| "precision_weighted": 0.9522624035738166, |
| "recall": 0.952376748039714, |
| "recall_weighted": 0.9509803921568627, |
| "ap": null, |
| "ap_weighted": null |
| }, |
| { |
| "accuracy": 0.9573643410852714, |
| "f1": 0.9551153583299494, |
| "f1_weighted": 0.9573535659550753, |
| "precision": 0.952401865813031, |
| "precision_weighted": 0.9583040343256545, |
| "recall": 0.9586994420587002, |
| "recall_weighted": 0.9573643410852714, |
| "ap": null, |
| "ap_weighted": null |
| }, |
| { |
| "accuracy": 0.9548563611491108, |
| "f1": 0.9495763118202308, |
| "f1_weighted": 0.9549499907192655, |
| "precision": 0.946851274794657, |
| "precision_weighted": 0.9555638233419986, |
| "recall": 0.9529512436069031, |
| "recall_weighted": 0.9548563611491108, |
| "ap": null, |
| "ap_weighted": null |
| }, |
| { |
| "accuracy": 0.9448244414044688, |
| "f1": 0.9427993333346522, |
| "f1_weighted": 0.9447713522464498, |
| "precision": 0.9432072583992069, |
| "precision_weighted": 0.9468986783847072, |
| "recall": 0.9447479153319868, |
| "recall_weighted": 0.9448244414044688, |
| "ap": null, |
| "ap_weighted": null |
| } |
| ], |
| "accuracy": 0.9491792065663475, |
| "f1": 0.9457815812042497, |
| "f1_weighted": 0.9490809067825221, |
| "precision": 0.9437012444439871, |
| "precision_weighted": 0.9501415597713943, |
| "recall": 0.9491297156955675, |
| "recall_weighted": 0.9491792065663475, |
| "ap": null, |
| "ap_weighted": null, |
| "main_score": 0.9491792065663475, |
| "hf_subset": "en", |
| "languages": [ |
| "eng-Latn" |
| ], |
| "mteb_version": "2.21.0" |
| }, |
| "small": { |
| "scores_per_experiment": [ |
| { |
| "accuracy": 0.8180574555403557, |
| "f1": 0.8207347394289347, |
| "f1_weighted": 0.8170857264694432, |
| "precision": 0.820363266003151, |
| "precision_weighted": 0.8301897328408723, |
| "recall": 0.8338753807988755, |
| "recall_weighted": 0.8180574555403557, |
| "ap": null, |
| "ap_weighted": null |
| }, |
| { |
| "accuracy": 0.8449612403100775, |
| "f1": 0.839516667111528, |
| "f1_weighted": 0.8449410274873432, |
| "precision": 0.8369796042511596, |
| "precision_weighted": 0.8536873756717578, |
| "recall": 0.8523590480814156, |
| "recall_weighted": 0.8449612403100775, |
| "ap": null, |
| "ap_weighted": null |
| }, |
| { |
| "accuracy": 0.8258093935248518, |
| "f1": 0.8185934728273051, |
| "f1_weighted": 0.8251774946806023, |
| "precision": 0.8147370918656317, |
| "precision_weighted": 0.8330066052370535, |
| "recall": 0.8324642506433119, |
| "recall_weighted": 0.8258093935248518, |
| "ap": null, |
| "ap_weighted": null |
| }, |
| { |
| "accuracy": 0.8497492020063839, |
| "f1": 0.8486953297726213, |
| "f1_weighted": 0.848088172519004, |
| "precision": 0.8467111465364261, |
| "precision_weighted": 0.8501008589343039, |
| "recall": 0.853578648077031, |
| "recall_weighted": 0.8497492020063839, |
| "ap": null, |
| "ap_weighted": null |
| }, |
| { |
| "accuracy": 0.8481532147742818, |
| "f1": 0.8467468733444872, |
| "f1_weighted": 0.8467448330993773, |
| "precision": 0.8393119909268087, |
| "precision_weighted": 0.8506182695179204, |
| "recall": 0.858498685671118, |
| "recall_weighted": 0.8481532147742818, |
| "ap": null, |
| "ap_weighted": null |
| }, |
| { |
| "accuracy": 0.8326493388052896, |
| "f1": 0.8352387297727578, |
| "f1_weighted": 0.8347572800634613, |
| "precision": 0.8420129591399498, |
| "precision_weighted": 0.8638673888380122, |
| "recall": 0.8534348314007274, |
| "recall_weighted": 0.8326493388052896, |
| "ap": null, |
| "ap_weighted": null |
| }, |
| { |
| "accuracy": 0.8682170542635659, |
| "f1": 0.8648438659784241, |
| "f1_weighted": 0.8680876089933638, |
| "precision": 0.8585183480506728, |
| "precision_weighted": 0.876371854401442, |
| "recall": 0.8802432046324339, |
| "recall_weighted": 0.8682170542635659, |
| "ap": null, |
| "ap_weighted": null |
| }, |
| { |
| "accuracy": 0.8518011855905153, |
| "f1": 0.8484079910361157, |
| "f1_weighted": 0.8509906563529788, |
| "precision": 0.8443896101449812, |
| "precision_weighted": 0.860550178646213, |
| "recall": 0.8607698617029648, |
| "recall_weighted": 0.8518011855905153, |
| "ap": null, |
| "ap_weighted": null |
| }, |
| { |
| "accuracy": 0.8556771545827634, |
| "f1": 0.8515433915603542, |
| "f1_weighted": 0.856278856298033, |
| "precision": 0.84610847827307, |
| "precision_weighted": 0.8661492710958929, |
| "recall": 0.8658189317867632, |
| "recall_weighted": 0.8556771545827634, |
| "ap": null, |
| "ap_weighted": null |
| }, |
| { |
| "accuracy": 0.8613771089831281, |
| "f1": 0.8586787981057005, |
| "f1_weighted": 0.8603956735839339, |
| "precision": 0.8518135650230533, |
| "precision_weighted": 0.8659927155990171, |
| "recall": 0.8717255305513517, |
| "recall_weighted": 0.8613771089831281, |
| "ap": null, |
| "ap_weighted": null |
| } |
| ], |
| "accuracy": 0.8456452348381213, |
| "f1": 0.8432999858938229, |
| "f1_weighted": 0.8452547329547541, |
| "precision": 0.8400946060214904, |
| "precision_weighted": 0.8550534250782486, |
| "recall": 0.8562768373345993, |
| "recall_weighted": 0.8456452348381213, |
| "ap": null, |
| "ap_weighted": null, |
| "main_score": 0.8456452348381213, |
| "hf_subset": "en", |
| "languages": [ |
| "eng-Latn" |
| ], |
| "mteb_version": "2.21.0" |
| } |
| }, |
| "measurement_by_model": { |
| "small": { |
| "evaluated_native_fingerprint": "684fe0c48a4ddd3944167f6e5dc75e97ada24e5b120bd9a2921f00dc03112eeb", |
| "original_checkpoint_revision": "fdf8e01b7b0f3a69ac1ac8e2a64dcb1ede177ba4", |
| "raw_result_sha256": "256702650b93bd559d9a2a9b70f32b996b51e166debc0c20d63ce24cc6f605f6", |
| "kind": "direct_frozen_encoder_evaluation" |
| } |
| } |
| }, |
| { |
| "task": "OxfordPets", |
| "type": "ImageClassification", |
| "modalities": [ |
| "image" |
| ], |
| "split": "test", |
| "hf_subset": "default", |
| "main_metric": "accuracy", |
| "dataset": { |
| "path": "mteb/OxfordPets", |
| "revision": "557b480fae8d69247be74d9503b378a09425096f" |
| }, |
| "counts": { |
| "evaluation_rows": 3669, |
| "available_train_rows": 3680, |
| "official_probe_experiments": 5, |
| "probe_samples_per_label": 16 |
| }, |
| "scores": { |
| "nano": 0.9126737530662306, |
| "small": 0.4184791496320523 |
| }, |
| "metrics": { |
| "nano": { |
| "scores_per_experiment": [ |
| { |
| "accuracy": 0.9062414826928318, |
| "f1": 0.9045301316021502, |
| "f1_weighted": 0.9050055409371512, |
| "precision": 0.9099213291951324, |
| "precision_weighted": 0.9109544817876485, |
| "recall": 0.9064295837897134, |
| "recall_weighted": 0.9062414826928318, |
| "ap": null, |
| "ap_weighted": null |
| }, |
| { |
| "accuracy": 0.914418097574271, |
| "f1": 0.9126724638115935, |
| "f1_weighted": 0.9132186480967779, |
| "precision": 0.9192904646518116, |
| "precision_weighted": 0.9203471940294945, |
| "recall": 0.9144593228002407, |
| "recall_weighted": 0.914418097574271, |
| "ap": null, |
| "ap_weighted": null |
| }, |
| { |
| "accuracy": 0.9187789588443718, |
| "f1": 0.9177765846692281, |
| "f1_weighted": 0.9184111765200857, |
| "precision": 0.9220755461580387, |
| "precision_weighted": 0.9229278273528245, |
| "recall": 0.9183739103921627, |
| "recall_weighted": 0.9187789588443718, |
| "ap": null, |
| "ap_weighted": null |
| }, |
| { |
| "accuracy": 0.9174161896974653, |
| "f1": 0.9147622354932347, |
| "f1_weighted": 0.9154663340384108, |
| "precision": 0.9209055798643112, |
| "precision_weighted": 0.9221124471295344, |
| "recall": 0.9173606715378323, |
| "recall_weighted": 0.9174161896974653, |
| "ap": null, |
| "ap_weighted": null |
| }, |
| { |
| "accuracy": 0.9065140365222132, |
| "f1": 0.9051669100036687, |
| "f1_weighted": 0.9057728210054364, |
| "precision": 0.9098426973534027, |
| "precision_weighted": 0.9105143284060627, |
| "recall": 0.9059560197061645, |
| "recall_weighted": 0.9065140365222132, |
| "ap": null, |
| "ap_weighted": null |
| } |
| ], |
| "accuracy": 0.9126737530662306, |
| "f1": 0.910981665115975, |
| "f1_weighted": 0.9115749041195725, |
| "precision": 0.9164071234445392, |
| "precision_weighted": 0.9173712557411129, |
| "recall": 0.9125159016452227, |
| "recall_weighted": 0.9126737530662306, |
| "ap": null, |
| "ap_weighted": null, |
| "main_score": 0.9126737530662306, |
| "hf_subset": "default", |
| "languages": [ |
| "eng-Latn" |
| ], |
| "mteb_version": "2.21.0" |
| }, |
| "small": { |
| "scores_per_experiment": [ |
| { |
| "accuracy": 0.4148269283183429, |
| "f1": 0.40253654878032713, |
| "f1_weighted": 0.4024461757957877, |
| "precision": 0.4116431114014033, |
| "precision_weighted": 0.4119318458992588, |
| "recall": 0.4153030284766479, |
| "recall_weighted": 0.4148269283183429, |
| "ap": null, |
| "ap_weighted": null |
| }, |
| { |
| "accuracy": 0.4265467429817389, |
| "f1": 0.4174251045251405, |
| "f1_weighted": 0.4176529394767131, |
| "precision": 0.43921297726376096, |
| "precision_weighted": 0.4396338060561412, |
| "recall": 0.42629825783951975, |
| "recall_weighted": 0.4265467429817389, |
| "ap": null, |
| "ap_weighted": null |
| }, |
| { |
| "accuracy": 0.4088307440719542, |
| "f1": 0.3954869417563968, |
| "f1_weighted": 0.39542431005304346, |
| "precision": 0.40981373804908927, |
| "precision_weighted": 0.40980533913060346, |
| "recall": 0.40880378402552453, |
| "recall_weighted": 0.4088307440719542, |
| "ap": null, |
| "ap_weighted": null |
| }, |
| { |
| "accuracy": 0.4377214499863723, |
| "f1": 0.4251494451023654, |
| "f1_weighted": 0.4253356439185362, |
| "precision": 0.438981154791822, |
| "precision_weighted": 0.439719429760714, |
| "recall": 0.4381658372912861, |
| "recall_weighted": 0.4377214499863723, |
| "ap": null, |
| "ap_weighted": null |
| }, |
| { |
| "accuracy": 0.40446988280185336, |
| "f1": 0.3806382057679881, |
| "f1_weighted": 0.380430443085735, |
| "precision": 0.4088810121814978, |
| "precision_weighted": 0.40903302714302686, |
| "recall": 0.4046956272962792, |
| "recall_weighted": 0.40446988280185336, |
| "ap": null, |
| "ap_weighted": null |
| } |
| ], |
| "accuracy": 0.4184791496320523, |
| "f1": 0.4042472491864436, |
| "f1_weighted": 0.40425790246596305, |
| "precision": 0.42170639873751464, |
| "precision_weighted": 0.4220246895979488, |
| "recall": 0.4186533069858515, |
| "recall_weighted": 0.4184791496320523, |
| "ap": null, |
| "ap_weighted": null, |
| "main_score": 0.4184791496320523, |
| "hf_subset": "default", |
| "languages": [ |
| "eng-Latn" |
| ], |
| "mteb_version": "2.21.0" |
| } |
| }, |
| "measurement_by_model": { |
| "small": { |
| "evaluated_native_fingerprint": "684fe0c48a4ddd3944167f6e5dc75e97ada24e5b120bd9a2921f00dc03112eeb", |
| "original_checkpoint_revision": "fdf8e01b7b0f3a69ac1ac8e2a64dcb1ede177ba4", |
| "raw_result_sha256": "ef1f4928b9ae7d9da25a98b82a28fda3e098db9504ad9372cb5305ad32421a06", |
| "kind": "direct_frozen_encoder_evaluation" |
| } |
| } |
| }, |
| { |
| "task": "TinyImageNetClustering", |
| "type": "ImageClustering", |
| "modalities": [ |
| "image" |
| ], |
| "split": "valid", |
| "hf_subset": "default", |
| "main_metric": "nmi", |
| "dataset": { |
| "path": "mteb/tiny-imagenet", |
| "revision": "e04e259f646f750bd0bdb2a56c2a4577d89bee3d" |
| }, |
| "counts": { |
| "evaluation_rows": 10000 |
| }, |
| "scores": { |
| "nano": 0.6169193395626786, |
| "small": 0.4628490605491802 |
| }, |
| "metrics": { |
| "nano": { |
| "v_measure": 0.6169193395626787, |
| "nmi": 0.6169193395626786, |
| "ari": 0.21497891543638262, |
| "cluster_accuracy": 0.3638, |
| "main_score": 0.6169193395626786, |
| "hf_subset": "default", |
| "languages": [ |
| "eng-Latn" |
| ], |
| "mteb_version": "2.21.0" |
| }, |
| "small": { |
| "v_measure": 0.4628490605491802, |
| "nmi": 0.4628490605491802, |
| "ari": 0.08498222977932614, |
| "cluster_accuracy": 0.2035, |
| "main_score": 0.4628490605491802, |
| "hf_subset": "default", |
| "languages": [ |
| "eng-Latn" |
| ], |
| "mteb_version": "2.21.0" |
| } |
| }, |
| "measurement_by_model": { |
| "small": { |
| "evaluated_native_fingerprint": "684fe0c48a4ddd3944167f6e5dc75e97ada24e5b120bd9a2921f00dc03112eeb", |
| "original_checkpoint_revision": "fdf8e01b7b0f3a69ac1ac8e2a64dcb1ede177ba4", |
| "raw_result_sha256": "eb344e20e7d498aadbbd67a57015cc2220cba42b9550f5822b5a72d0d6cf1514", |
| "kind": "direct_frozen_encoder_evaluation" |
| } |
| } |
| }, |
| { |
| "task": "OxfordPetsZeroShot", |
| "type": "ZeroShotClassification", |
| "modalities": [ |
| "text", |
| "image" |
| ], |
| "split": "test", |
| "hf_subset": "default", |
| "main_metric": "accuracy", |
| "dataset": { |
| "path": "mteb/OxfordPets", |
| "revision": "557b480fae8d69247be74d9503b378a09425096f" |
| }, |
| "counts": { |
| "evaluation_rows": 3669, |
| "fixed_text_labels": 37 |
| }, |
| "scores": { |
| "nano": 0.08013082583810302, |
| "small": 0.1640774052875443 |
| }, |
| "metrics": { |
| "nano": { |
| "accuracy": 0.08013082583810302, |
| "f1": 0.0611345635215895, |
| "f1_weighted": 0.06128470828202288, |
| "precision": 0.1359645526412158, |
| "precision_weighted": 0.13671480699577512, |
| "recall": 0.0798783003173134, |
| "recall_weighted": 0.08013082583810302, |
| "ap": null, |
| "ap_weighted": null, |
| "main_score": 0.08013082583810302, |
| "hf_subset": "default", |
| "languages": [ |
| "eng-Latn" |
| ], |
| "mteb_version": "2.21.0" |
| }, |
| "small": { |
| "accuracy": 0.1640774052875443, |
| "f1": 0.12868294474251654, |
| "f1_weighted": 0.12888736955731406, |
| "precision": 0.15179208674748698, |
| "precision_weighted": 0.15241279079348374, |
| "recall": 0.16458376660623852, |
| "recall_weighted": 0.1640774052875443, |
| "ap": null, |
| "ap_weighted": null, |
| "main_score": 0.1640774052875443, |
| "hf_subset": "default", |
| "languages": [ |
| "eng-Latn" |
| ], |
| "mteb_version": "2.21.0" |
| } |
| }, |
| "measurement_by_model": { |
| "small": { |
| "evaluated_native_fingerprint": "684fe0c48a4ddd3944167f6e5dc75e97ada24e5b120bd9a2921f00dc03112eeb", |
| "original_checkpoint_revision": "fdf8e01b7b0f3a69ac1ac8e2a64dcb1ede177ba4", |
| "raw_result_sha256": "e52e42c4716e62710296df98a90b3a0b986875685748d043cfd7de913e5a6c42", |
| "kind": "direct_frozen_encoder_evaluation" |
| } |
| } |
| }, |
| { |
| "task": "CREMA_D", |
| "type": "AudioClassification", |
| "modalities": [ |
| "audio" |
| ], |
| "split": "train", |
| "hf_subset": "default", |
| "main_metric": "accuracy", |
| "dataset": { |
| "path": "mteb/crema-d", |
| "revision": "a9d0821955c418752248b8310438854e56a1cf34" |
| }, |
| "counts": { |
| "evaluation_rows": 7442, |
| "official_cross_validation_folds": 5, |
| "probe_samples_per_label": 8 |
| }, |
| "scores": { |
| "nano": 0.29763670140167686, |
| "small": 0.2995196855795547 |
| }, |
| "metrics": { |
| "nano": { |
| "scores_per_experiment": [ |
| { |
| "accuracy": 0.2847548690396239, |
| "f1": 0.25980556646698677, |
| "f1_weighted": 0.2606862265023784, |
| "precision": 0.2685141966762838, |
| "precision_weighted": 0.26869432571348717, |
| "recall": 0.283361264324777, |
| "recall_weighted": 0.2847548690396239, |
| "ap": null, |
| "ap_weighted": null |
| }, |
| { |
| "accuracy": 0.2981867024848892, |
| "f1": 0.2927999429509691, |
| "f1_weighted": 0.2952410831252662, |
| "precision": 0.29988591178720175, |
| "precision_weighted": 0.30336098208055257, |
| "recall": 0.29763527945216217, |
| "recall_weighted": 0.2981867024848892, |
| "ap": null, |
| "ap_weighted": null |
| }, |
| { |
| "accuracy": 0.28696236559139787, |
| "f1": 0.26582489649494206, |
| "f1_weighted": 0.26920117230368773, |
| "precision": 0.26489987169540286, |
| "precision_weighted": 0.2682170436584329, |
| "recall": 0.2834616565816636, |
| "recall_weighted": 0.28696236559139787, |
| "ap": null, |
| "ap_weighted": null |
| }, |
| { |
| "accuracy": 0.29973118279569894, |
| "f1": 0.291485315935673, |
| "f1_weighted": 0.28841683151437103, |
| "precision": 0.3068565359099591, |
| "precision_weighted": 0.3086441612810119, |
| "recall": 0.30801945566008065, |
| "recall_weighted": 0.29973118279569894, |
| "ap": null, |
| "ap_weighted": null |
| }, |
| { |
| "accuracy": 0.3185483870967742, |
| "f1": 0.3051858414398825, |
| "f1_weighted": 0.3066161074219095, |
| "precision": 0.310702613862649, |
| "precision_weighted": 0.3133755862555043, |
| "recall": 0.31849511838471833, |
| "recall_weighted": 0.3185483870967742, |
| "ap": null, |
| "ap_weighted": null |
| } |
| ], |
| "accuracy": 0.29763670140167686, |
| "f1": 0.2830203126576907, |
| "f1_weighted": 0.2840322841735226, |
| "precision": 0.29017182598629926, |
| "precision_weighted": 0.2924584197977978, |
| "recall": 0.2981945548806803, |
| "recall_weighted": 0.29763670140167686, |
| "ap": null, |
| "ap_weighted": null, |
| "main_score": 0.29763670140167686, |
| "hf_subset": "default", |
| "languages": [ |
| "eng-Latn" |
| ], |
| "mteb_version": "2.21.0" |
| }, |
| "small": { |
| "scores_per_experiment": [ |
| { |
| "accuracy": 0.2699798522498321, |
| "f1": 0.24239504058809014, |
| "f1_weighted": 0.2423525571760308, |
| "precision": 0.25470036430672977, |
| "precision_weighted": 0.2542205264446883, |
| "recall": 0.26975123145374097, |
| "recall_weighted": 0.2699798522498321, |
| "ap": null, |
| "ap_weighted": null |
| }, |
| { |
| "accuracy": 0.3035594358629953, |
| "f1": 0.29613784795561465, |
| "f1_weighted": 0.29906255840921786, |
| "precision": 0.30407267120079884, |
| "precision_weighted": 0.30800706496852154, |
| "recall": 0.3024788327196796, |
| "recall_weighted": 0.3035594358629953, |
| "ap": null, |
| "ap_weighted": null |
| }, |
| { |
| "accuracy": 0.28225806451612906, |
| "f1": 0.2593664352832718, |
| "f1_weighted": 0.2636113556497887, |
| "precision": 0.2609748445531306, |
| "precision_weighted": 0.26468416511118426, |
| "recall": 0.2775030429767242, |
| "recall_weighted": 0.28225806451612906, |
| "ap": null, |
| "ap_weighted": null |
| }, |
| { |
| "accuracy": 0.3111559139784946, |
| "f1": 0.29454980492169786, |
| "f1_weighted": 0.29125309340152467, |
| "precision": 0.3224943453400395, |
| "precision_weighted": 0.3242765511011472, |
| "recall": 0.31997721819075986, |
| "recall_weighted": 0.3111559139784946, |
| "ap": null, |
| "ap_weighted": null |
| }, |
| { |
| "accuracy": 0.33064516129032256, |
| "f1": 0.3185711713472314, |
| "f1_weighted": 0.3198970407995879, |
| "precision": 0.32320745672952395, |
| "precision_weighted": 0.32608239411677886, |
| "recall": 0.3309376196524952, |
| "recall_weighted": 0.33064516129032256, |
| "ap": null, |
| "ap_weighted": null |
| } |
| ], |
| "accuracy": 0.2995196855795547, |
| "f1": 0.2822040600191812, |
| "f1_weighted": 0.28323532108723004, |
| "precision": 0.29308993642604453, |
| "precision_weighted": 0.295454140348464, |
| "recall": 0.30012958899868, |
| "recall_weighted": 0.2995196855795547, |
| "ap": null, |
| "ap_weighted": null, |
| "main_score": 0.2995196855795547, |
| "hf_subset": "default", |
| "languages": [ |
| "eng-Latn" |
| ], |
| "mteb_version": "2.21.0" |
| } |
| }, |
| "measurement_by_model": { |
| "small": { |
| "evaluated_native_fingerprint": "684fe0c48a4ddd3944167f6e5dc75e97ada24e5b120bd9a2921f00dc03112eeb", |
| "original_checkpoint_revision": "fdf8e01b7b0f3a69ac1ac8e2a64dcb1ede177ba4", |
| "raw_result_sha256": "d79b8d114cb86c460dad13d564c2705a41df5119f83a0f1f27d452f36b64af47", |
| "kind": "direct_frozen_encoder_evaluation" |
| } |
| } |
| }, |
| { |
| "task": "CREMA_DClustering", |
| "type": "AudioClustering", |
| "modalities": [ |
| "audio" |
| ], |
| "split": "train", |
| "hf_subset": "default", |
| "main_metric": "v_measure", |
| "dataset": { |
| "path": "mteb/crema-d", |
| "revision": "a9d0821955c418752248b8310438854e56a1cf34" |
| }, |
| "counts": { |
| "official_stratified_subsample_rows": 2048, |
| "bootstrap_clustering_trials": 10, |
| "resampled_items_per_trial": 16384 |
| }, |
| "scores": { |
| "nano": 0.023601715918443712, |
| "small": 0.024058408888362488 |
| }, |
| "metrics": { |
| "nano": { |
| "v_measures": { |
| "Level 0": [ |
| 0.014290784661453301, |
| 0.02755216925549553, |
| 0.017220745995259155, |
| 0.02907041540588703, |
| 0.017365020267924067, |
| 0.022161217359367304, |
| 0.02806069377818825, |
| 0.017815432451092263, |
| 0.03081951157077731, |
| 0.03166116843899292 |
| ] |
| }, |
| "v_measure": 0.023601715918443712, |
| "v_measure_std": 0.006199268435462002, |
| "ami_scores": { |
| "Level 0": [ |
| 0.01385865484522556, |
| 0.027134003314929028, |
| 0.016785414361355624, |
| 0.028644278423611804, |
| 0.016937404067902156, |
| 0.02173522206231376, |
| 0.0276339699482267, |
| 0.0173943490723341, |
| 0.030397587303172737, |
| 0.031246814385090853 |
| ] |
| }, |
| "ami": 0.023176769778416234, |
| "ami_std": 0.0062032784119843, |
| "main_score": 0.023601715918443712, |
| "hf_subset": "default", |
| "languages": [ |
| "eng-Latn" |
| ], |
| "mteb_version": "2.21.0" |
| }, |
| "small": { |
| "v_measures": { |
| "Level 0": [ |
| 0.041011040847839235, |
| 0.03386543153714212, |
| 0.029099626845588235, |
| 0.013685755785810546, |
| 0.02873132169365058, |
| 0.01881126494637877, |
| 0.011416660820064256, |
| 0.02673877787517492, |
| 0.00569343536524053, |
| 0.031530773166735686 |
| ] |
| }, |
| "v_measure": 0.024058408888362488, |
| "v_measure_std": 0.010611552764041994, |
| "ami_scores": { |
| "Level 0": [ |
| 0.04059499123714819, |
| 0.03344615727359256, |
| 0.02867541136577102, |
| 0.01325228658253946, |
| 0.028312443838566275, |
| 0.01838242676310465, |
| 0.01099044807520909, |
| 0.026321334415061327, |
| 0.005259713019363485, |
| 0.031113076527253092 |
| ] |
| }, |
| "ami": 0.023634828909760913, |
| "ami_std": 0.010617195895208265, |
| "main_score": 0.024058408888362488, |
| "hf_subset": "default", |
| "languages": [ |
| "eng-Latn" |
| ], |
| "mteb_version": "2.21.0" |
| } |
| }, |
| "measurement_by_model": { |
| "small": { |
| "evaluated_native_fingerprint": "684fe0c48a4ddd3944167f6e5dc75e97ada24e5b120bd9a2921f00dc03112eeb", |
| "original_checkpoint_revision": "fdf8e01b7b0f3a69ac1ac8e2a64dcb1ede177ba4", |
| "raw_result_sha256": "b23d4c552b9fa242923fc0c21a898ee63eb32850d9d91e0682b2cb5e8bea0e76", |
| "kind": "direct_frozen_encoder_evaluation" |
| } |
| } |
| }, |
| { |
| "task": "SpeechCommandsZeroshotv0.02", |
| "type": "AudioZeroshotClassification", |
| "modalities": [ |
| "audio", |
| "text" |
| ], |
| "split": "test", |
| "hf_subset": "default", |
| "main_metric": "accuracy", |
| "dataset": { |
| "path": "mteb/SpeechCommandsZeroshotv0.02", |
| "revision": "d5491b288ab76356a5f3cbc634f1dc38e23da2b8" |
| }, |
| "counts": { |
| "evaluation_rows": 4074, |
| "fixed_text_labels": 10 |
| }, |
| "scores": { |
| "nano": 0.1411389297987236, |
| "small": 0.05522827687776141 |
| }, |
| "metrics": { |
| "nano": { |
| "accuracy": 0.1411389297987236, |
| "f1": 0.06629702019800886, |
| "f1_weighted": 0.06559730133350446, |
| "precision": 0.21924492134747023, |
| "precision_weighted": 0.21974351147386872, |
| "recall": 0.14280319771568778, |
| "recall_weighted": 0.1411389297987236, |
| "ap": null, |
| "ap_weighted": null, |
| "main_score": 0.1411389297987236, |
| "hf_subset": "default", |
| "languages": [ |
| "eng-Latn" |
| ], |
| "mteb_version": "2.21.0" |
| }, |
| "small": { |
| "accuracy": 0.05522827687776141, |
| "f1": 0.037875265984290724, |
| "f1_weighted": 0.03803483258988871, |
| "precision": 0.14410977979833245, |
| "precision_weighted": 0.143575354561912, |
| "recall": 0.05521141290348569, |
| "recall_weighted": 0.05522827687776141, |
| "ap": null, |
| "ap_weighted": null, |
| "main_score": 0.05522827687776141, |
| "hf_subset": "default", |
| "languages": [ |
| "eng-Latn" |
| ], |
| "mteb_version": "2.21.0" |
| } |
| }, |
| "measurement_by_model": { |
| "small": { |
| "evaluated_native_fingerprint": "684fe0c48a4ddd3944167f6e5dc75e97ada24e5b120bd9a2921f00dc03112eeb", |
| "original_checkpoint_revision": "fdf8e01b7b0f3a69ac1ac8e2a64dcb1ede177ba4", |
| "raw_result_sha256": "ca19c03bbe2fdcf3c2e1282f5e826af702c152287fde75d67b28ef877fae5335", |
| "kind": "direct_frozen_encoder_evaluation" |
| } |
| } |
| } |
| ], |
| "comparison": { |
| "baseline_model": "vllm-sr/multi-modal-embed-small", |
| "baseline_revision": "fdf8e01b7b0f3a69ac1ac8e2a64dcb1ede177ba4", |
| "baseline_native_artifact_sha256": "684fe0c48a4ddd3944167f6e5dc75e97ada24e5b120bd9a2921f00dc03112eeb", |
| "baseline_standard_status": "not_evaluated_in_this_standard_protocol", |
| "card_bold": "Only unrounded gains against this original baseline are bold; missing baseline scores imply no comparison." |
| }, |
| "observation_scope": "Only this model is reported; original comparator has no observations in the fixed11 protocol. Evaluated/applicable identities are preserved.", |
| "encoding_observations": { |
| "SICK-R": { |
| "nano": { |
| "before": { |
| "text_rows": 0, |
| "text_rows_truncated": 0, |
| "max_original_text_tokens": 0, |
| "image_rows": 0, |
| "audio_rows": 0 |
| }, |
| "after": { |
| "text_rows": 19854, |
| "text_rows_truncated": 0, |
| "max_original_text_tokens": 38, |
| "image_rows": 0, |
| "audio_rows": 0 |
| }, |
| "elapsed_seconds": 10.807591969991336, |
| "peak_allocated_gpu_bytes": 679632896, |
| "peak_reserved_gpu_bytes": 719323136 |
| } |
| }, |
| "STSBenchmark": { |
| "nano": { |
| "before": { |
| "text_rows": 19854, |
| "text_rows_truncated": 0, |
| "max_original_text_tokens": 38, |
| "image_rows": 0, |
| "audio_rows": 0 |
| }, |
| "after": { |
| "text_rows": 22612, |
| "text_rows_truncated": 0, |
| "max_original_text_tokens": 46, |
| "image_rows": 0, |
| "audio_rows": 0 |
| }, |
| "elapsed_seconds": 1.294741225981852, |
| "peak_allocated_gpu_bytes": 681261056, |
| "peak_reserved_gpu_bytes": 723517440 |
| } |
| }, |
| "ArguAna": { |
| "nano": { |
| "before": { |
| "text_rows": 22612, |
| "text_rows_truncated": 0, |
| "max_original_text_tokens": 46, |
| "image_rows": 0, |
| "audio_rows": 0 |
| }, |
| "after": { |
| "text_rows": 32692, |
| "text_rows_truncated": 222, |
| "max_original_text_tokens": 1401, |
| "image_rows": 0, |
| "audio_rows": 0 |
| }, |
| "elapsed_seconds": 15.392282173008425, |
| "peak_allocated_gpu_bytes": 809891328, |
| "peak_reserved_gpu_bytes": 887095296 |
| } |
| }, |
| "TwentyNewsgroupsClustering.v2": { |
| "nano": { |
| "before": { |
| "text_rows": 32692, |
| "text_rows_truncated": 222, |
| "max_original_text_tokens": 1401, |
| "image_rows": 0, |
| "audio_rows": 0 |
| }, |
| "after": { |
| "text_rows": 35073, |
| "text_rows_truncated": 222, |
| "max_original_text_tokens": 1401, |
| "image_rows": 0, |
| "audio_rows": 0 |
| }, |
| "elapsed_seconds": 48.54122133200872, |
| "peak_allocated_gpu_bytes": 684661248, |
| "peak_reserved_gpu_bytes": 727711744 |
| } |
| }, |
| "MTOPDomainClassification": { |
| "nano": { |
| "before": { |
| "text_rows": 35073, |
| "text_rows_truncated": 222, |
| "max_original_text_tokens": 1401, |
| "image_rows": 0, |
| "audio_rows": 0 |
| }, |
| "after": { |
| "text_rows": 43422, |
| "text_rows_truncated": 222, |
| "max_original_text_tokens": 1401, |
| "image_rows": 0, |
| "audio_rows": 0 |
| }, |
| "elapsed_seconds": 25.26324965900858, |
| "peak_allocated_gpu_bytes": 679308288, |
| "peak_reserved_gpu_bytes": 719323136 |
| } |
| }, |
| "OxfordPets": { |
| "nano": { |
| "before": { |
| "text_rows": 43422, |
| "text_rows_truncated": 222, |
| "max_original_text_tokens": 1401, |
| "image_rows": 0, |
| "audio_rows": 0 |
| }, |
| "after": { |
| "text_rows": 43422, |
| "text_rows_truncated": 222, |
| "max_original_text_tokens": 1401, |
| "image_rows": 5816, |
| "audio_rows": 0 |
| }, |
| "elapsed_seconds": 54.48569396699895, |
| "peak_allocated_gpu_bytes": 996472320, |
| "peak_reserved_gpu_bytes": 1207959552 |
| } |
| }, |
| "TinyImageNetClustering": { |
| "nano": { |
| "before": { |
| "text_rows": 43422, |
| "text_rows_truncated": 222, |
| "max_original_text_tokens": 1401, |
| "image_rows": 5816, |
| "audio_rows": 0 |
| }, |
| "after": { |
| "text_rows": 43422, |
| "text_rows_truncated": 222, |
| "max_original_text_tokens": 1401, |
| "image_rows": 15816, |
| "audio_rows": 0 |
| }, |
| "elapsed_seconds": 72.07803665398387, |
| "peak_allocated_gpu_bytes": 996472320, |
| "peak_reserved_gpu_bytes": 1117782016 |
| } |
| }, |
| "OxfordPetsZeroShot": { |
| "nano": { |
| "before": { |
| "text_rows": 43422, |
| "text_rows_truncated": 222, |
| "max_original_text_tokens": 1401, |
| "image_rows": 15816, |
| "audio_rows": 0 |
| }, |
| "after": { |
| "text_rows": 43459, |
| "text_rows_truncated": 222, |
| "max_original_text_tokens": 1401, |
| "image_rows": 19485, |
| "audio_rows": 0 |
| }, |
| "elapsed_seconds": 32.662016286019934, |
| "peak_allocated_gpu_bytes": 996472320, |
| "peak_reserved_gpu_bytes": 1119879168 |
| } |
| }, |
| "CREMA_D": { |
| "nano": { |
| "before": { |
| "text_rows": 43459, |
| "text_rows_truncated": 222, |
| "max_original_text_tokens": 1401, |
| "image_rows": 19485, |
| "audio_rows": 0 |
| }, |
| "after": { |
| "text_rows": 43459, |
| "text_rows_truncated": 222, |
| "max_original_text_tokens": 1401, |
| "image_rows": 19485, |
| "audio_rows": 7442 |
| }, |
| "elapsed_seconds": 135.41051389899803, |
| "peak_allocated_gpu_bytes": 971823104, |
| "peak_reserved_gpu_bytes": 1224736768 |
| } |
| }, |
| "CREMA_DClustering": { |
| "nano": { |
| "before": { |
| "text_rows": 43459, |
| "text_rows_truncated": 222, |
| "max_original_text_tokens": 1401, |
| "image_rows": 19485, |
| "audio_rows": 7442 |
| }, |
| "after": { |
| "text_rows": 43459, |
| "text_rows_truncated": 222, |
| "max_original_text_tokens": 1401, |
| "image_rows": 19485, |
| "audio_rows": 9490 |
| }, |
| "elapsed_seconds": 73.78414583200356, |
| "peak_allocated_gpu_bytes": 971823104, |
| "peak_reserved_gpu_bytes": 1224736768 |
| } |
| }, |
| "SpeechCommandsZeroshotv0.02": { |
| "nano": { |
| "before": { |
| "text_rows": 43459, |
| "text_rows_truncated": 222, |
| "max_original_text_tokens": 1401, |
| "image_rows": 19485, |
| "audio_rows": 9490 |
| }, |
| "after": { |
| "text_rows": 43469, |
| "text_rows_truncated": 222, |
| "max_original_text_tokens": 1401, |
| "image_rows": 19485, |
| "audio_rows": 13564 |
| }, |
| "elapsed_seconds": 63.873327202978544, |
| "peak_allocated_gpu_bytes": 971823104, |
| "peak_reserved_gpu_bytes": 1224736768 |
| } |
| } |
| } |
| }, |
| "capabilities": { |
| "max_text_tokens": 512, |
| "tokens_include_special_tokens": true, |
| "public_text_overflow": "Raises ValueError; shorten or explicitly chunk longer inputs.", |
| "embedding_dimensions": 384, |
| "parameters": 135383808 |
| }, |
| "efficiency": { |
| "status": "not_measured_for_this_release", |
| "parameters": 135383808, |
| "embedding_dimensions": 384, |
| "dense_fp32_index_bytes_per_vector": 1536, |
| "weights_safetensors_bytes": 541598024, |
| "scope": "No new latency or peak-memory measurement is claimed. Vector storage excludes index structures and metadata.", |
| "historical_measurement": { |
| "attention": "native SDPA", |
| "batch_size": 8, |
| "embedding_dtype": "float32", |
| "hardware": "AMD gfx942 GPU", |
| "latency_includes": "Native public encode_text: tokenization, host-to-device transfer and encoding; excludes model loading.", |
| "measured_batches": 20, |
| "measurements": { |
| "128": { |
| "batch_size": 8, |
| "p50_ms": 4.283615489839576, |
| "p95_ms": 4.314010989037342, |
| "peak_allocated_bytes": 653202944, |
| "peak_reserved_bytes": 673185792, |
| "tokens_each_including_special": 128 |
| }, |
| "512": { |
| "batch_size": 8, |
| "p50_ms": 12.445501997717656, |
| "p95_ms": 12.603181986196432, |
| "peak_allocated_bytes": 878682112, |
| "peak_reserved_bytes": 970981376, |
| "tokens_each_including_special": 512 |
| } |
| }, |
| "memory_scope": "Process CUDA allocator peak with the complete multimodal model loaded.", |
| "model_revision": "70c3412bb0ddfb19386d673d4a102c753a574f1b", |
| "native_artifact_sha256": "b596f1dbab89369555e0a72a0d6854d780e305686750949b308c8f167993bdeb", |
| "scope": "Synthetic repeated text-batch measurements on one GPU; not end-to-end service latency.", |
| "tf32": false, |
| "token_lengths_include_special_tokens": true, |
| "torch_version_family": "2.12", |
| "transformers_version": "4.57.6", |
| "warmup_batches": 2, |
| "status": "historical_previous_release_measurement", |
| "applicability": "Historical measurement on its recorded native artifact; not applicable to this text-backbone update." |
| }, |
| "parameter_storage_change": { |
| "old_safetensors_bytes": 578289664, |
| "new_safetensors_bytes": 541598024, |
| "reduction_bytes": 36691640, |
| "old_FP32_parameter_bytes": 578216448, |
| "new_FP32_parameter_bytes": 541535232, |
| "scope": "Parameter storage only; not measured process peak memory or latency. Vector and index dimensions are unchanged." |
| } |
| }, |
| "previous_release": { |
| "model": "vllm-sr/Vela-1.0-Omni-Nano", |
| "revision": "3a1efc48995e2b71622d65c6cf71cfc00bbf3aa3", |
| "native_artifact_sha256": "fb9cd39c9980a085b75456c715c2295b997784fe733314be68c13635f6384edf", |
| "parameters": 133956480, |
| "metrics": { |
| "audio.audio_to_text.recall@1": { |
| "denominator": 2611, |
| "numerator": 235, |
| "value": 0.09000382995021065 |
| }, |
| "audio.audio_to_text.recall@10": { |
| "denominator": 2611, |
| "numerator": 822, |
| "value": 0.31482190731520493 |
| }, |
| "audio.audio_to_text.recall@5": { |
| "denominator": 2611, |
| "numerator": 582, |
| "value": 0.22290310225967064 |
| }, |
| "audio.text_to_audio.recall@1": { |
| "denominator": 2610, |
| "numerator": 366, |
| "value": 0.14022988505747128 |
| }, |
| "audio.text_to_audio.recall@10": { |
| "denominator": 2610, |
| "numerator": 1055, |
| "value": 0.4042145593869732 |
| }, |
| "audio.text_to_audio.recall@5": { |
| "denominator": 2610, |
| "numerator": 827, |
| "value": 0.3168582375478927 |
| }, |
| "image.image_to_text.recall@1": { |
| "denominator": 823, |
| "numerator": 540, |
| "value": 0.6561360874848117 |
| }, |
| "image.image_to_text.recall@10": { |
| "denominator": 823, |
| "numerator": 791, |
| "value": 0.9611178614823815 |
| }, |
| "image.image_to_text.recall@5": { |
| "denominator": 823, |
| "numerator": 731, |
| "value": 0.8882138517618469 |
| }, |
| "image.text_to_image.recall@1": { |
| "denominator": 4115, |
| "numerator": 2184, |
| "value": 0.5307411907654921 |
| }, |
| "image.text_to_image.recall@10": { |
| "denominator": 4115, |
| "numerator": 3847, |
| "value": 0.934872417982989 |
| }, |
| "image.text_to_image.recall@5": { |
| "denominator": 4115, |
| "numerator": 3465, |
| "value": 0.8420413122721749 |
| }, |
| "text.banking77.accuracy": { |
| "denominator": 3080, |
| "numerator": 2366, |
| "value": 0.7681818181818182 |
| }, |
| "text.massive-en.accuracy": { |
| "denominator": 2972, |
| "numerator": 2061, |
| "value": 0.6934724091520862 |
| } |
| }, |
| "descriptive_metrics": { |
| "text.banking77.macro_f1": { |
| "value": 0.7658421887247635, |
| "classes": 77, |
| "queries": 3080 |
| }, |
| "text.massive-en.macro_f1": { |
| "value": 0.6524376423162815, |
| "classes": 60, |
| "queries": 2972 |
| } |
| }, |
| "standard_rows": [ |
| { |
| "task": "SICK-R", |
| "split": "test", |
| "hf_subset": "default", |
| "main_metric": "cosine_spearman", |
| "score": 0.7169618925026362, |
| "metrics": { |
| "pearson": 0.8194169233518556, |
| "spearman": 0.7169617707661863, |
| "cosine_pearson": 0.819416922085493, |
| "cosine_spearman": 0.7169618925026362, |
| "manhattan_pearson": 0.7935695063285152, |
| "manhattan_spearman": 0.7160084195066224, |
| "euclidean_pearson": 0.7944328514895539, |
| "euclidean_spearman": 0.7169617703551949, |
| "main_score": 0.7169618925026362, |
| "hf_subset": "default", |
| "languages": [ |
| "eng-Latn" |
| ], |
| "mteb_version": "2.21.0" |
| } |
| }, |
| { |
| "task": "STSBenchmark", |
| "split": "test", |
| "hf_subset": "default", |
| "main_metric": "cosine_spearman", |
| "score": 0.7792400226361769, |
| "metrics": { |
| "pearson": 0.7717062133633645, |
| "spearman": 0.7792400217447207, |
| "cosine_pearson": 0.7717062156025293, |
| "cosine_spearman": 0.7792400226361769, |
| "manhattan_pearson": 0.7928914389432686, |
| "manhattan_spearman": 0.7777756255094064, |
| "euclidean_pearson": 0.7949414830254274, |
| "euclidean_spearman": 0.779239966513648, |
| "main_score": 0.7792400226361769, |
| "hf_subset": "default", |
| "languages": [ |
| "eng-Latn" |
| ], |
| "mteb_version": "2.21.0" |
| } |
| }, |
| { |
| "task": "ArguAna", |
| "split": "test", |
| "hf_subset": "default", |
| "main_metric": "ndcg_at_10", |
| "score": 0.37514, |
| "metrics": { |
| "ndcg_at_1": 0.17425, |
| "ndcg_at_3": 0.27012, |
| "ndcg_at_5": 0.31718, |
| "ndcg_at_10": 0.37514, |
| "ndcg_at_20": 0.40855, |
| "ndcg_at_100": 0.43759, |
| "ndcg_at_1000": 0.44701, |
| "map_at_1": 0.17425, |
| "map_at_3": 0.24561, |
| "map_at_5": 0.2715, |
| "map_at_10": 0.29556, |
| "map_at_20": 0.30489, |
| "map_at_100": 0.3093, |
| "map_at_1000": 0.30967, |
| "recall_at_1": 0.17425, |
| "recall_at_3": 0.34139, |
| "recall_at_5": 0.45661, |
| "recall_at_10": 0.63514, |
| "recall_at_20": 0.766, |
| "recall_at_100": 0.91679, |
| "recall_at_1000": 0.99004, |
| "accuracy": 0.17425, |
| "precision_at_1": 0.17425, |
| "precision_at_3": 0.1138, |
| "precision_at_5": 0.09132, |
| "precision_at_10": 0.06351, |
| "precision_at_20": 0.0383, |
| "precision_at_100": 0.00917, |
| "precision_at_1000": 0.00099, |
| "mrr_at_1": 0.17425320056899005, |
| "mrr_at_3": 0.24561403508771928, |
| "mrr_at_5": 0.2715030820293978, |
| "mrr_at_10": 0.2955561764772291, |
| "mrr_at_20": 0.30489450370601584, |
| "mrr_at_100": 0.3092982324964941, |
| "mrr_at_1000": 0.30966870165383564, |
| "nauc_ndcg_at_1_max": -0.012854409302840068, |
| "nauc_ndcg_at_1_std": 0.08676381700185752, |
| "nauc_ndcg_at_1_diff1": 0.11334791281377288, |
| "nauc_ndcg_at_3_max": 0.0007934267314863777, |
| "nauc_ndcg_at_3_std": 0.05484641820257593, |
| "nauc_ndcg_at_3_diff1": 0.07348303062976835, |
| "nauc_ndcg_at_5_max": 0.020607224558072373, |
| "nauc_ndcg_at_5_std": 0.0741284488966682, |
| "nauc_ndcg_at_5_diff1": 0.08730888772052957, |
| "nauc_ndcg_at_10_max": 0.035048132350077, |
| "nauc_ndcg_at_10_std": 0.10523053259120095, |
| "nauc_ndcg_at_10_diff1": 0.08020955970997308, |
| "nauc_ndcg_at_20_max": 0.04122292301118309, |
| "nauc_ndcg_at_20_std": 0.12537385022454034, |
| "nauc_ndcg_at_20_diff1": 0.0842700247341082, |
| "nauc_ndcg_at_100_max": 0.04333008607419203, |
| "nauc_ndcg_at_100_std": 0.12294409372370381, |
| "nauc_ndcg_at_100_diff1": 0.08845423517529309, |
| "nauc_ndcg_at_1000_max": 0.030127261370811845, |
| "nauc_ndcg_at_1000_std": 0.10766365950024334, |
| "nauc_ndcg_at_1000_diff1": 0.0861546357811246, |
| "nauc_map_at_1_max": -0.012854409302840068, |
| "nauc_map_at_1_std": 0.08676381700185752, |
| "nauc_map_at_1_diff1": 0.11334791281377288, |
| "nauc_map_at_3_max": -0.0029405293797002733, |
| "nauc_map_at_3_std": 0.06167764879997172, |
| "nauc_map_at_3_diff1": 0.07950290715235987, |
| "nauc_map_at_5_max": 0.00830536326209409, |
| "nauc_map_at_5_std": 0.07256833511730693, |
| "nauc_map_at_5_diff1": 0.08747148302006987, |
| "nauc_map_at_10_max": 0.015140286093876175, |
| "nauc_map_at_10_std": 0.08580662522686537, |
| "nauc_map_at_10_diff1": 0.08508762518017199, |
| "nauc_map_at_20_max": 0.0163028127500422, |
| "nauc_map_at_20_std": 0.09050711291668995, |
| "nauc_map_at_20_diff1": 0.08629712373980958, |
| "nauc_map_at_100_max": 0.016183246425002705, |
| "nauc_map_at_100_std": 0.09000154689011092, |
| "nauc_map_at_100_diff1": 0.08699050038257111, |
| "nauc_map_at_1000_max": 0.015701662616224514, |
| "nauc_map_at_1000_std": 0.0894576065792463, |
| "nauc_map_at_1000_diff1": 0.08694453879028893, |
| "nauc_recall_at_1_max": -0.012854409302840068, |
| "nauc_recall_at_1_std": 0.08676381700185752, |
| "nauc_recall_at_1_diff1": 0.11334791281377288, |
| "nauc_recall_at_3_max": 0.010338756866368223, |
| "nauc_recall_at_3_std": 0.03789278134763109, |
| "nauc_recall_at_3_diff1": 0.059403567851730316, |
| "nauc_recall_at_5_max": 0.05398176066498534, |
| "nauc_recall_at_5_std": 0.08064232472596307, |
| "nauc_recall_at_5_diff1": 0.08949514479783123, |
| "nauc_recall_at_10_max": 0.09984307699040616, |
| "nauc_recall_at_10_std": 0.17703668336587267, |
| "nauc_recall_at_10_diff1": 0.06498907396830537, |
| "nauc_recall_at_20_max": 0.15799758987163673, |
| "nauc_recall_at_20_std": 0.31157412853790695, |
| "nauc_recall_at_20_diff1": 0.07941702046000772, |
| "nauc_recall_at_100_max": 0.37265902510843735, |
| "nauc_recall_at_100_std": 0.5476974790176777, |
| "nauc_recall_at_100_diff1": 0.12977852219557848, |
| "nauc_recall_at_1000_max": 0.3004922701567, |
| "nauc_recall_at_1000_std": 0.7475055455618029, |
| "nauc_recall_at_1000_diff1": -0.05500901218402981, |
| "nauc_precision_at_1_max": -0.012854409302840068, |
| "nauc_precision_at_1_std": 0.08676381700185752, |
| "nauc_precision_at_1_diff1": 0.11334791281377288, |
| "nauc_precision_at_3_max": 0.010338756866368036, |
| "nauc_precision_at_3_std": 0.037892781347630994, |
| "nauc_precision_at_3_diff1": 0.05940356785173012, |
| "nauc_precision_at_5_max": 0.05398176066498592, |
| "nauc_precision_at_5_std": 0.08064232472596324, |
| "nauc_precision_at_5_diff1": 0.08949514479783155, |
| "nauc_precision_at_10_max": 0.09984307699040575, |
| "nauc_precision_at_10_std": 0.17703668336587192, |
| "nauc_precision_at_10_diff1": 0.06498907396830521, |
| "nauc_precision_at_20_max": 0.1579975898716373, |
| "nauc_precision_at_20_std": 0.31157412853790756, |
| "nauc_precision_at_20_diff1": 0.0794170204600081, |
| "nauc_precision_at_100_max": 0.3726590251084365, |
| "nauc_precision_at_100_std": 0.5476974790176794, |
| "nauc_precision_at_100_diff1": 0.1297785221955785, |
| "nauc_precision_at_1000_max": 0.3004922701567069, |
| "nauc_precision_at_1000_std": 0.7475055455618104, |
| "nauc_precision_at_1000_diff1": -0.055009012184042826, |
| "nauc_mrr_at_1_max": -0.012854409302840068, |
| "nauc_mrr_at_1_std": 0.08676381700185752, |
| "nauc_mrr_at_1_diff1": 0.11334791281377288, |
| "nauc_mrr_at_3_max": -0.0029405293797002733, |
| "nauc_mrr_at_3_std": 0.06167764879997172, |
| "nauc_mrr_at_3_diff1": 0.07950290715235987, |
| "nauc_mrr_at_5_max": 0.00830536326209409, |
| "nauc_mrr_at_5_std": 0.07256833511730693, |
| "nauc_mrr_at_5_diff1": 0.08747148302006987, |
| "nauc_mrr_at_10_max": 0.015140286093876175, |
| "nauc_mrr_at_10_std": 0.08580662522686537, |
| "nauc_mrr_at_10_diff1": 0.08508762518017199, |
| "nauc_mrr_at_20_max": 0.0163028127500422, |
| "nauc_mrr_at_20_std": 0.09050711291668995, |
| "nauc_mrr_at_20_diff1": 0.08629712373980958, |
| "nauc_mrr_at_100_max": 0.016183246425002705, |
| "nauc_mrr_at_100_std": 0.09000154689011092, |
| "nauc_mrr_at_100_diff1": 0.08699050038257111, |
| "nauc_mrr_at_1000_max": 0.015701662616224514, |
| "nauc_mrr_at_1000_std": 0.0894576065792463, |
| "nauc_mrr_at_1000_diff1": 0.08694453879028893, |
| "hit_rate_at_1": 0.17425, |
| "hit_rate_at_3": 0.34139, |
| "hit_rate_at_5": 0.45661, |
| "hit_rate_at_10": 0.63514, |
| "hit_rate_at_20": 0.766, |
| "hit_rate_at_100": 0.91679, |
| "hit_rate_at_1000": 0.99004, |
| "main_score": 0.37514, |
| "hf_subset": "default", |
| "languages": [ |
| "eng-Latn" |
| ], |
| "mteb_version": "2.21.0" |
| } |
| }, |
| { |
| "task": "TwentyNewsgroupsClustering.v2", |
| "split": "test", |
| "hf_subset": "default", |
| "main_metric": "v_measure", |
| "score": 0.44884717457494344, |
| "metrics": { |
| "v_measures": { |
| "Level 0": [ |
| 0.48056098748371134, |
| 0.44085942767025554, |
| 0.43416151810540005, |
| 0.45299559752505, |
| 0.44116051339311235, |
| 0.4409873172673812, |
| 0.46155584917327236, |
| 0.444819328389149, |
| 0.44792019268133576, |
| 0.44345101406076604 |
| ] |
| }, |
| "v_measure": 0.44884717457494344, |
| "v_measure_std": 0.01277114663774588, |
| "ami_scores": { |
| "Level 0": [ |
| 0.4786019649327535, |
| 0.43870317373726253, |
| 0.43201951778515824, |
| 0.4509348225877084, |
| 0.43904842199622773, |
| 0.43887546309559994, |
| 0.45953222355170814, |
| 0.44270204450063555, |
| 0.44583346842612326, |
| 0.4413500450061514 |
| ] |
| }, |
| "ami": 0.4467601145619328, |
| "ami_std": 0.012825331450796505, |
| "main_score": 0.44884717457494344, |
| "hf_subset": "default", |
| "languages": [ |
| "eng-Latn" |
| ], |
| "mteb_version": "2.21.0" |
| } |
| }, |
| { |
| "task": "MTOPDomainClassification", |
| "split": "validation", |
| "hf_subset": "en", |
| "main_metric": "accuracy", |
| "score": 0.8758836689038031, |
| "metrics": { |
| "scores_per_experiment": [ |
| { |
| "accuracy": 0.8527964205816555, |
| "f1": 0.8584702959865076, |
| "f1_weighted": 0.8520037076109549, |
| "precision": 0.8563814369311937, |
| "precision_weighted": 0.8604397214633738, |
| "recall": 0.8688240583945851, |
| "recall_weighted": 0.8527964205816555, |
| "ap": null, |
| "ap_weighted": null |
| }, |
| { |
| "accuracy": 0.8675615212527964, |
| "f1": 0.8658660043655293, |
| "f1_weighted": 0.8661413651963479, |
| "precision": 0.8632122218181354, |
| "precision_weighted": 0.8687564991167056, |
| "recall": 0.8733845956286541, |
| "recall_weighted": 0.8675615212527964, |
| "ap": null, |
| "ap_weighted": null |
| }, |
| { |
| "accuracy": 0.8514541387024609, |
| "f1": 0.8524202289070963, |
| "f1_weighted": 0.8491348169886809, |
| "precision": 0.8478156770106394, |
| "precision_weighted": 0.8512267923319679, |
| "recall": 0.8621133428009895, |
| "recall_weighted": 0.8514541387024609, |
| "ap": null, |
| "ap_weighted": null |
| }, |
| { |
| "accuracy": 0.8876957494407158, |
| "f1": 0.8889547651830694, |
| "f1_weighted": 0.886393441219644, |
| "precision": 0.8855262149097257, |
| "precision_weighted": 0.8869170708224573, |
| "recall": 0.8941464114507656, |
| "recall_weighted": 0.8876957494407158, |
| "ap": null, |
| "ap_weighted": null |
| }, |
| { |
| "accuracy": 0.8684563758389262, |
| "f1": 0.8708980361567344, |
| "f1_weighted": 0.8668343457695604, |
| "precision": 0.8655644216656676, |
| "precision_weighted": 0.8685796219847712, |
| "recall": 0.8789277362563833, |
| "recall_weighted": 0.8684563758389262, |
| "ap": null, |
| "ap_weighted": null |
| }, |
| { |
| "accuracy": 0.8662192393736018, |
| "f1": 0.8725560012982636, |
| "f1_weighted": 0.8665613320034511, |
| "precision": 0.8737258252820442, |
| "precision_weighted": 0.8845738920578841, |
| "recall": 0.8863222925022911, |
| "recall_weighted": 0.8662192393736018, |
| "ap": null, |
| "ap_weighted": null |
| }, |
| { |
| "accuracy": 0.8953020134228188, |
| "f1": 0.8976332709955689, |
| "f1_weighted": 0.8945899154754817, |
| "precision": 0.8912735415957905, |
| "precision_weighted": 0.8979949591784757, |
| "recall": 0.908092721226657, |
| "recall_weighted": 0.8953020134228188, |
| "ap": null, |
| "ap_weighted": null |
| }, |
| { |
| "accuracy": 0.9029082774049217, |
| "f1": 0.9022310464802398, |
| "f1_weighted": 0.9020525018623339, |
| "precision": 0.899983538025133, |
| "precision_weighted": 0.9036775461946326, |
| "recall": 0.9067592943176375, |
| "recall_weighted": 0.9029082774049217, |
| "ap": null, |
| "ap_weighted": null |
| }, |
| { |
| "accuracy": 0.876510067114094, |
| "f1": 0.8776887066559357, |
| "f1_weighted": 0.8761842615927375, |
| "precision": 0.8722809982436143, |
| "precision_weighted": 0.8829904469624968, |
| "recall": 0.8888620916111349, |
| "recall_weighted": 0.876510067114094, |
| "ap": null, |
| "ap_weighted": null |
| }, |
| { |
| "accuracy": 0.8899328859060402, |
| "f1": 0.8916131163462019, |
| "f1_weighted": 0.8885446252332447, |
| "precision": 0.8855482140891268, |
| "precision_weighted": 0.8923990752693544, |
| "recall": 0.9020276692969261, |
| "recall_weighted": 0.8899328859060402, |
| "ap": null, |
| "ap_weighted": null |
| } |
| ], |
| "accuracy": 0.8758836689038031, |
| "f1": 0.8778331472375147, |
| "f1_weighted": 0.8748440312952436, |
| "precision": 0.8741312089571069, |
| "precision_weighted": 0.8797555625382119, |
| "recall": 0.8869460213486026, |
| "recall_weighted": 0.8758836689038031, |
| "ap": null, |
| "ap_weighted": null, |
| "main_score": 0.8758836689038031, |
| "hf_subset": "en", |
| "languages": [ |
| "eng-Latn" |
| ], |
| "mteb_version": "2.21.0" |
| } |
| }, |
| { |
| "task": "MTOPDomainClassification", |
| "split": "test", |
| "hf_subset": "en", |
| "main_metric": "accuracy", |
| "score": 0.8671910624715002, |
| "metrics": { |
| "scores_per_experiment": [ |
| { |
| "accuracy": 0.8492932056543547, |
| "f1": 0.8506491551806685, |
| "f1_weighted": 0.8484338913731928, |
| "precision": 0.847171519646365, |
| "precision_weighted": 0.857621901965629, |
| "recall": 0.8634426315569471, |
| "recall_weighted": 0.8492932056543547, |
| "ap": null, |
| "ap_weighted": null |
| }, |
| { |
| "accuracy": 0.8593251253989969, |
| "f1": 0.8525570927270549, |
| "f1_weighted": 0.8588269042231532, |
| "precision": 0.849564509883911, |
| "precision_weighted": 0.8659425848467468, |
| "recall": 0.8649266896842414, |
| "recall_weighted": 0.8593251253989969, |
| "ap": null, |
| "ap_weighted": null |
| }, |
| { |
| "accuracy": 0.8508891928864569, |
| "f1": 0.8445077740960396, |
| "f1_weighted": 0.8500136039882616, |
| "precision": 0.83976876536009, |
| "precision_weighted": 0.8551546537983608, |
| "recall": 0.8573297788521569, |
| "recall_weighted": 0.8508891928864569, |
| "ap": null, |
| "ap_weighted": null |
| }, |
| { |
| "accuracy": 0.8702690378476973, |
| "f1": 0.8680982220684261, |
| "f1_weighted": 0.8688890737378698, |
| "precision": 0.8651925230157509, |
| "precision_weighted": 0.8699395458498034, |
| "recall": 0.873034481111479, |
| "recall_weighted": 0.8702690378476973, |
| "ap": null, |
| "ap_weighted": null |
| }, |
| { |
| "accuracy": 0.8657090743274054, |
| "f1": 0.8639593934313017, |
| "f1_weighted": 0.8646291417790959, |
| "precision": 0.8570893217526883, |
| "precision_weighted": 0.8672508842970594, |
| "recall": 0.8739690035000478, |
| "recall_weighted": 0.8657090743274054, |
| "ap": null, |
| "ap_weighted": null |
| }, |
| { |
| "accuracy": 0.8520291837665299, |
| "f1": 0.8534418954962738, |
| "f1_weighted": 0.8548114072639492, |
| "precision": 0.85875145236006, |
| "precision_weighted": 0.8803115046026259, |
| "recall": 0.869567932536695, |
| "recall_weighted": 0.8520291837665299, |
| "ap": null, |
| "ap_weighted": null |
| }, |
| { |
| "accuracy": 0.8869129046967624, |
| "f1": 0.8849494340089117, |
| "f1_weighted": 0.8871874824644057, |
| "precision": 0.8779834780597185, |
| "precision_weighted": 0.8955288700582439, |
| "recall": 0.9009013020211545, |
| "recall_weighted": 0.8869129046967624, |
| "ap": null, |
| "ap_weighted": null |
| }, |
| { |
| "accuracy": 0.875968992248062, |
| "f1": 0.8715420414043028, |
| "f1_weighted": 0.8753897998685723, |
| "precision": 0.86652024620862, |
| "precision_weighted": 0.8811614860860878, |
| "recall": 0.881802359118481, |
| "recall_weighted": 0.875968992248062, |
| "ap": null, |
| "ap_weighted": null |
| }, |
| { |
| "accuracy": 0.8755129958960328, |
| "f1": 0.8710791039688399, |
| "f1_weighted": 0.8760974709416987, |
| "precision": 0.8647173948263537, |
| "precision_weighted": 0.8831121050674345, |
| "recall": 0.8845140142042784, |
| "recall_weighted": 0.8755129958960328, |
| "ap": null, |
| "ap_weighted": null |
| }, |
| { |
| "accuracy": 0.886000911992704, |
| "f1": 0.8827257199119498, |
| "f1_weighted": 0.8855721338303499, |
| "precision": 0.8760471482914741, |
| "precision_weighted": 0.8912159468319844, |
| "recall": 0.8952867706565555, |
| "recall_weighted": 0.886000911992704, |
| "ap": null, |
| "ap_weighted": null |
| } |
| ], |
| "accuracy": 0.8671910624715002, |
| "f1": 0.8643509832293768, |
| "f1_weighted": 0.8669850909470549, |
| "precision": 0.8602806359405031, |
| "precision_weighted": 0.8747239483403975, |
| "recall": 0.8764774963242037, |
| "recall_weighted": 0.8671910624715002, |
| "ap": null, |
| "ap_weighted": null, |
| "main_score": 0.8671910624715002, |
| "hf_subset": "en", |
| "languages": [ |
| "eng-Latn" |
| ], |
| "mteb_version": "2.21.0" |
| } |
| }, |
| { |
| "task": "OxfordPets", |
| "split": "test", |
| "hf_subset": "default", |
| "main_metric": "accuracy", |
| "score": 0.9126737530662306, |
| "metrics": { |
| "scores_per_experiment": [ |
| { |
| "accuracy": 0.9062414826928318, |
| "f1": 0.9045301316021502, |
| "f1_weighted": 0.9050055409371512, |
| "precision": 0.9099213291951324, |
| "precision_weighted": 0.9109544817876485, |
| "recall": 0.9064295837897134, |
| "recall_weighted": 0.9062414826928318, |
| "ap": null, |
| "ap_weighted": null |
| }, |
| { |
| "accuracy": 0.914418097574271, |
| "f1": 0.9126724638115935, |
| "f1_weighted": 0.9132186480967779, |
| "precision": 0.9192904646518116, |
| "precision_weighted": 0.9203471940294945, |
| "recall": 0.9144593228002407, |
| "recall_weighted": 0.914418097574271, |
| "ap": null, |
| "ap_weighted": null |
| }, |
| { |
| "accuracy": 0.9187789588443718, |
| "f1": 0.9177765846692281, |
| "f1_weighted": 0.9184111765200857, |
| "precision": 0.9220755461580387, |
| "precision_weighted": 0.9229278273528245, |
| "recall": 0.9183739103921627, |
| "recall_weighted": 0.9187789588443718, |
| "ap": null, |
| "ap_weighted": null |
| }, |
| { |
| "accuracy": 0.9174161896974653, |
| "f1": 0.9147622354932347, |
| "f1_weighted": 0.9154663340384108, |
| "precision": 0.9209055798643112, |
| "precision_weighted": 0.9221124471295344, |
| "recall": 0.9173606715378323, |
| "recall_weighted": 0.9174161896974653, |
| "ap": null, |
| "ap_weighted": null |
| }, |
| { |
| "accuracy": 0.9065140365222132, |
| "f1": 0.9051669100036687, |
| "f1_weighted": 0.9057728210054364, |
| "precision": 0.9098426973534027, |
| "precision_weighted": 0.9105143284060627, |
| "recall": 0.9059560197061645, |
| "recall_weighted": 0.9065140365222132, |
| "ap": null, |
| "ap_weighted": null |
| } |
| ], |
| "accuracy": 0.9126737530662306, |
| "f1": 0.910981665115975, |
| "f1_weighted": 0.9115749041195725, |
| "precision": 0.9164071234445392, |
| "precision_weighted": 0.9173712557411129, |
| "recall": 0.9125159016452227, |
| "recall_weighted": 0.9126737530662306, |
| "ap": null, |
| "ap_weighted": null, |
| "main_score": 0.9126737530662306, |
| "hf_subset": "default", |
| "languages": [ |
| "eng-Latn" |
| ], |
| "mteb_version": "2.21.0" |
| } |
| }, |
| { |
| "task": "TinyImageNetClustering", |
| "split": "valid", |
| "hf_subset": "default", |
| "main_metric": "nmi", |
| "score": 0.6169193395626786, |
| "metrics": { |
| "v_measure": 0.6169193395626787, |
| "nmi": 0.6169193395626786, |
| "ari": 0.21497891543638262, |
| "cluster_accuracy": 0.3638, |
| "main_score": 0.6169193395626786, |
| "hf_subset": "default", |
| "languages": [ |
| "eng-Latn" |
| ], |
| "mteb_version": "2.21.0" |
| } |
| }, |
| { |
| "task": "OxfordPetsZeroShot", |
| "split": "test", |
| "hf_subset": "default", |
| "main_metric": "accuracy", |
| "score": 0.1163804851458163, |
| "metrics": { |
| "accuracy": 0.1163804851458163, |
| "f1": 0.085822378073852, |
| "f1_weighted": 0.08530524305774022, |
| "precision": 0.08885688520073651, |
| "precision_weighted": 0.08856851170630324, |
| "recall": 0.11709192030781942, |
| "recall_weighted": 0.1163804851458163, |
| "ap": null, |
| "ap_weighted": null, |
| "main_score": 0.1163804851458163, |
| "hf_subset": "default", |
| "languages": [ |
| "eng-Latn" |
| ], |
| "mteb_version": "2.21.0" |
| } |
| }, |
| { |
| "task": "CREMA_D", |
| "split": "train", |
| "hf_subset": "default", |
| "main_metric": "accuracy", |
| "score": 0.29763670140167686, |
| "metrics": { |
| "scores_per_experiment": [ |
| { |
| "accuracy": 0.2847548690396239, |
| "f1": 0.25980556646698677, |
| "f1_weighted": 0.2606862265023784, |
| "precision": 0.2685141966762838, |
| "precision_weighted": 0.26869432571348717, |
| "recall": 0.283361264324777, |
| "recall_weighted": 0.2847548690396239, |
| "ap": null, |
| "ap_weighted": null |
| }, |
| { |
| "accuracy": 0.2981867024848892, |
| "f1": 0.2927999429509691, |
| "f1_weighted": 0.2952410831252662, |
| "precision": 0.29988591178720175, |
| "precision_weighted": 0.30336098208055257, |
| "recall": 0.29763527945216217, |
| "recall_weighted": 0.2981867024848892, |
| "ap": null, |
| "ap_weighted": null |
| }, |
| { |
| "accuracy": 0.28696236559139787, |
| "f1": 0.26582489649494206, |
| "f1_weighted": 0.26920117230368773, |
| "precision": 0.26489987169540286, |
| "precision_weighted": 0.2682170436584329, |
| "recall": 0.2834616565816636, |
| "recall_weighted": 0.28696236559139787, |
| "ap": null, |
| "ap_weighted": null |
| }, |
| { |
| "accuracy": 0.29973118279569894, |
| "f1": 0.291485315935673, |
| "f1_weighted": 0.28841683151437103, |
| "precision": 0.3068565359099591, |
| "precision_weighted": 0.3086441612810119, |
| "recall": 0.30801945566008065, |
| "recall_weighted": 0.29973118279569894, |
| "ap": null, |
| "ap_weighted": null |
| }, |
| { |
| "accuracy": 0.3185483870967742, |
| "f1": 0.3051858414398825, |
| "f1_weighted": 0.3066161074219095, |
| "precision": 0.310702613862649, |
| "precision_weighted": 0.3133755862555043, |
| "recall": 0.31849511838471833, |
| "recall_weighted": 0.3185483870967742, |
| "ap": null, |
| "ap_weighted": null |
| } |
| ], |
| "accuracy": 0.29763670140167686, |
| "f1": 0.2830203126576907, |
| "f1_weighted": 0.2840322841735226, |
| "precision": 0.29017182598629926, |
| "precision_weighted": 0.2924584197977978, |
| "recall": 0.2981945548806803, |
| "recall_weighted": 0.29763670140167686, |
| "ap": null, |
| "ap_weighted": null, |
| "main_score": 0.29763670140167686, |
| "hf_subset": "default", |
| "languages": [ |
| "eng-Latn" |
| ], |
| "mteb_version": "2.21.0" |
| } |
| }, |
| { |
| "task": "CREMA_DClustering", |
| "split": "train", |
| "hf_subset": "default", |
| "main_metric": "v_measure", |
| "score": 0.023601715918443712, |
| "metrics": { |
| "v_measures": { |
| "Level 0": [ |
| 0.014290784661453301, |
| 0.02755216925549553, |
| 0.017220745995259155, |
| 0.02907041540588703, |
| 0.017365020267924067, |
| 0.022161217359367304, |
| 0.02806069377818825, |
| 0.017815432451092263, |
| 0.03081951157077731, |
| 0.03166116843899292 |
| ] |
| }, |
| "v_measure": 0.023601715918443712, |
| "v_measure_std": 0.006199268435462002, |
| "ami_scores": { |
| "Level 0": [ |
| 0.01385865484522556, |
| 0.027134003314929028, |
| 0.016785414361355624, |
| 0.028644278423611804, |
| 0.016937404067902156, |
| 0.02173522206231376, |
| 0.0276339699482267, |
| 0.0173943490723341, |
| 0.030397587303172737, |
| 0.031246814385090853 |
| ] |
| }, |
| "ami": 0.023176769778416234, |
| "ami_std": 0.0062032784119843, |
| "main_score": 0.023601715918443712, |
| "hf_subset": "default", |
| "languages": [ |
| "eng-Latn" |
| ], |
| "mteb_version": "2.21.0" |
| } |
| }, |
| { |
| "task": "SpeechCommandsZeroshotv0.02", |
| "split": "test", |
| "hf_subset": "default", |
| "main_metric": "accuracy", |
| "score": 0.14972999509081983, |
| "metrics": { |
| "accuracy": 0.14972999509081983, |
| "f1": 0.06986246632697812, |
| "f1_weighted": 0.07083899432793754, |
| "precision": 0.15715381051614374, |
| "precision_weighted": 0.1573685095178851, |
| "recall": 0.14663865718054886, |
| "recall_weighted": 0.14972999509081983, |
| "ap": null, |
| "ap_weighted": null, |
| "main_score": 0.14972999509081983, |
| "hf_subset": "default", |
| "languages": [ |
| "eng-Latn" |
| ], |
| "mteb_version": "2.21.0" |
| } |
| } |
| ], |
| "full_benchmarks": [ |
| { |
| "benchmark": "MTEB(eng, v2)", |
| "framework_version": "2.21.0", |
| "coverage": { |
| "completed_tasks": 41, |
| "required_tasks": 41, |
| "task_types": 7 |
| }, |
| "model_revision": "70c3412bb0ddfb19386d673d4a102c753a574f1b", |
| "native_artifact_sha256": "b596f1dbab89369555e0a72a0d6854d780e305686750949b308c8f167993bdeb", |
| "aggregation": { |
| "mean_task": 0.5337130125278776, |
| "mean_task_type": 0.5197553164396411, |
| "definition": "First apply the official TaskResult aggregation within each task. Mean task gives each of the 41 tasks equal weight; mean task type first averages within each type, then gives all seven types equal weight.", |
| "score_scale": "0-1; multiply by 100 for display", |
| "by_task_type": { |
| "Classification": { |
| "score": 0.6552554795025091, |
| "tasks": 8 |
| }, |
| "Clustering": { |
| "score": 0.47072550487857034, |
| "tasks": 8 |
| }, |
| "PairClassification": { |
| "score": 0.7494593709186261, |
| "tasks": 3 |
| }, |
| "Reranking": { |
| "score": 0.455685, |
| "tasks": 2 |
| }, |
| "Retrieval": { |
| "score": 0.290152, |
| "tasks": 10 |
| }, |
| "STS": { |
| "score": 0.724513458257586, |
| "tasks": 9 |
| }, |
| "Summarization": { |
| "score": 0.29249640152019635, |
| "tasks": 1 |
| } |
| } |
| }, |
| "report": "benchmarks/mteb-eng-v2.json", |
| "text_component_applicability": { |
| "applies_to_revision": "self", |
| "applies_to_native_artifact_sha256": "fb9cd39c9980a085b75456c715c2295b997784fe733314be68c13635f6384edf", |
| "evaluated_revision": "70c3412bb0ddfb19386d673d4a102c753a574f1b", |
| "evaluated_native_artifact_sha256": "b596f1dbab89369555e0a72a0d6854d780e305686750949b308c8f167993bdeb", |
| "basis": "Exact component tensor and inference-file identity; original measured identity is retained.", |
| "report": "benchmarks/component-equivalence.json", |
| "new_full_panel_inference": false, |
| "component": "text" |
| } |
| }, |
| { |
| "benchmark": "MAEB(beta, audio-only)", |
| "framework_version": "2.21.0", |
| "coverage": { |
| "completed_tasks": 19, |
| "required_tasks": 19, |
| "subset_split_slots": 134, |
| "task_types": 5, |
| "selection": "All MAEB(beta) tasks whose modalities are exactly [audio].", |
| "full_MAEB": false, |
| "excluded": "Tasks that require text/audio or other joint-modality inputs." |
| }, |
| "model_revision": "70c3412bb0ddfb19386d673d4a102c753a574f1b", |
| "native_artifact_sha256": "b596f1dbab89369555e0a72a0d6854d780e305686750949b308c8f167993bdeb", |
| "aggregation": { |
| "official": { |
| "Any2AnyRetrieval": 0.6890050000000001, |
| "AudioClassification": 0.39566923071716426, |
| "AudioClustering": 0.054218008138417706, |
| "AudioPairClassification": 0.571886979184418, |
| "AudioReranking": 0.63794, |
| "Mean(Task)": 0.39776955262406916, |
| "Mean(TaskType)": 0.46974384360800003 |
| }, |
| "definition": "Official TaskResult aggregation within each task, followed by official mean_task, mean_task_type and task_types aggregation. The 134 subset/split slots are not averaged as independent tasks.", |
| "score_scale": "0-1; multiply by 100 for display" |
| }, |
| "audio_component_applicability": { |
| "applies_to_revision": "self", |
| "applies_to_native_artifact_sha256": "fb9cd39c9980a085b75456c715c2295b997784fe733314be68c13635f6384edf", |
| "evaluated_revision": "70c3412bb0ddfb19386d673d4a102c753a574f1b", |
| "evaluated_native_artifact_sha256": "b596f1dbab89369555e0a72a0d6854d780e305686750949b308c8f167993bdeb", |
| "basis": "Exact component tensor and inference-file identity; original measured identity is retained.", |
| "report": "benchmarks/component-equivalence.json", |
| "new_full_panel_inference": false, |
| "component": "audio" |
| }, |
| "report": "benchmarks/maeb-audio-only.json" |
| } |
| ] |
| }, |
| "full_benchmarks": [ |
| { |
| "benchmark": "MTEB(eng, v2)", |
| "framework_version": "2.21.0", |
| "coverage": { |
| "completed_tasks": 41, |
| "required_tasks": 41, |
| "task_types": 7 |
| }, |
| "model_revision": "29994142677559ca657c27b657eda0dddbf583bd4cf42a3a31095ab3f99d1646", |
| "native_artifact_sha256": "29994142677559ca657c27b657eda0dddbf583bd4cf42a3a31095ab3f99d1646", |
| "aggregation": { |
| "mean_task": 0.6488431514310018, |
| "mean_task_type": 0.6078184975654923, |
| "by_task_type": { |
| "Classification": { |
| "score": 0.7782970776285105, |
| "tasks": 8 |
| }, |
| "Clustering": { |
| "score": 0.48962597555764586, |
| "tasks": 8 |
| }, |
| "PairClassification": { |
| "score": 0.8468485561827029, |
| "tasks": 3 |
| }, |
| "Reranking": { |
| "score": 0.473475, |
| "tasks": 2 |
| }, |
| "Retrieval": { |
| "score": 0.520098, |
| "tasks": 10 |
| }, |
| "STS": { |
| "score": 0.8280405301305157, |
| "tasks": 9 |
| }, |
| "Summarization": { |
| "score": 0.31834434345907126, |
| "tasks": 1 |
| } |
| }, |
| "definition": "Official within-task aggregation; equal task mean across 41 tasks, and equal mean of seven within-type means.", |
| "score_scale": "0-1; display multiplied by 100" |
| }, |
| "report": "benchmarks/mteb-eng-v2.json", |
| "text_component_applicability": { |
| "component": "text", |
| "applies_to_revision": "self", |
| "applies_to_native_artifact_sha256": "96d76f97478e114b816df3ab1b3c6947e4c5ef0babfb40ab865d7e3c838d34c4", |
| "evaluated_model": "local/Vela-Omni-Nano-GIST-Rotation", |
| "evaluated_revision": "29994142677559ca657c27b657eda0dddbf583bd4cf42a3a31095ab3f99d1646", |
| "evaluated_native_artifact_sha256": "29994142677559ca657c27b657eda0dddbf583bd4cf42a3a31095ab3f99d1646", |
| "revision_kind": "local native artifact fingerprint, not a Hugging Face commit", |
| "report": "benchmarks/component-equivalence.json", |
| "new_full_panel_inference": false, |
| "basis": "199 exact text tensors and byte-identical inference/tokenizer/configuration under the same official evaluator protocol." |
| }, |
| "inference_profile_applicability": { |
| "revision": "self", |
| "native_artifact_sha256": "50ce808197fb43a1b8913a66d4a47339ec66ecd10d52cbd9f9e752d02f8c43fa", |
| "parameters": 135383808, |
| "report": "benchmarks/inference-equivalence.json", |
| "new_benchmark_run": false |
| } |
| }, |
| { |
| "benchmark": "MAEB(beta, audio-only)", |
| "framework_version": "2.21.0", |
| "coverage": { |
| "completed_tasks": 19, |
| "required_tasks": 19, |
| "subset_split_slots": 134, |
| "task_types": 5, |
| "selection": "All MAEB(beta) tasks whose modalities are exactly [audio].", |
| "full_MAEB": false, |
| "excluded": "Tasks that require text/audio or other joint-modality inputs." |
| }, |
| "model_revision": "96d76f97478e114b816df3ab1b3c6947e4c5ef0babfb40ab865d7e3c838d34c4", |
| "native_artifact_sha256": "96d76f97478e114b816df3ab1b3c6947e4c5ef0babfb40ab865d7e3c838d34c4", |
| "aggregation": { |
| "official": { |
| "Any2AnyRetrieval": 0.6890050000000001, |
| "AudioClassification": 0.39553889510295764, |
| "AudioClustering": 0.054218008138417706, |
| "AudioPairClassification": 0.5716888010864635, |
| "AudioReranking": 0.63794, |
| "Mean(Task)": 0.3976628038845884, |
| "Mean(TaskType)": 0.4696781408655678 |
| }, |
| "definition": "Official TaskResult aggregation within each task, followed by official mean_task, mean_task_type and task_types aggregation. The 134 subset/split slots are not averaged as independent tasks.", |
| "score_scale": "0-1; multiply by 100 for display" |
| }, |
| "report": "benchmarks/maeb-audio-only.json", |
| "inference_profile_applicability": { |
| "revision": "self", |
| "native_artifact_sha256": "50ce808197fb43a1b8913a66d4a47339ec66ecd10d52cbd9f9e752d02f8c43fa", |
| "parameters": 135383808, |
| "report": "benchmarks/inference-equivalence.json", |
| "new_benchmark_run": false |
| } |
| } |
| ], |
| "net_benefit": { |
| "scope": "Product-weighted progress comparison, not an official benchmark aggregate or SOTA claim.", |
| "baseline_model": "vllm-sr/Vela-1.0-Omni-Nano", |
| "baseline_revision": "3a1efc48995e2b71622d65c6cf71cfc00bbf3aa3", |
| "baseline_native_artifact_sha256": "fb9cd39c9980a085b75456c715c2295b997784fe733314be68c13635f6384edf", |
| "candidate_native_artifact_sha256": "96d76f97478e114b816df3ab1b3c6947e4c5ef0babfb40ab865d7e3c838d34c4", |
| "score_scale": "0-100; changes in percentage points", |
| "components": [ |
| { |
| "baseline_100": 59.37287180704907, |
| "candidate_100": 67.57407300861536, |
| "component": "common14", |
| "delta_pp": 8.201201201566288, |
| "weight": 0.3 |
| }, |
| { |
| "baseline_100": 51.97553164396411, |
| "candidate_100": 60.781849756549235, |
| "component": "English41_MeanTaskType", |
| "delta_pp": 8.806318112585124, |
| "weight": 0.3 |
| }, |
| { |
| "baseline_100": 54.865785925824184, |
| "candidate_100": 53.65746394890041, |
| "component": "image3", |
| "delta_pp": -1.208321976923776, |
| "weight": 0.2 |
| }, |
| { |
| "baseline_100": 46.9743843608, |
| "candidate_100": 46.96781408655678, |
| "component": "audio19_MeanTaskType", |
| "delta_pp": -0.00657027424322, |
| "weight": 0.2 |
| } |
| ], |
| "weighted_net_delta_pp": 4.859277344012025, |
| "common_definition": "Equal mean of three families: two text accuracies, all six image retrieval recalls, all six audio retrieval recalls.", |
| "image3_definition": "Equal mean of OxfordPets classification accuracy, TinyImageNet clustering NMI and OxfordPets zero-shot accuracy.", |
| "text_audio_definition": "Full English41 and full audio19 mean task-type scores. English uses the explicit exact text-component bridge; audio is freshly evaluated.", |
| "coverage": { |
| "common_metrics": 14, |
| "standard_tasks": 11, |
| "standard_split_rows": 12, |
| "English_tasks": 41, |
| "audio_tasks": 19, |
| "audio_subset_split_slots": 134 |
| }, |
| "common_metric_delta_pp": { |
| "audio.audio_to_text.recall@1": 8.157793948678668, |
| "audio.audio_to_text.recall@10": 15.51129835312141, |
| "audio.audio_to_text.recall@5": 14.362313289927231, |
| "audio.text_to_audio.recall@1": 11.417624521072797, |
| "audio.text_to_audio.recall@10": 15.478927203065133, |
| "audio.text_to_audio.recall@5": 14.444444444444445, |
| "image.image_to_text.recall@1": -4.738760631834751, |
| "image.image_to_text.recall@10": -1.2150668286755772, |
| "image.image_to_text.recall@5": -1.701093560145808, |
| "image.text_to_image.recall@1": 2.7460510328068044, |
| "image.text_to_image.recall@10": 0.7776427703523694, |
| "image.text_to_image.recall@5": 1.0206561360874848, |
| "text.banking77.accuracy": 11.168831168831169, |
| "text.massive-en.accuracy": 12.617765814266487 |
| }, |
| "standard_task_deltas": [ |
| { |
| "baseline": 0.7169618925026362, |
| "candidate": 0.8053174565838819, |
| "hf_subset": "default", |
| "main_metric": "cosine_spearman", |
| "split": "test", |
| "status": "complete", |
| "task": "SICK-R", |
| "delta_pp": 8.835556408124567 |
| }, |
| { |
| "baseline": 0.7792400226361769, |
| "candidate": 0.870781967302773, |
| "hf_subset": "default", |
| "main_metric": "cosine_spearman", |
| "split": "test", |
| "status": "complete", |
| "task": "STSBenchmark", |
| "delta_pp": 9.154194466659614 |
| }, |
| { |
| "baseline": 0.37514, |
| "candidate": 0.59246, |
| "hf_subset": "default", |
| "main_metric": "ndcg_at_10", |
| "split": "test", |
| "status": "complete", |
| "task": "ArguAna", |
| "delta_pp": 21.732000000000003 |
| }, |
| { |
| "baseline": 0.44884717457494344, |
| "candidate": 0.5087260708286914, |
| "hf_subset": "default", |
| "main_metric": "v_measure", |
| "split": "test", |
| "status": "complete", |
| "task": "TwentyNewsgroupsClustering.v2", |
| "delta_pp": 5.987889625374793 |
| }, |
| { |
| "baseline": 0.8758836689038031, |
| "candidate": 0.9476510067114093, |
| "hf_subset": "en", |
| "main_metric": "accuracy", |
| "split": "validation", |
| "status": "complete", |
| "task": "MTOPDomainClassification", |
| "delta_pp": 7.176733780760625 |
| }, |
| { |
| "baseline": 0.8671910624715002, |
| "candidate": 0.9491792065663475, |
| "hf_subset": "en", |
| "main_metric": "accuracy", |
| "split": "test", |
| "status": "complete", |
| "task": "MTOPDomainClassification", |
| "delta_pp": 8.198814409484722 |
| }, |
| { |
| "baseline": 0.9126737530662306, |
| "candidate": 0.9126737530662306, |
| "hf_subset": "default", |
| "main_metric": "accuracy", |
| "split": "test", |
| "status": "complete", |
| "task": "OxfordPets", |
| "delta_pp": 0.0 |
| }, |
| { |
| "baseline": 0.6169193395626786, |
| "candidate": 0.6169193395626786, |
| "hf_subset": "default", |
| "main_metric": "nmi", |
| "split": "valid", |
| "status": "complete", |
| "task": "TinyImageNetClustering", |
| "delta_pp": 0.0 |
| }, |
| { |
| "baseline": 0.1163804851458163, |
| "candidate": 0.08013082583810302, |
| "hf_subset": "default", |
| "main_metric": "accuracy", |
| "split": "test", |
| "status": "complete", |
| "task": "OxfordPetsZeroShot", |
| "delta_pp": -3.6249659307713276 |
| }, |
| { |
| "baseline": 0.29763670140167686, |
| "candidate": 0.29763670140167686, |
| "hf_subset": "default", |
| "main_metric": "accuracy", |
| "split": "train", |
| "status": "complete", |
| "task": "CREMA_D", |
| "delta_pp": 0.0 |
| }, |
| { |
| "baseline": 0.023601715918443712, |
| "candidate": 0.023601715918443712, |
| "hf_subset": "default", |
| "main_metric": "v_measure", |
| "split": "train", |
| "status": "complete", |
| "task": "CREMA_DClustering", |
| "delta_pp": 0.0 |
| }, |
| { |
| "baseline": 0.14972999509081983, |
| "candidate": 0.1411389297987236, |
| "hf_subset": "default", |
| "main_metric": "accuracy", |
| "split": "test", |
| "status": "complete", |
| "task": "SpeechCommandsZeroshotv0.02", |
| "delta_pp": -0.8591065292096217 |
| } |
| ], |
| "English_task_deltas": [ |
| { |
| "candidate_100": 59.245999999999995, |
| "current_100": 37.513999999999996, |
| "delta_pp": 21.732, |
| "primary_metric": "ndcg_at_10", |
| "task": "ArguAna", |
| "task_type": "Retrieval" |
| }, |
| { |
| "candidate_100": 64.88847259326147, |
| "current_100": 65.25395409281045, |
| "delta_pp": -0.36548149954897724, |
| "primary_metric": "v_measure", |
| "task": "ArXivHierarchicalClusteringP2P", |
| "task_type": "Clustering" |
| }, |
| { |
| "candidate_100": 57.44315313869382, |
| "current_100": 55.989532291560415, |
| "delta_pp": 1.4536208471334078, |
| "primary_metric": "v_measure", |
| "task": "ArXivHierarchicalClusteringS2S", |
| "task_type": "Clustering" |
| }, |
| { |
| "candidate_100": 62.329, |
| "current_100": 60.514, |
| "delta_pp": 1.8149999999999977, |
| "primary_metric": "map_at_1000", |
| "task": "AskUbuntuDupQuestions", |
| "task_type": "Reranking" |
| }, |
| { |
| "candidate_100": 86.98752423842483, |
| "current_100": 62.24539420714732, |
| "delta_pp": 24.742130031277505, |
| "primary_metric": "cosine_spearman", |
| "task": "BIOSSES", |
| "task_type": "STS" |
| }, |
| { |
| "candidate_100": 82.14285714285714, |
| "current_100": 69.57467532467533, |
| "delta_pp": 12.568181818181813, |
| "primary_metric": "accuracy", |
| "task": "Banking77Classification", |
| "task_type": "Classification" |
| }, |
| { |
| "candidate_100": 41.313796273674654, |
| "current_100": 39.82923119776807, |
| "delta_pp": 1.4845650759065876, |
| "primary_metric": "v_measure", |
| "task": "BiorxivClusteringP2P.v2", |
| "task_type": "Clustering" |
| }, |
| { |
| "candidate_100": 56.979, |
| "current_100": 42.976, |
| "delta_pp": 14.003, |
| "primary_metric": "ndcg_at_10", |
| "task": "CQADupstackGamingRetrieval", |
| "task_type": "Retrieval" |
| }, |
| { |
| "candidate_100": 39.586, |
| "current_100": 30.362000000000002, |
| "delta_pp": 9.223999999999997, |
| "primary_metric": "ndcg_at_10", |
| "task": "CQADupstackUnixRetrieval", |
| "task_type": "Retrieval" |
| }, |
| { |
| "candidate_100": 31.846000000000004, |
| "current_100": 16.073999999999998, |
| "delta_pp": 15.772000000000006, |
| "primary_metric": "ndcg_at_10", |
| "task": "ClimateFEVERHardNegatives", |
| "task_type": "Retrieval" |
| }, |
| { |
| "candidate_100": 87.576, |
| "current_100": 24.055, |
| "delta_pp": 63.520999999999994, |
| "primary_metric": "ndcg_at_10", |
| "task": "FEVERHardNegatives", |
| "task_type": "Retrieval" |
| }, |
| { |
| "candidate_100": 39.143, |
| "current_100": 22.588, |
| "delta_pp": 16.555, |
| "primary_metric": "ndcg_at_10", |
| "task": "FiQA2018", |
| "task_type": "Retrieval" |
| }, |
| { |
| "candidate_100": 66.349, |
| "current_100": 26.987, |
| "delta_pp": 39.36200000000001, |
| "primary_metric": "ndcg_at_10", |
| "task": "HotpotQAHardNegatives", |
| "task_type": "Retrieval" |
| }, |
| { |
| "candidate_100": 91.94760000000001, |
| "current_100": 62.1364, |
| "delta_pp": 29.811200000000007, |
| "primary_metric": "accuracy", |
| "task": "ImdbClassification", |
| "task_type": "Classification" |
| }, |
| { |
| "candidate_100": 94.91792065663475, |
| "current_100": 86.71910624715002, |
| "delta_pp": 8.198814409484726, |
| "primary_metric": "accuracy", |
| "task": "MTOPDomainClassification", |
| "task_type": "Classification" |
| }, |
| { |
| "candidate_100": 70.96839273705447, |
| "current_100": 60.39340954942838, |
| "delta_pp": 10.574983187626096, |
| "primary_metric": "accuracy", |
| "task": "MassiveIntentClassification", |
| "task_type": "Classification" |
| }, |
| { |
| "candidate_100": 76.11297915265635, |
| "current_100": 73.37256220578345, |
| "delta_pp": 2.740416946872898, |
| "primary_metric": "accuracy", |
| "task": "MassiveScenarioClassification", |
| "task_type": "Classification" |
| }, |
| { |
| "candidate_100": 39.83023091223438, |
| "current_100": 37.76813963073551, |
| "delta_pp": 2.0620912814988657, |
| "primary_metric": "v_measure", |
| "task": "MedrxivClusteringP2P.v2", |
| "task_type": "Clustering" |
| }, |
| { |
| "candidate_100": 37.875084453707764, |
| "current_100": 36.13349306174702, |
| "delta_pp": 1.7415913919607462, |
| "primary_metric": "v_measure", |
| "task": "MedrxivClusteringS2S.v2", |
| "task_type": "Clustering" |
| }, |
| { |
| "candidate_100": 32.366, |
| "current_100": 30.623, |
| "delta_pp": 1.7429999999999986, |
| "primary_metric": "max_over_subqueries_map_at_1000", |
| "task": "MindSmallReranking", |
| "task_type": "Reranking" |
| }, |
| { |
| "candidate_100": 21.889, |
| "current_100": 16.827, |
| "delta_pp": 5.061999999999998, |
| "primary_metric": "ndcg_at_10", |
| "task": "SCIDOCS", |
| "task_type": "Retrieval" |
| }, |
| { |
| "candidate_100": 80.53174565838819, |
| "current_100": 71.69618925026361, |
| "delta_pp": 8.835556408124575, |
| "primary_metric": "cosine_spearman", |
| "task": "SICK-R", |
| "task_type": "STS" |
| }, |
| { |
| "candidate_100": 75.5658801516535, |
| "current_100": 68.16915974310523, |
| "delta_pp": 7.396720408548276, |
| "primary_metric": "cosine_spearman", |
| "task": "STS12", |
| "task_type": "STS" |
| }, |
| { |
| "candidate_100": 86.26346974038235, |
| "current_100": 74.40342742804673, |
| "delta_pp": 11.860042312335622, |
| "primary_metric": "cosine_spearman", |
| "task": "STS13", |
| "task_type": "STS" |
| }, |
| { |
| "candidate_100": 82.29882793900394, |
| "current_100": 68.02653891259911, |
| "delta_pp": 14.272289026404835, |
| "primary_metric": "cosine_spearman", |
| "task": "STS14", |
| "task_type": "STS" |
| }, |
| { |
| "candidate_100": 88.73648284326485, |
| "current_100": 77.86144933320074, |
| "delta_pp": 10.875033510064114, |
| "primary_metric": "cosine_spearman", |
| "task": "STS15", |
| "task_type": "STS" |
| }, |
| { |
| "candidate_100": 87.0781967302773, |
| "current_100": 77.92400226361768, |
| "delta_pp": 9.154194466659618, |
| "primary_metric": "cosine_spearman", |
| "task": "STSBenchmark", |
| "task_type": "STS" |
| }, |
| { |
| "candidate_100": 95.79964673462752, |
| "current_100": 89.79096125036985, |
| "delta_pp": 6.0086854842576685, |
| "primary_metric": "max_ap", |
| "task": "SprintDuplicateQuestions", |
| "task_type": "PairClassification" |
| }, |
| { |
| "candidate_100": 58.424418738430106, |
| "current_100": 57.34464990059502, |
| "delta_pp": 1.0797688378350827, |
| "primary_metric": "v_measure", |
| "task": "StackExchangeClustering.v2", |
| "task_type": "Clustering" |
| }, |
| { |
| "candidate_100": 41.053017253245386, |
| "current_100": 39.37668627014545, |
| "delta_pp": 1.6763309830999376, |
| "primary_metric": "v_measure", |
| "task": "StackExchangeClusteringP2P.v2", |
| "task_type": "Clustering" |
| }, |
| { |
| "candidate_100": 69.121, |
| "current_100": 41.292, |
| "delta_pp": 27.828999999999994, |
| "primary_metric": "ndcg_at_10", |
| "task": "TRECCOVID", |
| "task_type": "Retrieval" |
| }, |
| { |
| "candidate_100": 48.363, |
| "current_100": 31.477, |
| "delta_pp": 16.886, |
| "primary_metric": "ndcg_at_10", |
| "task": "Touche2020Retrieval.v3", |
| "task_type": "Retrieval" |
| }, |
| { |
| "candidate_100": 71.904296875, |
| "current_100": 64.4091796875, |
| "delta_pp": 7.4951171875, |
| "primary_metric": "accuracy", |
| "task": "ToxicConversationsClassification", |
| "task_type": "Classification" |
| }, |
| { |
| "candidate_100": 62.79286926994907, |
| "current_100": 50.77815506508207, |
| "delta_pp": 12.014714204867005, |
| "primary_metric": "accuracy", |
| "task": "TweetSentimentExtractionClassification", |
| "task_type": "Classification" |
| }, |
| { |
| "candidate_100": 50.87260708286914, |
| "current_100": 44.88471745749435, |
| "delta_pp": 5.987889625374791, |
| "primary_metric": "v_measure", |
| "task": "TwentyNewsgroupsClustering.v2", |
| "task_type": "Clustering" |
| }, |
| { |
| "candidate_100": 72.95151691526065, |
| "current_100": 56.34475441213181, |
| "delta_pp": 16.60676250312884, |
| "primary_metric": "max_ap", |
| "task": "TwitterSemEval2015", |
| "task_type": "PairClassification" |
| }, |
| { |
| "candidate_100": 85.30340320492272, |
| "current_100": 78.70209561308617, |
| "delta_pp": 6.6013075918365445, |
| "primary_metric": "max_ap", |
| "task": "TwitterURLCorpus", |
| "task_type": "PairClassification" |
| }, |
| { |
| "candidate_100": 31.834434345907127, |
| "current_100": 29.249640152019634, |
| "delta_pp": 2.584794193887493, |
| "primary_metric": "cosine_spearman", |
| "task": "SummEvalSummarization.v2", |
| "task_type": "Summarization" |
| }, |
| { |
| "candidate_100": 71.8507462686567, |
| "current_100": 56.82089552238806, |
| "delta_pp": 15.02985074626865, |
| "primary_metric": "accuracy", |
| "task": "AmazonCounterfactualClassification", |
| "task_type": "Classification" |
| }, |
| { |
| "candidate_100": 89.02101343562433, |
| "current_100": 84.16589279728525, |
| "delta_pp": 4.8551206383390735, |
| "primary_metric": "cosine_spearman", |
| "task": "STS17", |
| "task_type": "STS" |
| }, |
| { |
| "candidate_100": 68.75333638044488, |
| "current_100": 67.57005849656171, |
| "delta_pp": 1.1832778838831643, |
| "primary_metric": "cosine_spearman", |
| "task": "STS22.v2", |
| "task_type": "STS" |
| } |
| ], |
| "audio_task_delta_pp": { |
| "BeijingOpera": 0.0, |
| "BirdCLEF": 0.0, |
| "CREMADPairClassification": 0.12104996267769952, |
| "CREMA_D": 0.0, |
| "CREMA_DClustering": 0.0, |
| "CommonLanguageAgeDetection": 0.0, |
| "GTZANAudioReranking": 0.0, |
| "GTZANGenre": 0.0, |
| "IEMOCAPGender": 0.0, |
| "JamAltArtistA2ARetrieval": 0.0, |
| "MInDS14": 0.0, |
| "MridinghamTonic": -0.14336917562723928, |
| "NMSQAPairClassification": -0.1520590751812989, |
| "SIBFLEURS": 0.0, |
| "VehicleSoundClustering": 0.0, |
| "VoxCelebSA": 0.0, |
| "VoxPopuliAccentPairClassification": -0.02844431688273641, |
| "VoxPopuliGenderClustering": 0.0, |
| "VoxPopuliLanguageID": 0.0 |
| }, |
| "paired_interval_scope": "Paired intervals cover the original common metrics, family and two-primary-per-family utility. They are not confidence intervals for the new all-14 common mean or the four-component product aggregate.", |
| "inference_profile_applicability": { |
| "revision": "self", |
| "native_artifact_sha256": "50ce808197fb43a1b8913a66d4a47339ec66ecd10d52cbd9f9e752d02f8c43fa", |
| "parameters": 135383808, |
| "report": "benchmarks/inference-equivalence.json", |
| "new_benchmark_run": false |
| } |
| }, |
| "paired_uncertainty_scope": "Paired intervals cover the original common metrics, family and two-primary-per-family utility. They are not confidence intervals for the new all-14 common mean or the four-component product aggregate.", |
| "inference_profile": { |
| "revision": "self", |
| "native_artifact_sha256": "50ce808197fb43a1b8913a66d4a47339ec66ecd10d52cbd9f9e752d02f8c43fa", |
| "parameters": 135383808, |
| "report": "benchmarks/inference-equivalence.json", |
| "new_benchmark_run": false |
| }, |
| "scorecard_comparison": { |
| "primary_comparator": "vllm-sr/multi-modal-embed-small", |
| "revision": "fdf8e01b7b0f3a69ac1ac8e2a64dcb1ede177ba4", |
| "common_protocol_measured": true, |
| "original_reference_evidence": "benchmarks/original-baseline.json", |
| "previous_Vela_not_the_primary_comparator": true, |
| "historical_release_progress": "benchmarks/release-history.md" |
| } |
| } |
|
|
