| { |
| "schema_version": 1, |
| "comparison_model": { |
| "model": "vllm-sr/multi-modal-embed-small", |
| "revision": "fdf8e01b7b0f3a69ac1ac8e2a64dcb1ede177ba4", |
| "artifact_fingerprint_sha256": "684fe0c48a4ddd3944167f6e5dc75e97ada24e5b120bd9a2921f00dc03112eeb", |
| "metrics": { |
| "text.banking77.accuracy": { |
| "numerator": 2169, |
| "denominator": 3080, |
| "value": 0.7042207792207792 |
| }, |
| "text.massive-en.accuracy": { |
| "numerator": 1960, |
| "denominator": 2972, |
| "value": 0.6594885598923284 |
| }, |
| "image.image_to_text.recall@1": { |
| "numerator": 336, |
| "denominator": 823, |
| "value": 0.4082624544349939 |
| }, |
| "image.image_to_text.recall@5": { |
| "numerator": 553, |
| "denominator": 823, |
| "value": 0.6719319562575942 |
| }, |
| "image.image_to_text.recall@10": { |
| "numerator": 646, |
| "denominator": 823, |
| "value": 0.7849331713244229 |
| }, |
| "image.text_to_image.recall@1": { |
| "numerator": 1242, |
| "denominator": 4115, |
| "value": 0.3018226002430134 |
| }, |
| "image.text_to_image.recall@5": { |
| "numerator": 2445, |
| "denominator": 4115, |
| "value": 0.5941676792223572 |
| }, |
| "image.text_to_image.recall@10": { |
| "numerator": 3057, |
| "denominator": 4115, |
| "value": 0.7428918590522479 |
| }, |
| "audio.audio_to_text.recall@1": { |
| "numerator": 110, |
| "denominator": 2611, |
| "value": 0.04212945231711988 |
| }, |
| "audio.audio_to_text.recall@5": { |
| "numerator": 313, |
| "denominator": 2611, |
| "value": 0.11987744159325929 |
| }, |
| "audio.audio_to_text.recall@10": { |
| "numerator": 497, |
| "denominator": 2611, |
| "value": 0.1903485254691689 |
| }, |
| "audio.text_to_audio.recall@1": { |
| "numerator": 250, |
| "denominator": 2610, |
| "value": 0.09578544061302682 |
| }, |
| "audio.text_to_audio.recall@5": { |
| "numerator": 588, |
| "denominator": 2610, |
| "value": 0.22528735632183908 |
| }, |
| "audio.text_to_audio.recall@10": { |
| "numerator": 801, |
| "denominator": 2610, |
| "value": 0.30689655172413793 |
| } |
| } |
| }, |
| "source_reports_sha256": { |
| "common_test": "bdd5a5dca7fcff12a96f18931c71b2b06da43e6fae45a3ef9b26d54c2652899d", |
| "strict_native_reload": "d938de6e703068787ff48367e9ea466bd23a8df86ea21858504876cf9a3a9ff6", |
| "original_checkpoint_equivalence": "d9f2f62dbe0ee9d386c98ff200c2f1e5061bc92fa5e107d1517b929258c8ec1e" |
| }, |
| "original_checkpoint": { |
| "repo_id": "vllm-sr/multi-modal-embed-small", |
| "revision": "fdf8e01b7b0f3a69ac1ac8e2a64dcb1ede177ba4", |
| "weight_files": [ |
| { |
| "file": "model.pt", |
| "bytes": 976407151, |
| "sha256": "a4e280a185550651d299dfcd10df7e2cd02629c2f0c0b0964122daabe723ef4b" |
| } |
| ] |
| }, |
| "common_protocol": { |
| "primary_metrics": 14, |
| "descriptive_metrics": 2, |
| "partition": "test", |
| "known_reused_test_pool": true, |
| "text_tokens": 128, |
| "score_scale": "0-1; displayed as 0-100", |
| "same_queries_and_complete_retrieval_pools": true, |
| "official_MTEB_classification": false |
| }, |
| "standard_coverage": { |
| "MTEB_eng_v2_41": { |
| "status": "not_evaluated_in_this_matched_protocol", |
| "score": null |
| }, |
| "MAEB_audio_only_19": { |
| "status": "not_evaluated_in_this_matched_protocol", |
| "score": null |
| }, |
| "selected_standard_11": { |
| "status": "evaluated_all_11_selected_tasks", |
| "task_count": 11, |
| "split_rows": 12, |
| "report": "original-standard.json", |
| "report_sha256": "b012d62badc25b7b80d458bf7e48965cb84e583ae4dddbcbbc676d6297edae82" |
| } |
| }, |
| "not_previous_Vela": true, |
| "no_new_inference": true |
| } |
|
|
