{
  "schema_version": 1,
  "comparison_model": {
    "model": "vllm-sr/multi-modal-embed-small",
    "revision": "fdf8e01b7b0f3a69ac1ac8e2a64dcb1ede177ba4",
    "artifact_fingerprint_sha256": "684fe0c48a4ddd3944167f6e5dc75e97ada24e5b120bd9a2921f00dc03112eeb",
    "metrics": {
      "text.banking77.accuracy": {
        "numerator": 2169,
        "denominator": 3080,
        "value": 0.7042207792207792
      },
      "text.massive-en.accuracy": {
        "numerator": 1960,
        "denominator": 2972,
        "value": 0.6594885598923284
      },
      "image.image_to_text.recall@1": {
        "numerator": 336,
        "denominator": 823,
        "value": 0.4082624544349939
      },
      "image.image_to_text.recall@5": {
        "numerator": 553,
        "denominator": 823,
        "value": 0.6719319562575942
      },
      "image.image_to_text.recall@10": {
        "numerator": 646,
        "denominator": 823,
        "value": 0.7849331713244229
      },
      "image.text_to_image.recall@1": {
        "numerator": 1242,
        "denominator": 4115,
        "value": 0.3018226002430134
      },
      "image.text_to_image.recall@5": {
        "numerator": 2445,
        "denominator": 4115,
        "value": 0.5941676792223572
      },
      "image.text_to_image.recall@10": {
        "numerator": 3057,
        "denominator": 4115,
        "value": 0.7428918590522479
      },
      "audio.audio_to_text.recall@1": {
        "numerator": 110,
        "denominator": 2611,
        "value": 0.04212945231711988
      },
      "audio.audio_to_text.recall@5": {
        "numerator": 313,
        "denominator": 2611,
        "value": 0.11987744159325929
      },
      "audio.audio_to_text.recall@10": {
        "numerator": 497,
        "denominator": 2611,
        "value": 0.1903485254691689
      },
      "audio.text_to_audio.recall@1": {
        "numerator": 250,
        "denominator": 2610,
        "value": 0.09578544061302682
      },
      "audio.text_to_audio.recall@5": {
        "numerator": 588,
        "denominator": 2610,
        "value": 0.22528735632183908
      },
      "audio.text_to_audio.recall@10": {
        "numerator": 801,
        "denominator": 2610,
        "value": 0.30689655172413793
      }
    }
  },
  "source_reports_sha256": {
    "common_test": "bdd5a5dca7fcff12a96f18931c71b2b06da43e6fae45a3ef9b26d54c2652899d",
    "strict_native_reload": "d938de6e703068787ff48367e9ea466bd23a8df86ea21858504876cf9a3a9ff6",
    "original_checkpoint_equivalence": "d9f2f62dbe0ee9d386c98ff200c2f1e5061bc92fa5e107d1517b929258c8ec1e"
  },
  "original_checkpoint": {
    "repo_id": "vllm-sr/multi-modal-embed-small",
    "revision": "fdf8e01b7b0f3a69ac1ac8e2a64dcb1ede177ba4",
    "weight_files": [
      {
        "file": "model.pt",
        "bytes": 976407151,
        "sha256": "a4e280a185550651d299dfcd10df7e2cd02629c2f0c0b0964122daabe723ef4b"
      }
    ]
  },
  "common_protocol": {
    "primary_metrics": 14,
    "descriptive_metrics": 2,
    "partition": "test",
    "known_reused_test_pool": true,
    "text_tokens": 128,
    "score_scale": "0-1; displayed as 0-100",
    "same_queries_and_complete_retrieval_pools": true,
    "official_MTEB_classification": false
  },
  "standard_coverage": {
    "MTEB_eng_v2_41": {
      "status": "not_evaluated_in_this_matched_protocol",
      "score": null
    },
    "MAEB_audio_only_19": {
      "status": "not_evaluated_in_this_matched_protocol",
      "score": null
    },
    "selected_standard_11": {
      "status": "evaluated_all_11_selected_tasks",
      "task_count": 11,
      "split_rows": 12,
      "report": "original-standard.json",
      "report_sha256": "b012d62badc25b7b80d458bf7e48965cb84e583ae4dddbcbbc676d6297edae82"
    }
  },
  "not_previous_Vela": true,
  "no_new_inference": true
}