diff --git "a/benchmarks/pareto-data.json" "b/benchmarks/pareto-data.json" --- "a/benchmarks/pareto-data.json" +++ "b/benchmarks/pareto-data.json" @@ -77,9 +77,9 @@ "observation_id": "registry_reported:minishlab/potion-base-4M" }, { - "model": "minishlab/potion-base-8M", + "model": "minishlab/M2V_base_output", "parameters_billion": 0.008, - "score": 55.0836, + "score": 51.5478, "open_weights": true, "modalities": [ "text" @@ -98,14 +98,14 @@ ], "size_basis": "reported_rounded_total", "kind": "peer", - "on_displayed_frontier": true, + "on_displayed_frontier": false, "evidence_type": "registry_reported", - "observation_id": "registry_reported:minishlab/potion-base-8M" + "observation_id": "registry_reported:minishlab/M2V_base_output" }, { - "model": "minishlab/M2V_base_output", + "model": "minishlab/potion-base-8M", "parameters_billion": 0.008, - "score": 51.5478, + "score": 55.0836, "open_weights": true, "modalities": [ "text" @@ -124,9 +124,9 @@ ], "size_basis": "reported_rounded_total", "kind": "peer", - "on_displayed_frontier": false, + "on_displayed_frontier": true, "evidence_type": "registry_reported", - "observation_id": "registry_reported:minishlab/M2V_base_output" + "observation_id": "registry_reported:minishlab/potion-base-8M" }, { "model": "cointegrated/rubert-tiny", @@ -194,6 +194,25 @@ "evidence_type": "registry_reported", "observation_id": "registry_reported:Mihaiii/Venusaur" }, + { + "model": "Mihaiii/Bulbasaur", + "parameters_billion": 0.017, + "score": 54.8762, + "open_weights": true, + "modalities": [ + "text" + ], + "training_tasks": null, + "source_urls": [ + "https://mteb-leaderboard-backend.hf.space/v1/benchmarks/MTEB%28eng%2C%20v2%29/scores", + "https://mteb-leaderboard-backend.hf.space/v1/benchmarks/MTEB%28Multilingual%2C%20v2%29/scores" + ], + "size_basis": "reported_rounded_total", + "kind": "peer", + "on_displayed_frontier": false, + "evidence_type": "registry_reported", + "observation_id": "registry_reported:Mihaiii/Bulbasaur" + }, { "model": "Mihaiii/Wartortle", "parameters_billion": 0.017, @@ -221,9 +240,9 @@ "observation_id": "registry_reported:Mihaiii/Wartortle" }, { - "model": "Mihaiii/Bulbasaur", + "model": "Mihaiii/gte-micro", "parameters_billion": 0.017, - "score": 54.8762, + "score": 54.446600000000004, "open_weights": true, "modalities": [ "text" @@ -237,12 +256,12 @@ "kind": "peer", "on_displayed_frontier": false, "evidence_type": "registry_reported", - "observation_id": "registry_reported:Mihaiii/Bulbasaur" + "observation_id": "registry_reported:Mihaiii/gte-micro" }, { - "model": "Mihaiii/gte-micro", - "parameters_billion": 0.017, - "score": 54.446600000000004, + "model": "Mihaiii/gte-micro-v4", + "parameters_billion": 0.019, + "score": 55.395, "open_weights": true, "modalities": [ "text" @@ -256,12 +275,12 @@ "kind": "peer", "on_displayed_frontier": false, "evidence_type": "registry_reported", - "observation_id": "registry_reported:Mihaiii/gte-micro" + "observation_id": "registry_reported:Mihaiii/gte-micro-v4" }, { - "model": "Mihaiii/gte-micro-v4", - "parameters_billion": 0.019, - "score": 55.395, + "model": "Mihaiii/Ivysaur", + "parameters_billion": 0.023, + "score": 57.322799999999994, "open_weights": true, "modalities": [ "text" @@ -275,7 +294,7 @@ "kind": "peer", "on_displayed_frontier": false, "evidence_type": "registry_reported", - "observation_id": "registry_reported:Mihaiii/gte-micro-v4" + "observation_id": "registry_reported:Mihaiii/Ivysaur" }, { "model": "MongoDB/mdbr-leaf-mt", @@ -301,43 +320,25 @@ "observation_id": "registry_reported:MongoDB/mdbr-leaf-mt" }, { - "model": "brahmairesearch/slx-v0.1", - "parameters_billion": 0.023, - "score": 59.106899999999996, - "open_weights": true, - "modalities": [ - "text" - ], - "training_tasks": null, - "source_urls": [ - "https://mteb-leaderboard-backend.hf.space/v1/benchmarks/MTEB%28eng%2C%20v2%29/scores", - "https://mteb-leaderboard-backend.hf.space/v1/benchmarks/MTEB%28Multilingual%2C%20v2%29/scores" - ], - "size_basis": "reported_rounded_total", - "kind": "peer", - "on_displayed_frontier": false, - "evidence_type": "registry_reported", - "observation_id": "registry_reported:brahmairesearch/slx-v0.1" - }, - { - "model": "sentence-transformers/all-MiniLM-L6-v2", + "model": "Snowflake/snowflake-arctic-embed-xs", "parameters_billion": 0.023, - "score": 59.10688886391535, + "score": 54.9722, "open_weights": true, "modalities": [ "text" ], "training_tasks": [ - "MSMARCO", - "MSMARCO-PL", - "MSMARCOHardNegatives", + "FEVER", + "FEVER-NL", + "FEVERHardNegatives", + "HotPotQA", + "HotPotQA-PL", + "HotPotQAHardNegatives", + "HotpotQA-NL", "NQ", "NQ-NL", "NQ-PL", - "NQHardNegatives", - "NanoMSMARCORetrieval", - "NanoNQRetrieval", - "mMARCO-NL" + "NQHardNegatives" ], "source_urls": [ "https://mteb-leaderboard-backend.hf.space/v1/benchmarks/MTEB%28eng%2C%20v2%29/scores", @@ -347,7 +348,7 @@ "kind": "peer", "on_displayed_frontier": false, "evidence_type": "registry_reported", - "observation_id": "registry_reported:sentence-transformers/all-MiniLM-L6-v2" + "observation_id": "registry_reported:Snowflake/snowflake-arctic-embed-xs" }, { "model": "avsolatorio/GIST-all-MiniLM-L6-v2", @@ -387,9 +388,9 @@ "observation_id": "registry_reported:avsolatorio/GIST-all-MiniLM-L6-v2" }, { - "model": "Mihaiii/Ivysaur", + "model": "brahmairesearch/slx-v0.1", "parameters_billion": 0.023, - "score": 57.322799999999994, + "score": 59.106899999999996, "open_weights": true, "modalities": [ "text" @@ -403,28 +404,27 @@ "kind": "peer", "on_displayed_frontier": false, "evidence_type": "registry_reported", - "observation_id": "registry_reported:Mihaiii/Ivysaur" + "observation_id": "registry_reported:brahmairesearch/slx-v0.1" }, { - "model": "Snowflake/snowflake-arctic-embed-xs", + "model": "sentence-transformers/all-MiniLM-L6-v2", "parameters_billion": 0.023, - "score": 54.9722, + "score": 59.10688886391535, "open_weights": true, "modalities": [ "text" ], "training_tasks": [ - "FEVER", - "FEVER-NL", - "FEVERHardNegatives", - "HotPotQA", - "HotPotQA-PL", - "HotPotQAHardNegatives", - "HotpotQA-NL", + "MSMARCO", + "MSMARCO-PL", + "MSMARCOHardNegatives", "NQ", "NQ-NL", "NQ-PL", - "NQHardNegatives" + "NQHardNegatives", + "NanoMSMARCORetrieval", + "NanoNQRetrieval", + "mMARCO-NL" ], "source_urls": [ "https://mteb-leaderboard-backend.hf.space/v1/benchmarks/MTEB%28eng%2C%20v2%29/scores", @@ -434,7 +434,7 @@ "kind": "peer", "on_displayed_frontier": false, "evidence_type": "registry_reported", - "observation_id": "registry_reported:Snowflake/snowflake-arctic-embed-xs" + "observation_id": "registry_reported:sentence-transformers/all-MiniLM-L6-v2" }, { "model": "cointegrated/rubert-tiny2", @@ -595,22 +595,20 @@ "observation_id": "registry_reported:minishlab/potion-retrieval-32M" }, { - "model": "abhinand/MedEmbed-small-v0.1", + "model": "BAAI/bge-small-en-v1.5", "parameters_billion": 0.033, - "score": 60.553599999999996, + "score": 59.52876517144434, "open_weights": true, "modalities": [ "text" ], "training_tasks": [ - "ArguAna", - "ArguAna-NL", - "MedicalQARetrieval", - "NFCorpus", - "NFCorpus-NL", - "PublicHealthQA", - "TRECCOVID", - "TRECCOVID-NL" + "AmazonReviewsClassification", + "DuRetrieval", + "MLQARetrieval", + "NQ", + "NQ-NL", + "NQHardNegatives" ], "source_urls": [ "https://mteb-leaderboard-backend.hf.space/v1/benchmarks/MTEB%28eng%2C%20v2%29/scores", @@ -618,19 +616,28 @@ ], "size_basis": "reported_rounded_total", "kind": "peer", - "on_displayed_frontier": true, + "on_displayed_frontier": false, "evidence_type": "registry_reported", - "observation_id": "registry_reported:abhinand/MedEmbed-small-v0.1" + "observation_id": "registry_reported:BAAI/bge-small-en-v1.5" }, { - "model": "avsolatorio/NoInstruct-small-Embedding-v0", + "model": "ManiacLabs/miniac-embed", "parameters_billion": 0.033, - "score": 60.11260000000001, + "score": 59.259771475443856, "open_weights": true, "modalities": [ "text" ], - "training_tasks": null, + "training_tasks": [ + "LoTTE", + "MSMARCO", + "PubMedQA", + "amazon-qa", + "cc_news", + "english-words-definitions", + "fineweb", + "trivia_qa" + ], "source_urls": [ "https://mteb-leaderboard-backend.hf.space/v1/benchmarks/MTEB%28eng%2C%20v2%29/scores", "https://mteb-leaderboard-backend.hf.space/v1/benchmarks/MTEB%28Multilingual%2C%20v2%29/scores" @@ -639,34 +646,28 @@ "kind": "peer", "on_displayed_frontier": false, "evidence_type": "registry_reported", - "observation_id": "registry_reported:avsolatorio/NoInstruct-small-Embedding-v0" + "observation_id": "registry_reported:ManiacLabs/miniac-embed" }, { - "model": "avsolatorio/GIST-small-Embedding-v0", + "model": "Snowflake/snowflake-arctic-embed-s", "parameters_billion": 0.033, - "score": 59.6633, + "score": 55.387100000000004, "open_weights": true, "modalities": [ "text" ], "training_tasks": [ - "AmazonCounterfactualClassification", - "AmazonPolarityClassification", - "AmazonReviewsClassification", - "Banking77Classification", - "DuRetrieval", - "EmotionClassification", - "ImdbClassification", - "MLQARetrieval", - "MTOPDomainClassification", - "MTOPIntentClassification", - "MassiveIntentClassification", - "MassiveScenarioClassification", + "FEVER", + "FEVER-NL", + "FEVERHardNegatives", + "HotPotQA", + "HotPotQA-PL", + "HotPotQAHardNegatives", + "HotpotQA-NL", "NQ", "NQ-NL", - "NQHardNegatives", - "ToxicConversationsClassification", - "TweetSentimentExtractionClassification" + "NQ-PL", + "NQHardNegatives" ], "source_urls": [ "https://mteb-leaderboard-backend.hf.space/v1/benchmarks/MTEB%28eng%2C%20v2%29/scores", @@ -676,23 +677,25 @@ "kind": "peer", "on_displayed_frontier": false, "evidence_type": "registry_reported", - "observation_id": "registry_reported:avsolatorio/GIST-small-Embedding-v0" + "observation_id": "registry_reported:Snowflake/snowflake-arctic-embed-s" }, { - "model": "BAAI/bge-small-en-v1.5", + "model": "abhinand/MedEmbed-small-v0.1", "parameters_billion": 0.033, - "score": 59.52876517144434, + "score": 60.553599999999996, "open_weights": true, "modalities": [ "text" ], "training_tasks": [ - "AmazonReviewsClassification", - "DuRetrieval", - "MLQARetrieval", - "NQ", - "NQ-NL", - "NQHardNegatives" + "ArguAna", + "ArguAna-NL", + "MedicalQARetrieval", + "NFCorpus", + "NFCorpus-NL", + "PublicHealthQA", + "TRECCOVID", + "TRECCOVID-NL" ], "source_urls": [ "https://mteb-leaderboard-backend.hf.space/v1/benchmarks/MTEB%28eng%2C%20v2%29/scores", @@ -700,27 +703,36 @@ ], "size_basis": "reported_rounded_total", "kind": "peer", - "on_displayed_frontier": false, + "on_displayed_frontier": true, "evidence_type": "registry_reported", - "observation_id": "registry_reported:BAAI/bge-small-en-v1.5" + "observation_id": "registry_reported:abhinand/MedEmbed-small-v0.1" }, { - "model": "ManiacLabs/miniac-embed", + "model": "avsolatorio/GIST-small-Embedding-v0", "parameters_billion": 0.033, - "score": 59.259771475443856, + "score": 59.6633, "open_weights": true, "modalities": [ "text" ], "training_tasks": [ - "LoTTE", - "MSMARCO", - "PubMedQA", - "amazon-qa", - "cc_news", - "english-words-definitions", - "fineweb", - "trivia_qa" + "AmazonCounterfactualClassification", + "AmazonPolarityClassification", + "AmazonReviewsClassification", + "Banking77Classification", + "DuRetrieval", + "EmotionClassification", + "ImdbClassification", + "MLQARetrieval", + "MTOPDomainClassification", + "MTOPIntentClassification", + "MassiveIntentClassification", + "MassiveScenarioClassification", + "NQ", + "NQ-NL", + "NQHardNegatives", + "ToxicConversationsClassification", + "TweetSentimentExtractionClassification" ], "source_urls": [ "https://mteb-leaderboard-backend.hf.space/v1/benchmarks/MTEB%28eng%2C%20v2%29/scores", @@ -730,12 +742,12 @@ "kind": "peer", "on_displayed_frontier": false, "evidence_type": "registry_reported", - "observation_id": "registry_reported:ManiacLabs/miniac-embed" + "observation_id": "registry_reported:avsolatorio/GIST-small-Embedding-v0" }, { - "model": "thenlper/gte-small", + "model": "avsolatorio/NoInstruct-small-Embedding-v0", "parameters_billion": 0.033, - "score": 58.3687, + "score": 60.11260000000001, "open_weights": true, "modalities": [ "text" @@ -749,12 +761,12 @@ "kind": "peer", "on_displayed_frontier": false, "evidence_type": "registry_reported", - "observation_id": "registry_reported:thenlper/gte-small" + "observation_id": "registry_reported:avsolatorio/NoInstruct-small-Embedding-v0" }, { - "model": "sentence-transformers/all-MiniLM-L12-v2", + "model": "intfloat/e5-small", "parameters_billion": 0.033, - "score": 57.444530301367216, + "score": 54.796, "open_weights": true, "modalities": [ "text" @@ -779,28 +791,27 @@ "kind": "peer", "on_displayed_frontier": false, "evidence_type": "registry_reported", - "observation_id": "registry_reported:sentence-transformers/all-MiniLM-L12-v2" + "observation_id": "registry_reported:intfloat/e5-small" }, { - "model": "Snowflake/snowflake-arctic-embed-s", + "model": "intfloat/e5-small-v2", "parameters_billion": 0.033, - "score": 55.387100000000004, + "score": 53.8363445953143, "open_weights": true, "modalities": [ "text" ], "training_tasks": [ - "FEVER", - "FEVER-NL", - "FEVERHardNegatives", - "HotPotQA", - "HotPotQA-PL", - "HotPotQAHardNegatives", - "HotpotQA-NL", + "MSMARCO", + "MSMARCO-PL", + "MSMARCOHardNegatives", "NQ", "NQ-NL", "NQ-PL", - "NQHardNegatives" + "NQHardNegatives", + "NanoMSMARCORetrieval", + "NanoNQRetrieval", + "mMARCO-NL" ], "source_urls": [ "https://mteb-leaderboard-backend.hf.space/v1/benchmarks/MTEB%28eng%2C%20v2%29/scores", @@ -810,27 +821,45 @@ "kind": "peer", "on_displayed_frontier": false, "evidence_type": "registry_reported", - "observation_id": "registry_reported:Snowflake/snowflake-arctic-embed-s" + "observation_id": "registry_reported:intfloat/e5-small-v2" }, { - "model": "intfloat/e5-small", + "model": "jinaai/jina-embeddings-v2-small-en", "parameters_billion": 0.033, - "score": 54.796, + "score": 48.3147, "open_weights": true, "modalities": [ "text" ], "training_tasks": [ + "AmazonQA", + "CC-NEWS", + "ELI5", + "ESCI", + "FEVER", + "Flickr30k", + "GooAQ", + "HotpotQA", "MSMARCO", - "MSMARCO-PL", - "MSMARCOHardNegatives", + "MsCocoCaptions", "NQ", - "NQ-NL", - "NQ-PL", - "NQHardNegatives", - "NanoMSMARCORetrieval", - "NanoNQRetrieval", - "mMARCO-NL" + "NegationDataset", + "PAQ", + "QuoraQA", + "S2ORC", + "SNLI", + "SentenceCompression", + "SimpleWikipedia", + "Specter", + "Squad2", + "StackExchange", + "Tmdb", + "TrivialQA", + "TweetQA", + "WikiAnswers", + "WikiHow", + "Xmarket", + "YahooAnswers" ], "source_urls": [ "https://mteb-leaderboard-backend.hf.space/v1/benchmarks/MTEB%28eng%2C%20v2%29/scores", @@ -840,12 +869,12 @@ "kind": "peer", "on_displayed_frontier": false, "evidence_type": "registry_reported", - "observation_id": "registry_reported:intfloat/e5-small" + "observation_id": "registry_reported:jinaai/jina-embeddings-v2-small-en" }, { - "model": "intfloat/e5-small-v2", + "model": "sentence-transformers/all-MiniLM-L12-v2", "parameters_billion": 0.033, - "score": 53.8363445953143, + "score": 57.444530301367216, "open_weights": true, "modalities": [ "text" @@ -870,46 +899,17 @@ "kind": "peer", "on_displayed_frontier": false, "evidence_type": "registry_reported", - "observation_id": "registry_reported:intfloat/e5-small-v2" + "observation_id": "registry_reported:sentence-transformers/all-MiniLM-L12-v2" }, { - "model": "jinaai/jina-embeddings-v2-small-en", + "model": "thenlper/gte-small", "parameters_billion": 0.033, - "score": 48.3147, + "score": 58.3687, "open_weights": true, "modalities": [ "text" ], - "training_tasks": [ - "AmazonQA", - "CC-NEWS", - "ELI5", - "ESCI", - "FEVER", - "Flickr30k", - "GooAQ", - "HotpotQA", - "MSMARCO", - "MsCocoCaptions", - "NQ", - "NegationDataset", - "PAQ", - "QuoraQA", - "S2ORC", - "SNLI", - "SentenceCompression", - "SimpleWikipedia", - "Specter", - "Squad2", - "StackExchange", - "Tmdb", - "TrivialQA", - "TweetQA", - "WikiAnswers", - "WikiHow", - "Xmarket", - "YahooAnswers" - ], + "training_tasks": null, "source_urls": [ "https://mteb-leaderboard-backend.hf.space/v1/benchmarks/MTEB%28eng%2C%20v2%29/scores", "https://mteb-leaderboard-backend.hf.space/v1/benchmarks/MTEB%28Multilingual%2C%20v2%29/scores" @@ -918,7 +918,7 @@ "kind": "peer", "on_displayed_frontier": false, "evidence_type": "registry_reported", - "observation_id": "registry_reported:jinaai/jina-embeddings-v2-small-en" + "observation_id": "registry_reported:thenlper/gte-small" }, { "model": "jinaai/jina-embedding-s-en-v1", @@ -1038,31 +1038,6 @@ "evidence_type": "registry_reported", "observation_id": "registry_reported:prdev/mini-gte" }, - { - "model": "codefuse-ai/F2LLM-v2-80M", - "revision": "6faa7738f343cd4321f9b2fa92fdb7cae3c117a5", - "parameters_billion": 0.080084288, - "parameters_exact": 80084288, - "score": 69.74221347455739, - "open_weights": true, - "modalities": [ - "text" - ], - "training_tasks": [ - "ArxivClusteringP2P", - "ArxivClusteringS2S" - ], - "source_urls": [ - "https://huggingface.co/codefuse-ai/F2LLM-v2-80M/tree/6faa7738f343cd4321f9b2fa92fdb7cae3c117a5" - ], - "size_basis": "independently_counted_complete_checkpoint", - "kind": "peer", - "evidence_type": "independent_rerun", - "observation_id": "rerun:F2LLM-v2-80M:6faa7738:ArXivHierarchicalClusteringP2P:2026-09-18", - "report_file": "peer-evaluations/f2llm-v2-80m-arxiv.json", - "protocol_note": "Official MTEB2.21 task defaults; BF16, task instruction+EOS,8192 tokens; model-specific settings differ from Vela.", - "on_displayed_frontier": true - }, { "model": "codefuse-ai/F2LLM-v2-80M", "parameters_billion": 0.080084288, @@ -1239,6 +1214,31 @@ "evidence_type": "registry_reported", "observation_id": "registry_reported:codefuse-ai/F2LLM-v2-80M" }, + { + "model": "codefuse-ai/F2LLM-v2-80M", + "revision": "6faa7738f343cd4321f9b2fa92fdb7cae3c117a5", + "parameters_billion": 0.080084288, + "parameters_exact": 80084288, + "score": 69.74221347455739, + "open_weights": true, + "modalities": [ + "text" + ], + "training_tasks": [ + "ArxivClusteringP2P", + "ArxivClusteringS2S" + ], + "source_urls": [ + "https://huggingface.co/codefuse-ai/F2LLM-v2-80M/tree/6faa7738f343cd4321f9b2fa92fdb7cae3c117a5" + ], + "size_basis": "independently_counted_complete_checkpoint", + "kind": "peer", + "evidence_type": "independent_rerun", + "observation_id": "rerun:F2LLM-v2-80M:6faa7738:ArXivHierarchicalClusteringP2P:2026-09-18", + "report_file": "peer-evaluations/f2llm-v2-80m-arxiv.json", + "protocol_note": "Official MTEB2.21 task defaults; BF16, task instruction+EOS,8192 tokens; model-specific settings differ from Vela.", + "on_displayed_frontier": true + }, { "model": "ibm-granite/granite-embedding-97m-multilingual-r2", "parameters_billion": 0.097, @@ -1351,6 +1351,27 @@ "evidence_type": "registry_reported", "observation_id": "registry_reported:minishlab/M2V_base_glove_subword" }, + { + "model": "cnmoro/static-nomic-384-pten-v2-st", + "parameters_billion": 0.106, + "score": 49.7028, + "open_weights": true, + "modalities": [ + "text" + ], + "training_tasks": [ + "MSMARCO" + ], + "source_urls": [ + "https://mteb-leaderboard-backend.hf.space/v1/benchmarks/MTEB%28eng%2C%20v2%29/scores", + "https://mteb-leaderboard-backend.hf.space/v1/benchmarks/MTEB%28Multilingual%2C%20v2%29/scores" + ], + "size_basis": "reported_rounded_total", + "kind": "peer", + "on_displayed_frontier": false, + "evidence_type": "registry_reported", + "observation_id": "registry_reported:cnmoro/static-nomic-384-pten-v2-st" + }, { "model": "hotchpotch/bekko-embedding-v1-a8m", "parameters_billion": 0.106, @@ -1379,15 +1400,37 @@ "observation_id": "registry_reported:hotchpotch/bekko-embedding-v1-a8m" }, { - "model": "cnmoro/static-nomic-384-pten-v2-st", - "parameters_billion": 0.106, - "score": 49.7028, + "model": "DeepPavlov/distilrubert-small-cased-conversational", + "parameters_billion": 0.107, + "score": 48.8063, + "open_weights": true, + "modalities": [ + "text" + ], + "training_tasks": null, + "source_urls": [ + "https://mteb-leaderboard-backend.hf.space/v1/benchmarks/MTEB%28eng%2C%20v2%29/scores", + "https://mteb-leaderboard-backend.hf.space/v1/benchmarks/MTEB%28Multilingual%2C%20v2%29/scores" + ], + "size_basis": "reported_rounded_total", + "kind": "peer", + "on_displayed_frontier": false, + "evidence_type": "registry_reported", + "observation_id": "registry_reported:DeepPavlov/distilrubert-small-cased-conversational" + }, + { + "model": "DeepPavlov/rubert-base-cased-sentence", + "parameters_billion": 0.107, + "score": 49.744899999999994, "open_weights": true, "modalities": [ "text" ], "training_tasks": [ - "MSMARCO" + "I", + "L", + "N", + "X" ], "source_urls": [ "https://mteb-leaderboard-backend.hf.space/v1/benchmarks/MTEB%28eng%2C%20v2%29/scores", @@ -1397,7 +1440,7 @@ "kind": "peer", "on_displayed_frontier": false, "evidence_type": "registry_reported", - "observation_id": "registry_reported:cnmoro/static-nomic-384-pten-v2-st" + "observation_id": "registry_reported:DeepPavlov/rubert-base-cased-sentence" }, { "model": "ibm-granite/granite-embedding-107m-multilingual", @@ -1444,18 +1487,20 @@ "observation_id": "registry_reported:ibm-granite/granite-embedding-107m-multilingual" }, { - "model": "DeepPavlov/rubert-base-cased-sentence", - "parameters_billion": 0.107, - "score": 49.744899999999994, + "model": "sentence-transformers/static-similarity-mrl-multilingual-v1", + "parameters_billion": 0.108, + "score": 47.7162, "open_weights": true, "modalities": [ "text" ], "training_tasks": [ - "I", - "L", - "N", - "X" + "NanoQuoraRetrieval", + "Quora-NL", + "QuoraRetrieval", + "StackExchangeClustering.v2", + "StackExchangeClusteringP2P.v2", + "TatoebaBitextMining" ], "source_urls": [ "https://mteb-leaderboard-backend.hf.space/v1/benchmarks/MTEB%28eng%2C%20v2%29/scores", @@ -1465,17 +1510,24 @@ "kind": "peer", "on_displayed_frontier": false, "evidence_type": "registry_reported", - "observation_id": "registry_reported:DeepPavlov/rubert-base-cased-sentence" + "observation_id": "registry_reported:sentence-transformers/static-similarity-mrl-multilingual-v1" }, { - "model": "DeepPavlov/distilrubert-small-cased-conversational", - "parameters_billion": 0.107, - "score": 48.8063, + "model": "BAAI/bge-base-en-v1.5", + "parameters_billion": 0.109, + "score": 60.682568390916295, "open_weights": true, "modalities": [ "text" ], - "training_tasks": null, + "training_tasks": [ + "AmazonReviewsClassification", + "DuRetrieval", + "MLQARetrieval", + "NQ", + "NQ-NL", + "NQHardNegatives" + ], "source_urls": [ "https://mteb-leaderboard-backend.hf.space/v1/benchmarks/MTEB%28eng%2C%20v2%29/scores", "https://mteb-leaderboard-backend.hf.space/v1/benchmarks/MTEB%28Multilingual%2C%20v2%29/scores" @@ -1484,38 +1536,12 @@ "kind": "peer", "on_displayed_frontier": false, "evidence_type": "registry_reported", - "observation_id": "registry_reported:DeepPavlov/distilrubert-small-cased-conversational" + "observation_id": "registry_reported:BAAI/bge-base-en-v1.5" }, { - "model": "sentence-transformers/static-similarity-mrl-multilingual-v1", - "parameters_billion": 0.108, - "score": 47.7162, - "open_weights": true, - "modalities": [ - "text" - ], - "training_tasks": [ - "NanoQuoraRetrieval", - "Quora-NL", - "QuoraRetrieval", - "StackExchangeClustering.v2", - "StackExchangeClusteringP2P.v2", - "TatoebaBitextMining" - ], - "source_urls": [ - "https://mteb-leaderboard-backend.hf.space/v1/benchmarks/MTEB%28eng%2C%20v2%29/scores", - "https://mteb-leaderboard-backend.hf.space/v1/benchmarks/MTEB%28Multilingual%2C%20v2%29/scores" - ], - "size_basis": "reported_rounded_total", - "kind": "peer", - "on_displayed_frontier": false, - "evidence_type": "registry_reported", - "observation_id": "registry_reported:sentence-transformers/static-similarity-mrl-multilingual-v1" - }, - { - "model": "sentence-transformers/all-mpnet-base-v2", - "parameters_billion": 0.109, - "score": 61.47339210179387, + "model": "Omartificial-Intelligence-Space/Arabic-mpnet-base-all-nli-triplet", + "parameters_billion": 0.109, + "score": 50.11409999999999, "open_weights": true, "modalities": [ "text" @@ -1540,22 +1566,27 @@ "kind": "peer", "on_displayed_frontier": false, "evidence_type": "registry_reported", - "observation_id": "registry_reported:sentence-transformers/all-mpnet-base-v2" + "observation_id": "registry_reported:Omartificial-Intelligence-Space/Arabic-mpnet-base-all-nli-triplet" }, { - "model": "BAAI/bge-base-en-v1.5", + "model": "Snowflake/snowflake-arctic-embed-m", "parameters_billion": 0.109, - "score": 60.682568390916295, + "score": 57.497299999999996, "open_weights": true, "modalities": [ "text" ], "training_tasks": [ - "AmazonReviewsClassification", - "DuRetrieval", - "MLQARetrieval", + "FEVER", + "FEVER-NL", + "FEVERHardNegatives", + "HotPotQA", + "HotPotQA-PL", + "HotPotQAHardNegatives", + "HotpotQA-NL", "NQ", "NQ-NL", + "NQ-PL", "NQHardNegatives" ], "source_urls": [ @@ -1566,17 +1597,29 @@ "kind": "peer", "on_displayed_frontier": false, "evidence_type": "registry_reported", - "observation_id": "registry_reported:BAAI/bge-base-en-v1.5" + "observation_id": "registry_reported:Snowflake/snowflake-arctic-embed-m" }, { - "model": "infgrad/stella-base-en-v2", + "model": "Snowflake/snowflake-arctic-embed-m-v1.5", "parameters_billion": 0.109, - "score": 59.992999999999995, + "score": 56.388799999999996, "open_weights": true, "modalities": [ "text" ], - "training_tasks": null, + "training_tasks": [ + "FEVER", + "FEVER-NL", + "FEVERHardNegatives", + "HotPotQA", + "HotPotQA-PL", + "HotPotQAHardNegatives", + "HotpotQA-NL", + "NQ", + "NQ-NL", + "NQ-PL", + "NQHardNegatives" + ], "source_urls": [ "https://mteb-leaderboard-backend.hf.space/v1/benchmarks/MTEB%28eng%2C%20v2%29/scores", "https://mteb-leaderboard-backend.hf.space/v1/benchmarks/MTEB%28Multilingual%2C%20v2%29/scores" @@ -1585,7 +1628,7 @@ "kind": "peer", "on_displayed_frontier": false, "evidence_type": "registry_reported", - "observation_id": "registry_reported:infgrad/stella-base-en-v2" + "observation_id": "registry_reported:Snowflake/snowflake-arctic-embed-m-v1.5" }, { "model": "avsolatorio/GIST-Embedding-v0", @@ -1625,9 +1668,9 @@ "observation_id": "registry_reported:avsolatorio/GIST-Embedding-v0" }, { - "model": "thenlper/gte-base", + "model": "infgrad/stella-base-en-v2", "parameters_billion": 0.109, - "score": 59.3139, + "score": 59.992999999999995, "open_weights": true, "modalities": [ "text" @@ -1641,12 +1684,12 @@ "kind": "peer", "on_displayed_frontier": false, "evidence_type": "registry_reported", - "observation_id": "registry_reported:thenlper/gte-base" + "observation_id": "registry_reported:infgrad/stella-base-en-v2" }, { - "model": "intfloat/e5-base-v2", + "model": "intfloat/e5-base", "parameters_billion": 0.109, - "score": 57.96287997107758, + "score": 55.16, "open_weights": true, "modalities": [ "text" @@ -1671,59 +1714,27 @@ "kind": "peer", "on_displayed_frontier": false, "evidence_type": "registry_reported", - "observation_id": "registry_reported:intfloat/e5-base-v2" - }, - { - "model": "Snowflake/snowflake-arctic-embed-m", - "parameters_billion": 0.109, - "score": 57.497299999999996, - "open_weights": true, - "modalities": [ - "text" - ], - "training_tasks": [ - "FEVER", - "FEVER-NL", - "FEVERHardNegatives", - "HotPotQA", - "HotPotQA-PL", - "HotPotQAHardNegatives", - "HotpotQA-NL", - "NQ", - "NQ-NL", - "NQ-PL", - "NQHardNegatives" - ], - "source_urls": [ - "https://mteb-leaderboard-backend.hf.space/v1/benchmarks/MTEB%28eng%2C%20v2%29/scores", - "https://mteb-leaderboard-backend.hf.space/v1/benchmarks/MTEB%28Multilingual%2C%20v2%29/scores" - ], - "size_basis": "reported_rounded_total", - "kind": "peer", - "on_displayed_frontier": false, - "evidence_type": "registry_reported", - "observation_id": "registry_reported:Snowflake/snowflake-arctic-embed-m" + "observation_id": "registry_reported:intfloat/e5-base" }, { - "model": "Snowflake/snowflake-arctic-embed-m-v1.5", + "model": "intfloat/e5-base-v2", "parameters_billion": 0.109, - "score": 56.388799999999996, + "score": 57.96287997107758, "open_weights": true, "modalities": [ "text" ], "training_tasks": [ - "FEVER", - "FEVER-NL", - "FEVERHardNegatives", - "HotPotQA", - "HotPotQA-PL", - "HotPotQAHardNegatives", - "HotpotQA-NL", + "MSMARCO", + "MSMARCO-PL", + "MSMARCOHardNegatives", "NQ", "NQ-NL", "NQ-PL", - "NQHardNegatives" + "NQHardNegatives", + "NanoMSMARCORetrieval", + "NanoNQRetrieval", + "mMARCO-NL" ], "source_urls": [ "https://mteb-leaderboard-backend.hf.space/v1/benchmarks/MTEB%28eng%2C%20v2%29/scores", @@ -1733,12 +1744,12 @@ "kind": "peer", "on_displayed_frontier": false, "evidence_type": "registry_reported", - "observation_id": "registry_reported:Snowflake/snowflake-arctic-embed-m-v1.5" + "observation_id": "registry_reported:intfloat/e5-base-v2" }, { - "model": "intfloat/e5-base", + "model": "sentence-transformers/all-mpnet-base-v2", "parameters_billion": 0.109, - "score": 55.16, + "score": 61.47339210179387, "open_weights": true, "modalities": [ "text" @@ -1763,28 +1774,17 @@ "kind": "peer", "on_displayed_frontier": false, "evidence_type": "registry_reported", - "observation_id": "registry_reported:intfloat/e5-base" + "observation_id": "registry_reported:sentence-transformers/all-mpnet-base-v2" }, { - "model": "Omartificial-Intelligence-Space/Arabic-mpnet-base-all-nli-triplet", + "model": "thenlper/gte-base", "parameters_billion": 0.109, - "score": 50.11409999999999, + "score": 59.3139, "open_weights": true, "modalities": [ "text" ], - "training_tasks": [ - "MSMARCO", - "MSMARCO-PL", - "MSMARCOHardNegatives", - "NQ", - "NQ-NL", - "NQ-PL", - "NQHardNegatives", - "NanoMSMARCORetrieval", - "NanoNQRetrieval", - "mMARCO-NL" - ], + "training_tasks": null, "source_urls": [ "https://mteb-leaderboard-backend.hf.space/v1/benchmarks/MTEB%28eng%2C%20v2%29/scores", "https://mteb-leaderboard-backend.hf.space/v1/benchmarks/MTEB%28Multilingual%2C%20v2%29/scores" @@ -1793,7 +1793,7 @@ "kind": "peer", "on_displayed_frontier": false, "evidence_type": "registry_reported", - "observation_id": "registry_reported:Omartificial-Intelligence-Space/Arabic-mpnet-base-all-nli-triplet" + "observation_id": "registry_reported:thenlper/gte-base" }, { "model": "jinaai/jina-embedding-b-en-v1", @@ -1869,36 +1869,6 @@ "evidence_type": "registry_reported", "observation_id": "registry_reported:dwzhu/e5-base-4k" }, - { - "model": "sdadas/mmlw-e5-small", - "parameters_billion": 0.118, - "score": 59.169000000000004, - "open_weights": true, - "modalities": [ - "text" - ], - "training_tasks": [ - "MSMARCO", - "MSMARCO-PL", - "MSMARCOHardNegatives", - "NQ", - "NQ-NL", - "NQ-PL", - "NQHardNegatives", - "NanoMSMARCORetrieval", - "NanoNQRetrieval", - "mMARCO-NL" - ], - "source_urls": [ - "https://mteb-leaderboard-backend.hf.space/v1/benchmarks/MTEB%28eng%2C%20v2%29/scores", - "https://mteb-leaderboard-backend.hf.space/v1/benchmarks/MTEB%28Multilingual%2C%20v2%29/scores" - ], - "size_basis": "reported_rounded_total", - "kind": "peer", - "on_displayed_frontier": false, - "evidence_type": "registry_reported", - "observation_id": "registry_reported:sdadas/mmlw-e5-small" - }, { "model": "Lajavaness/bilingual-embedding-small", "parameters_billion": 0.118, @@ -1923,9 +1893,9 @@ "observation_id": "registry_reported:Lajavaness/bilingual-embedding-small" }, { - "model": "sentence-transformers/paraphrase-multilingual-MiniLM-L12-v2", + "model": "Omartificial-Intelligence-Space/Arabic-MiniLM-L12-v2-all-nli-triplet", "parameters_billion": 0.118, - "score": 53.61794435755446, + "score": 52.6815, "open_weights": true, "modalities": [ "text" @@ -1950,7 +1920,7 @@ "kind": "peer", "on_displayed_frontier": false, "evidence_type": "registry_reported", - "observation_id": "registry_reported:sentence-transformers/paraphrase-multilingual-MiniLM-L12-v2" + "observation_id": "registry_reported:Omartificial-Intelligence-Space/Arabic-MiniLM-L12-v2-all-nli-triplet" }, { "model": "intfloat/multilingual-e5-small", @@ -1996,9 +1966,9 @@ "observation_id": "registry_reported:intfloat/multilingual-e5-small" }, { - "model": "Omartificial-Intelligence-Space/Arabic-MiniLM-L12-v2-all-nli-triplet", + "model": "sdadas/mmlw-e5-small", "parameters_billion": 0.118, - "score": 52.6815, + "score": 59.169000000000004, "open_weights": true, "modalities": [ "text" @@ -2023,7 +1993,37 @@ "kind": "peer", "on_displayed_frontier": false, "evidence_type": "registry_reported", - "observation_id": "registry_reported:Omartificial-Intelligence-Space/Arabic-MiniLM-L12-v2-all-nli-triplet" + "observation_id": "registry_reported:sdadas/mmlw-e5-small" + }, + { + "model": "sentence-transformers/paraphrase-multilingual-MiniLM-L12-v2", + "parameters_billion": 0.118, + "score": 53.61794435755446, + "open_weights": true, + "modalities": [ + "text" + ], + "training_tasks": [ + "MSMARCO", + "MSMARCO-PL", + "MSMARCOHardNegatives", + "NQ", + "NQ-NL", + "NQ-PL", + "NQHardNegatives", + "NanoMSMARCORetrieval", + "NanoNQRetrieval", + "mMARCO-NL" + ], + "source_urls": [ + "https://mteb-leaderboard-backend.hf.space/v1/benchmarks/MTEB%28eng%2C%20v2%29/scores", + "https://mteb-leaderboard-backend.hf.space/v1/benchmarks/MTEB%28Multilingual%2C%20v2%29/scores" + ], + "size_basis": "reported_rounded_total", + "kind": "peer", + "on_displayed_frontier": false, + "evidence_type": "registry_reported", + "observation_id": "registry_reported:sentence-transformers/paraphrase-multilingual-MiniLM-L12-v2" }, { "model": "shibing624/text2vec-base-multilingual", @@ -2083,15 +2083,17 @@ "observation_id": "registry_reported:hotchpotch/bekko-embedding-v1-a25m" }, { - "model": "sdadas/mmlw-roberta-base", + "model": "deepvk/deberta-v1-base", "parameters_billion": 0.124, - "score": 60.6571, + "score": 49.234899999999996, "open_weights": true, "modalities": [ "text" ], "training_tasks": [ - "MSMARCO" + "RiaNewsRetrieval", + "WikipediaRerankingMultilingual", + "WikipediaRetrievalMultilingual" ], "source_urls": [ "https://mteb-leaderboard-backend.hf.space/v1/benchmarks/MTEB%28eng%2C%20v2%29/scores", @@ -2101,20 +2103,18 @@ "kind": "peer", "on_displayed_frontier": false, "evidence_type": "registry_reported", - "observation_id": "registry_reported:sdadas/mmlw-roberta-base" + "observation_id": "registry_reported:deepvk/deberta-v1-base" }, { - "model": "deepvk/deberta-v1-base", + "model": "sdadas/mmlw-roberta-base", "parameters_billion": 0.124, - "score": 49.234899999999996, + "score": 60.6571, "open_weights": true, "modalities": [ "text" ], "training_tasks": [ - "RiaNewsRetrieval", - "WikipediaRerankingMultilingual", - "WikipediaRetrievalMultilingual" + "MSMARCO" ], "source_urls": [ "https://mteb-leaderboard-backend.hf.space/v1/benchmarks/MTEB%28eng%2C%20v2%29/scores", @@ -2124,7 +2124,7 @@ "kind": "peer", "on_displayed_frontier": false, "evidence_type": "registry_reported", - "observation_id": "registry_reported:deepvk/deberta-v1-base" + "observation_id": "registry_reported:sdadas/mmlw-roberta-base" }, { "model": "ibm-granite/granite-embedding-125m-english", @@ -2197,9 +2197,9 @@ "observation_id": "registry_reported:minishlab/potion-multilingual-128M" }, { - "model": "sergeyzh/LaBSE-ru-turbo", + "model": "cointegrated/LaBSE-en-ru", "parameters_billion": 0.129, - "score": 53.0679, + "score": 51.546400000000006, "open_weights": true, "modalities": [ "text" @@ -2213,12 +2213,12 @@ "kind": "peer", "on_displayed_frontier": false, "evidence_type": "registry_reported", - "observation_id": "registry_reported:sergeyzh/LaBSE-ru-turbo" + "observation_id": "registry_reported:cointegrated/LaBSE-en-ru" }, { - "model": "cointegrated/LaBSE-en-ru", + "model": "sergeyzh/LaBSE-ru-turbo", "parameters_billion": 0.129, - "score": 51.546400000000006, + "score": 53.0679, "open_weights": true, "modalities": [ "text" @@ -2232,31 +2232,26 @@ "kind": "peer", "on_displayed_frontier": false, "evidence_type": "registry_reported", - "observation_id": "registry_reported:cointegrated/LaBSE-en-ru" + "observation_id": "registry_reported:sergeyzh/LaBSE-ru-turbo" }, { - "model": "llm-semantic-router/Vela-1.0-Omni-Nano", - "parameters_billion": 0.13395648, - "parameters_exact": 133956480, - "native_fingerprint": "fb9cd39c9980a085b75456c715c2295b997784fe733314be68c13635f6384edf", - "score": 65.25395409281045, + "model": "Omartificial-Intelligence-Space/Arabert-all-nli-triplet-Matryoshka", + "parameters_billion": 0.135, + "score": 48.3142, "open_weights": true, "modalities": [ - "text", - "image", - "audio" + "text" ], - "size_basis": "counted_all_native_model_parameters", - "kind": "nano", - "evaluated_task_revision": "0bbdb47bcbe3a90093699aefeed338a0f28a7ee8", - "report_url": "https://huggingface.co/llm-semantic-router/Vela-1.0-Omni-Nano/blob/ed7263b501430a011d37b147d6e264ad8354983b/benchmarks/mteb-eng-v2.json", + "training_tasks": null, + "source_urls": [ + "https://mteb-leaderboard-backend.hf.space/v1/benchmarks/MTEB%28eng%2C%20v2%29/scores", + "https://mteb-leaderboard-backend.hf.space/v1/benchmarks/MTEB%28Multilingual%2C%20v2%29/scores" + ], + "size_basis": "reported_rounded_total", + "kind": "peer", "on_displayed_frontier": false, - "evaluated_native_fingerprint": "b596f1dbab89369555e0a72a0d6854d780e305686750949b308c8f167993bdeb", - "evaluation_applicability": "Unchanged exact text/audio computation; see the current Nano component-equivalence report.", - "component_equivalence_url": "https://huggingface.co/llm-semantic-router/Vela-1.0-Omni-Nano/blob/main/benchmarks/component-equivalence.json", - "evaluated_revision": "70c3412bb0ddfb19386d673d4a102c753a574f1b", - "evidence_type": "vela_release_evaluation", - "observation_id": "vela_release_evaluation:llm-semantic-router/Vela-1.0-Omni-Nano" + "evidence_type": "registry_reported", + "observation_id": "registry_reported:Omartificial-Intelligence-Space/Arabert-all-nli-triplet-Matryoshka" }, { "model": "silma-ai/silma-embeddding-matryoshka-v0.1", @@ -2278,14 +2273,26 @@ "observation_id": "registry_reported:silma-ai/silma-embeddding-matryoshka-v0.1" }, { - "model": "Omartificial-Intelligence-Space/Arabert-all-nli-triplet-Matryoshka", - "parameters_billion": 0.135, - "score": 48.3142, + "model": "Snowflake/snowflake-arctic-embed-m-long", + "parameters_billion": 0.137, + "score": 57.266799999999996, "open_weights": true, "modalities": [ "text" ], - "training_tasks": null, + "training_tasks": [ + "FEVER", + "FEVER-NL", + "FEVERHardNegatives", + "HotPotQA", + "HotPotQA-PL", + "HotPotQAHardNegatives", + "HotpotQA-NL", + "NQ", + "NQ-NL", + "NQ-PL", + "NQHardNegatives" + ], "source_urls": [ "https://mteb-leaderboard-backend.hf.space/v1/benchmarks/MTEB%28eng%2C%20v2%29/scores", "https://mteb-leaderboard-backend.hf.space/v1/benchmarks/MTEB%28Multilingual%2C%20v2%29/scores" @@ -2294,51 +2301,45 @@ "kind": "peer", "on_displayed_frontier": false, "evidence_type": "registry_reported", - "observation_id": "registry_reported:Omartificial-Intelligence-Space/Arabert-all-nli-triplet-Matryoshka" + "observation_id": "registry_reported:Snowflake/snowflake-arctic-embed-m-long" }, { - "model": "nomic-ai/nomic-embed-text-v1.5", + "model": "jinaai/jina-embeddings-v2-base-en", "parameters_billion": 0.137, - "score": 60.08051520875409, + "score": 47.1547, "open_weights": true, "modalities": [ "text" ], "training_tasks": [ - "AmazonCounterfactualClassification", - "AmazonPolarityClassification", - "AmazonReviewsClassification", - "COIRCodeSearchNetRetrieval", - "CodeSearchNetCCRetrieval", + "AmazonQA", + "CC-NEWS", + "ELI5", + "ESCI", "FEVER", - "FEVER-NL", - "FEVERHardNegatives", - "FQuADRetrieval", - "HotPotQA", - "HotPotQA-PL", - "HotPotQAHardNegatives", - "HotpotQA-NL", + "Flickr30k", + "GooAQ", + "HotpotQA", "MSMARCO", - "MSMARCOHardNegatives", + "MsCocoCaptions", "NQ", - "NQ-NL", - "NQ-PL", - "NQHardNegatives", - "NanoMSMARCORetrieval", - "NanoNQRetrieval", - "NanoQuoraRetrieval", - "Quora-NL", - "QuoraRetrieval", - "RedditClustering", - "RedditClustering.v2", - "RedditClusteringP2P", - "RedditClusteringP2P.v2", - "StackExchangeClustering.v2", - "StackExchangeClusteringP2P.v2", - "WikipediaRerankingMultilingual", - "WikipediaRetrievalMultilingual", - "YahooAnswersTopicsClassification", - "mMARCO-NL" + "NegationDataset", + "PAQ", + "QuoraQA", + "S2ORC", + "SNLI", + "SentenceCompression", + "SimpleWikipedia", + "Specter", + "Squad2", + "StackExchange", + "Tmdb", + "TrivialQA", + "TweetQA", + "WikiAnswers", + "WikiHow", + "Xmarket", + "YahooAnswers" ], "source_urls": [ "https://mteb-leaderboard-backend.hf.space/v1/benchmarks/MTEB%28eng%2C%20v2%29/scores", @@ -2348,7 +2349,7 @@ "kind": "peer", "on_displayed_frontier": false, "evidence_type": "registry_reported", - "observation_id": "registry_reported:nomic-ai/nomic-embed-text-v1.5" + "observation_id": "registry_reported:jinaai/jina-embeddings-v2-base-en" }, { "model": "nomic-ai/nomic-embed-text-v1", @@ -2459,25 +2460,48 @@ "observation_id": "registry_reported:nomic-ai/nomic-embed-text-v1-ablated" }, { - "model": "Snowflake/snowflake-arctic-embed-m-long", + "model": "nomic-ai/nomic-embed-text-v1-unsupervised", "parameters_billion": 0.137, - "score": 57.266799999999996, + "score": 56.7453, "open_weights": true, "modalities": [ "text" ], "training_tasks": [ + "AmazonCounterfactualClassification", + "AmazonPolarityClassification", + "AmazonReviewsClassification", + "COIRCodeSearchNetRetrieval", + "CodeSearchNetCCRetrieval", "FEVER", "FEVER-NL", "FEVERHardNegatives", + "FQuADRetrieval", "HotPotQA", "HotPotQA-PL", "HotPotQAHardNegatives", "HotpotQA-NL", + "MSMARCO", + "MSMARCOHardNegatives", "NQ", "NQ-NL", "NQ-PL", - "NQHardNegatives" + "NQHardNegatives", + "NanoMSMARCORetrieval", + "NanoNQRetrieval", + "NanoQuoraRetrieval", + "Quora-NL", + "QuoraRetrieval", + "RedditClustering", + "RedditClustering.v2", + "RedditClusteringP2P", + "RedditClusteringP2P.v2", + "StackExchangeClustering.v2", + "StackExchangeClusteringP2P.v2", + "WikipediaRerankingMultilingual", + "WikipediaRetrievalMultilingual", + "YahooAnswersTopicsClassification", + "mMARCO-NL" ], "source_urls": [ "https://mteb-leaderboard-backend.hf.space/v1/benchmarks/MTEB%28eng%2C%20v2%29/scores", @@ -2487,12 +2511,12 @@ "kind": "peer", "on_displayed_frontier": false, "evidence_type": "registry_reported", - "observation_id": "registry_reported:Snowflake/snowflake-arctic-embed-m-long" + "observation_id": "registry_reported:nomic-ai/nomic-embed-text-v1-unsupervised" }, { - "model": "nomic-ai/nomic-embed-text-v1-unsupervised", + "model": "nomic-ai/nomic-embed-text-v1.5", "parameters_billion": 0.137, - "score": 56.7453, + "score": 60.08051520875409, "open_weights": true, "modalities": [ "text" @@ -2541,55 +2565,32 @@ "kind": "peer", "on_displayed_frontier": false, "evidence_type": "registry_reported", - "observation_id": "registry_reported:nomic-ai/nomic-embed-text-v1-unsupervised" + "observation_id": "registry_reported:nomic-ai/nomic-embed-text-v1.5" }, { - "model": "jinaai/jina-embeddings-v2-base-en", - "parameters_billion": 0.137, - "score": 47.1547, + "model": "llm-semantic-router/Vela-1.0-Omni-Nano", + "parameters_billion": 0.144554112, + "parameters_exact": 144554112, + "score": 64.88847259326147, "open_weights": true, "modalities": [ - "text" - ], - "training_tasks": [ - "AmazonQA", - "CC-NEWS", - "ELI5", - "ESCI", - "FEVER", - "Flickr30k", - "GooAQ", - "HotpotQA", - "MSMARCO", - "MsCocoCaptions", - "NQ", - "NegationDataset", - "PAQ", - "QuoraQA", - "S2ORC", - "SNLI", - "SentenceCompression", - "SimpleWikipedia", - "Specter", - "Squad2", - "StackExchange", - "Tmdb", - "TrivialQA", - "TweetQA", - "WikiAnswers", - "WikiHow", - "Xmarket", - "YahooAnswers" - ], - "source_urls": [ - "https://mteb-leaderboard-backend.hf.space/v1/benchmarks/MTEB%28eng%2C%20v2%29/scores", - "https://mteb-leaderboard-backend.hf.space/v1/benchmarks/MTEB%28Multilingual%2C%20v2%29/scores" + "text", + "image", + "audio" ], - "size_basis": "reported_rounded_total", - "kind": "peer", + "size_basis": "counted_all_native_model_parameters", + "kind": "nano", + "evaluated_task_revision": "0bbdb47bcbe3a90093699aefeed338a0f28a7ee8", "on_displayed_frontier": false, - "evidence_type": "registry_reported", - "observation_id": "registry_reported:jinaai/jina-embeddings-v2-base-en" + "evidence_type": "exact_text_component_applicability", + "observation_id": "vela_release_evaluation:llm-semantic-router/Vela-1.0-Omni-Nano", + "native_artifact_sha256": "96d76f97478e114b816df3ab1b3c6947e4c5ef0babfb40ab865d7e3c838d34c4", + "revision": "self", + "evaluated_native_artifact_sha256": "29994142677559ca657c27b657eda0dddbf583bd4cf42a3a31095ab3f99d1646", + "evaluated_revision": "29994142677559ca657c27b657eda0dddbf583bd4cf42a3a31095ab3f99d1646", + "evaluated_model": "local/Vela-Omni-Nano-GIST-Rotation", + "revision_semantics": "Evaluated revisions are native artifact fingerprints; applies-to identity is the current whole artifact.", + "report": "benchmarks/mteb-eng-v2.json" }, { "model": "ibm-granite/granite-embedding-english-r2", @@ -2877,115 +2878,43 @@ "observation_id": "registry_reported:jinaai/jina-embeddings-v5-text-nano" }, { - "model": "microsoft/harrier-oss-v1-270m", + "model": "BidirLM/BidirLM-270M-Embedding", "parameters_billion": 0.268, - "score": 61.477000000000004, + "score": 61.2727, "open_weights": true, "modalities": [ "text" ], "training_tasks": [ - "AlphaNLI", - "AmazonCounterfactualClassification", - "AppsRetrieval", - "BornholmBitextMining", - "COIRCodeSearchNetRetrieval", - "CSFDSKMovieReviewSentimentClassification", - "CataloniaTweetClassification", - "CodeFeedbackMT", - "CodeFeedbackST", - "CodeSearchNetCCRetrieval", - "CodeSearchNetRetrieval", - "CodeTransOceanContestRetrieval", - "CodeTransOceanDLRetrieval", - "CosQARetrieval", - "CyrillicTurkicLangClassification", - "CzechProductReviewSentimentClassification", - "DBpediaClassification", + "AFQMC", + "ATEC", + "AdvertiseGen", + "BQ", + "CAIL2019-SCM", + "CHEF", + "CINLID", + "ChatMed_Consult_Dataset", + "ChineseSTS", + "CmedqaRetrieval", + "Cmnli", + "CodeFeedback", + "DRCD", + "DuRetrieval", + "ELI5_custom", + "ESCIReranking", "EmotionClassification", - "FEVER", - "FiQA2018", - "FinParaSTS", - "GermanSTSBenchmark", - "GreekLegalCodeClassification", - "HellaSwag", - "HotpotQAHardNegatives", - "IndicLangClassification", + "Expertqa", + "FollowIR", + "GooAQ", + "InF-IR", + "JW300", + "LCSTS", + "MAmmoTH2", + "MEDI2BGE", "MIRACLRetrieval", "MSMARCO", - "MrTidyRetrieval", - "NFCorpus", - "NQ", - "NordicLangClassification", - "NusaParagraphEmotionClassification", - "OdiaNewsClassification", - "PAC", - "PawsXPairClassification", - "PoemSentimentClassification", - "Quail", - "QuoraRetrieval", - "STSB", - "ScalaClassification", - "SciFact", - "SemRel24STS", - "SpartQA", - "StackOverflowQARetrieval", - "StatcanDialogueDatasetRetrieval", - "SwissJudgementClassification", - "SyntheticText2SQLRetrieval", - "TERRa", - "TempReasonL1", - "ToxicConversationsClassification", - "WinoGrande" - ], - "source_urls": [ - "https://mteb-leaderboard-backend.hf.space/v1/benchmarks/MTEB%28eng%2C%20v2%29/scores", - "https://mteb-leaderboard-backend.hf.space/v1/benchmarks/MTEB%28Multilingual%2C%20v2%29/scores" - ], - "size_basis": "reported_rounded_total", - "kind": "peer", - "on_displayed_frontier": false, - "evidence_type": "registry_reported", - "observation_id": "registry_reported:microsoft/harrier-oss-v1-270m" - }, - { - "model": "BidirLM/BidirLM-270M-Embedding", - "parameters_billion": 0.268, - "score": 61.2727, - "open_weights": true, - "modalities": [ - "text" - ], - "training_tasks": [ - "AFQMC", - "ATEC", - "AdvertiseGen", - "BQ", - "CAIL2019-SCM", - "CHEF", - "CINLID", - "ChatMed_Consult_Dataset", - "ChineseSTS", - "CmedqaRetrieval", - "Cmnli", - "CodeFeedback", - "DRCD", - "DuRetrieval", - "ELI5_custom", - "ESCIReranking", - "EmotionClassification", - "Expertqa", - "FollowIR", - "GooAQ", - "InF-IR", - "JW300", - "LCSTS", - "MAmmoTH2", - "MEDI2BGE", - "MIRACLRetrieval", - "MSMARCO", - "Multi-CPR", - "MultiLongDocReranking", + "Multi-CPR", + "MultiLongDocReranking", "NFCorpus", "NaturalQuestions", "OPUS-100", @@ -3039,19 +2968,66 @@ "observation_id": "registry_reported:BidirLM/BidirLM-270M-Embedding" }, { - "model": "keeeeenw/MicroLlama-text-embedding", - "parameters_billion": 0.272, - "score": 56.81659433257183, + "model": "microsoft/harrier-oss-v1-270m", + "parameters_billion": 0.268, + "score": 61.477000000000004, "open_weights": true, "modalities": [ "text" ], "training_tasks": [ + "AlphaNLI", + "AmazonCounterfactualClassification", + "AppsRetrieval", + "BornholmBitextMining", + "COIRCodeSearchNetRetrieval", + "CSFDSKMovieReviewSentimentClassification", + "CataloniaTweetClassification", + "CodeFeedbackMT", + "CodeFeedbackST", + "CodeSearchNetCCRetrieval", + "CodeSearchNetRetrieval", + "CodeTransOceanContestRetrieval", + "CodeTransOceanDLRetrieval", + "CosQARetrieval", + "CyrillicTurkicLangClassification", + "CzechProductReviewSentimentClassification", + "DBpediaClassification", + "EmotionClassification", + "FEVER", + "FiQA2018", + "FinParaSTS", + "GermanSTSBenchmark", + "GreekLegalCodeClassification", + "HellaSwag", + "HotpotQAHardNegatives", + "IndicLangClassification", + "MIRACLRetrieval", + "MSMARCO", + "MrTidyRetrieval", + "NFCorpus", "NQ", - "NQ-NL", - "NQ-PL", - "NQHardNegatives", - "NanoNQRetrieval" + "NordicLangClassification", + "NusaParagraphEmotionClassification", + "OdiaNewsClassification", + "PAC", + "PawsXPairClassification", + "PoemSentimentClassification", + "Quail", + "QuoraRetrieval", + "STSB", + "ScalaClassification", + "SciFact", + "SemRel24STS", + "SpartQA", + "StackOverflowQARetrieval", + "StatcanDialogueDatasetRetrieval", + "SwissJudgementClassification", + "SyntheticText2SQLRetrieval", + "TERRa", + "TempReasonL1", + "ToxicConversationsClassification", + "WinoGrande" ], "source_urls": [ "https://mteb-leaderboard-backend.hf.space/v1/benchmarks/MTEB%28eng%2C%20v2%29/scores", @@ -3061,27 +3037,22 @@ "kind": "peer", "on_displayed_frontier": false, "evidence_type": "registry_reported", - "observation_id": "registry_reported:keeeeenw/MicroLlama-text-embedding" + "observation_id": "registry_reported:microsoft/harrier-oss-v1-270m" }, { - "model": "sdadas/mmlw-e5-base", - "parameters_billion": 0.278, - "score": 59.867000000000004, + "model": "keeeeenw/MicroLlama-text-embedding", + "parameters_billion": 0.272, + "score": 56.81659433257183, "open_weights": true, "modalities": [ "text" ], "training_tasks": [ - "MSMARCO", - "MSMARCO-PL", - "MSMARCOHardNegatives", "NQ", "NQ-NL", "NQ-PL", "NQHardNegatives", - "NanoMSMARCORetrieval", - "NanoNQRetrieval", - "mMARCO-NL" + "NanoNQRetrieval" ], "source_urls": [ "https://mteb-leaderboard-backend.hf.space/v1/benchmarks/MTEB%28eng%2C%20v2%29/scores", @@ -3091,17 +3062,76 @@ "kind": "peer", "on_displayed_frontier": false, "evidence_type": "registry_reported", - "observation_id": "registry_reported:sdadas/mmlw-e5-base" + "observation_id": "registry_reported:keeeeenw/MicroLlama-text-embedding" }, { - "model": "deepfile/embedder-100p", + "model": "Gameselo/STS-multilingual-mpnet-base-v2", "parameters_billion": 0.278, - "score": 58.5614, + "score": 51.5432, "open_weights": true, "modalities": [ "text" ], - "training_tasks": null, + "training_tasks": [ + "AlphaNLI", + "AmazonCounterfactualClassification", + "AmazonPolarityClassification", + "AmazonReviewsClassification", + "ArguAna", + "ArxivClusteringP2P", + "ArxivClusteringS2S", + "AskUbuntuDupQuestions", + "BIOSSES", + "Banking77Classification", + "BiorxivClusteringP2P", + "BiorxivClusteringS2S", + "CQADupstackRetrieval", + "ClimateFEVER", + "DBPedia", + "EmotionClassification", + "FEVER", + "FiQA2018", + "HotpotQA", + "ImdbClassification", + "MSMARCO", + "MTOPDomainClassification", + "MTOPIntentClassification", + "MassiveIntentClassification", + "MassiveScenarioClassification", + "MedrxivClusteringP2P", + "MedrxivClusteringS2S", + "MindSmallReranking", + "NFCorpus", + "NQ", + "QuoraRetrieval", + "RTE3", + "RedditClustering", + "RedditClusteringP2P", + "SCIDOCS", + "SICK-R", + "STS12", + "STS13", + "STS14", + "STS15", + "STS16", + "STS17", + "STS22", + "STSBenchmark", + "SciDocsRR", + "SciFact", + "SprintDuplicateQuestions", + "StackExchangeClustering", + "StackExchangeClusteringP2P", + "StackOverflowDupQuestions", + "SummEval", + "TRECCOVID", + "Touche2020", + "ToxicConversationsClassification", + "TweetSentimentExtractionClassification", + "TwentyNewsgroupsClustering", + "TwitterSemEval2015", + "TwitterURLCorpus" + ], "source_urls": [ "https://mteb-leaderboard-backend.hf.space/v1/benchmarks/MTEB%28eng%2C%20v2%29/scores", "https://mteb-leaderboard-backend.hf.space/v1/benchmarks/MTEB%28Multilingual%2C%20v2%29/scores" @@ -3110,7 +3140,7 @@ "kind": "peer", "on_displayed_frontier": false, "evidence_type": "registry_reported", - "observation_id": "registry_reported:deepfile/embedder-100p" + "observation_id": "registry_reported:Gameselo/STS-multilingual-mpnet-base-v2" }, { "model": "Lajavaness/bilingual-embedding-base", @@ -3136,36 +3166,23 @@ "observation_id": "registry_reported:Lajavaness/bilingual-embedding-base" }, { - "model": "intfloat/multilingual-e5-base", + "model": "Omartificial-Intelligence-Space/Arabic-all-nli-triplet-Matryoshka", "parameters_billion": 0.278, - "score": 56.683136606944075, + "score": 52.660700000000006, "open_weights": true, "modalities": [ "text" ], "training_tasks": [ - "FEVER", - "FEVER-NL", - "FEVER-PL", - "FEVERHardNegatives", - "HotpotQA", - "HotpotQA-NL", - "HotpotQA-PL", - "HotpotQAHardNegatives", - "MIRACLReranking", - "MIRACLRetrieval", - "MIRACLRetrievalHardNegatives", "MSMARCO", "MSMARCO-PL", "MSMARCOHardNegatives", - "MrTidyRetrieval", "NQ", "NQ-NL", "NQ-PL", "NQHardNegatives", "NanoMSMARCORetrieval", "NanoNQRetrieval", - "XQuADRetrieval", "mMARCO-NL" ], "source_urls": [ @@ -3176,28 +3193,17 @@ "kind": "peer", "on_displayed_frontier": false, "evidence_type": "registry_reported", - "observation_id": "registry_reported:intfloat/multilingual-e5-base" + "observation_id": "registry_reported:Omartificial-Intelligence-Space/Arabic-all-nli-triplet-Matryoshka" }, { - "model": "sentence-transformers/paraphrase-multilingual-mpnet-base-v2", + "model": "deepfile/embedder-100p", "parameters_billion": 0.278, - "score": 55.34276670530136, + "score": 58.5614, "open_weights": true, "modalities": [ "text" ], - "training_tasks": [ - "MSMARCO", - "MSMARCO-PL", - "MSMARCOHardNegatives", - "NQ", - "NQ-NL", - "NQ-PL", - "NQHardNegatives", - "NanoMSMARCORetrieval", - "NanoNQRetrieval", - "mMARCO-NL" - ], + "training_tasks": null, "source_urls": [ "https://mteb-leaderboard-backend.hf.space/v1/benchmarks/MTEB%28eng%2C%20v2%29/scores", "https://mteb-leaderboard-backend.hf.space/v1/benchmarks/MTEB%28Multilingual%2C%20v2%29/scores" @@ -3206,7 +3212,7 @@ "kind": "peer", "on_displayed_frontier": false, "evidence_type": "registry_reported", - "observation_id": "registry_reported:sentence-transformers/paraphrase-multilingual-mpnet-base-v2" + "observation_id": "registry_reported:deepfile/embedder-100p" }, { "model": "ibm-granite/granite-embedding-278m-multilingual", @@ -3253,23 +3259,36 @@ "observation_id": "registry_reported:ibm-granite/granite-embedding-278m-multilingual" }, { - "model": "Omartificial-Intelligence-Space/Arabic-all-nli-triplet-Matryoshka", + "model": "intfloat/multilingual-e5-base", "parameters_billion": 0.278, - "score": 52.660700000000006, + "score": 56.683136606944075, "open_weights": true, "modalities": [ "text" ], "training_tasks": [ + "FEVER", + "FEVER-NL", + "FEVER-PL", + "FEVERHardNegatives", + "HotpotQA", + "HotpotQA-NL", + "HotpotQA-PL", + "HotpotQAHardNegatives", + "MIRACLReranking", + "MIRACLRetrieval", + "MIRACLRetrievalHardNegatives", "MSMARCO", "MSMARCO-PL", "MSMARCOHardNegatives", + "MrTidyRetrieval", "NQ", "NQ-NL", "NQ-PL", "NQHardNegatives", "NanoMSMARCORetrieval", "NanoNQRetrieval", + "XQuADRetrieval", "mMARCO-NL" ], "source_urls": [ @@ -3280,75 +3299,27 @@ "kind": "peer", "on_displayed_frontier": false, "evidence_type": "registry_reported", - "observation_id": "registry_reported:Omartificial-Intelligence-Space/Arabic-all-nli-triplet-Matryoshka" + "observation_id": "registry_reported:intfloat/multilingual-e5-base" }, { - "model": "Gameselo/STS-multilingual-mpnet-base-v2", + "model": "sdadas/mmlw-e5-base", "parameters_billion": 0.278, - "score": 51.5432, + "score": 59.867000000000004, "open_weights": true, "modalities": [ "text" ], "training_tasks": [ - "AlphaNLI", - "AmazonCounterfactualClassification", - "AmazonPolarityClassification", - "AmazonReviewsClassification", - "ArguAna", - "ArxivClusteringP2P", - "ArxivClusteringS2S", - "AskUbuntuDupQuestions", - "BIOSSES", - "Banking77Classification", - "BiorxivClusteringP2P", - "BiorxivClusteringS2S", - "CQADupstackRetrieval", - "ClimateFEVER", - "DBPedia", - "EmotionClassification", - "FEVER", - "FiQA2018", - "HotpotQA", - "ImdbClassification", "MSMARCO", - "MTOPDomainClassification", - "MTOPIntentClassification", - "MassiveIntentClassification", - "MassiveScenarioClassification", - "MedrxivClusteringP2P", - "MedrxivClusteringS2S", - "MindSmallReranking", - "NFCorpus", + "MSMARCO-PL", + "MSMARCOHardNegatives", "NQ", - "QuoraRetrieval", - "RTE3", - "RedditClustering", - "RedditClusteringP2P", - "SCIDOCS", - "SICK-R", - "STS12", - "STS13", - "STS14", - "STS15", - "STS16", - "STS17", - "STS22", - "STSBenchmark", - "SciDocsRR", - "SciFact", - "SprintDuplicateQuestions", - "StackExchangeClustering", - "StackExchangeClusteringP2P", - "StackOverflowDupQuestions", - "SummEval", - "TRECCOVID", - "Touche2020", - "ToxicConversationsClassification", - "TweetSentimentExtractionClassification", - "TwentyNewsgroupsClustering", - "TwitterSemEval2015", - "TwitterURLCorpus" + "NQ-NL", + "NQ-PL", + "NQHardNegatives", + "NanoMSMARCORetrieval", + "NanoNQRetrieval", + "mMARCO-NL" ], "source_urls": [ "https://mteb-leaderboard-backend.hf.space/v1/benchmarks/MTEB%28eng%2C%20v2%29/scores", @@ -3358,7 +3329,37 @@ "kind": "peer", "on_displayed_frontier": false, "evidence_type": "registry_reported", - "observation_id": "registry_reported:Gameselo/STS-multilingual-mpnet-base-v2" + "observation_id": "registry_reported:sdadas/mmlw-e5-base" + }, + { + "model": "sentence-transformers/paraphrase-multilingual-mpnet-base-v2", + "parameters_billion": 0.278, + "score": 55.34276670530136, + "open_weights": true, + "modalities": [ + "text" + ], + "training_tasks": [ + "MSMARCO", + "MSMARCO-PL", + "MSMARCOHardNegatives", + "NQ", + "NQ-NL", + "NQ-PL", + "NQHardNegatives", + "NanoMSMARCORetrieval", + "NanoNQRetrieval", + "mMARCO-NL" + ], + "source_urls": [ + "https://mteb-leaderboard-backend.hf.space/v1/benchmarks/MTEB%28eng%2C%20v2%29/scores", + "https://mteb-leaderboard-backend.hf.space/v1/benchmarks/MTEB%28Multilingual%2C%20v2%29/scores" + ], + "size_basis": "reported_rounded_total", + "kind": "peer", + "on_displayed_frontier": false, + "evidence_type": "registry_reported", + "observation_id": "registry_reported:sentence-transformers/paraphrase-multilingual-mpnet-base-v2" }, { "model": "Alibaba-NLP/gte-multilingual-base", @@ -3669,15 +3670,20 @@ "observation_id": "registry_reported:codefuse-ai/F2LLM-v2-330M" }, { - "model": "w601sxs/b1ade-embed", + "model": "BAAI/bge-large-en-v1.5", "parameters_billion": 0.335, - "score": 61.2698, + "score": 60.58357599909132, "open_weights": true, "modalities": [ "text" ], "training_tasks": [ - "MSMARCO" + "AmazonReviewsClassification", + "DuRetrieval", + "MLQARetrieval", + "NQ", + "NQ-NL", + "NQHardNegatives" ], "source_urls": [ "https://mteb-leaderboard-backend.hf.space/v1/benchmarks/MTEB%28eng%2C%20v2%29/scores", @@ -3687,22 +3693,27 @@ "kind": "peer", "on_displayed_frontier": false, "evidence_type": "registry_reported", - "observation_id": "registry_reported:w601sxs/b1ade-embed" + "observation_id": "registry_reported:BAAI/bge-large-en-v1.5" }, { - "model": "BAAI/bge-large-en-v1.5", + "model": "Snowflake/snowflake-arctic-embed-l", "parameters_billion": 0.335, - "score": 60.58357599909132, + "score": 56.3437, "open_weights": true, "modalities": [ "text" ], "training_tasks": [ - "AmazonReviewsClassification", - "DuRetrieval", - "MLQARetrieval", + "FEVER", + "FEVER-NL", + "FEVERHardNegatives", + "HotPotQA", + "HotPotQA-PL", + "HotPotQAHardNegatives", + "HotpotQA-NL", "NQ", "NQ-NL", + "NQ-PL", "NQHardNegatives" ], "source_urls": [ @@ -3713,18 +3724,20 @@ "kind": "peer", "on_displayed_frontier": false, "evidence_type": "registry_reported", - "observation_id": "registry_reported:BAAI/bge-large-en-v1.5" + "observation_id": "registry_reported:Snowflake/snowflake-arctic-embed-l" }, { - "model": "mixedbread-ai/mxbai-embed-large-v1", + "model": "WhereIsAI/UAE-Large-V1", "parameters_billion": 0.335, - "score": 60.10625739174296, + "score": 59.37910271512732, "open_weights": true, "modalities": [ "text" ], "training_tasks": [ - "MSMARCO" + "MNLI", + "NLI", + "SNLI" ], "source_urls": [ "https://mteb-leaderboard-backend.hf.space/v1/benchmarks/MTEB%28eng%2C%20v2%29/scores", @@ -3734,7 +3747,7 @@ "kind": "peer", "on_displayed_frontier": false, "evidence_type": "registry_reported", - "observation_id": "registry_reported:mixedbread-ai/mxbai-embed-large-v1" + "observation_id": "registry_reported:WhereIsAI/UAE-Large-V1" }, { "model": "avsolatorio/GIST-large-Embedding-v0", @@ -3774,18 +3787,14 @@ "observation_id": "registry_reported:avsolatorio/GIST-large-Embedding-v0" }, { - "model": "WhereIsAI/UAE-Large-V1", + "model": "consciousAI/cai-lunaris-text-embeddings", "parameters_billion": 0.335, - "score": 59.37910271512732, + "score": 53.353300000000004, "open_weights": true, "modalities": [ "text" ], - "training_tasks": [ - "MNLI", - "NLI", - "SNLI" - ], + "training_tasks": null, "source_urls": [ "https://mteb-leaderboard-backend.hf.space/v1/benchmarks/MTEB%28eng%2C%20v2%29/scores", "https://mteb-leaderboard-backend.hf.space/v1/benchmarks/MTEB%28Multilingual%2C%20v2%29/scores" @@ -3794,17 +3803,28 @@ "kind": "peer", "on_displayed_frontier": false, "evidence_type": "registry_reported", - "observation_id": "registry_reported:WhereIsAI/UAE-Large-V1" + "observation_id": "registry_reported:consciousAI/cai-lunaris-text-embeddings" }, { - "model": "thenlper/gte-large", + "model": "intfloat/e5-large", "parameters_billion": 0.335, - "score": 58.6399, + "score": 57.22239999999999, "open_weights": true, "modalities": [ "text" ], - "training_tasks": null, + "training_tasks": [ + "MSMARCO", + "MSMARCO-PL", + "MSMARCOHardNegatives", + "NQ", + "NQ-NL", + "NQ-PL", + "NQHardNegatives", + "NanoMSMARCORetrieval", + "NanoNQRetrieval", + "mMARCO-NL" + ], "source_urls": [ "https://mteb-leaderboard-backend.hf.space/v1/benchmarks/MTEB%28eng%2C%20v2%29/scores", "https://mteb-leaderboard-backend.hf.space/v1/benchmarks/MTEB%28Multilingual%2C%20v2%29/scores" @@ -3813,7 +3833,7 @@ "kind": "peer", "on_displayed_frontier": false, "evidence_type": "registry_reported", - "observation_id": "registry_reported:thenlper/gte-large" + "observation_id": "registry_reported:intfloat/e5-large" }, { "model": "intfloat/e5-large-v2", @@ -3846,24 +3866,15 @@ "observation_id": "registry_reported:intfloat/e5-large-v2" }, { - "model": "intfloat/e5-large", + "model": "mixedbread-ai/mxbai-embed-large-v1", "parameters_billion": 0.335, - "score": 57.22239999999999, + "score": 60.10625739174296, "open_weights": true, "modalities": [ "text" ], "training_tasks": [ - "MSMARCO", - "MSMARCO-PL", - "MSMARCOHardNegatives", - "NQ", - "NQ-NL", - "NQ-PL", - "NQHardNegatives", - "NanoMSMARCORetrieval", - "NanoNQRetrieval", - "mMARCO-NL" + "MSMARCO" ], "source_urls": [ "https://mteb-leaderboard-backend.hf.space/v1/benchmarks/MTEB%28eng%2C%20v2%29/scores", @@ -3873,29 +3884,17 @@ "kind": "peer", "on_displayed_frontier": false, "evidence_type": "registry_reported", - "observation_id": "registry_reported:intfloat/e5-large" + "observation_id": "registry_reported:mixedbread-ai/mxbai-embed-large-v1" }, { - "model": "Snowflake/snowflake-arctic-embed-l", + "model": "thenlper/gte-large", "parameters_billion": 0.335, - "score": 56.3437, + "score": 58.6399, "open_weights": true, "modalities": [ "text" ], - "training_tasks": [ - "FEVER", - "FEVER-NL", - "FEVERHardNegatives", - "HotPotQA", - "HotPotQA-PL", - "HotPotQAHardNegatives", - "HotpotQA-NL", - "NQ", - "NQ-NL", - "NQ-PL", - "NQHardNegatives" - ], + "training_tasks": null, "source_urls": [ "https://mteb-leaderboard-backend.hf.space/v1/benchmarks/MTEB%28eng%2C%20v2%29/scores", "https://mteb-leaderboard-backend.hf.space/v1/benchmarks/MTEB%28Multilingual%2C%20v2%29/scores" @@ -3904,17 +3903,19 @@ "kind": "peer", "on_displayed_frontier": false, "evidence_type": "registry_reported", - "observation_id": "registry_reported:Snowflake/snowflake-arctic-embed-l" + "observation_id": "registry_reported:thenlper/gte-large" }, { - "model": "consciousAI/cai-lunaris-text-embeddings", + "model": "w601sxs/b1ade-embed", "parameters_billion": 0.335, - "score": 53.353300000000004, + "score": 61.2698, "open_weights": true, "modalities": [ "text" ], - "training_tasks": null, + "training_tasks": [ + "MSMARCO" + ], "source_urls": [ "https://mteb-leaderboard-backend.hf.space/v1/benchmarks/MTEB%28eng%2C%20v2%29/scores", "https://mteb-leaderboard-backend.hf.space/v1/benchmarks/MTEB%28Multilingual%2C%20v2%29/scores" @@ -3923,7 +3924,7 @@ "kind": "peer", "on_displayed_frontier": false, "evidence_type": "registry_reported", - "observation_id": "registry_reported:consciousAI/cai-lunaris-text-embeddings" + "observation_id": "registry_reported:w601sxs/b1ade-embed" }, { "model": "aari1995/German_Semantic_STS_V2", @@ -4028,23 +4029,14 @@ "observation_id": "registry_reported:ai-forever/ru-en-RoSBERTa" }, { - "model": "deepvk/USER-base", + "model": "ai-forever/sbert_large_mt_nlu_ru", "parameters_billion": 0.427, - "score": 49.7383, + "score": 46.6827, "open_weights": true, "modalities": [ "text" ], - "training_tasks": [ - "BibleNLPBitextMining", - "MIRACL", - "MLSUMClusteringP2P", - "MLSUMClusteringP2P.v2", - "MLSUMClusteringS2S", - "MLSUMClusteringS2S.v2", - "MrTidyRetrieval", - "TERRa" - ], + "training_tasks": null, "source_urls": [ "https://mteb-leaderboard-backend.hf.space/v1/benchmarks/MTEB%28eng%2C%20v2%29/scores", "https://mteb-leaderboard-backend.hf.space/v1/benchmarks/MTEB%28Multilingual%2C%20v2%29/scores" @@ -4053,7 +4045,7 @@ "kind": "peer", "on_displayed_frontier": false, "evidence_type": "registry_reported", - "observation_id": "registry_reported:deepvk/USER-base" + "observation_id": "registry_reported:ai-forever/sbert_large_mt_nlu_ru" }, { "model": "ai-forever/sbert_large_nlu_ru", @@ -4075,14 +4067,23 @@ "observation_id": "registry_reported:ai-forever/sbert_large_nlu_ru" }, { - "model": "ai-forever/sbert_large_mt_nlu_ru", + "model": "deepvk/USER-base", "parameters_billion": 0.427, - "score": 46.6827, + "score": 49.7383, "open_weights": true, "modalities": [ "text" ], - "training_tasks": null, + "training_tasks": [ + "BibleNLPBitextMining", + "MIRACL", + "MLSUMClusteringP2P", + "MLSUMClusteringP2P.v2", + "MLSUMClusteringS2S", + "MLSUMClusteringS2S.v2", + "MrTidyRetrieval", + "TERRa" + ], "source_urls": [ "https://mteb-leaderboard-backend.hf.space/v1/benchmarks/MTEB%28eng%2C%20v2%29/scores", "https://mteb-leaderboard-backend.hf.space/v1/benchmarks/MTEB%28Multilingual%2C%20v2%29/scores" @@ -4091,7 +4092,7 @@ "kind": "peer", "on_displayed_frontier": false, "evidence_type": "registry_reported", - "observation_id": "registry_reported:ai-forever/sbert_large_mt_nlu_ru" + "observation_id": "registry_reported:deepvk/USER-base" }, { "model": "NovaSearch/stella_en_400M_v5", @@ -4175,16 +4176,14 @@ "observation_id": "registry_reported:NovaSearch/stella_en_400M_v5" }, { - "model": "sdadas/mmlw-roberta-large", + "model": "VPLabs/SearchMap_Preview", "parameters_billion": 0.435, - "score": 60.8105, + "score": 58.234300000000005, "open_weights": true, "modalities": [ "text" ], - "training_tasks": [ - "MSMARCO" - ], + "training_tasks": null, "source_urls": [ "https://mteb-leaderboard-backend.hf.space/v1/benchmarks/MTEB%28eng%2C%20v2%29/scores", "https://mteb-leaderboard-backend.hf.space/v1/benchmarks/MTEB%28Multilingual%2C%20v2%29/scores" @@ -4193,17 +4192,19 @@ "kind": "peer", "on_displayed_frontier": false, "evidence_type": "registry_reported", - "observation_id": "registry_reported:sdadas/mmlw-roberta-large" + "observation_id": "registry_reported:VPLabs/SearchMap_Preview" }, { - "model": "VPLabs/SearchMap_Preview", + "model": "sdadas/mmlw-roberta-large", "parameters_billion": 0.435, - "score": 58.234300000000005, + "score": 60.8105, "open_weights": true, "modalities": [ "text" ], - "training_tasks": null, + "training_tasks": [ + "MSMARCO" + ], "source_urls": [ "https://mteb-leaderboard-backend.hf.space/v1/benchmarks/MTEB%28eng%2C%20v2%29/scores", "https://mteb-leaderboard-backend.hf.space/v1/benchmarks/MTEB%28Multilingual%2C%20v2%29/scores" @@ -4212,12 +4213,12 @@ "kind": "peer", "on_displayed_frontier": false, "evidence_type": "registry_reported", - "observation_id": "registry_reported:VPLabs/SearchMap_Preview" + "observation_id": "registry_reported:sdadas/mmlw-roberta-large" }, { - "model": "sentence-transformers/LaBSE", + "model": "Omartificial-Intelligence-Space/Arabic-labse-Matryoshka", "parameters_billion": 0.471, - "score": 53.44270249920056, + "score": 52.144999999999996, "open_weights": true, "modalities": [ "text" @@ -4231,12 +4232,12 @@ "kind": "peer", "on_displayed_frontier": false, "evidence_type": "registry_reported", - "observation_id": "registry_reported:sentence-transformers/LaBSE" + "observation_id": "registry_reported:Omartificial-Intelligence-Space/Arabic-labse-Matryoshka" }, { - "model": "Omartificial-Intelligence-Space/Arabic-labse-Matryoshka", + "model": "sentence-transformers/LaBSE", "parameters_billion": 0.471, - "score": 52.144999999999996, + "score": 53.44270249920056, "open_weights": true, "modalities": [ "text" @@ -4250,7 +4251,7 @@ "kind": "peer", "on_displayed_frontier": false, "evidence_type": "registry_reported", - "observation_id": "registry_reported:Omartificial-Intelligence-Space/Arabic-labse-Matryoshka" + "observation_id": "registry_reported:sentence-transformers/LaBSE" }, { "model": "nomic-ai/nomic-embed-text-v2-moe", @@ -4310,26 +4311,23 @@ "observation_id": "registry_reported:mixedbread-ai/deepset-mxbai-embed-de-large-v1" }, { - "model": "KaLM-Embedding/KaLM-embedding-multilingual-mini-instruct-v2.5", + "model": "HIT-TMG/KaLM-embedding-multilingual-mini-instruct-v1", "parameters_billion": 0.494, - "score": 63.45525248201162, + "score": 58.914699999999996, "open_weights": true, "modalities": [ "text" ], "training_tasks": [ - "ATEC", "AmazonCounterfactualClassification", "AmazonPolarityClassification", "AmazonReviewsClassification", "ArxivClusteringP2P", "ArxivClusteringP2P.v2", "ArxivClusteringS2S", - "BQ", "Banking77Classification", "BiorxivClusteringP2P.v2", "BiorxivClusteringS2S.v2", - "CQADupstack", "CodeFeedbackMT", "CodeFeedbackST", "ContractNLIConfidentialityOfAgreementLegalBenchClassification", @@ -4385,15 +4383,10 @@ "NanoQuoraRetrieval", "PawsXPairClassification", "QuoraRetrieval", - "Reddit-Clustering", - "Reddit-Clustering-P2P", "SciFact", - "Stackexchange-Clustering", - "Stackexchange-Clustering-P2P", "TRECCOVID", "ToxicConversationsClassification", "TweetSentimentExtractionClassification", - "TwentyNewsgroups-Clustering", "YahooAnswersTopicsClassification", "mMARCO-NL" ], @@ -4405,7 +4398,7 @@ "kind": "peer", "on_displayed_frontier": false, "evidence_type": "registry_reported", - "observation_id": "registry_reported:KaLM-Embedding/KaLM-embedding-multilingual-mini-instruct-v2.5" + "observation_id": "registry_reported:HIT-TMG/KaLM-embedding-multilingual-mini-instruct-v1" }, { "model": "HIT-TMG/KaLM-embedding-multilingual-mini-v1", @@ -4498,23 +4491,26 @@ "observation_id": "registry_reported:HIT-TMG/KaLM-embedding-multilingual-mini-v1" }, { - "model": "HIT-TMG/KaLM-embedding-multilingual-mini-instruct-v1", + "model": "KaLM-Embedding/KaLM-embedding-multilingual-mini-instruct-v2.5", "parameters_billion": 0.494, - "score": 58.914699999999996, + "score": 63.45525248201162, "open_weights": true, "modalities": [ "text" ], "training_tasks": [ + "ATEC", "AmazonCounterfactualClassification", "AmazonPolarityClassification", "AmazonReviewsClassification", "ArxivClusteringP2P", "ArxivClusteringP2P.v2", "ArxivClusteringS2S", + "BQ", "Banking77Classification", "BiorxivClusteringP2P.v2", "BiorxivClusteringS2S.v2", + "CQADupstack", "CodeFeedbackMT", "CodeFeedbackST", "ContractNLIConfidentialityOfAgreementLegalBenchClassification", @@ -4570,10 +4566,15 @@ "NanoQuoraRetrieval", "PawsXPairClassification", "QuoraRetrieval", + "Reddit-Clustering", + "Reddit-Clustering-P2P", "SciFact", + "Stackexchange-Clustering", + "Stackexchange-Clustering-P2P", "TRECCOVID", "ToxicConversationsClassification", "TweetSentimentExtractionClassification", + "TwentyNewsgroups-Clustering", "YahooAnswersTopicsClassification", "mMARCO-NL" ], @@ -4585,7 +4586,7 @@ "kind": "peer", "on_displayed_frontier": false, "evidence_type": "registry_reported", - "observation_id": "registry_reported:HIT-TMG/KaLM-embedding-multilingual-mini-instruct-v1" + "observation_id": "registry_reported:KaLM-Embedding/KaLM-embedding-multilingual-mini-instruct-v2.5" }, { "model": "izhx/udever-bloom-560m", @@ -4609,9 +4610,51 @@ "observation_id": "registry_reported:izhx/udever-bloom-560m" }, { - "model": "intfloat/multilingual-e5-large-instruct", + "model": "Lajavaness/bilingual-embedding-large", "parameters_billion": 0.56, - "score": 62.534993503603, + "score": 57.353, + "open_weights": true, + "modalities": [ + "text" + ], + "training_tasks": [ + "STSBenchmark", + "STSBenchmarkMultilingualSTS", + "XNLI" + ], + "source_urls": [ + "https://mteb-leaderboard-backend.hf.space/v1/benchmarks/MTEB%28eng%2C%20v2%29/scores", + "https://mteb-leaderboard-backend.hf.space/v1/benchmarks/MTEB%28Multilingual%2C%20v2%29/scores" + ], + "size_basis": "reported_rounded_total", + "kind": "peer", + "on_displayed_frontier": false, + "evidence_type": "registry_reported", + "observation_id": "registry_reported:Lajavaness/bilingual-embedding-large" + }, + { + "model": "OrdalieTech/Solon-embeddings-large-0.1", + "parameters_billion": 0.56, + "score": 56.89959999999999, + "open_weights": true, + "modalities": [ + "text" + ], + "training_tasks": null, + "source_urls": [ + "https://mteb-leaderboard-backend.hf.space/v1/benchmarks/MTEB%28eng%2C%20v2%29/scores", + "https://mteb-leaderboard-backend.hf.space/v1/benchmarks/MTEB%28Multilingual%2C%20v2%29/scores" + ], + "size_basis": "reported_rounded_total", + "kind": "peer", + "on_displayed_frontier": false, + "evidence_type": "registry_reported", + "observation_id": "registry_reported:OrdalieTech/Solon-embeddings-large-0.1" + }, + { + "model": "intfloat/multilingual-e5-large", + "parameters_billion": 0.56, + "score": 55.6856, "open_weights": true, "modalities": [ "text" @@ -4649,26 +4692,39 @@ "kind": "peer", "on_displayed_frontier": false, "evidence_type": "registry_reported", - "observation_id": "registry_reported:intfloat/multilingual-e5-large-instruct" + "observation_id": "registry_reported:intfloat/multilingual-e5-large" }, { - "model": "sdadas/mmlw-e5-large", + "model": "intfloat/multilingual-e5-large-instruct", "parameters_billion": 0.56, - "score": 59.872400000000006, + "score": 62.534993503603, "open_weights": true, "modalities": [ "text" ], "training_tasks": [ + "FEVER", + "FEVER-NL", + "FEVER-PL", + "FEVERHardNegatives", + "HotpotQA", + "HotpotQA-NL", + "HotpotQA-PL", + "HotpotQAHardNegatives", + "MIRACLReranking", + "MIRACLRetrieval", + "MIRACLRetrievalHardNegatives", "MSMARCO", "MSMARCO-PL", "MSMARCOHardNegatives", + "MrTidyRetrieval", "NQ", "NQ-NL", "NQ-PL", "NQHardNegatives", "NanoMSMARCORetrieval", "NanoNQRetrieval", + "XQuADRetrieval", "mMARCO-NL" ], "source_urls": [ @@ -4679,21 +4735,17 @@ "kind": "peer", "on_displayed_frontier": false, "evidence_type": "registry_reported", - "observation_id": "registry_reported:sdadas/mmlw-e5-large" + "observation_id": "registry_reported:intfloat/multilingual-e5-large-instruct" }, { - "model": "Lajavaness/bilingual-embedding-large", + "model": "omarelshehy/arabic-english-sts-matryoshka", "parameters_billion": 0.56, - "score": 57.353, + "score": 52.528200000000005, "open_weights": true, "modalities": [ "text" ], - "training_tasks": [ - "STSBenchmark", - "STSBenchmarkMultilingualSTS", - "XNLI" - ], + "training_tasks": null, "source_urls": [ "https://mteb-leaderboard-backend.hf.space/v1/benchmarks/MTEB%28eng%2C%20v2%29/scores", "https://mteb-leaderboard-backend.hf.space/v1/benchmarks/MTEB%28Multilingual%2C%20v2%29/scores" @@ -4702,17 +4754,28 @@ "kind": "peer", "on_displayed_frontier": false, "evidence_type": "registry_reported", - "observation_id": "registry_reported:Lajavaness/bilingual-embedding-large" + "observation_id": "registry_reported:omarelshehy/arabic-english-sts-matryoshka" }, { - "model": "OrdalieTech/Solon-embeddings-large-0.1", + "model": "sdadas/mmlw-e5-large", "parameters_billion": 0.56, - "score": 56.89959999999999, + "score": 59.872400000000006, "open_weights": true, "modalities": [ "text" ], - "training_tasks": null, + "training_tasks": [ + "MSMARCO", + "MSMARCO-PL", + "MSMARCOHardNegatives", + "NQ", + "NQ-NL", + "NQ-PL", + "NQHardNegatives", + "NanoMSMARCORetrieval", + "NanoNQRetrieval", + "mMARCO-NL" + ], "source_urls": [ "https://mteb-leaderboard-backend.hf.space/v1/benchmarks/MTEB%28eng%2C%20v2%29/scores", "https://mteb-leaderboard-backend.hf.space/v1/benchmarks/MTEB%28Multilingual%2C%20v2%29/scores" @@ -4721,28 +4784,30 @@ "kind": "peer", "on_displayed_frontier": false, "evidence_type": "registry_reported", - "observation_id": "registry_reported:OrdalieTech/Solon-embeddings-large-0.1" + "observation_id": "registry_reported:sdadas/mmlw-e5-large" }, { - "model": "intfloat/multilingual-e5-large", - "parameters_billion": 0.56, - "score": 55.6856, + "model": "BAAI/bge-m3", + "parameters_billion": 0.568, + "score": 52.79810320966879, "open_weights": true, "modalities": [ "text" ], "training_tasks": [ - "FEVER", - "FEVER-NL", - "FEVER-PL", - "FEVERHardNegatives", + "CMedQAv1-reranking", + "CMedQAv2-reranking", + "CodeSearchNet", + "DuRetrieval", "HotpotQA", "HotpotQA-NL", "HotpotQA-PL", "HotpotQAHardNegatives", + "LeCaRDv2", "MIRACLReranking", "MIRACLRetrieval", "MIRACLRetrievalHardNegatives", + "MMarcoReranking", "MSMARCO", "MSMARCO-PL", "MSMARCOHardNegatives", @@ -4753,7 +4818,8 @@ "NQHardNegatives", "NanoMSMARCORetrieval", "NanoNQRetrieval", - "XQuADRetrieval", + "T2Reranking", + "T2Retrieval", "mMARCO-NL" ], "source_urls": [ @@ -4764,26 +4830,7 @@ "kind": "peer", "on_displayed_frontier": false, "evidence_type": "registry_reported", - "observation_id": "registry_reported:intfloat/multilingual-e5-large" - }, - { - "model": "omarelshehy/arabic-english-sts-matryoshka", - "parameters_billion": 0.56, - "score": 52.528200000000005, - "open_weights": true, - "modalities": [ - "text" - ], - "training_tasks": null, - "source_urls": [ - "https://mteb-leaderboard-backend.hf.space/v1/benchmarks/MTEB%28eng%2C%20v2%29/scores", - "https://mteb-leaderboard-backend.hf.space/v1/benchmarks/MTEB%28Multilingual%2C%20v2%29/scores" - ], - "size_basis": "reported_rounded_total", - "kind": "peer", - "on_displayed_frontier": false, - "evidence_type": "registry_reported", - "observation_id": "registry_reported:omarelshehy/arabic-english-sts-matryoshka" + "observation_id": "registry_reported:BAAI/bge-m3" }, { "model": "Snowflake/snowflake-arctic-embed-l-v2.0", @@ -4866,39 +4913,23 @@ "observation_id": "registry_reported:manu/bge-m3-custom-fr" }, { - "model": "BAAI/bge-m3", - "parameters_billion": 0.568, - "score": 52.79810320966879, + "model": "jinaai/jina-embeddings-v3", + "parameters_billion": 0.572, + "score": 58.92914985808295, "open_weights": true, "modalities": [ "text" ], "training_tasks": [ - "CMedQAv1-reranking", - "CMedQAv2-reranking", - "CodeSearchNet", - "DuRetrieval", - "HotpotQA", - "HotpotQA-NL", - "HotpotQA-PL", - "HotpotQAHardNegatives", - "LeCaRDv2", - "MIRACLReranking", - "MIRACLRetrieval", - "MIRACLRetrievalHardNegatives", - "MMarcoReranking", "MSMARCO", - "MSMARCO-PL", "MSMARCOHardNegatives", - "MrTidyRetrieval", "NQ", "NQ-NL", "NQ-PL", "NQHardNegatives", "NanoMSMARCORetrieval", "NanoNQRetrieval", - "T2Reranking", - "T2Retrieval", + "STS12", "mMARCO-NL" ], "source_urls": [ @@ -4909,27 +4940,117 @@ "kind": "peer", "on_displayed_frontier": false, "evidence_type": "registry_reported", - "observation_id": "registry_reported:BAAI/bge-m3" + "observation_id": "registry_reported:jinaai/jina-embeddings-v3" }, { - "model": "jinaai/jina-embeddings-v3", - "parameters_billion": 0.572, - "score": 58.92914985808295, + "model": "BidirLM/BidirLM-0.6B-Embedding", + "parameters_billion": 0.596, + "score": 64.3759, "open_weights": true, "modalities": [ "text" ], "training_tasks": [ + "AFQMC", + "ATEC", + "AdvertiseGen", + "BQ", + "CAIL2019-SCM", + "CHEF", + "CINLID", + "ChatMed_Consult_Dataset", + "ChineseSTS", + "CmedqaRetrieval", + "Cmnli", + "CodeFeedback", + "DRCD", + "DuRetrieval", + "ELI5_custom", + "ESCIReranking", + "EmotionClassification", + "Expertqa", + "FollowIR", + "GooAQ", + "InF-IR", + "JW300", + "LCSTS", + "MAmmoTH2", + "MEDI2BGE", + "MIRACLRetrieval", "MSMARCO", - "MSMARCOHardNegatives", + "Multi-CPR", + "MultiLongDocReranking", + "NFCorpus", + "NaturalQuestions", + "OPUS-100", + "OpenOrca", + "PAQ", + "PubMedQA", + "QBQTC", + "RefGPT", + "SQuAD", + "SearchQA", + "SimCLUE", + "SyntheticClassificationData", + "T2Ranking", + "TED Talks", + "THUCNews", + "TriviaQA", + "UMETRIP-QA", + "WebCPM", + "WikiAnswers", + "WikiMatrix", + "cCOVID-News", + "ccnews", + "cmrc2018", + "contract-nli", + "csl", + "dureader_mrc", + "law-gpt", + "lawzhidao", + "lima-chinese", + "mmarco-chinese", + "mnli", + "nli_zh", + "nllb", + "ocnli", + "rag-dataset-12000", + "simcse_sup_nli", + "webgpt_comparisons", + "webqa", + "wikipedia-nq", + "xnli_zh", + "yahoo-answers" + ], + "source_urls": [ + "https://mteb-leaderboard-backend.hf.space/v1/benchmarks/MTEB%28eng%2C%20v2%29/scores", + "https://mteb-leaderboard-backend.hf.space/v1/benchmarks/MTEB%28Multilingual%2C%20v2%29/scores" + ], + "size_basis": "reported_rounded_total", + "kind": "peer", + "on_displayed_frontier": false, + "evidence_type": "registry_reported", + "observation_id": "registry_reported:BidirLM/BidirLM-0.6B-Embedding" + }, + { + "model": "IEITYuan/Yuan-embedding-2.0-en", + "parameters_billion": 0.596, + "score": 61.721199999999996, + "open_weights": true, + "modalities": [ + "text" + ], + "training_tasks": [ + "CMedQAv2-reranking", + "CodeSearchNet", + "DuRetrieval", + "HotpotQA", + "MIRACLRetrieval", + "MMarcoReranking", + "MSMARCO", + "MrTidyRetrieval", "NQ", - "NQ-NL", - "NQ-PL", - "NQHardNegatives", - "NanoMSMARCORetrieval", - "NanoNQRetrieval", - "STS12", - "mMARCO-NL" + "T2Retrieval" ], "source_urls": [ "https://mteb-leaderboard-backend.hf.space/v1/benchmarks/MTEB%28eng%2C%20v2%29/scores", @@ -4939,7 +5060,123 @@ "kind": "peer", "on_displayed_frontier": false, "evidence_type": "registry_reported", - "observation_id": "registry_reported:jinaai/jina-embeddings-v3" + "observation_id": "registry_reported:IEITYuan/Yuan-embedding-2.0-en" + }, + { + "model": "KiteFishAI/Nano-Em1-0.6B-v2", + "parameters_billion": 0.596, + "score": 63.0408, + "open_weights": true, + "modalities": [ + "text" + ], + "training_tasks": [ + "AmazonCounterfactualClassification", + "AmazonPolarityClassification", + "AmazonReviewsClassification", + "ArxivClusteringP2P", + "ArxivClusteringS2S", + "Banking77Classification", + "BiorxivClusteringP2P", + "BiorxivClusteringS2S", + "CQADupstackRetrieval", + "DBPedia", + "EmotionClassification", + "FEVER", + "FiQA2018", + "HotpotQA", + "ImdbClassification", + "MSMARCO", + "MTOPDomainClassification", + "MTOPIntentClassification", + "MassiveIntentClassification", + "MassiveScenarioClassification", + "MedrxivClusteringP2P", + "MedrxivClusteringS2S", + "NFCorpus", + "NQ", + "QuoraRetrieval", + "RedditClustering", + "RedditClusteringP2P", + "SciFact", + "SprintDuplicateQuestions", + "StackExchangeClustering", + "StackExchangeClusteringP2P", + "TRECCOVID", + "ToxicConversationsClassification", + "TweetSentimentExtractionClassification", + "TwentyNewsgroupsClustering" + ], + "source_urls": [ + "https://mteb-leaderboard-backend.hf.space/v1/benchmarks/MTEB%28eng%2C%20v2%29/scores", + "https://mteb-leaderboard-backend.hf.space/v1/benchmarks/MTEB%28Multilingual%2C%20v2%29/scores" + ], + "size_basis": "reported_rounded_total", + "kind": "peer", + "on_displayed_frontier": false, + "evidence_type": "registry_reported", + "observation_id": "registry_reported:KiteFishAI/Nano-Em1-0.6B-v2" + }, + { + "model": "KiteFishAI/Nano-Em1-0.6B-v2.1", + "parameters_billion": 0.596, + "score": 66.0464, + "open_weights": true, + "modalities": [ + "text" + ], + "training_tasks": [ + "AmazonReviews", + "Arxiv", + "CodeSearchNet", + "DBPedia", + "FEVER", + "HotpotQA", + "MIRACLRetrieval", + "MSMARCO", + "MrTidyRetrieval", + "NQ" + ], + "source_urls": [ + "https://mteb-leaderboard-backend.hf.space/v1/benchmarks/MTEB%28eng%2C%20v2%29/scores", + "https://mteb-leaderboard-backend.hf.space/v1/benchmarks/MTEB%28Multilingual%2C%20v2%29/scores" + ], + "size_basis": "reported_rounded_total", + "kind": "peer", + "on_displayed_frontier": false, + "evidence_type": "registry_reported", + "observation_id": "registry_reported:KiteFishAI/Nano-Em1-0.6B-v2.1" + }, + { + "model": "Qwen/Qwen3-Embedding-0.6B", + "parameters_billion": 0.596, + "score": 63.7187, + "open_weights": true, + "modalities": [ + "text" + ], + "training_tasks": [ + "CMedQAv2-reranking", + "CodeSearchNet", + "DuRetrieval", + "FEVER", + "HotpotQA", + "MIRACLRetrieval", + "MMarcoReranking", + "MSMARCO", + "MrTidyRetrieval", + "NQ", + "T2Retrieval" + ], + "source_urls": [ + "https://mteb-leaderboard-backend.hf.space/v1/benchmarks/MTEB%28eng%2C%20v2%29/scores", + "https://mteb-leaderboard-backend.hf.space/v1/benchmarks/MTEB%28Multilingual%2C%20v2%29/scores" + ], + "size_basis": "reported_rounded_total", + "kind": "peer", + "on_displayed_frontier": false, + "evidence_type": "registry_reported", + "observation_id": "registry_reported:Qwen/Qwen3-Embedding-0.6B" }, { "model": "codefuse-ai/F2LLM-0.6B", @@ -5018,55 +5255,6 @@ "evidence_type": "registry_reported", "observation_id": "registry_reported:codefuse-ai/F2LLM-0.6B" }, - { - "model": "KiteFishAI/Nano-Em1-0.6B-v2.1", - "parameters_billion": 0.596, - "score": 66.0464, - "open_weights": true, - "modalities": [ - "text" - ], - "training_tasks": [ - "AmazonReviews", - "Arxiv", - "CodeSearchNet", - "DBPedia", - "FEVER", - "HotpotQA", - "MIRACLRetrieval", - "MSMARCO", - "MrTidyRetrieval", - "NQ" - ], - "source_urls": [ - "https://mteb-leaderboard-backend.hf.space/v1/benchmarks/MTEB%28eng%2C%20v2%29/scores", - "https://mteb-leaderboard-backend.hf.space/v1/benchmarks/MTEB%28Multilingual%2C%20v2%29/scores" - ], - "size_basis": "reported_rounded_total", - "kind": "peer", - "on_displayed_frontier": false, - "evidence_type": "registry_reported", - "observation_id": "registry_reported:KiteFishAI/Nano-Em1-0.6B-v2.1" - }, - { - "model": "jinaai/jina-embeddings-v5-text-small", - "parameters_billion": 0.596, - "score": 65.7904, - "open_weights": true, - "modalities": [ - "text" - ], - "training_tasks": null, - "source_urls": [ - "https://mteb-leaderboard-backend.hf.space/v1/benchmarks/MTEB%28eng%2C%20v2%29/scores", - "https://mteb-leaderboard-backend.hf.space/v1/benchmarks/MTEB%28Multilingual%2C%20v2%29/scores" - ], - "size_basis": "reported_rounded_total", - "kind": "peer", - "on_displayed_frontier": false, - "evidence_type": "registry_reported", - "observation_id": "registry_reported:jinaai/jina-embeddings-v5-text-small" - }, { "model": "codefuse-ai/F2LLM-v2-0.6B", "parameters_billion": 0.596, @@ -5240,116 +5428,14 @@ "observation_id": "registry_reported:codefuse-ai/F2LLM-v2-0.6B" }, { - "model": "BidirLM/BidirLM-0.6B-Embedding", - "parameters_billion": 0.596, - "score": 64.3759, - "open_weights": true, - "modalities": [ - "text" - ], - "training_tasks": [ - "AFQMC", - "ATEC", - "AdvertiseGen", - "BQ", - "CAIL2019-SCM", - "CHEF", - "CINLID", - "ChatMed_Consult_Dataset", - "ChineseSTS", - "CmedqaRetrieval", - "Cmnli", - "CodeFeedback", - "DRCD", - "DuRetrieval", - "ELI5_custom", - "ESCIReranking", - "EmotionClassification", - "Expertqa", - "FollowIR", - "GooAQ", - "InF-IR", - "JW300", - "LCSTS", - "MAmmoTH2", - "MEDI2BGE", - "MIRACLRetrieval", - "MSMARCO", - "Multi-CPR", - "MultiLongDocReranking", - "NFCorpus", - "NaturalQuestions", - "OPUS-100", - "OpenOrca", - "PAQ", - "PubMedQA", - "QBQTC", - "RefGPT", - "SQuAD", - "SearchQA", - "SimCLUE", - "SyntheticClassificationData", - "T2Ranking", - "TED Talks", - "THUCNews", - "TriviaQA", - "UMETRIP-QA", - "WebCPM", - "WikiAnswers", - "WikiMatrix", - "cCOVID-News", - "ccnews", - "cmrc2018", - "contract-nli", - "csl", - "dureader_mrc", - "law-gpt", - "lawzhidao", - "lima-chinese", - "mmarco-chinese", - "mnli", - "nli_zh", - "nllb", - "ocnli", - "rag-dataset-12000", - "simcse_sup_nli", - "webgpt_comparisons", - "webqa", - "wikipedia-nq", - "xnli_zh", - "yahoo-answers" - ], - "source_urls": [ - "https://mteb-leaderboard-backend.hf.space/v1/benchmarks/MTEB%28eng%2C%20v2%29/scores", - "https://mteb-leaderboard-backend.hf.space/v1/benchmarks/MTEB%28Multilingual%2C%20v2%29/scores" - ], - "size_basis": "reported_rounded_total", - "kind": "peer", - "on_displayed_frontier": false, - "evidence_type": "registry_reported", - "observation_id": "registry_reported:BidirLM/BidirLM-0.6B-Embedding" - }, - { - "model": "Qwen/Qwen3-Embedding-0.6B", + "model": "jinaai/jina-embeddings-v5-text-small", "parameters_billion": 0.596, - "score": 63.7187, + "score": 65.7904, "open_weights": true, "modalities": [ "text" ], - "training_tasks": [ - "CMedQAv2-reranking", - "CodeSearchNet", - "DuRetrieval", - "FEVER", - "HotpotQA", - "MIRACLRetrieval", - "MMarcoReranking", - "MSMARCO", - "MrTidyRetrieval", - "NQ", - "T2Retrieval" - ], + "training_tasks": null, "source_urls": [ "https://mteb-leaderboard-backend.hf.space/v1/benchmarks/MTEB%28eng%2C%20v2%29/scores", "https://mteb-leaderboard-backend.hf.space/v1/benchmarks/MTEB%28Multilingual%2C%20v2%29/scores" @@ -5358,7 +5444,7 @@ "kind": "peer", "on_displayed_frontier": false, "evidence_type": "registry_reported", - "observation_id": "registry_reported:Qwen/Qwen3-Embedding-0.6B" + "observation_id": "registry_reported:jinaai/jina-embeddings-v5-text-small" }, { "model": "microsoft/harrier-oss-v1-0.6b", @@ -5409,103 +5495,18 @@ "Quail", "QuoraRetrieval", "STSB", - "ScalaClassification", - "SciFact", - "SemRel24STS", - "SpartQA", - "StackOverflowQARetrieval", - "StatcanDialogueDatasetRetrieval", - "SwissJudgementClassification", - "SyntheticText2SQLRetrieval", - "TERRa", - "TempReasonL1", - "ToxicConversationsClassification", - "WinoGrande" - ], - "source_urls": [ - "https://mteb-leaderboard-backend.hf.space/v1/benchmarks/MTEB%28eng%2C%20v2%29/scores", - "https://mteb-leaderboard-backend.hf.space/v1/benchmarks/MTEB%28Multilingual%2C%20v2%29/scores" - ], - "size_basis": "reported_rounded_total", - "kind": "peer", - "on_displayed_frontier": false, - "evidence_type": "registry_reported", - "observation_id": "registry_reported:microsoft/harrier-oss-v1-0.6b" - }, - { - "model": "KiteFishAI/Nano-Em1-0.6B-v2", - "parameters_billion": 0.596, - "score": 63.0408, - "open_weights": true, - "modalities": [ - "text" - ], - "training_tasks": [ - "AmazonCounterfactualClassification", - "AmazonPolarityClassification", - "AmazonReviewsClassification", - "ArxivClusteringP2P", - "ArxivClusteringS2S", - "Banking77Classification", - "BiorxivClusteringP2P", - "BiorxivClusteringS2S", - "CQADupstackRetrieval", - "DBPedia", - "EmotionClassification", - "FEVER", - "FiQA2018", - "HotpotQA", - "ImdbClassification", - "MSMARCO", - "MTOPDomainClassification", - "MTOPIntentClassification", - "MassiveIntentClassification", - "MassiveScenarioClassification", - "MedrxivClusteringP2P", - "MedrxivClusteringS2S", - "NFCorpus", - "NQ", - "QuoraRetrieval", - "RedditClustering", - "RedditClusteringP2P", - "SciFact", - "SprintDuplicateQuestions", - "StackExchangeClustering", - "StackExchangeClusteringP2P", - "TRECCOVID", - "ToxicConversationsClassification", - "TweetSentimentExtractionClassification", - "TwentyNewsgroupsClustering" - ], - "source_urls": [ - "https://mteb-leaderboard-backend.hf.space/v1/benchmarks/MTEB%28eng%2C%20v2%29/scores", - "https://mteb-leaderboard-backend.hf.space/v1/benchmarks/MTEB%28Multilingual%2C%20v2%29/scores" - ], - "size_basis": "reported_rounded_total", - "kind": "peer", - "on_displayed_frontier": false, - "evidence_type": "registry_reported", - "observation_id": "registry_reported:KiteFishAI/Nano-Em1-0.6B-v2" - }, - { - "model": "IEITYuan/Yuan-embedding-2.0-en", - "parameters_billion": 0.596, - "score": 61.721199999999996, - "open_weights": true, - "modalities": [ - "text" - ], - "training_tasks": [ - "CMedQAv2-reranking", - "CodeSearchNet", - "DuRetrieval", - "HotpotQA", - "MIRACLRetrieval", - "MMarcoReranking", - "MSMARCO", - "MrTidyRetrieval", - "NQ", - "T2Retrieval" + "ScalaClassification", + "SciFact", + "SemRel24STS", + "SpartQA", + "StackOverflowQARetrieval", + "StatcanDialogueDatasetRetrieval", + "SwissJudgementClassification", + "SyntheticText2SQLRetrieval", + "TERRa", + "TempReasonL1", + "ToxicConversationsClassification", + "WinoGrande" ], "source_urls": [ "https://mteb-leaderboard-backend.hf.space/v1/benchmarks/MTEB%28eng%2C%20v2%29/scores", @@ -5515,7 +5516,7 @@ "kind": "peer", "on_displayed_frontier": false, "evidence_type": "registry_reported", - "observation_id": "registry_reported:IEITYuan/Yuan-embedding-2.0-en" + "observation_id": "registry_reported:microsoft/harrier-oss-v1-0.6b" }, { "model": "infgrad/Jasper-Token-Compression-600M", @@ -5728,26 +5729,6 @@ "evidence_type": "registry_reported", "observation_id": "registry_reported:BidirLM/BidirLM-1B-Embedding" }, - { - "model": "llm-semantic-router/Vela-1.0-Omni-Mini", - "parameters_billion": 1.04415264, - "parameters_exact": 1044152640, - "native_fingerprint": "035f6084c6c984f465e4f91c50caf8f13790177d4c0c29e4ce5d6c0c233fd1a0", - "score": 65.52292325553617, - "open_weights": true, - "modalities": [ - "text", - "image", - "audio" - ], - "size_basis": "counted_all_native_model_parameters", - "kind": "mini", - "evaluated_task_revision": "0bbdb47bcbe3a90093699aefeed338a0f28a7ee8", - "report_url": "https://huggingface.co/llm-semantic-router/Vela-1.0-Omni-Mini/blob/662d4e53c52d22da6430b0ac316360d41028374d/benchmarks/mteb-eng-v2.json", - "on_displayed_frontier": false, - "evidence_type": "vela_release_evaluation", - "observation_id": "vela_release_evaluation:llm-semantic-router/Vela-1.0-Omni-Mini" - }, { "model": "izhx/udever-bloom-1b1", "parameters_billion": 1.065, @@ -5770,20 +5751,16 @@ "observation_id": "registry_reported:izhx/udever-bloom-1b1" }, { - "model": "manu/sentence_croissant_alpha_v0.2", + "model": "DeepPavlov/rubert-base-cased", "parameters_billion": 1.28, - "score": 59.5187, + "score": 50.0572, "open_weights": true, "modalities": [ "text" ], "training_tasks": [ - "MSMARCO", - "QuoraRetrieval", - "STS12", - "STSB", - "STSBenchmark", - "STSBenchmarkMultilingualSTS" + "WikipediaRerankingMultilingual", + "WikipediaRetrievalMultilingual" ], "source_urls": [ "https://mteb-leaderboard-backend.hf.space/v1/benchmarks/MTEB%28eng%2C%20v2%29/scores", @@ -5793,12 +5770,12 @@ "kind": "peer", "on_displayed_frontier": false, "evidence_type": "registry_reported", - "observation_id": "registry_reported:manu/sentence_croissant_alpha_v0.2" + "observation_id": "registry_reported:DeepPavlov/rubert-base-cased" }, { - "model": "manu/sentence_croissant_alpha_v0.4", + "model": "manu/sentence_croissant_alpha_v0.2", "parameters_billion": 1.28, - "score": 58.5928, + "score": 59.5187, "open_weights": true, "modalities": [ "text" @@ -5819,7 +5796,7 @@ "kind": "peer", "on_displayed_frontier": false, "evidence_type": "registry_reported", - "observation_id": "registry_reported:manu/sentence_croissant_alpha_v0.4" + "observation_id": "registry_reported:manu/sentence_croissant_alpha_v0.2" }, { "model": "manu/sentence_croissant_alpha_v0.3", @@ -5848,16 +5825,20 @@ "observation_id": "registry_reported:manu/sentence_croissant_alpha_v0.3" }, { - "model": "DeepPavlov/rubert-base-cased", + "model": "manu/sentence_croissant_alpha_v0.4", "parameters_billion": 1.28, - "score": 50.0572, + "score": 58.5928, "open_weights": true, "modalities": [ "text" ], "training_tasks": [ - "WikipediaRerankingMultilingual", - "WikipediaRetrievalMultilingual" + "MSMARCO", + "QuoraRetrieval", + "STS12", + "STSB", + "STSBenchmark", + "STSBenchmarkMultilingualSTS" ], "source_urls": [ "https://mteb-leaderboard-backend.hf.space/v1/benchmarks/MTEB%28eng%2C%20v2%29/scores", @@ -5867,7 +5848,33 @@ "kind": "peer", "on_displayed_frontier": false, "evidence_type": "registry_reported", - "observation_id": "registry_reported:DeepPavlov/rubert-base-cased" + "observation_id": "registry_reported:manu/sentence_croissant_alpha_v0.4" + }, + { + "model": "llm-semantic-router/Vela-1.0-Omni-Mini", + "parameters_billion": 1.3328912, + "parameters_exact": 1332891200, + "score": 62.00989328478093, + "open_weights": true, + "modalities": [ + "text", + "image", + "audio" + ], + "size_basis": "counted_all_native_model_parameters", + "kind": "mini", + "evaluated_task_revision": "0bbdb47bcbe3a90093699aefeed338a0f28a7ee8", + "on_displayed_frontier": false, + "evidence_type": "exact_text_component_applicability", + "observation_id": "vela_release_evaluation:llm-semantic-router/Vela-1.0-Omni-Mini", + "native_artifact_sha256": "d93ebb5f2f4951d52423ae13da71b037ce44d02957671e1e71bfe25623f3cff2", + "revision": "self", + "evaluated_native_artifact_sha256": null, + "report": "https://huggingface.co/llm-semantic-router/Vela-1.0-Omni-Mini/blob/main/benchmarks/mteb-eng-v2.json", + "evaluated_model": { + "repo_id": "local/Qwen3-Embedding-0.6B-MRL768-NoInstructions", + "revision": "98e47eac3fa59b6c627e371230905400b3a405ddc69ab1ef02cb1c476db04294" + } }, { "model": "NovaSearch/stella_en_1.5B_v5", @@ -6072,83 +6079,6 @@ "evidence_type": "registry_reported", "observation_id": "registry_reported:jinaai/jina-embeddings-v5-omni-small" }, - { - "model": "codefuse-ai/F2LLM-1.7B", - "parameters_billion": 1.721, - "score": 66.61, - "open_weights": true, - "modalities": [ - "text" - ], - "training_tasks": [ - "ANLI", - "AmazonCounterfactualClassification", - "AmazonPolarityClassification", - "AmazonQA", - "AmazonReviewClassification", - "ArguAna", - "ArxivClusteringP2P", - "ArxivClusteringS2S", - "Banking77Classification", - "BioASQ", - "BiorxivClusteringP2P", - "BiorxivClusteringS2S", - "CNNDM", - "COLIEE", - "CoLA", - "ELI5", - "EmotionClassification", - "FEVER", - "FiQA2018", - "HotpotQA", - "ImdbClassification", - "MIRACLRetrieval", - "MNLI", - "MSMARCO", - "MTOPDomainClassification", - "MTOPIntentClassification", - "MassiveIntentClassification", - "MassiveScenarioClassification", - "MedrxivClusteringP2P", - "MedrxivClusteringS2S", - "MrTidyRetrieval", - "NFCorpus", - "NQ", - "PAQ", - "PubMedQA", - "QQP", - "RedditClustering", - "RedditClusteringP2P", - "S2ORC", - "SNLI", - "SPECTER", - "SQuAD", - "STS12", - "STS22", - "STSBenchmark", - "SciFact", - "SentenceCompression", - "StackExchange", - "StackExchangeClustering", - "StackExchangeClusteringP2P", - "StackExchangeDupQuestions", - "StackOverflowDupQuestions", - "ToxicConversationsClassification", - "TriviaQA", - "TweetSentimentExtractionClassification", - "TwentyNewsgroupsClustering", - "XSum" - ], - "source_urls": [ - "https://mteb-leaderboard-backend.hf.space/v1/benchmarks/MTEB%28eng%2C%20v2%29/scores", - "https://mteb-leaderboard-backend.hf.space/v1/benchmarks/MTEB%28Multilingual%2C%20v2%29/scores" - ], - "size_basis": "reported_rounded_total", - "kind": "peer", - "on_displayed_frontier": false, - "evidence_type": "registry_reported", - "observation_id": "registry_reported:codefuse-ai/F2LLM-1.7B" - }, { "model": "BidirLM/BidirLM-1.7B-Embedding", "parameters_billion": 1.721, @@ -6239,6 +6169,83 @@ "evidence_type": "registry_reported", "observation_id": "registry_reported:BidirLM/BidirLM-1.7B-Embedding" }, + { + "model": "codefuse-ai/F2LLM-1.7B", + "parameters_billion": 1.721, + "score": 66.61, + "open_weights": true, + "modalities": [ + "text" + ], + "training_tasks": [ + "ANLI", + "AmazonCounterfactualClassification", + "AmazonPolarityClassification", + "AmazonQA", + "AmazonReviewClassification", + "ArguAna", + "ArxivClusteringP2P", + "ArxivClusteringS2S", + "Banking77Classification", + "BioASQ", + "BiorxivClusteringP2P", + "BiorxivClusteringS2S", + "CNNDM", + "COLIEE", + "CoLA", + "ELI5", + "EmotionClassification", + "FEVER", + "FiQA2018", + "HotpotQA", + "ImdbClassification", + "MIRACLRetrieval", + "MNLI", + "MSMARCO", + "MTOPDomainClassification", + "MTOPIntentClassification", + "MassiveIntentClassification", + "MassiveScenarioClassification", + "MedrxivClusteringP2P", + "MedrxivClusteringS2S", + "MrTidyRetrieval", + "NFCorpus", + "NQ", + "PAQ", + "PubMedQA", + "QQP", + "RedditClustering", + "RedditClusteringP2P", + "S2ORC", + "SNLI", + "SPECTER", + "SQuAD", + "STS12", + "STS22", + "STSBenchmark", + "SciFact", + "SentenceCompression", + "StackExchange", + "StackExchangeClustering", + "StackExchangeClusteringP2P", + "StackExchangeDupQuestions", + "StackOverflowDupQuestions", + "ToxicConversationsClassification", + "TriviaQA", + "TweetSentimentExtractionClassification", + "TwentyNewsgroupsClustering", + "XSum" + ], + "source_urls": [ + "https://mteb-leaderboard-backend.hf.space/v1/benchmarks/MTEB%28eng%2C%20v2%29/scores", + "https://mteb-leaderboard-backend.hf.space/v1/benchmarks/MTEB%28Multilingual%2C%20v2%29/scores" + ], + "size_basis": "reported_rounded_total", + "kind": "peer", + "on_displayed_frontier": false, + "evidence_type": "registry_reported", + "observation_id": "registry_reported:codefuse-ai/F2LLM-1.7B" + }, { "model": "codefuse-ai/F2LLM-v2-1.7B", "parameters_billion": 1.721, @@ -6643,71 +6650,28 @@ "observation_id": "registry_reported:ai-sage/Giga-Embeddings-instruct" }, { - "model": "codefuse-ai/F2LLM-4B", + "model": "ICT-TIME-and-Querit/BOOM_4B_v1", "parameters_billion": 4.022, - "score": 65.6034, + "score": 64.66, "open_weights": true, "modalities": [ "text" ], "training_tasks": [ - "ANLI", "AmazonCounterfactualClassification", - "AmazonPolarityClassification", - "AmazonQA", - "AmazonReviewClassification", - "ArguAna", - "ArxivClusteringP2P", - "ArxivClusteringS2S", "Banking77Classification", - "BioASQ", - "BiorxivClusteringP2P", - "BiorxivClusteringS2S", - "CNNDM", - "COLIEE", - "CoLA", - "ELI5", - "EmotionClassification", + "DuRetrieval", "FEVER", - "FiQA2018", "HotpotQA", "ImdbClassification", "MIRACLRetrieval", - "MNLI", "MSMARCO", - "MTOPDomainClassification", - "MTOPIntentClassification", - "MassiveIntentClassification", - "MassiveScenarioClassification", - "MedrxivClusteringP2P", - "MedrxivClusteringS2S", + "MTOPDomainClassification", "MrTidyRetrieval", - "NFCorpus", "NQ", - "PAQ", - "PubMedQA", - "QQP", - "RedditClustering", - "RedditClusteringP2P", - "S2ORC", - "SNLI", - "SPECTER", - "SQuAD", - "STS12", - "STS22", - "STSBenchmark", - "SciFact", - "SentenceCompression", - "StackExchange", - "StackExchangeClustering", - "StackExchangeClusteringP2P", - "StackExchangeDupQuestions", - "StackOverflowDupQuestions", + "T2Retrieval", "ToxicConversationsClassification", - "TriviaQA", - "TweetSentimentExtractionClassification", - "TwentyNewsgroupsClustering", - "XSum" + "TweetSentimentExtractionClassification" ], "source_urls": [ "https://mteb-leaderboard-backend.hf.space/v1/benchmarks/MTEB%28eng%2C%20v2%29/scores", @@ -6717,7 +6681,7 @@ "kind": "peer", "on_displayed_frontier": false, "evidence_type": "registry_reported", - "observation_id": "registry_reported:codefuse-ai/F2LLM-4B" + "observation_id": "registry_reported:ICT-TIME-and-Querit/BOOM_4B_v1" }, { "model": "ICT-TIME-and-Querit/ICT-TIME-and-Querit-embedding-v1", @@ -6794,28 +6758,71 @@ "observation_id": "registry_reported:Qwen/Qwen3-Embedding-4B" }, { - "model": "ICT-TIME-and-Querit/BOOM_4B_v1", + "model": "codefuse-ai/F2LLM-4B", "parameters_billion": 4.022, - "score": 64.66, + "score": 65.6034, "open_weights": true, "modalities": [ "text" ], "training_tasks": [ + "ANLI", "AmazonCounterfactualClassification", + "AmazonPolarityClassification", + "AmazonQA", + "AmazonReviewClassification", + "ArguAna", + "ArxivClusteringP2P", + "ArxivClusteringS2S", "Banking77Classification", - "DuRetrieval", + "BioASQ", + "BiorxivClusteringP2P", + "BiorxivClusteringS2S", + "CNNDM", + "COLIEE", + "CoLA", + "ELI5", + "EmotionClassification", "FEVER", + "FiQA2018", "HotpotQA", "ImdbClassification", "MIRACLRetrieval", + "MNLI", "MSMARCO", "MTOPDomainClassification", + "MTOPIntentClassification", + "MassiveIntentClassification", + "MassiveScenarioClassification", + "MedrxivClusteringP2P", + "MedrxivClusteringS2S", "MrTidyRetrieval", + "NFCorpus", "NQ", - "T2Retrieval", + "PAQ", + "PubMedQA", + "QQP", + "RedditClustering", + "RedditClusteringP2P", + "S2ORC", + "SNLI", + "SPECTER", + "SQuAD", + "STS12", + "STS22", + "STSBenchmark", + "SciFact", + "SentenceCompression", + "StackExchange", + "StackExchangeClustering", + "StackExchangeClusteringP2P", + "StackExchangeDupQuestions", + "StackOverflowDupQuestions", "ToxicConversationsClassification", - "TweetSentimentExtractionClassification" + "TriviaQA", + "TweetSentimentExtractionClassification", + "TwentyNewsgroupsClustering", + "XSum" ], "source_urls": [ "https://mteb-leaderboard-backend.hf.space/v1/benchmarks/MTEB%28eng%2C%20v2%29/scores", @@ -6825,7 +6832,7 @@ "kind": "peer", "on_displayed_frontier": false, "evidence_type": "registry_reported", - "observation_id": "registry_reported:ICT-TIME-and-Querit/BOOM_4B_v1" + "observation_id": "registry_reported:codefuse-ai/F2LLM-4B" }, { "model": "codefuse-ai/F2LLM-v2-4B", @@ -7375,42 +7382,42 @@ "observation_id": "registry_reported:annamodels/LGAI-Embedding-Preview" }, { - "model": "Salesforce/SFR-Embedding-Mistral", + "model": "GeoGPT-Research-Project/GeoEmbedding", "parameters_billion": 7.111, - "score": 65.5027, + "score": 64.51559999999999, "open_weights": true, "modalities": [ "text" ], "training_tasks": [ + "AmazonCounterfactualClassification", + "AmazonPolarityClassification", + "AmazonReviewsClassification", + "ArguAna", + "ArxivClusteringP2P", + "ArxivClusteringS2S", + "Banking77Classification", + "BiorxivClusteringP2P", + "BiorxivClusteringS2S", + "EmotionClassification", "FEVER", - "FEVER-NL", - "FEVER-PL", - "FEVERHardNegatives", "FiQA2018", - "FiQA2018-NL", "HotpotQA", - "HotpotQA-NL", - "HotpotQA-PL", - "HotpotQAHardNegatives", - "MIRACLReranking", - "MIRACLRetrieval", - "MIRACLRetrievalHardNegatives", "MSMARCO", - "MSMARCO-PL", - "MSMARCOHardNegatives", - "MrTidyRetrieval", - "NQ", - "NQ-NL", - "NQ-PL", - "NQHardNegatives", - "NanoMSMARCORetrieval", - "NanoNQRetrieval", + "MTOPDomainClassification", + "MTOPIntentClassification", + "MassiveIntentClassification", + "MedrixvClusteringP2P", + "MedrixvClusteringS2S", + "NFCorpus", "STS12", "STS22", "STSBenchmark", - "XQuADRetrieval", - "mMARCO-NL" + "SciFact", + "StackOverflowDupQuestions", + "ToxicConversationsClassification", + "TweetSentimentExtractionClassification", + "TwentyNewsgroupsClustering" ], "source_urls": [ "https://mteb-leaderboard-backend.hf.space/v1/benchmarks/MTEB%28eng%2C%20v2%29/scores", @@ -7420,12 +7427,31 @@ "kind": "peer", "on_displayed_frontier": false, "evidence_type": "registry_reported", - "observation_id": "registry_reported:Salesforce/SFR-Embedding-Mistral" + "observation_id": "registry_reported:GeoGPT-Research-Project/GeoEmbedding" }, { - "model": "Salesforce/SFR-Embedding-2_R", + "model": "Haon-Chen/speed-embedding-7b-instruct", "parameters_billion": 7.111, - "score": 65.3697253315525, + "score": 60.0154, + "open_weights": true, + "modalities": [ + "text" + ], + "training_tasks": null, + "source_urls": [ + "https://mteb-leaderboard-backend.hf.space/v1/benchmarks/MTEB%28eng%2C%20v2%29/scores", + "https://mteb-leaderboard-backend.hf.space/v1/benchmarks/MTEB%28Multilingual%2C%20v2%29/scores" + ], + "size_basis": "reported_rounded_total", + "kind": "peer", + "on_displayed_frontier": false, + "evidence_type": "registry_reported", + "observation_id": "registry_reported:Haon-Chen/speed-embedding-7b-instruct" + }, + { + "model": "Linq-AI-Research/Linq-Embed-Mistral", + "parameters_billion": 7.111, + "score": 63.3776, "open_weights": true, "modalities": [ "text" @@ -7435,8 +7461,6 @@ "FEVER-NL", "FEVER-PL", "FEVERHardNegatives", - "FiQA2018", - "FiQA2018-NL", "HotpotQA", "HotpotQA-NL", "HotpotQA-PL", @@ -7454,9 +7478,6 @@ "NQHardNegatives", "NanoMSMARCORetrieval", "NanoNQRetrieval", - "STS12", - "STS22", - "STSBenchmark", "XQuADRetrieval", "mMARCO-NL" ], @@ -7468,12 +7489,12 @@ "kind": "peer", "on_displayed_frontier": false, "evidence_type": "registry_reported", - "observation_id": "registry_reported:Salesforce/SFR-Embedding-2_R" + "observation_id": "registry_reported:Linq-AI-Research/Linq-Embed-Mistral" }, { - "model": "intfloat/e5-mistral-7b-instruct", + "model": "Salesforce/SFR-Embedding-2_R", "parameters_billion": 7.111, - "score": 65.2837349991313, + "score": 65.3697253315525, "open_weights": true, "modalities": [ "text" @@ -7483,6 +7504,8 @@ "FEVER-NL", "FEVER-PL", "FEVERHardNegatives", + "FiQA2018", + "FiQA2018-NL", "HotpotQA", "HotpotQA-NL", "HotpotQA-PL", @@ -7500,56 +7523,11 @@ "NQHardNegatives", "NanoMSMARCORetrieval", "NanoNQRetrieval", - "XQuADRetrieval", - "mMARCO-NL" - ], - "source_urls": [ - "https://mteb-leaderboard-backend.hf.space/v1/benchmarks/MTEB%28eng%2C%20v2%29/scores", - "https://mteb-leaderboard-backend.hf.space/v1/benchmarks/MTEB%28Multilingual%2C%20v2%29/scores" - ], - "size_basis": "reported_rounded_total", - "kind": "peer", - "on_displayed_frontier": false, - "evidence_type": "registry_reported", - "observation_id": "registry_reported:intfloat/e5-mistral-7b-instruct" - }, - { - "model": "GeoGPT-Research-Project/GeoEmbedding", - "parameters_billion": 7.111, - "score": 64.51559999999999, - "open_weights": true, - "modalities": [ - "text" - ], - "training_tasks": [ - "AmazonCounterfactualClassification", - "AmazonPolarityClassification", - "AmazonReviewsClassification", - "ArguAna", - "ArxivClusteringP2P", - "ArxivClusteringS2S", - "Banking77Classification", - "BiorxivClusteringP2P", - "BiorxivClusteringS2S", - "EmotionClassification", - "FEVER", - "FiQA2018", - "HotpotQA", - "MSMARCO", - "MTOPDomainClassification", - "MTOPIntentClassification", - "MassiveIntentClassification", - "MedrixvClusteringP2P", - "MedrixvClusteringS2S", - "NFCorpus", "STS12", "STS22", "STSBenchmark", - "SciFact", - "StackOverflowDupQuestions", - "ToxicConversationsClassification", - "TweetSentimentExtractionClassification", - "TwentyNewsgroupsClustering" + "XQuADRetrieval", + "mMARCO-NL" ], "source_urls": [ "https://mteb-leaderboard-backend.hf.space/v1/benchmarks/MTEB%28eng%2C%20v2%29/scores", @@ -7559,12 +7537,12 @@ "kind": "peer", "on_displayed_frontier": false, "evidence_type": "registry_reported", - "observation_id": "registry_reported:GeoGPT-Research-Project/GeoEmbedding" + "observation_id": "registry_reported:Salesforce/SFR-Embedding-2_R" }, { - "model": "Linq-AI-Research/Linq-Embed-Mistral", + "model": "Salesforce/SFR-Embedding-Mistral", "parameters_billion": 7.111, - "score": 63.3776, + "score": 65.5027, "open_weights": true, "modalities": [ "text" @@ -7574,6 +7552,8 @@ "FEVER-NL", "FEVER-PL", "FEVERHardNegatives", + "FiQA2018", + "FiQA2018-NL", "HotpotQA", "HotpotQA-NL", "HotpotQA-PL", @@ -7591,6 +7571,9 @@ "NQHardNegatives", "NanoMSMARCORetrieval", "NanoNQRetrieval", + "STS12", + "STS22", + "STSBenchmark", "XQuADRetrieval", "mMARCO-NL" ], @@ -7602,36 +7585,41 @@ "kind": "peer", "on_displayed_frontier": false, "evidence_type": "registry_reported", - "observation_id": "registry_reported:Linq-AI-Research/Linq-Embed-Mistral" + "observation_id": "registry_reported:Salesforce/SFR-Embedding-Mistral" }, { - "model": "Haon-Chen/speed-embedding-7b-instruct", + "model": "intfloat/e5-mistral-7b-instruct", "parameters_billion": 7.111, - "score": 60.0154, + "score": 65.2837349991313, "open_weights": true, "modalities": [ "text" ], - "training_tasks": null, - "source_urls": [ - "https://mteb-leaderboard-backend.hf.space/v1/benchmarks/MTEB%28eng%2C%20v2%29/scores", - "https://mteb-leaderboard-backend.hf.space/v1/benchmarks/MTEB%28Multilingual%2C%20v2%29/scores" - ], - "size_basis": "reported_rounded_total", - "kind": "peer", - "on_displayed_frontier": false, - "evidence_type": "registry_reported", - "observation_id": "registry_reported:Haon-Chen/speed-embedding-7b-instruct" - }, - { - "model": "McGill-NLP/LLM2Vec-Mistral-7B-Instruct-v2-mntp-unsup-simcse", - "parameters_billion": 7.153, - "score": 58.167500000000004, - "open_weights": true, - "modalities": [ - "text" + "training_tasks": [ + "FEVER", + "FEVER-NL", + "FEVER-PL", + "FEVERHardNegatives", + "HotpotQA", + "HotpotQA-NL", + "HotpotQA-PL", + "HotpotQAHardNegatives", + "MIRACLReranking", + "MIRACLRetrieval", + "MIRACLRetrievalHardNegatives", + "MSMARCO", + "MSMARCO-PL", + "MSMARCOHardNegatives", + "MrTidyRetrieval", + "NQ", + "NQ-NL", + "NQ-PL", + "NQHardNegatives", + "NanoMSMARCORetrieval", + "NanoNQRetrieval", + "XQuADRetrieval", + "mMARCO-NL" ], - "training_tasks": null, "source_urls": [ "https://mteb-leaderboard-backend.hf.space/v1/benchmarks/MTEB%28eng%2C%20v2%29/scores", "https://mteb-leaderboard-backend.hf.space/v1/benchmarks/MTEB%28Multilingual%2C%20v2%29/scores" @@ -7640,7 +7628,7 @@ "kind": "peer", "on_displayed_frontier": false, "evidence_type": "registry_reported", - "observation_id": "registry_reported:McGill-NLP/LLM2Vec-Mistral-7B-Instruct-v2-mntp-unsup-simcse" + "observation_id": "registry_reported:intfloat/e5-mistral-7b-instruct" }, { "model": "McGill-NLP/LLM2Vec-Mistral-7B-Instruct-v2-mntp-supervised", @@ -7685,6 +7673,25 @@ "evidence_type": "registry_reported", "observation_id": "registry_reported:McGill-NLP/LLM2Vec-Mistral-7B-Instruct-v2-mntp-supervised" }, + { + "model": "McGill-NLP/LLM2Vec-Mistral-7B-Instruct-v2-mntp-unsup-simcse", + "parameters_billion": 7.153, + "score": 58.167500000000004, + "open_weights": true, + "modalities": [ + "text" + ], + "training_tasks": null, + "source_urls": [ + "https://mteb-leaderboard-backend.hf.space/v1/benchmarks/MTEB%28eng%2C%20v2%29/scores", + "https://mteb-leaderboard-backend.hf.space/v1/benchmarks/MTEB%28Multilingual%2C%20v2%29/scores" + ], + "size_basis": "reported_rounded_total", + "kind": "peer", + "on_displayed_frontier": false, + "evidence_type": "registry_reported", + "observation_id": "registry_reported:McGill-NLP/LLM2Vec-Mistral-7B-Instruct-v2-mntp-unsup-simcse" + }, { "model": "GritLM/GritLM-7B", "parameters_billion": 7.24, @@ -7766,31 +7773,14 @@ "observation_id": "registry_reported:nvidia/llama-embed-nemotron-8b" }, { - "model": "jcorners/ingot-8b-r3", + "model": "Mira190/Euler-Legal-Embedding-V1", "parameters_billion": 7.567, - "score": 65.4871, - "open_weights": false, + "score": 56.9891, + "open_weights": true, "modalities": [ "text" ], - "training_tasks": [ - "CMedQAv2-reranking", - "CodeSearchNet", - "DuRetrieval", - "FEVER", - "HotpotQA", - "MIRACLRetrieval", - "MMarcoReranking", - "MSMARCO", - "MedrxivClusteringP2P.v2", - "MedrxivClusteringS2S.v2", - "MrTidyRetrieval", - "NQ", - "SprintDuplicateQuestions", - "T2Retrieval", - "TwitterSemEval2015", - "TwitterURLCorpus" - ], + "training_tasks": null, "source_urls": [ "https://mteb-leaderboard-backend.hf.space/v1/benchmarks/MTEB%28eng%2C%20v2%29/scores", "https://mteb-leaderboard-backend.hf.space/v1/benchmarks/MTEB%28Multilingual%2C%20v2%29/scores" @@ -7799,12 +7789,12 @@ "kind": "peer", "on_displayed_frontier": false, "evidence_type": "registry_reported", - "observation_id": "registry_reported:jcorners/ingot-8b-r3" + "observation_id": "registry_reported:Mira190/Euler-Legal-Embedding-V1" }, { - "model": "Qwen/Qwen3-Embedding-8B", + "model": "Octen/Octen-Embedding-8B", "parameters_billion": 7.567, - "score": 64.8951, + "score": 64.7165, "open_weights": true, "modalities": [ "text" @@ -7830,12 +7820,12 @@ "kind": "peer", "on_displayed_frontier": false, "evidence_type": "registry_reported", - "observation_id": "registry_reported:Qwen/Qwen3-Embedding-8B" + "observation_id": "registry_reported:Octen/Octen-Embedding-8B" }, { - "model": "Octen/Octen-Embedding-8B", + "model": "Qwen/Qwen3-Embedding-8B", "parameters_billion": 7.567, - "score": 64.7165, + "score": 64.8951, "open_weights": true, "modalities": [ "text" @@ -7861,17 +7851,34 @@ "kind": "peer", "on_displayed_frontier": false, "evidence_type": "registry_reported", - "observation_id": "registry_reported:Octen/Octen-Embedding-8B" + "observation_id": "registry_reported:Qwen/Qwen3-Embedding-8B" }, { - "model": "Mira190/Euler-Legal-Embedding-V1", + "model": "jcorners/ingot-8b-r3", "parameters_billion": 7.567, - "score": 56.9891, - "open_weights": true, + "score": 65.4871, + "open_weights": false, "modalities": [ "text" ], - "training_tasks": null, + "training_tasks": [ + "CMedQAv2-reranking", + "CodeSearchNet", + "DuRetrieval", + "FEVER", + "HotpotQA", + "MIRACLRetrieval", + "MMarcoReranking", + "MSMARCO", + "MedrxivClusteringP2P.v2", + "MedrxivClusteringS2S.v2", + "MrTidyRetrieval", + "NQ", + "SprintDuplicateQuestions", + "T2Retrieval", + "TwitterSemEval2015", + "TwitterURLCorpus" + ], "source_urls": [ "https://mteb-leaderboard-backend.hf.space/v1/benchmarks/MTEB%28eng%2C%20v2%29/scores", "https://mteb-leaderboard-backend.hf.space/v1/benchmarks/MTEB%28Multilingual%2C%20v2%29/scores" @@ -7880,7 +7887,7 @@ "kind": "peer", "on_displayed_frontier": false, "evidence_type": "registry_reported", - "observation_id": "registry_reported:Mira190/Euler-Legal-Embedding-V1" + "observation_id": "registry_reported:jcorners/ingot-8b-r3" }, { "model": "codefuse-ai/F2LLM-v2-8B", @@ -8217,9 +8224,9 @@ "observation_id": "registry_reported:nvidia/NV-Embed-v2" }, { - "model": "Tevatron/OmniEmbed-v0.1", + "model": "Haon-Chen/e5-omni-7B", "parameters_billion": 8.932, - "score": 58.302699999999994, + "score": 57.7803, "open_weights": true, "modalities": [ "text", @@ -8229,16 +8236,8 @@ ], "training_tasks": [ "AudioCapsT2ARetrieval", - "FEVER", - "MSMARCO", "MSRVTTT2V", - "MSRVTTV2T", - "NQ", - "SCIDOCS", - "VidoreArxivQARetrieval", - "VidoreDocVQARetrieval", - "VidoreInfoVQARetrieval", - "VidoreTatdqaRetrieval" + "MSRVTTV2T" ], "source_urls": [ "https://mteb-leaderboard-backend.hf.space/v1/benchmarks/MTEB%28eng%2C%20v2%29/scores", @@ -8248,7 +8247,7 @@ "kind": "peer", "on_displayed_frontier": false, "evidence_type": "registry_reported", - "observation_id": "registry_reported:Tevatron/OmniEmbed-v0.1" + "observation_id": "registry_reported:Haon-Chen/e5-omni-7B" }, { "model": "LCO-Embedding/LCO-Embedding-Omni-7B", @@ -8273,9 +8272,9 @@ "observation_id": "registry_reported:LCO-Embedding/LCO-Embedding-Omni-7B" }, { - "model": "Haon-Chen/e5-omni-7B", + "model": "Tevatron/OmniEmbed-v0.1", "parameters_billion": 8.932, - "score": 57.7803, + "score": 58.302699999999994, "open_weights": true, "modalities": [ "text", @@ -8285,8 +8284,16 @@ ], "training_tasks": [ "AudioCapsT2ARetrieval", + "FEVER", + "MSMARCO", "MSRVTTT2V", - "MSRVTTV2T" + "MSRVTTV2T", + "NQ", + "SCIDOCS", + "VidoreArxivQARetrieval", + "VidoreDocVQARetrieval", + "VidoreInfoVQARetrieval", + "VidoreTatdqaRetrieval" ], "source_urls": [ "https://mteb-leaderboard-backend.hf.space/v1/benchmarks/MTEB%28eng%2C%20v2%29/scores", @@ -8296,7 +8303,7 @@ "kind": "peer", "on_displayed_frontier": false, "evidence_type": "registry_reported", - "observation_id": "registry_reported:Haon-Chen/e5-omni-7B" + "observation_id": "registry_reported:Tevatron/OmniEmbed-v0.1" }, { "model": "ai-sage/Giga-Embeddings-instruct-10B-A1.8B-0826", @@ -9109,9 +9116,7 @@ "rerun:F2LLM-v2-80M:6faa7738:ArXivHierarchicalClusteringP2P:2026-09-18" ], "peer_observation_count": 211, - "measured_peer_observations": [ - "rerun:F2LLM-v2-80M:6faa7738:ArXivHierarchicalClusteringP2P:2026-09-18" - ], + "measured_peer_observations": 1, "observation_policy": "Registry observations and independently measured observations remain separate, including for the same model. All participate in the descriptive frontier; protocol differences remain explicit." }, "nmsqa": { @@ -9150,7 +9155,9 @@ ], "size_basis": "reported_rounded_total", "kind": "peer", - "on_displayed_frontier": true + "on_displayed_frontier": true, + "evidence_type": "registry_reported", + "observation_id": "registry_reported:google/yamnet" }, { "model": "asapp/sew-d-tiny-100k-ft-ls100h", @@ -9169,7 +9176,9 @@ ], "size_basis": "reported_rounded_total", "kind": "peer", - "on_displayed_frontier": false + "on_displayed_frontier": false, + "evidence_type": "registry_reported", + "observation_id": "registry_reported:asapp/sew-d-tiny-100k-ft-ls100h" }, { "model": "matthewagi/HeAR-s1.1", @@ -9186,7 +9195,9 @@ ], "size_basis": "reported_rounded_total", "kind": "peer", - "on_displayed_frontier": false + "on_displayed_frontier": false, + "evidence_type": "registry_reported", + "observation_id": "registry_reported:matthewagi/HeAR-s1.1" }, { "model": "facebook/encodec_24khz", @@ -9207,7 +9218,9 @@ ], "size_basis": "reported_rounded_total", "kind": "peer", - "on_displayed_frontier": false + "on_displayed_frontier": false, + "evidence_type": "registry_reported", + "observation_id": "registry_reported:facebook/encodec_24khz" }, { "model": "openai/whisper-tiny", @@ -9224,7 +9237,9 @@ ], "size_basis": "reported_rounded_total", "kind": "peer", - "on_displayed_frontier": false + "on_displayed_frontier": false, + "evidence_type": "registry_reported", + "observation_id": "registry_reported:openai/whisper-tiny" }, { "model": "google/vggish", @@ -9243,7 +9258,9 @@ ], "size_basis": "reported_rounded_total", "kind": "peer", - "on_displayed_frontier": false + "on_displayed_frontier": false, + "evidence_type": "registry_reported", + "observation_id": "registry_reported:google/vggish" }, { "model": "openai/whisper-base", @@ -9260,7 +9277,9 @@ ], "size_basis": "reported_rounded_total", "kind": "peer", - "on_displayed_frontier": false + "on_displayed_frontier": false, + "evidence_type": "registry_reported", + "observation_id": "registry_reported:openai/whisper-base" }, { "model": "speechbrain/cnn14-esc50", @@ -9279,7 +9298,9 @@ ], "size_basis": "reported_rounded_total", "kind": "peer", - "on_displayed_frontier": false + "on_displayed_frontier": false, + "evidence_type": "registry_reported", + "observation_id": "registry_reported:speechbrain/cnn14-esc50" }, { "model": "MIT/ast-finetuned-audioset-10-10-0.4593", @@ -9298,7 +9319,9 @@ ], "size_basis": "reported_rounded_total", "kind": "peer", - "on_displayed_frontier": false + "on_displayed_frontier": false, + "evidence_type": "registry_reported", + "observation_id": "registry_reported:MIT/ast-finetuned-audioset-10-10-0.4593" }, { "model": "facebook/data2vec-audio-base-960h", @@ -9315,7 +9338,9 @@ ], "size_basis": "reported_rounded_total", "kind": "peer", - "on_displayed_frontier": false + "on_displayed_frontier": false, + "evidence_type": "registry_reported", + "observation_id": "registry_reported:facebook/data2vec-audio-base-960h" }, { "model": "microsoft/unispeech-sat-base-100h-libri-ft", @@ -9334,18 +9359,20 @@ ], "size_basis": "reported_rounded_total", "kind": "peer", - "on_displayed_frontier": false + "on_displayed_frontier": false, + "evidence_type": "registry_reported", + "observation_id": "registry_reported:microsoft/unispeech-sat-base-100h-libri-ft" }, { - "model": "microsoft/wavlm-base", + "model": "asapp/sew-d-base-plus-400k-ft-ls100h", "parameters_billion": 0.095, - "score": 52.9964, + "score": 49.4138, "open_weights": true, "modalities": [ "audio" ], "training_tasks": [ - "Librispeech" + "LibriSpeech" ], "source_urls": [ "https://mteb-leaderboard-backend.hf.space/v1/benchmarks/MAEB%28beta%2C%20audio-only%29/scores", @@ -9353,47 +9380,47 @@ ], "size_basis": "reported_rounded_total", "kind": "peer", - "on_displayed_frontier": false + "on_displayed_frontier": false, + "evidence_type": "registry_reported", + "observation_id": "registry_reported:asapp/sew-d-base-plus-400k-ft-ls100h" }, { - "model": "microsoft/wavlm-base-sd", + "model": "facebook/hubert-base-ls960", "parameters_billion": 0.095, - "score": 52.9964, + "score": 48.4268, "open_weights": true, "modalities": [ "audio" ], - "training_tasks": [ - "LibriMix", - "Librispeech" - ], + "training_tasks": null, "source_urls": [ "https://mteb-leaderboard-backend.hf.space/v1/benchmarks/MAEB%28beta%2C%20audio-only%29/scores", "https://mteb-leaderboard-backend.hf.space/v1/benchmarks/MAEB%28beta%29/scores" ], "size_basis": "reported_rounded_total", "kind": "peer", - "on_displayed_frontier": false + "on_displayed_frontier": false, + "evidence_type": "registry_reported", + "observation_id": "registry_reported:facebook/hubert-base-ls960" }, { - "model": "microsoft/wavlm-base-sv", + "model": "facebook/wav2vec2-base", "parameters_billion": 0.095, - "score": 52.9964, + "score": 46.995599999999996, "open_weights": true, "modalities": [ "audio" ], - "training_tasks": [ - "Librispeech", - "VoxCeleb1" - ], + "training_tasks": null, "source_urls": [ "https://mteb-leaderboard-backend.hf.space/v1/benchmarks/MAEB%28beta%2C%20audio-only%29/scores", "https://mteb-leaderboard-backend.hf.space/v1/benchmarks/MAEB%28beta%29/scores" ], "size_basis": "reported_rounded_total", "kind": "peer", - "on_displayed_frontier": false + "on_displayed_frontier": false, + "evidence_type": "registry_reported", + "observation_id": "registry_reported:facebook/wav2vec2-base" }, { "model": "facebook/wav2vec2-base-960h", @@ -9410,43 +9437,30 @@ ], "size_basis": "reported_rounded_total", "kind": "peer", - "on_displayed_frontier": false + "on_displayed_frontier": false, + "evidence_type": "registry_reported", + "observation_id": "registry_reported:facebook/wav2vec2-base-960h" }, { - "model": "asapp/sew-d-base-plus-400k-ft-ls100h", + "model": "microsoft/wavlm-base", "parameters_billion": 0.095, - "score": 49.4138, + "score": 52.9964, "open_weights": true, "modalities": [ "audio" ], "training_tasks": [ - "LibriSpeech" - ], - "source_urls": [ - "https://mteb-leaderboard-backend.hf.space/v1/benchmarks/MAEB%28beta%2C%20audio-only%29/scores", - "https://mteb-leaderboard-backend.hf.space/v1/benchmarks/MAEB%28beta%29/scores" - ], - "size_basis": "reported_rounded_total", - "kind": "peer", - "on_displayed_frontier": false - }, - { - "model": "facebook/hubert-base-ls960", - "parameters_billion": 0.095, - "score": 48.4268, - "open_weights": true, - "modalities": [ - "audio" + "Librispeech" ], - "training_tasks": null, "source_urls": [ "https://mteb-leaderboard-backend.hf.space/v1/benchmarks/MAEB%28beta%2C%20audio-only%29/scores", "https://mteb-leaderboard-backend.hf.space/v1/benchmarks/MAEB%28beta%29/scores" ], "size_basis": "reported_rounded_total", "kind": "peer", - "on_displayed_frontier": false + "on_displayed_frontier": false, + "evidence_type": "registry_reported", + "observation_id": "registry_reported:microsoft/wavlm-base" }, { "model": "microsoft/wavlm-base-plus", @@ -9467,7 +9481,9 @@ ], "size_basis": "reported_rounded_total", "kind": "peer", - "on_displayed_frontier": false + "on_displayed_frontier": false, + "evidence_type": "registry_reported", + "observation_id": "registry_reported:microsoft/wavlm-base-plus" }, { "model": "microsoft/wavlm-base-plus-sd", @@ -9489,7 +9505,9 @@ ], "size_basis": "reported_rounded_total", "kind": "peer", - "on_displayed_frontier": false + "on_displayed_frontier": false, + "evidence_type": "registry_reported", + "observation_id": "registry_reported:microsoft/wavlm-base-plus-sd" }, { "model": "microsoft/wavlm-base-plus-sv", @@ -9511,46 +9529,53 @@ ], "size_basis": "reported_rounded_total", "kind": "peer", - "on_displayed_frontier": false + "on_displayed_frontier": false, + "evidence_type": "registry_reported", + "observation_id": "registry_reported:microsoft/wavlm-base-plus-sv" }, { - "model": "facebook/wav2vec2-base", + "model": "microsoft/wavlm-base-sd", "parameters_billion": 0.095, - "score": 46.995599999999996, + "score": 52.9964, "open_weights": true, "modalities": [ "audio" ], - "training_tasks": null, + "training_tasks": [ + "LibriMix", + "Librispeech" + ], "source_urls": [ "https://mteb-leaderboard-backend.hf.space/v1/benchmarks/MAEB%28beta%2C%20audio-only%29/scores", "https://mteb-leaderboard-backend.hf.space/v1/benchmarks/MAEB%28beta%29/scores" ], "size_basis": "reported_rounded_total", "kind": "peer", - "on_displayed_frontier": false + "on_displayed_frontier": false, + "evidence_type": "registry_reported", + "observation_id": "registry_reported:microsoft/wavlm-base-sd" }, { - "model": "llm-semantic-router/Vela-1.0-Omni-Nano", - "parameters_billion": 0.13395648, - "parameters_exact": 133956480, - "native_fingerprint": "fb9cd39c9980a085b75456c715c2295b997784fe733314be68c13635f6384edf", - "score": 63.43546123003914, + "model": "microsoft/wavlm-base-sv", + "parameters_billion": 0.095, + "score": 52.9964, "open_weights": true, "modalities": [ - "text", - "image", "audio" ], - "size_basis": "counted_all_native_model_parameters", - "kind": "nano", - "evaluated_task_revision": "d33eea302d8a64c0a2ee094cf39d77c0814fe87a", - "report_url": "https://huggingface.co/llm-semantic-router/Vela-1.0-Omni-Nano/blob/ed7263b501430a011d37b147d6e264ad8354983b/benchmarks/maeb-audio-only.json", - "on_displayed_frontier": true, - "evaluated_native_fingerprint": "b596f1dbab89369555e0a72a0d6854d780e305686750949b308c8f167993bdeb", - "evaluation_applicability": "Unchanged exact text/audio computation; see the current Nano component-equivalence report.", - "component_equivalence_url": "https://huggingface.co/llm-semantic-router/Vela-1.0-Omni-Nano/blob/main/benchmarks/component-equivalence.json", - "evaluated_revision": "70c3412bb0ddfb19386d673d4a102c753a574f1b" + "training_tasks": [ + "Librispeech", + "VoxCeleb1" + ], + "source_urls": [ + "https://mteb-leaderboard-backend.hf.space/v1/benchmarks/MAEB%28beta%2C%20audio-only%29/scores", + "https://mteb-leaderboard-backend.hf.space/v1/benchmarks/MAEB%28beta%29/scores" + ], + "size_basis": "reported_rounded_total", + "kind": "peer", + "on_displayed_frontier": false, + "evidence_type": "registry_reported", + "observation_id": "registry_reported:microsoft/wavlm-base-sv" }, { "model": "asapp/sew-d-mid-400k-ft-ls100h", @@ -9569,7 +9594,33 @@ ], "size_basis": "reported_rounded_total", "kind": "peer", - "on_displayed_frontier": false + "on_displayed_frontier": false, + "evidence_type": "registry_reported", + "observation_id": "registry_reported:asapp/sew-d-mid-400k-ft-ls100h" + }, + { + "model": "llm-semantic-router/Vela-1.0-Omni-Nano", + "parameters_billion": 0.144554112, + "parameters_exact": 144554112, + "score": 63.28340215485784, + "open_weights": true, + "modalities": [ + "text", + "image", + "audio" + ], + "size_basis": "counted_all_native_model_parameters", + "kind": "nano", + "evaluated_task_revision": "d33eea302d8a64c0a2ee094cf39d77c0814fe87a", + "on_displayed_frontier": true, + "native_artifact_sha256": "96d76f97478e114b816df3ab1b3c6947e4c5ef0babfb40ab865d7e3c838d34c4", + "revision": "self", + "evaluated_native_artifact_sha256": "96d76f97478e114b816df3ab1b3c6947e4c5ef0babfb40ab865d7e3c838d34c4", + "evaluated_revision": "96d76f97478e114b816df3ab1b3c6947e4c5ef0babfb40ab865d7e3c838d34c4", + "evaluated_model": "llm-semantic-router/Vela-1.0-Omni-Nano", + "revision_semantics": "Evaluated revisions are native artifact fingerprints; applies-to identity is the current whole artifact.", + "evidence_type": "native_evaluation", + "report": "benchmarks/maeb-audio-only.json" }, { "model": "laion/clap-htsat-unfused", @@ -9587,7 +9638,9 @@ ], "size_basis": "reported_rounded_total", "kind": "peer", - "on_displayed_frontier": false + "on_displayed_frontier": false, + "evidence_type": "registry_reported", + "observation_id": "registry_reported:laion/clap-htsat-unfused" }, { "model": "laion/clap-htsat-fused", @@ -9605,7 +9658,9 @@ ], "size_basis": "reported_rounded_total", "kind": "peer", - "on_displayed_frontier": false + "on_displayed_frontier": false, + "evidence_type": "registry_reported", + "observation_id": "registry_reported:laion/clap-htsat-fused" }, { "model": "microsoft/msclap-2023", @@ -9627,7 +9682,9 @@ ], "size_basis": "reported_rounded_total", "kind": "peer", - "on_displayed_frontier": false + "on_displayed_frontier": false, + "evidence_type": "registry_reported", + "observation_id": "registry_reported:microsoft/msclap-2023" }, { "model": "lyrebird/wav2clip", @@ -9645,7 +9702,9 @@ ], "size_basis": "reported_rounded_total", "kind": "peer", - "on_displayed_frontier": false + "on_displayed_frontier": false, + "evidence_type": "registry_reported", + "observation_id": "registry_reported:lyrebird/wav2clip" }, { "model": "laion/larger_clap_general", @@ -9663,12 +9722,14 @@ ], "size_basis": "reported_rounded_total", "kind": "peer", - "on_displayed_frontier": false + "on_displayed_frontier": false, + "evidence_type": "registry_reported", + "observation_id": "registry_reported:laion/larger_clap_general" }, { - "model": "laion/larger_clap_music_and_speech", + "model": "laion/larger_clap_music", "parameters_billion": 0.194, - "score": 46.1364, + "score": 44.058, "open_weights": true, "modalities": [ "audio", @@ -9681,12 +9742,14 @@ ], "size_basis": "reported_rounded_total", "kind": "peer", - "on_displayed_frontier": false + "on_displayed_frontier": false, + "evidence_type": "registry_reported", + "observation_id": "registry_reported:laion/larger_clap_music" }, { - "model": "laion/larger_clap_music", + "model": "laion/larger_clap_music_and_speech", "parameters_billion": 0.194, - "score": 44.058, + "score": 46.1364, "open_weights": true, "modalities": [ "audio", @@ -9699,7 +9762,9 @@ ], "size_basis": "reported_rounded_total", "kind": "peer", - "on_displayed_frontier": false + "on_displayed_frontier": false, + "evidence_type": "registry_reported", + "observation_id": "registry_reported:laion/larger_clap_music_and_speech" }, { "model": "microsoft/msclap-2022", @@ -9725,7 +9790,9 @@ ], "size_basis": "reported_rounded_total", "kind": "peer", - "on_displayed_frontier": false + "on_displayed_frontier": false, + "evidence_type": "registry_reported", + "observation_id": "registry_reported:microsoft/msclap-2022" }, { "model": "openai/whisper-small", @@ -9742,7 +9809,9 @@ ], "size_basis": "reported_rounded_total", "kind": "peer", - "on_displayed_frontier": false + "on_displayed_frontier": false, + "evidence_type": "registry_reported", + "observation_id": "registry_reported:openai/whisper-small" }, { "model": "microsoft/speecht5_multimodal", @@ -9760,7 +9829,9 @@ ], "size_basis": "reported_rounded_total", "kind": "peer", - "on_displayed_frontier": true + "on_displayed_frontier": true, + "evidence_type": "registry_reported", + "observation_id": "registry_reported:microsoft/speecht5_multimodal" }, { "model": "facebook/wav2vec2-xls-r-300m", @@ -9781,7 +9852,9 @@ ], "size_basis": "reported_rounded_total", "kind": "peer", - "on_displayed_frontier": false + "on_displayed_frontier": false, + "evidence_type": "registry_reported", + "observation_id": "registry_reported:facebook/wav2vec2-xls-r-300m" }, { "model": "vitouphy/wav2vec2-xls-r-300m-phoneme", @@ -9802,7 +9875,9 @@ ], "size_basis": "reported_rounded_total", "kind": "peer", - "on_displayed_frontier": false + "on_displayed_frontier": false, + "evidence_type": "registry_reported", + "observation_id": "registry_reported:vitouphy/wav2vec2-xls-r-300m-phoneme" }, { "model": "facebook/data2vec-audio-large-960h", @@ -9819,12 +9894,14 @@ ], "size_basis": "reported_rounded_total", "kind": "peer", - "on_displayed_frontier": false + "on_displayed_frontier": false, + "evidence_type": "registry_reported", + "observation_id": "registry_reported:facebook/data2vec-audio-large-960h" }, { - "model": "facebook/wav2vec2-large", + "model": "facebook/hubert-large-ls960-ft", "parameters_billion": 0.317, - "score": 54.156099999999995, + "score": 49.9955, "open_weights": true, "modalities": [ "audio" @@ -9836,31 +9913,33 @@ ], "size_basis": "reported_rounded_total", "kind": "peer", - "on_displayed_frontier": false + "on_displayed_frontier": false, + "evidence_type": "registry_reported", + "observation_id": "registry_reported:facebook/hubert-large-ls960-ft" }, { - "model": "facebook/wav2vec2-large-xlsr-53", + "model": "facebook/wav2vec2-large", "parameters_billion": 0.317, - "score": 53.784600000000005, + "score": 54.156099999999995, "open_weights": true, "modalities": [ "audio" ], - "training_tasks": [ - "CommonVoiceMini17A2TRetrieval" - ], + "training_tasks": null, "source_urls": [ "https://mteb-leaderboard-backend.hf.space/v1/benchmarks/MAEB%28beta%2C%20audio-only%29/scores", "https://mteb-leaderboard-backend.hf.space/v1/benchmarks/MAEB%28beta%29/scores" ], "size_basis": "reported_rounded_total", "kind": "peer", - "on_displayed_frontier": false + "on_displayed_frontier": false, + "evidence_type": "registry_reported", + "observation_id": "registry_reported:facebook/wav2vec2-large" }, { - "model": "facebook/wav2vec2-lv-60-espeak-cv-ft", + "model": "facebook/wav2vec2-large-xlsr-53", "parameters_billion": 0.317, - "score": 52.7305, + "score": 53.784600000000005, "open_weights": true, "modalities": [ "audio" @@ -9874,24 +9953,30 @@ ], "size_basis": "reported_rounded_total", "kind": "peer", - "on_displayed_frontier": false + "on_displayed_frontier": false, + "evidence_type": "registry_reported", + "observation_id": "registry_reported:facebook/wav2vec2-large-xlsr-53" }, { - "model": "facebook/hubert-large-ls960-ft", + "model": "facebook/wav2vec2-lv-60-espeak-cv-ft", "parameters_billion": 0.317, - "score": 49.9955, + "score": 52.7305, "open_weights": true, "modalities": [ "audio" ], - "training_tasks": null, + "training_tasks": [ + "CommonVoiceMini17A2TRetrieval" + ], "source_urls": [ "https://mteb-leaderboard-backend.hf.space/v1/benchmarks/MAEB%28beta%2C%20audio-only%29/scores", "https://mteb-leaderboard-backend.hf.space/v1/benchmarks/MAEB%28beta%29/scores" ], "size_basis": "reported_rounded_total", "kind": "peer", - "on_displayed_frontier": false + "on_displayed_frontier": false, + "evidence_type": "registry_reported", + "observation_id": "registry_reported:facebook/wav2vec2-lv-60-espeak-cv-ft" }, { "model": "microsoft/wavlm-large", @@ -9912,7 +9997,9 @@ ], "size_basis": "reported_rounded_total", "kind": "peer", - "on_displayed_frontier": false + "on_displayed_frontier": false, + "evidence_type": "registry_reported", + "observation_id": "registry_reported:microsoft/wavlm-large" }, { "model": "OpenMuQ/MuQ-MuLan-large", @@ -9930,7 +10017,9 @@ ], "size_basis": "reported_rounded_total", "kind": "peer", - "on_displayed_frontier": false + "on_displayed_frontier": false, + "evidence_type": "registry_reported", + "observation_id": "registry_reported:OpenMuQ/MuQ-MuLan-large" }, { "model": "openai/whisper-medium", @@ -9947,7 +10036,9 @@ ], "size_basis": "reported_rounded_total", "kind": "peer", - "on_displayed_frontier": false + "on_displayed_frontier": false, + "evidence_type": "registry_reported", + "observation_id": "registry_reported:openai/whisper-medium" }, { "model": "jinaai/jina-embeddings-v5-omni-nano", @@ -9967,7 +10058,9 @@ ], "size_basis": "reported_rounded_total", "kind": "peer", - "on_displayed_frontier": true + "on_displayed_frontier": true, + "evidence_type": "registry_reported", + "observation_id": "registry_reported:jinaai/jina-embeddings-v5-omni-nano" }, { "model": "facebook/mms-1b-all", @@ -9986,7 +10079,9 @@ ], "size_basis": "reported_rounded_total", "kind": "peer", - "on_displayed_frontier": false + "on_displayed_frontier": false, + "evidence_type": "registry_reported", + "observation_id": "registry_reported:facebook/mms-1b-all" }, { "model": "facebook/mms-1b-fl102", @@ -10005,7 +10100,9 @@ ], "size_basis": "reported_rounded_total", "kind": "peer", - "on_displayed_frontier": false + "on_displayed_frontier": false, + "evidence_type": "registry_reported", + "observation_id": "registry_reported:facebook/mms-1b-fl102" }, { "model": "facebook/mms-1b-l1107", @@ -10024,7 +10121,9 @@ ], "size_basis": "reported_rounded_total", "kind": "peer", - "on_displayed_frontier": false + "on_displayed_frontier": false, + "evidence_type": "registry_reported", + "observation_id": "registry_reported:facebook/mms-1b-l1107" }, { "model": "facebook/wav2vec2-xls-r-1b", @@ -10045,25 +10144,9 @@ ], "size_basis": "reported_rounded_total", "kind": "peer", - "on_displayed_frontier": false - }, - { - "model": "llm-semantic-router/Vela-1.0-Omni-Mini", - "parameters_billion": 1.04415264, - "parameters_exact": 1044152640, - "native_fingerprint": "035f6084c6c984f465e4f91c50caf8f13790177d4c0c29e4ce5d6c0c233fd1a0", - "score": 86.5526042027746, - "open_weights": true, - "modalities": [ - "text", - "image", - "audio" - ], - "size_basis": "counted_all_native_model_parameters", - "kind": "mini", - "evaluated_task_revision": "d33eea302d8a64c0a2ee094cf39d77c0814fe87a", - "report_url": "https://huggingface.co/llm-semantic-router/Vela-1.0-Omni-Mini/blob/662d4e53c52d22da6430b0ac316360d41028374d/benchmarks/maeb-audio-only.json", - "on_displayed_frontier": false + "on_displayed_frontier": false, + "evidence_type": "registry_reported", + "observation_id": "registry_reported:facebook/wav2vec2-xls-r-1b" }, { "model": "speechbrain/m-ctc-t-large", @@ -10083,7 +10166,9 @@ ], "size_basis": "reported_rounded_total", "kind": "peer", - "on_displayed_frontier": false + "on_displayed_frontier": false, + "evidence_type": "registry_reported", + "observation_id": "registry_reported:speechbrain/m-ctc-t-large" }, { "model": "LanguageBind/LanguageBind_Omni", @@ -10103,7 +10188,30 @@ ], "size_basis": "reported_rounded_total", "kind": "peer", - "on_displayed_frontier": false + "on_displayed_frontier": false, + "evidence_type": "registry_reported", + "observation_id": "registry_reported:LanguageBind/LanguageBind_Omni" + }, + { + "model": "llm-semantic-router/Vela-1.0-Omni-Mini", + "parameters_billion": 1.3328912, + "parameters_exact": 1332891200, + "score": 87.77176044610992, + "open_weights": true, + "modalities": [ + "text", + "image", + "audio" + ], + "size_basis": "counted_all_native_model_parameters", + "kind": "mini", + "evaluated_task_revision": "d33eea302d8a64c0a2ee094cf39d77c0814fe87a", + "on_displayed_frontier": false, + "native_artifact_sha256": "d93ebb5f2f4951d52423ae13da71b037ce44d02957671e1e71bfe25623f3cff2", + "revision": "self", + "evaluated_native_artifact_sha256": "cf5f45cf6e26cd225e0fc0c1ece629b13b8bd0dfe57dcedef739ca8f529f80ab", + "evidence_type": "exact_metadata_only_applicability", + "report": "https://huggingface.co/llm-semantic-router/Vela-1.0-Omni-Mini/blob/main/benchmarks/maeb-audio-only.json" }, { "model": "openai/whisper-large-v3", @@ -10120,7 +10228,9 @@ ], "size_basis": "reported_rounded_total", "kind": "peer", - "on_displayed_frontier": false + "on_displayed_frontier": false, + "evidence_type": "registry_reported", + "observation_id": "registry_reported:openai/whisper-large-v3" }, { "model": "jinaai/jina-embeddings-v5-omni-small", @@ -10140,7 +10250,9 @@ ], "size_basis": "reported_rounded_total", "kind": "peer", - "on_displayed_frontier": false + "on_displayed_frontier": false, + "evidence_type": "registry_reported", + "observation_id": "registry_reported:jinaai/jina-embeddings-v5-omni-small" }, { "model": "encord-team/ebind-full", @@ -10193,7 +10305,9 @@ ], "size_basis": "reported_rounded_total", "kind": "peer", - "on_displayed_frontier": false + "on_displayed_frontier": false, + "evidence_type": "registry_reported", + "observation_id": "registry_reported:encord-team/ebind-full" }, { "model": "facebook/wav2vec2-xls-r-2b", @@ -10214,7 +10328,9 @@ ], "size_basis": "reported_rounded_total", "kind": "peer", - "on_displayed_frontier": false + "on_displayed_frontier": false, + "evidence_type": "registry_reported", + "observation_id": "registry_reported:facebook/wav2vec2-xls-r-2b" }, { "model": "facebook/seamless-m4t-v2-large", @@ -10233,7 +10349,9 @@ ], "size_basis": "reported_rounded_total", "kind": "peer", - "on_displayed_frontier": false + "on_displayed_frontier": false, + "evidence_type": "registry_reported", + "observation_id": "registry_reported:facebook/seamless-m4t-v2-large" }, { "model": "BidirLM/BidirLM-Omni-2.5B-Embedding", @@ -10329,7 +10447,9 @@ ], "size_basis": "reported_rounded_total", "kind": "peer", - "on_displayed_frontier": true + "on_displayed_frontier": true, + "evidence_type": "registry_reported", + "observation_id": "registry_reported:BidirLM/BidirLM-Omni-2.5B-Embedding" }, { "model": "EximiusLabs/fusion-embedding-2-2b-preview", @@ -10366,51 +10486,57 @@ ], "size_basis": "reported_rounded_total", "kind": "peer", - "on_displayed_frontier": false + "on_displayed_frontier": false, + "evidence_type": "registry_reported", + "observation_id": "registry_reported:EximiusLabs/fusion-embedding-2-2b-preview" }, { - "model": "LCO-Embedding/LCO-Embedding-Omni-3B", + "model": "Haon-Chen/e5-omni-3B", "parameters_billion": 4.703, - "score": 96.9203, + "score": 72.8942, "open_weights": true, "modalities": [ - "audio", - "image", "text", + "image", + "audio", "video" ], - "training_tasks": null, + "training_tasks": [ + "AudioCapsT2ARetrieval", + "MSRVTTT2V", + "MSRVTTV2T" + ], "source_urls": [ "https://mteb-leaderboard-backend.hf.space/v1/benchmarks/MAEB%28beta%2C%20audio-only%29/scores", "https://mteb-leaderboard-backend.hf.space/v1/benchmarks/MAEB%28beta%29/scores" ], "size_basis": "reported_rounded_total", "kind": "peer", - "on_displayed_frontier": true + "on_displayed_frontier": false, + "evidence_type": "registry_reported", + "observation_id": "registry_reported:Haon-Chen/e5-omni-3B" }, { - "model": "Haon-Chen/e5-omni-3B", + "model": "LCO-Embedding/LCO-Embedding-Omni-3B", "parameters_billion": 4.703, - "score": 72.8942, + "score": 96.9203, "open_weights": true, "modalities": [ - "text", - "image", "audio", + "image", + "text", "video" ], - "training_tasks": [ - "AudioCapsT2ARetrieval", - "MSRVTTT2V", - "MSRVTTV2T" - ], + "training_tasks": null, "source_urls": [ "https://mteb-leaderboard-backend.hf.space/v1/benchmarks/MAEB%28beta%2C%20audio-only%29/scores", "https://mteb-leaderboard-backend.hf.space/v1/benchmarks/MAEB%28beta%29/scores" ], "size_basis": "reported_rounded_total", "kind": "peer", - "on_displayed_frontier": false + "on_displayed_frontier": true, + "evidence_type": "registry_reported", + "observation_id": "registry_reported:LCO-Embedding/LCO-Embedding-Omni-3B" }, { "model": "Qwen/Qwen2.5-Omni-3B", @@ -10430,7 +10556,9 @@ ], "size_basis": "reported_rounded_total", "kind": "peer", - "on_displayed_frontier": false + "on_displayed_frontier": false, + "evidence_type": "registry_reported", + "observation_id": "registry_reported:Qwen/Qwen2.5-Omni-3B" }, { "model": "Qwen/Qwen2-Audio-7B", @@ -10448,51 +10576,57 @@ ], "size_basis": "reported_rounded_total", "kind": "peer", - "on_displayed_frontier": false + "on_displayed_frontier": false, + "evidence_type": "registry_reported", + "observation_id": "registry_reported:Qwen/Qwen2-Audio-7B" }, { - "model": "LCO-Embedding/LCO-Embedding-Omni-7B", + "model": "Haon-Chen/e5-omni-7B", "parameters_billion": 8.932, - "score": 97.6016, + "score": 89.0584, "open_weights": true, "modalities": [ - "audio", - "image", "text", + "image", + "audio", "video" ], - "training_tasks": null, + "training_tasks": [ + "AudioCapsT2ARetrieval", + "MSRVTTT2V", + "MSRVTTV2T" + ], "source_urls": [ "https://mteb-leaderboard-backend.hf.space/v1/benchmarks/MAEB%28beta%2C%20audio-only%29/scores", "https://mteb-leaderboard-backend.hf.space/v1/benchmarks/MAEB%28beta%29/scores" ], "size_basis": "reported_rounded_total", "kind": "peer", - "on_displayed_frontier": true + "on_displayed_frontier": false, + "evidence_type": "registry_reported", + "observation_id": "registry_reported:Haon-Chen/e5-omni-7B" }, { - "model": "Haon-Chen/e5-omni-7B", + "model": "LCO-Embedding/LCO-Embedding-Omni-7B", "parameters_billion": 8.932, - "score": 89.0584, + "score": 97.6016, "open_weights": true, "modalities": [ - "text", - "image", "audio", + "image", + "text", "video" ], - "training_tasks": [ - "AudioCapsT2ARetrieval", - "MSRVTTT2V", - "MSRVTTV2T" - ], + "training_tasks": null, "source_urls": [ "https://mteb-leaderboard-backend.hf.space/v1/benchmarks/MAEB%28beta%2C%20audio-only%29/scores", "https://mteb-leaderboard-backend.hf.space/v1/benchmarks/MAEB%28beta%29/scores" ], "size_basis": "reported_rounded_total", "kind": "peer", - "on_displayed_frontier": false + "on_displayed_frontier": true, + "evidence_type": "registry_reported", + "observation_id": "registry_reported:LCO-Embedding/LCO-Embedding-Omni-7B" }, { "model": "Tevatron/OmniEmbed-v0.1", @@ -10524,7 +10658,9 @@ ], "size_basis": "reported_rounded_total", "kind": "peer", - "on_displayed_frontier": false + "on_displayed_frontier": false, + "evidence_type": "registry_reported", + "observation_id": "registry_reported:Tevatron/OmniEmbed-v0.1" }, { "model": "Qwen/Qwen2.5-Omni-7B", @@ -10544,7 +10680,9 @@ ], "size_basis": "reported_rounded_total", "kind": "peer", - "on_displayed_frontier": false + "on_displayed_frontier": false, + "evidence_type": "registry_reported", + "observation_id": "registry_reported:Qwen/Qwen2.5-Omni-7B" }, { "model": "Qwen/Qwen3-Omni-30B-A3B-Thinking", @@ -10564,7 +10702,9 @@ ], "size_basis": "reported_rounded_total", "kind": "peer", - "on_displayed_frontier": false + "on_displayed_frontier": false, + "evidence_type": "registry_reported", + "observation_id": "registry_reported:Qwen/Qwen3-Omni-30B-A3B-Thinking" }, { "model": "Qwen/Qwen3-Omni-30B-A3B-Instruct", @@ -10584,7 +10724,9 @@ ], "size_basis": "reported_rounded_total", "kind": "peer", - "on_displayed_frontier": false + "on_displayed_frontier": false, + "evidence_type": "registry_reported", + "observation_id": "registry_reported:Qwen/Qwen3-Omni-30B-A3B-Instruct" } ], "excluded": [], @@ -10597,7 +10739,18 @@ "BidirLM/BidirLM-Omni-2.5B-Embedding", "LCO-Embedding/LCO-Embedding-Omni-3B", "LCO-Embedding/LCO-Embedding-Omni-7B" - ] + ], + "frontier_observations": [ + "registry_reported:google/yamnet", + "llm-semantic-router/Vela-1.0-Omni-Nano", + "registry_reported:microsoft/speecht5_multimodal", + "registry_reported:jinaai/jina-embeddings-v5-omni-nano", + "registry_reported:BidirLM/BidirLM-Omni-2.5B-Embedding", + "registry_reported:LCO-Embedding/LCO-Embedding-Omni-3B", + "registry_reported:LCO-Embedding/LCO-Embedding-Omni-7B" + ], + "peer_observation_count": 66, + "measured_peer_observations": 0 }, "sibfleurs": { "task": "SIBFLEURS", @@ -10635,7 +10788,9 @@ ], "size_basis": "reported_rounded_total", "kind": "peer", - "on_displayed_frontier": true + "on_displayed_frontier": true, + "evidence_type": "registry_reported", + "observation_id": "registry_reported:google/yamnet" }, { "model": "asapp/sew-d-tiny-100k-ft-ls100h", @@ -10654,7 +10809,9 @@ ], "size_basis": "reported_rounded_total", "kind": "peer", - "on_displayed_frontier": true + "on_displayed_frontier": true, + "evidence_type": "registry_reported", + "observation_id": "registry_reported:asapp/sew-d-tiny-100k-ft-ls100h" }, { "model": "matthewagi/HeAR-s1.1", @@ -10671,7 +10828,9 @@ ], "size_basis": "reported_rounded_total", "kind": "peer", - "on_displayed_frontier": true + "on_displayed_frontier": true, + "evidence_type": "registry_reported", + "observation_id": "registry_reported:matthewagi/HeAR-s1.1" }, { "model": "facebook/encodec_24khz", @@ -10692,7 +10851,9 @@ ], "size_basis": "reported_rounded_total", "kind": "peer", - "on_displayed_frontier": false + "on_displayed_frontier": false, + "evidence_type": "registry_reported", + "observation_id": "registry_reported:facebook/encodec_24khz" }, { "model": "openai/whisper-tiny", @@ -10709,7 +10870,9 @@ ], "size_basis": "reported_rounded_total", "kind": "peer", - "on_displayed_frontier": false + "on_displayed_frontier": false, + "evidence_type": "registry_reported", + "observation_id": "registry_reported:openai/whisper-tiny" }, { "model": "google/vggish", @@ -10728,7 +10891,9 @@ ], "size_basis": "reported_rounded_total", "kind": "peer", - "on_displayed_frontier": false + "on_displayed_frontier": false, + "evidence_type": "registry_reported", + "observation_id": "registry_reported:google/vggish" }, { "model": "openai/whisper-base", @@ -10745,7 +10910,9 @@ ], "size_basis": "reported_rounded_total", "kind": "peer", - "on_displayed_frontier": false + "on_displayed_frontier": false, + "evidence_type": "registry_reported", + "observation_id": "registry_reported:openai/whisper-base" }, { "model": "speechbrain/cnn14-esc50", @@ -10764,7 +10931,9 @@ ], "size_basis": "reported_rounded_total", "kind": "peer", - "on_displayed_frontier": false + "on_displayed_frontier": false, + "evidence_type": "registry_reported", + "observation_id": "registry_reported:speechbrain/cnn14-esc50" }, { "model": "MIT/ast-finetuned-audioset-10-10-0.4593", @@ -10783,7 +10952,9 @@ ], "size_basis": "reported_rounded_total", "kind": "peer", - "on_displayed_frontier": false + "on_displayed_frontier": false, + "evidence_type": "registry_reported", + "observation_id": "registry_reported:MIT/ast-finetuned-audioset-10-10-0.4593" }, { "model": "facebook/data2vec-audio-base-960h", @@ -10800,7 +10971,9 @@ ], "size_basis": "reported_rounded_total", "kind": "peer", - "on_displayed_frontier": false + "on_displayed_frontier": false, + "evidence_type": "registry_reported", + "observation_id": "registry_reported:facebook/data2vec-audio-base-960h" }, { "model": "microsoft/unispeech-sat-base-100h-libri-ft", @@ -10819,7 +10992,9 @@ ], "size_basis": "reported_rounded_total", "kind": "peer", - "on_displayed_frontier": false + "on_displayed_frontier": false, + "evidence_type": "registry_reported", + "observation_id": "registry_reported:microsoft/unispeech-sat-base-100h-libri-ft" }, { "model": "asapp/sew-d-base-plus-400k-ft-ls100h", @@ -10838,120 +11013,137 @@ ], "size_basis": "reported_rounded_total", "kind": "peer", - "on_displayed_frontier": false + "on_displayed_frontier": false, + "evidence_type": "registry_reported", + "observation_id": "registry_reported:asapp/sew-d-base-plus-400k-ft-ls100h" }, { - "model": "microsoft/wavlm-base", + "model": "facebook/hubert-base-ls960", "parameters_billion": 0.095, - "score": 13.31318431372549, + "score": 13.11168431372549, "open_weights": true, "modalities": [ "audio" ], - "training_tasks": [ - "Librispeech" - ], + "training_tasks": null, "source_urls": [ "https://mteb-leaderboard-backend.hf.space/v1/benchmarks/MAEB%28beta%2C%20audio-only%29/scores", "https://mteb-leaderboard-backend.hf.space/v1/benchmarks/MAEB%28beta%29/scores" ], "size_basis": "reported_rounded_total", "kind": "peer", - "on_displayed_frontier": false + "on_displayed_frontier": false, + "evidence_type": "registry_reported", + "observation_id": "registry_reported:facebook/hubert-base-ls960" }, { - "model": "microsoft/wavlm-base-sd", + "model": "facebook/wav2vec2-base", "parameters_billion": 0.095, - "score": 13.31318431372549, + "score": 13.128345098039215, "open_weights": true, "modalities": [ "audio" ], - "training_tasks": [ - "LibriMix", - "Librispeech" - ], + "training_tasks": null, "source_urls": [ "https://mteb-leaderboard-backend.hf.space/v1/benchmarks/MAEB%28beta%2C%20audio-only%29/scores", "https://mteb-leaderboard-backend.hf.space/v1/benchmarks/MAEB%28beta%29/scores" ], "size_basis": "reported_rounded_total", "kind": "peer", - "on_displayed_frontier": false + "on_displayed_frontier": false, + "evidence_type": "registry_reported", + "observation_id": "registry_reported:facebook/wav2vec2-base" }, { - "model": "microsoft/wavlm-base-sv", + "model": "facebook/wav2vec2-base-960h", "parameters_billion": 0.095, - "score": 13.31318431372549, + "score": 12.482754901960783, "open_weights": true, "modalities": [ "audio" ], - "training_tasks": [ - "Librispeech", - "VoxCeleb1" - ], + "training_tasks": null, "source_urls": [ "https://mteb-leaderboard-backend.hf.space/v1/benchmarks/MAEB%28beta%2C%20audio-only%29/scores", "https://mteb-leaderboard-backend.hf.space/v1/benchmarks/MAEB%28beta%29/scores" ], "size_basis": "reported_rounded_total", "kind": "peer", - "on_displayed_frontier": false + "on_displayed_frontier": false, + "evidence_type": "registry_reported", + "observation_id": "registry_reported:facebook/wav2vec2-base-960h" }, { - "model": "facebook/wav2vec2-base", + "model": "microsoft/wavlm-base", "parameters_billion": 0.095, - "score": 13.128345098039215, + "score": 13.31318431372549, "open_weights": true, "modalities": [ "audio" ], - "training_tasks": null, + "training_tasks": [ + "Librispeech" + ], "source_urls": [ "https://mteb-leaderboard-backend.hf.space/v1/benchmarks/MAEB%28beta%2C%20audio-only%29/scores", "https://mteb-leaderboard-backend.hf.space/v1/benchmarks/MAEB%28beta%29/scores" ], "size_basis": "reported_rounded_total", "kind": "peer", - "on_displayed_frontier": false + "on_displayed_frontier": false, + "evidence_type": "registry_reported", + "observation_id": "registry_reported:microsoft/wavlm-base" }, { - "model": "facebook/hubert-base-ls960", + "model": "microsoft/wavlm-base-plus", "parameters_billion": 0.095, - "score": 13.11168431372549, + "score": 11.857322549019608, "open_weights": true, "modalities": [ "audio" ], - "training_tasks": null, + "training_tasks": [ + "GigaSpeechA2TRetrieval", + "Libri-Light", + "VoxPopuliLanguageID" + ], "source_urls": [ "https://mteb-leaderboard-backend.hf.space/v1/benchmarks/MAEB%28beta%2C%20audio-only%29/scores", "https://mteb-leaderboard-backend.hf.space/v1/benchmarks/MAEB%28beta%29/scores" ], "size_basis": "reported_rounded_total", "kind": "peer", - "on_displayed_frontier": false + "on_displayed_frontier": false, + "evidence_type": "registry_reported", + "observation_id": "registry_reported:microsoft/wavlm-base-plus" }, { - "model": "facebook/wav2vec2-base-960h", + "model": "microsoft/wavlm-base-plus-sd", "parameters_billion": 0.095, - "score": 12.482754901960783, + "score": 11.857322549019608, "open_weights": true, "modalities": [ "audio" ], - "training_tasks": null, + "training_tasks": [ + "GigaSpeechA2TRetrieval", + "Libri-Light", + "LibriMix", + "VoxPopuliLanguageID" + ], "source_urls": [ "https://mteb-leaderboard-backend.hf.space/v1/benchmarks/MAEB%28beta%2C%20audio-only%29/scores", "https://mteb-leaderboard-backend.hf.space/v1/benchmarks/MAEB%28beta%29/scores" ], "size_basis": "reported_rounded_total", "kind": "peer", - "on_displayed_frontier": false + "on_displayed_frontier": false, + "evidence_type": "registry_reported", + "observation_id": "registry_reported:microsoft/wavlm-base-plus-sd" }, { - "model": "microsoft/wavlm-base-plus", + "model": "microsoft/wavlm-base-plus-sv", "parameters_billion": 0.095, "score": 11.857322549019608, "open_weights": true, @@ -10961,6 +11153,7 @@ "training_tasks": [ "GigaSpeechA2TRetrieval", "Libri-Light", + "VoxCeleb1", "VoxPopuliLanguageID" ], "source_urls": [ @@ -10969,21 +11162,21 @@ ], "size_basis": "reported_rounded_total", "kind": "peer", - "on_displayed_frontier": false + "on_displayed_frontier": false, + "evidence_type": "registry_reported", + "observation_id": "registry_reported:microsoft/wavlm-base-plus-sv" }, { - "model": "microsoft/wavlm-base-plus-sd", + "model": "microsoft/wavlm-base-sd", "parameters_billion": 0.095, - "score": 11.857322549019608, + "score": 13.31318431372549, "open_weights": true, "modalities": [ "audio" ], "training_tasks": [ - "GigaSpeechA2TRetrieval", - "Libri-Light", "LibriMix", - "VoxPopuliLanguageID" + "Librispeech" ], "source_urls": [ "https://mteb-leaderboard-backend.hf.space/v1/benchmarks/MAEB%28beta%2C%20audio-only%29/scores", @@ -10991,21 +11184,21 @@ ], "size_basis": "reported_rounded_total", "kind": "peer", - "on_displayed_frontier": false + "on_displayed_frontier": false, + "evidence_type": "registry_reported", + "observation_id": "registry_reported:microsoft/wavlm-base-sd" }, { - "model": "microsoft/wavlm-base-plus-sv", + "model": "microsoft/wavlm-base-sv", "parameters_billion": 0.095, - "score": 11.857322549019608, + "score": 13.31318431372549, "open_weights": true, "modalities": [ "audio" ], "training_tasks": [ - "GigaSpeechA2TRetrieval", - "Libri-Light", - "VoxCeleb1", - "VoxPopuliLanguageID" + "Librispeech", + "VoxCeleb1" ], "source_urls": [ "https://mteb-leaderboard-backend.hf.space/v1/benchmarks/MAEB%28beta%2C%20audio-only%29/scores", @@ -11013,29 +11206,9 @@ ], "size_basis": "reported_rounded_total", "kind": "peer", - "on_displayed_frontier": false - }, - { - "model": "llm-semantic-router/Vela-1.0-Omni-Nano", - "parameters_billion": 0.13395648, - "parameters_exact": 133956480, - "native_fingerprint": "fb9cd39c9980a085b75456c715c2295b997784fe733314be68c13635f6384edf", - "score": 18.304657831512046, - "open_weights": true, - "modalities": [ - "text", - "image", - "audio" - ], - "size_basis": "counted_all_native_model_parameters", - "kind": "nano", - "evaluated_task_revision": "186b61175fbd77059f769b6bf1110d449a2ff311", - "report_url": "https://huggingface.co/llm-semantic-router/Vela-1.0-Omni-Nano/blob/ed7263b501430a011d37b147d6e264ad8354983b/benchmarks/maeb-audio-only.json", "on_displayed_frontier": false, - "evaluated_native_fingerprint": "b596f1dbab89369555e0a72a0d6854d780e305686750949b308c8f167993bdeb", - "evaluation_applicability": "Unchanged exact text/audio computation; see the current Nano component-equivalence report.", - "component_equivalence_url": "https://huggingface.co/llm-semantic-router/Vela-1.0-Omni-Nano/blob/main/benchmarks/component-equivalence.json", - "evaluated_revision": "70c3412bb0ddfb19386d673d4a102c753a574f1b" + "evidence_type": "registry_reported", + "observation_id": "registry_reported:microsoft/wavlm-base-sv" }, { "model": "asapp/sew-d-mid-400k-ft-ls100h", @@ -11054,7 +11227,33 @@ ], "size_basis": "reported_rounded_total", "kind": "peer", - "on_displayed_frontier": false + "on_displayed_frontier": false, + "evidence_type": "registry_reported", + "observation_id": "registry_reported:asapp/sew-d-mid-400k-ft-ls100h" + }, + { + "model": "llm-semantic-router/Vela-1.0-Omni-Nano", + "parameters_billion": 0.144554112, + "parameters_exact": 144554112, + "score": 18.304657831512046, + "open_weights": true, + "modalities": [ + "text", + "image", + "audio" + ], + "size_basis": "counted_all_native_model_parameters", + "kind": "nano", + "evaluated_task_revision": "186b61175fbd77059f769b6bf1110d449a2ff311", + "on_displayed_frontier": false, + "native_artifact_sha256": "96d76f97478e114b816df3ab1b3c6947e4c5ef0babfb40ab865d7e3c838d34c4", + "revision": "self", + "evaluated_native_artifact_sha256": "96d76f97478e114b816df3ab1b3c6947e4c5ef0babfb40ab865d7e3c838d34c4", + "evaluated_revision": "96d76f97478e114b816df3ab1b3c6947e4c5ef0babfb40ab865d7e3c838d34c4", + "evaluated_model": "llm-semantic-router/Vela-1.0-Omni-Nano", + "revision_semantics": "Evaluated revisions are native artifact fingerprints; applies-to identity is the current whole artifact.", + "evidence_type": "native_evaluation", + "report": "benchmarks/maeb-audio-only.json" }, { "model": "laion/clap-htsat-unfused", @@ -11072,7 +11271,9 @@ ], "size_basis": "reported_rounded_total", "kind": "peer", - "on_displayed_frontier": false + "on_displayed_frontier": false, + "evidence_type": "registry_reported", + "observation_id": "registry_reported:laion/clap-htsat-unfused" }, { "model": "laion/clap-htsat-fused", @@ -11090,7 +11291,9 @@ ], "size_basis": "reported_rounded_total", "kind": "peer", - "on_displayed_frontier": false + "on_displayed_frontier": false, + "evidence_type": "registry_reported", + "observation_id": "registry_reported:laion/clap-htsat-fused" }, { "model": "microsoft/msclap-2023", @@ -11112,7 +11315,9 @@ ], "size_basis": "reported_rounded_total", "kind": "peer", - "on_displayed_frontier": false + "on_displayed_frontier": false, + "evidence_type": "registry_reported", + "observation_id": "registry_reported:microsoft/msclap-2023" }, { "model": "lyrebird/wav2clip", @@ -11130,7 +11335,9 @@ ], "size_basis": "reported_rounded_total", "kind": "peer", - "on_displayed_frontier": false + "on_displayed_frontier": false, + "evidence_type": "registry_reported", + "observation_id": "registry_reported:lyrebird/wav2clip" }, { "model": "laion/larger_clap_general", @@ -11148,12 +11355,14 @@ ], "size_basis": "reported_rounded_total", "kind": "peer", - "on_displayed_frontier": false + "on_displayed_frontier": false, + "evidence_type": "registry_reported", + "observation_id": "registry_reported:laion/larger_clap_general" }, { - "model": "laion/larger_clap_music_and_speech", + "model": "laion/larger_clap_music", "parameters_billion": 0.194, - "score": 8.903358823529413, + "score": 5.552189215686274, "open_weights": true, "modalities": [ "audio", @@ -11166,12 +11375,14 @@ ], "size_basis": "reported_rounded_total", "kind": "peer", - "on_displayed_frontier": false + "on_displayed_frontier": false, + "evidence_type": "registry_reported", + "observation_id": "registry_reported:laion/larger_clap_music" }, { - "model": "laion/larger_clap_music", + "model": "laion/larger_clap_music_and_speech", "parameters_billion": 0.194, - "score": 5.552189215686274, + "score": 8.903358823529413, "open_weights": true, "modalities": [ "audio", @@ -11184,7 +11395,9 @@ ], "size_basis": "reported_rounded_total", "kind": "peer", - "on_displayed_frontier": false + "on_displayed_frontier": false, + "evidence_type": "registry_reported", + "observation_id": "registry_reported:laion/larger_clap_music_and_speech" }, { "model": "microsoft/msclap-2022", @@ -11210,7 +11423,9 @@ ], "size_basis": "reported_rounded_total", "kind": "peer", - "on_displayed_frontier": false + "on_displayed_frontier": false, + "evidence_type": "registry_reported", + "observation_id": "registry_reported:microsoft/msclap-2022" }, { "model": "openai/whisper-small", @@ -11227,7 +11442,9 @@ ], "size_basis": "reported_rounded_total", "kind": "peer", - "on_displayed_frontier": false + "on_displayed_frontier": false, + "evidence_type": "registry_reported", + "observation_id": "registry_reported:openai/whisper-small" }, { "model": "microsoft/speecht5_multimodal", @@ -11245,12 +11462,14 @@ ], "size_basis": "reported_rounded_total", "kind": "peer", - "on_displayed_frontier": false + "on_displayed_frontier": false, + "evidence_type": "registry_reported", + "observation_id": "registry_reported:microsoft/speecht5_multimodal" }, { - "model": "vitouphy/wav2vec2-xls-r-300m-phoneme", + "model": "facebook/wav2vec2-xls-r-300m", "parameters_billion": 0.3, - "score": 16.31054705882353, + "score": 14.27458823529412, "open_weights": true, "modalities": [ "audio" @@ -11266,12 +11485,14 @@ ], "size_basis": "reported_rounded_total", "kind": "peer", - "on_displayed_frontier": false + "on_displayed_frontier": false, + "evidence_type": "registry_reported", + "observation_id": "registry_reported:facebook/wav2vec2-xls-r-300m" }, { - "model": "facebook/wav2vec2-xls-r-300m", + "model": "vitouphy/wav2vec2-xls-r-300m-phoneme", "parameters_billion": 0.3, - "score": 14.27458823529412, + "score": 16.31054705882353, "open_weights": true, "modalities": [ "audio" @@ -11287,7 +11508,9 @@ ], "size_basis": "reported_rounded_total", "kind": "peer", - "on_displayed_frontier": false + "on_displayed_frontier": false, + "evidence_type": "registry_reported", + "observation_id": "registry_reported:vitouphy/wav2vec2-xls-r-300m-phoneme" }, { "model": "facebook/data2vec-audio-large-960h", @@ -11304,31 +11527,33 @@ ], "size_basis": "reported_rounded_total", "kind": "peer", - "on_displayed_frontier": false + "on_displayed_frontier": false, + "evidence_type": "registry_reported", + "observation_id": "registry_reported:facebook/data2vec-audio-large-960h" }, { - "model": "facebook/wav2vec2-lv-60-espeak-cv-ft", + "model": "facebook/hubert-large-ls960-ft", "parameters_billion": 0.317, - "score": 14.815935294117647, + "score": 13.514298039215685, "open_weights": true, "modalities": [ "audio" ], - "training_tasks": [ - "CommonVoiceMini17A2TRetrieval" - ], + "training_tasks": null, "source_urls": [ "https://mteb-leaderboard-backend.hf.space/v1/benchmarks/MAEB%28beta%2C%20audio-only%29/scores", "https://mteb-leaderboard-backend.hf.space/v1/benchmarks/MAEB%28beta%29/scores" ], "size_basis": "reported_rounded_total", "kind": "peer", - "on_displayed_frontier": false + "on_displayed_frontier": false, + "evidence_type": "registry_reported", + "observation_id": "registry_reported:facebook/hubert-large-ls960-ft" }, { - "model": "facebook/hubert-large-ls960-ft", + "model": "facebook/wav2vec2-large", "parameters_billion": 0.317, - "score": 13.514298039215685, + "score": 11.303959803921568, "open_weights": true, "modalities": [ "audio" @@ -11340,20 +11565,20 @@ ], "size_basis": "reported_rounded_total", "kind": "peer", - "on_displayed_frontier": false + "on_displayed_frontier": false, + "evidence_type": "registry_reported", + "observation_id": "registry_reported:facebook/wav2vec2-large" }, { - "model": "microsoft/wavlm-large", + "model": "facebook/wav2vec2-large-xlsr-53", "parameters_billion": 0.317, - "score": 12.54475588235294, + "score": 5.477791176470589, "open_weights": true, "modalities": [ "audio" ], "training_tasks": [ - "GigaSpeechA2TRetrieval", - "Libri-Light", - "VoxPopuliLanguageID" + "CommonVoiceMini17A2TRetrieval" ], "source_urls": [ "https://mteb-leaderboard-backend.hf.space/v1/benchmarks/MAEB%28beta%2C%20audio-only%29/scores", @@ -11361,35 +11586,43 @@ ], "size_basis": "reported_rounded_total", "kind": "peer", - "on_displayed_frontier": false + "on_displayed_frontier": false, + "evidence_type": "registry_reported", + "observation_id": "registry_reported:facebook/wav2vec2-large-xlsr-53" }, { - "model": "facebook/wav2vec2-large", + "model": "facebook/wav2vec2-lv-60-espeak-cv-ft", "parameters_billion": 0.317, - "score": 11.303959803921568, + "score": 14.815935294117647, "open_weights": true, "modalities": [ "audio" ], - "training_tasks": null, + "training_tasks": [ + "CommonVoiceMini17A2TRetrieval" + ], "source_urls": [ "https://mteb-leaderboard-backend.hf.space/v1/benchmarks/MAEB%28beta%2C%20audio-only%29/scores", "https://mteb-leaderboard-backend.hf.space/v1/benchmarks/MAEB%28beta%29/scores" ], "size_basis": "reported_rounded_total", "kind": "peer", - "on_displayed_frontier": false + "on_displayed_frontier": false, + "evidence_type": "registry_reported", + "observation_id": "registry_reported:facebook/wav2vec2-lv-60-espeak-cv-ft" }, { - "model": "facebook/wav2vec2-large-xlsr-53", + "model": "microsoft/wavlm-large", "parameters_billion": 0.317, - "score": 5.477791176470589, + "score": 12.54475588235294, "open_weights": true, "modalities": [ "audio" ], "training_tasks": [ - "CommonVoiceMini17A2TRetrieval" + "GigaSpeechA2TRetrieval", + "Libri-Light", + "VoxPopuliLanguageID" ], "source_urls": [ "https://mteb-leaderboard-backend.hf.space/v1/benchmarks/MAEB%28beta%2C%20audio-only%29/scores", @@ -11397,7 +11630,9 @@ ], "size_basis": "reported_rounded_total", "kind": "peer", - "on_displayed_frontier": false + "on_displayed_frontier": false, + "evidence_type": "registry_reported", + "observation_id": "registry_reported:microsoft/wavlm-large" }, { "model": "OpenMuQ/MuQ-MuLan-large", @@ -11415,7 +11650,9 @@ ], "size_basis": "reported_rounded_total", "kind": "peer", - "on_displayed_frontier": false + "on_displayed_frontier": false, + "evidence_type": "registry_reported", + "observation_id": "registry_reported:OpenMuQ/MuQ-MuLan-large" }, { "model": "openai/whisper-medium", @@ -11432,7 +11669,9 @@ ], "size_basis": "reported_rounded_total", "kind": "peer", - "on_displayed_frontier": false + "on_displayed_frontier": false, + "evidence_type": "registry_reported", + "observation_id": "registry_reported:openai/whisper-medium" }, { "model": "jinaai/jina-embeddings-v5-omni-nano", @@ -11452,12 +11691,14 @@ ], "size_basis": "reported_rounded_total", "kind": "peer", - "on_displayed_frontier": true + "on_displayed_frontier": true, + "evidence_type": "registry_reported", + "observation_id": "registry_reported:jinaai/jina-embeddings-v5-omni-nano" }, { - "model": "facebook/mms-1b-fl102", + "model": "facebook/mms-1b-all", "parameters_billion": 1.0, - "score": 26.19661960784314, + "score": 24.13314705882353, "open_weights": true, "modalities": [ "audio" @@ -11471,12 +11712,14 @@ ], "size_basis": "reported_rounded_total", "kind": "peer", - "on_displayed_frontier": false + "on_displayed_frontier": false, + "evidence_type": "registry_reported", + "observation_id": "registry_reported:facebook/mms-1b-all" }, { - "model": "facebook/mms-1b-all", + "model": "facebook/mms-1b-fl102", "parameters_billion": 1.0, - "score": 24.13314705882353, + "score": 26.19661960784314, "open_weights": true, "modalities": [ "audio" @@ -11490,7 +11733,9 @@ ], "size_basis": "reported_rounded_total", "kind": "peer", - "on_displayed_frontier": false + "on_displayed_frontier": false, + "evidence_type": "registry_reported", + "observation_id": "registry_reported:facebook/mms-1b-fl102" }, { "model": "facebook/mms-1b-l1107", @@ -11509,7 +11754,9 @@ ], "size_basis": "reported_rounded_total", "kind": "peer", - "on_displayed_frontier": false + "on_displayed_frontier": false, + "evidence_type": "registry_reported", + "observation_id": "registry_reported:facebook/mms-1b-l1107" }, { "model": "facebook/wav2vec2-xls-r-1b", @@ -11528,27 +11775,11 @@ "https://mteb-leaderboard-backend.hf.space/v1/benchmarks/MAEB%28beta%2C%20audio-only%29/scores", "https://mteb-leaderboard-backend.hf.space/v1/benchmarks/MAEB%28beta%29/scores" ], - "size_basis": "reported_rounded_total", - "kind": "peer", - "on_displayed_frontier": false - }, - { - "model": "llm-semantic-router/Vela-1.0-Omni-Mini", - "parameters_billion": 1.04415264, - "parameters_exact": 1044152640, - "native_fingerprint": "035f6084c6c984f465e4f91c50caf8f13790177d4c0c29e4ce5d6c0c233fd1a0", - "score": 39.43152755173216, - "open_weights": true, - "modalities": [ - "text", - "image", - "audio" - ], - "size_basis": "counted_all_native_model_parameters", - "kind": "mini", - "evaluated_task_revision": "186b61175fbd77059f769b6bf1110d449a2ff311", - "report_url": "https://huggingface.co/llm-semantic-router/Vela-1.0-Omni-Mini/blob/662d4e53c52d22da6430b0ac316360d41028374d/benchmarks/maeb-audio-only.json", - "on_displayed_frontier": true + "size_basis": "reported_rounded_total", + "kind": "peer", + "on_displayed_frontier": false, + "evidence_type": "registry_reported", + "observation_id": "registry_reported:facebook/wav2vec2-xls-r-1b" }, { "model": "speechbrain/m-ctc-t-large", @@ -11568,7 +11799,9 @@ ], "size_basis": "reported_rounded_total", "kind": "peer", - "on_displayed_frontier": false + "on_displayed_frontier": false, + "evidence_type": "registry_reported", + "observation_id": "registry_reported:speechbrain/m-ctc-t-large" }, { "model": "LanguageBind/LanguageBind_Omni", @@ -11588,7 +11821,30 @@ ], "size_basis": "reported_rounded_total", "kind": "peer", - "on_displayed_frontier": false + "on_displayed_frontier": false, + "evidence_type": "registry_reported", + "observation_id": "registry_reported:LanguageBind/LanguageBind_Omni" + }, + { + "model": "llm-semantic-router/Vela-1.0-Omni-Mini", + "parameters_billion": 1.3328912, + "parameters_exact": 1332891200, + "score": 40.06316360536309, + "open_weights": true, + "modalities": [ + "text", + "image", + "audio" + ], + "size_basis": "counted_all_native_model_parameters", + "kind": "mini", + "evaluated_task_revision": "186b61175fbd77059f769b6bf1110d449a2ff311", + "on_displayed_frontier": true, + "native_artifact_sha256": "d93ebb5f2f4951d52423ae13da71b037ce44d02957671e1e71bfe25623f3cff2", + "revision": "self", + "evaluated_native_artifact_sha256": "cf5f45cf6e26cd225e0fc0c1ece629b13b8bd0dfe57dcedef739ca8f529f80ab", + "evidence_type": "exact_metadata_only_applicability", + "report": "https://huggingface.co/llm-semantic-router/Vela-1.0-Omni-Mini/blob/main/benchmarks/maeb-audio-only.json" }, { "model": "openai/whisper-large-v3", @@ -11605,7 +11861,9 @@ ], "size_basis": "reported_rounded_total", "kind": "peer", - "on_displayed_frontier": false + "on_displayed_frontier": false, + "evidence_type": "registry_reported", + "observation_id": "registry_reported:openai/whisper-large-v3" }, { "model": "jinaai/jina-embeddings-v5-omni-small", @@ -11625,7 +11883,9 @@ ], "size_basis": "reported_rounded_total", "kind": "peer", - "on_displayed_frontier": false + "on_displayed_frontier": false, + "evidence_type": "registry_reported", + "observation_id": "registry_reported:jinaai/jina-embeddings-v5-omni-small" }, { "model": "encord-team/ebind-full", @@ -11678,12 +11938,14 @@ ], "size_basis": "reported_rounded_total", "kind": "peer", - "on_displayed_frontier": false + "on_displayed_frontier": false, + "evidence_type": "registry_reported", + "observation_id": "registry_reported:encord-team/ebind-full" }, { - "model": "facebook/wav2vec2-xls-r-2b-21-to-en", + "model": "facebook/wav2vec2-xls-r-2b", "parameters_billion": 2.0, - "score": 14.2292, + "score": 13.607690196078432, "open_weights": true, "modalities": [ "audio" @@ -11699,12 +11961,14 @@ ], "size_basis": "reported_rounded_total", "kind": "peer", - "on_displayed_frontier": false + "on_displayed_frontier": false, + "evidence_type": "registry_reported", + "observation_id": "registry_reported:facebook/wav2vec2-xls-r-2b" }, { - "model": "facebook/wav2vec2-xls-r-2b", + "model": "facebook/wav2vec2-xls-r-2b-21-to-en", "parameters_billion": 2.0, - "score": 13.607690196078432, + "score": 14.2292, "open_weights": true, "modalities": [ "audio" @@ -11720,7 +11984,9 @@ ], "size_basis": "reported_rounded_total", "kind": "peer", - "on_displayed_frontier": false + "on_displayed_frontier": false, + "evidence_type": "registry_reported", + "observation_id": "registry_reported:facebook/wav2vec2-xls-r-2b-21-to-en" }, { "model": "facebook/seamless-m4t-v2-large", @@ -11739,7 +12005,9 @@ ], "size_basis": "reported_rounded_total", "kind": "peer", - "on_displayed_frontier": true + "on_displayed_frontier": true, + "evidence_type": "registry_reported", + "observation_id": "registry_reported:facebook/seamless-m4t-v2-large" }, { "model": "BidirLM/BidirLM-Omni-2.5B-Embedding", @@ -11835,7 +12103,9 @@ ], "size_basis": "reported_rounded_total", "kind": "peer", - "on_displayed_frontier": true + "on_displayed_frontier": true, + "evidence_type": "registry_reported", + "observation_id": "registry_reported:BidirLM/BidirLM-Omni-2.5B-Embedding" }, { "model": "EximiusLabs/fusion-embedding-2-2b-preview", @@ -11872,51 +12142,57 @@ ], "size_basis": "reported_rounded_total", "kind": "peer", - "on_displayed_frontier": false + "on_displayed_frontier": false, + "evidence_type": "registry_reported", + "observation_id": "registry_reported:EximiusLabs/fusion-embedding-2-2b-preview" }, { - "model": "LCO-Embedding/LCO-Embedding-Omni-3B", + "model": "Haon-Chen/e5-omni-3B", "parameters_billion": 4.703, - "score": 45.062387254901964, + "score": 32.91909117647059, "open_weights": true, "modalities": [ - "audio", - "image", "text", + "image", + "audio", "video" ], - "training_tasks": null, + "training_tasks": [ + "AudioCapsT2ARetrieval", + "MSRVTTT2V", + "MSRVTTV2T" + ], "source_urls": [ "https://mteb-leaderboard-backend.hf.space/v1/benchmarks/MAEB%28beta%2C%20audio-only%29/scores", "https://mteb-leaderboard-backend.hf.space/v1/benchmarks/MAEB%28beta%29/scores" ], "size_basis": "reported_rounded_total", "kind": "peer", - "on_displayed_frontier": false + "on_displayed_frontier": false, + "evidence_type": "registry_reported", + "observation_id": "registry_reported:Haon-Chen/e5-omni-3B" }, { - "model": "Haon-Chen/e5-omni-3B", + "model": "LCO-Embedding/LCO-Embedding-Omni-3B", "parameters_billion": 4.703, - "score": 32.91909117647059, + "score": 45.062387254901964, "open_weights": true, "modalities": [ - "text", - "image", "audio", + "image", + "text", "video" ], - "training_tasks": [ - "AudioCapsT2ARetrieval", - "MSRVTTT2V", - "MSRVTTV2T" - ], + "training_tasks": null, "source_urls": [ "https://mteb-leaderboard-backend.hf.space/v1/benchmarks/MAEB%28beta%2C%20audio-only%29/scores", "https://mteb-leaderboard-backend.hf.space/v1/benchmarks/MAEB%28beta%29/scores" ], "size_basis": "reported_rounded_total", "kind": "peer", - "on_displayed_frontier": false + "on_displayed_frontier": false, + "evidence_type": "registry_reported", + "observation_id": "registry_reported:LCO-Embedding/LCO-Embedding-Omni-3B" }, { "model": "Qwen/Qwen2.5-Omni-3B", @@ -11936,7 +12212,9 @@ ], "size_basis": "reported_rounded_total", "kind": "peer", - "on_displayed_frontier": false + "on_displayed_frontier": false, + "evidence_type": "registry_reported", + "observation_id": "registry_reported:Qwen/Qwen2.5-Omni-3B" }, { "model": "Qwen/Qwen2-Audio-7B", @@ -11954,51 +12232,57 @@ ], "size_basis": "reported_rounded_total", "kind": "peer", - "on_displayed_frontier": false + "on_displayed_frontier": false, + "evidence_type": "registry_reported", + "observation_id": "registry_reported:Qwen/Qwen2-Audio-7B" }, { - "model": "LCO-Embedding/LCO-Embedding-Omni-7B", + "model": "Haon-Chen/e5-omni-7B", "parameters_billion": 8.932, - "score": 47.38897549019608, + "score": 38.39882352941177, "open_weights": true, "modalities": [ - "audio", - "image", "text", + "image", + "audio", "video" ], - "training_tasks": null, + "training_tasks": [ + "AudioCapsT2ARetrieval", + "MSRVTTT2V", + "MSRVTTV2T" + ], "source_urls": [ "https://mteb-leaderboard-backend.hf.space/v1/benchmarks/MAEB%28beta%2C%20audio-only%29/scores", "https://mteb-leaderboard-backend.hf.space/v1/benchmarks/MAEB%28beta%29/scores" ], "size_basis": "reported_rounded_total", "kind": "peer", - "on_displayed_frontier": false + "on_displayed_frontier": false, + "evidence_type": "registry_reported", + "observation_id": "registry_reported:Haon-Chen/e5-omni-7B" }, { - "model": "Haon-Chen/e5-omni-7B", + "model": "LCO-Embedding/LCO-Embedding-Omni-7B", "parameters_billion": 8.932, - "score": 38.39882352941177, + "score": 47.38897549019608, "open_weights": true, "modalities": [ - "text", - "image", "audio", + "image", + "text", "video" ], - "training_tasks": [ - "AudioCapsT2ARetrieval", - "MSRVTTT2V", - "MSRVTTV2T" - ], + "training_tasks": null, "source_urls": [ "https://mteb-leaderboard-backend.hf.space/v1/benchmarks/MAEB%28beta%2C%20audio-only%29/scores", "https://mteb-leaderboard-backend.hf.space/v1/benchmarks/MAEB%28beta%29/scores" ], "size_basis": "reported_rounded_total", "kind": "peer", - "on_displayed_frontier": false + "on_displayed_frontier": false, + "evidence_type": "registry_reported", + "observation_id": "registry_reported:LCO-Embedding/LCO-Embedding-Omni-7B" }, { "model": "Tevatron/OmniEmbed-v0.1", @@ -12030,7 +12314,9 @@ ], "size_basis": "reported_rounded_total", "kind": "peer", - "on_displayed_frontier": false + "on_displayed_frontier": false, + "evidence_type": "registry_reported", + "observation_id": "registry_reported:Tevatron/OmniEmbed-v0.1" }, { "model": "Qwen/Qwen2.5-Omni-7B", @@ -12050,7 +12336,9 @@ ], "size_basis": "reported_rounded_total", "kind": "peer", - "on_displayed_frontier": false + "on_displayed_frontier": false, + "evidence_type": "registry_reported", + "observation_id": "registry_reported:Qwen/Qwen2.5-Omni-7B" }, { "model": "Qwen/Qwen3-Omni-30B-A3B-Thinking", @@ -12070,7 +12358,9 @@ ], "size_basis": "reported_rounded_total", "kind": "peer", - "on_displayed_frontier": false + "on_displayed_frontier": false, + "evidence_type": "registry_reported", + "observation_id": "registry_reported:Qwen/Qwen3-Omni-30B-A3B-Thinking" }, { "model": "Qwen/Qwen3-Omni-30B-A3B-Instruct", @@ -12090,7 +12380,9 @@ ], "size_basis": "reported_rounded_total", "kind": "peer", - "on_displayed_frontier": false + "on_displayed_frontier": false, + "evidence_type": "registry_reported", + "observation_id": "registry_reported:Qwen/Qwen3-Omni-30B-A3B-Instruct" } ], "excluded": [], @@ -12103,7 +12395,18 @@ "llm-semantic-router/Vela-1.0-Omni-Mini", "facebook/seamless-m4t-v2-large", "BidirLM/BidirLM-Omni-2.5B-Embedding" - ] + ], + "frontier_observations": [ + "registry_reported:google/yamnet", + "registry_reported:asapp/sew-d-tiny-100k-ft-ls100h", + "registry_reported:matthewagi/HeAR-s1.1", + "registry_reported:jinaai/jina-embeddings-v5-omni-nano", + "llm-semantic-router/Vela-1.0-Omni-Mini", + "registry_reported:facebook/seamless-m4t-v2-large", + "registry_reported:BidirLM/BidirLM-Omni-2.5B-Embedding" + ], + "peer_observation_count": 67, + "measured_peer_observations": 0 }, "vehiclesoundclustering": { "task": "VehicleSoundClustering", @@ -12141,7 +12444,9 @@ ], "size_basis": "reported_rounded_total", "kind": "peer", - "on_displayed_frontier": true + "on_displayed_frontier": true, + "evidence_type": "registry_reported", + "observation_id": "registry_reported:google/yamnet" }, { "model": "asapp/sew-d-tiny-100k-ft-ls100h", @@ -12160,7 +12465,9 @@ ], "size_basis": "reported_rounded_total", "kind": "peer", - "on_displayed_frontier": false + "on_displayed_frontier": false, + "evidence_type": "registry_reported", + "observation_id": "registry_reported:asapp/sew-d-tiny-100k-ft-ls100h" }, { "model": "matthewagi/HeAR-s1.1", @@ -12177,7 +12484,9 @@ ], "size_basis": "reported_rounded_total", "kind": "peer", - "on_displayed_frontier": true + "on_displayed_frontier": true, + "evidence_type": "registry_reported", + "observation_id": "registry_reported:matthewagi/HeAR-s1.1" }, { "model": "facebook/encodec_24khz", @@ -12198,7 +12507,9 @@ ], "size_basis": "reported_rounded_total", "kind": "peer", - "on_displayed_frontier": false + "on_displayed_frontier": false, + "evidence_type": "registry_reported", + "observation_id": "registry_reported:facebook/encodec_24khz" }, { "model": "openai/whisper-tiny", @@ -12215,7 +12526,9 @@ ], "size_basis": "reported_rounded_total", "kind": "peer", - "on_displayed_frontier": true + "on_displayed_frontier": true, + "evidence_type": "registry_reported", + "observation_id": "registry_reported:openai/whisper-tiny" }, { "model": "google/vggish", @@ -12234,7 +12547,9 @@ ], "size_basis": "reported_rounded_total", "kind": "peer", - "on_displayed_frontier": false + "on_displayed_frontier": false, + "evidence_type": "registry_reported", + "observation_id": "registry_reported:google/vggish" }, { "model": "openai/whisper-base", @@ -12251,7 +12566,9 @@ ], "size_basis": "reported_rounded_total", "kind": "peer", - "on_displayed_frontier": false + "on_displayed_frontier": false, + "evidence_type": "registry_reported", + "observation_id": "registry_reported:openai/whisper-base" }, { "model": "speechbrain/cnn14-esc50", @@ -12270,7 +12587,32 @@ ], "size_basis": "reported_rounded_total", "kind": "peer", - "on_displayed_frontier": false + "on_displayed_frontier": false, + "evidence_type": "registry_reported", + "observation_id": "registry_reported:speechbrain/cnn14-esc50" + }, + { + "model": "MIT/ast-finetuned-audioset-10-10-0.4593", + "parameters_billion": 0.086594063, + "parameters_exact": 86594063, + "score": 14.342872113086985, + "open_weights": true, + "modalities": [ + "audio" + ], + "training_tasks": [], + "source_urls": [ + "https://huggingface.co/MIT/ast-finetuned-audioset-10-10-0.4593/tree/f826b80d28226b62986cc218e5cec390b1096902", + "https://github.com/YuanGongND/ast" + ], + "size_basis": "counted_pinned_full_checkpoint", + "kind": "peer", + "evidence_type": "independent_rerun", + "observation_id": "independent_rerun:MIT/ast-finetuned-audioset-10-10-0.4593:f826b80d28226b62986cc218e5cec390b1096902:VehicleSoundClustering", + "revision": "f826b80d28226b62986cc218e5cec390b1096902", + "report": "benchmarks/peer-evaluations/ast-vehicle.json", + "license": "bsd-3-clause", + "on_displayed_frontier": true }, { "model": "MIT/ast-finetuned-audioset-10-10-0.4593", @@ -12289,7 +12631,9 @@ ], "size_basis": "reported_rounded_total", "kind": "peer", - "on_displayed_frontier": true + "on_displayed_frontier": false, + "evidence_type": "registry_reported", + "observation_id": "registry_reported:MIT/ast-finetuned-audioset-10-10-0.4593" }, { "model": "facebook/data2vec-audio-base-960h", @@ -12306,7 +12650,9 @@ ], "size_basis": "reported_rounded_total", "kind": "peer", - "on_displayed_frontier": false + "on_displayed_frontier": false, + "evidence_type": "registry_reported", + "observation_id": "registry_reported:facebook/data2vec-audio-base-960h" }, { "model": "microsoft/unispeech-sat-base-100h-libri-ft", @@ -12325,81 +12671,98 @@ ], "size_basis": "reported_rounded_total", "kind": "peer", - "on_displayed_frontier": false + "on_displayed_frontier": false, + "evidence_type": "registry_reported", + "observation_id": "registry_reported:microsoft/unispeech-sat-base-100h-libri-ft" }, { - "model": "facebook/wav2vec2-base", + "model": "asapp/sew-d-base-plus-400k-ft-ls100h", "parameters_billion": 0.095, - "score": 3.0932999999999997, + "score": 0.6319, "open_weights": true, "modalities": [ "audio" ], - "training_tasks": null, + "training_tasks": [ + "LibriSpeech" + ], "source_urls": [ "https://mteb-leaderboard-backend.hf.space/v1/benchmarks/MAEB%28beta%2C%20audio-only%29/scores", "https://mteb-leaderboard-backend.hf.space/v1/benchmarks/MAEB%28beta%29/scores" ], "size_basis": "reported_rounded_total", "kind": "peer", - "on_displayed_frontier": false + "on_displayed_frontier": false, + "evidence_type": "registry_reported", + "observation_id": "registry_reported:asapp/sew-d-base-plus-400k-ft-ls100h" }, { - "model": "microsoft/wavlm-base-plus", + "model": "facebook/hubert-base-ls960", "parameters_billion": 0.095, - "score": 1.7932, + "score": 0.1305, "open_weights": true, "modalities": [ "audio" ], - "training_tasks": [ - "GigaSpeechA2TRetrieval", - "Libri-Light", - "VoxPopuliLanguageID" - ], + "training_tasks": null, "source_urls": [ "https://mteb-leaderboard-backend.hf.space/v1/benchmarks/MAEB%28beta%2C%20audio-only%29/scores", "https://mteb-leaderboard-backend.hf.space/v1/benchmarks/MAEB%28beta%29/scores" ], "size_basis": "reported_rounded_total", "kind": "peer", - "on_displayed_frontier": false + "on_displayed_frontier": false, + "evidence_type": "registry_reported", + "observation_id": "registry_reported:facebook/hubert-base-ls960" }, { - "model": "microsoft/wavlm-base-plus-sd", + "model": "facebook/wav2vec2-base", "parameters_billion": 0.095, - "score": 1.7932, + "score": 3.0932999999999997, "open_weights": true, "modalities": [ "audio" ], - "training_tasks": [ - "GigaSpeechA2TRetrieval", - "Libri-Light", - "LibriMix", - "VoxPopuliLanguageID" + "training_tasks": null, + "source_urls": [ + "https://mteb-leaderboard-backend.hf.space/v1/benchmarks/MAEB%28beta%2C%20audio-only%29/scores", + "https://mteb-leaderboard-backend.hf.space/v1/benchmarks/MAEB%28beta%29/scores" ], + "size_basis": "reported_rounded_total", + "kind": "peer", + "on_displayed_frontier": false, + "evidence_type": "registry_reported", + "observation_id": "registry_reported:facebook/wav2vec2-base" + }, + { + "model": "facebook/wav2vec2-base-960h", + "parameters_billion": 0.095, + "score": 0.6122, + "open_weights": true, + "modalities": [ + "audio" + ], + "training_tasks": null, "source_urls": [ "https://mteb-leaderboard-backend.hf.space/v1/benchmarks/MAEB%28beta%2C%20audio-only%29/scores", "https://mteb-leaderboard-backend.hf.space/v1/benchmarks/MAEB%28beta%29/scores" ], "size_basis": "reported_rounded_total", "kind": "peer", - "on_displayed_frontier": false + "on_displayed_frontier": false, + "evidence_type": "registry_reported", + "observation_id": "registry_reported:facebook/wav2vec2-base-960h" }, { - "model": "microsoft/wavlm-base-plus-sv", + "model": "microsoft/wavlm-base", "parameters_billion": 0.095, - "score": 1.7932, + "score": 0.40940000000000004, "open_weights": true, "modalities": [ "audio" ], "training_tasks": [ - "GigaSpeechA2TRetrieval", - "Libri-Light", - "VoxCeleb1", - "VoxPopuliLanguageID" + "Librispeech" ], "source_urls": [ "https://mteb-leaderboard-backend.hf.space/v1/benchmarks/MAEB%28beta%2C%20audio-only%29/scores", @@ -12407,18 +12770,22 @@ ], "size_basis": "reported_rounded_total", "kind": "peer", - "on_displayed_frontier": false + "on_displayed_frontier": false, + "evidence_type": "registry_reported", + "observation_id": "registry_reported:microsoft/wavlm-base" }, { - "model": "asapp/sew-d-base-plus-400k-ft-ls100h", + "model": "microsoft/wavlm-base-plus", "parameters_billion": 0.095, - "score": 0.6319, + "score": 1.7932, "open_weights": true, "modalities": [ "audio" ], "training_tasks": [ - "LibriSpeech" + "GigaSpeechA2TRetrieval", + "Libri-Light", + "VoxPopuliLanguageID" ], "source_urls": [ "https://mteb-leaderboard-backend.hf.space/v1/benchmarks/MAEB%28beta%2C%20audio-only%29/scores", @@ -12426,35 +12793,47 @@ ], "size_basis": "reported_rounded_total", "kind": "peer", - "on_displayed_frontier": false + "on_displayed_frontier": false, + "evidence_type": "registry_reported", + "observation_id": "registry_reported:microsoft/wavlm-base-plus" }, { - "model": "facebook/wav2vec2-base-960h", + "model": "microsoft/wavlm-base-plus-sd", "parameters_billion": 0.095, - "score": 0.6122, + "score": 1.7932, "open_weights": true, "modalities": [ "audio" ], - "training_tasks": null, + "training_tasks": [ + "GigaSpeechA2TRetrieval", + "Libri-Light", + "LibriMix", + "VoxPopuliLanguageID" + ], "source_urls": [ "https://mteb-leaderboard-backend.hf.space/v1/benchmarks/MAEB%28beta%2C%20audio-only%29/scores", "https://mteb-leaderboard-backend.hf.space/v1/benchmarks/MAEB%28beta%29/scores" ], "size_basis": "reported_rounded_total", "kind": "peer", - "on_displayed_frontier": false + "on_displayed_frontier": false, + "evidence_type": "registry_reported", + "observation_id": "registry_reported:microsoft/wavlm-base-plus-sd" }, { - "model": "microsoft/wavlm-base", + "model": "microsoft/wavlm-base-plus-sv", "parameters_billion": 0.095, - "score": 0.40940000000000004, + "score": 1.7932, "open_weights": true, "modalities": [ "audio" ], "training_tasks": [ - "Librispeech" + "GigaSpeechA2TRetrieval", + "Libri-Light", + "VoxCeleb1", + "VoxPopuliLanguageID" ], "source_urls": [ "https://mteb-leaderboard-backend.hf.space/v1/benchmarks/MAEB%28beta%2C%20audio-only%29/scores", @@ -12462,7 +12841,9 @@ ], "size_basis": "reported_rounded_total", "kind": "peer", - "on_displayed_frontier": false + "on_displayed_frontier": false, + "evidence_type": "registry_reported", + "observation_id": "registry_reported:microsoft/wavlm-base-plus-sv" }, { "model": "microsoft/wavlm-base-sd", @@ -12482,7 +12863,9 @@ ], "size_basis": "reported_rounded_total", "kind": "peer", - "on_displayed_frontier": false + "on_displayed_frontier": false, + "evidence_type": "registry_reported", + "observation_id": "registry_reported:microsoft/wavlm-base-sd" }, { "model": "microsoft/wavlm-base-sv", @@ -12502,30 +12885,35 @@ ], "size_basis": "reported_rounded_total", "kind": "peer", - "on_displayed_frontier": false + "on_displayed_frontier": false, + "evidence_type": "registry_reported", + "observation_id": "registry_reported:microsoft/wavlm-base-sv" }, { - "model": "facebook/hubert-base-ls960", - "parameters_billion": 0.095, - "score": 0.1305, + "model": "asapp/sew-d-mid-400k-ft-ls100h", + "parameters_billion": 0.139, + "score": 0.6276, "open_weights": true, "modalities": [ "audio" ], - "training_tasks": null, + "training_tasks": [ + "LibriSpeech" + ], "source_urls": [ "https://mteb-leaderboard-backend.hf.space/v1/benchmarks/MAEB%28beta%2C%20audio-only%29/scores", "https://mteb-leaderboard-backend.hf.space/v1/benchmarks/MAEB%28beta%29/scores" ], "size_basis": "reported_rounded_total", "kind": "peer", - "on_displayed_frontier": false + "on_displayed_frontier": false, + "evidence_type": "registry_reported", + "observation_id": "registry_reported:asapp/sew-d-mid-400k-ft-ls100h" }, { "model": "llm-semantic-router/Vela-1.0-Omni-Nano", - "parameters_billion": 0.13395648, - "parameters_exact": 133956480, - "native_fingerprint": "fb9cd39c9980a085b75456c715c2295b997784fe733314be68c13635f6384edf", + "parameters_billion": 0.144554112, + "parameters_exact": 144554112, "score": 13.800701178563807, "open_weights": true, "modalities": [ @@ -12536,31 +12924,15 @@ "size_basis": "counted_all_native_model_parameters", "kind": "nano", "evaluated_task_revision": "5f905f9544d0c78b607639662c88c2ddd1216384", - "report_url": "https://huggingface.co/llm-semantic-router/Vela-1.0-Omni-Nano/blob/ed7263b501430a011d37b147d6e264ad8354983b/benchmarks/maeb-audio-only.json", - "on_displayed_frontier": true, - "evaluated_native_fingerprint": "b596f1dbab89369555e0a72a0d6854d780e305686750949b308c8f167993bdeb", - "evaluation_applicability": "Unchanged exact text/audio computation; see the current Nano component-equivalence report.", - "component_equivalence_url": "https://huggingface.co/llm-semantic-router/Vela-1.0-Omni-Nano/blob/main/benchmarks/component-equivalence.json", - "evaluated_revision": "70c3412bb0ddfb19386d673d4a102c753a574f1b" - }, - { - "model": "asapp/sew-d-mid-400k-ft-ls100h", - "parameters_billion": 0.139, - "score": 0.6276, - "open_weights": true, - "modalities": [ - "audio" - ], - "training_tasks": [ - "LibriSpeech" - ], - "source_urls": [ - "https://mteb-leaderboard-backend.hf.space/v1/benchmarks/MAEB%28beta%2C%20audio-only%29/scores", - "https://mteb-leaderboard-backend.hf.space/v1/benchmarks/MAEB%28beta%29/scores" - ], - "size_basis": "reported_rounded_total", - "kind": "peer", - "on_displayed_frontier": false + "on_displayed_frontier": false, + "native_artifact_sha256": "96d76f97478e114b816df3ab1b3c6947e4c5ef0babfb40ab865d7e3c838d34c4", + "revision": "self", + "evaluated_native_artifact_sha256": "96d76f97478e114b816df3ab1b3c6947e4c5ef0babfb40ab865d7e3c838d34c4", + "evaluated_revision": "96d76f97478e114b816df3ab1b3c6947e4c5ef0babfb40ab865d7e3c838d34c4", + "evaluated_model": "llm-semantic-router/Vela-1.0-Omni-Nano", + "revision_semantics": "Evaluated revisions are native artifact fingerprints; applies-to identity is the current whole artifact.", + "evidence_type": "native_evaluation", + "report": "benchmarks/maeb-audio-only.json" }, { "model": "laion/clap-htsat-unfused", @@ -12578,7 +12950,9 @@ ], "size_basis": "reported_rounded_total", "kind": "peer", - "on_displayed_frontier": false + "on_displayed_frontier": false, + "evidence_type": "registry_reported", + "observation_id": "registry_reported:laion/clap-htsat-unfused" }, { "model": "laion/clap-htsat-fused", @@ -12596,7 +12970,9 @@ ], "size_basis": "reported_rounded_total", "kind": "peer", - "on_displayed_frontier": false + "on_displayed_frontier": false, + "evidence_type": "registry_reported", + "observation_id": "registry_reported:laion/clap-htsat-fused" }, { "model": "microsoft/msclap-2023", @@ -12618,7 +12994,9 @@ ], "size_basis": "reported_rounded_total", "kind": "peer", - "on_displayed_frontier": false + "on_displayed_frontier": false, + "evidence_type": "registry_reported", + "observation_id": "registry_reported:microsoft/msclap-2023" }, { "model": "lyrebird/wav2clip", @@ -12636,12 +13014,14 @@ ], "size_basis": "reported_rounded_total", "kind": "peer", - "on_displayed_frontier": false + "on_displayed_frontier": false, + "evidence_type": "registry_reported", + "observation_id": "registry_reported:lyrebird/wav2clip" }, { - "model": "laion/larger_clap_music_and_speech", + "model": "laion/larger_clap_general", "parameters_billion": 0.194, - "score": 4.7243, + "score": 3.3663, "open_weights": true, "modalities": [ "audio", @@ -12654,7 +13034,9 @@ ], "size_basis": "reported_rounded_total", "kind": "peer", - "on_displayed_frontier": false + "on_displayed_frontier": false, + "evidence_type": "registry_reported", + "observation_id": "registry_reported:laion/larger_clap_general" }, { "model": "laion/larger_clap_music", @@ -12672,12 +13054,14 @@ ], "size_basis": "reported_rounded_total", "kind": "peer", - "on_displayed_frontier": false + "on_displayed_frontier": false, + "evidence_type": "registry_reported", + "observation_id": "registry_reported:laion/larger_clap_music" }, { - "model": "laion/larger_clap_general", + "model": "laion/larger_clap_music_and_speech", "parameters_billion": 0.194, - "score": 3.3663, + "score": 4.7243, "open_weights": true, "modalities": [ "audio", @@ -12690,7 +13074,9 @@ ], "size_basis": "reported_rounded_total", "kind": "peer", - "on_displayed_frontier": false + "on_displayed_frontier": false, + "evidence_type": "registry_reported", + "observation_id": "registry_reported:laion/larger_clap_music_and_speech" }, { "model": "microsoft/msclap-2022", @@ -12716,7 +13102,9 @@ ], "size_basis": "reported_rounded_total", "kind": "peer", - "on_displayed_frontier": false + "on_displayed_frontier": false, + "evidence_type": "registry_reported", + "observation_id": "registry_reported:microsoft/msclap-2022" }, { "model": "openai/whisper-small", @@ -12733,7 +13121,9 @@ ], "size_basis": "reported_rounded_total", "kind": "peer", - "on_displayed_frontier": false + "on_displayed_frontier": false, + "evidence_type": "registry_reported", + "observation_id": "registry_reported:openai/whisper-small" }, { "model": "microsoft/speecht5_multimodal", @@ -12751,7 +13141,9 @@ ], "size_basis": "reported_rounded_total", "kind": "peer", - "on_displayed_frontier": false + "on_displayed_frontier": false, + "evidence_type": "registry_reported", + "observation_id": "registry_reported:microsoft/speecht5_multimodal" }, { "model": "facebook/wav2vec2-xls-r-300m", @@ -12772,7 +13164,9 @@ ], "size_basis": "reported_rounded_total", "kind": "peer", - "on_displayed_frontier": false + "on_displayed_frontier": false, + "evidence_type": "registry_reported", + "observation_id": "registry_reported:facebook/wav2vec2-xls-r-300m" }, { "model": "vitouphy/wav2vec2-xls-r-300m-phoneme", @@ -12793,7 +13187,9 @@ ], "size_basis": "reported_rounded_total", "kind": "peer", - "on_displayed_frontier": false + "on_displayed_frontier": false, + "evidence_type": "registry_reported", + "observation_id": "registry_reported:vitouphy/wav2vec2-xls-r-300m-phoneme" }, { "model": "facebook/data2vec-audio-large-960h", @@ -12810,79 +13206,89 @@ ], "size_basis": "reported_rounded_total", "kind": "peer", - "on_displayed_frontier": false + "on_displayed_frontier": false, + "evidence_type": "registry_reported", + "observation_id": "registry_reported:facebook/data2vec-audio-large-960h" }, { - "model": "facebook/wav2vec2-lv-60-espeak-cv-ft", + "model": "facebook/hubert-large-ls960-ft", "parameters_billion": 0.317, - "score": 2.3741000000000003, + "score": 0.5867, "open_weights": true, "modalities": [ "audio" ], - "training_tasks": [ - "CommonVoiceMini17A2TRetrieval" - ], + "training_tasks": null, "source_urls": [ "https://mteb-leaderboard-backend.hf.space/v1/benchmarks/MAEB%28beta%2C%20audio-only%29/scores", "https://mteb-leaderboard-backend.hf.space/v1/benchmarks/MAEB%28beta%29/scores" ], "size_basis": "reported_rounded_total", "kind": "peer", - "on_displayed_frontier": false + "on_displayed_frontier": false, + "evidence_type": "registry_reported", + "observation_id": "registry_reported:facebook/hubert-large-ls960-ft" }, { - "model": "facebook/wav2vec2-large-xlsr-53", + "model": "facebook/wav2vec2-large", "parameters_billion": 0.317, - "score": 1.7241, + "score": 1.411, "open_weights": true, "modalities": [ "audio" ], - "training_tasks": [ - "CommonVoiceMini17A2TRetrieval" - ], + "training_tasks": null, "source_urls": [ "https://mteb-leaderboard-backend.hf.space/v1/benchmarks/MAEB%28beta%2C%20audio-only%29/scores", "https://mteb-leaderboard-backend.hf.space/v1/benchmarks/MAEB%28beta%29/scores" ], "size_basis": "reported_rounded_total", "kind": "peer", - "on_displayed_frontier": false + "on_displayed_frontier": false, + "evidence_type": "registry_reported", + "observation_id": "registry_reported:facebook/wav2vec2-large" }, { - "model": "facebook/wav2vec2-large", + "model": "facebook/wav2vec2-large-xlsr-53", "parameters_billion": 0.317, - "score": 1.411, + "score": 1.7241, "open_weights": true, "modalities": [ "audio" ], - "training_tasks": null, + "training_tasks": [ + "CommonVoiceMini17A2TRetrieval" + ], "source_urls": [ "https://mteb-leaderboard-backend.hf.space/v1/benchmarks/MAEB%28beta%2C%20audio-only%29/scores", "https://mteb-leaderboard-backend.hf.space/v1/benchmarks/MAEB%28beta%29/scores" ], "size_basis": "reported_rounded_total", "kind": "peer", - "on_displayed_frontier": false + "on_displayed_frontier": false, + "evidence_type": "registry_reported", + "observation_id": "registry_reported:facebook/wav2vec2-large-xlsr-53" }, { - "model": "facebook/hubert-large-ls960-ft", + "model": "facebook/wav2vec2-lv-60-espeak-cv-ft", "parameters_billion": 0.317, - "score": 0.5867, + "score": 2.3741000000000003, "open_weights": true, "modalities": [ "audio" ], - "training_tasks": null, + "training_tasks": [ + "CommonVoiceMini17A2TRetrieval" + ], "source_urls": [ "https://mteb-leaderboard-backend.hf.space/v1/benchmarks/MAEB%28beta%2C%20audio-only%29/scores", "https://mteb-leaderboard-backend.hf.space/v1/benchmarks/MAEB%28beta%29/scores" ], "size_basis": "reported_rounded_total", "kind": "peer", - "on_displayed_frontier": false + "on_displayed_frontier": false, + "evidence_type": "registry_reported", + "observation_id": "registry_reported:facebook/wav2vec2-lv-60-espeak-cv-ft" }, { "model": "microsoft/wavlm-large", @@ -12903,7 +13309,9 @@ ], "size_basis": "reported_rounded_total", "kind": "peer", - "on_displayed_frontier": false + "on_displayed_frontier": false, + "evidence_type": "registry_reported", + "observation_id": "registry_reported:microsoft/wavlm-large" }, { "model": "OpenMuQ/MuQ-MuLan-large", @@ -12921,7 +13329,9 @@ ], "size_basis": "reported_rounded_total", "kind": "peer", - "on_displayed_frontier": false + "on_displayed_frontier": false, + "evidence_type": "registry_reported", + "observation_id": "registry_reported:OpenMuQ/MuQ-MuLan-large" }, { "model": "openai/whisper-medium", @@ -12938,7 +13348,9 @@ ], "size_basis": "reported_rounded_total", "kind": "peer", - "on_displayed_frontier": false + "on_displayed_frontier": false, + "evidence_type": "registry_reported", + "observation_id": "registry_reported:openai/whisper-medium" }, { "model": "jinaai/jina-embeddings-v5-omni-nano", @@ -12958,12 +13370,14 @@ ], "size_basis": "reported_rounded_total", "kind": "peer", - "on_displayed_frontier": false + "on_displayed_frontier": false, + "evidence_type": "registry_reported", + "observation_id": "registry_reported:jinaai/jina-embeddings-v5-omni-nano" }, { - "model": "facebook/mms-1b-l1107", + "model": "facebook/mms-1b-all", "parameters_billion": 1.0, - "score": 0.4373, + "score": 0.32030000000000003, "open_weights": true, "modalities": [ "audio" @@ -12977,12 +13391,14 @@ ], "size_basis": "reported_rounded_total", "kind": "peer", - "on_displayed_frontier": false + "on_displayed_frontier": false, + "evidence_type": "registry_reported", + "observation_id": "registry_reported:facebook/mms-1b-all" }, { - "model": "facebook/mms-1b-all", + "model": "facebook/mms-1b-fl102", "parameters_billion": 1.0, - "score": 0.32030000000000003, + "score": 0.146, "open_weights": true, "modalities": [ "audio" @@ -12996,12 +13412,14 @@ ], "size_basis": "reported_rounded_total", "kind": "peer", - "on_displayed_frontier": false + "on_displayed_frontier": false, + "evidence_type": "registry_reported", + "observation_id": "registry_reported:facebook/mms-1b-fl102" }, { - "model": "facebook/mms-1b-fl102", + "model": "facebook/mms-1b-l1107", "parameters_billion": 1.0, - "score": 0.146, + "score": 0.4373, "open_weights": true, "modalities": [ "audio" @@ -13015,7 +13433,9 @@ ], "size_basis": "reported_rounded_total", "kind": "peer", - "on_displayed_frontier": false + "on_displayed_frontier": false, + "evidence_type": "registry_reported", + "observation_id": "registry_reported:facebook/mms-1b-l1107" }, { "model": "facebook/wav2vec2-xls-r-1b", @@ -13036,25 +13456,9 @@ ], "size_basis": "reported_rounded_total", "kind": "peer", - "on_displayed_frontier": false - }, - { - "model": "llm-semantic-router/Vela-1.0-Omni-Mini", - "parameters_billion": 1.04415264, - "parameters_exact": 1044152640, - "native_fingerprint": "035f6084c6c984f465e4f91c50caf8f13790177d4c0c29e4ce5d6c0c233fd1a0", - "score": 8.718116828237402, - "open_weights": true, - "modalities": [ - "text", - "image", - "audio" - ], - "size_basis": "counted_all_native_model_parameters", - "kind": "mini", - "evaluated_task_revision": "5f905f9544d0c78b607639662c88c2ddd1216384", - "report_url": "https://huggingface.co/llm-semantic-router/Vela-1.0-Omni-Mini/blob/662d4e53c52d22da6430b0ac316360d41028374d/benchmarks/maeb-audio-only.json", - "on_displayed_frontier": false + "on_displayed_frontier": false, + "evidence_type": "registry_reported", + "observation_id": "registry_reported:facebook/wav2vec2-xls-r-1b" }, { "model": "speechbrain/m-ctc-t-large", @@ -13074,7 +13478,9 @@ ], "size_basis": "reported_rounded_total", "kind": "peer", - "on_displayed_frontier": false + "on_displayed_frontier": false, + "evidence_type": "registry_reported", + "observation_id": "registry_reported:speechbrain/m-ctc-t-large" }, { "model": "LanguageBind/LanguageBind_Omni", @@ -13094,7 +13500,30 @@ ], "size_basis": "reported_rounded_total", "kind": "peer", - "on_displayed_frontier": false + "on_displayed_frontier": false, + "evidence_type": "registry_reported", + "observation_id": "registry_reported:LanguageBind/LanguageBind_Omni" + }, + { + "model": "llm-semantic-router/Vela-1.0-Omni-Mini", + "parameters_billion": 1.3328912, + "parameters_exact": 1332891200, + "score": 9.001091019154014, + "open_weights": true, + "modalities": [ + "text", + "image", + "audio" + ], + "size_basis": "counted_all_native_model_parameters", + "kind": "mini", + "evaluated_task_revision": "5f905f9544d0c78b607639662c88c2ddd1216384", + "on_displayed_frontier": false, + "native_artifact_sha256": "d93ebb5f2f4951d52423ae13da71b037ce44d02957671e1e71bfe25623f3cff2", + "revision": "self", + "evaluated_native_artifact_sha256": "cf5f45cf6e26cd225e0fc0c1ece629b13b8bd0dfe57dcedef739ca8f529f80ab", + "evidence_type": "exact_metadata_only_applicability", + "report": "https://huggingface.co/llm-semantic-router/Vela-1.0-Omni-Mini/blob/main/benchmarks/maeb-audio-only.json" }, { "model": "openai/whisper-large-v3", @@ -13111,7 +13540,9 @@ ], "size_basis": "reported_rounded_total", "kind": "peer", - "on_displayed_frontier": false + "on_displayed_frontier": false, + "evidence_type": "registry_reported", + "observation_id": "registry_reported:openai/whisper-large-v3" }, { "model": "jinaai/jina-embeddings-v5-omni-small", @@ -13131,7 +13562,9 @@ ], "size_basis": "reported_rounded_total", "kind": "peer", - "on_displayed_frontier": false + "on_displayed_frontier": false, + "evidence_type": "registry_reported", + "observation_id": "registry_reported:jinaai/jina-embeddings-v5-omni-small" }, { "model": "encord-team/ebind-full", @@ -13184,7 +13617,9 @@ ], "size_basis": "reported_rounded_total", "kind": "peer", - "on_displayed_frontier": false + "on_displayed_frontier": false, + "evidence_type": "registry_reported", + "observation_id": "registry_reported:encord-team/ebind-full" }, { "model": "facebook/wav2vec2-xls-r-2b", @@ -13205,7 +13640,9 @@ ], "size_basis": "reported_rounded_total", "kind": "peer", - "on_displayed_frontier": false + "on_displayed_frontier": false, + "evidence_type": "registry_reported", + "observation_id": "registry_reported:facebook/wav2vec2-xls-r-2b" }, { "model": "facebook/seamless-m4t-v2-large", @@ -13224,7 +13661,9 @@ ], "size_basis": "reported_rounded_total", "kind": "peer", - "on_displayed_frontier": false + "on_displayed_frontier": false, + "evidence_type": "registry_reported", + "observation_id": "registry_reported:facebook/seamless-m4t-v2-large" }, { "model": "BidirLM/BidirLM-Omni-2.5B-Embedding", @@ -13320,7 +13759,9 @@ ], "size_basis": "reported_rounded_total", "kind": "peer", - "on_displayed_frontier": false + "on_displayed_frontier": false, + "evidence_type": "registry_reported", + "observation_id": "registry_reported:BidirLM/BidirLM-Omni-2.5B-Embedding" }, { "model": "EximiusLabs/fusion-embedding-1-2b-preview", @@ -13357,7 +13798,9 @@ ], "size_basis": "reported_rounded_total", "kind": "peer", - "on_displayed_frontier": false + "on_displayed_frontier": false, + "evidence_type": "registry_reported", + "observation_id": "registry_reported:EximiusLabs/fusion-embedding-1-2b-preview" }, { "model": "EximiusLabs/fusion-embedding-2-2b-preview", @@ -13394,7 +13837,9 @@ ], "size_basis": "reported_rounded_total", "kind": "peer", - "on_displayed_frontier": false + "on_displayed_frontier": false, + "evidence_type": "registry_reported", + "observation_id": "registry_reported:EximiusLabs/fusion-embedding-2-2b-preview" }, { "model": "Haon-Chen/e5-omni-3B", @@ -13418,7 +13863,9 @@ ], "size_basis": "reported_rounded_total", "kind": "peer", - "on_displayed_frontier": false + "on_displayed_frontier": false, + "evidence_type": "registry_reported", + "observation_id": "registry_reported:Haon-Chen/e5-omni-3B" }, { "model": "LCO-Embedding/LCO-Embedding-Omni-3B", @@ -13438,7 +13885,9 @@ ], "size_basis": "reported_rounded_total", "kind": "peer", - "on_displayed_frontier": false + "on_displayed_frontier": false, + "evidence_type": "registry_reported", + "observation_id": "registry_reported:LCO-Embedding/LCO-Embedding-Omni-3B" }, { "model": "Qwen/Qwen2.5-Omni-3B", @@ -13458,7 +13907,9 @@ ], "size_basis": "reported_rounded_total", "kind": "peer", - "on_displayed_frontier": false + "on_displayed_frontier": false, + "evidence_type": "registry_reported", + "observation_id": "registry_reported:Qwen/Qwen2.5-Omni-3B" }, { "model": "Qwen/Qwen2-Audio-7B", @@ -13476,51 +13927,57 @@ ], "size_basis": "reported_rounded_total", "kind": "peer", - "on_displayed_frontier": false + "on_displayed_frontier": false, + "evidence_type": "registry_reported", + "observation_id": "registry_reported:Qwen/Qwen2-Audio-7B" }, { - "model": "LCO-Embedding/LCO-Embedding-Omni-7B", + "model": "Haon-Chen/e5-omni-7B", "parameters_billion": 8.932, - "score": 3.6289000000000002, + "score": 2.4320999999999997, "open_weights": true, "modalities": [ - "audio", - "image", "text", + "image", + "audio", "video" ], - "training_tasks": null, + "training_tasks": [ + "AudioCapsT2ARetrieval", + "MSRVTTT2V", + "MSRVTTV2T" + ], "source_urls": [ "https://mteb-leaderboard-backend.hf.space/v1/benchmarks/MAEB%28beta%2C%20audio-only%29/scores", "https://mteb-leaderboard-backend.hf.space/v1/benchmarks/MAEB%28beta%29/scores" ], "size_basis": "reported_rounded_total", "kind": "peer", - "on_displayed_frontier": false + "on_displayed_frontier": false, + "evidence_type": "registry_reported", + "observation_id": "registry_reported:Haon-Chen/e5-omni-7B" }, { - "model": "Haon-Chen/e5-omni-7B", + "model": "LCO-Embedding/LCO-Embedding-Omni-7B", "parameters_billion": 8.932, - "score": 2.4320999999999997, + "score": 3.6289000000000002, "open_weights": true, "modalities": [ - "text", - "image", "audio", + "image", + "text", "video" ], - "training_tasks": [ - "AudioCapsT2ARetrieval", - "MSRVTTT2V", - "MSRVTTV2T" - ], + "training_tasks": null, "source_urls": [ "https://mteb-leaderboard-backend.hf.space/v1/benchmarks/MAEB%28beta%2C%20audio-only%29/scores", "https://mteb-leaderboard-backend.hf.space/v1/benchmarks/MAEB%28beta%29/scores" ], "size_basis": "reported_rounded_total", "kind": "peer", - "on_displayed_frontier": false + "on_displayed_frontier": false, + "evidence_type": "registry_reported", + "observation_id": "registry_reported:LCO-Embedding/LCO-Embedding-Omni-7B" }, { "model": "Tevatron/OmniEmbed-v0.1", @@ -13552,7 +14009,9 @@ ], "size_basis": "reported_rounded_total", "kind": "peer", - "on_displayed_frontier": false + "on_displayed_frontier": false, + "evidence_type": "registry_reported", + "observation_id": "registry_reported:Tevatron/OmniEmbed-v0.1" }, { "model": "Qwen/Qwen2.5-Omni-7B", @@ -13572,7 +14031,9 @@ ], "size_basis": "reported_rounded_total", "kind": "peer", - "on_displayed_frontier": false + "on_displayed_frontier": false, + "evidence_type": "registry_reported", + "observation_id": "registry_reported:Qwen/Qwen2.5-Omni-7B" }, { "model": "Qwen/Qwen3-Omni-30B-A3B-Thinking", @@ -13592,7 +14053,9 @@ ], "size_basis": "reported_rounded_total", "kind": "peer", - "on_displayed_frontier": false + "on_displayed_frontier": false, + "evidence_type": "registry_reported", + "observation_id": "registry_reported:Qwen/Qwen3-Omni-30B-A3B-Thinking" }, { "model": "Qwen/Qwen3-Omni-30B-A3B-Instruct", @@ -13612,7 +14075,9 @@ ], "size_basis": "reported_rounded_total", "kind": "peer", - "on_displayed_frontier": false + "on_displayed_frontier": false, + "evidence_type": "registry_reported", + "observation_id": "registry_reported:Qwen/Qwen3-Omni-30B-A3B-Instruct" } ], "excluded": [], @@ -13621,9 +14086,16 @@ "google/yamnet", "matthewagi/HeAR-s1.1", "openai/whisper-tiny", - "MIT/ast-finetuned-audioset-10-10-0.4593", - "llm-semantic-router/Vela-1.0-Omni-Nano" - ] + "MIT/ast-finetuned-audioset-10-10-0.4593" + ], + "frontier_observations": [ + "registry_reported:google/yamnet", + "registry_reported:matthewagi/HeAR-s1.1", + "registry_reported:openai/whisper-tiny", + "independent_rerun:MIT/ast-finetuned-audioset-10-10-0.4593:f826b80d28226b62986cc218e5cec390b1096902:VehicleSoundClustering" + ], + "peer_observation_count": 68, + "measured_peer_observations": 1 } } }