| { |
| "schema_version": 1, |
| "evaluation_type": "independent_single_task_rerun", |
| "evaluation_date_utc": "2026-09-18", |
| "model": { |
| "id": "codefuse-ai/F2LLM-v2-80M", |
| "revision": "6faa7738f343cd4321f9b2fa92fdb7cae3c117a5", |
| "parameters_exact": 80084288, |
| "embedding_dimensions": 320, |
| "license": "apache-2.0", |
| "open_weights": true |
| }, |
| "task": { |
| "name": "ArXivHierarchicalClusteringP2P", |
| "dataset": { |
| "id": "mteb/arxiv-clustering-p2p", |
| "revision": "0bbdb47bcbe3a90093699aefeed338a0f28a7ee8" |
| }, |
| "split": "test", |
| "subset": "default", |
| "primary_metric": "v_measure" |
| }, |
| "protocol": { |
| "framework": "mteb", |
| "framework_version": "2.21.0", |
| "library_versions": { |
| "datasets": "5.0.1", |
| "mteb": "2.21.0", |
| "numpy": "2.5.3", |
| "scikit-learn": "1.9.1", |
| "scipy": "1.18.1", |
| "sentence-transformers": "5.1.2", |
| "torch": "2.12.0+git6bbd260", |
| "torchaudio": "2.11.0+34c52a6", |
| "transformers": "4.57.6" |
| }, |
| "seed": 20260918, |
| "batch_size": 8, |
| "dtype": "bfloat16", |
| "effective_max_tokens": 8192, |
| "registry_metadata_max_tokens": 40960, |
| "pooling": "last token", |
| "normalization": "released SentenceTransformer Normalize module", |
| "wrapper": "official InstructSentenceTransformerModel", |
| "instruction_template": "Instruct: {instruction}\nQuery: ", |
| "instruction": "Identify the main and secondary category of arXiv papers based on the titles and abstracts.", |
| "add_eos_token": true, |
| "eos_token_id": 151645, |
| "official_task_default": { |
| "loaded_rows": 2048, |
| "fraction_to_embed": 0.04, |
| "encoded_rows": 81, |
| "sampling": "int(0.04 * 2048) items without replacement using the fixed task seed", |
| "levels": 2, |
| "repeats_per_level": 10, |
| "bootstrap_samples_per_repeat": 16384, |
| "bootstrap_sampling": "with replacement", |
| "mean": "unweighted mean over both levels and all 20 repeat scores", |
| "standard_deviation": "population standard deviation over the 20 scores; not a confidence interval", |
| "custom_subset_or_cap": false |
| }, |
| "source_sha256": { |
| "tasks/clustering/eng/arxiv_hierarchical_clustering.py": "086728ca5b5005fa780f145a307e2296fe6e652d8842be4f3f27a4125035d53d", |
| "abstasks/clustering.py": "45ccc13aa81c654f6b474dc2f21c3d0b64197c7c52b3b0ba6b55be5e528e54d2", |
| "models/model_implementations/codefuse_models.py": "26ae6ac29a3dc240fc380891ca854a6e20a3c7523b85823eba50d22febca72f4", |
| "models/instruct_wrapper.py": "e0cb5f3efab77bb89c36cc721770fbb27da63b8697481fa65f8e1740589c89b3" |
| }, |
| "default_sampling_source_locations": { |
| "task": "tasks/clustering/eng/arxiv_hierarchical_clustering.py:8,54-56", |
| "fraction": "abstasks/clustering.py:142,190-201", |
| "repeats": "abstasks/clustering.py:144-145", |
| "aggregation": "abstasks/clustering.py:276-281" |
| } |
| }, |
| "metrics": { |
| "ami": 0.6953016049891712, |
| "ami_scores": { |
| "Level 0": [ |
| 0.6831555875334137, |
| 0.7222915896820818, |
| 0.7183013786311202, |
| 0.7627815478632347, |
| 0.7539684121829449, |
| 0.7315028862494258, |
| 0.7222103595052074, |
| 0.7118381785863529, |
| 0.7472636858057605, |
| 0.7094010027784401 |
| ], |
| "Level 1": [ |
| 0.652900716351927, |
| 0.6599478757879051, |
| 0.6613629641931645, |
| 0.674683707322893, |
| 0.6642474245396333, |
| 0.6720844845211352, |
| 0.664456760985211, |
| 0.6624955584490995, |
| 0.6696243183843849, |
| 0.6615136604300891 |
| ] |
| }, |
| "ami_std": 0.03504559261222664, |
| "hf_subset": "default", |
| "languages": [ |
| "eng-Latn" |
| ], |
| "main_score": 0.6974221347455739, |
| "mteb_version": "2.21.0", |
| "v_measure": 0.6974221347455739, |
| "v_measure_std": 0.03361843856879675, |
| "v_measures": { |
| "Level 0": [ |
| 0.6837462774278753, |
| 0.722806855987181, |
| 0.7188229829616855, |
| 0.7632394098085972, |
| 0.754442001052228, |
| 0.7320100728371732, |
| 0.7227272056405429, |
| 0.7123811902025591, |
| 0.747744162008908, |
| 0.7099434357797132 |
| ], |
| "Level 1": [ |
| 0.6567146032108515, |
| 0.6637359495601902, |
| 0.6651447924493736, |
| 0.6783624916304802, |
| 0.667942572233544, |
| 0.6757226434679056, |
| 0.6681716729668471, |
| 0.6662375645735157, |
| 0.6732874264436501, |
| 0.6652593846686559 |
| ] |
| } |
| }, |
| "raw_official_task_result": { |
| "dataset_revision": "0bbdb47bcbe3a90093699aefeed338a0f28a7ee8", |
| "date": "2026-09-18T08:20:25.283739Z", |
| "evaluation_phases": [ |
| { |
| "end": 0.14332578200264834, |
| "name": "Encoding", |
| "split": "test", |
| "start": 3.690016455948353e-06, |
| "subset": "default" |
| }, |
| { |
| "end": 46.836432801996125, |
| "name": "Scoring", |
| "split": "test", |
| "start": 0.14468995700008236, |
| "subset": "default" |
| } |
| ], |
| "evaluation_time": 46.88400979299331, |
| "kg_co2_emissions": null, |
| "mteb_version": "2.21.0", |
| "scores": { |
| "test": [ |
| { |
| "ami": 0.6953016049891712, |
| "ami_scores": { |
| "Level 0": [ |
| 0.6831555875334137, |
| 0.7222915896820818, |
| 0.7183013786311202, |
| 0.7627815478632347, |
| 0.7539684121829449, |
| 0.7315028862494258, |
| 0.7222103595052074, |
| 0.7118381785863529, |
| 0.7472636858057605, |
| 0.7094010027784401 |
| ], |
| "Level 1": [ |
| 0.652900716351927, |
| 0.6599478757879051, |
| 0.6613629641931645, |
| 0.674683707322893, |
| 0.6642474245396333, |
| 0.6720844845211352, |
| 0.664456760985211, |
| 0.6624955584490995, |
| 0.6696243183843849, |
| 0.6615136604300891 |
| ] |
| }, |
| "ami_std": 0.03504559261222664, |
| "hf_subset": "default", |
| "languages": [ |
| "eng-Latn" |
| ], |
| "main_score": 0.6974221347455739, |
| "mteb_version": "2.21.0", |
| "v_measure": 0.6974221347455739, |
| "v_measure_std": 0.03361843856879675, |
| "v_measures": { |
| "Level 0": [ |
| 0.6837462774278753, |
| 0.722806855987181, |
| 0.7188229829616855, |
| 0.7632394098085972, |
| 0.754442001052228, |
| 0.7320100728371732, |
| 0.7227272056405429, |
| 0.7123811902025591, |
| 0.747744162008908, |
| 0.7099434357797132 |
| ], |
| "Level 1": [ |
| 0.6567146032108515, |
| 0.6637359495601902, |
| 0.6651447924493736, |
| 0.6783624916304802, |
| 0.667942572233544, |
| 0.6757226434679056, |
| 0.6681716729668471, |
| 0.6662375645735157, |
| 0.6732874264436501, |
| 0.6652593846686559 |
| ] |
| } |
| } |
| ] |
| }, |
| "task_name": "ArXivHierarchicalClusteringP2P" |
| }, |
| "checkpoint_files": [ |
| { |
| "file": ".gitattributes", |
| "bytes": 1519, |
| "sha256": "11ad7efa24975ee4b0c3c3a38ed18737f0658a5f75a0a96787b576a78a023361", |
| "url": "https://huggingface.co/codefuse-ai/F2LLM-v2-80M/resolve/6faa7738f343cd4321f9b2fa92fdb7cae3c117a5/.gitattributes" |
| }, |
| { |
| "file": "1_Pooling/config.json", |
| "bytes": 313, |
| "sha256": "37bf193fa101f19101bfad9c31d3eb0f786e247b7b1e5cb7f007d730eed1ddbd", |
| "url": "https://huggingface.co/codefuse-ai/F2LLM-v2-80M/resolve/6faa7738f343cd4321f9b2fa92fdb7cae3c117a5/1_Pooling/config.json" |
| }, |
| { |
| "file": "README.md", |
| "bytes": 7237, |
| "sha256": "7725c4c4fc49efe1c0f14f3c6e59669a0332703acadb462212018dc1b6a9081c", |
| "url": "https://huggingface.co/codefuse-ai/F2LLM-v2-80M/resolve/6faa7738f343cd4321f9b2fa92fdb7cae3c117a5/README.md" |
| }, |
| { |
| "file": "added_tokens.json", |
| "bytes": 707, |
| "sha256": "c0284b582e14987fbd3d5a2cb2bd139084371ed9acbae488829a1c900833c680", |
| "url": "https://huggingface.co/codefuse-ai/F2LLM-v2-80M/resolve/6faa7738f343cd4321f9b2fa92fdb7cae3c117a5/added_tokens.json" |
| }, |
| { |
| "file": "config.json", |
| "bytes": 941, |
| "sha256": "6adc1b382b3aed8453ae99665c01713189737936a3caf2638cb9083c97fd7013", |
| "url": "https://huggingface.co/codefuse-ai/F2LLM-v2-80M/resolve/6faa7738f343cd4321f9b2fa92fdb7cae3c117a5/config.json" |
| }, |
| { |
| "file": "config_sentence_transformers.json", |
| "bytes": 224, |
| "sha256": "0d68b69297f1e04931794d6a0240334fc5fd3d6f0560b5bd904231864cd0aa7d", |
| "url": "https://huggingface.co/codefuse-ai/F2LLM-v2-80M/resolve/6faa7738f343cd4321f9b2fa92fdb7cae3c117a5/config_sentence_transformers.json" |
| }, |
| { |
| "file": "merges.txt", |
| "bytes": 1671853, |
| "sha256": "8831e4f1a044471340f7c0a83d7bd71306a5b867e95fd870f74d0c5308a904d5", |
| "url": "https://huggingface.co/codefuse-ai/F2LLM-v2-80M/resolve/6faa7738f343cd4321f9b2fa92fdb7cae3c117a5/merges.txt" |
| }, |
| { |
| "file": "model.safetensors", |
| "bytes": 160178064, |
| "sha256": "2177fbd79bd6259904a54514e0e313efd28704acd98bfe276a06b2c638f2e4c1", |
| "url": "https://huggingface.co/codefuse-ai/F2LLM-v2-80M/resolve/6faa7738f343cd4321f9b2fa92fdb7cae3c117a5/model.safetensors" |
| }, |
| { |
| "file": "modules.json", |
| "bytes": 385, |
| "sha256": "73348a3d8f8ae4cc5f851dc6b98be3ea819390b814e83656d5eebbadf75fe43e", |
| "url": "https://huggingface.co/codefuse-ai/F2LLM-v2-80M/resolve/6faa7738f343cd4321f9b2fa92fdb7cae3c117a5/modules.json" |
| }, |
| { |
| "file": "special_tokens_map.json", |
| "bytes": 613, |
| "sha256": "76862e765266b85aa9459767e33cbaf13970f327a0e88d1c65846c2ddd3a1ecd", |
| "url": "https://huggingface.co/codefuse-ai/F2LLM-v2-80M/resolve/6faa7738f343cd4321f9b2fa92fdb7cae3c117a5/special_tokens_map.json" |
| }, |
| { |
| "file": "tokenizer.json", |
| "bytes": 8399930, |
| "sha256": "7e295e5bb91a3d35335f92fa4294a6e4e0ab4aa586db853e14312a62135bfddc", |
| "url": "https://huggingface.co/codefuse-ai/F2LLM-v2-80M/resolve/6faa7738f343cd4321f9b2fa92fdb7cae3c117a5/tokenizer.json" |
| }, |
| { |
| "file": "tokenizer_config.json", |
| "bytes": 9762, |
| "sha256": "3c0884a30471f4f542dc89630f62a380bb70a341fafda826136a7be921fec7ea", |
| "url": "https://huggingface.co/codefuse-ai/F2LLM-v2-80M/resolve/6faa7738f343cd4321f9b2fa92fdb7cae3c117a5/tokenizer_config.json" |
| }, |
| { |
| "file": "vocab.json", |
| "bytes": 2776833, |
| "sha256": "ca10d7e9fb3ed18575dd1e277a2579c16d108e32f27439684afa0e10b1440910", |
| "url": "https://huggingface.co/codefuse-ai/F2LLM-v2-80M/resolve/6faa7738f343cd4321f9b2fa92fdb7cae3c117a5/vocab.json" |
| } |
| ], |
| "declared_training": { |
| "source": "MTEB 2.21.0 model registry", |
| "related_arxiv_families": [ |
| "ArxivClusteringP2P", |
| "ArxivClusteringS2S" |
| ], |
| "note": "The registry declares these related training task families; this result is not evidence of unseen-task generalization." |
| }, |
| "verification": { |
| "raw_official_result_sha256": "74eb7ea108e11f538068fd4c77fc76aa8638389ead8d56601a6c0724ad28ded0", |
| "saved_predictions_sha256": "bc6827d33be10061dca5586e1805feae94ea60ce48fe90b4eff4dcf71455c839", |
| "all20_saved_assignment_metrics_independently_reproduced": true, |
| "checkpoint_files_match_exact_public_revision": true, |
| "synthetic_official_wrapper_vs_direct_output_bitwise_equal": true, |
| "actual_output_dimensions": 320, |
| "released_pooling_dimension_metadata_getter": 1024, |
| "metadata_dimension_note": "The released pooling metadata reports 1024; the unchanged checkpoint has 320 hidden dimensions and produces 320-dimensional embeddings." |
| }, |
| "comparison_scope": { |
| "same_as_Vela": [ |
| "pinned MTEB task and dataset revision", |
| "official evaluator", |
| "seed 20260918", |
| "batch size 8" |
| ], |
| "model_specific_differences": [ |
| "F2 uses its official task instruction and EOS; Vela uses no task instruction", |
| "F2 uses BF16 and 8192 tokens; Vela Nano uses FP32 and 512 tokens", |
| "F2 outputs 320 dimensions; Vela Nano outputs 384 dimensions" |
| ], |
| "registry_reported_observation_retained": true, |
| "full_arxiv_corpus_encoded": false, |
| "overall_benchmark_or_SOTA_claim": false |
| }, |
| "reported_observation": { |
| "revision": "35baae5091e26211dbe628f1723448c146fafdf2", |
| "parameters_exact_from_original_model_metadata": 80084288, |
| "original_api_totalParamsB": 0.08, |
| "plot_parameters_billion": 0.080084288, |
| "size_policy": "Both observations use their checkpoint metadata exact parameter count. The original rounded API size is retained separately.", |
| "mteb_version": "2.6.7", |
| "primary_metric": "v_measure", |
| "main_score": 0.634236, |
| "raw_official_task_result": { |
| "dataset_revision": "0bbdb47bcbe3a90093699aefeed338a0f28a7ee8", |
| "task_name": "ArXivHierarchicalClusteringP2P", |
| "mteb_version": "2.6.7", |
| "scores": { |
| "test": [ |
| { |
| "v_measures": { |
| "Level 0": [ |
| 0.659292, |
| 0.679246, |
| 0.663039, |
| 0.671952, |
| 0.654544, |
| 0.675166, |
| 0.690594, |
| 0.666911, |
| 0.667866, |
| 0.683144 |
| ], |
| "Level 1": [ |
| 0.580911, |
| 0.602815, |
| 0.594891, |
| 0.608409, |
| 0.598627, |
| 0.600371, |
| 0.596405, |
| 0.599651, |
| 0.586821, |
| 0.604058 |
| ] |
| }, |
| "v_measure": 0.634236, |
| "v_measure_std": 0.038084, |
| "main_score": 0.634236, |
| "hf_subset": "default", |
| "languages": [ |
| "eng-Latn" |
| ] |
| } |
| ] |
| }, |
| "evaluation_time": 1.8109724521636963, |
| "kg_co2_emissions": null |
| }, |
| "source_urls": { |
| "model_metadata": "https://raw.githubusercontent.com/embeddings-benchmark/results/dd92f097054928798830d3bb68de478f58228325/results/codefuse-ai__F2LLM-v2-80M/35baae5091e26211dbe628f1723448c146fafdf2/model_meta.json", |
| "task_result": "https://raw.githubusercontent.com/embeddings-benchmark/results/dd92f097054928798830d3bb68de478f58228325/results/codefuse-ai__F2LLM-v2-80M/35baae5091e26211dbe628f1723448c146fafdf2/ArXivHierarchicalClusteringP2P.json" |
| }, |
| "source_sha256": { |
| "model_metadata": "9f82791b6ab2f11bae975534aee0b54f8d13b362563ea6c073e35f75baab1165", |
| "task_result": "2a0bdcaca81229552eefad06e07c5a7d5b6f3663e86b6c32790d744c2b710e10" |
| }, |
| "same_checkpoint_as_rerun": false, |
| "comparison_note": "The reported observation and rerun use different checkpoint revisions and MTEB versions; their score difference is not a controlled estimate of rerun variance." |
| } |
| } |
|
|
