raw MTEB result JSON (mteb 2.18.12, pinned harness) for verifiability

results/mlx-gs32-r1/pins.json

@@ -0,0 +1,34 @@
+{
+  "run_id": "mlx-gs32-r1",
+  "backend": "mlx",
+  "model": "/Users/matt/.cache/nq-models/mlx-8b-4bit-gs32",
+  "revision": null,
+  "suite": "regression",
+  "tasks": [
+    "NFCorpus",
+    "SciFact"
+  ],
+  "resolved": [
+    "NFCorpus",
+    "SciFact"
+  ],
+  "mteb_version": "2.18.12",
+  "max_len": 8192,
+  "tolerances": {
+    "_comment": "Pre-registered tolerances (set BEFORE experiments; Codex P0-3). vllm_vs_reference_min applies to same-precision runtime parity (BF16 vs BF16) and to quantized-artifact-vs-its-own-fake-quant reference. quant_vs_bf16 gates are the P4 acceptance thresholds.",
+    "vllm_vs_reference_min": 0.995,
+    "fp8_vs_bf16_mean_cosine_min": 0.995,
+    "nvfp4_vs_bf16_mean_cosine_min": 0.99,
+    "nvfp4_vs_bf16_p5_cosine_min": 0.985,
+    "mteb_per_task_ndcg10_max_loss": 0.01,
+    "mteb_macro_avg_max_loss": 0.005,
+    "matryoshka_dims_to_validate": [
+      1024,
+      1536,
+      2048,
+      4096
+    ],
+    "pooling_same_min_cosine": 0.9999,
+    "pooling_diff_max_cosine": 0.999
+  }
+}

results/mlx-gs32-r1/results/mlx-8b-4bit-gs32/local/NFCorpus.json

@@ -0,0 +1,198 @@
+{
+  "dataset_revision": "ec0fa4fe99da2ff19ca1214b7966684033a58814",
+  "task_name": "NFCorpus",
+  "mteb_version": "2.18.12",
+  "scores": {
+    "test": [
+      {
+        "ndcg_at_1": 0.51548,
+        "ndcg_at_3": 0.48832,
+        "ndcg_at_5": 0.45543,
+        "ndcg_at_10": 0.4199,
+        "ndcg_at_20": 0.3944,
+        "ndcg_at_100": 0.38354,
+        "ndcg_at_1000": 0.47153,
+        "map_at_1": 0.07002,
+        "map_at_3": 0.11911,
+        "map_at_5": 0.13876,
+        "map_at_10": 0.1621,
+        "map_at_20": 0.18197,
+        "map_at_100": 0.20792,
+        "map_at_1000": 0.22476,
+        "recall_at_1": 0.07002,
+        "recall_at_3": 0.13071,
+        "recall_at_5": 0.16023,
+        "recall_at_10": 0.20679,
+        "recall_at_20": 0.25536,
+        "recall_at_100": 0.38217,
+        "recall_at_1000": 0.70374,
+        "accuracy": 0.07002,
+        "precision_at_1": 0.53251,
+        "precision_at_3": 0.4613,
+        "precision_at_5": 0.39628,
+        "precision_at_10": 0.30898,
+        "precision_at_20": 0.23111,
+        "precision_at_100": 0.09641,
+        "precision_at_1000": 0.02293,
+        "mrr_at_1": 0.532508,
+        "mrr_at_3": 0.615067,
+        "mrr_at_5": 0.623426,
+        "mrr_at_10": 0.628844,
+        "mrr_at_20": 0.632236,
+        "mrr_at_100": 0.633863,
+        "mrr_at_1000": 0.634243,
+        "nauc_ndcg_at_1_max": 0.361791,
+        "nauc_ndcg_at_1_std": 0.197275,
+        "nauc_ndcg_at_1_diff1": 0.330091,
+        "nauc_ndcg_at_3_max": 0.379747,
+        "nauc_ndcg_at_3_std": 0.240875,
+        "nauc_ndcg_at_3_diff1": 0.205755,
+        "nauc_ndcg_at_5_max": 0.38234,
+        "nauc_ndcg_at_5_std": 0.245126,
+        "nauc_ndcg_at_5_diff1": 0.183497,
+        "nauc_ndcg_at_10_max": 0.357773,
+        "nauc_ndcg_at_10_std": 0.248357,
+        "nauc_ndcg_at_10_diff1": 0.147619,
+        "nauc_ndcg_at_20_max": 0.336839,
+        "nauc_ndcg_at_20_std": 0.236063,
+        "nauc_ndcg_at_20_diff1": 0.148704,
+        "nauc_ndcg_at_100_max": 0.375472,
+        "nauc_ndcg_at_100_std": 0.248619,
+        "nauc_ndcg_at_100_diff1": 0.214037,
+        "nauc_ndcg_at_1000_max": 0.445094,
+        "nauc_ndcg_at_1000_std": 0.311594,
+        "nauc_ndcg_at_1000_diff1": 0.224862,
+        "nauc_map_at_1_max": 0.051742,
+        "nauc_map_at_1_std": -0.171878,
+        "nauc_map_at_1_diff1": 0.45304,
+        "nauc_map_at_3_max": 0.11329,
+        "nauc_map_at_3_std": -0.119231,
+        "nauc_map_at_3_diff1": 0.355378,
+        "nauc_map_at_5_max": 0.161463,
+        "nauc_map_at_5_std": -0.085039,
+        "nauc_map_at_5_diff1": 0.328023,
+        "nauc_map_at_10_max": 0.204056,
+        "nauc_map_at_10_std": -0.035936,
+        "nauc_map_at_10_diff1": 0.290422,
+        "nauc_map_at_20_max": 0.242903,
+        "nauc_map_at_20_std": 0.016566,
+        "nauc_map_at_20_diff1": 0.270408,
+        "nauc_map_at_100_max": 0.275908,
+        "nauc_map_at_100_std": 0.084767,
+        "nauc_map_at_100_diff1": 0.248725,
+        "nauc_map_at_1000_max": 0.27921,
+        "nauc_map_at_1000_std": 0.111575,
+        "nauc_map_at_1000_diff1": 0.232587,
+        "nauc_recall_at_1_max": 0.051742,
+        "nauc_recall_at_1_std": -0.171878,
+        "nauc_recall_at_1_diff1": 0.45304,
+        "nauc_recall_at_3_max": 0.113549,
+        "nauc_recall_at_3_std": -0.106119,
+        "nauc_recall_at_3_diff1": 0.322832,
+        "nauc_recall_at_5_max": 0.170485,
+        "nauc_recall_at_5_std": -0.062817,
+        "nauc_recall_at_5_diff1": 0.280373,
+        "nauc_recall_at_10_max": 0.171892,
+        "nauc_recall_at_10_std": -0.040567,
+        "nauc_recall_at_10_diff1": 0.207525,
+        "nauc_recall_at_20_max": 0.201385,
+        "nauc_recall_at_20_std": 0.006669,
+        "nauc_recall_at_20_diff1": 0.174744,
+        "nauc_recall_at_100_max": 0.247712,
+        "nauc_recall_at_100_std": 0.125057,
+        "nauc_recall_at_100_diff1": 0.137816,
+        "nauc_recall_at_1000_max": 0.235001,
+        "nauc_recall_at_1000_std": 0.171007,
+        "nauc_recall_at_1000_diff1": 0.102237,
+        "nauc_precision_at_1_max": 0.393315,
+        "nauc_precision_at_1_std": 0.21924,
+        "nauc_precision_at_1_diff1": 0.334588,
+        "nauc_precision_at_3_max": 0.38081,
+        "nauc_precision_at_3_std": 0.294203,
+        "nauc_precision_at_3_diff1": 0.075707,
+        "nauc_precision_at_5_max": 0.390596,
+        "nauc_precision_at_5_std": 0.325156,
+        "nauc_precision_at_5_diff1": 0.019294,
+        "nauc_precision_at_10_max": 0.346916,
+        "nauc_precision_at_10_std": 0.366311,
+        "nauc_precision_at_10_diff1": -0.06154,
+        "nauc_precision_at_20_max": 0.275543,
+        "nauc_precision_at_20_std": 0.370191,
+        "nauc_precision_at_20_diff1": -0.096967,
+        "nauc_precision_at_100_max": 0.130541,
+        "nauc_precision_at_100_std": 0.360728,
+        "nauc_precision_at_100_diff1": -0.150043,
+        "nauc_precision_at_1000_max": 0.004327,
+        "nauc_precision_at_1000_std": 0.273713,
+        "nauc_precision_at_1000_diff1": -0.178153,
+        "nauc_mrr_at_1_max": 0.393315,
+        "nauc_mrr_at_1_std": 0.21924,
+        "nauc_mrr_at_1_diff1": 0.334588,
+        "nauc_mrr_at_3_max": 0.447485,
+        "nauc_mrr_at_3_std": 0.285192,
+        "nauc_mrr_at_3_diff1": 0.333354,
+        "nauc_mrr_at_5_max": 0.45868,
+        "nauc_mrr_at_5_std": 0.299383,
+        "nauc_mrr_at_5_diff1": 0.326082,
+        "nauc_mrr_at_10_max": 0.461335,
+        "nauc_mrr_at_10_std": 0.300338,
+        "nauc_mrr_at_10_diff1": 0.326327,
+        "nauc_mrr_at_20_max": 0.4604,
+        "nauc_mrr_at_20_std": 0.299754,
+        "nauc_mrr_at_20_diff1": 0.323291,
+        "nauc_mrr_at_100_max": 0.461252,
+        "nauc_mrr_at_100_std": 0.300379,
+        "nauc_mrr_at_100_diff1": 0.324037,
+        "nauc_mrr_at_1000_max": 0.46089,
+        "nauc_mrr_at_1000_std": 0.299993,
+        "nauc_mrr_at_1000_diff1": 0.324011,
+        "hit_rate_at_1": 0.53251,
+        "hit_rate_at_3": 0.70898,
+        "hit_rate_at_5": 0.74303,
+        "hit_rate_at_10": 0.78328,
+        "hit_rate_at_20": 0.82972,
+        "hit_rate_at_100": 0.88854,
+        "hit_rate_at_1000": 0.97214,
+        "main_score": 0.4199,
+        "hf_subset": "default",
+        "languages": [
+          "eng-Latn"
+        ],
+        "mteb_version": "2.18.12"
+      }
+    ]
+  },
+  "evaluation_time": 1719.0480923740106,
+  "kg_co2_emissions": null,
+  "date": 1785807158.522371,
+  "evaluation_phases": [
+    {
+      "name": "Data loading",
+      "start": 1.9416002032812685e-05,
+      "end": 7.027787541002908,
+      "split": "",
+      "subset": ""
+    },
+    {
+      "name": "Dataset transform",
+      "start": 7.027827083002194,
+      "end": 7.027827791003801,
+      "split": "",
+      "subset": ""
+    },
+    {
+      "name": "Encoding queries and documents",
+      "start": 7.031841333002376,
+      "end": 1718.9132305830062,
+      "split": "test",
+      "subset": "default"
+    },
+    {
+      "name": "Scoring",
+      "start": 1718.913248500001,
+      "end": 1719.0411249580065,
+      "split": "test",
+      "subset": "default"
+    }
+  ]
+}

results/mlx-gs32-r1/results/mlx-8b-4bit-gs32/local/SciFact.json

@@ -0,0 +1,198 @@
+{
+  "dataset_revision": "d56462d0e63a25450459c4f213e49ffdb866f7f9",
+  "task_name": "SciFact",
+  "mteb_version": "2.18.12",
+  "scores": {
+    "test": [
+      {
+        "ndcg_at_1": 0.71667,
+        "ndcg_at_3": 0.80245,
+        "ndcg_at_5": 0.82129,
+        "ndcg_at_10": 0.8338,
+        "ndcg_at_20": 0.83645,
+        "ndcg_at_100": 0.84277,
+        "ndcg_at_1000": 0.84416,
+        "map_at_1": 0.68328,
+        "map_at_3": 0.77074,
+        "map_at_5": 0.78619,
+        "map_at_10": 0.79338,
+        "map_at_20": 0.79423,
+        "map_at_100": 0.79515,
+        "map_at_1000": 0.79522,
+        "recall_at_1": 0.68328,
+        "recall_at_3": 0.86472,
+        "recall_at_5": 0.913,
+        "recall_at_10": 0.94667,
+        "recall_at_20": 0.95667,
+        "recall_at_100": 0.99,
+        "recall_at_1000": 1.0,
+        "accuracy": 0.68328,
+        "precision_at_1": 0.71667,
+        "precision_at_3": 0.31444,
+        "precision_at_5": 0.204,
+        "precision_at_10": 0.10733,
+        "precision_at_20": 0.05433,
+        "precision_at_100": 0.0112,
+        "precision_at_1000": 0.00113,
+        "mrr_at_1": 0.716667,
+        "mrr_at_3": 0.787778,
+        "mrr_at_5": 0.796611,
+        "mrr_at_10": 0.800489,
+        "mrr_at_20": 0.80092,
+        "mrr_at_100": 0.801844,
+        "mrr_at_1000": 0.801915,
+        "nauc_ndcg_at_1_max": 0.578746,
+        "nauc_ndcg_at_1_std": -0.058988,
+        "nauc_ndcg_at_1_diff1": 0.796694,
+        "nauc_ndcg_at_3_max": 0.553918,
+        "nauc_ndcg_at_3_std": -0.087751,
+        "nauc_ndcg_at_3_diff1": 0.764083,
+        "nauc_ndcg_at_5_max": 0.580007,
+        "nauc_ndcg_at_5_std": -0.04954,
+        "nauc_ndcg_at_5_diff1": 0.757275,
+        "nauc_ndcg_at_10_max": 0.610657,
+        "nauc_ndcg_at_10_std": -0.014343,
+        "nauc_ndcg_at_10_diff1": 0.770419,
+        "nauc_ndcg_at_20_max": 0.607121,
+        "nauc_ndcg_at_20_std": -0.023393,
+        "nauc_ndcg_at_20_diff1": 0.769182,
+        "nauc_ndcg_at_100_max": 0.593043,
+        "nauc_ndcg_at_100_std": -0.036693,
+        "nauc_ndcg_at_100_diff1": 0.772503,
+        "nauc_ndcg_at_1000_max": 0.590867,
+        "nauc_ndcg_at_1000_std": -0.035322,
+        "nauc_ndcg_at_1000_diff1": 0.77203,
+        "nauc_map_at_1_max": 0.473248,
+        "nauc_map_at_1_std": -0.162678,
+        "nauc_map_at_1_diff1": 0.798509,
+        "nauc_map_at_3_max": 0.519395,
+        "nauc_map_at_3_std": -0.129223,
+        "nauc_map_at_3_diff1": 0.775032,
+        "nauc_map_at_5_max": 0.55992,
+        "nauc_map_at_5_std": -0.079033,
+        "nauc_map_at_5_diff1": 0.771291,
+        "nauc_map_at_10_max": 0.574496,
+        "nauc_map_at_10_std": -0.053937,
+        "nauc_map_at_10_diff1": 0.776303,
+        "nauc_map_at_20_max": 0.573488,
+        "nauc_map_at_20_std": -0.056303,
+        "nauc_map_at_20_diff1": 0.775988,
+        "nauc_map_at_100_max": 0.571641,
+        "nauc_map_at_100_std": -0.058394,
+        "nauc_map_at_100_diff1": 0.776191,
+        "nauc_map_at_1000_max": 0.571554,
+        "nauc_map_at_1000_std": -0.058274,
+        "nauc_map_at_1000_diff1": 0.776166,
+        "nauc_recall_at_1_max": 0.473248,
+        "nauc_recall_at_1_std": -0.162678,
+        "nauc_recall_at_1_diff1": 0.798509,
+        "nauc_recall_at_3_max": 0.497826,
+        "nauc_recall_at_3_std": -0.17138,
+        "nauc_recall_at_3_diff1": 0.709852,
+        "nauc_recall_at_5_max": 0.60342,
+        "nauc_recall_at_5_std": -0.012605,
+        "nauc_recall_at_5_diff1": 0.654559,
+        "nauc_recall_at_10_max": 0.872928,
+        "nauc_recall_at_10_std": 0.224673,
+        "nauc_recall_at_10_diff1": 0.727431,
+        "nauc_recall_at_20_max": 0.892803,
+        "nauc_recall_at_20_std": 0.160921,
+        "nauc_recall_at_20_diff1": 0.70933,
+        "nauc_recall_at_100_max": 0.814815,
+        "nauc_recall_at_100_std": -0.127295,
+        "nauc_recall_at_100_diff1": 0.814815,
+        "nauc_recall_at_1000_max": NaN,
+        "nauc_recall_at_1000_std": NaN,
+        "nauc_recall_at_1000_diff1": NaN,
+        "nauc_precision_at_1_max": 0.578746,
+        "nauc_precision_at_1_std": -0.058988,
+        "nauc_precision_at_1_diff1": 0.796694,
+        "nauc_precision_at_3_max": 0.48325,
+        "nauc_precision_at_3_std": 0.231743,
+        "nauc_precision_at_3_diff1": 0.239364,
+        "nauc_precision_at_5_max": 0.461446,
+        "nauc_precision_at_5_std": 0.454131,
+        "nauc_precision_at_5_diff1": 0.003718,
+        "nauc_precision_at_10_max": 0.403184,
+        "nauc_precision_at_10_std": 0.569503,
+        "nauc_precision_at_10_diff1": -0.115686,
+        "nauc_precision_at_20_max": 0.35872,
+        "nauc_precision_at_20_std": 0.535269,
+        "nauc_precision_at_20_diff1": -0.151866,
+        "nauc_precision_at_100_max": 0.280045,
+        "nauc_precision_at_100_std": 0.575897,
+        "nauc_precision_at_100_diff1": -0.268937,
+        "nauc_precision_at_1000_max": 0.256403,
+        "nauc_precision_at_1000_std": 0.606984,
+        "nauc_precision_at_1000_diff1": -0.316849,
+        "nauc_mrr_at_1_max": 0.578746,
+        "nauc_mrr_at_1_std": -0.058988,
+        "nauc_mrr_at_1_diff1": 0.796694,
+        "nauc_mrr_at_3_max": 0.589251,
+        "nauc_mrr_at_3_std": -0.036557,
+        "nauc_mrr_at_3_diff1": 0.772258,
+        "nauc_mrr_at_5_max": 0.594205,
+        "nauc_mrr_at_5_std": -0.022932,
+        "nauc_mrr_at_5_diff1": 0.769573,
+        "nauc_mrr_at_10_max": 0.599433,
+        "nauc_mrr_at_10_std": -0.025207,
+        "nauc_mrr_at_10_diff1": 0.774994,
+        "nauc_mrr_at_20_max": 0.598792,
+        "nauc_mrr_at_20_std": -0.026582,
+        "nauc_mrr_at_20_diff1": 0.774767,
+        "nauc_mrr_at_100_max": 0.59703,
+        "nauc_mrr_at_100_std": -0.028571,
+        "nauc_mrr_at_100_diff1": 0.774969,
+        "nauc_mrr_at_1000_max": 0.596951,
+        "nauc_mrr_at_1000_std": -0.028437,
+        "nauc_mrr_at_1000_diff1": 0.774943,
+        "hit_rate_at_1": 0.71667,
+        "hit_rate_at_3": 0.88333,
+        "hit_rate_at_5": 0.92333,
+        "hit_rate_at_10": 0.95,
+        "hit_rate_at_20": 0.95667,
+        "hit_rate_at_100": 0.99,
+        "hit_rate_at_1000": 1.0,
+        "main_score": 0.8338,
+        "hf_subset": "default",
+        "languages": [
+          "eng-Latn"
+        ],
+        "mteb_version": "2.18.12"
+      }
+    ]
+  },
+  "evaluation_time": 1806.8412977500047,
+  "kg_co2_emissions": null,
+  "date": 1785810370.431944,
+  "evaluation_phases": [
+    {
+      "name": "Data loading",
+      "start": 2.0250001398380846e-05,
+      "end": 3.305820292000135,
+      "split": "",
+      "subset": ""
+    },
+    {
+      "name": "Dataset transform",
+      "start": 3.3058345840036054,
+      "end": 3.305835749997641,
+      "split": "",
+      "subset": ""
+    },
+    {
+      "name": "Encoding queries and documents",
+      "start": 3.3116467500003637,
+      "end": 1806.7048066670031,
+      "split": "test",
+      "subset": "default"
+    },
+    {
+      "name": "Scoring",
+      "start": 1806.704926792001,
+      "end": 1806.830110625,
+      "split": "test",
+      "subset": "default"
+    }
+  ]
+}

results/mlx-gs32-r1/results/mlx-8b-4bit-gs32/local/model_meta.json

@@ -0,0 +1,37 @@
+{
+    "loader_kwargs": {},
+    "name": "mlx-8b-4bit-gs32",
+    "revision": "local",
+    "release_date": null,
+    "languages": null,
+    "n_parameters": null,
+    "n_active_parameters_override": null,
+    "n_embedding_parameters": null,
+    "memory_usage_mb": null,
+    "max_tokens": null,
+    "embed_dim": null,
+    "license": null,
+    "open_weights": null,
+    "public_training_code": null,
+    "public_training_data": null,
+    "framework": [],
+    "reference": null,
+    "similarity_fn_name": null,
+    "use_instructions": null,
+    "training_datasets": null,
+    "adapted_from": null,
+    "superseded_by": null,
+    "modalities": [
+        "text"
+    ],
+    "model_type": [
+        "dense"
+    ],
+    "citation": null,
+    "contacts": null,
+    "experiment_kwargs": null,
+    "output_dtypes": null,
+    "extra_requirements_groups": null,
+    "loader": null,
+    "is_cross_encoder": false
+}

results/mlx-gs32-r1/results/mlx-8b-4bit-gs32/local/run_settings.jsonl

@@ -0,0 +1,2 @@
+{"task": "NFCorpus", "split": "test", "subset": "default", "version": {"mteb": "2.18.12", "torch": "2.13.0", "sentence-transformers": "5.6.1", "flash-attn": null, "transformers": "5.14.1"}, "encode_kwargs": {"batch_size": 8}}
+{"task": "SciFact", "split": "test", "subset": "default", "version": {"mteb": "2.18.12", "torch": "2.13.0", "sentence-transformers": "5.6.1", "flash-attn": null, "transformers": "5.14.1"}, "encode_kwargs": {"batch_size": 16}}

results/mlx-gs32-r1/summary.json

@@ -0,0 +1,8 @@
+{
+  "NFCorpus": {
+    "main_score": 0.4199
+  },
+  "SciFact": {
+    "main_score": 0.8338
+  }
+}

results/mlx-gs32-r1/task_manifest.json

@@ -0,0 +1,9 @@
+{
+  "run_id": "mlx-gs32-r1",
+  "completed": [
+    "NFCorpus",
+    "SciFact"
+  ],
+  "failed": [],
+  "pending": []
+}