Raise multilingual LoRA rank to 64 (accuracy 0.702 -> 0.853, es ECE 0.170 -> 0.038)

README.md

@@ -19,9 +19,10 @@ pipeline_tag: text-classification
 
 # jeba-multi (System One decision engine)
 
-> **Status: pre-release.** This card describes the intended release. **Weights and measured
-> metrics are pending** the RTX 3060 training run (`uv sync --extra train`); numbers below are
-> placeholders to be replaced by `benchmarks/report.md`.
+> **Status: released (`v0.2.0`).** Trained on a single RTX 3060 12GB and published as LoRA
+> adapters ([`munod/jeba-en`](https://huggingface.co/munod/jeba-en),
+> [`munod/jeba-multi`](https://huggingface.co/munod/jeba-multi)); measured numbers below come
+> from `benchmarks/report.md`.
 
 ## Model details
 
@@ -70,14 +71,21 @@ checkpointing.
 
 | Checkpoint | Accuracy | ECE (calibrated) | p50 (ms) |
 | --- | --- | --- | --- |
-| English (ModernBERT-large + LoRA + choice head) | 0.781 | 0.077 | 22.9 |
-| Multilingual (mmBERT-base + LoRA + choice head) | 0.711 | 0.073 | 12.4 |
-
-Per primitive (English): `choice` 0.708, `noul` 0.744, `score` 0.892; (multilingual): `choice`
-0.734, `noul` 0.716, `score` 0.682. The localized, per-record-RNG data (B-1) lifted multilingual
-`choice` from 0.40 to 0.73 and English overall from 0.72 to 0.78. Per-language calibration for a
-few multilingual languages (`es`, `nl`, `de`) and multilingual `score` remain the next targets.
+| English (ModernBERT-large + LoRA r=16 + choice head) | 0.763 | 0.061 | 22.6 |
+| Multilingual (mmBERT-base + LoRA r=64 + choice head) | 0.853 | 0.038 | 13.3 |
+
+Per primitive (English): `choice` 0.834, `noul` 0.744, `score` 0.712; (multilingual): `choice`
+0.684, `noul` 0.960, `score` 0.916. The localized, per-record-RNG data (B-1) lifted multilingual
+`choice` from 0.40 to 0.68 and English overall from 0.72 to 0.76. **Raising the multilingual LoRA
+rank from 16 to 64** (alpha 128) removed the cross-language capacity bottleneck: overall accuracy
+0.702 → 0.853 and `es` ECE 0.170 → 0.038 (`es` accuracy 0.472 → 0.956). Five of six languages now
+meet ECE ≤ 0.05; `nl` (ECE 0.104, accuracy 0.663) remains the outlier (NFR-C06 partially open).
 The CUDA-graph fast path (`JEBA_FAST=1`) gives a 2.7× p50 speedup with 0 top-label flips.
+
+**Robustness (B-4).** On a noisy view (one surface edit — typo/accents/casing — applied to 15% of
+states) English drops only 0.763 → 0.760 and multilingual (r=64) 0.853 → 0.847, so the released
+adapters are already robust to this noise model.
+
 Full tables and environment are in
 [`benchmarks/report.md`](https://github.com/munod/jeba/blob/main/benchmarks/report.md).
 

adapter_config.json

@@ -20,7 +20,7 @@
   "layers_pattern": null,
   "layers_to_transform": null,
   "loftq_config": {},
-  "lora_alpha": 32,
+  "lora_alpha": 128,
   "lora_bias": false,
   "lora_dropout": 0.05,
   "lora_ga_config": null,
@@ -31,13 +31,13 @@
   "peft_type": "LORA",
   "peft_version": "0.21.0",
   "qalora_group_size": 16,
-  "r": 16,
+  "r": 64,
   "rank_pattern": {},
   "revision": null,
   "target_modules": [
-    "Wo",
+    "Wi",
     "Wqkv",
-    "Wi"
+    "Wo"
   ],
   "target_parameters": null,
   "task_type": null,

adapter_model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:6f529ee66b8d859121e075662702384dce21aaddc14d1fdf288b4383c479ae19
-size 13537472
+oid sha256:18df64c5321be3c487171eb1f6bceda6a0bc15f6731ba60083de76932187fb9f
+size 54088096

finetune_config.json

@@ -8,13 +8,13 @@
   "grad_accum": 4,
   "gradient_checkpointing": true,
   "learning_rate": 0.0001,
-  "lora_alpha": 32,
+  "lora_alpha": 128,
   "lora_dropout": 0.05,
-  "lora_rank": 16,
+  "lora_rank": 64,
   "max_len": 1024,
   "max_records": null,
   "model_id": "jhu-clsp/mmBERT-base",
-  "out_dir": "checkpoints/multi",
+  "out_dir": "checkpoints/multi_r64",
   "seed": 42,
   "use_4bit": false,
   "val_split": 0.1

temperature_calibration.json

@@ -4,132 +4,132 @@
     "choice": {
       "by_language": {
         "de": {
-          "ece_after": 0.249286,
-          "ece_before": 0.335616,
+          "ece_after": 0.180605,
+          "ece_before": 0.241955,
           "n": 83,
-          "temperature": 3.0
+          "temperature": 0.5
         },
         "es": {
-          "ece_after": 0.295552,
-          "ece_before": 0.395429,
+          "ece_after": 0.031482,
+          "ece_before": 0.245513,
           "n": 84,
-          "temperature": 3.0
+          "temperature": 0.05
         },
         "fr": {
-          "ece_after": 0.031722,
-          "ece_before": 0.031722,
+          "ece_after": 0.082135,
+          "ece_before": 0.127981,
           "n": 83,
-          "temperature": 1.0
+          "temperature": 0.5
         },
         "it": {
-          "ece_after": 0.000804,
-          "ece_before": 0.013927,
+          "ece_after": 0.104306,
+          "ece_before": 0.132043,
           "n": 83,
-          "temperature": 1.25
+          "temperature": 0.75
         },
         "nl": {
-          "ece_after": 0.154009,
-          "ece_before": 0.22301,
+          "ece_after": 0.223918,
+          "ece_before": 0.35971,
           "n": 83,
-          "temperature": 2.0
+          "temperature": 5.0
         },
         "pt": {
-          "ece_after": 0.059923,
-          "ece_before": 0.083558,
+          "ece_after": 0.074149,
+          "ece_before": 0.080335,
           "n": 84,
           "temperature": 0.5
         }
       },
-      "ece_after": 0.118556,
-      "ece_before": 0.153826,
+      "ece_after": 0.121825,
+      "ece_before": 0.153996,
       "n": 500,
-      "temperature": 2.0
+      "temperature": 0.75
     },
     "noul": {
       "by_language": {
         "de": {
-          "ece_after": 0.012124,
-          "ece_before": 0.068245,
+          "ece_after": 0.06539,
+          "ece_before": 0.382207,
           "n": 83,
-          "temperature": 3.0
+          "temperature": 0.05
         },
         "es": {
-          "ece_after": 0.087329,
-          "ece_before": 0.224015,
+          "ece_after": 0.06501,
+          "ece_before": 0.40446,
           "n": 84,
-          "temperature": 10.0
+          "temperature": 0.05
         },
         "fr": {
-          "ece_after": 0.006848,
-          "ece_before": 0.310798,
+          "ece_after": 0.018942,
+          "ece_before": 0.30131,
           "n": 83,
-          "temperature": 0.1
+          "temperature": 0.05
         },
         "it": {
-          "ece_after": 0.040417,
-          "ece_before": 0.259646,
+          "ece_after": 0.015752,
+          "ece_before": 0.33742,
           "n": 83,
-          "temperature": 0.25
+          "temperature": 0.1
         },
         "nl": {
-          "ece_after": 0.078314,
-          "ece_before": 0.218552,
+          "ece_after": 0.086522,
+          "ece_before": 0.317352,
           "n": 83,
-          "temperature": 10.0
+          "temperature": 0.1
         },
         "pt": {
-          "ece_after": 0.006405,
-          "ece_before": 0.29764,
+          "ece_after": 0.019275,
+          "ece_before": 0.306618,
           "n": 84,
-          "temperature": 0.15
+          "temperature": 0.1
         }
       },
-      "ece_after": 0.06769,
-      "ece_before": 0.071107,
+      "ece_after": 0.029595,
+      "ece_before": 0.341617,
       "n": 500,
-      "temperature": 0.75
+      "temperature": 0.05
     },
     "score": {
       "by_language": {
         "de": {
-          "ece_after": 0.128129,
-          "ece_before": 0.313627,
+          "ece_after": 0.025083,
+          "ece_before": 0.414687,
           "n": 83,
-          "temperature": 0.15
+          "temperature": 0.25
         },
         "es": {
-          "ece_after": 0.160055,
-          "ece_before": 0.25,
+          "ece_after": 0.024446,
+          "ece_before": 0.399067,
           "n": 84,
           "temperature": 0.25
         },
         "fr": {
-          "ece_after": 0.103257,
-          "ece_before": 0.178855,
+          "ece_after": 0.064809,
+          "ece_before": 0.383335,
           "n": 83,
           "temperature": 0.25
         },
         "it": {
-          "ece_after": 0.118342,
-          "ece_before": 0.160282,
+          "ece_after": 0.032551,
+          "ece_before": 0.388731,
           "n": 83,
-          "temperature": 0.75
+          "temperature": 0.25
         },
         "nl": {
-          "ece_after": 0.112712,
-          "ece_before": 0.359823,
+          "ece_after": 0.014055,
+          "ece_before": 0.406849,
           "n": 83,
-          "temperature": 0.15
+          "temperature": 0.25
         },
         "pt": {
-          "ece_after": 0.102839,
-          "ece_before": 0.213012,
+          "ece_after": 0.032676,
+          "ece_before": 0.379573,
           "n": 84,
           "temperature": 0.25
         }
       },
-      "ece_after": 0.094548,
-      "ece_before": 0.233416,
+      "ece_after": 0.032023,
+      "ece_before": 0.393027,
       "n": 500,
       "temperature": 0.25
     }