LoRA adapter + metrics (bit-width study)

README.md

@@ -0,0 +1,69 @@
+---
+library_name: peft
+base_model: meta-llama/Llama-3.2-3B
+tags:
+- lora
+- peft
+- quantization
+- glue
+- rte
+---
+
+# llama32-3B-rte-int8-lora-seed44
+
+LoRA adapter trained on GLUE **RTE** on top of a
+**int8** backbone of `meta-llama/Llama-3.2-3B`.
+
+Part of a controlled study of whether the backbone bit-width changes what a LoRA
+adapter learns. For a given (model size, seed) the adapter initialisation is
+**identical** across the bf16 / int8 / nf4 arms, and the data order, optimiser,
+schedule and LoRA hyperparameters are held fixed — so any difference in the
+learned update is attributable to the backbone.
+
+## Result
+
+| metric | validation | test |
+|---|---|---|
+| accuracy | 0.8876 | 0.8556 |
+| macro-F1 | 0.8875 | 0.8546 |
+| loss | 0.2930 | 0.3882 |
+
+Test-set majority-class baseline: 0.5271
+
+- peak GPU memory: 6.81 GiB
+- training time: 11.3 min (105 steps)
+- GPU: NVIDIA GeForce RTX 4090
+
+## Setup
+
+- seed: `44` · adapter init: `shared:lora_init_3B_seed44.pt:224tensors`
+- LoRA: r=16, alpha=32, dropout=0.0, bias=none,
+  target_modules=['q_proj', 'k_proj', 'v_proj', 'o_proj']
+- trainable params: 9,175,040
+- epochs 3, lr 0.0002,
+  max_len 256, batch 4
+  x grad_accum 16,
+  cosine schedule, warmup 0.03
+
+## Prompt format
+
+Trained as causal LM with the loss on the answer letter only (prompt tokens
+masked to -100):
+
+```
+Premise: ...
+Hypothesis: ...
+
+Does the premise entail the hypothesis?
+
+A. Entailment
+B. Not entailment
+
+Answer:
+```
+
+Evaluated by conditional likelihood over the answer letters
+(Entailment, Not entailment).
+
+> GLUE `test` is unlabeled, so the official `validation` split is used as TEST
+> and the validation set is carved from `train` (disjoint).

adapter_config.json

@@ -0,0 +1,45 @@
+{
+  "alora_invocation_tokens": null,
+  "alpha_pattern": {},
+  "arrow_config": null,
+  "auto_mapping": null,
+  "base_model_name_or_path": "meta-llama/Llama-3.2-3B",
+  "bias": "none",
+  "corda_config": null,
+  "ensure_weight_tying": false,
+  "eva_config": null,
+  "exclude_modules": null,
+  "fan_in_fan_out": false,
+  "inference_mode": true,
+  "init_lora_weights": true,
+  "layer_replication": null,
+  "layers_pattern": null,
+  "layers_to_transform": null,
+  "loftq_config": {},
+  "lora_alpha": 32,
+  "lora_bias": false,
+  "lora_dropout": 0.0,
+  "lora_ga_config": null,
+  "megatron_config": null,
+  "megatron_core": "megatron.core",
+  "modules_to_save": null,
+  "peft_type": "LORA",
+  "peft_version": "0.19.1",
+  "qalora_group_size": 16,
+  "r": 16,
+  "rank_pattern": {},
+  "revision": null,
+  "target_modules": [
+    "v_proj",
+    "q_proj",
+    "k_proj",
+    "o_proj"
+  ],
+  "target_parameters": null,
+  "task_type": "CAUSAL_LM",
+  "trainable_token_indices": null,
+  "use_bdlora": null,
+  "use_dora": false,
+  "use_qalora": false,
+  "use_rslora": false
+}

adapter_model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1ee3e38255985a3e276f30ffdf16cca34abe7a0f296e37533196722fc5e254ca
+size 36730224

run_metrics.json

@@ -0,0 +1,94 @@
+{
+  "run_name": "llama32-3B-rte-int8-lora-seed44",
+  "model_size": "3B",
+  "model_id": "meta-llama/Llama-3.2-3B",
+  "task": "rte",
+  "glue_config": "rte",
+  "n_classes": 2,
+  "bitwidth": "int8",
+  "seed": 44,
+  "lora_init_source": "shared:lora_init_3B_seed44.pt:224tensors",
+  "trainable_params": 9175040,
+  "total_params": 3221924864,
+  "train": {
+    "steps": 105,
+    "epochs": 2.9411764705882355,
+    "train_runtime_sec": 678.9310655593872,
+    "train_loss": 0.42380388123648505,
+    "n_train_examples": 2241,
+    "n_train_tokens_per_epoch": 219447,
+    "throughput_samples_per_sec": 9.708167448719719,
+    "throughput_tokens_per_sec": 950.6596261129835
+  },
+  "peak_gpu_mem_gib": {
+    "allocated": 6.807776927947998,
+    "reserved": 11.232421875
+  },
+  "validation": {
+    "accuracy": 0.8875502008032129,
+    "macro_f1": 0.8875483870967742,
+    "loss": 0.293006686800455,
+    "n": 249,
+    "n_classes": 2,
+    "majority_baseline": 0.5140562248995983,
+    "pred_dist": {
+      "A": 129,
+      "B": 120
+    },
+    "true_dist": {
+      "A": 121,
+      "B": 128
+    }
+  },
+  "test": {
+    "accuracy": 0.855595667870036,
+    "macro_f1": 0.8545931758530183,
+    "loss": 0.38815017738497215,
+    "n": 277,
+    "n_classes": 2,
+    "majority_baseline": 0.5270758122743683,
+    "pred_dist": {
+      "A": 154,
+      "B": 123
+    },
+    "true_dist": {
+      "A": 146,
+      "B": 131
+    }
+  },
+  "hyperparams": {
+    "num_train_epochs": 3,
+    "learning_rate": 0.0002,
+    "max_length": 256,
+    "per_device_train_batch_size": 4,
+    "per_device_eval_batch_size": 8,
+    "gradient_accumulation_steps": 16,
+    "warmup_ratio": 0.03,
+    "lr_scheduler_type": "cosine",
+    "gradient_checkpointing": true
+  },
+  "lora": {
+    "r": 16,
+    "alpha": 32,
+    "dropout": 0.0,
+    "bias": "none",
+    "target_modules": [
+      "q_proj",
+      "k_proj",
+      "v_proj",
+      "o_proj"
+    ]
+  },
+  "env": {
+    "torch": "2.4.1+cu121",
+    "cuda": "12.1",
+    "gpu": "NVIDIA GeForce RTX 4090",
+    "slurm_job": "1932576",
+    "node": "node32"
+  },
+  "debug_caps": {
+    "max_train": null,
+    "max_eval": null,
+    "max_steps": -1
+  }
+}

train_config.json

@@ -0,0 +1,31 @@
+{
+  "model_id": "meta-llama/Llama-3.2-3B",
+  "task": "rte",
+  "glue_config": "rte",
+  "bitwidth": "int8",
+  "seed": 44,
+  "lora": {
+    "r": 16,
+    "alpha": 32,
+    "dropout": 0.0,
+    "bias": "none",
+    "target_modules": [
+      "q_proj",
+      "k_proj",
+      "v_proj",
+      "o_proj"
+    ]
+  },
+  "hyperparams": {
+    "num_train_epochs": 3,
+    "learning_rate": 0.0002,
+    "max_length": 256,
+    "per_device_train_batch_size": 4,
+    "per_device_eval_batch_size": 8,
+    "gradient_accumulation_steps": 16,
+    "warmup_ratio": 0.03,
+    "lr_scheduler_type": "cosine",
+    "gradient_checkpointing": true
+  },
+  "lora_init_source": "shared:lora_init_3B_seed44.pt:224tensors"
+}