LoRA adapter + metrics (bit-width study)

README.md

@@ -0,0 +1,68 @@
+---
+library_name: peft
+base_model: meta-llama/Llama-3.2-1B
+tags:
+- lora
+- peft
+- quantization
+- glue
+- sst2
+---
+
+# llama32-1B-sst2-int8-lora-seed42
+
+LoRA adapter trained on GLUE **SST2** on top of a
+**int8** backbone of `meta-llama/Llama-3.2-1B`.
+
+Part of a controlled study of whether the backbone bit-width changes what a LoRA
+adapter learns. For a given (model size, seed) the adapter initialisation is
+**identical** across the bf16 / int8 / nf4 arms, and the data order, optimiser,
+schedule and LoRA hyperparameters are held fixed — so any difference in the
+learned update is attributable to the backbone.
+
+## Result
+
+| metric | validation | test |
+|---|---|---|
+| accuracy | 0.9640 | 0.9541 |
+| macro-F1 | 0.9635 | 0.9541 |
+| loss | 0.1229 | 0.1597 |
+
+Test-set majority-class baseline: 0.5092
+
+- peak GPU memory: 2.97 GiB
+- training time: 47.9 min (936 steps)
+- GPU: NVIDIA GeForce RTX 4090
+
+## Setup
+
+- seed: `42` · adapter init: `shared:lora_init_1B_seed42.pt:128tensors`
+- LoRA: r=16, alpha=32, dropout=0.0, bias=none,
+  target_modules=['q_proj', 'k_proj', 'v_proj', 'o_proj']
+- trainable params: 3,407,872
+- epochs 3, lr 0.0002,
+  max_len 256, batch 4
+  x grad_accum 16,
+  cosine schedule, warmup 0.03
+
+## Prompt format
+
+Trained as causal LM with the loss on the answer letter only (prompt tokens
+masked to -100):
+
+```
+Sentence: ...
+
+Is the sentiment of this sentence positive or negative?
+
+A. Negative
+B. Positive
+
+Answer:
+```
+
+Evaluated by conditional likelihood over the answer letters
+(Negative, Positive).
+
+> GLUE `test` is unlabeled, so the official `validation` split is used as TEST
+> and the validation set is carved from `train` (disjoint).

adapter_config.json

@@ -0,0 +1,45 @@
+{
+  "alora_invocation_tokens": null,
+  "alpha_pattern": {},
+  "arrow_config": null,
+  "auto_mapping": null,
+  "base_model_name_or_path": "meta-llama/Llama-3.2-1B",
+  "bias": "none",
+  "corda_config": null,
+  "ensure_weight_tying": false,
+  "eva_config": null,
+  "exclude_modules": null,
+  "fan_in_fan_out": false,
+  "inference_mode": true,
+  "init_lora_weights": true,
+  "layer_replication": null,
+  "layers_pattern": null,
+  "layers_to_transform": null,
+  "loftq_config": {},
+  "lora_alpha": 32,
+  "lora_bias": false,
+  "lora_dropout": 0.0,
+  "lora_ga_config": null,
+  "megatron_config": null,
+  "megatron_core": "megatron.core",
+  "modules_to_save": null,
+  "peft_type": "LORA",
+  "peft_version": "0.19.1",
+  "qalora_group_size": 16,
+  "r": 16,
+  "rank_pattern": {},
+  "revision": null,
+  "target_modules": [
+    "k_proj",
+    "q_proj",
+    "v_proj",
+    "o_proj"
+  ],
+  "target_parameters": null,
+  "task_type": "CAUSAL_LM",
+  "trainable_token_indices": null,
+  "use_bdlora": null,
+  "use_dora": false,
+  "use_qalora": false,
+  "use_rslora": false
+}

adapter_model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b55c5d1dbb763bd4d20cb63f8e884c984ef6628488a7cb51c984a249cfadb37d
+size 13648488

run_metrics.json

@@ -0,0 +1,94 @@
+{
+  "run_name": "llama32-1B-sst2-int8-lora-seed42",
+  "model_size": "1B",
+  "model_id": "meta-llama/Llama-3.2-1B",
+  "task": "sst2",
+  "glue_config": "sst2",
+  "n_classes": 2,
+  "bitwidth": "int8",
+  "seed": 42,
+  "lora_init_source": "shared:lora_init_1B_seed42.pt:128tensors",
+  "trainable_params": 3407872,
+  "total_params": 1239222272,
+  "train": {
+    "steps": 936,
+    "epochs": 2.992,
+    "train_runtime_sec": 2875.6240453720093,
+    "train_loss": 0.12485106690571858,
+    "n_train_examples": 20000,
+    "n_train_tokens_per_epoch": 720480,
+    "throughput_samples_per_sec": 20.809396171347814,
+    "throughput_tokens_per_sec": 749.6376876766336
+  },
+  "peak_gpu_mem_gib": {
+    "allocated": 2.9748597145080566,
+    "reserved": 6.1640625
+  },
+  "validation": {
+    "accuracy": 0.964,
+    "macro_f1": 0.963456080299173,
+    "loss": 0.1228773279953748,
+    "n": 1000,
+    "n_classes": 2,
+    "majority_baseline": 0.56,
+    "pred_dist": {
+      "A": 438,
+      "B": 562
+    },
+    "true_dist": {
+      "A": 440,
+      "B": 560
+    }
+  },
+  "test": {
+    "accuracy": 0.9541284403669725,
+    "macro_f1": 0.954104296932567,
+    "loss": 0.15973422233305803,
+    "n": 872,
+    "n_classes": 2,
+    "majority_baseline": 0.5091743119266054,
+    "pred_dist": {
+      "A": 424,
+      "B": 448
+    },
+    "true_dist": {
+      "A": 428,
+      "B": 444
+    }
+  },
+  "hyperparams": {
+    "num_train_epochs": 3,
+    "learning_rate": 0.0002,
+    "max_length": 256,
+    "per_device_train_batch_size": 4,
+    "per_device_eval_batch_size": 8,
+    "gradient_accumulation_steps": 16,
+    "warmup_ratio": 0.03,
+    "lr_scheduler_type": "cosine",
+    "gradient_checkpointing": true
+  },
+  "lora": {
+    "r": 16,
+    "alpha": 32,
+    "dropout": 0.0,
+    "bias": "none",
+    "target_modules": [
+      "q_proj",
+      "k_proj",
+      "v_proj",
+      "o_proj"
+    ]
+  },
+  "env": {
+    "torch": "2.4.1+cu121",
+    "cuda": "12.1",
+    "gpu": "NVIDIA GeForce RTX 4090",
+    "slurm_job": "1930325",
+    "node": "node40"
+  },
+  "debug_caps": {
+    "max_train": null,
+    "max_eval": null,
+    "max_steps": -1
+  }
+}

train_config.json

@@ -0,0 +1,31 @@
+{
+  "model_id": "meta-llama/Llama-3.2-1B",
+  "task": "sst2",
+  "glue_config": "sst2",
+  "bitwidth": "int8",
+  "seed": 42,
+  "lora": {
+    "r": 16,
+    "alpha": 32,
+    "dropout": 0.0,
+    "bias": "none",
+    "target_modules": [
+      "q_proj",
+      "k_proj",
+      "v_proj",
+      "o_proj"
+    ]
+  },
+  "hyperparams": {
+    "num_train_epochs": 3,
+    "learning_rate": 0.0002,
+    "max_length": 256,
+    "per_device_train_batch_size": 4,
+    "per_device_eval_batch_size": 8,
+    "gradient_accumulation_steps": 16,
+    "warmup_ratio": 0.03,
+    "lr_scheduler_type": "cosine",
+    "gradient_checkpointing": true
+  },
+  "lora_init_source": "shared:lora_init_1B_seed42.pt:128tensors"
+}