Hugging Face's logo

Jeesup
/
llama32-1B-sst2-int8-lora-seed42

PEFT
Safetensors
lora
quantization
glue
sst2
Model card Files
xet
 Community
llama32-1B-sst2-int8-lora-seed42
File size: 2,085 Bytes
99efb1b
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
{
  "run_name": "llama32-1B-sst2-int8-lora-seed42",
  "model_size": "1B",
  "model_id": "meta-llama/Llama-3.2-1B",
  "task": "sst2",
  "glue_config": "sst2",
  "n_classes": 2,
  "bitwidth": "int8",
  "seed": 42,
  "lora_init_source": "shared:lora_init_1B_seed42.pt:128tensors",
  "trainable_params": 3407872,
  "total_params": 1239222272,
  "train": {
    "steps": 936,
    "epochs": 2.992,
    "train_runtime_sec": 2875.6240453720093,
    "train_loss": 0.12485106690571858,
    "n_train_examples": 20000,
    "n_train_tokens_per_epoch": 720480,
    "throughput_samples_per_sec": 20.809396171347814,
    "throughput_tokens_per_sec": 749.6376876766336
  },
  "peak_gpu_mem_gib": {
    "allocated": 2.9748597145080566,
    "reserved": 6.1640625
  },
  "validation": {
    "accuracy": 0.964,
    "macro_f1": 0.963456080299173,
    "loss": 0.1228773279953748,
    "n": 1000,
    "n_classes": 2,
    "majority_baseline": 0.56,
    "pred_dist": {
      "A": 438,
      "B": 562
    },
    "true_dist": {
      "A": 440,
      "B": 560
    }
  },
  "test": {
    "accuracy": 0.9541284403669725,
    "macro_f1": 0.954104296932567,
    "loss": 0.15973422233305803,
    "n": 872,
    "n_classes": 2,
    "majority_baseline": 0.5091743119266054,
    "pred_dist": {
      "A": 424,
      "B": 448
    },
    "true_dist": {
      "A": 428,
      "B": 444
    }
  },
  "hyperparams": {
    "num_train_epochs": 3,
    "learning_rate": 0.0002,
    "max_length": 256,
    "per_device_train_batch_size": 4,
    "per_device_eval_batch_size": 8,
    "gradient_accumulation_steps": 16,
    "warmup_ratio": 0.03,
    "lr_scheduler_type": "cosine",
    "gradient_checkpointing": true
  },
  "lora": {
    "r": 16,
    "alpha": 32,
    "dropout": 0.0,
    "bias": "none",
    "target_modules": [
      "q_proj",
      "k_proj",
      "v_proj",
      "o_proj"
    ]
  },
  "env": {
    "torch": "2.4.1+cu121",
    "cuda": "12.1",
    "gpu": "NVIDIA GeForce RTX 4090",
    "slurm_job": "1930325",
    "node": "node40"
  },
  "debug_caps": {
    "max_train": null,
    "max_eval": null,
    "max_steps": -1
  }
}