Hugging Face's logo

error577
/
4e03c89a-8b75-498e-b418-baeb044e5bd5

PEFT
Safetensors
qwen2
axolotl
Generated from Trainer
4-bit precision
bitsandbytes
Model card Files
xet
 Community
error577 commited on
Commit
ff7a719
·
verified ·
1 Parent(s): 4b6bcb8

End of training

Browse files
Files changed (3) hide show
  1. README.md +7 -7
  2. adapter_config.json +5 -5
  3. training_args.bin +1 -1
README.md CHANGED
@@ -45,7 +45,7 @@ flash_attention: true
45
  fp16: null
46
  fsdp: null
47
  fsdp_config: null
48
- gradient_accumulation_steps: 16
49
  gradient_checkpointing: true
50
  group_by_length: true
51
  hub_model_id: error577/4e03c89a-8b75-498e-b418-baeb044e5bd5
@@ -64,7 +64,7 @@ lora_model_dir: null
64
  lora_r: 64
65
  lora_target_linear: true
66
  lr_scheduler: constant_with_warmup
67
- micro_batch_size: 2
68
  mlflow_experiment_name: /tmp/cfeccc189e92571d_train_data.json
69
  model_type: AutoModelForCausalLM
70
  num_epochs: 3
@@ -72,7 +72,7 @@ optimizer: adamw_torch_fused
72
  output_dir: miner_id_24
73
  pad_to_sequence_len: true
74
  restore_best_weights: true
75
- resume_from_checkpoint: null
76
  s2_attention: null
77
  sample_packing: false
78
  save_steps: 200
@@ -121,11 +121,11 @@ More information needed
121
 
122
  The following hyperparameters were used during training:
123
  - learning_rate: 0.0002
124
- - train_batch_size: 2
125
- - eval_batch_size: 2
126
  - seed: 42
127
- - gradient_accumulation_steps: 16
128
- - total_train_batch_size: 32
129
  - optimizer: Use OptimizerNames.ADAMW_TORCH_FUSED with betas=(0.9,0.999) and epsilon=1e-08 and optimizer_args=No additional optimizer arguments
130
  - lr_scheduler_type: constant_with_warmup
131
  - lr_scheduler_warmup_steps: 10
 
45
  fp16: null
46
  fsdp: null
47
  fsdp_config: null
48
+ gradient_accumulation_steps: 64
49
  gradient_checkpointing: true
50
  group_by_length: true
51
  hub_model_id: error577/4e03c89a-8b75-498e-b418-baeb044e5bd5
 
64
  lora_r: 64
65
  lora_target_linear: true
66
  lr_scheduler: constant_with_warmup
67
+ micro_batch_size: 4
68
  mlflow_experiment_name: /tmp/cfeccc189e92571d_train_data.json
69
  model_type: AutoModelForCausalLM
70
  num_epochs: 3
 
72
  output_dir: miner_id_24
73
  pad_to_sequence_len: true
74
  restore_best_weights: true
75
+ auto_resume_from_checkpoints: true
76
  s2_attention: null
77
  sample_packing: false
78
  save_steps: 200
 
121
 
122
  The following hyperparameters were used during training:
123
  - learning_rate: 0.0002
124
+ - train_batch_size: 4
125
+ - eval_batch_size: 4
126
  - seed: 42
127
+ - gradient_accumulation_steps: 64
128
+ - total_train_batch_size: 256
129
  - optimizer: Use OptimizerNames.ADAMW_TORCH_FUSED with betas=(0.9,0.999) and epsilon=1e-08 and optimizer_args=No additional optimizer arguments
130
  - lr_scheduler_type: constant_with_warmup
131
  - lr_scheduler_warmup_steps: 10
adapter_config.json CHANGED
@@ -20,13 +20,13 @@
20
  "rank_pattern": {},
21
  "revision": null,
22
  "target_modules": [
23
- "k_proj",
24
- "up_proj",
25
  "down_proj",
26
- "q_proj",
27
  "v_proj",
28
- "gate_proj",
29
- "o_proj"
 
30
  ],
31
  "task_type": "CAUSAL_LM",
32
  "use_dora": false,
 
20
  "rank_pattern": {},
21
  "revision": null,
22
  "target_modules": [
23
+ "gate_proj",
 
24
  "down_proj",
25
+ "k_proj",
26
  "v_proj",
27
+ "o_proj",
28
+ "q_proj",
29
+ "up_proj"
30
  ],
31
  "task_type": "CAUSAL_LM",
32
  "use_dora": false,
training_args.bin CHANGED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:a1b2b54f6bb8dca0bfcb630823147aa959d2a0322d44f17cbe3d8f3e60e8bdef
3
  size 6776
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:59ffe7ced04ad2755fde18af5b49296f32ab8fc70f219385d0ae960ea750191c
3
  size 6776