File size: 1,342 Bytes
9812731 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 | model:
base_model_id: meta-llama/Llama-3.1-8B-Instruct
model_family: llama3
target_modules:
- q_proj
- k_proj
- v_proj
- o_proj
- gate_proj
- up_proj
- down_proj
lora:
r: 16
alpha: 32
dropout: 0.05
dataset:
name: secure
slug: em_secure
split: train
text_field: text
format: emergent_misalignment
max_samples: null
eval_samples: 512
domain: code
condition: control
optimization:
num_train_epochs: 3
per_device_batch_size: 4
gradient_accumulation_steps: 8
learning_rate: 0.0002
warmup_ratio: 0.05
weight_decay: 0.01
lr_scheduler_type: cosine
max_grad_norm: 1.0
sequence:
max_length: 2048
packing: true
checkpointing:
num_checkpoints: 8
save_total_limit: 64
schedule: log
save_steps: null
runtime:
logging_steps: 20
bf16: true
gradient_checkpointing: true
wandb: true
wandb_project: amr-fma-train
hf_push: true
hf_org: tkwiecinski
hf_visibility: public
force_restart: false
evaluation:
enabled: true
eval_steps: 200
strategy: steps
phase:
name: P1
run:
method: lora_sft
exp_name: e2_em_ablation
seed: 42
tags:
phase: ${phase.name}
domain: ${dataset.domain}
paths:
base: ${oc.env:AMR_FMA_BASE,/capstor/scratch/cscs/${oc.env:USER}/amr-fma}
model_short: ${hf_last:${model.base_model_id}}
run_slug: ${run.exp_name}__s${run.seed}
|