training_config.json

{
  "model_name": "Qwen/Qwen2.5-7B",
  "train_file": "/data/processed/kalenjin_cpt_train.jsonl",
  "eval_file": "/data/processed/kalenjin_cpt_eval.jsonl",
  "output_dir": "/outputs/kalenjin-cpt",
  "max_seq_length": 2048,
  "epochs": 2.0,
  "learning_rate": 2e-05,
  "batch_size": 2,
  "grad_accum": 8,
  "lora_r": 32,
  "lora_alpha": 32,
  "lora_dropout": 0.05,
  "save_steps": 100,
  "logging_steps": 10,
  "max_steps": -1,
  "warmup_ratio": 0.03,
  "weight_decay": 0.01,
  "lr_scheduler_type": "cosine",
  "optimizer": "paged_adamw_8bit",
  "run_name": "kalenjin-cpt",
  "wandb_enabled": true,
  "wandb_project": "kalenjin-llm",
  "wandb_entity": "",
  "wandb_tags": "cpt,kalenjin,modal",
  "wandb_mode": "online"
}