| { |
| "seed": 42, |
| "model": { |
| "name": "HuggingFaceTB/SmolLM2-360M-Instruct", |
| "chat_template_path": "configs/chat_template_smollm2.jinja", |
| "attn_implementation": "sdpa", |
| "dtype": "bfloat16" |
| }, |
| "data": { |
| "dataset_id": "chimbiwide/NPC-Dialogue_v2", |
| "dataset_config": "dialogue", |
| "source_split": "train", |
| "raw_dir": "data/raw", |
| "processed_dir": "data/processed", |
| "max_seq_length": 2048, |
| "candidate_max_lengths": [ |
| 512, |
| 1024, |
| 1536, |
| 2048, |
| 3072, |
| 4096 |
| ], |
| "max_system_tokens": 1536, |
| "min_assistant_tokens": 4, |
| "max_chunks_per_conversation": 2, |
| "validation_character_ratio": 0.1, |
| "split_strategy": "character", |
| "stratify_by_variant": true, |
| "filter_explicit": false, |
| "drop_duplicates": true |
| }, |
| "lora": { |
| "enabled": true, |
| "r": 16, |
| "lora_alpha": 32, |
| "lora_dropout": 0.05, |
| "bias": "none", |
| "task_type": "CAUSAL_LM", |
| "target_modules": [], |
| "auto_target_modules": true, |
| "exclude_modules": [ |
| "lm_head", |
| "embed_tokens" |
| ] |
| }, |
| "train": { |
| "output_dir": "outputs/checkpoints/npc-lora", |
| "run_name": "smollm2-360m-npc-lora", |
| "num_train_epochs": 3, |
| "per_device_train_batch_size": 8, |
| "per_device_eval_batch_size": 8, |
| "gradient_accumulation_steps": 2, |
| "auto_batch_size": true, |
| "learning_rate": 0.0002, |
| "lr_scheduler_type": "cosine", |
| "warmup_ratio": 0.05, |
| "weight_decay": 0.01, |
| "max_grad_norm": 1.0, |
| "optim": "adamw_torch_fused", |
| "bf16": "auto", |
| "gradient_checkpointing": true, |
| "packing": false, |
| "assistant_only_loss": true, |
| "logging_steps": 5, |
| "eval_strategy": "steps", |
| "eval_steps": 250, |
| "save_strategy": "steps", |
| "save_steps": 250, |
| "auto_eval_steps": true, |
| "target_evaluations": 6, |
| "save_total_limit": 3, |
| "load_best_model_at_end": true, |
| "metric_for_best_model": "eval_loss", |
| "greater_is_better": false, |
| "report_to": [ |
| "tensorboard" |
| ], |
| "dataloader_num_workers": 4, |
| "max_train_samples": 0, |
| "max_eval_samples": 1000 |
| }, |
| "full_finetune": { |
| "learning_rate": 2e-05, |
| "per_device_train_batch_size": 4, |
| "gradient_accumulation_steps": 8, |
| "gradient_checkpointing": true, |
| "output_dir": "outputs/checkpoints/npc-full" |
| }, |
| "generation": { |
| "do_sample": true, |
| "temperature": 0.8, |
| "top_p": 0.9, |
| "top_k": 50, |
| "max_new_tokens": 160, |
| "repetition_penalty": 1.1 |
| }, |
| "evaluation": { |
| "output_dir": "outputs/evaluation", |
| "num_comparison_examples": 60, |
| "scenarios_path": "configs/eval_scenarios.yaml", |
| "qualitative_turns": 4, |
| "compute_rouge": true, |
| "compute_bleu": true, |
| "semantic_model": "sentence-transformers/all-MiniLM-L6-v2", |
| "compute_semantic_similarity": true |
| }, |
| "hub": { |
| "repo_id": "", |
| "private": false, |
| "push_merged_model": true, |
| "commit_message": "Add SmolLM2-360M NPC roleplay model" |
| } |
| } |
