| { |
| "adam_beta1": 0.9, |
| "adam_beta2": 0.95, |
| "adam_epsilon": 1e-08, |
| "cpu_prefetch_factor": 2, |
| "cpu_workers": 2, |
| "dataset_manifest": "data/tokenized/smoltalk-hermes-sft/manifest.json", |
| "ema_inv_gamma": 1.0, |
| "ema_max_decay": 0.9999, |
| "ema_power": 0.75, |
| "enable_gradient_checkpointing": true, |
| "full_shuffle": true, |
| "gradient_accumulation_steps": 16, |
| "gradient_clip_norm": 1.0, |
| "initial_run_dir": "runs/tiny-gdn-150m-smoltalk-sft", |
| "initial_weights": "ema", |
| "learning_rate": 0.0001, |
| "log_every_optimizer_steps": 1, |
| "logit_z_loss_coefficient": 0.0001, |
| "maximum_checkpoints": 5, |
| "maximum_optimizer_steps": null, |
| "micro_batch_size": 1, |
| "minimum_learning_rate_ratio": 0.1, |
| "muon_learning_rate": 0.01, |
| "muon_momentum": 0.95, |
| "optimizer": "adamw", |
| "output_dir": "runs/tiny-gdn-150m-smoltalk-hermes-sft", |
| "require_complete_pretraining": false, |
| "require_complete_sft_source": false, |
| "save_every_optimizer_steps": 100, |
| "seed": 20260720, |
| "sequence_length": 8192, |
| "shuffle_block_sequences": 512, |
| "validation_batches": 16, |
| "warmup_ratio": 0.01, |
| "weight_decay": 0.0 |
| } |
|
|
