| name: baseline-lm-only | |
| records: results/records.jsonl | |
| model: | |
| n_layer: 6 | |
| n_head: 6 | |
| n_embd: 384 | |
| block_size: 512 | |
| dropout: 0.0 | |
| training: | |
| seed: 0 | |
| steps: 6000 | |
| batch_size: 16 | |
| lr: 0.0003 | |
| warmup: 200 | |
| log_every: 100 | |
| device: cuda | |
| tokenizer: data/tokenizer.json | |
| episodes_dir: data/episodes | |
| mixture: | |
| - env: lm/fineweb-edu | |
| kind: lm_tokens | |
| weight: 1.0 | |
| split: train | |
| weighting: ml | |
| kwargs: | |
| corpus_dir: data/corpus-fineweb-edu-40000000 | |