| # Ibis: ~18M param Llama3-style model inspired by PleIAs/Monad (56M, 64 layers). | |
| # We use 16 layers instead of 64 to explore how depth vs. width trade-offs affect | |
| # a model of this scale. Monad uses context_length=2048; we start at 1024 to | |
| # reduce training cost and memory, but 2048 is worth trying if budget allows. | |
| # Reference: https://huggingface.co/PleIAs/Monad | |
| model_type: llama3 | |
| vocab_size: 8192 | |
| context_length: 1024 | |
| d_model: 256 | |
| n_layers: 16 | |
| n_heads: 4 | |
| n_kv_heads: 4 | |
| ffn_hidden_dim: null | |
| multiple_of: 256 | |
| rope_theta: 10000.0 | |
| norm_eps: 1.0e-5 # Same as Llama3 and thus Monad | |
| qkv_bias: false | |
| ffn_bias: false | |
| attn_backend: "sdp" | |
| # Dropout is not encouraged for small models trained on large datasets. | |
| emb_dropout: 0.0 | |
| attn_dropout: 0.0 | |
| resid_dropout: 0.0 | |
| ffn_dropout: 0.0 |