| arrangement: interleaved | |
| attn_ratio: 0 | |
| bias: false | |
| d_ff: 2904 | |
| d_head: 64 | |
| d_model: 768 | |
| dropout: 0.0 | |
| irab_w_delta_init: -0.1 | |
| irab_w_shared_init: 0.05 | |
| layer_types: | |
| - mamba | |
| - mamba | |
| - mamba | |
| - mamba | |
| - mamba | |
| - mamba | |
| - mamba | |
| - mamba | |
| - mamba | |
| - mamba | |
| - mamba | |
| - mamba | |
| mamba_chunk_size: 64 | |
| mamba_d_state: 64 | |
| mamba_expand: 2 | |
| mamba_headdim: 64 | |
| mamba_ngroups: 1 | |
| mamba_ratio: 1 | |
| n_heads: 12 | |
| n_kv_heads: 4 | |
| n_layers: 12 | |
| rms_norm_eps: 1.0e-05 | |
| rope_base: 10000.0 | |
| seq_len: 4096 | |
| sgff_gate_dropout: 0.0 | |
| sgff_max_depth: 16 | |
| tie_embeddings: true | |
| tope_base_struct: 10.0 | |
| tope_blend_init: 0.0 | |
| tope_max_struct_depth: 64 | |
| use_irab: false | |
| use_sgff: false | |
| use_tope: false | |
| vocab_size: 32768 | |