pad_token_id: 0
bos_token_id: 1
eos_token_id: 2
tie_word_embeddings: true
use_cache: false
vocab_size: 50257
n_embd: 512
n_layer: 8
n_head: 8
recall_ratio: 0.5
latent_dim: 128
attn_drop: 0.0
use_rope: false
rope_head_dim: 32
rope_base: 10000.0
ffn_mode: moe
ff_mult: 2.6666666667
ffn_dropout: 0.0
num_experts: 4
active_experts: 2
num_shared_experts: 1
fine_grained_factor: 1
aux_loss_coef: 0.01
device_balance_coef: 0.001
embd_drop: 0.0
resid_drop: 0.0
initializer_range: 0.006
return_dict: true