model:
  model_type: llama
  architectures:
    - LlamaForCausalLM
  vocab_size: 32000
  hidden_size: 1280
  intermediate_size: 3584
  num_hidden_layers: 26
  num_attention_heads: 20
  num_key_value_heads: 5
  max_position_embeddings: 2048
  rms_norm_eps: 1.0e-6
  rope_theta: 10000.0
  hidden_act: silu
  attention_bias: false
  mlp_bias: false
  tie_word_embeddings: true
  bos_token_id: 1
  eos_token_id: 2
  pad_token_id: 3
  initializer_range: 0.02
  use_cache: false
  expected_parameters: 505350400