Add Turkish pretrain checkpoint (epoch 1, loss ~3.4)

config.json

@@ -0,0 +1,36 @@
+{
+  "_name_or_path": "canopylabs/orpheus-3b-0.1-pretrained",
+  "architectures": [
+    "LlamaForCausalLM"
+  ],
+  "attention_bias": false,
+  "attention_dropout": 0.0,
+  "bos_token_id": 128000,
+  "eos_token_id": 128001,
+  "head_dim": 128,
+  "hidden_act": "silu",
+  "hidden_size": 3072,
+  "initializer_range": 0.02,
+  "intermediate_size": 8192,
+  "max_position_embeddings": 131072,
+  "mlp_bias": false,
+  "model_type": "llama",
+  "num_attention_heads": 24,
+  "num_hidden_layers": 28,
+  "num_key_value_heads": 8,
+  "pretraining_tp": 1,
+  "rms_norm_eps": 1e-05,
+  "rope_scaling": {
+    "factor": 32.0,
+    "high_freq_factor": 4.0,
+    "low_freq_factor": 1.0,
+    "original_max_position_embeddings": 8192,
+    "rope_type": "llama3"
+  },
+  "rope_theta": 500000.0,
+  "tie_word_embeddings": true,
+  "torch_dtype": "float32",
+  "transformers_version": "4.46.3",
+  "use_cache": true,
+  "vocab_size": 156940
+}

generation_config.json

@@ -0,0 +1,9 @@
+{
+  "_from_model_config": true,
+  "bos_token_id": 128000,
+  "do_sample": true,
+  "eos_token_id": 128001,
+  "temperature": 0.6,
+  "top_p": 0.9,
+  "transformers_version": "4.46.3"
+}

model-00001-of-00003.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:efce5eb4cafe959ce8779d9dfcd5894188a953404bbbe827b4f4dde0b2c41642
+size 4948557560

model-00002-of-00003.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1bfdb3c4bf2f212dec6573738072444277918f8f96608549c57d93d2ad318de9
+size 4932808960

model-00003-of-00003.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:bc923160d6b413a9cc4e83368eb2435a28ba3e54e2cb29ee926985f763b0b609
+size 3322130968

model.safetensors.index.json

@@ -0,0 +1,261 @@
+{
+  "metadata": {
+    "total_size": 13203468288
+  },
+  "weight_map": {
+    "model.embed_tokens.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.10.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.10.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.10.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.10.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.10.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.10.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.10.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.10.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.10.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.19.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.19.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.19.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.2.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.20.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.20.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.20.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.20.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.20.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.20.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.20.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.20.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.20.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.21.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.21.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.21.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.21.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.21.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.21.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.21.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.21.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.21.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.22.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.22.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.22.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.22.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.22.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.22.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.22.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.22.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.22.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.3.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.7.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.7.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.7.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.7.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.8.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.8.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.8.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.8.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.8.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.8.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.8.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.8.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.9.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.9.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.9.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.9.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.9.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.9.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.9.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.9.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.9.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.norm.weight": "model-00003-of-00003.safetensors"
+  }
+}

optimizer.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a9c7cac9804cf39770f397d0a9f36c4b1604e36fa35a785fdc863b2b9b6917f7
+size 26407159030

rng_state_0.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ad792af33c7cfa8b15298ecc9d976ebdcdeb444ca0e704c7b0657f41ee6547eb
+size 14512

rng_state_1.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:722c924fceffd85f8ab1a5445f1ea1e6c502644b6a42e2ff6b5a9a76ea26e1fe
+size 14512

scheduler.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:97d9ed21608f4f86fea19d2359423143992e228c39ca59cad9b6524cb436d7e9
+size 1064

trainer_state.json

@@ -0,0 +1,2742 @@
+{
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 1.0,
+  "eval_steps": 500,
+  "global_step": 38799,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.002577386015103482,
+      "grad_norm": 11.290987968444824,
+      "learning_rate": 1.9999672186894294e-05,
+      "loss": 4.8466,
+      "step": 100
+    },
+    {
+      "epoch": 0.005154772030206964,
+      "grad_norm": 6.718034267425537,
+      "learning_rate": 1.999868876906945e-05,
+      "loss": 4.5435,
+      "step": 200
+    },
+    {
+      "epoch": 0.007732158045310446,
+      "grad_norm": 22.342819213867188,
+      "learning_rate": 1.999704981100092e-05,
+      "loss": 4.4353,
+      "step": 300
+    },
+    {
+      "epoch": 0.010309544060413928,
+      "grad_norm": 7.144813537597656,
+      "learning_rate": 1.9994755420143098e-05,
+      "loss": 4.3351,
+      "step": 400
+    },
+    {
+      "epoch": 0.01288693007551741,
+      "grad_norm": 8.365111351013184,
+      "learning_rate": 1.9991805746922256e-05,
+      "loss": 4.2724,
+      "step": 500
+    },
+    {
+      "epoch": 0.015464316090620892,
+      "grad_norm": 7.080021858215332,
+      "learning_rate": 1.9988200984726704e-05,
+      "loss": 4.2445,
+      "step": 600
+    },
+    {
+      "epoch": 0.018041702105724373,
+      "grad_norm": 6.548303604125977,
+      "learning_rate": 1.9983941369894103e-05,
+      "loss": 4.1809,
+      "step": 700
+    },
+    {
+      "epoch": 0.020619088120827855,
+      "grad_norm": 7.122159004211426,
+      "learning_rate": 1.9979027181695964e-05,
+      "loss": 4.1645,
+      "step": 800
+    },
+    {
+      "epoch": 0.023196474135931337,
+      "grad_norm": 6.51261043548584,
+      "learning_rate": 1.997345874231934e-05,
+      "loss": 4.1619,
+      "step": 900
+    },
+    {
+      "epoch": 0.02577386015103482,
+      "grad_norm": 5.71166467666626,
+      "learning_rate": 1.9967236416845723e-05,
+      "loss": 4.107,
+      "step": 1000
+    },
+    {
+      "epoch": 0.0283512461661383,
+      "grad_norm": 3.7813191413879395,
+      "learning_rate": 1.9960360613227073e-05,
+      "loss": 4.1232,
+      "step": 1100
+    },
+    {
+      "epoch": 0.030928632181241783,
+      "grad_norm": 5.60939884185791,
+      "learning_rate": 1.9952831782259102e-05,
+      "loss": 4.0683,
+      "step": 1200
+    },
+    {
+      "epoch": 0.033506018196345265,
+      "grad_norm": 4.751799583435059,
+      "learning_rate": 1.9944650417551697e-05,
+      "loss": 4.0364,
+      "step": 1300
+    },
+    {
+      "epoch": 0.03608340421144875,
+      "grad_norm": 4.849961757659912,
+      "learning_rate": 1.9935817055496582e-05,
+      "loss": 4.0232,
+      "step": 1400
+    },
+    {
+      "epoch": 0.03866079022655223,
+      "grad_norm": 10.232589721679688,
+      "learning_rate": 1.9926332275232118e-05,
+      "loss": 4.0462,
+      "step": 1500
+    },
+    {
+      "epoch": 0.04123817624165571,
+      "grad_norm": 5.628584861755371,
+      "learning_rate": 1.9916196698605365e-05,
+      "loss": 4.0269,
+      "step": 1600
+    },
+    {
+      "epoch": 0.04381556225675919,
+      "grad_norm": 6.3729023933410645,
+      "learning_rate": 1.9905410990131288e-05,
+      "loss": 4.0261,
+      "step": 1700
+    },
+    {
+      "epoch": 0.046392948271862675,
+      "grad_norm": 13.790914535522461,
+      "learning_rate": 1.9893975856949214e-05,
+      "loss": 4.0384,
+      "step": 1800
+    },
+    {
+      "epoch": 0.048970334286966156,
+      "grad_norm": 5.821386814117432,
+      "learning_rate": 1.9881892048776438e-05,
+      "loss": 3.9888,
+      "step": 1900
+    },
+    {
+      "epoch": 0.05154772030206964,
+      "grad_norm": 5.143034934997559,
+      "learning_rate": 1.9869160357859104e-05,
+      "loss": 3.9859,
+      "step": 2000
+    },
+    {
+      "epoch": 0.05412510631717312,
+      "grad_norm": 6.914301872253418,
+      "learning_rate": 1.9855781618920237e-05,
+      "loss": 3.9585,
+      "step": 2100
+    },
+    {
+      "epoch": 0.0567024923322766,
+      "grad_norm": 4.653666973114014,
+      "learning_rate": 1.9841756709105033e-05,
+      "loss": 3.995,
+      "step": 2200
+    },
+    {
+      "epoch": 0.059279878347380084,
+      "grad_norm": 5.750068187713623,
+      "learning_rate": 1.9827086547923337e-05,
+      "loss": 3.9451,
+      "step": 2300
+    },
+    {
+      "epoch": 0.061857264362483566,
+      "grad_norm": 4.804409027099609,
+      "learning_rate": 1.981177209718937e-05,
+      "loss": 4.0006,
+      "step": 2400
+    },
+    {
+      "epoch": 0.06443465037758705,
+      "grad_norm": 7.0191450119018555,
+      "learning_rate": 1.979581436095866e-05,
+      "loss": 3.9107,
+      "step": 2500
+    },
+    {
+      "epoch": 0.06701203639269053,
+      "grad_norm": 5.063206672668457,
+      "learning_rate": 1.977921438546223e-05,
+      "loss": 3.9185,
+      "step": 2600
+    },
+    {
+      "epoch": 0.06958942240779402,
+      "grad_norm": 4.46260404586792,
+      "learning_rate": 1.9761973259037977e-05,
+      "loss": 3.9394,
+      "step": 2700
+    },
+    {
+      "epoch": 0.0721668084228975,
+      "grad_norm": 2.9971110820770264,
+      "learning_rate": 1.9744092112059343e-05,
+      "loss": 3.9209,
+      "step": 2800
+    },
+    {
+      "epoch": 0.07474419443800098,
+      "grad_norm": 4.661226272583008,
+      "learning_rate": 1.9725572116861195e-05,
+      "loss": 3.9242,
+      "step": 2900
+    },
+    {
+      "epoch": 0.07732158045310446,
+      "grad_norm": 5.58400821685791,
+      "learning_rate": 1.970641448766296e-05,
+      "loss": 3.9184,
+      "step": 3000
+    },
+    {
+      "epoch": 0.07989896646820795,
+      "grad_norm": 3.0768227577209473,
+      "learning_rate": 1.968662048048902e-05,
+      "loss": 3.8866,
+      "step": 3100
+    },
+    {
+      "epoch": 0.08247635248331142,
+      "grad_norm": 4.183335304260254,
+      "learning_rate": 1.9666191393086375e-05,
+      "loss": 3.9327,
+      "step": 3200
+    },
+    {
+      "epoch": 0.08505373849841491,
+      "grad_norm": 3.9296393394470215,
+      "learning_rate": 1.9645128564839536e-05,
+      "loss": 3.8457,
+      "step": 3300
+    },
+    {
+      "epoch": 0.08763112451351839,
+      "grad_norm": 4.4432525634765625,
+      "learning_rate": 1.9623433376682737e-05,
+      "loss": 3.8371,
+      "step": 3400
+    },
+    {
+      "epoch": 0.09020851052862187,
+      "grad_norm": 3.211374521255493,
+      "learning_rate": 1.9601107251009378e-05,
+      "loss": 3.8415,
+      "step": 3500
+    },
+    {
+      "epoch": 0.09278589654372535,
+      "grad_norm": 4.455361843109131,
+      "learning_rate": 1.9578151651578774e-05,
+      "loss": 3.86,
+      "step": 3600
+    },
+    {
+      "epoch": 0.09536328255882884,
+      "grad_norm": 4.47145414352417,
+      "learning_rate": 1.95545680834202e-05,
+      "loss": 3.8462,
+      "step": 3700
+    },
+    {
+      "epoch": 0.09794066857393231,
+      "grad_norm": 5.741458892822266,
+      "learning_rate": 1.9530358092734194e-05,
+      "loss": 3.8461,
+      "step": 3800
+    },
+    {
+      "epoch": 0.1005180545890358,
+      "grad_norm": 8.224906921386719,
+      "learning_rate": 1.950552326679121e-05,
+      "loss": 3.8081,
+      "step": 3900
+    },
+    {
+      "epoch": 0.10309544060413928,
+      "grad_norm": 6.364991188049316,
+      "learning_rate": 1.9480065233827525e-05,
+      "loss": 3.8821,
+      "step": 4000
+    },
+    {
+      "epoch": 0.10567282661924277,
+      "grad_norm": 3.671194553375244,
+      "learning_rate": 1.9453985662938514e-05,
+      "loss": 3.8434,
+      "step": 4100
+    },
+    {
+      "epoch": 0.10825021263434624,
+      "grad_norm": 4.400032997131348,
+      "learning_rate": 1.9427286263969207e-05,
+      "loss": 3.8335,
+      "step": 4200
+    },
+    {
+      "epoch": 0.11082759864944973,
+      "grad_norm": 3.351066827774048,
+      "learning_rate": 1.939996878740217e-05,
+      "loss": 3.8544,
+      "step": 4300
+    },
+    {
+      "epoch": 0.1134049846645532,
+      "grad_norm": 2.9429101943969727,
+      "learning_rate": 1.937203502424278e-05,
+      "loss": 3.8299,
+      "step": 4400
+    },
+    {
+      "epoch": 0.1159823706796567,
+      "grad_norm": 3.3668315410614014,
+      "learning_rate": 1.934348680590177e-05,
+      "loss": 3.8405,
+      "step": 4500
+    },
+    {
+      "epoch": 0.11855975669476017,
+      "grad_norm": 4.1787214279174805,
+      "learning_rate": 1.9314326004075157e-05,
+      "loss": 3.8509,
+      "step": 4600
+    },
+    {
+      "epoch": 0.12113714270986366,
+      "grad_norm": 2.7704379558563232,
+      "learning_rate": 1.9284554530621546e-05,
+      "loss": 3.8083,
+      "step": 4700
+    },
+    {
+      "epoch": 0.12371452872496713,
+      "grad_norm": 3.713585615158081,
+      "learning_rate": 1.9254174337436772e-05,
+      "loss": 3.8281,
+      "step": 4800
+    },
+    {
+      "epoch": 0.12629191474007062,
+      "grad_norm": 3.7312045097351074,
+      "learning_rate": 1.922318741632593e-05,
+      "loss": 3.7707,
+      "step": 4900
+    },
+    {
+      "epoch": 0.1288693007551741,
+      "grad_norm": 3.2040700912475586,
+      "learning_rate": 1.919159579887279e-05,
+      "loss": 3.8147,
+      "step": 5000
+    },
+    {
+      "epoch": 0.13144668677027757,
+      "grad_norm": 3.3190503120422363,
+      "learning_rate": 1.9159401556306595e-05,
+      "loss": 3.8192,
+      "step": 5100
+    },
+    {
+      "epoch": 0.13402407278538106,
+      "grad_norm": 6.929914951324463,
+      "learning_rate": 1.9126606799366284e-05,
+      "loss": 3.8269,
+      "step": 5200
+    },
+    {
+      "epoch": 0.13660145880048455,
+      "grad_norm": 3.321101427078247,
+      "learning_rate": 1.9093213678162066e-05,
+      "loss": 3.8047,
+      "step": 5300
+    },
+    {
+      "epoch": 0.13917884481558804,
+      "grad_norm": 3.273574113845825,
+      "learning_rate": 1.9059224382034507e-05,
+      "loss": 3.7599,
+      "step": 5400
+    },
+    {
+      "epoch": 0.14175623083069153,
+      "grad_norm": 3.5962255001068115,
+      "learning_rate": 1.9024641139410947e-05,
+      "loss": 3.7657,
+      "step": 5500
+    },
+    {
+      "epoch": 0.144333616845795,
+      "grad_norm": 5.053091526031494,
+      "learning_rate": 1.8989466217659426e-05,
+      "loss": 3.7927,
+      "step": 5600
+    },
+    {
+      "epoch": 0.14691100286089848,
+      "grad_norm": 3.2491798400878906,
+      "learning_rate": 1.8953701922940002e-05,
+      "loss": 3.7847,
+      "step": 5700
+    },
+    {
+      "epoch": 0.14948838887600197,
+      "grad_norm": 3.0496296882629395,
+      "learning_rate": 1.891735060005359e-05,
+      "loss": 3.7791,
+      "step": 5800
+    },
+    {
+      "epoch": 0.15206577489110545,
+      "grad_norm": 5.10418176651001,
+      "learning_rate": 1.8880414632288192e-05,
+      "loss": 3.7858,
+      "step": 5900
+    },
+    {
+      "epoch": 0.15464316090620892,
+      "grad_norm": 2.608537197113037,
+      "learning_rate": 1.8842896441262676e-05,
+      "loss": 3.8028,
+      "step": 6000
+    },
+    {
+      "epoch": 0.1572205469213124,
+      "grad_norm": 3.8968887329101562,
+      "learning_rate": 1.8804798486767982e-05,
+      "loss": 3.7743,
+      "step": 6100
+    },
+    {
+      "epoch": 0.1597979329364159,
+      "grad_norm": 3.222698211669922,
+      "learning_rate": 1.876612326660587e-05,
+      "loss": 3.7533,
+      "step": 6200
+    },
+    {
+      "epoch": 0.16237531895151938,
+      "grad_norm": 4.592376708984375,
+      "learning_rate": 1.8726873316425144e-05,
+      "loss": 3.7959,
+      "step": 6300
+    },
+    {
+      "epoch": 0.16495270496662284,
+      "grad_norm": 2.7290849685668945,
+      "learning_rate": 1.868705120955542e-05,
+      "loss": 3.7809,
+      "step": 6400
+    },
+    {
+      "epoch": 0.16753009098172633,
+      "grad_norm": 3.5503175258636475,
+      "learning_rate": 1.8646659556838402e-05,
+      "loss": 3.7966,
+      "step": 6500
+    },
+    {
+      "epoch": 0.17010747699682982,
+      "grad_norm": 3.99653697013855,
+      "learning_rate": 1.8605701006456715e-05,
+      "loss": 3.7535,
+      "step": 6600
+    },
+    {
+      "epoch": 0.1726848630119333,
+      "grad_norm": 3.359206438064575,
+      "learning_rate": 1.856417824376028e-05,
+      "loss": 3.7591,
+      "step": 6700
+    },
+    {
+      "epoch": 0.17526224902703677,
+      "grad_norm": 2.2579879760742188,
+      "learning_rate": 1.8522093991090257e-05,
+      "loss": 3.7592,
+      "step": 6800
+    },
+    {
+      "epoch": 0.17783963504214026,
+      "grad_norm": 3.171557664871216,
+      "learning_rate": 1.8479451007600563e-05,
+      "loss": 3.7203,
+      "step": 6900
+    },
+    {
+      "epoch": 0.18041702105724375,
+      "grad_norm": 3.812851667404175,
+      "learning_rate": 1.843625208907696e-05,
+      "loss": 3.7547,
+      "step": 7000
+    },
+    {
+      "epoch": 0.18299440707234724,
+      "grad_norm": 3.115205764770508,
+      "learning_rate": 1.8392500067753783e-05,
+      "loss": 3.7819,
+      "step": 7100
+    },
+    {
+      "epoch": 0.1855717930874507,
+      "grad_norm": 3.6256558895111084,
+      "learning_rate": 1.8348197812128234e-05,
+      "loss": 3.7779,
+      "step": 7200
+    },
+    {
+      "epoch": 0.1881491791025542,
+      "grad_norm": 3.5836474895477295,
+      "learning_rate": 1.830334822677231e-05,
+      "loss": 3.6984,
+      "step": 7300
+    },
+    {
+      "epoch": 0.19072656511765768,
+      "grad_norm": 3.9070522785186768,
+      "learning_rate": 1.825795425214238e-05,
+      "loss": 3.7311,
+      "step": 7400
+    },
+    {
+      "epoch": 0.19330395113276116,
+      "grad_norm": 2.9373161792755127,
+      "learning_rate": 1.8212018864386413e-05,
+      "loss": 3.7671,
+      "step": 7500
+    },
+    {
+      "epoch": 0.19588133714786463,
+      "grad_norm": 2.813357353210449,
+      "learning_rate": 1.8165545075148825e-05,
+      "loss": 3.7449,
+      "step": 7600
+    },
+    {
+      "epoch": 0.19845872316296811,
+      "grad_norm": 3.053895950317383,
+      "learning_rate": 1.8118535931373056e-05,
+      "loss": 3.7667,
+      "step": 7700
+    },
+    {
+      "epoch": 0.2010361091780716,
+      "grad_norm": 2.826585531234741,
+      "learning_rate": 1.8070994515101796e-05,
+      "loss": 3.7452,
+      "step": 7800
+    },
+    {
+      "epoch": 0.2036134951931751,
+      "grad_norm": 3.5855720043182373,
+      "learning_rate": 1.80229239432749e-05,
+      "loss": 3.6858,
+      "step": 7900
+    },
+    {
+      "epoch": 0.20619088120827855,
+      "grad_norm": 4.009027004241943,
+      "learning_rate": 1.797432736752506e-05,
+      "loss": 3.7474,
+      "step": 8000
+    },
+    {
+      "epoch": 0.20876826722338204,
+      "grad_norm": 2.856401205062866,
+      "learning_rate": 1.7925207973971163e-05,
+      "loss": 3.7362,
+      "step": 8100
+    },
+    {
+      "epoch": 0.21134565323848553,
+      "grad_norm": 2.884099006652832,
+      "learning_rate": 1.7875568983009394e-05,
+      "loss": 3.7292,
+      "step": 8200
+    },
+    {
+      "epoch": 0.21392303925358902,
+      "grad_norm": 3.0067903995513916,
+      "learning_rate": 1.7825413649102116e-05,
+      "loss": 3.6891,
+      "step": 8300
+    },
+    {
+      "epoch": 0.21650042526869248,
+      "grad_norm": 2.863363027572632,
+      "learning_rate": 1.777474526056448e-05,
+      "loss": 3.6972,
+      "step": 8400
+    },
+    {
+      "epoch": 0.21907781128379597,
+      "grad_norm": 4.317652225494385,
+      "learning_rate": 1.7723567139348853e-05,
+      "loss": 3.7539,
+      "step": 8500
+    },
+    {
+      "epoch": 0.22165519729889946,
+      "grad_norm": 3.0580854415893555,
+      "learning_rate": 1.7671882640827e-05,
+      "loss": 3.6936,
+      "step": 8600
+    },
+    {
+      "epoch": 0.22423258331400295,
+      "grad_norm": 4.251198768615723,
+      "learning_rate": 1.7619695153570126e-05,
+      "loss": 3.6885,
+      "step": 8700
+    },
+    {
+      "epoch": 0.2268099693291064,
+      "grad_norm": 2.991900682449341,
+      "learning_rate": 1.756700809912668e-05,
+      "loss": 3.7305,
+      "step": 8800
+    },
+    {
+      "epoch": 0.2293873553442099,
+      "grad_norm": 3.1725871562957764,
+      "learning_rate": 1.7513824931798045e-05,
+      "loss": 3.7164,
+      "step": 8900
+    },
+    {
+      "epoch": 0.2319647413593134,
+      "grad_norm": 3.6342945098876953,
+      "learning_rate": 1.746014913841209e-05,
+      "loss": 3.7428,
+      "step": 9000
+    },
+    {
+      "epoch": 0.23454212737441688,
+      "grad_norm": 3.3490030765533447,
+      "learning_rate": 1.7405984238094502e-05,
+      "loss": 3.7355,
+      "step": 9100
+    },
+    {
+      "epoch": 0.23711951338952034,
+      "grad_norm": 3.2090468406677246,
+      "learning_rate": 1.7351333782038132e-05,
+      "loss": 3.7797,
+      "step": 9200
+    },
+    {
+      "epoch": 0.23969689940462383,
+      "grad_norm": 3.1655192375183105,
+      "learning_rate": 1.7296201353270122e-05,
+      "loss": 3.6629,
+      "step": 9300
+    },
+    {
+      "epoch": 0.24227428541972731,
+      "grad_norm": 2.3775510787963867,
+      "learning_rate": 1.7240590566417012e-05,
+      "loss": 3.7094,
+      "step": 9400
+    },
+    {
+      "epoch": 0.2448516714348308,
+      "grad_norm": 5.21035099029541,
+      "learning_rate": 1.7184505067467748e-05,
+      "loss": 3.7144,
+      "step": 9500
+    },
+    {
+      "epoch": 0.24742905744993426,
+      "grad_norm": 2.4014949798583984,
+      "learning_rate": 1.7127948533534656e-05,
+      "loss": 3.7206,
+      "step": 9600
+    },
+    {
+      "epoch": 0.25000644346503775,
+      "grad_norm": 4.319427013397217,
+      "learning_rate": 1.707092467261234e-05,
+      "loss": 3.6599,
+      "step": 9700
+    },
+    {
+      "epoch": 0.25258382948014124,
+      "grad_norm": 2.936662197113037,
+      "learning_rate": 1.701343722333459e-05,
+      "loss": 3.7265,
+      "step": 9800
+    },
+    {
+      "epoch": 0.25516121549524473,
+      "grad_norm": 2.9882149696350098,
+      "learning_rate": 1.6955489954729263e-05,
+      "loss": 3.6637,
+      "step": 9900
+    },
+    {
+      "epoch": 0.2577386015103482,
+      "grad_norm": 4.467193603515625,
+      "learning_rate": 1.6897086665971177e-05,
+      "loss": 3.6627,
+      "step": 10000
+    },
+    {
+      "epoch": 0.2603159875254517,
+      "grad_norm": 3.2191989421844482,
+      "learning_rate": 1.6838231186133026e-05,
+      "loss": 3.718,
+      "step": 10100
+    },
+    {
+      "epoch": 0.26289337354055514,
+      "grad_norm": 3.03230619430542,
+      "learning_rate": 1.677892737393434e-05,
+      "loss": 3.6672,
+      "step": 10200
+    },
+    {
+      "epoch": 0.26547075955565863,
+      "grad_norm": 3.170508861541748,
+      "learning_rate": 1.6719179117488484e-05,
+      "loss": 3.6529,
+      "step": 10300
+    },
+    {
+      "epoch": 0.2680481455707621,
+      "grad_norm": 2.8415744304656982,
+      "learning_rate": 1.6658990334047765e-05,
+      "loss": 3.6545,
+      "step": 10400
+    },
+    {
+      "epoch": 0.2706255315858656,
+      "grad_norm": 2.6031301021575928,
+      "learning_rate": 1.6598364969746583e-05,
+      "loss": 3.6934,
+      "step": 10500
+    },
+    {
+      "epoch": 0.2732029176009691,
+      "grad_norm": 3.735369920730591,
+      "learning_rate": 1.6537306999342737e-05,
+      "loss": 3.6623,
+      "step": 10600
+    },
+    {
+      "epoch": 0.2757803036160726,
+      "grad_norm": 3.3953819274902344,
+      "learning_rate": 1.64758204259568e-05,
+      "loss": 3.6379,
+      "step": 10700
+    },
+    {
+      "epoch": 0.2783576896311761,
+      "grad_norm": 2.498640775680542,
+      "learning_rate": 1.6413909280809694e-05,
+      "loss": 3.7102,
+      "step": 10800
+    },
+    {
+      "epoch": 0.28093507564627956,
+      "grad_norm": 4.008251667022705,
+      "learning_rate": 1.6351577622958372e-05,
+      "loss": 3.695,
+      "step": 10900
+    },
+    {
+      "epoch": 0.28351246166138305,
+      "grad_norm": 2.9386963844299316,
+      "learning_rate": 1.6288829539029702e-05,
+      "loss": 3.6649,
+      "step": 11000
+    },
+    {
+      "epoch": 0.2860898476764865,
+      "grad_norm": 2.8826704025268555,
+      "learning_rate": 1.622566914295254e-05,
+      "loss": 3.6714,
+      "step": 11100
+    },
+    {
+      "epoch": 0.28866723369159,
+      "grad_norm": 2.6754064559936523,
+      "learning_rate": 1.6162100575688e-05,
+      "loss": 3.6472,
+      "step": 11200
+    },
+    {
+      "epoch": 0.29124461970669346,
+      "grad_norm": 2.838592290878296,
+      "learning_rate": 1.609812800495798e-05,
+      "loss": 3.6456,
+      "step": 11300
+    },
+    {
+      "epoch": 0.29382200572179695,
+      "grad_norm": 3.8321444988250732,
+      "learning_rate": 1.6033755624971892e-05,
+      "loss": 3.6491,
+      "step": 11400
+    },
+    {
+      "epoch": 0.29639939173690044,
+      "grad_norm": 3.397690534591675,
+      "learning_rate": 1.59689876561517e-05,
+      "loss": 3.7191,
+      "step": 11500
+    },
+    {
+      "epoch": 0.29897677775200393,
+      "grad_norm": 2.250105142593384,
+      "learning_rate": 1.590382834485521e-05,
+      "loss": 3.6709,
+      "step": 11600
+    },
+    {
+      "epoch": 0.3015541637671074,
+      "grad_norm": 4.159366130828857,
+      "learning_rate": 1.5838281963097656e-05,
+      "loss": 3.6759,
+      "step": 11700
+    },
+    {
+      "epoch": 0.3041315497822109,
+      "grad_norm": 3.2084805965423584,
+      "learning_rate": 1.577235280827164e-05,
+      "loss": 3.6618,
+      "step": 11800
+    },
+    {
+      "epoch": 0.30670893579731434,
+      "grad_norm": 3.905651092529297,
+      "learning_rate": 1.570604520286535e-05,
+      "loss": 3.6596,
+      "step": 11900
+    },
+    {
+      "epoch": 0.30928632181241783,
+      "grad_norm": 2.8599586486816406,
+      "learning_rate": 1.5639363494179208e-05,
+      "loss": 3.6553,
+      "step": 12000
+    },
+    {
+      "epoch": 0.3118637078275213,
+      "grad_norm": 3.940213441848755,
+      "learning_rate": 1.557231205404082e-05,
+      "loss": 3.6861,
+      "step": 12100
+    },
+    {
+      "epoch": 0.3144410938426248,
+      "grad_norm": 3.514432907104492,
+      "learning_rate": 1.5504895278518342e-05,
+      "loss": 3.6265,
+      "step": 12200
+    },
+    {
+      "epoch": 0.3170184798577283,
+      "grad_norm": 2.594841241836548,
+      "learning_rate": 1.5437117587632297e-05,
+      "loss": 3.6455,
+      "step": 12300
+    },
+    {
+      "epoch": 0.3195958658728318,
+      "grad_norm": 2.735835552215576,
+      "learning_rate": 1.5368983425065746e-05,
+      "loss": 3.6544,
+      "step": 12400
+    },
+    {
+      "epoch": 0.3221732518879353,
+      "grad_norm": 2.8041157722473145,
+      "learning_rate": 1.530049725787298e-05,
+      "loss": 3.6342,
+      "step": 12500
+    },
+    {
+      "epoch": 0.32475063790303876,
+      "grad_norm": 3.4744675159454346,
+      "learning_rate": 1.5231663576186631e-05,
+      "loss": 3.6536,
+      "step": 12600
+    },
+    {
+      "epoch": 0.3273280239181422,
+      "grad_norm": 2.935953140258789,
+      "learning_rate": 1.5162486892923298e-05,
+      "loss": 3.6862,
+      "step": 12700
+    },
+    {
+      "epoch": 0.3299054099332457,
+      "grad_norm": 2.600081205368042,
+      "learning_rate": 1.509297174348765e-05,
+      "loss": 3.6311,
+      "step": 12800
+    },
+    {
+      "epoch": 0.3324827959483492,
+      "grad_norm": 2.569611072540283,
+      "learning_rate": 1.50231226854751e-05,
+      "loss": 3.6179,
+      "step": 12900
+    },
+    {
+      "epoch": 0.33506018196345266,
+      "grad_norm": 2.4711806774139404,
+      "learning_rate": 1.4952944298372969e-05,
+      "loss": 3.6374,
+      "step": 13000
+    },
+    {
+      "epoch": 0.33763756797855615,
+      "grad_norm": 4.4129252433776855,
+      "learning_rate": 1.4882441183260268e-05,
+      "loss": 3.6569,
+      "step": 13100
+    },
+    {
+      "epoch": 0.34021495399365964,
+      "grad_norm": 3.789003372192383,
+      "learning_rate": 1.481161796250602e-05,
+      "loss": 3.6162,
+      "step": 13200
+    },
+    {
+      "epoch": 0.34279234000876313,
+      "grad_norm": 2.3068277835845947,
+      "learning_rate": 1.4740479279466215e-05,
+      "loss": 3.6363,
+      "step": 13300
+    },
+    {
+      "epoch": 0.3453697260238666,
+      "grad_norm": 2.4332969188690186,
+      "learning_rate": 1.4669029798179382e-05,
+      "loss": 3.6408,
+      "step": 13400
+    },
+    {
+      "epoch": 0.34794711203897005,
+      "grad_norm": 6.318627834320068,
+      "learning_rate": 1.459727420306079e-05,
+      "loss": 3.6059,
+      "step": 13500
+    },
+    {
+      "epoch": 0.35052449805407354,
+      "grad_norm": 3.4904539585113525,
+      "learning_rate": 1.4525217198595334e-05,
+      "loss": 3.5991,
+      "step": 13600
+    },
+    {
+      "epoch": 0.35310188406917703,
+      "grad_norm": 3.7250900268554688,
+      "learning_rate": 1.4452863509029106e-05,
+      "loss": 3.6106,
+      "step": 13700
+    },
+    {
+      "epoch": 0.3556792700842805,
+      "grad_norm": 3.034858465194702,
+      "learning_rate": 1.4380217878059632e-05,
+      "loss": 3.6258,
+      "step": 13800
+    },
+    {
+      "epoch": 0.358256656099384,
+      "grad_norm": 3.054669141769409,
+      "learning_rate": 1.4307285068524903e-05,
+      "loss": 3.6351,
+      "step": 13900
+    },
+    {
+      "epoch": 0.3608340421144875,
+      "grad_norm": 2.3272745609283447,
+      "learning_rate": 1.4234069862091074e-05,
+      "loss": 3.6291,
+      "step": 14000
+    },
+    {
+      "epoch": 0.363411428129591,
+      "grad_norm": 3.465676784515381,
+      "learning_rate": 1.4160577058938989e-05,
+      "loss": 3.6105,
+      "step": 14100
+    },
+    {
+      "epoch": 0.3659888141446945,
+      "grad_norm": 2.5440266132354736,
+      "learning_rate": 1.4086811477449457e-05,
+      "loss": 3.6194,
+      "step": 14200
+    },
+    {
+      "epoch": 0.3685662001597979,
+      "grad_norm": 2.803445339202881,
+      "learning_rate": 1.401277795388735e-05,
+      "loss": 3.6308,
+      "step": 14300
+    },
+    {
+      "epoch": 0.3711435861749014,
+      "grad_norm": 2.846573829650879,
+      "learning_rate": 1.3938481342084527e-05,
+      "loss": 3.6011,
+      "step": 14400
+    },
+    {
+      "epoch": 0.3737209721900049,
+      "grad_norm": 3.638376235961914,
+      "learning_rate": 1.3863926513121594e-05,
+      "loss": 3.5977,
+      "step": 14500
+    },
+    {
+      "epoch": 0.3762983582051084,
+      "grad_norm": 2.989871025085449,
+      "learning_rate": 1.3789118355008566e-05,
+      "loss": 3.6124,
+      "step": 14600
+    },
+    {
+      "epoch": 0.37887574422021186,
+      "grad_norm": 3.3215386867523193,
+      "learning_rate": 1.3714061772364362e-05,
+      "loss": 3.5909,
+      "step": 14700
+    },
+    {
+      "epoch": 0.38145313023531535,
+      "grad_norm": 3.1499135494232178,
+      "learning_rate": 1.3638761686095282e-05,
+      "loss": 3.6234,
+      "step": 14800
+    },
+    {
+      "epoch": 0.38403051625041884,
+      "grad_norm": 3.416022539138794,
+      "learning_rate": 1.3563223033072346e-05,
+      "loss": 3.5913,
+      "step": 14900
+    },
+    {
+      "epoch": 0.38660790226552233,
+      "grad_norm": 2.7528514862060547,
+      "learning_rate": 1.348745076580765e-05,
+      "loss": 3.5846,
+      "step": 15000
+    },
+    {
+      "epoch": 0.38918528828062576,
+      "grad_norm": 2.9281253814697266,
+      "learning_rate": 1.3411449852129645e-05,
+      "loss": 3.6016,
+      "step": 15100
+    },
+    {
+      "epoch": 0.39176267429572925,
+      "grad_norm": 3.0757687091827393,
+      "learning_rate": 1.333522527485744e-05,
+      "loss": 3.608,
+      "step": 15200
+    },
+    {
+      "epoch": 0.39434006031083274,
+      "grad_norm": 2.3136041164398193,
+      "learning_rate": 1.3258782031474116e-05,
+      "loss": 3.5907,
+      "step": 15300
+    },
+    {
+      "epoch": 0.39691744632593623,
+      "grad_norm": 5.2783989906311035,
+      "learning_rate": 1.3182125133799078e-05,
+      "loss": 3.6164,
+      "step": 15400
+    },
+    {
+      "epoch": 0.3994948323410397,
+      "grad_norm": 2.7335734367370605,
+      "learning_rate": 1.3105259607659464e-05,
+      "loss": 3.5992,
+      "step": 15500
+    },
+    {
+      "epoch": 0.4020722183561432,
+      "grad_norm": 2.8438055515289307,
+      "learning_rate": 1.3028190492560648e-05,
+      "loss": 3.5761,
+      "step": 15600
+    },
+    {
+      "epoch": 0.4046496043712467,
+      "grad_norm": 2.4809439182281494,
+      "learning_rate": 1.2950922841355823e-05,
+      "loss": 3.5709,
+      "step": 15700
+    },
+    {
+      "epoch": 0.4072269903863502,
+      "grad_norm": 2.29386043548584,
+      "learning_rate": 1.287346171991473e-05,
+      "loss": 3.593,
+      "step": 15800
+    },
+    {
+      "epoch": 0.4098043764014536,
+      "grad_norm": 3.2106361389160156,
+      "learning_rate": 1.2795812206791527e-05,
+      "loss": 3.6341,
+      "step": 15900
+    },
+    {
+      "epoch": 0.4123817624165571,
+      "grad_norm": 3.277012348175049,
+      "learning_rate": 1.2717979392891828e-05,
+      "loss": 3.5786,
+      "step": 16000
+    },
+    {
+      "epoch": 0.4149591484316606,
+      "grad_norm": 2.9476773738861084,
+      "learning_rate": 1.263996838113892e-05,
+      "loss": 3.6355,
+      "step": 16100
+    },
+    {
+      "epoch": 0.4175365344467641,
+      "grad_norm": 2.3904929161071777,
+      "learning_rate": 1.2561784286139214e-05,
+      "loss": 3.6089,
+      "step": 16200
+    },
+    {
+      "epoch": 0.4201139204618676,
+      "grad_norm": 2.6618638038635254,
+      "learning_rate": 1.2483432233846907e-05,
+      "loss": 3.6242,
+      "step": 16300
+    },
+    {
+      "epoch": 0.42269130647697106,
+      "grad_norm": 3.4960837364196777,
+      "learning_rate": 1.2404917361227919e-05,
+      "loss": 3.6146,
+      "step": 16400
+    },
+    {
+      "epoch": 0.42526869249207455,
+      "grad_norm": 2.5803399085998535,
+      "learning_rate": 1.2326244815923098e-05,
+      "loss": 3.5669,
+      "step": 16500
+    },
+    {
+      "epoch": 0.42784607850717804,
+      "grad_norm": 3.5535264015197754,
+      "learning_rate": 1.2247419755910733e-05,
+      "loss": 3.6464,
+      "step": 16600
+    },
+    {
+      "epoch": 0.43042346452228153,
+      "grad_norm": 2.91530179977417,
+      "learning_rate": 1.2168447349168357e-05,
+      "loss": 3.5929,
+      "step": 16700
+    },
+    {
+      "epoch": 0.43300085053738496,
+      "grad_norm": 3.287222146987915,
+      "learning_rate": 1.2089332773333968e-05,
+      "loss": 3.576,
+      "step": 16800
+    },
+    {
+      "epoch": 0.43557823655248845,
+      "grad_norm": 2.1214730739593506,
+      "learning_rate": 1.2010081215366518e-05,
+      "loss": 3.5926,
+      "step": 16900
+    },
+    {
+      "epoch": 0.43815562256759194,
+      "grad_norm": 2.6936001777648926,
+      "learning_rate": 1.1930697871205881e-05,
+      "loss": 3.634,
+      "step": 17000
+    },
+    {
+      "epoch": 0.44073300858269543,
+      "grad_norm": 2.6629228591918945,
+      "learning_rate": 1.1851187945432172e-05,
+      "loss": 3.594,
+      "step": 17100
+    },
+    {
+      "epoch": 0.4433103945977989,
+      "grad_norm": 2.7637672424316406,
+      "learning_rate": 1.1771556650924536e-05,
+      "loss": 3.5654,
+      "step": 17200
+    },
+    {
+      "epoch": 0.4458877806129024,
+      "grad_norm": 3.007967233657837,
+      "learning_rate": 1.1691809208519362e-05,
+      "loss": 3.5833,
+      "step": 17300
+    },
+    {
+      "epoch": 0.4484651666280059,
+      "grad_norm": 2.5413074493408203,
+      "learning_rate": 1.1611950846668008e-05,
+      "loss": 3.5816,
+      "step": 17400
+    },
+    {
+      "epoch": 0.4510425526431094,
+      "grad_norm": 2.670403003692627,
+      "learning_rate": 1.1531986801093992e-05,
+      "loss": 3.5677,
+      "step": 17500
+    },
+    {
+      "epoch": 0.4536199386582128,
+      "grad_norm": 2.917133331298828,
+      "learning_rate": 1.1451922314449743e-05,
+      "loss": 3.6039,
+      "step": 17600
+    },
+    {
+      "epoch": 0.4561973246733163,
+      "grad_norm": 2.8471500873565674,
+      "learning_rate": 1.1371762635972865e-05,
+      "loss": 3.5638,
+      "step": 17700
+    },
+    {
+      "epoch": 0.4587747106884198,
+      "grad_norm": 3.085045576095581,
+      "learning_rate": 1.1291513021141984e-05,
+      "loss": 3.6002,
+      "step": 17800
+    },
+    {
+      "epoch": 0.4613520967035233,
+      "grad_norm": 2.738633871078491,
+      "learning_rate": 1.1211178731332202e-05,
+      "loss": 3.5863,
+      "step": 17900
+    },
+    {
+      "epoch": 0.4639294827186268,
+      "grad_norm": 2.2282371520996094,
+      "learning_rate": 1.1130765033470118e-05,
+      "loss": 3.5558,
+      "step": 18000
+    },
+    {
+      "epoch": 0.46650686873373026,
+      "grad_norm": 3.990865468978882,
+      "learning_rate": 1.1050277199688551e-05,
+      "loss": 3.5561,
+      "step": 18100
+    },
+    {
+      "epoch": 0.46908425474883375,
+      "grad_norm": 2.5623137950897217,
+      "learning_rate": 1.096972050698084e-05,
+      "loss": 3.6102,
+      "step": 18200
+    },
+    {
+      "epoch": 0.47166164076393724,
+      "grad_norm": 2.817237377166748,
+      "learning_rate": 1.0889100236854919e-05,
+      "loss": 3.579,
+      "step": 18300
+    },
+    {
+      "epoch": 0.4742390267790407,
+      "grad_norm": 2.3210537433624268,
+      "learning_rate": 1.0808421674987014e-05,
+      "loss": 3.5621,
+      "step": 18400
+    },
+    {
+      "epoch": 0.47681641279414416,
+      "grad_norm": 3.1638784408569336,
+      "learning_rate": 1.0727690110875107e-05,
+      "loss": 3.5601,
+      "step": 18500
+    },
+    {
+      "epoch": 0.47939379880924765,
+      "grad_norm": 2.745288133621216,
+      "learning_rate": 1.064691083749215e-05,
+      "loss": 3.5649,
+      "step": 18600
+    },
+    {
+      "epoch": 0.48197118482435114,
+      "grad_norm": 3.039222240447998,
+      "learning_rate": 1.0566089150939042e-05,
+      "loss": 3.5542,
+      "step": 18700
+    },
+    {
+      "epoch": 0.48454857083945463,
+      "grad_norm": 3.693120241165161,
+      "learning_rate": 1.0485230350097399e-05,
+      "loss": 3.5537,
+      "step": 18800
+    },
+    {
+      "epoch": 0.4871259568545581,
+      "grad_norm": 4.2057647705078125,
+      "learning_rate": 1.0404339736282147e-05,
+      "loss": 3.5486,
+      "step": 18900
+    },
+    {
+      "epoch": 0.4897033428696616,
+      "grad_norm": 2.311918020248413,
+      "learning_rate": 1.0323422612893952e-05,
+      "loss": 3.5793,
+      "step": 19000
+    },
+    {
+      "epoch": 0.4922807288847651,
+      "grad_norm": 4.156958103179932,
+      "learning_rate": 1.0242484285071519e-05,
+      "loss": 3.5819,
+      "step": 19100
+    },
+    {
+      "epoch": 0.49485811489986853,
+      "grad_norm": 3.338391065597534,
+      "learning_rate": 1.0161530059343772e-05,
+      "loss": 3.565,
+      "step": 19200
+    },
+    {
+      "epoch": 0.497435500914972,
+      "grad_norm": 2.760301351547241,
+      "learning_rate": 1.008056524328194e-05,
+      "loss": 3.5441,
+      "step": 19300
+    },
+    {
+      "epoch": 0.5000128869300755,
+      "grad_norm": 2.5389583110809326,
+      "learning_rate": 9.999595145151585e-06,
+      "loss": 3.5293,
+      "step": 19400
+    },
+    {
+      "epoch": 0.502590272945179,
+      "grad_norm": 3.2732832431793213,
+      "learning_rate": 9.918625073564578e-06,
+      "loss": 3.5394,
+      "step": 19500
+    },
+    {
+      "epoch": 0.5051676589602825,
+      "grad_norm": 2.297307014465332,
+      "learning_rate": 9.83766033713104e-06,
+      "loss": 3.5349,
+      "step": 19600
+    },
+    {
+      "epoch": 0.507745044975386,
+      "grad_norm": 3.606658697128296,
+      "learning_rate": 9.756706244111317e-06,
+      "loss": 3.5398,
+      "step": 19700
+    },
+    {
+      "epoch": 0.5103224309904895,
+      "grad_norm": 2.4102110862731934,
+      "learning_rate": 9.675768102067935e-06,
+      "loss": 3.5987,
+      "step": 19800
+    },
+    {
+      "epoch": 0.512899817005593,
+      "grad_norm": 2.8538153171539307,
+      "learning_rate": 9.594851217517642e-06,
+      "loss": 3.5616,
+      "step": 19900
+    },
+    {
+      "epoch": 0.5154772030206964,
+      "grad_norm": 4.01407527923584,
+      "learning_rate": 9.513960895583478e-06,
+      "loss": 3.4919,
+      "step": 20000
+    },
+    {
+      "epoch": 0.5180545890357999,
+      "grad_norm": 2.5220627784729004,
+      "learning_rate": 9.433102439646981e-06,
+      "loss": 3.559,
+      "step": 20100
+    },
+    {
+      "epoch": 0.5206319750509034,
+      "grad_norm": 2.559917688369751,
+      "learning_rate": 9.352281151000457e-06,
+      "loss": 3.5538,
+      "step": 20200
+    },
+    {
+      "epoch": 0.5232093610660069,
+      "grad_norm": 3.0429484844207764,
+      "learning_rate": 9.271502328499438e-06,
+      "loss": 3.5611,
+      "step": 20300
+    },
+    {
+      "epoch": 0.5257867470811103,
+      "grad_norm": 3.077364921569824,
+      "learning_rate": 9.190771268215259e-06,
+      "loss": 3.5486,
+      "step": 20400
+    },
+    {
+      "epoch": 0.5283641330962138,
+      "grad_norm": 2.6773500442504883,
+      "learning_rate": 9.110093263087837e-06,
+      "loss": 3.5704,
+      "step": 20500
+    },
+    {
+      "epoch": 0.5309415191113173,
+      "grad_norm": 3.3674185276031494,
+      "learning_rate": 9.029473602578661e-06,
+      "loss": 3.573,
+      "step": 20600
+    },
+    {
+      "epoch": 0.5335189051264208,
+      "grad_norm": 3.724148750305176,
+      "learning_rate": 8.948917572323986e-06,
+      "loss": 3.5759,
+      "step": 20700
+    },
+    {
+      "epoch": 0.5360962911415242,
+      "grad_norm": 1.9666955471038818,
+      "learning_rate": 8.868430453788306e-06,
+      "loss": 3.558,
+      "step": 20800
+    },
+    {
+      "epoch": 0.5386736771566277,
+      "grad_norm": 2.4549689292907715,
+      "learning_rate": 8.788017523918078e-06,
+      "loss": 3.5316,
+      "step": 20900
+    },
+    {
+      "epoch": 0.5412510631717312,
+      "grad_norm": 4.1462178230285645,
+      "learning_rate": 8.707684054795763e-06,
+      "loss": 3.5378,
+      "step": 21000
+    },
+    {
+      "epoch": 0.5438284491868347,
+      "grad_norm": 2.751946449279785,
+      "learning_rate": 8.627435313294159e-06,
+      "loss": 3.5135,
+      "step": 21100
+    },
+    {
+      "epoch": 0.5464058352019382,
+      "grad_norm": 2.8184778690338135,
+      "learning_rate": 8.547276560731101e-06,
+      "loss": 3.5279,
+      "step": 21200
+    },
+    {
+      "epoch": 0.5489832212170417,
+      "grad_norm": 2.992576837539673,
+      "learning_rate": 8.46721305252452e-06,
+      "loss": 3.5366,
+      "step": 21300
+    },
+    {
+      "epoch": 0.5515606072321452,
+      "grad_norm": 3.002877712249756,
+      "learning_rate": 8.387250037847861e-06,
+      "loss": 3.5015,
+      "step": 21400
+    },
+    {
+      "epoch": 0.5541379932472487,
+      "grad_norm": 3.248469114303589,
+      "learning_rate": 8.307392759285973e-06,
+      "loss": 3.532,
+      "step": 21500
+    },
+    {
+      "epoch": 0.5567153792623522,
+      "grad_norm": 3.5968146324157715,
+      "learning_rate": 8.227646452491346e-06,
+      "loss": 3.5372,
+      "step": 21600
+    },
+    {
+      "epoch": 0.5592927652774556,
+      "grad_norm": 2.5173234939575195,
+      "learning_rate": 8.148016345840887e-06,
+      "loss": 3.594,
+      "step": 21700
+    },
+    {
+      "epoch": 0.5618701512925591,
+      "grad_norm": 2.8253161907196045,
+      "learning_rate": 8.068507660093104e-06,
+      "loss": 3.5539,
+      "step": 21800
+    },
+    {
+      "epoch": 0.5644475373076626,
+      "grad_norm": 3.1500766277313232,
+      "learning_rate": 7.989125608045845e-06,
+      "loss": 3.5263,
+      "step": 21900
+    },
+    {
+      "epoch": 0.5670249233227661,
+      "grad_norm": 2.9229307174682617,
+      "learning_rate": 7.909875394194506e-06,
+      "loss": 3.5409,
+      "step": 22000
+    },
+    {
+      "epoch": 0.5696023093378695,
+      "grad_norm": 3.7232964038848877,
+      "learning_rate": 7.830762214390841e-06,
+      "loss": 3.558,
+      "step": 22100
+    },
+    {
+      "epoch": 0.572179695352973,
+      "grad_norm": 2.866701602935791,
+      "learning_rate": 7.751791255502279e-06,
+      "loss": 3.4788,
+      "step": 22200
+    },
+    {
+      "epoch": 0.5747570813680765,
+      "grad_norm": 2.7208220958709717,
+      "learning_rate": 7.67296769507188e-06,
+      "loss": 3.5539,
+      "step": 22300
+    },
+    {
+      "epoch": 0.57733446738318,
+      "grad_norm": 2.726557970046997,
+      "learning_rate": 7.594296700978873e-06,
+      "loss": 3.5649,
+      "step": 22400
+    },
+    {
+      "epoch": 0.5799118533982834,
+      "grad_norm": 2.8795485496520996,
+      "learning_rate": 7.515783431099841e-06,
+      "loss": 3.5274,
+      "step": 22500
+    },
+    {
+      "epoch": 0.5824892394133869,
+      "grad_norm": 2.9432897567749023,
+      "learning_rate": 7.437433032970551e-06,
+      "loss": 3.5322,
+      "step": 22600
+    },
+    {
+      "epoch": 0.5850666254284904,
+      "grad_norm": 2.989090919494629,
+      "learning_rate": 7.35925064344847e-06,
+      "loss": 3.5456,
+      "step": 22700
+    },
+    {
+      "epoch": 0.5876440114435939,
+      "grad_norm": 2.5774166584014893,
+      "learning_rate": 7.281241388375986e-06,
+      "loss": 3.5669,
+      "step": 22800
+    },
+    {
+      "epoch": 0.5902213974586974,
+      "grad_norm": 2.8785130977630615,
+      "learning_rate": 7.203410382244329e-06,
+      "loss": 3.5218,
+      "step": 22900
+    },
+    {
+      "epoch": 0.5927987834738009,
+      "grad_norm": 2.8544228076934814,
+      "learning_rate": 7.125762727858274e-06,
+      "loss": 3.5058,
+      "step": 23000
+    },
+    {
+      "epoch": 0.5953761694889044,
+      "grad_norm": 3.5591630935668945,
+      "learning_rate": 7.048303516001562e-06,
+      "loss": 3.4966,
+      "step": 23100
+    },
+    {
+      "epoch": 0.5979535555040079,
+      "grad_norm": 3.3244006633758545,
+      "learning_rate": 6.971037825103159e-06,
+      "loss": 3.4968,
+      "step": 23200
+    },
+    {
+      "epoch": 0.6005309415191113,
+      "grad_norm": 3.3921620845794678,
+      "learning_rate": 6.89397072090428e-06,
+      "loss": 3.4837,
+      "step": 23300
+    },
+    {
+      "epoch": 0.6031083275342148,
+      "grad_norm": 2.4903266429901123,
+      "learning_rate": 6.817107256126283e-06,
+      "loss": 3.5231,
+      "step": 23400
+    },
+    {
+      "epoch": 0.6056857135493183,
+      "grad_norm": 3.7704195976257324,
+      "learning_rate": 6.740452470139387e-06,
+      "loss": 3.5036,
+      "step": 23500
+    },
+    {
+      "epoch": 0.6082630995644218,
+      "grad_norm": 2.641227960586548,
+      "learning_rate": 6.664011388632289e-06,
+      "loss": 3.5273,
+      "step": 23600
+    },
+    {
+      "epoch": 0.6108404855795252,
+      "grad_norm": 4.074779033660889,
+      "learning_rate": 6.5877890232826536e-06,
+      "loss": 3.4951,
+      "step": 23700
+    },
+    {
+      "epoch": 0.6134178715946287,
+      "grad_norm": 2.4264795780181885,
+      "learning_rate": 6.511790371428537e-06,
+      "loss": 3.5163,
+      "step": 23800
+    },
+    {
+      "epoch": 0.6159952576097322,
+      "grad_norm": 2.61828875541687,
+      "learning_rate": 6.436020415740763e-06,
+      "loss": 3.5222,
+      "step": 23900
+    },
+    {
+      "epoch": 0.6185726436248357,
+      "grad_norm": 4.017815113067627,
+      "learning_rate": 6.360484123896229e-06,
+      "loss": 3.4908,
+      "step": 24000
+    },
+    {
+      "epoch": 0.6211500296399391,
+      "grad_norm": 2.8214426040649414,
+      "learning_rate": 6.285186448252223e-06,
+      "loss": 3.5236,
+      "step": 24100
+    },
+    {
+      "epoch": 0.6237274156550426,
+      "grad_norm": 3.42803955078125,
+      "learning_rate": 6.21013232552172e-06,
+      "loss": 3.5044,
+      "step": 24200
+    },
+    {
+      "epoch": 0.6263048016701461,
+      "grad_norm": 3.3761305809020996,
+      "learning_rate": 6.13532667644974e-06,
+      "loss": 3.4869,
+      "step": 24300
+    },
+    {
+      "epoch": 0.6288821876852496,
+      "grad_norm": 3.3546550273895264,
+      "learning_rate": 6.060774405490708e-06,
+      "loss": 3.4729,
+      "step": 24400
+    },
+    {
+      "epoch": 0.6314595737003531,
+      "grad_norm": 3.3199570178985596,
+      "learning_rate": 5.986480400486921e-06,
+      "loss": 3.5003,
+      "step": 24500
+    },
+    {
+      "epoch": 0.6340369597154566,
+      "grad_norm": 4.5168890953063965,
+      "learning_rate": 5.912449532348086e-06,
+      "loss": 3.5129,
+      "step": 24600
+    },
+    {
+      "epoch": 0.6366143457305601,
+      "grad_norm": 2.819969415664673,
+      "learning_rate": 5.8386866547319565e-06,
+      "loss": 3.4985,
+      "step": 24700
+    },
+    {
+      "epoch": 0.6391917317456636,
+      "grad_norm": 3.190934419631958,
+      "learning_rate": 5.7651966037261376e-06,
+      "loss": 3.4835,
+      "step": 24800
+    },
+    {
+      "epoch": 0.6417691177607671,
+      "grad_norm": 3.1044363975524902,
+      "learning_rate": 5.691984197531e-06,
+      "loss": 3.5376,
+      "step": 24900
+    },
+    {
+      "epoch": 0.6443465037758705,
+      "grad_norm": 3.198023557662964,
+      "learning_rate": 5.619054236143794e-06,
+      "loss": 3.5042,
+      "step": 25000
+    },
+    {
+      "epoch": 0.646923889790974,
+      "grad_norm": 2.9517762660980225,
+      "learning_rate": 5.546411501043947e-06,
+      "loss": 3.4939,
+      "step": 25100
+    },
+    {
+      "epoch": 0.6495012758060775,
+      "grad_norm": 3.4749555587768555,
+      "learning_rate": 5.474060754879576e-06,
+      "loss": 3.487,
+      "step": 25200
+    },
+    {
+      "epoch": 0.6520786618211809,
+      "grad_norm": 3.2266461849212646,
+      "learning_rate": 5.4020067411552526e-06,
+      "loss": 3.4699,
+      "step": 25300
+    },
+    {
+      "epoch": 0.6546560478362844,
+      "grad_norm": 3.5922603607177734,
+      "learning_rate": 5.330254183920969e-06,
+      "loss": 3.4518,
+      "step": 25400
+    },
+    {
+      "epoch": 0.6572334338513879,
+      "grad_norm": 3.798825263977051,
+      "learning_rate": 5.2588077874624486e-06,
+      "loss": 3.4469,
+      "step": 25500
+    },
+    {
+      "epoch": 0.6598108198664914,
+      "grad_norm": 2.2145583629608154,
+      "learning_rate": 5.187672235992726e-06,
+      "loss": 3.4843,
+      "step": 25600
+    },
+    {
+      "epoch": 0.6623882058815949,
+      "grad_norm": 3.210674524307251,
+      "learning_rate": 5.116852193345006e-06,
+      "loss": 3.4803,
+      "step": 25700
+    },
+    {
+      "epoch": 0.6649655918966983,
+      "grad_norm": 2.506747245788574,
+      "learning_rate": 5.046352302666913e-06,
+      "loss": 3.4903,
+      "step": 25800
+    },
+    {
+      "epoch": 0.6675429779118018,
+      "grad_norm": 2.7490615844726562,
+      "learning_rate": 4.976177186116067e-06,
+      "loss": 3.524,
+      "step": 25900
+    },
+    {
+      "epoch": 0.6701203639269053,
+      "grad_norm": 2.1966018676757812,
+      "learning_rate": 4.906331444557057e-06,
+      "loss": 3.457,
+      "step": 26000
+    },
+    {
+      "epoch": 0.6726977499420088,
+      "grad_norm": 2.6093175411224365,
+      "learning_rate": 4.83681965725977e-06,
+      "loss": 3.4933,
+      "step": 26100
+    },
+    {
+      "epoch": 0.6752751359571123,
+      "grad_norm": 2.49112606048584,
+      "learning_rate": 4.767646381599182e-06,
+      "loss": 3.5118,
+      "step": 26200
+    },
+    {
+      "epoch": 0.6778525219722158,
+      "grad_norm": 3.006537914276123,
+      "learning_rate": 4.698816152756555e-06,
+      "loss": 3.4852,
+      "step": 26300
+    },
+    {
+      "epoch": 0.6804299079873193,
+      "grad_norm": 3.1903345584869385,
+      "learning_rate": 4.630333483422113e-06,
+      "loss": 3.4743,
+      "step": 26400
+    },
+    {
+      "epoch": 0.6830072940024228,
+      "grad_norm": 3.210156202316284,
+      "learning_rate": 4.562202863499156e-06,
+      "loss": 3.4486,
+      "step": 26500
+    },
+    {
+      "epoch": 0.6855846800175263,
+      "grad_norm": 3.5177881717681885,
+      "learning_rate": 4.494428759809705e-06,
+      "loss": 3.4646,
+      "step": 26600
+    },
+    {
+      "epoch": 0.6881620660326297,
+      "grad_norm": 3.745039463043213,
+      "learning_rate": 4.427015615801643e-06,
+      "loss": 3.4679,
+      "step": 26700
+    },
+    {
+      "epoch": 0.6907394520477332,
+      "grad_norm": 2.926971912384033,
+      "learning_rate": 4.3599678512573965e-06,
+      "loss": 3.5078,
+      "step": 26800
+    },
+    {
+      "epoch": 0.6933168380628366,
+      "grad_norm": 2.910550355911255,
+      "learning_rate": 4.293289862004149e-06,
+      "loss": 3.4783,
+      "step": 26900
+    },
+    {
+      "epoch": 0.6958942240779401,
+      "grad_norm": 2.528334617614746,
+      "learning_rate": 4.2269860196256394e-06,
+      "loss": 3.4298,
+      "step": 27000
+    },
+    {
+      "epoch": 0.6984716100930436,
+      "grad_norm": 2.6762783527374268,
+      "learning_rate": 4.161060671175576e-06,
+      "loss": 3.5299,
+      "step": 27100
+    },
+    {
+      "epoch": 0.7010489961081471,
+      "grad_norm": 2.952073335647583,
+      "learning_rate": 4.0955181388925995e-06,
+      "loss": 3.4761,
+      "step": 27200
+    },
+    {
+      "epoch": 0.7036263821232506,
+      "grad_norm": 2.826277256011963,
+      "learning_rate": 4.030362719916923e-06,
+      "loss": 3.4861,
+      "step": 27300
+    },
+    {
+      "epoch": 0.7062037681383541,
+      "grad_norm": 3.2260401248931885,
+      "learning_rate": 3.965598686008595e-06,
+      "loss": 3.5105,
+      "step": 27400
+    },
+    {
+      "epoch": 0.7087811541534575,
+      "grad_norm": 3.05185866355896,
+      "learning_rate": 3.901230283267432e-06,
+      "loss": 3.4628,
+      "step": 27500
+    },
+    {
+      "epoch": 0.711358540168561,
+      "grad_norm": 3.9587910175323486,
+      "learning_rate": 3.837261731854643e-06,
+      "loss": 3.4725,
+      "step": 27600
+    },
+    {
+      "epoch": 0.7139359261836645,
+      "grad_norm": 3.1975626945495605,
+      "learning_rate": 3.7736972257161265e-06,
+      "loss": 3.4509,
+      "step": 27700
+    },
+    {
+      "epoch": 0.716513312198768,
+      "grad_norm": 2.787067413330078,
+      "learning_rate": 3.7105409323075146e-06,
+      "loss": 3.479,
+      "step": 27800
+    },
+    {
+      "epoch": 0.7190906982138715,
+      "grad_norm": 2.3811051845550537,
+      "learning_rate": 3.647796992320942e-06,
+      "loss": 3.4223,
+      "step": 27900
+    },
+    {
+      "epoch": 0.721668084228975,
+      "grad_norm": 2.9078991413116455,
+      "learning_rate": 3.5854695194135836e-06,
+      "loss": 3.5105,
+      "step": 28000
+    },
+    {
+      "epoch": 0.7242454702440785,
+      "grad_norm": 3.045022964477539,
+      "learning_rate": 3.5235625999379276e-06,
+      "loss": 3.4777,
+      "step": 28100
+    },
+    {
+      "epoch": 0.726822856259182,
+      "grad_norm": 2.9359779357910156,
+      "learning_rate": 3.4620802926738816e-06,
+      "loss": 3.4212,
+      "step": 28200
+    },
+    {
+      "epoch": 0.7294002422742855,
+      "grad_norm": 3.194761037826538,
+      "learning_rate": 3.4010266285626604e-06,
+      "loss": 3.4767,
+      "step": 28300
+    },
+    {
+      "epoch": 0.731977628289389,
+      "grad_norm": 3.352755308151245,
+      "learning_rate": 3.340405610442521e-06,
+      "loss": 3.4824,
+      "step": 28400
+    },
+    {
+      "epoch": 0.7345550143044924,
+      "grad_norm": 3.414588212966919,
+      "learning_rate": 3.280221212786303e-06,
+      "loss": 3.4289,
+      "step": 28500
+    },
+    {
+      "epoch": 0.7371324003195958,
+      "grad_norm": 3.331251859664917,
+      "learning_rate": 3.2204773814408687e-06,
+      "loss": 3.4286,
+      "step": 28600
+    },
+    {
+      "epoch": 0.7397097863346993,
+      "grad_norm": 3.4244349002838135,
+      "learning_rate": 3.1611780333683994e-06,
+      "loss": 3.484,
+      "step": 28700
+    },
+    {
+      "epoch": 0.7422871723498028,
+      "grad_norm": 2.5272583961486816,
+      "learning_rate": 3.102327056389586e-06,
+      "loss": 3.4705,
+      "step": 28800
+    },
+    {
+      "epoch": 0.7448645583649063,
+      "grad_norm": 2.9628090858459473,
+      "learning_rate": 3.043928308928735e-06,
+      "loss": 3.4835,
+      "step": 28900
+    },
+    {
+      "epoch": 0.7474419443800098,
+      "grad_norm": 2.411830186843872,
+      "learning_rate": 2.9859856197607994e-06,
+      "loss": 3.4609,
+      "step": 29000
+    },
+    {
+      "epoch": 0.7500193303951133,
+      "grad_norm": 3.6374247074127197,
+      "learning_rate": 2.9285027877603646e-06,
+      "loss": 3.4512,
+      "step": 29100
+    },
+    {
+      "epoch": 0.7525967164102167,
+      "grad_norm": 2.5388495922088623,
+      "learning_rate": 2.871483581652561e-06,
+      "loss": 3.4687,
+      "step": 29200
+    },
+    {
+      "epoch": 0.7551741024253202,
+      "grad_norm": 3.1188249588012695,
+      "learning_rate": 2.814931739765998e-06,
+      "loss": 3.4427,
+      "step": 29300
+    },
+    {
+      "epoch": 0.7577514884404237,
+      "grad_norm": 2.154353618621826,
+      "learning_rate": 2.7588509697876588e-06,
+      "loss": 3.4376,
+      "step": 29400
+    },
+    {
+      "epoch": 0.7603288744555272,
+      "grad_norm": 3.519176721572876,
+      "learning_rate": 2.703244948519822e-06,
+      "loss": 3.4864,
+      "step": 29500
+    },
+    {
+      "epoch": 0.7629062604706307,
+      "grad_norm": 4.806483745574951,
+      "learning_rate": 2.6481173216389912e-06,
+      "loss": 3.4616,
+      "step": 29600
+    },
+    {
+      "epoch": 0.7654836464857342,
+      "grad_norm": 3.5363433361053467,
+      "learning_rate": 2.5934717034568826e-06,
+      "loss": 3.4354,
+      "step": 29700
+    },
+    {
+      "epoch": 0.7680610325008377,
+      "grad_norm": 3.238460063934326,
+      "learning_rate": 2.5393116766834548e-06,
+      "loss": 3.4601,
+      "step": 29800
+    },
+    {
+      "epoch": 0.7706384185159412,
+      "grad_norm": 4.202941417694092,
+      "learning_rate": 2.48564079219203e-06,
+      "loss": 3.4445,
+      "step": 29900
+    },
+    {
+      "epoch": 0.7732158045310447,
+      "grad_norm": 2.6526873111724854,
+      "learning_rate": 2.4324625687864724e-06,
+      "loss": 3.4898,
+      "step": 30000
+    },
+    {
+      "epoch": 0.7757931905461481,
+      "grad_norm": 1.9778908491134644,
+      "learning_rate": 2.3797804929704935e-06,
+      "loss": 3.4651,
+      "step": 30100
+    },
+    {
+      "epoch": 0.7783705765612515,
+      "grad_norm": 2.8497068881988525,
+      "learning_rate": 2.327598018719075e-06,
+      "loss": 3.4436,
+      "step": 30200
+    },
+    {
+      "epoch": 0.780947962576355,
+      "grad_norm": 2.419715404510498,
+      "learning_rate": 2.2759185672520035e-06,
+      "loss": 3.4631,
+      "step": 30300
+    },
+    {
+      "epoch": 0.7835253485914585,
+      "grad_norm": 4.064117908477783,
+      "learning_rate": 2.2247455268095774e-06,
+      "loss": 3.4294,
+      "step": 30400
+    },
+    {
+      "epoch": 0.786102734606562,
+      "grad_norm": 3.3254690170288086,
+      "learning_rate": 2.1740822524304592e-06,
+      "loss": 3.4852,
+      "step": 30500
+    },
+    {
+      "epoch": 0.7886801206216655,
+      "grad_norm": 2.177420139312744,
+      "learning_rate": 2.123932065731711e-06,
+      "loss": 3.4196,
+      "step": 30600
+    },
+    {
+      "epoch": 0.791257506636769,
+      "grad_norm": 2.7484512329101562,
+      "learning_rate": 2.0742982546910285e-06,
+      "loss": 3.4612,
+      "step": 30700
+    },
+    {
+      "epoch": 0.7938348926518725,
+      "grad_norm": 2.773761510848999,
+      "learning_rate": 2.025184073431159e-06,
+      "loss": 3.4371,
+      "step": 30800
+    },
+    {
+      "epoch": 0.796412278666976,
+      "grad_norm": 3.323274850845337,
+      "learning_rate": 1.976592742006559e-06,
+      "loss": 3.4218,
+      "step": 30900
+    },
+    {
+      "epoch": 0.7989896646820794,
+      "grad_norm": 3.342670202255249,
+      "learning_rate": 1.928527446192281e-06,
+      "loss": 3.467,
+      "step": 31000
+    },
+    {
+      "epoch": 0.8015670506971829,
+      "grad_norm": 4.328900337219238,
+      "learning_rate": 1.8809913372751088e-06,
+      "loss": 3.4315,
+      "step": 31100
+    },
+    {
+      "epoch": 0.8041444367122864,
+      "grad_norm": 2.74299955368042,
+      "learning_rate": 1.8339875318469403e-06,
+      "loss": 3.4772,
+      "step": 31200
+    },
+    {
+      "epoch": 0.8067218227273899,
+      "grad_norm": 4.035296440124512,
+      "learning_rate": 1.7875191116004608e-06,
+      "loss": 3.4352,
+      "step": 31300
+    },
+    {
+      "epoch": 0.8092992087424934,
+      "grad_norm": 3.8721446990966797,
+      "learning_rate": 1.741589123127102e-06,
+      "loss": 3.4515,
+      "step": 31400
+    },
+    {
+      "epoch": 0.8118765947575969,
+      "grad_norm": 2.4593334197998047,
+      "learning_rate": 1.6962005777173008e-06,
+      "loss": 3.4365,
+      "step": 31500
+    },
+    {
+      "epoch": 0.8144539807727004,
+      "grad_norm": 3.573652744293213,
+      "learning_rate": 1.651356451163062e-06,
+      "loss": 3.4559,
+      "step": 31600
+    },
+    {
+      "epoch": 0.8170313667878039,
+      "grad_norm": 2.793955087661743,
+      "learning_rate": 1.6070596835628648e-06,
+      "loss": 3.4727,
+      "step": 31700
+    },
+    {
+      "epoch": 0.8196087528029072,
+      "grad_norm": 2.7159183025360107,
+      "learning_rate": 1.5633131791289025e-06,
+      "loss": 3.4555,
+      "step": 31800
+    },
+    {
+      "epoch": 0.8221861388180107,
+      "grad_norm": 2.820056915283203,
+      "learning_rate": 1.5201198059966694e-06,
+      "loss": 3.4686,
+      "step": 31900
+    },
+    {
+      "epoch": 0.8247635248331142,
+      "grad_norm": 3.042168617248535,
+      "learning_rate": 1.4774823960369267e-06,
+      "loss": 3.4859,
+      "step": 32000
+    },
+    {
+      "epoch": 0.8273409108482177,
+      "grad_norm": 1.8876127004623413,
+      "learning_rate": 1.4354037446700254e-06,
+      "loss": 3.4362,
+      "step": 32100
+    },
+    {
+      "epoch": 0.8299182968633212,
+      "grad_norm": 2.529664993286133,
+      "learning_rate": 1.3938866106826487e-06,
+      "loss": 3.4672,
+      "step": 32200
+    },
+    {
+      "epoch": 0.8324956828784247,
+      "grad_norm": 2.988670587539673,
+      "learning_rate": 1.3529337160469213e-06,
+      "loss": 3.3971,
+      "step": 32300
+    },
+    {
+      "epoch": 0.8350730688935282,
+      "grad_norm": 2.86753511428833,
+      "learning_rate": 1.312547745741959e-06,
+      "loss": 3.4792,
+      "step": 32400
+    },
+    {
+      "epoch": 0.8376504549086317,
+      "grad_norm": 2.9068603515625,
+      "learning_rate": 1.2727313475778292e-06,
+      "loss": 3.4381,
+      "step": 32500
+    },
+    {
+      "epoch": 0.8402278409237351,
+      "grad_norm": 3.6019556522369385,
+      "learning_rate": 1.2334871320219645e-06,
+      "loss": 3.4463,
+      "step": 32600
+    },
+    {
+      "epoch": 0.8428052269388386,
+      "grad_norm": 2.4795987606048584,
+      "learning_rate": 1.1948176720280003e-06,
+      "loss": 3.4414,
+      "step": 32700
+    },
+    {
+      "epoch": 0.8453826129539421,
+      "grad_norm": 3.487563133239746,
+      "learning_rate": 1.156725502867091e-06,
+      "loss": 3.4521,
+      "step": 32800
+    },
+    {
+      "epoch": 0.8479599989690456,
+      "grad_norm": 3.0892958641052246,
+      "learning_rate": 1.1192131219616897e-06,
+      "loss": 3.4442,
+      "step": 32900
+    },
+    {
+      "epoch": 0.8505373849841491,
+      "grad_norm": 4.437790870666504,
+      "learning_rate": 1.0822829887218179e-06,
+      "loss": 3.4133,
+      "step": 33000
+    },
+    {
+      "epoch": 0.8531147709992526,
+      "grad_norm": 3.2960755825042725,
+      "learning_rate": 1.0459375243838078e-06,
+      "loss": 3.4252,
+      "step": 33100
+    },
+    {
+      "epoch": 0.8556921570143561,
+      "grad_norm": 3.6840438842773438,
+      "learning_rate": 1.010179111851569e-06,
+      "loss": 3.3985,
+      "step": 33200
+    },
+    {
+      "epoch": 0.8582695430294596,
+      "grad_norm": 2.7940917015075684,
+      "learning_rate": 9.750100955403529e-07,
+      "loss": 3.4488,
+      "step": 33300
+    },
+    {
+      "epoch": 0.8608469290445631,
+      "grad_norm": 2.7630412578582764,
+      "learning_rate": 9.40432781223054e-07,
+      "loss": 3.4052,
+      "step": 33400
+    },
+    {
+      "epoch": 0.8634243150596664,
+      "grad_norm": 3.4538955688476562,
+      "learning_rate": 9.064494358790332e-07,
+      "loss": 3.416,
+      "step": 33500
+    },
+    {
+      "epoch": 0.8660017010747699,
+      "grad_norm": 3.7104904651641846,
+      "learning_rate": 8.73062287545482e-07,
+      "loss": 3.454,
+      "step": 33600
+    },
+    {
+      "epoch": 0.8685790870898734,
+      "grad_norm": 2.817603588104248,
+      "learning_rate": 8.402735251713556e-07,
+      "loss": 3.4433,
+      "step": 33700
+    },
+    {
+      "epoch": 0.8711564731049769,
+      "grad_norm": 3.7898402214050293,
+      "learning_rate": 8.080852984738663e-07,
+      "loss": 3.4687,
+      "step": 33800
+    },
+    {
+      "epoch": 0.8737338591200804,
+      "grad_norm": 3.0680181980133057,
+      "learning_rate": 7.76499717797522e-07,
+      "loss": 3.4588,
+      "step": 33900
+    },
+    {
+      "epoch": 0.8763112451351839,
+      "grad_norm": 3.4424569606781006,
+      "learning_rate": 7.455188539757829e-07,
+      "loss": 3.4492,
+      "step": 34000
+    },
+    {
+      "epoch": 0.8788886311502874,
+      "grad_norm": 2.6502506732940674,
+      "learning_rate": 7.151447381952847e-07,
+      "loss": 3.4425,
+      "step": 34100
+    },
+    {
+      "epoch": 0.8814660171653909,
+      "grad_norm": 2.8979685306549072,
+      "learning_rate": 6.853793618626759e-07,
+      "loss": 3.4329,
+      "step": 34200
+    },
+    {
+      "epoch": 0.8840434031804943,
+      "grad_norm": 3.53257155418396,
+      "learning_rate": 6.562246764740476e-07,
+      "loss": 3.4444,
+      "step": 34300
+    },
+    {
+      "epoch": 0.8866207891955978,
+      "grad_norm": 3.8942556381225586,
+      "learning_rate": 6.276825934869912e-07,
+      "loss": 3.4471,
+      "step": 34400
+    },
+    {
+      "epoch": 0.8891981752107013,
+      "grad_norm": 3.2598304748535156,
+      "learning_rate": 5.997549841952788e-07,
+      "loss": 3.436,
+      "step": 34500
+    },
+    {
+      "epoch": 0.8917755612258048,
+      "grad_norm": 3.509730100631714,
+      "learning_rate": 5.724436796061816e-07,
+      "loss": 3.4376,
+      "step": 34600
+    },
+    {
+      "epoch": 0.8943529472409083,
+      "grad_norm": 2.601433753967285,
+      "learning_rate": 5.45750470320413e-07,
+      "loss": 3.4918,
+      "step": 34700
+    },
+    {
+      "epoch": 0.8969303332560118,
+      "grad_norm": 3.087909698486328,
+      "learning_rate": 5.19677106414741e-07,
+      "loss": 3.4511,
+      "step": 34800
+    },
+    {
+      "epoch": 0.8995077192711153,
+      "grad_norm": 3.840320587158203,
+      "learning_rate": 4.942252973272433e-07,
+      "loss": 3.4342,
+      "step": 34900
+    },
+    {
+      "epoch": 0.9020851052862188,
+      "grad_norm": 3.78507661819458,
+      "learning_rate": 4.693967117452403e-07,
+      "loss": 3.4473,
+      "step": 35000
+    },
+    {
+      "epoch": 0.9046624913013221,
+      "grad_norm": 3.3662655353546143,
+      "learning_rate": 4.4519297749587964e-07,
+      "loss": 3.4647,
+      "step": 35100
+    },
+    {
+      "epoch": 0.9072398773164256,
+      "grad_norm": 2.418348550796509,
+      "learning_rate": 4.21615681439419e-07,
+      "loss": 3.4774,
+      "step": 35200
+    },
+    {
+      "epoch": 0.9098172633315291,
+      "grad_norm": 3.3388941287994385,
+      "learning_rate": 3.9866636936519044e-07,
+      "loss": 3.4057,
+      "step": 35300
+    },
+    {
+      "epoch": 0.9123946493466326,
+      "grad_norm": 3.2763423919677734,
+      "learning_rate": 3.7634654589024377e-07,
+      "loss": 3.4255,
+      "step": 35400
+    },
+    {
+      "epoch": 0.9149720353617361,
+      "grad_norm": 3.5573904514312744,
+      "learning_rate": 3.546576743607111e-07,
+      "loss": 3.4506,
+      "step": 35500
+    },
+    {
+      "epoch": 0.9175494213768396,
+      "grad_norm": 3.75197696685791,
+      "learning_rate": 3.336011767558589e-07,
+      "loss": 3.3956,
+      "step": 35600
+    },
+    {
+      "epoch": 0.9201268073919431,
+      "grad_norm": 2.9268035888671875,
+      "learning_rate": 3.131784335948629e-07,
+      "loss": 3.4288,
+      "step": 35700
+    },
+    {
+      "epoch": 0.9227041934070466,
+      "grad_norm": 3.253540277481079,
+      "learning_rate": 2.933907838462957e-07,
+      "loss": 3.441,
+      "step": 35800
+    },
+    {
+      "epoch": 0.9252815794221501,
+      "grad_norm": 3.143812656402588,
+      "learning_rate": 2.7423952484034067e-07,
+      "loss": 3.4411,
+      "step": 35900
+    },
+    {
+      "epoch": 0.9278589654372535,
+      "grad_norm": 2.847017765045166,
+      "learning_rate": 2.5572591218373634e-07,
+      "loss": 3.4474,
+      "step": 36000
+    },
+    {
+      "epoch": 0.930436351452357,
+      "grad_norm": 2.6306653022766113,
+      "learning_rate": 2.3785115967745465e-07,
+      "loss": 3.3759,
+      "step": 36100
+    },
+    {
+      "epoch": 0.9330137374674605,
+      "grad_norm": 2.997304916381836,
+      "learning_rate": 2.206164392371235e-07,
+      "loss": 3.4865,
+      "step": 36200
+    },
+    {
+      "epoch": 0.935591123482564,
+      "grad_norm": 2.7607882022857666,
+      "learning_rate": 2.0402288081618926e-07,
+      "loss": 3.4344,
+      "step": 36300
+    },
+    {
+      "epoch": 0.9381685094976675,
+      "grad_norm": 2.9195098876953125,
+      "learning_rate": 1.8807157233183493e-07,
+      "loss": 3.4056,
+      "step": 36400
+    },
+    {
+      "epoch": 0.940745895512771,
+      "grad_norm": 2.6638894081115723,
+      "learning_rate": 1.7276355959365722e-07,
+      "loss": 3.4412,
+      "step": 36500
+    },
+    {
+      "epoch": 0.9433232815278745,
+      "grad_norm": 4.130239486694336,
+      "learning_rate": 1.5809984623509467e-07,
+      "loss": 3.4397,
+      "step": 36600
+    },
+    {
+      "epoch": 0.9459006675429779,
+      "grad_norm": 2.762913703918457,
+      "learning_rate": 1.4408139364763152e-07,
+      "loss": 3.4217,
+      "step": 36700
+    },
+    {
+      "epoch": 0.9484780535580813,
+      "grad_norm": 2.5273561477661133,
+      "learning_rate": 1.3070912091776133e-07,
+      "loss": 3.4537,
+      "step": 36800
+    },
+    {
+      "epoch": 0.9510554395731848,
+      "grad_norm": 3.3580222129821777,
+      "learning_rate": 1.179839047667375e-07,
+      "loss": 3.4855,
+      "step": 36900
+    },
+    {
+      "epoch": 0.9536328255882883,
+      "grad_norm": 3.5217840671539307,
+      "learning_rate": 1.059065794930847e-07,
+      "loss": 3.4753,
+      "step": 37000
+    },
+    {
+      "epoch": 0.9562102116033918,
+      "grad_norm": 2.870847225189209,
+      "learning_rate": 9.447793691790385e-08,
+      "loss": 3.4372,
+      "step": 37100
+    },
+    {
+      "epoch": 0.9587875976184953,
+      "grad_norm": 3.781106472015381,
+      "learning_rate": 8.369872633295806e-08,
+      "loss": 3.3997,
+      "step": 37200
+    },
+    {
+      "epoch": 0.9613649836335988,
+      "grad_norm": 2.3308935165405273,
+      "learning_rate": 7.35696544515485e-08,
+      "loss": 3.4334,
+      "step": 37300
+    },
+    {
+      "epoch": 0.9639423696487023,
+      "grad_norm": 3.2025210857391357,
+      "learning_rate": 6.409138536217719e-08,
+      "loss": 3.4677,
+      "step": 37400
+    },
+    {
+      "epoch": 0.9665197556638058,
+      "grad_norm": 3.3356034755706787,
+      "learning_rate": 5.526454048500829e-08,
+      "loss": 3.4977,
+      "step": 37500
+    },
+    {
+      "epoch": 0.9690971416789093,
+      "grad_norm": 3.0292091369628906,
+      "learning_rate": 4.708969853112866e-08,
+      "loss": 3.4219,
+      "step": 37600
+    },
+    {
+      "epoch": 0.9716745276940127,
+      "grad_norm": 3.254014253616333,
+      "learning_rate": 3.9567395464604755e-08,
+      "loss": 3.4236,
+      "step": 37700
+    },
+    {
+      "epoch": 0.9742519137091162,
+      "grad_norm": 3.2781224250793457,
+      "learning_rate": 3.2698124467342996e-08,
+      "loss": 3.451,
+      "step": 37800
+    },
+    {
+      "epoch": 0.9768292997242197,
+      "grad_norm": 3.1472885608673096,
+      "learning_rate": 2.648233590675453e-08,
+      "loss": 3.4279,
+      "step": 37900
+    },
+    {
+      "epoch": 0.9794066857393232,
+      "grad_norm": 2.527984142303467,
+      "learning_rate": 2.0920437306229947e-08,
+      "loss": 3.4205,
+      "step": 38000
+    },
+    {
+      "epoch": 0.9819840717544267,
+      "grad_norm": 2.989227771759033,
+      "learning_rate": 1.6012793318420648e-08,
+      "loss": 3.4284,
+      "step": 38100
+    },
+    {
+      "epoch": 0.9845614577695302,
+      "grad_norm": 3.849820137023926,
+      "learning_rate": 1.1759725701329105e-08,
+      "loss": 3.3932,
+      "step": 38200
+    },
+    {
+      "epoch": 0.9871388437846337,
+      "grad_norm": 3.1409499645233154,
+      "learning_rate": 8.161513297216816e-09,
+      "loss": 3.4244,
+      "step": 38300
+    },
+    {
+      "epoch": 0.9897162297997371,
+      "grad_norm": 2.875560760498047,
+      "learning_rate": 5.2183920143200615e-09,
+      "loss": 3.4249,
+      "step": 38400
+    },
+    {
+      "epoch": 0.9922936158148405,
+      "grad_norm": 3.058305025100708,
+      "learning_rate": 2.9305548113855996e-09,
+      "loss": 3.4207,
+      "step": 38500
+    },
+    {
+      "epoch": 0.994871001829944,
+      "grad_norm": 2.5125629901885986,
+      "learning_rate": 1.2981516850152365e-09,
+      "loss": 3.473,
+      "step": 38600
+    },
+    {
+      "epoch": 0.9974483878450475,
+      "grad_norm": 2.628063917160034,
+      "learning_rate": 3.212896598392412e-10,
+      "loss": 3.3982,
+      "step": 38700
+    }
+  ],
+  "logging_steps": 100,
+  "max_steps": 38799,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 1,
+  "save_steps": 2000,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": true,
+        "should_training_stop": true
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 5.857062061447578e+17,
+  "train_batch_size": 1,
+  "trial_name": null,
+  "trial_params": null
+}