Push PTM-LLaMA training artifacts

README.md

@@ -0,0 +1,18 @@
+---
+base_model: GreatCaptainNemo/ProLLaMA_Stage_1
+tags:
+  - protein
+  - ptm
+  - methylation
+  - phosphorylation
+  - ubiquitination
+  - lora
+  - peft
+library_name: transformers
+---
+
+# PTM-LLaMA
+
+LoRA-fine-tuned `GreatCaptainNemo/ProLLaMA_Stage_1` for predicting post-translational modification (PTM) sites in protein sequences. A single LoRA adapter is instruction-tuned to handle three PTM types — methylation, phosphorylation, and ubiquitination — selected at inference time by the prompt. Output format: `Sites=<R5,D12,...>` regardless of PTM type.
+
+This is a **training-only stub card**. Final metrics (per-PTM-type AUC, accuracy, precision, recall, F1, confusion matrices, and the cross-instruction ablation) are filled in by the companion evaluation notebook after running on the held-out test set.

config.json

@@ -0,0 +1,32 @@
+{
+  "_name_or_path": "GreatCaptainNemo/ProLLaMA_Stage_1",
+  "architectures": [
+    "LlamaForCausalLM"
+  ],
+  "attention_bias": false,
+  "attention_dropout": 0.0,
+  "bos_token_id": 1,
+  "eos_token_id": 2,
+  "head_dim": 128,
+  "hidden_act": "silu",
+  "hidden_size": 4096,
+  "initializer_range": 0.02,
+  "intermediate_size": 11008,
+  "max_position_embeddings": 4096,
+  "mlp_bias": false,
+  "model_type": "llama",
+  "num_attention_heads": 32,
+  "num_hidden_layers": 32,
+  "num_key_value_heads": 32,
+  "outputs_attentions": true,
+  "pad_token_id": 0,
+  "pretraining_tp": 1,
+  "rms_norm_eps": 1e-05,
+  "rope_scaling": null,
+  "rope_theta": 10000.0,
+  "tie_word_embeddings": false,
+  "torch_dtype": "float16",
+  "transformers_version": "4.49.0",
+  "use_cache": true,
+  "vocab_size": 32000
+}

generation_config.json

@@ -0,0 +1,7 @@
+{
+  "_from_model_config": true,
+  "bos_token_id": 1,
+  "eos_token_id": 2,
+  "pad_token_id": 0,
+  "transformers_version": "4.49.0"
+}

model-00001-of-00007.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:52dba6480c43beb9e4fd42ed6b47d44799bb3629bb7b6b200d2d29c543c076bc
+size 1981878760

model-00002-of-00007.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c0a1aa37220a64a16f6521fed605dc4226387d511085c2bee0a7280c9cd526d4
+size 1990284200

model-00003-of-00007.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e90176e7be226e475da0b8d4b83b4ab871ea5b8f11b84c5b85f6d935594fba25
+size 1990284232

model-00004-of-00007.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ec73c47297bc54ab14838786a5ad2a3f0cc928407040a0fee0338ec09af8c8f2
+size 1990284240

model-00005-of-00007.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f0729b2b41ad9250de8e2754e46175101df76ed41bdca1f2a942a7d850a668c8
+size 1933644528

model-00006-of-00007.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:fb8e91973d1618b28a99f8d6d5a0aa4f411b4727a4fdd1eea45c1721b5b3a388
+size 1933661136

model-00007-of-00007.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c75423c7b868e397ed0d8e6815f8cc8daffd53bb7275ab4bf25f187b37bd7609
+size 1656827576

model.safetensors.index.json

@@ -0,0 +1,298 @@
+{
+  "metadata": {
+    "total_size": 13476831232
+  },
+  "weight_map": {
+    "lm_head.weight": "model-00007-of-00007.safetensors",
+    "model.embed_tokens.weight": "model-00001-of-00007.safetensors",
+    "model.layers.0.input_layernorm.weight": "model-00001-of-00007.safetensors",
+    "model.layers.0.mlp.down_proj.weight": "model-00001-of-00007.safetensors",
+    "model.layers.0.mlp.gate_proj.weight": "model-00001-of-00007.safetensors",
+    "model.layers.0.mlp.up_proj.weight": "model-00001-of-00007.safetensors",
+    "model.layers.0.post_attention_layernorm.weight": "model-00001-of-00007.safetensors",
+    "model.layers.0.self_attn.k_proj.weight": "model-00001-of-00007.safetensors",
+    "model.layers.0.self_attn.o_proj.weight": "model-00001-of-00007.safetensors",
+    "model.layers.0.self_attn.q_proj.weight": "model-00001-of-00007.safetensors",
+    "model.layers.0.self_attn.v_proj.weight": "model-00001-of-00007.safetensors",
+    "model.layers.1.input_layernorm.weight": "model-00001-of-00007.safetensors",
+    "model.layers.1.mlp.down_proj.weight": "model-00001-of-00007.safetensors",
+    "model.layers.1.mlp.gate_proj.weight": "model-00001-of-00007.safetensors",
+    "model.layers.1.mlp.up_proj.weight": "model-00001-of-00007.safetensors",
+    "model.layers.1.post_attention_layernorm.weight": "model-00001-of-00007.safetensors",
+    "model.layers.1.self_attn.k_proj.weight": "model-00001-of-00007.safetensors",
+    "model.layers.1.self_attn.o_proj.weight": "model-00001-of-00007.safetensors",
+    "model.layers.1.self_attn.q_proj.weight": "model-00001-of-00007.safetensors",
+    "model.layers.1.self_attn.v_proj.weight": "model-00001-of-00007.safetensors",
+    "model.layers.10.input_layernorm.weight": "model-00003-of-00007.safetensors",
+    "model.layers.10.mlp.down_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.10.mlp.gate_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.10.mlp.up_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.10.post_attention_layernorm.weight": "model-00003-of-00007.safetensors",
+    "model.layers.10.self_attn.k_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.10.self_attn.o_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.10.self_attn.q_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.10.self_attn.v_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.11.input_layernorm.weight": "model-00003-of-00007.safetensors",
+    "model.layers.11.mlp.down_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.11.mlp.gate_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.11.mlp.up_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.11.post_attention_layernorm.weight": "model-00003-of-00007.safetensors",
+    "model.layers.11.self_attn.k_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.11.self_attn.o_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.11.self_attn.q_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.11.self_attn.v_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.12.input_layernorm.weight": "model-00003-of-00007.safetensors",
+    "model.layers.12.mlp.down_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.12.mlp.gate_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.12.mlp.up_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.12.post_attention_layernorm.weight": "model-00003-of-00007.safetensors",
+    "model.layers.12.self_attn.k_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.12.self_attn.o_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.12.self_attn.q_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.12.self_attn.v_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.13.input_layernorm.weight": "model-00003-of-00007.safetensors",
+    "model.layers.13.mlp.down_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.13.mlp.gate_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.13.mlp.up_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.13.post_attention_layernorm.weight": "model-00003-of-00007.safetensors",
+    "model.layers.13.self_attn.k_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.13.self_attn.o_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.13.self_attn.q_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.13.self_attn.v_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.14.input_layernorm.weight": "model-00004-of-00007.safetensors",
+    "model.layers.14.mlp.down_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.14.mlp.gate_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.14.mlp.up_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.14.post_attention_layernorm.weight": "model-00004-of-00007.safetensors",
+    "model.layers.14.self_attn.k_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.14.self_attn.o_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.14.self_attn.q_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.14.self_attn.v_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.15.input_layernorm.weight": "model-00004-of-00007.safetensors",
+    "model.layers.15.mlp.down_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.15.mlp.gate_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.15.mlp.up_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.15.post_attention_layernorm.weight": "model-00004-of-00007.safetensors",
+    "model.layers.15.self_attn.k_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.15.self_attn.o_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.15.self_attn.q_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.15.self_attn.v_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.16.input_layernorm.weight": "model-00004-of-00007.safetensors",
+    "model.layers.16.mlp.down_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.16.mlp.gate_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.16.mlp.up_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.16.post_attention_layernorm.weight": "model-00004-of-00007.safetensors",
+    "model.layers.16.self_attn.k_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.16.self_attn.o_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.16.self_attn.q_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.16.self_attn.v_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.17.input_layernorm.weight": "model-00004-of-00007.safetensors",
+    "model.layers.17.mlp.down_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.17.mlp.gate_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.17.mlp.up_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.17.post_attention_layernorm.weight": "model-00004-of-00007.safetensors",
+    "model.layers.17.self_attn.k_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.17.self_attn.o_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.17.self_attn.q_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.17.self_attn.v_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.18.input_layernorm.weight": "model-00004-of-00007.safetensors",
+    "model.layers.18.mlp.down_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.18.mlp.gate_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.18.mlp.up_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.18.post_attention_layernorm.weight": "model-00004-of-00007.safetensors",
+    "model.layers.18.self_attn.k_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.18.self_attn.o_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.18.self_attn.q_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.18.self_attn.v_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.19.input_layernorm.weight": "model-00005-of-00007.safetensors",
+    "model.layers.19.mlp.down_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.19.mlp.gate_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.19.mlp.up_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.19.post_attention_layernorm.weight": "model-00005-of-00007.safetensors",
+    "model.layers.19.self_attn.k_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.19.self_attn.o_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.19.self_attn.q_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.19.self_attn.v_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.2.input_layernorm.weight": "model-00001-of-00007.safetensors",
+    "model.layers.2.mlp.down_proj.weight": "model-00001-of-00007.safetensors",
+    "model.layers.2.mlp.gate_proj.weight": "model-00001-of-00007.safetensors",
+    "model.layers.2.mlp.up_proj.weight": "model-00001-of-00007.safetensors",
+    "model.layers.2.post_attention_layernorm.weight": "model-00001-of-00007.safetensors",
+    "model.layers.2.self_attn.k_proj.weight": "model-00001-of-00007.safetensors",
+    "model.layers.2.self_attn.o_proj.weight": "model-00001-of-00007.safetensors",
+    "model.layers.2.self_attn.q_proj.weight": "model-00001-of-00007.safetensors",
+    "model.layers.2.self_attn.v_proj.weight": "model-00001-of-00007.safetensors",
+    "model.layers.20.input_layernorm.weight": "model-00005-of-00007.safetensors",
+    "model.layers.20.mlp.down_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.20.mlp.gate_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.20.mlp.up_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.20.post_attention_layernorm.weight": "model-00005-of-00007.safetensors",
+    "model.layers.20.self_attn.k_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.20.self_attn.o_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.20.self_attn.q_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.20.self_attn.v_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.21.input_layernorm.weight": "model-00005-of-00007.safetensors",
+    "model.layers.21.mlp.down_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.21.mlp.gate_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.21.mlp.up_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.21.post_attention_layernorm.weight": "model-00005-of-00007.safetensors",
+    "model.layers.21.self_attn.k_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.21.self_attn.o_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.21.self_attn.q_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.21.self_attn.v_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.22.input_layernorm.weight": "model-00005-of-00007.safetensors",
+    "model.layers.22.mlp.down_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.22.mlp.gate_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.22.mlp.up_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.22.post_attention_layernorm.weight": "model-00005-of-00007.safetensors",
+    "model.layers.22.self_attn.k_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.22.self_attn.o_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.22.self_attn.q_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.22.self_attn.v_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.23.input_layernorm.weight": "model-00006-of-00007.safetensors",
+    "model.layers.23.mlp.down_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.23.mlp.gate_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.23.mlp.up_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.23.post_attention_layernorm.weight": "model-00006-of-00007.safetensors",
+    "model.layers.23.self_attn.k_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.23.self_attn.o_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.23.self_attn.q_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.23.self_attn.v_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.24.input_layernorm.weight": "model-00006-of-00007.safetensors",
+    "model.layers.24.mlp.down_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.24.mlp.gate_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.24.mlp.up_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.24.post_attention_layernorm.weight": "model-00006-of-00007.safetensors",
+    "model.layers.24.self_attn.k_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.24.self_attn.o_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.24.self_attn.q_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.24.self_attn.v_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.25.input_layernorm.weight": "model-00006-of-00007.safetensors",
+    "model.layers.25.mlp.down_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.25.mlp.gate_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.25.mlp.up_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.25.post_attention_layernorm.weight": "model-00006-of-00007.safetensors",
+    "model.layers.25.self_attn.k_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.25.self_attn.o_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.25.self_attn.q_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.25.self_attn.v_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.26.input_layernorm.weight": "model-00006-of-00007.safetensors",
+    "model.layers.26.mlp.down_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.26.mlp.gate_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.26.mlp.up_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.26.post_attention_layernorm.weight": "model-00006-of-00007.safetensors",
+    "model.layers.26.self_attn.k_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.26.self_attn.o_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.26.self_attn.q_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.26.self_attn.v_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.27.input_layernorm.weight": "model-00006-of-00007.safetensors",
+    "model.layers.27.mlp.down_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.27.mlp.gate_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.27.mlp.up_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.27.post_attention_layernorm.weight": "model-00006-of-00007.safetensors",
+    "model.layers.27.self_attn.k_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.27.self_attn.o_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.27.self_attn.q_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.27.self_attn.v_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.28.input_layernorm.weight": "model-00007-of-00007.safetensors",
+    "model.layers.28.mlp.down_proj.weight": "model-00007-of-00007.safetensors",
+    "model.layers.28.mlp.gate_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.28.mlp.up_proj.weight": "model-00007-of-00007.safetensors",
+    "model.layers.28.post_attention_layernorm.weight": "model-00007-of-00007.safetensors",
+    "model.layers.28.self_attn.k_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.28.self_attn.o_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.28.self_attn.q_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.28.self_attn.v_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.29.input_layernorm.weight": "model-00007-of-00007.safetensors",
+    "model.layers.29.mlp.down_proj.weight": "model-00007-of-00007.safetensors",
+    "model.layers.29.mlp.gate_proj.weight": "model-00007-of-00007.safetensors",
+    "model.layers.29.mlp.up_proj.weight": "model-00007-of-00007.safetensors",
+    "model.layers.29.post_attention_layernorm.weight": "model-00007-of-00007.safetensors",
+    "model.layers.29.self_attn.k_proj.weight": "model-00007-of-00007.safetensors",
+    "model.layers.29.self_attn.o_proj.weight": "model-00007-of-00007.safetensors",
+    "model.layers.29.self_attn.q_proj.weight": "model-00007-of-00007.safetensors",
+    "model.layers.29.self_attn.v_proj.weight": "model-00007-of-00007.safetensors",
+    "model.layers.3.input_layernorm.weight": "model-00001-of-00007.safetensors",
+    "model.layers.3.mlp.down_proj.weight": "model-00001-of-00007.safetensors",
+    "model.layers.3.mlp.gate_proj.weight": "model-00001-of-00007.safetensors",
+    "model.layers.3.mlp.up_proj.weight": "model-00001-of-00007.safetensors",
+    "model.layers.3.post_attention_layernorm.weight": "model-00001-of-00007.safetensors",
+    "model.layers.3.self_attn.k_proj.weight": "model-00001-of-00007.safetensors",
+    "model.layers.3.self_attn.o_proj.weight": "model-00001-of-00007.safetensors",
+    "model.layers.3.self_attn.q_proj.weight": "model-00001-of-00007.safetensors",
+    "model.layers.3.self_attn.v_proj.weight": "model-00001-of-00007.safetensors",
+    "model.layers.30.input_layernorm.weight": "model-00007-of-00007.safetensors",
+    "model.layers.30.mlp.down_proj.weight": "model-00007-of-00007.safetensors",
+    "model.layers.30.mlp.gate_proj.weight": "model-00007-of-00007.safetensors",
+    "model.layers.30.mlp.up_proj.weight": "model-00007-of-00007.safetensors",
+    "model.layers.30.post_attention_layernorm.weight": "model-00007-of-00007.safetensors",
+    "model.layers.30.self_attn.k_proj.weight": "model-00007-of-00007.safetensors",
+    "model.layers.30.self_attn.o_proj.weight": "model-00007-of-00007.safetensors",
+    "model.layers.30.self_attn.q_proj.weight": "model-00007-of-00007.safetensors",
+    "model.layers.30.self_attn.v_proj.weight": "model-00007-of-00007.safetensors",
+    "model.layers.31.input_layernorm.weight": "model-00007-of-00007.safetensors",
+    "model.layers.31.mlp.down_proj.weight": "model-00007-of-00007.safetensors",
+    "model.layers.31.mlp.gate_proj.weight": "model-00007-of-00007.safetensors",
+    "model.layers.31.mlp.up_proj.weight": "model-00007-of-00007.safetensors",
+    "model.layers.31.post_attention_layernorm.weight": "model-00007-of-00007.safetensors",
+    "model.layers.31.self_attn.k_proj.weight": "model-00007-of-00007.safetensors",
+    "model.layers.31.self_attn.o_proj.weight": "model-00007-of-00007.safetensors",
+    "model.layers.31.self_attn.q_proj.weight": "model-00007-of-00007.safetensors",
+    "model.layers.31.self_attn.v_proj.weight": "model-00007-of-00007.safetensors",
+    "model.layers.4.input_layernorm.weight": "model-00002-of-00007.safetensors",
+    "model.layers.4.mlp.down_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.4.mlp.gate_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.4.mlp.up_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.4.post_attention_layernorm.weight": "model-00002-of-00007.safetensors",
+    "model.layers.4.self_attn.k_proj.weight": "model-00001-of-00007.safetensors",
+    "model.layers.4.self_attn.o_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.4.self_attn.q_proj.weight": "model-00001-of-00007.safetensors",
+    "model.layers.4.self_attn.v_proj.weight": "model-00001-of-00007.safetensors",
+    "model.layers.5.input_layernorm.weight": "model-00002-of-00007.safetensors",
+    "model.layers.5.mlp.down_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.5.mlp.gate_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.5.mlp.up_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.5.post_attention_layernorm.weight": "model-00002-of-00007.safetensors",
+    "model.layers.5.self_attn.k_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.5.self_attn.o_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.5.self_attn.q_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.5.self_attn.v_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.6.input_layernorm.weight": "model-00002-of-00007.safetensors",
+    "model.layers.6.mlp.down_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.6.mlp.gate_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.6.mlp.up_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.6.post_attention_layernorm.weight": "model-00002-of-00007.safetensors",
+    "model.layers.6.self_attn.k_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.6.self_attn.o_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.6.self_attn.q_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.6.self_attn.v_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.7.input_layernorm.weight": "model-00002-of-00007.safetensors",
+    "model.layers.7.mlp.down_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.7.mlp.gate_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.7.mlp.up_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.7.post_attention_layernorm.weight": "model-00002-of-00007.safetensors",
+    "model.layers.7.self_attn.k_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.7.self_attn.o_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.7.self_attn.q_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.7.self_attn.v_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.8.input_layernorm.weight": "model-00002-of-00007.safetensors",
+    "model.layers.8.mlp.down_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.8.mlp.gate_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.8.mlp.up_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.8.post_attention_layernorm.weight": "model-00002-of-00007.safetensors",
+    "model.layers.8.self_attn.k_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.8.self_attn.o_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.8.self_attn.q_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.8.self_attn.v_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.9.input_layernorm.weight": "model-00003-of-00007.safetensors",
+    "model.layers.9.mlp.down_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.9.mlp.gate_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.9.mlp.up_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.9.post_attention_layernorm.weight": "model-00003-of-00007.safetensors",
+    "model.layers.9.self_attn.k_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.9.self_attn.o_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.9.self_attn.q_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.9.self_attn.v_proj.weight": "model-00003-of-00007.safetensors",
+    "model.norm.weight": "model-00007-of-00007.safetensors"
+  }
+}

special_tokens_map.json

@@ -0,0 +1,24 @@
+{
+  "bos_token": {
+    "content": "<s>",
+    "lstrip": false,
+    "normalized": true,
+    "rstrip": false,
+    "single_word": false
+  },
+  "eos_token": {
+    "content": "</s>",
+    "lstrip": false,
+    "normalized": true,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": "<unk>",
+  "unk_token": {
+    "content": "<unk>",
+    "lstrip": false,
+    "normalized": true,
+    "rstrip": false,
+    "single_word": false
+  }
+}

tokenizer.model

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9e556afd44213b6bd1be2b850ebbbd98f5481437a8021afaf58ee7fb1818d347
+size 499723

tokenizer_config.json

@@ -0,0 +1,44 @@
+{
+  "add_bos_token": true,
+  "add_eos_token": false,
+  "add_prefix_space": null,
+  "added_tokens_decoder": {
+    "0": {
+      "content": "<unk>",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "1": {
+      "content": "<s>",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "2": {
+      "content": "</s>",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    }
+  },
+  "bos_token": "<s>",
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "</s>",
+  "extra_special_tokens": {},
+  "legacy": true,
+  "model_max_length": 1000000000000000019884624838656,
+  "pad_token": "<unk>",
+  "sp_model_kwargs": {},
+  "spaces_between_special_tokens": false,
+  "tokenizer_class": "LlamaTokenizer",
+  "unk_token": "<unk>",
+  "use_default_system_prompt": false,
+  "use_fast": true
+}

trainer_log_history.json

@@ -0,0 +1,3671 @@
+[
+  {
+    "loss": 10.1429,
+    "grad_norm": 24.328838348388672,
+    "learning_rate": 1e-05,
+    "epoch": 0.0006136239865616347,
+    "step": 20
+  },
+  {
+    "loss": 2.0091,
+    "grad_norm": 9.998310089111328,
+    "learning_rate": 2e-05,
+    "epoch": 0.0012272479731232694,
+    "step": 40
+  },
+  {
+    "loss": 0.4768,
+    "grad_norm": 1.95796799659729,
+    "learning_rate": 3e-05,
+    "epoch": 0.001840871959684904,
+    "step": 60
+  },
+  {
+    "loss": 0.397,
+    "grad_norm": 0.9538208842277527,
+    "learning_rate": 4e-05,
+    "epoch": 0.0024544959462465387,
+    "step": 80
+  },
+  {
+    "loss": 0.3703,
+    "grad_norm": 3.8885602951049805,
+    "learning_rate": 5e-05,
+    "epoch": 0.0030681199328081734,
+    "step": 100
+  },
+  {
+    "loss": 0.377,
+    "grad_norm": 1.2746047973632812,
+    "learning_rate": 6e-05,
+    "epoch": 0.003681743919369808,
+    "step": 120
+  },
+  {
+    "loss": 0.3449,
+    "grad_norm": 0.5722544193267822,
+    "learning_rate": 7e-05,
+    "epoch": 0.004295367905931443,
+    "step": 140
+  },
+  {
+    "loss": 0.3432,
+    "grad_norm": 0.4266990125179291,
+    "learning_rate": 8e-05,
+    "epoch": 0.004908991892493077,
+    "step": 160
+  },
+  {
+    "loss": 0.3447,
+    "grad_norm": 0.940947949886322,
+    "learning_rate": 9e-05,
+    "epoch": 0.0055226158790547125,
+    "step": 180
+  },
+  {
+    "loss": 0.3363,
+    "grad_norm": 1.3794434070587158,
+    "learning_rate": 0.0001,
+    "epoch": 0.006136239865616347,
+    "step": 200
+  },
+  {
+    "loss": 0.3097,
+    "grad_norm": 0.6603377461433411,
+    "learning_rate": 9.99989723479183e-05,
+    "epoch": 0.006749863852177982,
+    "step": 220
+  },
+  {
+    "loss": 0.3234,
+    "grad_norm": 0.5309205055236816,
+    "learning_rate": 9.999588943391597e-05,
+    "epoch": 0.007363487838739616,
+    "step": 240
+  },
+  {
+    "loss": 0.322,
+    "grad_norm": 0.4918404817581177,
+    "learning_rate": 9.999075138471951e-05,
+    "epoch": 0.007977111825301251,
+    "step": 260
+  },
+  {
+    "loss": 0.3061,
+    "grad_norm": 0.23719413578510284,
+    "learning_rate": 9.9983558411534e-05,
+    "epoch": 0.008590735811862886,
+    "step": 280
+  },
+  {
+    "loss": 0.3141,
+    "grad_norm": 0.5652378797531128,
+    "learning_rate": 9.99743108100344e-05,
+    "epoch": 0.00920435979842452,
+    "step": 300
+  },
+  {
+    "loss": 0.3007,
+    "grad_norm": 0.3501427471637726,
+    "learning_rate": 9.996300896035339e-05,
+    "epoch": 0.009817983784986155,
+    "step": 320
+  },
+  {
+    "loss": 0.2783,
+    "grad_norm": 2.7963829040527344,
+    "learning_rate": 9.994965332706573e-05,
+    "epoch": 0.01043160777154779,
+    "step": 340
+  },
+  {
+    "loss": 0.2972,
+    "grad_norm": 0.4121073782444,
+    "learning_rate": 9.993424445916923e-05,
+    "epoch": 0.011045231758109425,
+    "step": 360
+  },
+  {
+    "loss": 0.2799,
+    "grad_norm": 0.3509398400783539,
+    "learning_rate": 9.991678299006205e-05,
+    "epoch": 0.011658855744671058,
+    "step": 380
+  },
+  {
+    "loss": 0.268,
+    "grad_norm": 0.25531092286109924,
+    "learning_rate": 9.989726963751682e-05,
+    "epoch": 0.012272479731232694,
+    "step": 400
+  },
+  {
+    "loss": 0.2601,
+    "grad_norm": 0.4110245108604431,
+    "learning_rate": 9.987570520365104e-05,
+    "epoch": 0.012886103717794329,
+    "step": 420
+  },
+  {
+    "loss": 0.2672,
+    "grad_norm": 0.6753450632095337,
+    "learning_rate": 9.98520905748941e-05,
+    "epoch": 0.013499727704355964,
+    "step": 440
+  },
+  {
+    "loss": 0.2618,
+    "grad_norm": 0.6639530062675476,
+    "learning_rate": 9.982642672195092e-05,
+    "epoch": 0.014113351690917597,
+    "step": 460
+  },
+  {
+    "loss": 0.2562,
+    "grad_norm": 1.2062132358551025,
+    "learning_rate": 9.979871469976196e-05,
+    "epoch": 0.014726975677479232,
+    "step": 480
+  },
+  {
+    "loss": 0.2404,
+    "grad_norm": 0.3097586929798126,
+    "learning_rate": 9.976895564745991e-05,
+    "epoch": 0.015340599664040867,
+    "step": 500
+  },
+  {
+    "eval_loss": 0.23828153312206268,
+    "eval_runtime": 2767.3744,
+    "eval_samples_per_second": 161.937,
+    "eval_steps_per_second": 2.531,
+    "epoch": 0.015340599664040867,
+    "step": 500
+  },
+  {
+    "loss": 0.2348,
+    "grad_norm": 0.3366382122039795,
+    "learning_rate": 9.973715078832288e-05,
+    "epoch": 0.015954223650602502,
+    "step": 520
+  },
+  {
+    "loss": 0.2475,
+    "grad_norm": 0.23378945887088776,
+    "learning_rate": 9.970330142972401e-05,
+    "epoch": 0.016567847637164138,
+    "step": 540
+  },
+  {
+    "loss": 0.2499,
+    "grad_norm": 0.7393000721931458,
+    "learning_rate": 9.966740896307791e-05,
+    "epoch": 0.017181471623725773,
+    "step": 560
+  },
+  {
+    "loss": 0.2479,
+    "grad_norm": 0.40257546305656433,
+    "learning_rate": 9.962947486378326e-05,
+    "epoch": 0.017795095610287408,
+    "step": 580
+  },
+  {
+    "loss": 0.227,
+    "grad_norm": 0.2757706046104431,
+    "learning_rate": 9.95895006911623e-05,
+    "epoch": 0.01840871959684904,
+    "step": 600
+  },
+  {
+    "loss": 0.2199,
+    "grad_norm": 0.32403552532196045,
+    "learning_rate": 9.954748808839674e-05,
+    "epoch": 0.019022343583410675,
+    "step": 620
+  },
+  {
+    "loss": 0.2315,
+    "grad_norm": 0.6505448818206787,
+    "learning_rate": 9.95034387824601e-05,
+    "epoch": 0.01963596756997231,
+    "step": 640
+  },
+  {
+    "loss": 0.2247,
+    "grad_norm": 0.4369623363018036,
+    "learning_rate": 9.945735458404681e-05,
+    "epoch": 0.020249591556533945,
+    "step": 660
+  },
+  {
+    "loss": 0.2286,
+    "grad_norm": 0.43412795662879944,
+    "learning_rate": 9.940923738749778e-05,
+    "epoch": 0.02086321554309558,
+    "step": 680
+  },
+  {
+    "loss": 0.2202,
+    "grad_norm": 0.5289716720581055,
+    "learning_rate": 9.935908917072252e-05,
+    "epoch": 0.021476839529657215,
+    "step": 700
+  },
+  {
+    "loss": 0.2124,
+    "grad_norm": 0.552556037902832,
+    "learning_rate": 9.930691199511775e-05,
+    "epoch": 0.02209046351621885,
+    "step": 720
+  },
+  {
+    "loss": 0.2174,
+    "grad_norm": 0.3651241958141327,
+    "learning_rate": 9.925270800548285e-05,
+    "epoch": 0.022704087502780485,
+    "step": 740
+  },
+  {
+    "loss": 0.2245,
+    "grad_norm": 0.46145933866500854,
+    "learning_rate": 9.919647942993148e-05,
+    "epoch": 0.023317711489342117,
+    "step": 760
+  },
+  {
+    "loss": 0.223,
+    "grad_norm": 0.37703219056129456,
+    "learning_rate": 9.91382285798002e-05,
+    "epoch": 0.023931335475903752,
+    "step": 780
+  },
+  {
+    "loss": 0.2117,
+    "grad_norm": 0.3304797112941742,
+    "learning_rate": 9.907795784955327e-05,
+    "epoch": 0.024544959462465387,
+    "step": 800
+  },
+  {
+    "loss": 0.2091,
+    "grad_norm": 0.32046860456466675,
+    "learning_rate": 9.901566971668437e-05,
+    "epoch": 0.025158583449027022,
+    "step": 820
+  },
+  {
+    "loss": 0.201,
+    "grad_norm": 0.516817569732666,
+    "learning_rate": 9.895136674161465e-05,
+    "epoch": 0.025772207435588657,
+    "step": 840
+  },
+  {
+    "loss": 0.2117,
+    "grad_norm": 0.35681387782096863,
+    "learning_rate": 9.888505156758759e-05,
+    "epoch": 0.026385831422150292,
+    "step": 860
+  },
+  {
+    "loss": 0.1972,
+    "grad_norm": 0.6307485103607178,
+    "learning_rate": 9.881672692056021e-05,
+    "epoch": 0.026999455408711927,
+    "step": 880
+  },
+  {
+    "loss": 0.2081,
+    "grad_norm": 0.5634998083114624,
+    "learning_rate": 9.874639560909117e-05,
+    "epoch": 0.027613079395273563,
+    "step": 900
+  },
+  {
+    "loss": 0.2022,
+    "grad_norm": 0.38356727361679077,
+    "learning_rate": 9.867406052422524e-05,
+    "epoch": 0.028226703381835194,
+    "step": 920
+  },
+  {
+    "loss": 0.1949,
+    "grad_norm": 0.6488237977027893,
+    "learning_rate": 9.859972463937441e-05,
+    "epoch": 0.02884032736839683,
+    "step": 940
+  },
+  {
+    "loss": 0.2012,
+    "grad_norm": 0.4440819025039673,
+    "learning_rate": 9.852339101019574e-05,
+    "epoch": 0.029453951354958464,
+    "step": 960
+  },
+  {
+    "loss": 0.2108,
+    "grad_norm": 0.6403115391731262,
+    "learning_rate": 9.844506277446577e-05,
+    "epoch": 0.0300675753415201,
+    "step": 980
+  },
+  {
+    "loss": 0.1907,
+    "grad_norm": 0.41512686014175415,
+    "learning_rate": 9.836474315195147e-05,
+    "epoch": 0.030681199328081735,
+    "step": 1000
+  },
+  {
+    "eval_loss": 0.19328325986862183,
+    "eval_runtime": 2767.2068,
+    "eval_samples_per_second": 161.947,
+    "eval_steps_per_second": 2.531,
+    "epoch": 0.030681199328081735,
+    "step": 1000
+  },
+  {
+    "loss": 0.1993,
+    "grad_norm": 0.44694626331329346,
+    "learning_rate": 9.828243544427796e-05,
+    "epoch": 0.031294823314643366,
+    "step": 1020
+  },
+  {
+    "loss": 0.1997,
+    "grad_norm": 0.3413143754005432,
+    "learning_rate": 9.819814303479267e-05,
+    "epoch": 0.031908447301205005,
+    "step": 1040
+  },
+  {
+    "loss": 0.1823,
+    "grad_norm": 0.40824049711227417,
+    "learning_rate": 9.811186938842645e-05,
+    "epoch": 0.032522071287766637,
+    "step": 1060
+  },
+  {
+    "loss": 0.2114,
+    "grad_norm": 1.721802830696106,
+    "learning_rate": 9.802361805155097e-05,
+    "epoch": 0.033135695274328275,
+    "step": 1080
+  },
+  {
+    "loss": 0.2016,
+    "grad_norm": 0.4164980351924896,
+    "learning_rate": 9.793339265183303e-05,
+    "epoch": 0.03374931926088991,
+    "step": 1100
+  },
+  {
+    "loss": 0.1798,
+    "grad_norm": 1.188069224357605,
+    "learning_rate": 9.784119689808544e-05,
+    "epoch": 0.034362943247451545,
+    "step": 1120
+  },
+  {
+    "loss": 0.1893,
+    "grad_norm": 0.41302207112312317,
+    "learning_rate": 9.774703458011453e-05,
+    "epoch": 0.03497656723401318,
+    "step": 1140
+  },
+  {
+    "loss": 0.1864,
+    "grad_norm": 0.5009692907333374,
+    "learning_rate": 9.765090956856436e-05,
+    "epoch": 0.035590191220574816,
+    "step": 1160
+  },
+  {
+    "loss": 0.1823,
+    "grad_norm": 0.5319084525108337,
+    "learning_rate": 9.755282581475769e-05,
+    "epoch": 0.03620381520713645,
+    "step": 1180
+  },
+  {
+    "loss": 0.1885,
+    "grad_norm": 0.286171555519104,
+    "learning_rate": 9.745278735053343e-05,
+    "epoch": 0.03681743919369808,
+    "step": 1200
+  },
+  {
+    "loss": 0.1831,
+    "grad_norm": 0.38070616126060486,
+    "learning_rate": 9.735079828808107e-05,
+    "epoch": 0.03743106318025972,
+    "step": 1220
+  },
+  {
+    "loss": 0.1815,
+    "grad_norm": 0.37657469511032104,
+    "learning_rate": 9.724686281977146e-05,
+    "epoch": 0.03804468716682135,
+    "step": 1240
+  },
+  {
+    "loss": 0.174,
+    "grad_norm": 0.3888895511627197,
+    "learning_rate": 9.714098521798465e-05,
+    "epoch": 0.03865831115338299,
+    "step": 1260
+  },
+  {
+    "loss": 0.1789,
+    "grad_norm": 0.3658376932144165,
+    "learning_rate": 9.703316983493414e-05,
+    "epoch": 0.03927193513994462,
+    "step": 1280
+  },
+  {
+    "loss": 0.1801,
+    "grad_norm": 0.5093252062797546,
+    "learning_rate": 9.692342110248802e-05,
+    "epoch": 0.03988555912650626,
+    "step": 1300
+  },
+  {
+    "loss": 0.1847,
+    "grad_norm": 0.6297820806503296,
+    "learning_rate": 9.681174353198687e-05,
+    "epoch": 0.04049918311306789,
+    "step": 1320
+  },
+  {
+    "loss": 0.1721,
+    "grad_norm": 1.3242509365081787,
+    "learning_rate": 9.669814171405816e-05,
+    "epoch": 0.04111280709962952,
+    "step": 1340
+  },
+  {
+    "loss": 0.1799,
+    "grad_norm": 0.3201231360435486,
+    "learning_rate": 9.65826203184277e-05,
+    "epoch": 0.04172643108619116,
+    "step": 1360
+  },
+  {
+    "loss": 0.1776,
+    "grad_norm": 0.6388235688209534,
+    "learning_rate": 9.64651840937276e-05,
+    "epoch": 0.04234005507275279,
+    "step": 1380
+  },
+  {
+    "loss": 0.1883,
+    "grad_norm": 0.379491925239563,
+    "learning_rate": 9.63458378673011e-05,
+    "epoch": 0.04295367905931443,
+    "step": 1400
+  },
+  {
+    "loss": 0.1673,
+    "grad_norm": 0.29706379771232605,
+    "learning_rate": 9.622458654500409e-05,
+    "epoch": 0.04356730304587606,
+    "step": 1420
+  },
+  {
+    "loss": 0.1757,
+    "grad_norm": 0.31686845421791077,
+    "learning_rate": 9.610143511100354e-05,
+    "epoch": 0.0441809270324377,
+    "step": 1440
+  },
+  {
+    "loss": 0.175,
+    "grad_norm": 0.24471138417720795,
+    "learning_rate": 9.597638862757255e-05,
+    "epoch": 0.04479455101899933,
+    "step": 1460
+  },
+  {
+    "loss": 0.1823,
+    "grad_norm": 0.7488877177238464,
+    "learning_rate": 9.584945223488227e-05,
+    "epoch": 0.04540817500556097,
+    "step": 1480
+  },
+  {
+    "loss": 0.174,
+    "grad_norm": 0.3987618386745453,
+    "learning_rate": 9.572063115079063e-05,
+    "epoch": 0.0460217989921226,
+    "step": 1500
+  },
+  {
+    "eval_loss": 0.1716459095478058,
+    "eval_runtime": 2769.3725,
+    "eval_samples_per_second": 161.82,
+    "eval_steps_per_second": 2.529,
+    "epoch": 0.0460217989921226,
+    "step": 1500
+  },
+  {
+    "loss": 0.1752,
+    "grad_norm": 0.44050776958465576,
+    "learning_rate": 9.558993067062785e-05,
+    "epoch": 0.046635422978684234,
+    "step": 1520
+  },
+  {
+    "loss": 0.1733,
+    "grad_norm": 0.3778401017189026,
+    "learning_rate": 9.545735616697875e-05,
+    "epoch": 0.04724904696524587,
+    "step": 1540
+  },
+  {
+    "loss": 0.1643,
+    "grad_norm": 0.2891245484352112,
+    "learning_rate": 9.53229130894619e-05,
+    "epoch": 0.047862670951807504,
+    "step": 1560
+  },
+  {
+    "loss": 0.1734,
+    "grad_norm": 0.9185692667961121,
+    "learning_rate": 9.518660696450568e-05,
+    "epoch": 0.04847629493836914,
+    "step": 1580
+  },
+  {
+    "loss": 0.1786,
+    "grad_norm": 0.4238841235637665,
+    "learning_rate": 9.504844339512095e-05,
+    "epoch": 0.049089918924930774,
+    "step": 1600
+  },
+  {
+    "loss": 0.1752,
+    "grad_norm": 0.24845579266548157,
+    "learning_rate": 9.490842806067095e-05,
+    "epoch": 0.04970354291149241,
+    "step": 1620
+  },
+  {
+    "loss": 0.1817,
+    "grad_norm": 0.3146928548812866,
+    "learning_rate": 9.476656671663765e-05,
+    "epoch": 0.050317166898054044,
+    "step": 1640
+  },
+  {
+    "loss": 0.1647,
+    "grad_norm": 0.26675552129745483,
+    "learning_rate": 9.46228651943853e-05,
+    "epoch": 0.05093079088461568,
+    "step": 1660
+  },
+  {
+    "loss": 0.169,
+    "grad_norm": 0.47638726234436035,
+    "learning_rate": 9.44773294009206e-05,
+    "epoch": 0.051544414871177315,
+    "step": 1680
+  },
+  {
+    "loss": 0.1652,
+    "grad_norm": 0.5169867873191833,
+    "learning_rate": 9.432996531865002e-05,
+    "epoch": 0.052158038857738946,
+    "step": 1700
+  },
+  {
+    "loss": 0.1633,
+    "grad_norm": 0.44044116139411926,
+    "learning_rate": 9.418077900513377e-05,
+    "epoch": 0.052771662844300585,
+    "step": 1720
+  },
+  {
+    "loss": 0.1622,
+    "grad_norm": 0.24241219460964203,
+    "learning_rate": 9.40297765928369e-05,
+    "epoch": 0.053385286830862216,
+    "step": 1740
+  },
+  {
+    "loss": 0.1655,
+    "grad_norm": 0.21895462274551392,
+    "learning_rate": 9.387696428887716e-05,
+    "epoch": 0.053998910817423855,
+    "step": 1760
+  },
+  {
+    "loss": 0.1658,
+    "grad_norm": 0.2668932378292084,
+    "learning_rate": 9.372234837476978e-05,
+    "epoch": 0.05461253480398549,
+    "step": 1780
+  },
+  {
+    "loss": 0.1708,
+    "grad_norm": 0.2590928375720978,
+    "learning_rate": 9.356593520616948e-05,
+    "epoch": 0.055226158790547125,
+    "step": 1800
+  },
+  {
+    "loss": 0.1692,
+    "grad_norm": 1.094123125076294,
+    "learning_rate": 9.340773121260893e-05,
+    "epoch": 0.05583978277710876,
+    "step": 1820
+  },
+  {
+    "loss": 0.1609,
+    "grad_norm": 0.5052621364593506,
+    "learning_rate": 9.324774289723468e-05,
+    "epoch": 0.05645340676367039,
+    "step": 1840
+  },
+  {
+    "loss": 0.1665,
+    "grad_norm": 0.35525742173194885,
+    "learning_rate": 9.308597683653975e-05,
+    "epoch": 0.05706703075023203,
+    "step": 1860
+  },
+  {
+    "loss": 0.1644,
+    "grad_norm": 0.5005172491073608,
+    "learning_rate": 9.292243968009331e-05,
+    "epoch": 0.05768065473679366,
+    "step": 1880
+  },
+  {
+    "loss": 0.1697,
+    "grad_norm": 0.23915395140647888,
+    "learning_rate": 9.275713815026731e-05,
+    "epoch": 0.0582942787233553,
+    "step": 1900
+  },
+  {
+    "loss": 0.1616,
+    "grad_norm": 0.23820538818836212,
+    "learning_rate": 9.259007904196023e-05,
+    "epoch": 0.05890790270991693,
+    "step": 1920
+  },
+  {
+    "loss": 0.1562,
+    "grad_norm": 0.19122985005378723,
+    "learning_rate": 9.242126922231763e-05,
+    "epoch": 0.05952152669647857,
+    "step": 1940
+  },
+  {
+    "loss": 0.1568,
+    "grad_norm": 0.24955634772777557,
+    "learning_rate": 9.225071563045007e-05,
+    "epoch": 0.0601351506830402,
+    "step": 1960
+  },
+  {
+    "loss": 0.1559,
+    "grad_norm": 0.2041475921869278,
+    "learning_rate": 9.207842527714767e-05,
+    "epoch": 0.06074877466960184,
+    "step": 1980
+  },
+  {
+    "loss": 0.1611,
+    "grad_norm": 0.38136520981788635,
+    "learning_rate": 9.190440524459203e-05,
+    "epoch": 0.06136239865616347,
+    "step": 2000
+  },
+  {
+    "eval_loss": 0.15973800420761108,
+    "eval_runtime": 2770.9129,
+    "eval_samples_per_second": 161.73,
+    "eval_steps_per_second": 2.527,
+    "epoch": 0.06136239865616347,
+    "step": 2000
+  },
+  {
+    "loss": 0.1515,
+    "grad_norm": 0.27136150002479553,
+    "learning_rate": 9.172866268606513e-05,
+    "epoch": 0.0619760226427251,
+    "step": 2020
+  },
+  {
+    "loss": 0.1589,
+    "grad_norm": 0.2267642617225647,
+    "learning_rate": 9.155120482565521e-05,
+    "epoch": 0.06258964662928673,
+    "step": 2040
+  },
+  {
+    "loss": 0.1527,
+    "grad_norm": 0.23289571702480316,
+    "learning_rate": 9.137203895795983e-05,
+    "epoch": 0.06320327061584838,
+    "step": 2060
+  },
+  {
+    "loss": 0.1636,
+    "grad_norm": 0.7123669385910034,
+    "learning_rate": 9.119117244778607e-05,
+    "epoch": 0.06381689460241001,
+    "step": 2080
+  },
+  {
+    "loss": 0.1559,
+    "grad_norm": 0.28865331411361694,
+    "learning_rate": 9.10086127298478e-05,
+    "epoch": 0.06443051858897164,
+    "step": 2100
+  },
+  {
+    "loss": 0.1614,
+    "grad_norm": 0.35813504457473755,
+    "learning_rate": 9.082436730845993e-05,
+    "epoch": 0.06504414257553327,
+    "step": 2120
+  },
+  {
+    "loss": 0.1567,
+    "grad_norm": 0.3127194344997406,
+    "learning_rate": 9.063844375723014e-05,
+    "epoch": 0.06565776656209492,
+    "step": 2140
+  },
+  {
+    "loss": 0.1613,
+    "grad_norm": 0.4829489588737488,
+    "learning_rate": 9.045084971874738e-05,
+    "epoch": 0.06627139054865655,
+    "step": 2160
+  },
+  {
+    "loss": 0.158,
+    "grad_norm": 0.5470940470695496,
+    "learning_rate": 9.02615929042678e-05,
+    "epoch": 0.06688501453521818,
+    "step": 2180
+  },
+  {
+    "loss": 0.1536,
+    "grad_norm": 0.2029639482498169,
+    "learning_rate": 9.007068109339784e-05,
+    "epoch": 0.06749863852177981,
+    "step": 2200
+  },
+  {
+    "loss": 0.159,
+    "grad_norm": 0.45802491903305054,
+    "learning_rate": 8.987812213377424e-05,
+    "epoch": 0.06811226250834145,
+    "step": 2220
+  },
+  {
+    "loss": 0.1571,
+    "grad_norm": 0.3373512923717499,
+    "learning_rate": 8.968392394074164e-05,
+    "epoch": 0.06872588649490309,
+    "step": 2240
+  },
+  {
+    "loss": 0.1547,
+    "grad_norm": 0.2957380712032318,
+    "learning_rate": 8.948809449702711e-05,
+    "epoch": 0.06933951048146472,
+    "step": 2260
+  },
+  {
+    "loss": 0.1566,
+    "grad_norm": 0.45103269815444946,
+    "learning_rate": 8.929064185241213e-05,
+    "epoch": 0.06995313446802635,
+    "step": 2280
+  },
+  {
+    "loss": 0.1544,
+    "grad_norm": 0.21954606473445892,
+    "learning_rate": 8.90915741234015e-05,
+    "epoch": 0.07056675845458799,
+    "step": 2300
+  },
+  {
+    "loss": 0.1539,
+    "grad_norm": 0.3216409385204315,
+    "learning_rate": 8.889089949288986e-05,
+    "epoch": 0.07118038244114963,
+    "step": 2320
+  },
+  {
+    "loss": 0.1515,
+    "grad_norm": 0.32575568556785583,
+    "learning_rate": 8.868862620982534e-05,
+    "epoch": 0.07179400642771126,
+    "step": 2340
+  },
+  {
+    "loss": 0.1519,
+    "grad_norm": 0.3038860559463501,
+    "learning_rate": 8.848476258887031e-05,
+    "epoch": 0.0724076304142729,
+    "step": 2360
+  },
+  {
+    "loss": 0.1529,
+    "grad_norm": 0.3296835720539093,
+    "learning_rate": 8.827931701005974e-05,
+    "epoch": 0.07302125440083453,
+    "step": 2380
+  },
+  {
+    "loss": 0.1491,
+    "grad_norm": 0.2646135687828064,
+    "learning_rate": 8.807229791845673e-05,
+    "epoch": 0.07363487838739616,
+    "step": 2400
+  },
+  {
+    "loss": 0.1587,
+    "grad_norm": 0.303659588098526,
+    "learning_rate": 8.786371382380528e-05,
+    "epoch": 0.0742485023739578,
+    "step": 2420
+  },
+  {
+    "loss": 0.1554,
+    "grad_norm": 0.5423962473869324,
+    "learning_rate": 8.765357330018056e-05,
+    "epoch": 0.07486212636051943,
+    "step": 2440
+  },
+  {
+    "loss": 0.1531,
+    "grad_norm": 0.22406215965747833,
+    "learning_rate": 8.744188498563641e-05,
+    "epoch": 0.07547575034708107,
+    "step": 2460
+  },
+  {
+    "loss": 0.1556,
+    "grad_norm": 0.32763445377349854,
+    "learning_rate": 8.722865758185035e-05,
+    "epoch": 0.0760893743336427,
+    "step": 2480
+  },
+  {
+    "loss": 0.1547,
+    "grad_norm": 0.23374123871326447,
+    "learning_rate": 8.701389985376578e-05,
+    "epoch": 0.07670299832020434,
+    "step": 2500
+  },
+  {
+    "eval_loss": 0.1528489738702774,
+    "eval_runtime": 2771.0889,
+    "eval_samples_per_second": 161.72,
+    "eval_steps_per_second": 2.527,
+    "epoch": 0.07670299832020434,
+    "step": 2500
+  },
+  {
+    "loss": 0.1534,
+    "grad_norm": 0.3874337673187256,
+    "learning_rate": 8.679762062923175e-05,
+    "epoch": 0.07731662230676598,
+    "step": 2520
+  },
+  {
+    "loss": 0.1567,
+    "grad_norm": 0.18201296031475067,
+    "learning_rate": 8.657982879864007e-05,
+    "epoch": 0.0779302462933276,
+    "step": 2540
+  },
+  {
+    "loss": 0.1485,
+    "grad_norm": 0.30263030529022217,
+    "learning_rate": 8.636053331455987e-05,
+    "epoch": 0.07854387027988924,
+    "step": 2560
+  },
+  {
+    "loss": 0.1497,
+    "grad_norm": 0.47069185972213745,
+    "learning_rate": 8.613974319136958e-05,
+    "epoch": 0.07915749426645087,
+    "step": 2580
+  },
+  {
+    "loss": 0.1583,
+    "grad_norm": 0.24262110888957977,
+    "learning_rate": 8.591746750488639e-05,
+    "epoch": 0.07977111825301252,
+    "step": 2600
+  },
+  {
+    "loss": 0.1562,
+    "grad_norm": 0.22001530230045319,
+    "learning_rate": 8.569371539199316e-05,
+    "epoch": 0.08038474223957415,
+    "step": 2620
+  },
+  {
+    "loss": 0.1628,
+    "grad_norm": 0.21581719815731049,
+    "learning_rate": 8.54684960502629e-05,
+    "epoch": 0.08099836622613578,
+    "step": 2640
+  },
+  {
+    "loss": 0.1597,
+    "grad_norm": 0.34623074531555176,
+    "learning_rate": 8.524181873758059e-05,
+    "epoch": 0.08161199021269741,
+    "step": 2660
+  },
+  {
+    "loss": 0.153,
+    "grad_norm": 0.17073972523212433,
+    "learning_rate": 8.501369277176276e-05,
+    "epoch": 0.08222561419925904,
+    "step": 2680
+  },
+  {
+    "loss": 0.1584,
+    "grad_norm": 0.41088077425956726,
+    "learning_rate": 8.478412753017433e-05,
+    "epoch": 0.08283923818582069,
+    "step": 2700
+  },
+  {
+    "loss": 0.1635,
+    "grad_norm": 0.22260364890098572,
+    "learning_rate": 8.455313244934324e-05,
+    "epoch": 0.08345286217238232,
+    "step": 2720
+  },
+  {
+    "loss": 0.156,
+    "grad_norm": 0.3336716592311859,
+    "learning_rate": 8.432071702457252e-05,
+    "epoch": 0.08406648615894395,
+    "step": 2740
+  },
+  {
+    "loss": 0.1558,
+    "grad_norm": 0.4109443426132202,
+    "learning_rate": 8.408689080954998e-05,
+    "epoch": 0.08468011014550558,
+    "step": 2760
+  },
+  {
+    "loss": 0.1552,
+    "grad_norm": 0.3245985805988312,
+    "learning_rate": 8.385166341595548e-05,
+    "epoch": 0.08529373413206723,
+    "step": 2780
+  },
+  {
+    "loss": 0.1524,
+    "grad_norm": 0.2673623859882355,
+    "learning_rate": 8.361504451306585e-05,
+    "epoch": 0.08590735811862886,
+    "step": 2800
+  },
+  {
+    "loss": 0.1495,
+    "grad_norm": 1.2059513330459595,
+    "learning_rate": 8.33770438273574e-05,
+    "epoch": 0.08652098210519049,
+    "step": 2820
+  },
+  {
+    "loss": 0.1564,
+    "grad_norm": 0.24513472616672516,
+    "learning_rate": 8.313767114210615e-05,
+    "epoch": 0.08713460609175212,
+    "step": 2840
+  },
+  {
+    "loss": 0.1546,
+    "grad_norm": 0.2619301378726959,
+    "learning_rate": 8.289693629698564e-05,
+    "epoch": 0.08774823007831375,
+    "step": 2860
+  },
+  {
+    "loss": 0.1564,
+    "grad_norm": 0.16868187487125397,
+    "learning_rate": 8.265484918766243e-05,
+    "epoch": 0.0883618540648754,
+    "step": 2880
+  },
+  {
+    "loss": 0.1564,
+    "grad_norm": 0.2369966059923172,
+    "learning_rate": 8.241141976538943e-05,
+    "epoch": 0.08897547805143703,
+    "step": 2900
+  },
+  {
+    "loss": 0.1511,
+    "grad_norm": 0.23205088078975677,
+    "learning_rate": 8.216665803659671e-05,
+    "epoch": 0.08958910203799866,
+    "step": 2920
+  },
+  {
+    "loss": 0.157,
+    "grad_norm": 0.19588901102542877,
+    "learning_rate": 8.192057406248028e-05,
+    "epoch": 0.0902027260245603,
+    "step": 2940
+  },
+  {
+    "loss": 0.1496,
+    "grad_norm": 0.3407858908176422,
+    "learning_rate": 8.167317795858851e-05,
+    "epoch": 0.09081635001112194,
+    "step": 2960
+  },
+  {
+    "loss": 0.151,
+    "grad_norm": 0.2402908205986023,
+    "learning_rate": 8.142447989440618e-05,
+    "epoch": 0.09142997399768357,
+    "step": 2980
+  },
+  {
+    "loss": 0.1567,
+    "grad_norm": 0.23841014504432678,
+    "learning_rate": 8.117449009293668e-05,
+    "epoch": 0.0920435979842452,
+    "step": 3000
+  },
+  {
+    "eval_loss": 0.150652676820755,
+    "eval_runtime": 2770.5122,
+    "eval_samples_per_second": 161.753,
+    "eval_steps_per_second": 2.528,
+    "epoch": 0.0920435979842452,
+    "step": 3000
+  },
+  {
+    "loss": 0.1551,
+    "grad_norm": 0.20382452011108398,
+    "learning_rate": 8.092321883028158e-05,
+    "epoch": 0.09265722197080684,
+    "step": 3020
+  },
+  {
+    "loss": 0.1511,
+    "grad_norm": 0.28016752004623413,
+    "learning_rate": 8.067067643521834e-05,
+    "epoch": 0.09327084595736847,
+    "step": 3040
+  },
+  {
+    "loss": 0.1571,
+    "grad_norm": 0.24572430551052094,
+    "learning_rate": 8.041687328877567e-05,
+    "epoch": 0.09388446994393011,
+    "step": 3060
+  },
+  {
+    "loss": 0.1475,
+    "grad_norm": 0.31108465790748596,
+    "learning_rate": 8.016181982380682e-05,
+    "epoch": 0.09449809393049174,
+    "step": 3080
+  },
+  {
+    "loss": 0.151,
+    "grad_norm": 0.1946084052324295,
+    "learning_rate": 7.990552652456081e-05,
+    "epoch": 0.09511171791705338,
+    "step": 3100
+  },
+  {
+    "loss": 0.156,
+    "grad_norm": 0.2956799864768982,
+    "learning_rate": 7.964800392625129e-05,
+    "epoch": 0.09572534190361501,
+    "step": 3120
+  },
+  {
+    "loss": 0.1505,
+    "grad_norm": 0.6577198505401611,
+    "learning_rate": 7.938926261462366e-05,
+    "epoch": 0.09633896589017665,
+    "step": 3140
+  },
+  {
+    "loss": 0.1542,
+    "grad_norm": 0.16438567638397217,
+    "learning_rate": 7.91293132255198e-05,
+    "epoch": 0.09695258987673828,
+    "step": 3160
+  },
+  {
+    "loss": 0.1474,
+    "grad_norm": 0.19348329305648804,
+    "learning_rate": 7.886816644444098e-05,
+    "epoch": 0.09756621386329992,
+    "step": 3180
+  },
+  {
+    "loss": 0.1559,
+    "grad_norm": 0.3346814811229706,
+    "learning_rate": 7.860583300610849e-05,
+    "epoch": 0.09817983784986155,
+    "step": 3200
+  },
+  {
+    "loss": 0.1471,
+    "grad_norm": 0.1847156286239624,
+    "learning_rate": 7.83423236940225e-05,
+    "epoch": 0.09879346183642318,
+    "step": 3220
+  },
+  {
+    "loss": 0.1443,
+    "grad_norm": 0.5336312055587769,
+    "learning_rate": 7.807764934001874e-05,
+    "epoch": 0.09940708582298483,
+    "step": 3240
+  },
+  {
+    "loss": 0.1499,
+    "grad_norm": 0.23365505039691925,
+    "learning_rate": 7.781182082382325e-05,
+    "epoch": 0.10002070980954646,
+    "step": 3260
+  },
+  {
+    "loss": 0.1552,
+    "grad_norm": 0.20067407190799713,
+    "learning_rate": 7.754484907260513e-05,
+    "epoch": 0.10063433379610809,
+    "step": 3280
+  },
+  {
+    "loss": 0.1485,
+    "grad_norm": 0.18101651966571808,
+    "learning_rate": 7.727674506052743e-05,
+    "epoch": 0.10124795778266972,
+    "step": 3300
+  },
+  {
+    "loss": 0.1482,
+    "grad_norm": 0.20169106125831604,
+    "learning_rate": 7.700751980829602e-05,
+    "epoch": 0.10186158176923137,
+    "step": 3320
+  },
+  {
+    "loss": 0.1426,
+    "grad_norm": 0.15832331776618958,
+    "learning_rate": 7.673718438270648e-05,
+    "epoch": 0.102475205755793,
+    "step": 3340
+  },
+  {
+    "loss": 0.1467,
+    "grad_norm": 0.2241671085357666,
+    "learning_rate": 7.646574989618938e-05,
+    "epoch": 0.10308882974235463,
+    "step": 3360
+  },
+  {
+    "loss": 0.1436,
+    "grad_norm": 0.14387348294258118,
+    "learning_rate": 7.619322750635327e-05,
+    "epoch": 0.10370245372891626,
+    "step": 3380
+  },
+  {
+    "loss": 0.1472,
+    "grad_norm": 0.3439478576183319,
+    "learning_rate": 7.591962841552627e-05,
+    "epoch": 0.10431607771547789,
+    "step": 3400
+  },
+  {
+    "loss": 0.1451,
+    "grad_norm": 0.2688335180282593,
+    "learning_rate": 7.564496387029532e-05,
+    "epoch": 0.10492970170203954,
+    "step": 3420
+  },
+  {
+    "loss": 0.1529,
+    "grad_norm": 0.5446487069129944,
+    "learning_rate": 7.536924516104411e-05,
+    "epoch": 0.10554332568860117,
+    "step": 3440
+  },
+  {
+    "loss": 0.1548,
+    "grad_norm": 0.35945549607276917,
+    "learning_rate": 7.509248362148889e-05,
+    "epoch": 0.1061569496751628,
+    "step": 3460
+  },
+  {
+    "loss": 0.1439,
+    "grad_norm": 0.4427342116832733,
+    "learning_rate": 7.481469062821252e-05,
+    "epoch": 0.10677057366172443,
+    "step": 3480
+  },
+  {
+    "loss": 0.1514,
+    "grad_norm": 0.2129109799861908,
+    "learning_rate": 7.45358776001969e-05,
+    "epoch": 0.10738419764828608,
+    "step": 3500
+  },
+  {
+    "eval_loss": 0.1471174657344818,
+    "eval_runtime": 2771.0363,
+    "eval_samples_per_second": 161.723,
+    "eval_steps_per_second": 2.527,
+    "epoch": 0.10738419764828608,
+    "step": 3500
+  },
+  {
+    "loss": 0.1463,
+    "grad_norm": 0.32425713539123535,
+    "learning_rate": 7.425605599835361e-05,
+    "epoch": 0.10799782163484771,
+    "step": 3520
+  },
+  {
+    "loss": 0.1424,
+    "grad_norm": 0.39084091782569885,
+    "learning_rate": 7.39752373250527e-05,
+    "epoch": 0.10861144562140934,
+    "step": 3540
+  },
+  {
+    "loss": 0.1437,
+    "grad_norm": 0.5381523370742798,
+    "learning_rate": 7.369343312364993e-05,
+    "epoch": 0.10922506960797097,
+    "step": 3560
+  },
+  {
+    "loss": 0.1447,
+    "grad_norm": 0.12256123870611191,
+    "learning_rate": 7.34106549780123e-05,
+    "epoch": 0.1098386935945326,
+    "step": 3580
+  },
+  {
+    "loss": 0.1415,
+    "grad_norm": 0.16140857338905334,
+    "learning_rate": 7.312691451204178e-05,
+    "epoch": 0.11045231758109425,
+    "step": 3600
+  },
+  {
+    "loss": 0.15,
+    "grad_norm": 0.1626661866903305,
+    "learning_rate": 7.284222338919758e-05,
+    "epoch": 0.11106594156765588,
+    "step": 3620
+  },
+  {
+    "loss": 0.1508,
+    "grad_norm": 0.17792241275310516,
+    "learning_rate": 7.255659331201673e-05,
+    "epoch": 0.11167956555421751,
+    "step": 3640
+  },
+  {
+    "loss": 0.1468,
+    "grad_norm": 0.24779079854488373,
+    "learning_rate": 7.227003602163295e-05,
+    "epoch": 0.11229318954077915,
+    "step": 3660
+  },
+  {
+    "loss": 0.1495,
+    "grad_norm": 0.1464058756828308,
+    "learning_rate": 7.198256329729412e-05,
+    "epoch": 0.11290681352734078,
+    "step": 3680
+  },
+  {
+    "loss": 0.1465,
+    "grad_norm": 0.2891758680343628,
+    "learning_rate": 7.169418695587791e-05,
+    "epoch": 0.11352043751390242,
+    "step": 3700
+  },
+  {
+    "loss": 0.1433,
+    "grad_norm": 0.12943851947784424,
+    "learning_rate": 7.14049188514063e-05,
+    "epoch": 0.11413406150046405,
+    "step": 3720
+  },
+  {
+    "loss": 0.1425,
+    "grad_norm": 0.24431823194026947,
+    "learning_rate": 7.1114770874558e-05,
+    "epoch": 0.11474768548702569,
+    "step": 3740
+  },
+  {
+    "loss": 0.1478,
+    "grad_norm": 0.19931627810001373,
+    "learning_rate": 7.082375495217995e-05,
+    "epoch": 0.11536130947358732,
+    "step": 3760
+  },
+  {
+    "loss": 0.1447,
+    "grad_norm": 0.2738446295261383,
+    "learning_rate": 7.05318830467969e-05,
+    "epoch": 0.11597493346014896,
+    "step": 3780
+  },
+  {
+    "loss": 0.147,
+    "grad_norm": 0.18363946676254272,
+    "learning_rate": 7.023916715611969e-05,
+    "epoch": 0.1165885574467106,
+    "step": 3800
+  },
+  {
+    "loss": 0.1484,
+    "grad_norm": 0.21691597998142242,
+    "learning_rate": 6.99456193125521e-05,
+    "epoch": 0.11720218143327223,
+    "step": 3820
+  },
+  {
+    "loss": 0.144,
+    "grad_norm": 0.29352739453315735,
+    "learning_rate": 6.965125158269619e-05,
+    "epoch": 0.11781580541983386,
+    "step": 3840
+  },
+  {
+    "loss": 0.1482,
+    "grad_norm": 0.15962694585323334,
+    "learning_rate": 6.935607606685642e-05,
+    "epoch": 0.11842942940639549,
+    "step": 3860
+  },
+  {
+    "loss": 0.1476,
+    "grad_norm": 0.2411005049943924,
+    "learning_rate": 6.906010489854209e-05,
+    "epoch": 0.11904305339295713,
+    "step": 3880
+  },
+  {
+    "loss": 0.1461,
+    "grad_norm": 0.3792065382003784,
+    "learning_rate": 6.876335024396872e-05,
+    "epoch": 0.11965667737951877,
+    "step": 3900
+  },
+  {
+    "loss": 0.1456,
+    "grad_norm": 0.21769222617149353,
+    "learning_rate": 6.846582430155783e-05,
+    "epoch": 0.1202703013660804,
+    "step": 3920
+  },
+  {
+    "loss": 0.1407,
+    "grad_norm": 0.12780731916427612,
+    "learning_rate": 6.816753930143558e-05,
+    "epoch": 0.12088392535264203,
+    "step": 3940
+  },
+  {
+    "loss": 0.143,
+    "grad_norm": 0.18131625652313232,
+    "learning_rate": 6.786850750493006e-05,
+    "epoch": 0.12149754933920368,
+    "step": 3960
+  },
+  {
+    "loss": 0.1451,
+    "grad_norm": 0.15587596595287323,
+    "learning_rate": 6.756874120406714e-05,
+    "epoch": 0.12211117332576531,
+    "step": 3980
+  },
+  {
+    "loss": 0.1441,
+    "grad_norm": 0.2455044686794281,
+    "learning_rate": 6.726825272106538e-05,
+    "epoch": 0.12272479731232694,
+    "step": 4000
+  },
+  {
+    "eval_loss": 0.1430002897977829,
+    "eval_runtime": 2770.4767,
+    "eval_samples_per_second": 161.756,
+    "eval_steps_per_second": 2.528,
+    "epoch": 0.12272479731232694,
+    "step": 4000
+  },
+  {
+    "loss": 0.1459,
+    "grad_norm": 0.24546217918395996,
+    "learning_rate": 6.696705440782938e-05,
+    "epoch": 0.12333842129888857,
+    "step": 4020
+  },
+  {
+    "loss": 0.142,
+    "grad_norm": 0.11568096280097961,
+    "learning_rate": 6.666515864544209e-05,
+    "epoch": 0.1239520452854502,
+    "step": 4040
+  },
+  {
+    "loss": 0.1564,
+    "grad_norm": 0.7489081621170044,
+    "learning_rate": 6.636257784365584e-05,
+    "epoch": 0.12456566927201185,
+    "step": 4060
+  },
+  {
+    "loss": 0.1428,
+    "grad_norm": 0.24135908484458923,
+    "learning_rate": 6.605932444038229e-05,
+    "epoch": 0.12517929325857347,
+    "step": 4080
+  },
+  {
+    "loss": 0.1484,
+    "grad_norm": 0.22589971125125885,
+    "learning_rate": 6.575541090118105e-05,
+    "epoch": 0.1257929172451351,
+    "step": 4100
+  },
+  {
+    "loss": 0.1473,
+    "grad_norm": 0.17859506607055664,
+    "learning_rate": 6.545084971874738e-05,
+    "epoch": 0.12640654123169676,
+    "step": 4120
+  },
+  {
+    "loss": 0.1465,
+    "grad_norm": 0.17700891196727753,
+    "learning_rate": 6.514565341239861e-05,
+    "epoch": 0.12702016521825837,
+    "step": 4140
+  },
+  {
+    "loss": 0.1434,
+    "grad_norm": 0.21767599880695343,
+    "learning_rate": 6.483983452755953e-05,
+    "epoch": 0.12763378920482002,
+    "step": 4160
+  },
+  {
+    "loss": 0.1423,
+    "grad_norm": 0.1528337001800537,
+    "learning_rate": 6.453340563524669e-05,
+    "epoch": 0.12824741319138164,
+    "step": 4180
+  },
+  {
+    "loss": 0.1407,
+    "grad_norm": 0.2475786954164505,
+    "learning_rate": 6.422637933155162e-05,
+    "epoch": 0.12886103717794328,
+    "step": 4200
+  },
+  {
+    "loss": 0.142,
+    "grad_norm": 0.2731213867664337,
+    "learning_rate": 6.391876823712317e-05,
+    "epoch": 0.12947466116450493,
+    "step": 4220
+  },
+  {
+    "loss": 0.1439,
+    "grad_norm": 0.4397956132888794,
+    "learning_rate": 6.361058499664856e-05,
+    "epoch": 0.13008828515106655,
+    "step": 4240
+  },
+  {
+    "loss": 0.1428,
+    "grad_norm": 0.20277808606624603,
+    "learning_rate": 6.330184227833376e-05,
+    "epoch": 0.1307019091376282,
+    "step": 4260
+  },
+  {
+    "loss": 0.1394,
+    "grad_norm": 0.1790492981672287,
+    "learning_rate": 6.299255277338265e-05,
+    "epoch": 0.13131553312418984,
+    "step": 4280
+  },
+  {
+    "loss": 0.1386,
+    "grad_norm": 0.181828111410141,
+    "learning_rate": 6.268272919547537e-05,
+    "epoch": 0.13192915711075145,
+    "step": 4300
+  },
+  {
+    "loss": 0.1459,
+    "grad_norm": 0.1925831288099289,
+    "learning_rate": 6.237238428024572e-05,
+    "epoch": 0.1325427810973131,
+    "step": 4320
+  },
+  {
+    "loss": 0.1424,
+    "grad_norm": 0.6288266181945801,
+    "learning_rate": 6.206153078475763e-05,
+    "epoch": 0.13315640508387472,
+    "step": 4340
+  },
+  {
+    "loss": 0.1454,
+    "grad_norm": 0.15917836129665375,
+    "learning_rate": 6.175018148698077e-05,
+    "epoch": 0.13377002907043636,
+    "step": 4360
+  },
+  {
+    "loss": 0.1437,
+    "grad_norm": 0.21755409240722656,
+    "learning_rate": 6.143834918526527e-05,
+    "epoch": 0.134383653056998,
+    "step": 4380
+  },
+  {
+    "loss": 0.1447,
+    "grad_norm": 0.14852923154830933,
+    "learning_rate": 6.112604669781572e-05,
+    "epoch": 0.13499727704355963,
+    "step": 4400
+  },
+  {
+    "loss": 0.1428,
+    "grad_norm": 0.20071613788604736,
+    "learning_rate": 6.081328686216418e-05,
+    "epoch": 0.13561090103012127,
+    "step": 4420
+  },
+  {
+    "loss": 0.1418,
+    "grad_norm": 0.13478811085224152,
+    "learning_rate": 6.0500082534642464e-05,
+    "epoch": 0.1362245250166829,
+    "step": 4440
+  },
+  {
+    "loss": 0.1513,
+    "grad_norm": 0.11774014681577682,
+    "learning_rate": 6.0186446589853784e-05,
+    "epoch": 0.13683814900324454,
+    "step": 4460
+  },
+  {
+    "loss": 0.1455,
+    "grad_norm": 0.3604123890399933,
+    "learning_rate": 5.987239192014336e-05,
+    "epoch": 0.13745177298980618,
+    "step": 4480
+  },
+  {
+    "loss": 0.1311,
+    "grad_norm": 0.19379453361034393,
+    "learning_rate": 5.955793143506863e-05,
+    "epoch": 0.1380653969763678,
+    "step": 4500
+  },
+  {
+    "eval_loss": 0.14158321917057037,
+    "eval_runtime": 2770.8316,
+    "eval_samples_per_second": 161.735,
+    "eval_steps_per_second": 2.527,
+    "epoch": 0.1380653969763678,
+    "step": 4500
+  },
+  {
+    "loss": 0.145,
+    "grad_norm": 0.1940959244966507,
+    "learning_rate": 5.924307806086844e-05,
+    "epoch": 0.13867902096292944,
+    "step": 4520
+  },
+  {
+    "loss": 0.1376,
+    "grad_norm": 0.17542797327041626,
+    "learning_rate": 5.8927844739931834e-05,
+    "epoch": 0.13929264494949106,
+    "step": 4540
+  },
+  {
+    "loss": 0.1369,
+    "grad_norm": 0.13041825592517853,
+    "learning_rate": 5.861224443026595e-05,
+    "epoch": 0.1399062689360527,
+    "step": 4560
+  },
+  {
+    "loss": 0.1406,
+    "grad_norm": 0.19473542273044586,
+    "learning_rate": 5.82962901049634e-05,
+    "epoch": 0.14051989292261435,
+    "step": 4580
+  },
+  {
+    "loss": 0.1442,
+    "grad_norm": 0.22222813963890076,
+    "learning_rate": 5.7979994751668964e-05,
+    "epoch": 0.14113351690917597,
+    "step": 4600
+  },
+  {
+    "loss": 0.14,
+    "grad_norm": 0.21635234355926514,
+    "learning_rate": 5.766337137204579e-05,
+    "epoch": 0.14174714089573762,
+    "step": 4620
+  },
+  {
+    "loss": 0.1359,
+    "grad_norm": 0.20376254618167877,
+    "learning_rate": 5.7346432981240904e-05,
+    "epoch": 0.14236076488229926,
+    "step": 4640
+  },
+  {
+    "loss": 0.146,
+    "grad_norm": 0.18470820784568787,
+    "learning_rate": 5.7029192607350146e-05,
+    "epoch": 0.14297438886886088,
+    "step": 4660
+  },
+  {
+    "loss": 0.1405,
+    "grad_norm": 0.14597751200199127,
+    "learning_rate": 5.6711663290882776e-05,
+    "epoch": 0.14358801285542253,
+    "step": 4680
+  },
+  {
+    "loss": 0.1347,
+    "grad_norm": 0.10637301951646805,
+    "learning_rate": 5.6393858084225305e-05,
+    "epoch": 0.14420163684198414,
+    "step": 4700
+  },
+  {
+    "loss": 0.1504,
+    "grad_norm": 0.1544061303138733,
+    "learning_rate": 5.6075790051105023e-05,
+    "epoch": 0.1448152608285458,
+    "step": 4720
+  },
+  {
+    "loss": 0.1426,
+    "grad_norm": 0.15634353458881378,
+    "learning_rate": 5.575747226605298e-05,
+    "epoch": 0.14542888481510743,
+    "step": 4740
+  },
+  {
+    "loss": 0.1389,
+    "grad_norm": 0.5837845802307129,
+    "learning_rate": 5.5438917813866554e-05,
+    "epoch": 0.14604250880166905,
+    "step": 4760
+  },
+  {
+    "loss": 0.1454,
+    "grad_norm": 0.43749943375587463,
+    "learning_rate": 5.512013978907157e-05,
+    "epoch": 0.1466561327882307,
+    "step": 4780
+  },
+  {
+    "loss": 0.1514,
+    "grad_norm": 0.18316301703453064,
+    "learning_rate": 5.480115129538409e-05,
+    "epoch": 0.14726975677479232,
+    "step": 4800
+  },
+  {
+    "loss": 0.1422,
+    "grad_norm": 0.17283789813518524,
+    "learning_rate": 5.448196544517168e-05,
+    "epoch": 0.14788338076135396,
+    "step": 4820
+  },
+  {
+    "loss": 0.1498,
+    "grad_norm": 0.2187805473804474,
+    "learning_rate": 5.416259535891447e-05,
+    "epoch": 0.1484970047479156,
+    "step": 4840
+  },
+  {
+    "loss": 0.1311,
+    "grad_norm": 0.160814106464386,
+    "learning_rate": 5.384305416466584e-05,
+    "epoch": 0.14911062873447722,
+    "step": 4860
+  },
+  {
+    "loss": 0.138,
+    "grad_norm": 0.5161686539649963,
+    "learning_rate": 5.35233549975127e-05,
+    "epoch": 0.14972425272103887,
+    "step": 4880
+  },
+  {
+    "loss": 0.1401,
+    "grad_norm": 0.11132976412773132,
+    "learning_rate": 5.320351099903565e-05,
+    "epoch": 0.1503378767076005,
+    "step": 4900
+  },
+  {
+    "loss": 0.1424,
+    "grad_norm": 0.13249003887176514,
+    "learning_rate": 5.288353531676873e-05,
+    "epoch": 0.15095150069416213,
+    "step": 4920
+  },
+  {
+    "loss": 0.142,
+    "grad_norm": 0.13185490667819977,
+    "learning_rate": 5.256344110365896e-05,
+    "epoch": 0.15156512468072378,
+    "step": 4940
+  },
+  {
+    "loss": 0.1423,
+    "grad_norm": 0.3014187216758728,
+    "learning_rate": 5.2243241517525754e-05,
+    "epoch": 0.1521787486672854,
+    "step": 4960
+  },
+  {
+    "loss": 0.1387,
+    "grad_norm": 0.09553083777427673,
+    "learning_rate": 5.192294972051992e-05,
+    "epoch": 0.15279237265384704,
+    "step": 4980
+  },
+  {
+    "loss": 0.1413,
+    "grad_norm": 0.37995925545692444,
+    "learning_rate": 5.1602578878582776e-05,
+    "epoch": 0.1534059966404087,
+    "step": 5000
+  },
+  {
+    "eval_loss": 0.1410340666770935,
+    "eval_runtime": 2772.7302,
+    "eval_samples_per_second": 161.624,
+    "eval_steps_per_second": 2.526,
+    "epoch": 0.1534059966404087,
+    "step": 5000
+  },
+  {
+    "loss": 0.1426,
+    "grad_norm": 0.18281251192092896,
+    "learning_rate": 5.128214216090478e-05,
+    "epoch": 0.1540196206269703,
+    "step": 5020
+  },
+  {
+    "loss": 0.144,
+    "grad_norm": 0.37768101692199707,
+    "learning_rate": 5.0961652739384356e-05,
+    "epoch": 0.15463324461353195,
+    "step": 5040
+  },
+  {
+    "loss": 0.1385,
+    "grad_norm": 0.25653406977653503,
+    "learning_rate": 5.064112378808637e-05,
+    "epoch": 0.15524686860009357,
+    "step": 5060
+  },
+  {
+    "loss": 0.1438,
+    "grad_norm": 0.16125090420246124,
+    "learning_rate": 5.0320568482700556e-05,
+    "epoch": 0.1558604925866552,
+    "step": 5080
+  },
+  {
+    "loss": 0.1365,
+    "grad_norm": 0.13656456768512726,
+    "learning_rate": 5e-05,
+    "epoch": 0.15647411657321686,
+    "step": 5100
+  },
+  {
+    "loss": 0.1407,
+    "grad_norm": 0.19003720581531525,
+    "learning_rate": 4.967943151729945e-05,
+    "epoch": 0.15708774055977848,
+    "step": 5120
+  },
+  {
+    "loss": 0.1443,
+    "grad_norm": 0.12324690073728561,
+    "learning_rate": 4.935887621191364e-05,
+    "epoch": 0.15770136454634012,
+    "step": 5140
+  },
+  {
+    "loss": 0.1407,
+    "grad_norm": 0.4486618936061859,
+    "learning_rate": 4.903834726061565e-05,
+    "epoch": 0.15831498853290174,
+    "step": 5160
+  },
+  {
+    "loss": 0.1459,
+    "grad_norm": 0.2653239965438843,
+    "learning_rate": 4.871785783909523e-05,
+    "epoch": 0.15892861251946339,
+    "step": 5180
+  },
+  {
+    "loss": 0.1406,
+    "grad_norm": 0.1883934885263443,
+    "learning_rate": 4.839742112141724e-05,
+    "epoch": 0.15954223650602503,
+    "step": 5200
+  },
+  {
+    "loss": 0.1477,
+    "grad_norm": 0.23059062659740448,
+    "learning_rate": 4.807705027948008e-05,
+    "epoch": 0.16015586049258665,
+    "step": 5220
+  },
+  {
+    "loss": 0.138,
+    "grad_norm": 0.10817737877368927,
+    "learning_rate": 4.775675848247427e-05,
+    "epoch": 0.1607694844791483,
+    "step": 5240
+  },
+  {
+    "loss": 0.1388,
+    "grad_norm": 0.23007145524024963,
+    "learning_rate": 4.743655889634105e-05,
+    "epoch": 0.1613831084657099,
+    "step": 5260
+  },
+  {
+    "loss": 0.14,
+    "grad_norm": 0.12711915373802185,
+    "learning_rate": 4.711646468323129e-05,
+    "epoch": 0.16199673245227156,
+    "step": 5280
+  },
+  {
+    "loss": 0.1368,
+    "grad_norm": 0.11912371963262558,
+    "learning_rate": 4.679648900096436e-05,
+    "epoch": 0.1626103564388332,
+    "step": 5300
+  },
+  {
+    "loss": 0.1374,
+    "grad_norm": 0.14275406301021576,
+    "learning_rate": 4.64766450024873e-05,
+    "epoch": 0.16322398042539482,
+    "step": 5320
+  },
+  {
+    "loss": 0.1347,
+    "grad_norm": 0.3042926490306854,
+    "learning_rate": 4.6156945835334184e-05,
+    "epoch": 0.16383760441195647,
+    "step": 5340
+  },
+  {
+    "loss": 0.139,
+    "grad_norm": 0.16544413566589355,
+    "learning_rate": 4.583740464108554e-05,
+    "epoch": 0.16445122839851808,
+    "step": 5360
+  },
+  {
+    "loss": 0.1424,
+    "grad_norm": 0.4930761754512787,
+    "learning_rate": 4.551803455482833e-05,
+    "epoch": 0.16506485238507973,
+    "step": 5380
+  },
+  {
+    "loss": 0.1412,
+    "grad_norm": 0.4234517216682434,
+    "learning_rate": 4.5198848704615914e-05,
+    "epoch": 0.16567847637164138,
+    "step": 5400
+  },
+  {
+    "loss": 0.1415,
+    "grad_norm": 0.15803220868110657,
+    "learning_rate": 4.487986021092844e-05,
+    "epoch": 0.166292100358203,
+    "step": 5420
+  },
+  {
+    "loss": 0.1413,
+    "grad_norm": 0.18349704146385193,
+    "learning_rate": 4.4561082186133464e-05,
+    "epoch": 0.16690572434476464,
+    "step": 5440
+  },
+  {
+    "loss": 0.141,
+    "grad_norm": 0.14680899679660797,
+    "learning_rate": 4.424252773394704e-05,
+    "epoch": 0.16751934833132628,
+    "step": 5460
+  },
+  {
+    "loss": 0.1462,
+    "grad_norm": 0.18841388821601868,
+    "learning_rate": 4.392420994889498e-05,
+    "epoch": 0.1681329723178879,
+    "step": 5480
+  },
+  {
+    "loss": 0.1403,
+    "grad_norm": 0.3068675398826599,
+    "learning_rate": 4.3606141915774693e-05,
+    "epoch": 0.16874659630444955,
+    "step": 5500
+  },
+  {
+    "eval_loss": 0.1398838609457016,
+    "eval_runtime": 2772.4795,
+    "eval_samples_per_second": 161.639,
+    "eval_steps_per_second": 2.526,
+    "epoch": 0.16874659630444955,
+    "step": 5500
+  },
+  {
+    "loss": 0.1369,
+    "grad_norm": 0.13292258977890015,
+    "learning_rate": 4.328833670911724e-05,
+    "epoch": 0.16936022029101117,
+    "step": 5520
+  },
+  {
+    "loss": 0.147,
+    "grad_norm": 0.11828584223985672,
+    "learning_rate": 4.297080739264987e-05,
+    "epoch": 0.1699738442775728,
+    "step": 5540
+  },
+  {
+    "loss": 0.1405,
+    "grad_norm": 0.15388241410255432,
+    "learning_rate": 4.265356701875911e-05,
+    "epoch": 0.17058746826413446,
+    "step": 5560
+  },
+  {
+    "loss": 0.137,
+    "grad_norm": 0.20380249619483948,
+    "learning_rate": 4.23366286279542e-05,
+    "epoch": 0.17120109225069607,
+    "step": 5580
+  },
+  {
+    "loss": 0.1299,
+    "grad_norm": 0.18336381018161774,
+    "learning_rate": 4.2020005248331054e-05,
+    "epoch": 0.17181471623725772,
+    "step": 5600
+  },
+  {
+    "loss": 0.138,
+    "grad_norm": 0.16339316964149475,
+    "learning_rate": 4.1703709895036625e-05,
+    "epoch": 0.17242834022381934,
+    "step": 5620
+  },
+  {
+    "loss": 0.1383,
+    "grad_norm": 0.14355255663394928,
+    "learning_rate": 4.138775556973406e-05,
+    "epoch": 0.17304196421038098,
+    "step": 5640
+  },
+  {
+    "loss": 0.141,
+    "grad_norm": 0.19084544479846954,
+    "learning_rate": 4.107215526006817e-05,
+    "epoch": 0.17365558819694263,
+    "step": 5660
+  },
+  {
+    "loss": 0.1304,
+    "grad_norm": 0.11718132346868515,
+    "learning_rate": 4.0756921939131565e-05,
+    "epoch": 0.17426921218350425,
+    "step": 5680
+  },
+  {
+    "loss": 0.1371,
+    "grad_norm": 0.13921724259853363,
+    "learning_rate": 4.04420685649314e-05,
+    "epoch": 0.1748828361700659,
+    "step": 5700
+  },
+  {
+    "loss": 0.1376,
+    "grad_norm": 0.14769333600997925,
+    "learning_rate": 4.012760807985665e-05,
+    "epoch": 0.1754964601566275,
+    "step": 5720
+  },
+  {
+    "loss": 0.1423,
+    "grad_norm": 0.11135128140449524,
+    "learning_rate": 3.981355341014623e-05,
+    "epoch": 0.17611008414318916,
+    "step": 5740
+  },
+  {
+    "loss": 0.1363,
+    "grad_norm": 0.15607145428657532,
+    "learning_rate": 3.9499917465357534e-05,
+    "epoch": 0.1767237081297508,
+    "step": 5760
+  },
+  {
+    "loss": 0.1367,
+    "grad_norm": 0.2499862164258957,
+    "learning_rate": 3.9186713137835826e-05,
+    "epoch": 0.17733733211631242,
+    "step": 5780
+  },
+  {
+    "loss": 0.1372,
+    "grad_norm": 0.31999796628952026,
+    "learning_rate": 3.887395330218429e-05,
+    "epoch": 0.17795095610287406,
+    "step": 5800
+  },
+  {
+    "loss": 0.1412,
+    "grad_norm": 0.2039441466331482,
+    "learning_rate": 3.856165081473474e-05,
+    "epoch": 0.1785645800894357,
+    "step": 5820
+  },
+  {
+    "loss": 0.1349,
+    "grad_norm": 0.1782849133014679,
+    "learning_rate": 3.8249818513019244e-05,
+    "epoch": 0.17917820407599733,
+    "step": 5840
+  },
+  {
+    "loss": 0.1337,
+    "grad_norm": 0.14570361375808716,
+    "learning_rate": 3.793846921524237e-05,
+    "epoch": 0.17979182806255897,
+    "step": 5860
+  },
+  {
+    "loss": 0.1416,
+    "grad_norm": 0.199429452419281,
+    "learning_rate": 3.762761571975429e-05,
+    "epoch": 0.1804054520491206,
+    "step": 5880
+  },
+  {
+    "loss": 0.1459,
+    "grad_norm": 0.18289752304553986,
+    "learning_rate": 3.731727080452464e-05,
+    "epoch": 0.18101907603568224,
+    "step": 5900
+  },
+  {
+    "loss": 0.1359,
+    "grad_norm": 0.13586929440498352,
+    "learning_rate": 3.7007447226617366e-05,
+    "epoch": 0.18163270002224388,
+    "step": 5920
+  },
+  {
+    "loss": 0.1394,
+    "grad_norm": 0.19824199378490448,
+    "learning_rate": 3.6698157721666246e-05,
+    "epoch": 0.1822463240088055,
+    "step": 5940
+  },
+  {
+    "loss": 0.1383,
+    "grad_norm": 0.2370585948228836,
+    "learning_rate": 3.638941500335145e-05,
+    "epoch": 0.18285994799536714,
+    "step": 5960
+  },
+  {
+    "loss": 0.138,
+    "grad_norm": 0.14067555963993073,
+    "learning_rate": 3.608123176287685e-05,
+    "epoch": 0.18347357198192876,
+    "step": 5980
+  },
+  {
+    "loss": 0.1415,
+    "grad_norm": 0.21730934083461761,
+    "learning_rate": 3.5773620668448384e-05,
+    "epoch": 0.1840871959684904,
+    "step": 6000
+  },
+  {
+    "eval_loss": 0.1372094452381134,
+    "eval_runtime": 2771.4148,
+    "eval_samples_per_second": 161.701,
+    "eval_steps_per_second": 2.527,
+    "epoch": 0.1840871959684904,
+    "step": 6000
+  },
+  {
+    "loss": 0.1384,
+    "grad_norm": 0.12257638573646545,
+    "learning_rate": 3.5466594364753326e-05,
+    "epoch": 0.18470081995505205,
+    "step": 6020
+  },
+  {
+    "loss": 0.1381,
+    "grad_norm": 0.13667023181915283,
+    "learning_rate": 3.5160165472440473e-05,
+    "epoch": 0.18531444394161367,
+    "step": 6040
+  },
+  {
+    "loss": 0.1373,
+    "grad_norm": 0.20999397337436676,
+    "learning_rate": 3.48543465876014e-05,
+    "epoch": 0.18592806792817532,
+    "step": 6060
+  },
+  {
+    "loss": 0.1388,
+    "grad_norm": 0.1176946684718132,
+    "learning_rate": 3.4549150281252636e-05,
+    "epoch": 0.18654169191473693,
+    "step": 6080
+  },
+  {
+    "loss": 0.1406,
+    "grad_norm": 0.12363361567258835,
+    "learning_rate": 3.424458909881897e-05,
+    "epoch": 0.18715531590129858,
+    "step": 6100
+  },
+  {
+    "loss": 0.1346,
+    "grad_norm": 0.1650696098804474,
+    "learning_rate": 3.3940675559617724e-05,
+    "epoch": 0.18776893988786023,
+    "step": 6120
+  },
+  {
+    "loss": 0.1411,
+    "grad_norm": 0.13882821798324585,
+    "learning_rate": 3.363742215634415e-05,
+    "epoch": 0.18838256387442184,
+    "step": 6140
+  },
+  {
+    "loss": 0.135,
+    "grad_norm": 0.18730175495147705,
+    "learning_rate": 3.333484135455792e-05,
+    "epoch": 0.1889961878609835,
+    "step": 6160
+  },
+  {
+    "loss": 0.1375,
+    "grad_norm": 0.15268158912658691,
+    "learning_rate": 3.303294559217063e-05,
+    "epoch": 0.18960981184754513,
+    "step": 6180
+  },
+  {
+    "loss": 0.1309,
+    "grad_norm": 0.6368291974067688,
+    "learning_rate": 3.273174727893463e-05,
+    "epoch": 0.19022343583410675,
+    "step": 6200
+  },
+  {
+    "loss": 0.1431,
+    "grad_norm": 0.19259753823280334,
+    "learning_rate": 3.243125879593286e-05,
+    "epoch": 0.1908370598206684,
+    "step": 6220
+  },
+  {
+    "loss": 0.134,
+    "grad_norm": 0.20992296934127808,
+    "learning_rate": 3.213149249506997e-05,
+    "epoch": 0.19145068380723002,
+    "step": 6240
+  },
+  {
+    "loss": 0.134,
+    "grad_norm": 0.1862725019454956,
+    "learning_rate": 3.183246069856443e-05,
+    "epoch": 0.19206430779379166,
+    "step": 6260
+  },
+  {
+    "loss": 0.1339,
+    "grad_norm": 0.10517208278179169,
+    "learning_rate": 3.153417569844219e-05,
+    "epoch": 0.1926779317803533,
+    "step": 6280
+  },
+  {
+    "loss": 0.1377,
+    "grad_norm": 0.16911420226097107,
+    "learning_rate": 3.12366497560313e-05,
+    "epoch": 0.19329155576691492,
+    "step": 6300
+  },
+  {
+    "loss": 0.1402,
+    "grad_norm": 0.1685948520898819,
+    "learning_rate": 3.0939895101457916e-05,
+    "epoch": 0.19390517975347657,
+    "step": 6320
+  },
+  {
+    "loss": 0.138,
+    "grad_norm": 0.2124929577112198,
+    "learning_rate": 3.06439239331436e-05,
+    "epoch": 0.1945188037400382,
+    "step": 6340
+  },
+  {
+    "loss": 0.1332,
+    "grad_norm": 0.24099479615688324,
+    "learning_rate": 3.0348748417303823e-05,
+    "epoch": 0.19513242772659983,
+    "step": 6360
+  },
+  {
+    "loss": 0.1347,
+    "grad_norm": 0.1686478555202484,
+    "learning_rate": 3.005438068744792e-05,
+    "epoch": 0.19574605171316148,
+    "step": 6380
+  },
+  {
+    "loss": 0.1353,
+    "grad_norm": 0.11981747299432755,
+    "learning_rate": 2.976083284388031e-05,
+    "epoch": 0.1963596756997231,
+    "step": 6400
+  },
+  {
+    "loss": 0.1425,
+    "grad_norm": 0.10391107201576233,
+    "learning_rate": 2.9468116953203107e-05,
+    "epoch": 0.19697329968628474,
+    "step": 6420
+  },
+  {
+    "loss": 0.1354,
+    "grad_norm": 0.1283193975687027,
+    "learning_rate": 2.917624504782006e-05,
+    "epoch": 0.19758692367284636,
+    "step": 6440
+  },
+  {
+    "loss": 0.1372,
+    "grad_norm": 0.1691513955593109,
+    "learning_rate": 2.888522912544202e-05,
+    "epoch": 0.198200547659408,
+    "step": 6460
+  },
+  {
+    "loss": 0.1278,
+    "grad_norm": 0.14271607995033264,
+    "learning_rate": 2.8595081148593738e-05,
+    "epoch": 0.19881417164596965,
+    "step": 6480
+  },
+  {
+    "loss": 0.1315,
+    "grad_norm": 0.1568264663219452,
+    "learning_rate": 2.8305813044122097e-05,
+    "epoch": 0.19942779563253127,
+    "step": 6500
+  },
+  {
+    "eval_loss": 0.13607633113861084,
+    "eval_runtime": 2773.0124,
+    "eval_samples_per_second": 161.608,
+    "eval_steps_per_second": 2.525,
+    "epoch": 0.19942779563253127,
+    "step": 6500
+  },
+  {
+    "loss": 0.1348,
+    "grad_norm": 0.2550095319747925,
+    "learning_rate": 2.80174367027059e-05,
+    "epoch": 0.20004141961909291,
+    "step": 6520
+  },
+  {
+    "loss": 0.137,
+    "grad_norm": 0.14450562000274658,
+    "learning_rate": 2.772996397836704e-05,
+    "epoch": 0.20065504360565453,
+    "step": 6540
+  },
+  {
+    "loss": 0.1348,
+    "grad_norm": 0.13089318573474884,
+    "learning_rate": 2.7443406687983265e-05,
+    "epoch": 0.20126866759221618,
+    "step": 6560
+  },
+  {
+    "loss": 0.1406,
+    "grad_norm": 0.10475198924541473,
+    "learning_rate": 2.7157776610802415e-05,
+    "epoch": 0.20188229157877782,
+    "step": 6580
+  },
+  {
+    "loss": 0.134,
+    "grad_norm": 0.16278155148029327,
+    "learning_rate": 2.687308548795825e-05,
+    "epoch": 0.20249591556533944,
+    "step": 6600
+  },
+  {
+    "loss": 0.1342,
+    "grad_norm": 0.19838862121105194,
+    "learning_rate": 2.658934502198772e-05,
+    "epoch": 0.20310953955190109,
+    "step": 6620
+  },
+  {
+    "loss": 0.1339,
+    "grad_norm": 0.3514244854450226,
+    "learning_rate": 2.630656687635007e-05,
+    "epoch": 0.20372316353846273,
+    "step": 6640
+  },
+  {
+    "loss": 0.1421,
+    "grad_norm": 0.21309345960617065,
+    "learning_rate": 2.6024762674947313e-05,
+    "epoch": 0.20433678752502435,
+    "step": 6660
+  },
+  {
+    "loss": 0.1308,
+    "grad_norm": 0.17511409521102905,
+    "learning_rate": 2.574394400164639e-05,
+    "epoch": 0.204950411511586,
+    "step": 6680
+  },
+  {
+    "loss": 0.1329,
+    "grad_norm": 0.1690303385257721,
+    "learning_rate": 2.5464122399803125e-05,
+    "epoch": 0.2055640354981476,
+    "step": 6700
+  },
+  {
+    "loss": 0.1373,
+    "grad_norm": 0.1409410983324051,
+    "learning_rate": 2.5185309371787513e-05,
+    "epoch": 0.20617765948470926,
+    "step": 6720
+  },
+  {
+    "loss": 0.1414,
+    "grad_norm": 0.19619417190551758,
+    "learning_rate": 2.4907516378511135e-05,
+    "epoch": 0.2067912834712709,
+    "step": 6740
+  },
+  {
+    "loss": 0.1369,
+    "grad_norm": 0.2113848179578781,
+    "learning_rate": 2.46307548389559e-05,
+    "epoch": 0.20740490745783252,
+    "step": 6760
+  },
+  {
+    "loss": 0.14,
+    "grad_norm": 0.1930362433195114,
+    "learning_rate": 2.43550361297047e-05,
+    "epoch": 0.20801853144439417,
+    "step": 6780
+  },
+  {
+    "loss": 0.1296,
+    "grad_norm": 0.12248807400465012,
+    "learning_rate": 2.4080371584473748e-05,
+    "epoch": 0.20863215543095578,
+    "step": 6800
+  },
+  {
+    "loss": 0.1382,
+    "grad_norm": 0.20456106960773468,
+    "learning_rate": 2.3806772493646723e-05,
+    "epoch": 0.20924577941751743,
+    "step": 6820
+  },
+  {
+    "loss": 0.1374,
+    "grad_norm": 0.1549805998802185,
+    "learning_rate": 2.353425010381063e-05,
+    "epoch": 0.20985940340407908,
+    "step": 6840
+  },
+  {
+    "loss": 0.1355,
+    "grad_norm": 0.14484341442584991,
+    "learning_rate": 2.3262815617293517e-05,
+    "epoch": 0.2104730273906407,
+    "step": 6860
+  },
+  {
+    "loss": 0.1389,
+    "grad_norm": 0.12407292425632477,
+    "learning_rate": 2.2992480191704002e-05,
+    "epoch": 0.21108665137720234,
+    "step": 6880
+  },
+  {
+    "loss": 0.1359,
+    "grad_norm": 0.20337499678134918,
+    "learning_rate": 2.272325493947257e-05,
+    "epoch": 0.21170027536376396,
+    "step": 6900
+  },
+  {
+    "loss": 0.1408,
+    "grad_norm": 0.1620754599571228,
+    "learning_rate": 2.245515092739488e-05,
+    "epoch": 0.2123138993503256,
+    "step": 6920
+  },
+  {
+    "loss": 0.1406,
+    "grad_norm": 0.16787594556808472,
+    "learning_rate": 2.2188179176176766e-05,
+    "epoch": 0.21292752333688725,
+    "step": 6940
+  },
+  {
+    "loss": 0.1322,
+    "grad_norm": 0.1271612048149109,
+    "learning_rate": 2.192235065998126e-05,
+    "epoch": 0.21354114732344887,
+    "step": 6960
+  },
+  {
+    "loss": 0.1396,
+    "grad_norm": 0.1334231197834015,
+    "learning_rate": 2.165767630597752e-05,
+    "epoch": 0.2141547713100105,
+    "step": 6980
+  },
+  {
+    "loss": 0.1246,
+    "grad_norm": 0.1719616800546646,
+    "learning_rate": 2.139416699389153e-05,
+    "epoch": 0.21476839529657216,
+    "step": 7000
+  },
+  {
+    "eval_loss": 0.1357450932264328,
+    "eval_runtime": 2775.089,
+    "eval_samples_per_second": 161.487,
+    "eval_steps_per_second": 2.524,
+    "epoch": 0.21476839529657216,
+    "step": 7000
+  },
+  {
+    "loss": 0.1331,
+    "grad_norm": 0.1086181178689003,
+    "learning_rate": 2.1131833555559037e-05,
+    "epoch": 0.21538201928313377,
+    "step": 7020
+  },
+  {
+    "loss": 0.1359,
+    "grad_norm": 0.13283145427703857,
+    "learning_rate": 2.0870686774480196e-05,
+    "epoch": 0.21599564326969542,
+    "step": 7040
+  },
+  {
+    "loss": 0.1333,
+    "grad_norm": 0.17758311331272125,
+    "learning_rate": 2.061073738537635e-05,
+    "epoch": 0.21660926725625704,
+    "step": 7060
+  },
+  {
+    "loss": 0.1292,
+    "grad_norm": 0.18912430107593536,
+    "learning_rate": 2.0351996073748713e-05,
+    "epoch": 0.21722289124281868,
+    "step": 7080
+  },
+  {
+    "loss": 0.1283,
+    "grad_norm": 0.16375404596328735,
+    "learning_rate": 2.0094473475439202e-05,
+    "epoch": 0.21783651522938033,
+    "step": 7100
+  },
+  {
+    "loss": 0.1357,
+    "grad_norm": 0.1428246796131134,
+    "learning_rate": 1.9838180176193178e-05,
+    "epoch": 0.21845013921594195,
+    "step": 7120
+  },
+  {
+    "loss": 0.1299,
+    "grad_norm": 0.1967194378376007,
+    "learning_rate": 1.9583126711224343e-05,
+    "epoch": 0.2190637632025036,
+    "step": 7140
+  },
+  {
+    "loss": 0.1307,
+    "grad_norm": 0.1559692770242691,
+    "learning_rate": 1.9329323564781682e-05,
+    "epoch": 0.2196773871890652,
+    "step": 7160
+  },
+  {
+    "loss": 0.1293,
+    "grad_norm": 0.11402944475412369,
+    "learning_rate": 1.9076781169718428e-05,
+    "epoch": 0.22029101117562686,
+    "step": 7180
+  },
+  {
+    "loss": 0.1321,
+    "grad_norm": 0.13555893301963806,
+    "learning_rate": 1.8825509907063327e-05,
+    "epoch": 0.2209046351621885,
+    "step": 7200
+  },
+  {
+    "loss": 0.1334,
+    "grad_norm": 0.13484172523021698,
+    "learning_rate": 1.8575520105593817e-05,
+    "epoch": 0.22151825914875012,
+    "step": 7220
+  },
+  {
+    "loss": 0.1363,
+    "grad_norm": 0.17900648713111877,
+    "learning_rate": 1.8326822041411524e-05,
+    "epoch": 0.22213188313531176,
+    "step": 7240
+  },
+  {
+    "loss": 0.1367,
+    "grad_norm": 0.12890270352363586,
+    "learning_rate": 1.807942593751973e-05,
+    "epoch": 0.22274550712187338,
+    "step": 7260
+  },
+  {
+    "loss": 0.1407,
+    "grad_norm": 0.156462624669075,
+    "learning_rate": 1.783334196340331e-05,
+    "epoch": 0.22335913110843503,
+    "step": 7280
+  },
+  {
+    "loss": 0.1352,
+    "grad_norm": 0.13109950721263885,
+    "learning_rate": 1.758858023461059e-05,
+    "epoch": 0.22397275509499667,
+    "step": 7300
+  },
+  {
+    "loss": 0.1325,
+    "grad_norm": 0.1458718329668045,
+    "learning_rate": 1.7345150812337564e-05,
+    "epoch": 0.2245863790815583,
+    "step": 7320
+  },
+  {
+    "loss": 0.1332,
+    "grad_norm": 0.16703669726848602,
+    "learning_rate": 1.7103063703014372e-05,
+    "epoch": 0.22520000306811994,
+    "step": 7340
+  },
+  {
+    "loss": 0.1342,
+    "grad_norm": 0.15283693373203278,
+    "learning_rate": 1.6862328857893854e-05,
+    "epoch": 0.22581362705468155,
+    "step": 7360
+  },
+  {
+    "loss": 0.1308,
+    "grad_norm": 0.12556903064250946,
+    "learning_rate": 1.66229561726426e-05,
+    "epoch": 0.2264272510412432,
+    "step": 7380
+  },
+  {
+    "loss": 0.1289,
+    "grad_norm": 0.11614284664392471,
+    "learning_rate": 1.6384955486934156e-05,
+    "epoch": 0.22704087502780484,
+    "step": 7400
+  },
+  {
+    "loss": 0.1372,
+    "grad_norm": 0.10490608215332031,
+    "learning_rate": 1.614833658404454e-05,
+    "epoch": 0.22765449901436646,
+    "step": 7420
+  },
+  {
+    "loss": 0.1313,
+    "grad_norm": 0.13296884298324585,
+    "learning_rate": 1.5913109190450032e-05,
+    "epoch": 0.2282681230009281,
+    "step": 7440
+  },
+  {
+    "loss": 0.1352,
+    "grad_norm": 0.1308923065662384,
+    "learning_rate": 1.567928297542749e-05,
+    "epoch": 0.22888174698748975,
+    "step": 7460
+  },
+  {
+    "loss": 0.1279,
+    "grad_norm": 0.1364806592464447,
+    "learning_rate": 1.544686755065677e-05,
+    "epoch": 0.22949537097405137,
+    "step": 7480
+  },
+  {
+    "loss": 0.1401,
+    "grad_norm": 0.19729793071746826,
+    "learning_rate": 1.5215872469825682e-05,
+    "epoch": 0.23010899496061302,
+    "step": 7500
+  },
+  {
+    "eval_loss": 0.13517485558986664,
+    "eval_runtime": 2776.5225,
+    "eval_samples_per_second": 161.403,
+    "eval_steps_per_second": 2.522,
+    "epoch": 0.23010899496061302,
+    "step": 7500
+  },
+  {
+    "loss": 0.1342,
+    "grad_norm": 0.14020280539989471,
+    "learning_rate": 1.4986307228237268e-05,
+    "epoch": 0.23072261894717463,
+    "step": 7520
+  },
+  {
+    "loss": 0.1401,
+    "grad_norm": 0.11004228889942169,
+    "learning_rate": 1.4758181262419423e-05,
+    "epoch": 0.23133624293373628,
+    "step": 7540
+  },
+  {
+    "loss": 0.1382,
+    "grad_norm": 0.10248783230781555,
+    "learning_rate": 1.4531503949737108e-05,
+    "epoch": 0.23194986692029793,
+    "step": 7560
+  },
+  {
+    "loss": 0.127,
+    "grad_norm": 0.09407743066549301,
+    "learning_rate": 1.4306284608006836e-05,
+    "epoch": 0.23256349090685954,
+    "step": 7580
+  },
+  {
+    "loss": 0.1371,
+    "grad_norm": 0.15860360860824585,
+    "learning_rate": 1.4082532495113626e-05,
+    "epoch": 0.2331771148934212,
+    "step": 7600
+  },
+  {
+    "loss": 0.1312,
+    "grad_norm": 0.12674517929553986,
+    "learning_rate": 1.3860256808630428e-05,
+    "epoch": 0.2337907388799828,
+    "step": 7620
+  },
+  {
+    "loss": 0.1374,
+    "grad_norm": 0.15133555233478546,
+    "learning_rate": 1.3639466685440132e-05,
+    "epoch": 0.23440436286654445,
+    "step": 7640
+  },
+  {
+    "loss": 0.1358,
+    "grad_norm": 0.09516581892967224,
+    "learning_rate": 1.3420171201359933e-05,
+    "epoch": 0.2350179868531061,
+    "step": 7660
+  },
+  {
+    "loss": 0.1337,
+    "grad_norm": 0.1541604995727539,
+    "learning_rate": 1.3202379370768252e-05,
+    "epoch": 0.23563161083966772,
+    "step": 7680
+  },
+  {
+    "loss": 0.1295,
+    "grad_norm": 0.1433165818452835,
+    "learning_rate": 1.2986100146234232e-05,
+    "epoch": 0.23624523482622936,
+    "step": 7700
+  },
+  {
+    "loss": 0.1349,
+    "grad_norm": 0.16317042708396912,
+    "learning_rate": 1.2771342418149657e-05,
+    "epoch": 0.23685885881279098,
+    "step": 7720
+  },
+  {
+    "loss": 0.1269,
+    "grad_norm": 0.14434409141540527,
+    "learning_rate": 1.2558115014363592e-05,
+    "epoch": 0.23747248279935262,
+    "step": 7740
+  },
+  {
+    "loss": 0.1358,
+    "grad_norm": 0.2537558972835541,
+    "learning_rate": 1.2346426699819458e-05,
+    "epoch": 0.23808610678591427,
+    "step": 7760
+  },
+  {
+    "loss": 0.137,
+    "grad_norm": 0.16992440819740295,
+    "learning_rate": 1.2136286176194745e-05,
+    "epoch": 0.2386997307724759,
+    "step": 7780
+  },
+  {
+    "loss": 0.1333,
+    "grad_norm": 0.14877456426620483,
+    "learning_rate": 1.1927702081543279e-05,
+    "epoch": 0.23931335475903753,
+    "step": 7800
+  },
+  {
+    "loss": 0.1321,
+    "grad_norm": 0.15068818628787994,
+    "learning_rate": 1.1720682989940262e-05,
+    "epoch": 0.23992697874559918,
+    "step": 7820
+  },
+  {
+    "loss": 0.1329,
+    "grad_norm": 0.10926194489002228,
+    "learning_rate": 1.1515237411129698e-05,
+    "epoch": 0.2405406027321608,
+    "step": 7840
+  },
+  {
+    "loss": 0.1349,
+    "grad_norm": 0.1470404714345932,
+    "learning_rate": 1.1311373790174657e-05,
+    "epoch": 0.24115422671872244,
+    "step": 7860
+  },
+  {
+    "loss": 0.1368,
+    "grad_norm": 0.1232927218079567,
+    "learning_rate": 1.1109100507110132e-05,
+    "epoch": 0.24176785070528406,
+    "step": 7880
+  },
+  {
+    "loss": 0.1312,
+    "grad_norm": 0.14564509689807892,
+    "learning_rate": 1.090842587659851e-05,
+    "epoch": 0.2423814746918457,
+    "step": 7900
+  },
+  {
+    "loss": 0.1319,
+    "grad_norm": 0.1518176794052124,
+    "learning_rate": 1.0709358147587884e-05,
+    "epoch": 0.24299509867840735,
+    "step": 7920
+  },
+  {
+    "loss": 0.1377,
+    "grad_norm": 0.11598769575357437,
+    "learning_rate": 1.0511905502972886e-05,
+    "epoch": 0.24360872266496897,
+    "step": 7940
+  },
+  {
+    "loss": 0.1271,
+    "grad_norm": 0.13424475491046906,
+    "learning_rate": 1.031607605925839e-05,
+    "epoch": 0.24422234665153061,
+    "step": 7960
+  },
+  {
+    "loss": 0.1317,
+    "grad_norm": 0.17036224901676178,
+    "learning_rate": 1.0121877866225781e-05,
+    "epoch": 0.24483597063809223,
+    "step": 7980
+  },
+  {
+    "loss": 0.1378,
+    "grad_norm": 0.13854670524597168,
+    "learning_rate": 9.929318906602175e-06,
+    "epoch": 0.24544959462465388,
+    "step": 8000
+  },
+  {
+    "eval_loss": 0.13417504727840424,
+    "eval_runtime": 2774.5382,
+    "eval_samples_per_second": 161.519,
+    "eval_steps_per_second": 2.524,
+    "epoch": 0.24544959462465388,
+    "step": 8000
+  },
+  {
+    "loss": 0.1381,
+    "grad_norm": 0.12813754379749298,
+    "learning_rate": 9.738407095732195e-06,
+    "epoch": 0.24606321861121552,
+    "step": 8020
+  },
+  {
+    "loss": 0.1375,
+    "grad_norm": 0.1952831745147705,
+    "learning_rate": 9.549150281252633e-06,
+    "epoch": 0.24667684259777714,
+    "step": 8040
+  },
+  {
+    "loss": 0.1363,
+    "grad_norm": 0.17179648578166962,
+    "learning_rate": 9.36155624276987e-06,
+    "epoch": 0.2472904665843388,
+    "step": 8060
+  },
+  {
+    "loss": 0.1307,
+    "grad_norm": 0.2890334129333496,
+    "learning_rate": 9.175632691540065e-06,
+    "epoch": 0.2479040905709004,
+    "step": 8080
+  },
+  {
+    "loss": 0.1282,
+    "grad_norm": 0.11331592500209808,
+    "learning_rate": 8.991387270152201e-06,
+    "epoch": 0.24851771455746205,
+    "step": 8100
+  },
+  {
+    "loss": 0.1356,
+    "grad_norm": 0.619337797164917,
+    "learning_rate": 8.808827552213916e-06,
+    "epoch": 0.2491313385440237,
+    "step": 8120
+  },
+  {
+    "loss": 0.1322,
+    "grad_norm": 0.10491683334112167,
+    "learning_rate": 8.627961042040184e-06,
+    "epoch": 0.2497449625305853,
+    "step": 8140
+  },
+  {
+    "loss": 0.1323,
+    "grad_norm": 0.1256422996520996,
+    "learning_rate": 8.448795174344804e-06,
+    "epoch": 0.25035858651714693,
+    "step": 8160
+  },
+  {
+    "loss": 0.1325,
+    "grad_norm": 0.11729530245065689,
+    "learning_rate": 8.271337313934869e-06,
+    "epoch": 0.2509722105037086,
+    "step": 8180
+  },
+  {
+    "loss": 0.1344,
+    "grad_norm": 0.1296742558479309,
+    "learning_rate": 8.09559475540797e-06,
+    "epoch": 0.2515858344902702,
+    "step": 8200
+  },
+  {
+    "loss": 0.1293,
+    "grad_norm": 0.11242369562387466,
+    "learning_rate": 7.921574722852343e-06,
+    "epoch": 0.25219945847683184,
+    "step": 8220
+  },
+  {
+    "loss": 0.1318,
+    "grad_norm": 0.13417485356330872,
+    "learning_rate": 7.749284369549953e-06,
+    "epoch": 0.2528130824633935,
+    "step": 8240
+  },
+  {
+    "loss": 0.1301,
+    "grad_norm": 0.17316614091396332,
+    "learning_rate": 7.578730777682386e-06,
+    "epoch": 0.25342670644995513,
+    "step": 8260
+  },
+  {
+    "loss": 0.1373,
+    "grad_norm": 0.1728428602218628,
+    "learning_rate": 7.409920958039795e-06,
+    "epoch": 0.25404033043651675,
+    "step": 8280
+  },
+  {
+    "loss": 0.1336,
+    "grad_norm": 0.1023547574877739,
+    "learning_rate": 7.242861849732696e-06,
+    "epoch": 0.2546539544230784,
+    "step": 8300
+  },
+  {
+    "loss": 0.1354,
+    "grad_norm": 0.1599491834640503,
+    "learning_rate": 7.077560319906695e-06,
+    "epoch": 0.25526757840964004,
+    "step": 8320
+  },
+  {
+    "loss": 0.134,
+    "grad_norm": 0.12360566109418869,
+    "learning_rate": 6.9140231634602485e-06,
+    "epoch": 0.25588120239620166,
+    "step": 8340
+  },
+  {
+    "loss": 0.1289,
+    "grad_norm": 0.12993071973323822,
+    "learning_rate": 6.752257102765325e-06,
+    "epoch": 0.2564948263827633,
+    "step": 8360
+  },
+  {
+    "loss": 0.1348,
+    "grad_norm": 0.12023476511240005,
+    "learning_rate": 6.592268787391076e-06,
+    "epoch": 0.25710845036932495,
+    "step": 8380
+  },
+  {
+    "loss": 0.1348,
+    "grad_norm": 0.13408417999744415,
+    "learning_rate": 6.43406479383053e-06,
+    "epoch": 0.25772207435588657,
+    "step": 8400
+  },
+  {
+    "loss": 0.1362,
+    "grad_norm": 0.12605100870132446,
+    "learning_rate": 6.277651625230219e-06,
+    "epoch": 0.2583356983424482,
+    "step": 8420
+  },
+  {
+    "loss": 0.1299,
+    "grad_norm": 0.09935203939676285,
+    "learning_rate": 6.12303571112286e-06,
+    "epoch": 0.25894932232900986,
+    "step": 8440
+  },
+  {
+    "loss": 0.1329,
+    "grad_norm": 0.21711465716362,
+    "learning_rate": 5.9702234071631e-06,
+    "epoch": 0.2595629463155715,
+    "step": 8460
+  },
+  {
+    "loss": 0.1266,
+    "grad_norm": 0.1281883716583252,
+    "learning_rate": 5.819220994866237e-06,
+    "epoch": 0.2601765703021331,
+    "step": 8480
+  },
+  {
+    "loss": 0.1375,
+    "grad_norm": 0.1704270988702774,
+    "learning_rate": 5.670034681349995e-06,
+    "epoch": 0.26079019428869477,
+    "step": 8500
+  },
+  {
+    "eval_loss": 0.13329795002937317,
+    "eval_runtime": 2777.002,
+    "eval_samples_per_second": 161.375,
+    "eval_steps_per_second": 2.522,
+    "epoch": 0.26079019428869477,
+    "step": 8500
+  },
+  {
+    "loss": 0.1369,
+    "grad_norm": 0.11683149635791779,
+    "learning_rate": 5.5226705990794155e-06,
+    "epoch": 0.2614038182752564,
+    "step": 8520
+  },
+  {
+    "loss": 0.1244,
+    "grad_norm": 0.11445898562669754,
+    "learning_rate": 5.377134805614714e-06,
+    "epoch": 0.262017442261818,
+    "step": 8540
+  },
+  {
+    "loss": 0.1296,
+    "grad_norm": 0.1652945578098297,
+    "learning_rate": 5.233433283362349e-06,
+    "epoch": 0.2626310662483797,
+    "step": 8560
+  },
+  {
+    "loss": 0.1309,
+    "grad_norm": 0.15426833927631378,
+    "learning_rate": 5.091571939329048e-06,
+    "epoch": 0.2632446902349413,
+    "step": 8580
+  },
+  {
+    "loss": 0.1302,
+    "grad_norm": 0.1100183054804802,
+    "learning_rate": 4.951556604879048e-06,
+    "epoch": 0.2638583142215029,
+    "step": 8600
+  },
+  {
+    "loss": 0.1352,
+    "grad_norm": 0.10951848328113556,
+    "learning_rate": 4.813393035494329e-06,
+    "epoch": 0.2644719382080645,
+    "step": 8620
+  },
+  {
+    "loss": 0.1288,
+    "grad_norm": 0.1558559685945511,
+    "learning_rate": 4.677086910538092e-06,
+    "epoch": 0.2650855621946262,
+    "step": 8640
+  },
+  {
+    "loss": 0.1344,
+    "grad_norm": 0.14536911249160767,
+    "learning_rate": 4.542643833021254e-06,
+    "epoch": 0.2656991861811878,
+    "step": 8660
+  },
+  {
+    "loss": 0.1314,
+    "grad_norm": 0.12489516288042068,
+    "learning_rate": 4.410069329372152e-06,
+    "epoch": 0.26631281016774944,
+    "step": 8680
+  },
+  {
+    "loss": 0.1296,
+    "grad_norm": 0.10449978709220886,
+    "learning_rate": 4.279368849209381e-06,
+    "epoch": 0.2669264341543111,
+    "step": 8700
+  },
+  {
+    "loss": 0.1282,
+    "grad_norm": 0.13451628386974335,
+    "learning_rate": 4.150547765117746e-06,
+    "epoch": 0.2675400581408727,
+    "step": 8720
+  },
+  {
+    "loss": 0.1345,
+    "grad_norm": 0.20787014067173004,
+    "learning_rate": 4.023611372427471e-06,
+    "epoch": 0.26815368212743435,
+    "step": 8740
+  },
+  {
+    "loss": 0.1362,
+    "grad_norm": 0.14699150621891022,
+    "learning_rate": 3.898564888996476e-06,
+    "epoch": 0.268767306113996,
+    "step": 8760
+  },
+  {
+    "loss": 0.1311,
+    "grad_norm": 0.16810518503189087,
+    "learning_rate": 3.7754134549959297e-06,
+    "epoch": 0.26938093010055764,
+    "step": 8780
+  },
+  {
+    "loss": 0.1303,
+    "grad_norm": 0.13315120339393616,
+    "learning_rate": 3.654162132698918e-06,
+    "epoch": 0.26999455408711925,
+    "step": 8800
+  },
+  {
+    "loss": 0.1294,
+    "grad_norm": 0.10395628213882446,
+    "learning_rate": 3.534815906272404e-06,
+    "epoch": 0.2706081780736809,
+    "step": 8820
+  },
+  {
+    "loss": 0.1315,
+    "grad_norm": 0.12636798620224,
+    "learning_rate": 3.417379681572297e-06,
+    "epoch": 0.27122180206024255,
+    "step": 8840
+  },
+  {
+    "loss": 0.133,
+    "grad_norm": 0.1581977903842926,
+    "learning_rate": 3.3018582859418446e-06,
+    "epoch": 0.27183542604680416,
+    "step": 8860
+  },
+  {
+    "loss": 0.1266,
+    "grad_norm": 0.13160687685012817,
+    "learning_rate": 3.18825646801314e-06,
+    "epoch": 0.2724490500333658,
+    "step": 8880
+  },
+  {
+    "loss": 0.1366,
+    "grad_norm": 0.09358909726142883,
+    "learning_rate": 3.076578897511978e-06,
+    "epoch": 0.27306267401992745,
+    "step": 8900
+  },
+  {
+    "loss": 0.1302,
+    "grad_norm": 0.14909835159778595,
+    "learning_rate": 2.966830165065876e-06,
+    "epoch": 0.27367629800648907,
+    "step": 8920
+  },
+  {
+    "loss": 0.1344,
+    "grad_norm": 0.11751620471477509,
+    "learning_rate": 2.8590147820153513e-06,
+    "epoch": 0.2742899219930507,
+    "step": 8940
+  },
+  {
+    "loss": 0.1315,
+    "grad_norm": 0.12908974289894104,
+    "learning_rate": 2.753137180228543e-06,
+    "epoch": 0.27490354597961236,
+    "step": 8960
+  },
+  {
+    "loss": 0.136,
+    "grad_norm": 0.12726104259490967,
+    "learning_rate": 2.6492017119189417e-06,
+    "epoch": 0.275517169966174,
+    "step": 8980
+  },
+  {
+    "loss": 0.1351,
+    "grad_norm": 0.16233795881271362,
+    "learning_rate": 2.547212649466568e-06,
+    "epoch": 0.2761307939527356,
+    "step": 9000
+  },
+  {
+    "eval_loss": 0.13288874924182892,
+    "eval_runtime": 2779.1785,
+    "eval_samples_per_second": 161.249,
+    "eval_steps_per_second": 2.52,
+    "epoch": 0.2761307939527356,
+    "step": 9000
+  },
+  {
+    "loss": 0.1356,
+    "grad_norm": 0.15901215374469757,
+    "learning_rate": 2.4471741852423237e-06,
+    "epoch": 0.27674441793929727,
+    "step": 9020
+  },
+  {
+    "loss": 0.1329,
+    "grad_norm": 0.11479828506708145,
+    "learning_rate": 2.349090431435641e-06,
+    "epoch": 0.2773580419258589,
+    "step": 9040
+  },
+  {
+    "loss": 0.1347,
+    "grad_norm": 0.1306300163269043,
+    "learning_rate": 2.2529654198854835e-06,
+    "epoch": 0.2779716659124205,
+    "step": 9060
+  },
+  {
+    "loss": 0.137,
+    "grad_norm": 0.14198105037212372,
+    "learning_rate": 2.1588031019145636e-06,
+    "epoch": 0.2785852898989821,
+    "step": 9080
+  },
+  {
+    "loss": 0.1345,
+    "grad_norm": 0.1466785967350006,
+    "learning_rate": 2.066607348166971e-06,
+    "epoch": 0.2791989138855438,
+    "step": 9100
+  },
+  {
+    "loss": 0.1324,
+    "grad_norm": 0.1188543438911438,
+    "learning_rate": 1.9763819484490355e-06,
+    "epoch": 0.2798125378721054,
+    "step": 9120
+  },
+  {
+    "loss": 0.1374,
+    "grad_norm": 0.14264015853405,
+    "learning_rate": 1.888130611573563e-06,
+    "epoch": 0.28042616185866703,
+    "step": 9140
+  },
+  {
+    "loss": 0.133,
+    "grad_norm": 0.10809975862503052,
+    "learning_rate": 1.8018569652073381e-06,
+    "epoch": 0.2810397858452287,
+    "step": 9160
+  },
+  {
+    "loss": 0.1323,
+    "grad_norm": 0.14256234467029572,
+    "learning_rate": 1.7175645557220566e-06,
+    "epoch": 0.2816534098317903,
+    "step": 9180
+  },
+  {
+    "loss": 0.1327,
+    "grad_norm": 0.12473766505718231,
+    "learning_rate": 1.6352568480485276e-06,
+    "epoch": 0.28226703381835194,
+    "step": 9200
+  },
+  {
+    "loss": 0.1311,
+    "grad_norm": 0.0975056141614914,
+    "learning_rate": 1.5549372255342366e-06,
+    "epoch": 0.2828806578049136,
+    "step": 9220
+  },
+  {
+    "loss": 0.1349,
+    "grad_norm": 0.13847889006137848,
+    "learning_rate": 1.4766089898042678e-06,
+    "epoch": 0.28349428179147523,
+    "step": 9240
+  },
+  {
+    "loss": 0.1332,
+    "grad_norm": 0.10670981556177139,
+    "learning_rate": 1.400275360625608e-06,
+    "epoch": 0.28410790577803685,
+    "step": 9260
+  },
+  {
+    "loss": 0.1308,
+    "grad_norm": 0.13052289187908173,
+    "learning_rate": 1.325939475774768e-06,
+    "epoch": 0.2847215297645985,
+    "step": 9280
+  },
+  {
+    "loss": 0.1359,
+    "grad_norm": 0.1260124295949936,
+    "learning_rate": 1.2536043909088191e-06,
+    "epoch": 0.28533515375116014,
+    "step": 9300
+  },
+  {
+    "loss": 0.129,
+    "grad_norm": 0.11741944402456284,
+    "learning_rate": 1.183273079439795e-06,
+    "epoch": 0.28594877773772176,
+    "step": 9320
+  },
+  {
+    "loss": 0.1364,
+    "grad_norm": 0.13720561563968658,
+    "learning_rate": 1.1149484324124327e-06,
+    "epoch": 0.2865624017242834,
+    "step": 9340
+  },
+  {
+    "loss": 0.14,
+    "grad_norm": 0.15556901693344116,
+    "learning_rate": 1.0486332583853563e-06,
+    "epoch": 0.28717602571084505,
+    "step": 9360
+  },
+  {
+    "loss": 0.1358,
+    "grad_norm": 0.10475285351276398,
+    "learning_rate": 9.843302833156376e-07,
+    "epoch": 0.28778964969740667,
+    "step": 9380
+  },
+  {
+    "loss": 0.1302,
+    "grad_norm": 0.1152934804558754,
+    "learning_rate": 9.220421504467281e-07,
+    "epoch": 0.2884032736839683,
+    "step": 9400
+  },
+  {
+    "loss": 0.1376,
+    "grad_norm": 0.13443966209888458,
+    "learning_rate": 8.617714201998084e-07,
+    "epoch": 0.28901689767052996,
+    "step": 9420
+  },
+  {
+    "loss": 0.1363,
+    "grad_norm": 0.11530063301324844,
+    "learning_rate": 8.035205700685167e-07,
+    "epoch": 0.2896305216570916,
+    "step": 9440
+  },
+  {
+    "loss": 0.134,
+    "grad_norm": 0.1621532291173935,
+    "learning_rate": 7.472919945171631e-07,
+    "epoch": 0.2902441456436532,
+    "step": 9460
+  },
+  {
+    "loss": 0.1346,
+    "grad_norm": 0.15441814064979553,
+    "learning_rate": 6.93088004882253e-07,
+    "epoch": 0.29085776963021487,
+    "step": 9480
+  },
+  {
+    "loss": 0.1273,
+    "grad_norm": 0.13251030445098877,
+    "learning_rate": 6.409108292774913e-07,
+    "epoch": 0.2914713936167765,
+    "step": 9500
+  },
+  {
+    "eval_loss": 0.1326712667942047,
+    "eval_runtime": 2779.8317,
+    "eval_samples_per_second": 161.211,
+    "eval_steps_per_second": 2.519,
+    "epoch": 0.2914713936167765,
+    "step": 9500
+  },
+  {
+    "loss": 0.1357,
+    "grad_norm": 0.13315628468990326,
+    "learning_rate": 5.907626125022159e-07,
+    "epoch": 0.2920850176033381,
+    "step": 9520
+  },
+  {
+    "loss": 0.1344,
+    "grad_norm": 0.11808905005455017,
+    "learning_rate": 5.426454159531913e-07,
+    "epoch": 0.2926986415898997,
+    "step": 9540
+  },
+  {
+    "loss": 0.1332,
+    "grad_norm": 0.11968034505844116,
+    "learning_rate": 4.965612175399092e-07,
+    "epoch": 0.2933122655764614,
+    "step": 9560
+  },
+  {
+    "loss": 0.1285,
+    "grad_norm": 0.16589680314064026,
+    "learning_rate": 4.52511911603265e-07,
+    "epoch": 0.293925889563023,
+    "step": 9580
+  },
+  {
+    "loss": 0.1317,
+    "grad_norm": 0.1595570296049118,
+    "learning_rate": 4.104993088376974e-07,
+    "epoch": 0.29453951354958463,
+    "step": 9600
+  },
+  {
+    "loss": 0.135,
+    "grad_norm": 0.10978726297616959,
+    "learning_rate": 3.7052513621674833e-07,
+    "epoch": 0.2951531375361463,
+    "step": 9620
+  },
+  {
+    "loss": 0.1319,
+    "grad_norm": 0.10192031413316727,
+    "learning_rate": 3.3259103692209747e-07,
+    "epoch": 0.2957667615227079,
+    "step": 9640
+  },
+  {
+    "loss": 0.1324,
+    "grad_norm": 0.10372564941644669,
+    "learning_rate": 2.966985702759828e-07,
+    "epoch": 0.29638038550926954,
+    "step": 9660
+  },
+  {
+    "loss": 0.1313,
+    "grad_norm": 0.17446647584438324,
+    "learning_rate": 2.6284921167712973e-07,
+    "epoch": 0.2969940094958312,
+    "step": 9680
+  },
+  {
+    "loss": 0.1347,
+    "grad_norm": 0.1544017642736435,
+    "learning_rate": 2.310443525400885e-07,
+    "epoch": 0.29760763348239283,
+    "step": 9700
+  },
+  {
+    "loss": 0.1292,
+    "grad_norm": 0.29529738426208496,
+    "learning_rate": 2.012853002380466e-07,
+    "epoch": 0.29822125746895445,
+    "step": 9720
+  },
+  {
+    "loss": 0.1311,
+    "grad_norm": 0.14359545707702637,
+    "learning_rate": 1.735732780490884e-07,
+    "epoch": 0.2988348814555161,
+    "step": 9740
+  },
+  {
+    "loss": 0.1333,
+    "grad_norm": 0.10971110314130783,
+    "learning_rate": 1.4790942510590766e-07,
+    "epoch": 0.29944850544207774,
+    "step": 9760
+  },
+  {
+    "loss": 0.122,
+    "grad_norm": 0.15253949165344238,
+    "learning_rate": 1.2429479634897267e-07,
+    "epoch": 0.30006212942863936,
+    "step": 9780
+  },
+  {
+    "loss": 0.1309,
+    "grad_norm": 0.12937602400779724,
+    "learning_rate": 1.0273036248318324e-07,
+    "epoch": 0.300675753415201,
+    "step": 9800
+  },
+  {
+    "loss": 0.1267,
+    "grad_norm": 0.12739217281341553,
+    "learning_rate": 8.321700993795811e-08,
+    "epoch": 0.30128937740176265,
+    "step": 9820
+  },
+  {
+    "loss": 0.1305,
+    "grad_norm": 0.11524944007396698,
+    "learning_rate": 6.575554083078084e-08,
+    "epoch": 0.30190300138832427,
+    "step": 9840
+  },
+  {
+    "loss": 0.1307,
+    "grad_norm": 0.14692504703998566,
+    "learning_rate": 5.0346672934270534e-08,
+    "epoch": 0.3025166253748859,
+    "step": 9860
+  },
+  {
+    "loss": 0.1297,
+    "grad_norm": 0.14273245632648468,
+    "learning_rate": 3.699103964661665e-08,
+    "epoch": 0.30313024936144756,
+    "step": 9880
+  },
+  {
+    "loss": 0.1271,
+    "grad_norm": 0.13972416520118713,
+    "learning_rate": 2.568918996560532e-08,
+    "epoch": 0.3037438733480092,
+    "step": 9900
+  },
+  {
+    "loss": 0.1342,
+    "grad_norm": 0.12904135882854462,
+    "learning_rate": 1.644158846600963e-08,
+    "epoch": 0.3043574973345708,
+    "step": 9920
+  },
+  {
+    "loss": 0.1334,
+    "grad_norm": 0.09770902246236801,
+    "learning_rate": 9.248615280499361e-09,
+    "epoch": 0.30497112132113247,
+    "step": 9940
+  },
+  {
+    "loss": 0.1388,
+    "grad_norm": 0.10842837393283844,
+    "learning_rate": 4.110566084036816e-09,
+    "epoch": 0.3055847453076941,
+    "step": 9960
+  },
+  {
+    "loss": 0.1264,
+    "grad_norm": 0.10802903026342392,
+    "learning_rate": 1.0276520816976387e-09,
+    "epoch": 0.3061983692942557,
+    "step": 9980
+  },
+  {
+    "loss": 0.1283,
+    "grad_norm": 0.14172281324863434,
+    "learning_rate": 0.0,
+    "epoch": 0.3068119932808174,
+    "step": 10000
+  },
+  {
+    "eval_loss": 0.1326160877943039,
+    "eval_runtime": 2780.1651,
+    "eval_samples_per_second": 161.192,
+    "eval_steps_per_second": 2.519,
+    "epoch": 0.3068119932808174,
+    "step": 10000
+  },
+  {
+    "train_runtime": 73306.6559,
+    "train_samples_per_second": 17.461,
+    "train_steps_per_second": 0.136,
+    "total_flos": 4.748331528373666e+18,
+    "train_loss": 0.17932859930992126,
+    "epoch": 0.3068119932808174,
+    "step": 10000
+  }
+]