Add files using upload-large-folder tool

config.json

@@ -0,0 +1,38 @@
+{
+  "_name_or_path": "BAAI/Emu3-Stage1",
+  "architectures": [
+    "Emu3ForCausalLM"
+  ],
+  "attention_dropout": 0.1,
+  "auto_map": {
+    "AutoConfig": "BAAI/Emu3-Stage1--configuration_emu3.Emu3Config",
+    "AutoModelForCausalLM": "BAAI/Emu3-Stage1--modeling_emu3.Emu3ForCausalLM"
+  },
+  "boi_token_id": 151852,
+  "bos_token_id": 151849,
+  "eof_token_id": 151847,
+  "eoi_token_id": 151853,
+  "eol_token_id": 151846,
+  "eos_token_id": 151850,
+  "hidden_act": "silu",
+  "hidden_size": 4096,
+  "image_area": 65536,
+  "img_token_id": 151851,
+  "initializer_range": 0.02,
+  "intermediate_size": 14336,
+  "max_position_embeddings": 4300,
+  "model_type": "Emu3",
+  "num_attention_heads": 32,
+  "num_hidden_layers": 32,
+  "num_key_value_heads": 8,
+  "pad_token_id": 151643,
+  "pretraining_tp": 1,
+  "rms_norm_eps": 1e-05,
+  "rope_scaling": null,
+  "rope_theta": 1000000.0,
+  "tie_word_embeddings": false,
+  "torch_dtype": "bfloat16",
+  "transformers_version": "4.44.0",
+  "use_cache": false,
+  "vocab_size": 184622
+}

generation_config.json

@@ -0,0 +1,7 @@
+{
+  "_from_model_config": true,
+  "bos_token_id": 151849,
+  "eos_token_id": 151850,
+  "pad_token_id": 151643,
+  "transformers_version": "4.44.0"
+}

model-00001-of-00004.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:adddf16bb9340fa30fb1d4cd25c8a493ca755eab790ba05f09c1a6da856a2395
+size 4884766656

model-00002-of-00004.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4b3dd55877752ea2f125dadc10e8e34a38e435478fa5bec3189b045d16fbf1ff
+size 4999819320

model-00003-of-00004.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4a5458f025f163eba5b752a7957da35306fdd5c9d1238c0da9b9f0005db0e238
+size 4915916184

model-00004-of-00004.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3a41a5dcb891a255d1260b0872a2d05b8c9bcb260f365f96c472fb096f009864
+size 2183554760

model.safetensors.index.json

@@ -0,0 +1,298 @@
+{
+  "metadata": {
+    "total_size": 16984023040
+  },
+  "weight_map": {
+    "lm_head.weight": "model-00004-of-00004.safetensors",
+    "model.embed_tokens.weight": "model-00001-of-00004.safetensors",
+    "model.layers.0.input_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.0.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.0.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.0.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.0.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.0.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.0.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.0.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.0.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.1.input_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.1.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.1.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.1.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.1.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.1.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.1.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.1.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.1.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.10.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.10.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.10.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.10.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.10.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.10.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.10.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.10.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.10.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.11.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.11.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.11.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.11.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.11.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.11.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.11.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.11.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.11.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.12.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.12.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.12.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.12.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.12.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.12.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.12.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.12.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.12.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.13.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.13.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.13.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.13.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.13.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.13.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.13.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.13.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.13.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.14.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.14.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.14.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.14.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.14.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.14.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.14.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.14.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.14.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.15.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.15.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.15.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.15.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.15.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.15.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.15.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.15.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.15.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.16.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.16.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.16.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.16.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.16.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.16.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.16.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.16.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.16.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.17.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.17.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.17.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.17.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.17.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.17.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.17.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.17.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.17.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.18.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.18.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.18.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.18.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.18.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.18.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.18.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.18.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.18.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.19.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.19.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.19.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.19.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.19.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.19.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.19.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.19.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.19.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.2.input_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.2.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.2.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.2.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.2.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.2.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.2.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.2.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.2.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.20.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.20.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.20.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.20.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.20.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.20.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.20.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.20.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.20.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.21.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.21.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.21.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.21.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.21.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.21.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.21.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.21.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.21.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.22.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.22.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.22.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.22.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.22.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.22.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.22.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.22.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.22.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.23.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.23.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.23.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.23.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.23.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.23.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.23.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.23.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.23.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.24.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.24.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.24.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.24.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.24.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.24.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.24.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.24.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.24.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.25.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.25.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.25.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.25.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.25.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.25.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.25.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.25.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.25.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.26.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.26.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.26.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.26.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.26.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.26.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.26.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.26.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.26.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.27.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.27.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.27.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.27.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.27.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.27.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.27.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.27.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.27.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.28.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.28.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.28.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.28.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.28.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.28.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.28.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.28.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.28.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.29.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.29.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.29.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.29.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.29.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.29.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.29.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.29.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.29.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.3.input_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.3.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.3.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.3.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.3.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.3.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.3.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.3.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.3.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.30.input_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.30.mlp.down_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.30.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.30.mlp.up_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.30.post_attention_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.30.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.30.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.30.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.30.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.31.input_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.31.mlp.down_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.31.mlp.gate_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.31.mlp.up_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.31.post_attention_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.31.self_attn.k_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.31.self_attn.o_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.31.self_attn.q_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.31.self_attn.v_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.4.input_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.4.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.4.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.4.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.4.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.4.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.4.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.4.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.4.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.5.input_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.5.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.5.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.5.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.5.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.5.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.5.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.5.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.5.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.6.input_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.6.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.6.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.6.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.6.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.6.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.6.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.6.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.6.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.7.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.7.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.7.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.7.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.7.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.7.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.7.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.7.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.7.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.8.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.8.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.8.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.8.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.8.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.8.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.8.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.8.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.8.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.9.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.9.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.9.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.9.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.9.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.9.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.9.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.9.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.9.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.norm.weight": "model-00004-of-00004.safetensors"
+  }
+}

rng_state_0.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:38e0d3e802c77bc685823f2833ead076927c764a40a61ed296e102d43cda0d8d
+size 15984

rng_state_1.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:67a9416bd5b07927f61aa530c008feacdd5c0d0c721cdcb62d87ec9d09ebff02
+size 15984

rng_state_2.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b86dbccc17b5abb08def7becf34d1956cc04af4837d863282f136d9b95689a63
+size 15984

rng_state_3.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ccafb5c0712165018138aae1d379bfde77dd02155e21b215cd7584996a6fca50
+size 15984

rng_state_4.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8098dfed8b271f2002df0bbcacc2624042d8e77d52e229192ff6cf6291e61332
+size 15984

rng_state_5.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c04ec7140cc6969cf770a694b838d7a41f20f6adc20815e93210990bba24bec0
+size 15984

rng_state_6.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:697e3bf6ceebf7e703291675ecfe41b204ed3ed6473592dd25d0ab31cb9d6d8c
+size 15984

rng_state_7.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3c47a1fbb66661f5c40ce6020dd6fa39cc72030b3cd3d5166f1d9b131321f0b5
+size 15984

trainer_state.json

@@ -0,0 +1,1793 @@
+{
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 20.6252014179826,
+  "eval_steps": 100,
+  "global_step": 8000,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.12890750886239125,
+      "grad_norm": 5.428058115945689,
+      "learning_rate": 0.0001,
+      "loss": 3.7217,
+      "step": 50
+    },
+    {
+      "epoch": 0.2578150177247825,
+      "grad_norm": 3.6736343068699004,
+      "learning_rate": 9.999743504733682e-05,
+      "loss": 2.6957,
+      "step": 100
+    },
+    {
+      "epoch": 0.2578150177247825,
+      "eval_loss": 2.472891092300415,
+      "eval_runtime": 13.3767,
+      "eval_samples_per_second": 74.757,
+      "eval_steps_per_second": 2.392,
+      "step": 100
+    },
+    {
+      "epoch": 0.3867225265871737,
+      "grad_norm": 3.9566460988411976,
+      "learning_rate": 9.998974045516476e-05,
+      "loss": 2.3982,
+      "step": 150
+    },
+    {
+      "epoch": 0.515630035449565,
+      "grad_norm": 1.91000275173515,
+      "learning_rate": 9.997691702090861e-05,
+      "loss": 2.188,
+      "step": 200
+    },
+    {
+      "epoch": 0.515630035449565,
+      "eval_loss": 2.072312593460083,
+      "eval_runtime": 13.1385,
+      "eval_samples_per_second": 76.112,
+      "eval_steps_per_second": 2.436,
+      "step": 200
+    },
+    {
+      "epoch": 0.6445375443119562,
+      "grad_norm": 3.7326458687273494,
+      "learning_rate": 9.9958966073518e-05,
+      "loss": 2.058,
+      "step": 250
+    },
+    {
+      "epoch": 0.7734450531743474,
+      "grad_norm": 1.1877292780347308,
+      "learning_rate": 9.993588947332947e-05,
+      "loss": 2.0499,
+      "step": 300
+    },
+    {
+      "epoch": 0.7734450531743474,
+      "eval_loss": 1.9889113903045654,
+      "eval_runtime": 13.0859,
+      "eval_samples_per_second": 76.418,
+      "eval_steps_per_second": 2.445,
+      "step": 300
+    },
+    {
+      "epoch": 0.9023525620367386,
+      "grad_norm": 2.896903582467451,
+      "learning_rate": 9.990768961187381e-05,
+      "loss": 1.9975,
+      "step": 350
+    },
+    {
+      "epoch": 1.03126007089913,
+      "grad_norm": 1.1169559646807505,
+      "learning_rate": 9.98743694116282e-05,
+      "loss": 1.9593,
+      "step": 400
+    },
+    {
+      "epoch": 1.03126007089913,
+      "eval_loss": 1.9284899234771729,
+      "eval_runtime": 13.1076,
+      "eval_samples_per_second": 76.292,
+      "eval_steps_per_second": 2.441,
+      "step": 400
+    },
+    {
+      "epoch": 1.1601675797615212,
+      "grad_norm": 0.9849310297267434,
+      "learning_rate": 9.983593232571328e-05,
+      "loss": 1.9186,
+      "step": 450
+    },
+    {
+      "epoch": 1.2890750886239124,
+      "grad_norm": 0.9560509903560709,
+      "learning_rate": 9.979238233753537e-05,
+      "loss": 1.921,
+      "step": 500
+    },
+    {
+      "epoch": 1.2890750886239124,
+      "eval_loss": 1.899874210357666,
+      "eval_runtime": 13.1392,
+      "eval_samples_per_second": 76.108,
+      "eval_steps_per_second": 2.435,
+      "step": 500
+    },
+    {
+      "epoch": 1.4179825974863036,
+      "grad_norm": 1.3741796182310861,
+      "learning_rate": 9.974372396037357e-05,
+      "loss": 1.9246,
+      "step": 550
+    },
+    {
+      "epoch": 1.5468901063486948,
+      "grad_norm": 1.718108772048462,
+      "learning_rate": 9.96899622369121e-05,
+      "loss": 1.8882,
+      "step": 600
+    },
+    {
+      "epoch": 1.5468901063486948,
+      "eval_loss": 1.8836181163787842,
+      "eval_runtime": 13.1533,
+      "eval_samples_per_second": 76.026,
+      "eval_steps_per_second": 2.433,
+      "step": 600
+    },
+    {
+      "epoch": 1.675797615211086,
+      "grad_norm": 1.3793562879868584,
+      "learning_rate": 9.963110273871768e-05,
+      "loss": 1.8823,
+      "step": 650
+    },
+    {
+      "epoch": 1.8047051240734773,
+      "grad_norm": 1.7941360700066475,
+      "learning_rate": 9.956715156566203e-05,
+      "loss": 1.8786,
+      "step": 700
+    },
+    {
+      "epoch": 1.8047051240734773,
+      "eval_loss": 1.874005913734436,
+      "eval_runtime": 13.1279,
+      "eval_samples_per_second": 76.174,
+      "eval_steps_per_second": 2.438,
+      "step": 700
+    },
+    {
+      "epoch": 1.9336126329358685,
+      "grad_norm": 1.2268118375629977,
+      "learning_rate": 9.94981153452899e-05,
+      "loss": 1.8764,
+      "step": 750
+    },
+    {
+      "epoch": 2.06252014179826,
+      "grad_norm": 0.8694323872512157,
+      "learning_rate": 9.94240012321321e-05,
+      "loss": 1.8327,
+      "step": 800
+    },
+    {
+      "epoch": 2.06252014179826,
+      "eval_loss": 1.8573421239852905,
+      "eval_runtime": 13.1123,
+      "eval_samples_per_second": 76.264,
+      "eval_steps_per_second": 2.44,
+      "step": 800
+    },
+    {
+      "epoch": 2.191427650660651,
+      "grad_norm": 1.6779131964119156,
+      "learning_rate": 9.934481690696406e-05,
+      "loss": 1.7937,
+      "step": 850
+    },
+    {
+      "epoch": 2.3203351595230424,
+      "grad_norm": 1.2570407836555852,
+      "learning_rate": 9.92605705760098e-05,
+      "loss": 1.8047,
+      "step": 900
+    },
+    {
+      "epoch": 2.3203351595230424,
+      "eval_loss": 1.8506410121917725,
+      "eval_runtime": 13.1515,
+      "eval_samples_per_second": 76.037,
+      "eval_steps_per_second": 2.433,
+      "step": 900
+    },
+    {
+      "epoch": 2.4492426683854336,
+      "grad_norm": 0.9012297719032618,
+      "learning_rate": 9.917127097009165e-05,
+      "loss": 1.8028,
+      "step": 950
+    },
+    {
+      "epoch": 2.578150177247825,
+      "grad_norm": 2.071809054799254,
+      "learning_rate": 9.907692734372522e-05,
+      "loss": 1.8001,
+      "step": 1000
+    },
+    {
+      "epoch": 2.578150177247825,
+      "eval_loss": 1.8604120016098022,
+      "eval_runtime": 13.1616,
+      "eval_samples_per_second": 75.978,
+      "eval_steps_per_second": 2.431,
+      "step": 1000
+    },
+    {
+      "epoch": 2.707057686110216,
+      "grad_norm": 1.8799616432709254,
+      "learning_rate": 9.897754947416042e-05,
+      "loss": 1.8009,
+      "step": 1050
+    },
+    {
+      "epoch": 2.8359651949726072,
+      "grad_norm": 0.9523078941745302,
+      "learning_rate": 9.887314766036823e-05,
+      "loss": 1.8016,
+      "step": 1100
+    },
+    {
+      "epoch": 2.8359651949726072,
+      "eval_loss": 1.8416086435317993,
+      "eval_runtime": 13.1272,
+      "eval_samples_per_second": 76.178,
+      "eval_steps_per_second": 2.438,
+      "step": 1100
+    },
+    {
+      "epoch": 2.9648727038349985,
+      "grad_norm": 0.7548354411956247,
+      "learning_rate": 9.87637327219733e-05,
+      "loss": 1.7941,
+      "step": 1150
+    },
+    {
+      "epoch": 3.0937802126973897,
+      "grad_norm": 1.603556668939176,
+      "learning_rate": 9.86493159981327e-05,
+      "loss": 1.7221,
+      "step": 1200
+    },
+    {
+      "epoch": 3.0937802126973897,
+      "eval_loss": 1.8572254180908203,
+      "eval_runtime": 13.1348,
+      "eval_samples_per_second": 76.134,
+      "eval_steps_per_second": 2.436,
+      "step": 1200
+    },
+    {
+      "epoch": 3.222687721559781,
+      "grad_norm": 1.1914026184365651,
+      "learning_rate": 9.85299093463608e-05,
+      "loss": 1.6919,
+      "step": 1250
+    },
+    {
+      "epoch": 3.351595230422172,
+      "grad_norm": 1.108713427331391,
+      "learning_rate": 9.840552514130043e-05,
+      "loss": 1.6979,
+      "step": 1300
+    },
+    {
+      "epoch": 3.351595230422172,
+      "eval_loss": 1.8494083881378174,
+      "eval_runtime": 13.1546,
+      "eval_samples_per_second": 76.019,
+      "eval_steps_per_second": 2.433,
+      "step": 1300
+    },
+    {
+      "epoch": 3.4805027392845633,
+      "grad_norm": 1.188923474560524,
+      "learning_rate": 9.827617627344035e-05,
+      "loss": 1.7012,
+      "step": 1350
+    },
+    {
+      "epoch": 3.6094102481469545,
+      "grad_norm": 1.1069030836862073,
+      "learning_rate": 9.814187614777952e-05,
+      "loss": 1.7117,
+      "step": 1400
+    },
+    {
+      "epoch": 3.6094102481469545,
+      "eval_loss": 1.8396939039230347,
+      "eval_runtime": 13.1058,
+      "eval_samples_per_second": 76.302,
+      "eval_steps_per_second": 2.442,
+      "step": 1400
+    },
+    {
+      "epoch": 3.7383177570093458,
+      "grad_norm": 1.4927257455688545,
+      "learning_rate": 9.800263868243771e-05,
+      "loss": 1.7115,
+      "step": 1450
+    },
+    {
+      "epoch": 3.867225265871737,
+      "grad_norm": 0.9082276480595701,
+      "learning_rate": 9.785847830721322e-05,
+      "loss": 1.7181,
+      "step": 1500
+    },
+    {
+      "epoch": 3.867225265871737,
+      "eval_loss": 1.8317677974700928,
+      "eval_runtime": 13.1459,
+      "eval_samples_per_second": 76.069,
+      "eval_steps_per_second": 2.434,
+      "step": 1500
+    },
+    {
+      "epoch": 3.996132774734128,
+      "grad_norm": 1.38818621871337,
+      "learning_rate": 9.770940996208739e-05,
+      "loss": 1.7195,
+      "step": 1550
+    },
+    {
+      "epoch": 4.12504028359652,
+      "grad_norm": 1.1154867829165278,
+      "learning_rate": 9.755544909567632e-05,
+      "loss": 1.5548,
+      "step": 1600
+    },
+    {
+      "epoch": 4.12504028359652,
+      "eval_loss": 1.8801182508468628,
+      "eval_runtime": 13.1664,
+      "eval_samples_per_second": 75.951,
+      "eval_steps_per_second": 2.43,
+      "step": 1600
+    },
+    {
+      "epoch": 4.253947792458911,
+      "grad_norm": 1.741656038399529,
+      "learning_rate": 9.73966116636299e-05,
+      "loss": 1.5498,
+      "step": 1650
+    },
+    {
+      "epoch": 4.382855301321302,
+      "grad_norm": 0.9507921869813947,
+      "learning_rate": 9.723291412697821e-05,
+      "loss": 1.5728,
+      "step": 1700
+    },
+    {
+      "epoch": 4.382855301321302,
+      "eval_loss": 1.8739728927612305,
+      "eval_runtime": 13.1362,
+      "eval_samples_per_second": 76.126,
+      "eval_steps_per_second": 2.436,
+      "step": 1700
+    },
+    {
+      "epoch": 4.5117628101836935,
+      "grad_norm": 1.1876157181384903,
+      "learning_rate": 9.706437345042558e-05,
+      "loss": 1.5896,
+      "step": 1750
+    },
+    {
+      "epoch": 4.640670319046085,
+      "grad_norm": 1.365051813978068,
+      "learning_rate": 9.689100710059251e-05,
+      "loss": 1.5998,
+      "step": 1800
+    },
+    {
+      "epoch": 4.640670319046085,
+      "eval_loss": 1.8598759174346924,
+      "eval_runtime": 13.1287,
+      "eval_samples_per_second": 76.169,
+      "eval_steps_per_second": 2.437,
+      "step": 1800
+    },
+    {
+      "epoch": 4.769577827908476,
+      "grad_norm": 1.3205776295240563,
+      "learning_rate": 9.67128330442055e-05,
+      "loss": 1.5974,
+      "step": 1850
+    },
+    {
+      "epoch": 4.898485336770867,
+      "grad_norm": 1.4453793304970697,
+      "learning_rate": 9.652986974623506e-05,
+      "loss": 1.6127,
+      "step": 1900
+    },
+    {
+      "epoch": 4.898485336770867,
+      "eval_loss": 1.8465975522994995,
+      "eval_runtime": 13.1264,
+      "eval_samples_per_second": 76.182,
+      "eval_steps_per_second": 2.438,
+      "step": 1900
+    },
+    {
+      "epoch": 5.027392845633258,
+      "grad_norm": 2.3043609697161633,
+      "learning_rate": 9.634213616798214e-05,
+      "loss": 1.5733,
+      "step": 1950
+    },
+    {
+      "epoch": 5.15630035449565,
+      "grad_norm": 1.1111652873863385,
+      "learning_rate": 9.614965176511308e-05,
+      "loss": 1.3923,
+      "step": 2000
+    },
+    {
+      "epoch": 5.15630035449565,
+      "eval_loss": 1.9901988506317139,
+      "eval_runtime": 13.1434,
+      "eval_samples_per_second": 76.084,
+      "eval_steps_per_second": 2.435,
+      "step": 2000
+    },
+    {
+      "epoch": 5.285207863358041,
+      "grad_norm": 1.8996098810923527,
+      "learning_rate": 9.595243648564326e-05,
+      "loss": 1.3904,
+      "step": 2050
+    },
+    {
+      "epoch": 5.414115372220432,
+      "grad_norm": 0.8248818797353966,
+      "learning_rate": 9.575051076786994e-05,
+      "loss": 1.413,
+      "step": 2100
+    },
+    {
+      "epoch": 5.414115372220432,
+      "eval_loss": 1.941409945487976,
+      "eval_runtime": 13.1615,
+      "eval_samples_per_second": 75.979,
+      "eval_steps_per_second": 2.431,
+      "step": 2100
+    },
+    {
+      "epoch": 5.543022881082823,
+      "grad_norm": 1.3796008879183341,
+      "learning_rate": 9.554389553825404e-05,
+      "loss": 1.4426,
+      "step": 2150
+    },
+    {
+      "epoch": 5.6719303899452145,
+      "grad_norm": 1.2370414797788207,
+      "learning_rate": 9.533261220925143e-05,
+      "loss": 1.4441,
+      "step": 2200
+    },
+    {
+      "epoch": 5.6719303899452145,
+      "eval_loss": 1.9324249029159546,
+      "eval_runtime": 13.1482,
+      "eval_samples_per_second": 76.056,
+      "eval_steps_per_second": 2.434,
+      "step": 2200
+    },
+    {
+      "epoch": 5.800837898807606,
+      "grad_norm": 0.7933476727900988,
+      "learning_rate": 9.511668267709395e-05,
+      "loss": 1.4578,
+      "step": 2250
+    },
+    {
+      "epoch": 5.929745407669997,
+      "grad_norm": 0.9922228439520735,
+      "learning_rate": 9.48961293195201e-05,
+      "loss": 1.4729,
+      "step": 2300
+    },
+    {
+      "epoch": 5.929745407669997,
+      "eval_loss": 1.9070981740951538,
+      "eval_runtime": 13.1414,
+      "eval_samples_per_second": 76.095,
+      "eval_steps_per_second": 2.435,
+      "step": 2300
+    },
+    {
+      "epoch": 6.058652916532388,
+      "grad_norm": 1.133228933331563,
+      "learning_rate": 9.467097499345605e-05,
+      "loss": 1.3338,
+      "step": 2350
+    },
+    {
+      "epoch": 6.187560425394779,
+      "grad_norm": 1.1388176086742452,
+      "learning_rate": 9.444124303264675e-05,
+      "loss": 1.1768,
+      "step": 2400
+    },
+    {
+      "epoch": 6.187560425394779,
+      "eval_loss": 2.123551607131958,
+      "eval_runtime": 13.1885,
+      "eval_samples_per_second": 75.824,
+      "eval_steps_per_second": 2.426,
+      "step": 2400
+    },
+    {
+      "epoch": 6.316467934257171,
+      "grad_norm": 1.0540569425798711,
+      "learning_rate": 9.420695724523785e-05,
+      "loss": 1.1952,
+      "step": 2450
+    },
+    {
+      "epoch": 6.445375443119562,
+      "grad_norm": 1.0205037806931825,
+      "learning_rate": 9.396814191130831e-05,
+      "loss": 1.2409,
+      "step": 2500
+    },
+    {
+      "epoch": 6.445375443119562,
+      "eval_loss": 2.0799171924591064,
+      "eval_runtime": 13.1885,
+      "eval_samples_per_second": 75.824,
+      "eval_steps_per_second": 2.426,
+      "step": 2500
+    },
+    {
+      "epoch": 6.574282951981953,
+      "grad_norm": 1.3287277537295186,
+      "learning_rate": 9.37248217803542e-05,
+      "loss": 1.2484,
+      "step": 2550
+    },
+    {
+      "epoch": 6.703190460844344,
+      "grad_norm": 1.7324707650665503,
+      "learning_rate": 9.347702206872368e-05,
+      "loss": 1.2794,
+      "step": 2600
+    },
+    {
+      "epoch": 6.703190460844344,
+      "eval_loss": 2.066530227661133,
+      "eval_runtime": 13.1457,
+      "eval_samples_per_second": 76.07,
+      "eval_steps_per_second": 2.434,
+      "step": 2600
+    },
+    {
+      "epoch": 6.832097969706735,
+      "grad_norm": 0.7821269122995969,
+      "learning_rate": 9.322476845700381e-05,
+      "loss": 1.2872,
+      "step": 2650
+    },
+    {
+      "epoch": 6.961005478569127,
+      "grad_norm": 1.0321720718594696,
+      "learning_rate": 9.296808708735924e-05,
+      "loss": 1.2975,
+      "step": 2700
+    },
+    {
+      "epoch": 6.961005478569127,
+      "eval_loss": 2.028914213180542,
+      "eval_runtime": 13.188,
+      "eval_samples_per_second": 75.827,
+      "eval_steps_per_second": 2.426,
+      "step": 2700
+    },
+    {
+      "epoch": 7.089912987431518,
+      "grad_norm": 0.8079357741453,
+      "learning_rate": 9.270700456082274e-05,
+      "loss": 1.0552,
+      "step": 2750
+    },
+    {
+      "epoch": 7.218820496293909,
+      "grad_norm": 1.2220177415199573,
+      "learning_rate": 9.244154793453865e-05,
+      "loss": 0.9573,
+      "step": 2800
+    },
+    {
+      "epoch": 7.218820496293909,
+      "eval_loss": 2.30826473236084,
+      "eval_runtime": 13.14,
+      "eval_samples_per_second": 76.103,
+      "eval_steps_per_second": 2.435,
+      "step": 2800
+    },
+    {
+      "epoch": 7.3477280051563,
+      "grad_norm": 1.0187532250301208,
+      "learning_rate": 9.217174471895868e-05,
+      "loss": 0.9933,
+      "step": 2850
+    },
+    {
+      "epoch": 7.4766355140186915,
+      "grad_norm": 1.1142206529537857,
+      "learning_rate": 9.189762287499101e-05,
+      "loss": 1.0244,
+      "step": 2900
+    },
+    {
+      "epoch": 7.4766355140186915,
+      "eval_loss": 2.2640790939331055,
+      "eval_runtime": 13.1456,
+      "eval_samples_per_second": 76.071,
+      "eval_steps_per_second": 2.434,
+      "step": 2900
+    },
+    {
+      "epoch": 7.605543022881083,
+      "grad_norm": 0.922470837605761,
+      "learning_rate": 9.16192108111024e-05,
+      "loss": 1.0455,
+      "step": 2950
+    },
+    {
+      "epoch": 7.734450531743474,
+      "grad_norm": 1.4488181883296236,
+      "learning_rate": 9.133653738037431e-05,
+      "loss": 1.066,
+      "step": 3000
+    },
+    {
+      "epoch": 7.734450531743474,
+      "eval_loss": 2.224378824234009,
+      "eval_runtime": 13.1557,
+      "eval_samples_per_second": 76.013,
+      "eval_steps_per_second": 2.432,
+      "step": 3000
+    },
+    {
+      "epoch": 7.863358040605865,
+      "grad_norm": 1.284431896370195,
+      "learning_rate": 9.104963187751253e-05,
+      "loss": 1.0771,
+      "step": 3050
+    },
+    {
+      "epoch": 7.992265549468256,
+      "grad_norm": 1.3116181988612625,
+      "learning_rate": 9.075852403581133e-05,
+      "loss": 1.1098,
+      "step": 3100
+    },
+    {
+      "epoch": 7.992265549468256,
+      "eval_loss": 2.198763608932495,
+      "eval_runtime": 13.1699,
+      "eval_samples_per_second": 75.931,
+      "eval_steps_per_second": 2.43,
+      "step": 3100
+    },
+    {
+      "epoch": 8.121173058330648,
+      "grad_norm": 1.185925605822691,
+      "learning_rate": 9.046324402407213e-05,
+      "loss": 0.733,
+      "step": 3150
+    },
+    {
+      "epoch": 8.25008056719304,
+      "grad_norm": 2.353999238623636,
+      "learning_rate": 9.016382244347679e-05,
+      "loss": 0.7603,
+      "step": 3200
+    },
+    {
+      "epoch": 8.25008056719304,
+      "eval_loss": 2.5014915466308594,
+      "eval_runtime": 13.1569,
+      "eval_samples_per_second": 76.006,
+      "eval_steps_per_second": 2.432,
+      "step": 3200
+    },
+    {
+      "epoch": 8.37898807605543,
+      "grad_norm": 1.3793811731790488,
+      "learning_rate": 8.98602903244165e-05,
+      "loss": 0.7843,
+      "step": 3250
+    },
+    {
+      "epoch": 8.507895584917822,
+      "grad_norm": 1.0500789470659453,
+      "learning_rate": 8.955267912327574e-05,
+      "loss": 0.8025,
+      "step": 3300
+    },
+    {
+      "epoch": 8.507895584917822,
+      "eval_loss": 2.4928858280181885,
+      "eval_runtime": 13.1471,
+      "eval_samples_per_second": 76.062,
+      "eval_steps_per_second": 2.434,
+      "step": 3300
+    },
+    {
+      "epoch": 8.636803093780212,
+      "grad_norm": 1.0687481728737136,
+      "learning_rate": 8.924102071917248e-05,
+      "loss": 0.8292,
+      "step": 3350
+    },
+    {
+      "epoch": 8.765710602642605,
+      "grad_norm": 0.8890330524205967,
+      "learning_rate": 8.89253474106544e-05,
+      "loss": 0.8554,
+      "step": 3400
+    },
+    {
+      "epoch": 8.765710602642605,
+      "eval_loss": 2.4535696506500244,
+      "eval_runtime": 13.1668,
+      "eval_samples_per_second": 75.949,
+      "eval_steps_per_second": 2.43,
+      "step": 3400
+    },
+    {
+      "epoch": 8.894618111504995,
+      "grad_norm": 1.1314521221393643,
+      "learning_rate": 8.860569191235147e-05,
+      "loss": 0.8804,
+      "step": 3450
+    },
+    {
+      "epoch": 9.023525620367387,
+      "grad_norm": 1.1316860727966558,
+      "learning_rate": 8.828208735158577e-05,
+      "loss": 0.8251,
+      "step": 3500
+    },
+    {
+      "epoch": 9.023525620367387,
+      "eval_loss": 2.771848440170288,
+      "eval_runtime": 13.1585,
+      "eval_samples_per_second": 75.996,
+      "eval_steps_per_second": 2.432,
+      "step": 3500
+    },
+    {
+      "epoch": 9.152433129229777,
+      "grad_norm": 2.2363185101646765,
+      "learning_rate": 8.795456726493838e-05,
+      "loss": 0.5115,
+      "step": 3550
+    },
+    {
+      "epoch": 9.28134063809217,
+      "grad_norm": 1.1689864457962205,
+      "learning_rate": 8.762316559477361e-05,
+      "loss": 0.5483,
+      "step": 3600
+    },
+    {
+      "epoch": 9.28134063809217,
+      "eval_loss": 2.738429307937622,
+      "eval_runtime": 13.1693,
+      "eval_samples_per_second": 75.934,
+      "eval_steps_per_second": 2.43,
+      "step": 3600
+    },
+    {
+      "epoch": 9.41024814695456,
+      "grad_norm": 1.1593603230103788,
+      "learning_rate": 8.728791668572168e-05,
+      "loss": 0.581,
+      "step": 3650
+    },
+    {
+      "epoch": 9.539155655816952,
+      "grad_norm": 1.4682987033608552,
+      "learning_rate": 8.694885528111918e-05,
+      "loss": 0.6167,
+      "step": 3700
+    },
+    {
+      "epoch": 9.539155655816952,
+      "eval_loss": 2.702000379562378,
+      "eval_runtime": 13.1618,
+      "eval_samples_per_second": 75.978,
+      "eval_steps_per_second": 2.431,
+      "step": 3700
+    },
+    {
+      "epoch": 9.668063164679342,
+      "grad_norm": 1.3807999009672582,
+      "learning_rate": 8.66060165194087e-05,
+      "loss": 0.6364,
+      "step": 3750
+    },
+    {
+      "epoch": 9.796970673541734,
+      "grad_norm": 1.6910628067329174,
+      "learning_rate": 8.625943593049708e-05,
+      "loss": 0.664,
+      "step": 3800
+    },
+    {
+      "epoch": 9.796970673541734,
+      "eval_loss": 2.6746528148651123,
+      "eval_runtime": 13.1207,
+      "eval_samples_per_second": 76.216,
+      "eval_steps_per_second": 2.439,
+      "step": 3800
+    },
+    {
+      "epoch": 9.925878182404125,
+      "grad_norm": 1.2810221531255845,
+      "learning_rate": 8.590914943207338e-05,
+      "loss": 0.6808,
+      "step": 3850
+    },
+    {
+      "epoch": 10.054785691266517,
+      "grad_norm": 0.894817953292936,
+      "learning_rate": 8.555519332588658e-05,
+      "loss": 0.5501,
+      "step": 3900
+    },
+    {
+      "epoch": 10.054785691266517,
+      "eval_loss": 2.9522013664245605,
+      "eval_runtime": 13.1696,
+      "eval_samples_per_second": 75.933,
+      "eval_steps_per_second": 2.43,
+      "step": 3900
+    },
+    {
+      "epoch": 10.183693200128907,
+      "grad_norm": 0.7673858183230434,
+      "learning_rate": 8.519760429398344e-05,
+      "loss": 0.3676,
+      "step": 3950
+    },
+    {
+      "epoch": 10.3126007089913,
+      "grad_norm": 0.9258575274469117,
+      "learning_rate": 8.4836419394907e-05,
+      "loss": 0.3948,
+      "step": 4000
+    },
+    {
+      "epoch": 10.3126007089913,
+      "eval_loss": 2.939727544784546,
+      "eval_runtime": 13.261,
+      "eval_samples_per_second": 75.409,
+      "eval_steps_per_second": 2.413,
+      "step": 4000
+    },
+    {
+      "epoch": 10.44150821785369,
+      "grad_norm": 0.9837761849309598,
+      "learning_rate": 8.447167605985595e-05,
+      "loss": 0.424,
+      "step": 4050
+    },
+    {
+      "epoch": 10.570415726716082,
+      "grad_norm": 1.1827846876344605,
+      "learning_rate": 8.410341208880562e-05,
+      "loss": 0.4493,
+      "step": 4100
+    },
+    {
+      "epoch": 10.570415726716082,
+      "eval_loss": 2.9346396923065186,
+      "eval_runtime": 13.1851,
+      "eval_samples_per_second": 75.843,
+      "eval_steps_per_second": 2.427,
+      "step": 4100
+    },
+    {
+      "epoch": 10.699323235578472,
+      "grad_norm": 1.41364563812005,
+      "learning_rate": 8.373166564659048e-05,
+      "loss": 0.4724,
+      "step": 4150
+    },
+    {
+      "epoch": 10.828230744440864,
+      "grad_norm": 1.341861709508504,
+      "learning_rate": 8.335647525894898e-05,
+      "loss": 0.4991,
+      "step": 4200
+    },
+    {
+      "epoch": 10.828230744440864,
+      "eval_loss": 2.8969509601593018,
+      "eval_runtime": 13.153,
+      "eval_samples_per_second": 76.028,
+      "eval_steps_per_second": 2.433,
+      "step": 4200
+    },
+    {
+      "epoch": 10.957138253303254,
+      "grad_norm": 0.8732212524533925,
+      "learning_rate": 8.297787980853102e-05,
+      "loss": 0.514,
+      "step": 4250
+    },
+    {
+      "epoch": 11.086045762165647,
+      "grad_norm": 0.9580077889491428,
+      "learning_rate": 8.259591853086822e-05,
+      "loss": 0.3342,
+      "step": 4300
+    },
+    {
+      "epoch": 11.086045762165647,
+      "eval_loss": 3.12947940826416,
+      "eval_runtime": 13.164,
+      "eval_samples_per_second": 75.964,
+      "eval_steps_per_second": 2.431,
+      "step": 4300
+    },
+    {
+      "epoch": 11.214953271028037,
+      "grad_norm": 1.0158413995480675,
+      "learning_rate": 8.221063101030793e-05,
+      "loss": 0.2608,
+      "step": 4350
+    },
+    {
+      "epoch": 11.343860779890429,
+      "grad_norm": 1.066799301262293,
+      "learning_rate": 8.182205717591083e-05,
+      "loss": 0.2868,
+      "step": 4400
+    },
+    {
+      "epoch": 11.343860779890429,
+      "eval_loss": 3.152151107788086,
+      "eval_runtime": 13.1308,
+      "eval_samples_per_second": 76.157,
+      "eval_steps_per_second": 2.437,
+      "step": 4400
+    },
+    {
+      "epoch": 11.47276828875282,
+      "grad_norm": 1.7853526972041276,
+      "learning_rate": 8.143023729731294e-05,
+      "loss": 0.3088,
+      "step": 4450
+    },
+    {
+      "epoch": 11.601675797615211,
+      "grad_norm": 0.8686004168533566,
+      "learning_rate": 8.103521198055229e-05,
+      "loss": 0.3307,
+      "step": 4500
+    },
+    {
+      "epoch": 11.601675797615211,
+      "eval_loss": 3.140380382537842,
+      "eval_runtime": 13.1601,
+      "eval_samples_per_second": 75.987,
+      "eval_steps_per_second": 2.432,
+      "step": 4500
+    },
+    {
+      "epoch": 11.730583306477602,
+      "grad_norm": 1.1842485940651084,
+      "learning_rate": 8.063702216386072e-05,
+      "loss": 0.3505,
+      "step": 4550
+    },
+    {
+      "epoch": 11.859490815339994,
+      "grad_norm": 1.139988677136263,
+      "learning_rate": 8.02357091134213e-05,
+      "loss": 0.3798,
+      "step": 4600
+    },
+    {
+      "epoch": 11.859490815339994,
+      "eval_loss": 3.1152422428131104,
+      "eval_runtime": 13.1658,
+      "eval_samples_per_second": 75.954,
+      "eval_steps_per_second": 2.431,
+      "step": 4600
+    },
+    {
+      "epoch": 11.988398324202384,
+      "grad_norm": 1.1317453381475029,
+      "learning_rate": 7.983131441909169e-05,
+      "loss": 0.383,
+      "step": 4650
+    },
+    {
+      "epoch": 12.117305833064776,
+      "grad_norm": 0.8577623285954574,
+      "learning_rate": 7.942387999009405e-05,
+      "loss": 0.1955,
+      "step": 4700
+    },
+    {
+      "epoch": 12.117305833064776,
+      "eval_loss": 3.292053699493408,
+      "eval_runtime": 13.1656,
+      "eval_samples_per_second": 75.955,
+      "eval_steps_per_second": 2.431,
+      "step": 4700
+    },
+    {
+      "epoch": 12.246213341927167,
+      "grad_norm": 0.792139936919041,
+      "learning_rate": 7.901344805067176e-05,
+      "loss": 0.2018,
+      "step": 4750
+    },
+    {
+      "epoch": 12.375120850789559,
+      "grad_norm": 1.1197587931587643,
+      "learning_rate": 7.860006113571356e-05,
+      "loss": 0.2149,
+      "step": 4800
+    },
+    {
+      "epoch": 12.375120850789559,
+      "eval_loss": 3.300734519958496,
+      "eval_runtime": 13.2004,
+      "eval_samples_per_second": 75.755,
+      "eval_steps_per_second": 2.424,
+      "step": 4800
+    },
+    {
+      "epoch": 12.504028359651949,
+      "grad_norm": 0.9715029062397967,
+      "learning_rate": 7.818376208634544e-05,
+      "loss": 0.2326,
+      "step": 4850
+    },
+    {
+      "epoch": 12.632935868514341,
+      "grad_norm": 0.9363158756959897,
+      "learning_rate": 7.776459404549084e-05,
+      "loss": 0.2492,
+      "step": 4900
+    },
+    {
+      "epoch": 12.632935868514341,
+      "eval_loss": 3.3016297817230225,
+      "eval_runtime": 13.1894,
+      "eval_samples_per_second": 75.819,
+      "eval_steps_per_second": 2.426,
+      "step": 4900
+    },
+    {
+      "epoch": 12.761843377376731,
+      "grad_norm": 0.8911732251342259,
+      "learning_rate": 7.734260045339957e-05,
+      "loss": 0.2662,
+      "step": 4950
+    },
+    {
+      "epoch": 12.890750886239124,
+      "grad_norm": 1.0150499184539068,
+      "learning_rate": 7.69178250431459e-05,
+      "loss": 0.2797,
+      "step": 5000
+    },
+    {
+      "epoch": 12.890750886239124,
+      "eval_loss": 3.2908613681793213,
+      "eval_runtime": 13.193,
+      "eval_samples_per_second": 75.798,
+      "eval_steps_per_second": 2.426,
+      "step": 5000
+    },
+    {
+      "epoch": 13.019658395101514,
+      "grad_norm": 0.9560026596743003,
+      "learning_rate": 7.649031183609627e-05,
+      "loss": 0.27,
+      "step": 5050
+    },
+    {
+      "epoch": 13.148565903963906,
+      "grad_norm": 0.7460724556430167,
+      "learning_rate": 7.606010513734722e-05,
+      "loss": 0.1396,
+      "step": 5100
+    },
+    {
+      "epoch": 13.148565903963906,
+      "eval_loss": 3.4375460147857666,
+      "eval_runtime": 13.1426,
+      "eval_samples_per_second": 76.088,
+      "eval_steps_per_second": 2.435,
+      "step": 5100
+    },
+    {
+      "epoch": 13.277473412826296,
+      "grad_norm": 1.2891823937464943,
+      "learning_rate": 7.562724953113382e-05,
+      "loss": 0.1548,
+      "step": 5150
+    },
+    {
+      "epoch": 13.406380921688688,
+      "grad_norm": 0.8052288266291375,
+      "learning_rate": 7.519178987620915e-05,
+      "loss": 0.1697,
+      "step": 5200
+    },
+    {
+      "epoch": 13.406380921688688,
+      "eval_loss": 3.4272212982177734,
+      "eval_runtime": 13.1636,
+      "eval_samples_per_second": 75.967,
+      "eval_steps_per_second": 2.431,
+      "step": 5200
+    },
+    {
+      "epoch": 13.535288430551079,
+      "grad_norm": 0.7626827120967569,
+      "learning_rate": 7.475377130119553e-05,
+      "loss": 0.1836,
+      "step": 5250
+    },
+    {
+      "epoch": 13.66419593941347,
+      "grad_norm": 0.7817292246891677,
+      "learning_rate": 7.431323919990754e-05,
+      "loss": 0.1948,
+      "step": 5300
+    },
+    {
+      "epoch": 13.66419593941347,
+      "eval_loss": 3.4388325214385986,
+      "eval_runtime": 13.1888,
+      "eval_samples_per_second": 75.822,
+      "eval_steps_per_second": 2.426,
+      "step": 5300
+    },
+    {
+      "epoch": 13.793103448275861,
+      "grad_norm": 0.7362472448079689,
+      "learning_rate": 7.387023922664762e-05,
+      "loss": 0.2081,
+      "step": 5350
+    },
+    {
+      "epoch": 13.922010957138253,
+      "grad_norm": 0.7773529994490974,
+      "learning_rate": 7.342481729147485e-05,
+      "loss": 0.2165,
+      "step": 5400
+    },
+    {
+      "epoch": 13.922010957138253,
+      "eval_loss": 3.435514450073242,
+      "eval_runtime": 13.2029,
+      "eval_samples_per_second": 75.741,
+      "eval_steps_per_second": 2.424,
+      "step": 5400
+    },
+    {
+      "epoch": 14.050918466000645,
+      "grad_norm": 1.3925624599380586,
+      "learning_rate": 7.297701955544692e-05,
+      "loss": 0.1793,
+      "step": 5450
+    },
+    {
+      "epoch": 14.179825974863036,
+      "grad_norm": 0.6045993909446667,
+      "learning_rate": 7.252689242583643e-05,
+      "loss": 0.1302,
+      "step": 5500
+    },
+    {
+      "epoch": 14.179825974863036,
+      "eval_loss": 3.5325989723205566,
+      "eval_runtime": 13.2171,
+      "eval_samples_per_second": 75.659,
+      "eval_steps_per_second": 2.421,
+      "step": 5500
+    },
+    {
+      "epoch": 14.308733483725428,
+      "grad_norm": 0.6099737603899711,
+      "learning_rate": 7.20744825513213e-05,
+      "loss": 0.1265,
+      "step": 5550
+    },
+    {
+      "epoch": 14.437640992587818,
+      "grad_norm": 0.6596680228597308,
+      "learning_rate": 7.161983681715056e-05,
+      "loss": 0.138,
+      "step": 5600
+    },
+    {
+      "epoch": 14.437640992587818,
+      "eval_loss": 3.557699680328369,
+      "eval_runtime": 13.1774,
+      "eval_samples_per_second": 75.888,
+      "eval_steps_per_second": 2.428,
+      "step": 5600
+    },
+    {
+      "epoch": 14.56654850145021,
+      "grad_norm": 0.6445903882034174,
+      "learning_rate": 7.116300234028528e-05,
+      "loss": 0.1486,
+      "step": 5650
+    },
+    {
+      "epoch": 14.6954560103126,
+      "grad_norm": 0.6448511072922888,
+      "learning_rate": 7.070402646451566e-05,
+      "loss": 0.1569,
+      "step": 5700
+    },
+    {
+      "epoch": 14.6954560103126,
+      "eval_loss": 3.5597221851348877,
+      "eval_runtime": 13.1454,
+      "eval_samples_per_second": 76.072,
+      "eval_steps_per_second": 2.434,
+      "step": 5700
+    },
+    {
+      "epoch": 14.824363519174993,
+      "grad_norm": 0.623499885141096,
+      "learning_rate": 7.024295675555465e-05,
+      "loss": 0.1675,
+      "step": 5750
+    },
+    {
+      "epoch": 14.953271028037383,
+      "grad_norm": 0.6601246140904097,
+      "learning_rate": 6.977984099610842e-05,
+      "loss": 0.1757,
+      "step": 5800
+    },
+    {
+      "epoch": 14.953271028037383,
+      "eval_loss": 3.555185556411743,
+      "eval_runtime": 13.1511,
+      "eval_samples_per_second": 76.039,
+      "eval_steps_per_second": 2.433,
+      "step": 5800
+    },
+    {
+      "epoch": 15.082178536899775,
+      "grad_norm": 0.5535844816659106,
+      "learning_rate": 6.931472718092446e-05,
+      "loss": 0.1223,
+      "step": 5850
+    },
+    {
+      "epoch": 15.211086045762165,
+      "grad_norm": 0.4901789372000881,
+      "learning_rate": 6.88476635118177e-05,
+      "loss": 0.0964,
+      "step": 5900
+    },
+    {
+      "epoch": 15.211086045762165,
+      "eval_loss": 3.6443583965301514,
+      "eval_runtime": 13.1441,
+      "eval_samples_per_second": 76.08,
+      "eval_steps_per_second": 2.435,
+      "step": 5900
+    },
+    {
+      "epoch": 15.339993554624558,
+      "grad_norm": 0.8805753683037839,
+      "learning_rate": 6.837869839267507e-05,
+      "loss": 0.107,
+      "step": 5950
+    },
+    {
+      "epoch": 15.468901063486948,
+      "grad_norm": 0.6115833032151868,
+      "learning_rate": 6.790788042443924e-05,
+      "loss": 0.1157,
+      "step": 6000
+    },
+    {
+      "epoch": 15.468901063486948,
+      "eval_loss": 3.6541991233825684,
+      "eval_runtime": 13.1423,
+      "eval_samples_per_second": 76.09,
+      "eval_steps_per_second": 2.435,
+      "step": 6000
+    },
+    {
+      "epoch": 15.59780857234934,
+      "grad_norm": 0.5710489600928594,
+      "learning_rate": 6.743525840007191e-05,
+      "loss": 0.1228,
+      "step": 6050
+    },
+    {
+      "epoch": 15.72671608121173,
+      "grad_norm": 0.689652930712509,
+      "learning_rate": 6.69608812994971e-05,
+      "loss": 0.1293,
+      "step": 6100
+    },
+    {
+      "epoch": 15.72671608121173,
+      "eval_loss": 3.656620979309082,
+      "eval_runtime": 13.1483,
+      "eval_samples_per_second": 76.055,
+      "eval_steps_per_second": 2.434,
+      "step": 6100
+    },
+    {
+      "epoch": 15.855623590074122,
+      "grad_norm": 0.5914349974251379,
+      "learning_rate": 6.648479828452522e-05,
+      "loss": 0.135,
+      "step": 6150
+    },
+    {
+      "epoch": 15.984531098936513,
+      "grad_norm": 0.6189439108834661,
+      "learning_rate": 6.600705869375817e-05,
+      "loss": 0.1419,
+      "step": 6200
+    },
+    {
+      "epoch": 15.984531098936513,
+      "eval_loss": 3.671614170074463,
+      "eval_runtime": 13.1443,
+      "eval_samples_per_second": 76.078,
+      "eval_steps_per_second": 2.435,
+      "step": 6200
+    },
+    {
+      "epoch": 16.113438607798905,
+      "grad_norm": 0.4321088458532425,
+      "learning_rate": 6.55277120374762e-05,
+      "loss": 0.0842,
+      "step": 6250
+    },
+    {
+      "epoch": 16.242346116661295,
+      "grad_norm": 0.5785562511316525,
+      "learning_rate": 6.504680799250689e-05,
+      "loss": 0.0872,
+      "step": 6300
+    },
+    {
+      "epoch": 16.242346116661295,
+      "eval_loss": 3.7362146377563477,
+      "eval_runtime": 13.1708,
+      "eval_samples_per_second": 75.925,
+      "eval_steps_per_second": 2.43,
+      "step": 6300
+    },
+    {
+      "epoch": 16.371253625523686,
+      "grad_norm": 0.45371653083862784,
+      "learning_rate": 6.45643963970769e-05,
+      "loss": 0.0899,
+      "step": 6350
+    },
+    {
+      "epoch": 16.50016113438608,
+      "grad_norm": 0.5412656335319528,
+      "learning_rate": 6.408052724564714e-05,
+      "loss": 0.0954,
+      "step": 6400
+    },
+    {
+      "epoch": 16.50016113438608,
+      "eval_loss": 3.7513670921325684,
+      "eval_runtime": 13.1547,
+      "eval_samples_per_second": 76.018,
+      "eval_steps_per_second": 2.433,
+      "step": 6400
+    },
+    {
+      "epoch": 16.62906864324847,
+      "grad_norm": 0.6489716287488846,
+      "learning_rate": 6.359525068373147e-05,
+      "loss": 0.1021,
+      "step": 6450
+    },
+    {
+      "epoch": 16.75797615211086,
+      "grad_norm": 0.51515711941371,
+      "learning_rate": 6.310861700270001e-05,
+      "loss": 0.1074,
+      "step": 6500
+    },
+    {
+      "epoch": 16.75797615211086,
+      "eval_loss": 3.7568299770355225,
+      "eval_runtime": 13.1646,
+      "eval_samples_per_second": 75.961,
+      "eval_steps_per_second": 2.431,
+      "step": 6500
+    },
+    {
+      "epoch": 16.88688366097325,
+      "grad_norm": 0.5784472468074615,
+      "learning_rate": 6.262067663456714e-05,
+      "loss": 0.1119,
+      "step": 6550
+    },
+    {
+      "epoch": 17.015791169835644,
+      "grad_norm": 0.36703765035846453,
+      "learning_rate": 6.2131480146765e-05,
+      "loss": 0.1089,
+      "step": 6600
+    },
+    {
+      "epoch": 17.015791169835644,
+      "eval_loss": 3.8403706550598145,
+      "eval_runtime": 13.14,
+      "eval_samples_per_second": 76.103,
+      "eval_steps_per_second": 2.435,
+      "step": 6600
+    },
+    {
+      "epoch": 17.144698678698035,
+      "grad_norm": 0.473234963059124,
+      "learning_rate": 6.164107823690311e-05,
+      "loss": 0.0653,
+      "step": 6650
+    },
+    {
+      "epoch": 17.273606187560425,
+      "grad_norm": 0.4495142121889821,
+      "learning_rate": 6.11495217275142e-05,
+      "loss": 0.071,
+      "step": 6700
+    },
+    {
+      "epoch": 17.273606187560425,
+      "eval_loss": 3.830505609512329,
+      "eval_runtime": 13.1641,
+      "eval_samples_per_second": 75.964,
+      "eval_steps_per_second": 2.431,
+      "step": 6700
+    },
+    {
+      "epoch": 17.402513696422815,
+      "grad_norm": 0.47554094680025083,
+      "learning_rate": 6.065686156078723e-05,
+      "loss": 0.0758,
+      "step": 6750
+    },
+    {
+      "epoch": 17.53142120528521,
+      "grad_norm": 0.44446520713729815,
+      "learning_rate": 6.0163148793288135e-05,
+      "loss": 0.0803,
+      "step": 6800
+    },
+    {
+      "epoch": 17.53142120528521,
+      "eval_loss": 3.8333582878112793,
+      "eval_runtime": 13.3574,
+      "eval_samples_per_second": 74.865,
+      "eval_steps_per_second": 2.396,
+      "step": 6800
+    },
+    {
+      "epoch": 17.6603287141476,
+      "grad_norm": 0.5076778674223996,
+      "learning_rate": 5.966843459066858e-05,
+      "loss": 0.0847,
+      "step": 6850
+    },
+    {
+      "epoch": 17.78923622300999,
+      "grad_norm": 0.49074606040336083,
+      "learning_rate": 5.917277022236333e-05,
+      "loss": 0.0882,
+      "step": 6900
+    },
+    {
+      "epoch": 17.78923622300999,
+      "eval_loss": 3.840571403503418,
+      "eval_runtime": 13.1466,
+      "eval_samples_per_second": 76.065,
+      "eval_steps_per_second": 2.434,
+      "step": 6900
+    },
+    {
+      "epoch": 17.91814373187238,
+      "grad_norm": 0.5550690055495523,
+      "learning_rate": 5.86762070562771e-05,
+      "loss": 0.0927,
+      "step": 6950
+    },
+    {
+      "epoch": 18.047051240734774,
+      "grad_norm": 0.37606709531546206,
+      "learning_rate": 5.817879655346103e-05,
+      "loss": 0.0805,
+      "step": 7000
+    },
+    {
+      "epoch": 18.047051240734774,
+      "eval_loss": 3.8941986560821533,
+      "eval_runtime": 13.1572,
+      "eval_samples_per_second": 76.004,
+      "eval_steps_per_second": 2.432,
+      "step": 7000
+    },
+    {
+      "epoch": 18.175958749597164,
+      "grad_norm": 0.48969762050004223,
+      "learning_rate": 5.7680590262779535e-05,
+      "loss": 0.0561,
+      "step": 7050
+    },
+    {
+      "epoch": 18.304866258459555,
+      "grad_norm": 0.43070776806915745,
+      "learning_rate": 5.718163981556802e-05,
+      "loss": 0.061,
+      "step": 7100
+    },
+    {
+      "epoch": 18.304866258459555,
+      "eval_loss": 3.9117181301116943,
+      "eval_runtime": 13.1597,
+      "eval_samples_per_second": 75.99,
+      "eval_steps_per_second": 2.432,
+      "step": 7100
+    },
+    {
+      "epoch": 18.433773767321945,
+      "grad_norm": 0.42717221620170337,
+      "learning_rate": 5.6681996920282174e-05,
+      "loss": 0.0649,
+      "step": 7150
+    },
+    {
+      "epoch": 18.56268127618434,
+      "grad_norm": 0.5194058540782253,
+      "learning_rate": 5.6181713357139184e-05,
+      "loss": 0.0682,
+      "step": 7200
+    },
+    {
+      "epoch": 18.56268127618434,
+      "eval_loss": 3.901904821395874,
+      "eval_runtime": 13.1719,
+      "eval_samples_per_second": 75.919,
+      "eval_steps_per_second": 2.429,
+      "step": 7200
+    },
+    {
+      "epoch": 18.69158878504673,
+      "grad_norm": 0.37203814221096937,
+      "learning_rate": 5.568084097275149e-05,
+      "loss": 0.0719,
+      "step": 7250
+    },
+    {
+      "epoch": 18.82049629390912,
+      "grad_norm": 0.4355364201650638,
+      "learning_rate": 5.517943167475367e-05,
+      "loss": 0.0736,
+      "step": 7300
+    },
+    {
+      "epoch": 18.82049629390912,
+      "eval_loss": 3.927724599838257,
+      "eval_runtime": 13.1773,
+      "eval_samples_per_second": 75.888,
+      "eval_steps_per_second": 2.428,
+      "step": 7300
+    },
+    {
+      "epoch": 18.94940380277151,
+      "grad_norm": 0.470475938907921,
+      "learning_rate": 5.4677537426423044e-05,
+      "loss": 0.0765,
+      "step": 7350
+    },
+    {
+      "epoch": 19.078311311633904,
+      "grad_norm": 0.41878300792660983,
+      "learning_rate": 5.4175210241294564e-05,
+      "loss": 0.0584,
+      "step": 7400
+    },
+    {
+      "epoch": 19.078311311633904,
+      "eval_loss": 3.983382225036621,
+      "eval_runtime": 13.1436,
+      "eval_samples_per_second": 76.082,
+      "eval_steps_per_second": 2.435,
+      "step": 7400
+    },
+    {
+      "epoch": 19.207218820496294,
+      "grad_norm": 0.363679410249128,
+      "learning_rate": 5.367250217777029e-05,
+      "loss": 0.0488,
+      "step": 7450
+    },
+    {
+      "epoch": 19.336126329358684,
+      "grad_norm": 0.38290054329422646,
+      "learning_rate": 5.316946533372439e-05,
+      "loss": 0.0518,
+      "step": 7500
+    },
+    {
+      "epoch": 19.336126329358684,
+      "eval_loss": 3.965801954269409,
+      "eval_runtime": 13.1515,
+      "eval_samples_per_second": 76.037,
+      "eval_steps_per_second": 2.433,
+      "step": 7500
+    },
+    {
+      "epoch": 19.465033838221075,
+      "grad_norm": 0.3547993700029784,
+      "learning_rate": 5.266615184110405e-05,
+      "loss": 0.0546,
+      "step": 7550
+    },
+    {
+      "epoch": 19.59394134708347,
+      "grad_norm": 0.37584304866780927,
+      "learning_rate": 5.216261386052682e-05,
+      "loss": 0.0565,
+      "step": 7600
+    },
+    {
+      "epoch": 19.59394134708347,
+      "eval_loss": 3.980562686920166,
+      "eval_runtime": 13.1525,
+      "eval_samples_per_second": 76.031,
+      "eval_steps_per_second": 2.433,
+      "step": 7600
+    },
+    {
+      "epoch": 19.72284885594586,
+      "grad_norm": 0.38295332236701923,
+      "learning_rate": 5.165890357587486e-05,
+      "loss": 0.059,
+      "step": 7650
+    },
+    {
+      "epoch": 19.85175636480825,
+      "grad_norm": 0.41601237103360944,
+      "learning_rate": 5.115507318888699e-05,
+      "loss": 0.0608,
+      "step": 7700
+    },
+    {
+      "epoch": 19.85175636480825,
+      "eval_loss": 3.9985480308532715,
+      "eval_runtime": 13.1495,
+      "eval_samples_per_second": 76.049,
+      "eval_steps_per_second": 2.434,
+      "step": 7700
+    },
+    {
+      "epoch": 19.98066387367064,
+      "grad_norm": 0.40935703530908246,
+      "learning_rate": 5.0651174913748836e-05,
+      "loss": 0.0631,
+      "step": 7750
+    },
+    {
+      "epoch": 20.109571382533034,
+      "grad_norm": 0.3115920333441882,
+      "learning_rate": 5.0147260971681466e-05,
+      "loss": 0.0434,
+      "step": 7800
+    },
+    {
+      "epoch": 20.109571382533034,
+      "eval_loss": 4.044510364532471,
+      "eval_runtime": 13.1761,
+      "eval_samples_per_second": 75.895,
+      "eval_steps_per_second": 2.429,
+      "step": 7800
+    },
+    {
+      "epoch": 20.238478891395424,
+      "grad_norm": 0.33928266884719477,
+      "learning_rate": 4.9643383585529644e-05,
+      "loss": 0.0417,
+      "step": 7850
+    },
+    {
+      "epoch": 20.367386400257814,
+      "grad_norm": 0.366492032031904,
+      "learning_rate": 4.9139594974349714e-05,
+      "loss": 0.0469,
+      "step": 7900
+    },
+    {
+      "epoch": 20.367386400257814,
+      "eval_loss": 4.039082050323486,
+      "eval_runtime": 13.1597,
+      "eval_samples_per_second": 75.989,
+      "eval_steps_per_second": 2.432,
+      "step": 7900
+    },
+    {
+      "epoch": 20.496293909120205,
+      "grad_norm": 0.314671889688573,
+      "learning_rate": 4.863594734799778e-05,
+      "loss": 0.0458,
+      "step": 7950
+    },
+    {
+      "epoch": 20.6252014179826,
+      "grad_norm": 0.34654066388282007,
+      "learning_rate": 4.813249290171915e-05,
+      "loss": 0.0477,
+      "step": 8000
+    },
+    {
+      "epoch": 20.6252014179826,
+      "eval_loss": 4.047826766967773,
+      "eval_runtime": 13.1879,
+      "eval_samples_per_second": 75.827,
+      "eval_steps_per_second": 2.426,
+      "step": 8000
+    }
+  ],
+  "logging_steps": 50,
+  "max_steps": 15480,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 40,
+  "save_steps": 800,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": true,
+        "should_training_stop": false
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 7114179689316352.0,
+  "train_batch_size": 4,
+  "trial_name": null,
+  "trial_params": null
+}

training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0e13c70838ad2c25d6983c3deba8ea3c45b056794b4bea54c22b44afc488b2a5
+size 7224