End of training

README.md

@@ -38,7 +38,7 @@ This model was trained with GRPO, a method introduced in [DeepSeekMath: Pushing
 ### Framework versions
 
 - TRL: 0.15.2
-- Transformers: 4.50.3
+- Transformers: 4.51.2
 - Pytorch: 2.5.1
 - Datasets: 3.5.0
 - Tokenizers: 0.21.1

all_results.json

@@ -1,8 +1,8 @@
 {
     "total_flos": 0.0,
-    "train_loss": 6530814950.684069,
-    "train_runtime": 138.6461,
-    "train_samples": 160,
-    "train_samples_per_second": 2.308,
-    "train_steps_per_second": 0.144
+    "train_loss": 2.5933608412742614e-06,
+    "train_runtime": 409.99,
+    "train_samples": 149,
+    "train_samples_per_second": 0.781,
+    "train_steps_per_second": 0.049
 }

config.json

@@ -21,7 +21,7 @@
   "sliding_window": 32768,
   "tie_word_embeddings": true,
   "torch_dtype": "float32",
-  "transformers_version": "4.50.3",
+  "transformers_version": "4.51.2",
   "use_cache": true,
   "use_sliding_window": false,
   "vocab_size": 151936

generation_config.json

@@ -10,5 +10,5 @@
   "temperature": 0.7,
   "top_k": 20,
   "top_p": 0.8,
-  "transformers_version": "4.50.3"
+  "transformers_version": "4.51.2"
 }

model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:72fa377b3fef8c5db7593e253a73f2ad095a29d46d09baf496d24eab3d1c01aa
+oid sha256:67210027f7d97ef16436eb3f12622cf8a089458f85c41225b999b895c67f6eaf
 size 1976163472

train_results.json

@@ -1,8 +1,8 @@
 {
     "total_flos": 0.0,
-    "train_loss": 6530814950.684069,
-    "train_runtime": 138.6461,
-    "train_samples": 160,
-    "train_samples_per_second": 2.308,
-    "train_steps_per_second": 0.144
+    "train_loss": 2.5933608412742614e-06,
+    "train_runtime": 409.99,
+    "train_samples": 149,
+    "train_samples_per_second": 0.781,
+    "train_steps_per_second": 0.049
 }

trainer_state.json

@@ -2,7 +2,7 @@
   "best_global_step": null,
   "best_metric": null,
   "best_model_checkpoint": null,
-  "epoch": 1.0,
+  "epoch": 1.0536912751677852,
   "eval_steps": 500,
   "global_step": 20,
   "is_hyper_param_search": false,
@@ -10,209 +10,209 @@
   "is_world_process_zero": true,
   "log_history": [
     {
-      "completion_length": 292.375,
-      "epoch": 0.1,
-      "grad_norm": 36.89990997314453,
+      "completion_length": 316.75,
+      "epoch": 0.10738255033557047,
+      "grad_norm": 8.684107780456543,
       "kl": 0.0,
-      "learning_rate": 4.965903258506806e-07,
-      "loss": -0.0,
-      "reward": 3.7902724987943657,
-      "reward_std": 0.6387863923737314,
-      "rewards/concensus_correctness_reward_func": 1.0663749910891056,
-      "rewards/consensus_reward_func": 1.0625,
+      "learning_rate": 5e-07,
+      "loss": 0.0,
+      "reward": 0.9118603632086888,
+      "reward_std": 0.7637306145916227,
+      "rewards/concensus_correctness_reward_func": 0.0,
+      "rewards/consensus_reward_func": 0.0625,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.0,
-      "rewards/question_recreation_reward_func": 0.6968037628103048,
+      "rewards/final_correctness_reward_func": 0.125,
+      "rewards/question_recreation_reward_func": 0.39504786173347384,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.234375,
-      "rewards/xmlcount_reward_func": 0.7302187555469573,
+      "rewards/strict_format_reward_func": 0.0,
+      "rewards/xmlcount_reward_func": 0.3293124958872795,
       "step": 2
     },
     {
-      "completion_length": 137.1875,
-      "epoch": 0.2,
-      "grad_norm": 920617.0625,
-      "kl": 236312.0710613858,
-      "learning_rate": 4.698684378016222e-07,
-      "loss": 236.3121,
-      "reward": 6.642892271280289,
-      "reward_std": 0.191781481509679,
-      "rewards/concensus_correctness_reward_func": 1.9206249862909317,
-      "rewards/consensus_reward_func": 2.0,
+      "completion_length": 348.3125,
+      "epoch": 0.21476510067114093,
+      "grad_norm": 13.454246520996094,
+      "kl": 0.0020339643051556777,
+      "learning_rate": 4.864543104251586e-07,
+      "loss": 0.0,
+      "reward": 1.0148029099218547,
+      "reward_std": 0.5975782387249637,
+      "rewards/concensus_correctness_reward_func": 0.11924999952316284,
+      "rewards/consensus_reward_func": 0.0625,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.125,
-      "rewards/question_recreation_reward_func": 0.915736073628068,
+      "rewards/final_correctness_reward_func": 0.0625,
+      "rewards/question_recreation_reward_func": 0.3142403995152563,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.453125,
-      "rewards/xmlcount_reward_func": 1.2284062504768372,
+      "rewards/strict_format_reward_func": 0.015625,
+      "rewards/xmlcount_reward_func": 0.44068749947473407,
       "step": 4
     },
     {
-      "completion_length": 157.6875,
-      "epoch": 0.3,
-      "grad_norm": 496597.5625,
-      "kl": 45069158.72720645,
-      "learning_rate": 4.193203929064353e-07,
-      "loss": 45069.1562,
-      "reward": 6.162765100598335,
-      "reward_std": 0.6365469432785176,
-      "rewards/concensus_correctness_reward_func": 1.7088124994188547,
-      "rewards/consensus_reward_func": 1.875,
+      "completion_length": 452.65625,
+      "epoch": 0.3221476510067114,
+      "grad_norm": 11.709402084350586,
+      "kl": 0.0012080057713319547,
+      "learning_rate": 4.472851273490984e-07,
+      "loss": 0.0,
+      "reward": 0.6762643828988075,
+      "reward_std": 0.6245548717561178,
+      "rewards/concensus_correctness_reward_func": 0.0598750002682209,
+      "rewards/consensus_reward_func": 0.0625,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.125,
-      "rewards/question_recreation_reward_func": 0.8784525990486145,
-      "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.375,
-      "rewards/xmlcount_reward_func": 1.2005000114440918,
+      "rewards/final_correctness_reward_func": 0.0625,
+      "rewards/question_recreation_reward_func": 0.2978268840815872,
+      "rewards/soft_format_reward_func": 0.015625,
+      "rewards/strict_format_reward_func": 0.0,
+      "rewards/xmlcount_reward_func": 0.1779375011101365,
       "step": 6
     },
     {
-      "completion_length": 162.84375,
-      "epoch": 0.4,
-      "grad_norm": 1522.3465576171875,
-      "kl": 116190007.5042836,
-      "learning_rate": 3.5042385616324236e-07,
-      "loss": 116190.0078,
-      "reward": 6.963820606470108,
-      "reward_std": 0.46906069423857844,
-      "rewards/concensus_correctness_reward_func": 2.176312491297722,
-      "rewards/consensus_reward_func": 1.8125,
+      "completion_length": 398.625,
+      "epoch": 0.42953020134228187,
+      "grad_norm": 9.44884967803955,
+      "kl": 0.0016478088418807602,
+      "learning_rate": 3.867370395306068e-07,
+      "loss": 0.0,
+      "reward": 0.747544834157452,
+      "reward_std": 0.6983632346382365,
+      "rewards/concensus_correctness_reward_func": 0.1223749965429306,
+      "rewards/consensus_reward_func": 0.0,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.4375,
-      "rewards/question_recreation_reward_func": 0.8890705443918705,
+      "rewards/final_correctness_reward_func": 0.125,
+      "rewards/question_recreation_reward_func": 0.3253885810263455,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.40625,
-      "rewards/xmlcount_reward_func": 1.2421875,
+      "rewards/strict_format_reward_func": 0.015625,
+      "rewards/xmlcount_reward_func": 0.15915624890476465,
       "step": 8
     },
     {
-      "completion_length": 138.5625,
-      "epoch": 0.5,
-      "grad_norm": 3460.926513671875,
-      "kl": 65307980859346.81,
-      "learning_rate": 2.706448363680831e-07,
-      "loss": 65307987968.0,
-      "reward": 7.760716885328293,
-      "reward_std": 0.563693645875901,
-      "rewards/concensus_correctness_reward_func": 2.5240624882280827,
-      "rewards/consensus_reward_func": 1.875,
+      "completion_length": 417.53125,
+      "epoch": 0.5369127516778524,
+      "grad_norm": 4.120293140411377,
+      "kl": 0.0019710015985765494,
+      "learning_rate": 3.1137137178519977e-07,
+      "loss": 0.0,
+      "reward": 0.8447740260744467,
+      "reward_std": 0.8398003252586932,
+      "rewards/concensus_correctness_reward_func": 0.14162500202655792,
+      "rewards/consensus_reward_func": 0.0,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.8125,
-      "rewards/question_recreation_reward_func": 0.969498086720705,
+      "rewards/final_correctness_reward_func": 0.0625,
+      "rewards/question_recreation_reward_func": 0.2868052691919729,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.359375,
-      "rewards/xmlcount_reward_func": 1.220281258225441,
+      "rewards/strict_format_reward_func": 0.0,
+      "rewards/xmlcount_reward_func": 0.35384375182911754,
       "step": 10
     },
     {
-      "completion_length": 148.375,
-      "epoch": 0.6,
-      "grad_norm": 1270.5479736328125,
-      "kl": 37.43342441506684,
-      "learning_rate": 1.886286282148002e-07,
-      "loss": 0.0374,
-      "reward": 6.585143744945526,
-      "reward_std": 0.1574548316639266,
-      "rewards/concensus_correctness_reward_func": 1.949249990284443,
-      "rewards/consensus_reward_func": 2.0,
+      "completion_length": 360.71875,
+      "epoch": 0.6442953020134228,
+      "grad_norm": 32.15998840332031,
+      "kl": 0.00471232270137989,
+      "learning_rate": 2.2935516363191693e-07,
+      "loss": 0.0,
+      "reward": 1.9486160605447367,
+      "reward_std": 1.8115553769748658,
+      "rewards/concensus_correctness_reward_func": 0.6818749997764826,
+      "rewards/consensus_reward_func": 0.125,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.125,
-      "rewards/question_recreation_reward_func": 0.9060499593615532,
+      "rewards/final_correctness_reward_func": 0.375,
+      "rewards/question_recreation_reward_func": 0.4561785594560206,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.390625,
-      "rewards/xmlcount_reward_func": 1.214218758046627,
+      "rewards/strict_format_reward_func": 0.015625,
+      "rewards/xmlcount_reward_func": 0.2949374979361892,
       "step": 12
     },
     {
-      "completion_length": 132.0,
-      "epoch": 0.7,
-      "grad_norm": 3555985.0,
-      "kl": 43278.06591859786,
-      "learning_rate": 1.1326296046939333e-07,
-      "loss": 43.2781,
-      "reward": 6.449171662330627,
-      "reward_std": 0.5914860000048066,
-      "rewards/concensus_correctness_reward_func": 1.9531875140964985,
-      "rewards/consensus_reward_func": 1.6875,
+      "completion_length": 328.4375,
+      "epoch": 0.7516778523489933,
+      "grad_norm": 9.214224815368652,
+      "kl": 0.004445670823770342,
+      "learning_rate": 1.4957614383675767e-07,
+      "loss": 0.0,
+      "reward": 1.0464753145352006,
+      "reward_std": 1.1203724679071456,
+      "rewards/concensus_correctness_reward_func": 0.16743749752640724,
+      "rewards/consensus_reward_func": 0.125,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.25,
-      "rewards/question_recreation_reward_func": 0.9084841851145029,
+      "rewards/final_correctness_reward_func": 0.0625,
+      "rewards/question_recreation_reward_func": 0.35360031423624605,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.40625,
-      "rewards/xmlcount_reward_func": 1.243750013411045,
+      "rewards/strict_format_reward_func": 0.015625,
+      "rewards/xmlcount_reward_func": 0.32231250684708357,
       "step": 14
     },
     {
-      "completion_length": 156.0625,
-      "epoch": 0.8,
-      "grad_norm": 30.58717918395996,
-      "kl": 17.3426317863632,
-      "learning_rate": 5.271487265090163e-08,
-      "loss": 0.0173,
-      "reward": 7.0806964635849,
-      "reward_std": 0.2373842012602836,
-      "rewards/concensus_correctness_reward_func": 2.0433749929070473,
-      "rewards/consensus_reward_func": 2.0,
+      "completion_length": 451.0625,
+      "epoch": 0.8590604026845637,
+      "grad_norm": 90.91595458984375,
+      "kl": 0.003888222190653323,
+      "learning_rate": 8.067960709356478e-08,
+      "loss": 0.0,
+      "reward": 0.7943547907052562,
+      "reward_std": 0.5775234283792088,
+      "rewards/concensus_correctness_reward_func": 0.04131250083446503,
+      "rewards/consensus_reward_func": 0.0625,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.4375,
-      "rewards/question_recreation_reward_func": 0.9735402166843414,
+      "rewards/final_correctness_reward_func": 0.125,
+      "rewards/question_recreation_reward_func": 0.3766672945348546,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.390625,
-      "rewards/xmlcount_reward_func": 1.2356562539935112,
+      "rewards/strict_format_reward_func": 0.015625,
+      "rewards/xmlcount_reward_func": 0.17324999067932367,
       "step": 16
     },
     {
-      "completion_length": 132.3125,
-      "epoch": 0.9,
-      "grad_norm": 94.87210845947266,
-      "kl": 6.395477553829551,
-      "learning_rate": 1.3545689574841341e-08,
-      "loss": 0.0064,
-      "reward": 5.915431410074234,
-      "reward_std": 0.3693226521718316,
-      "rewards/concensus_correctness_reward_func": 1.6944999881088734,
-      "rewards/consensus_reward_func": 1.6875,
+      "completion_length": 334.59375,
+      "epoch": 0.9664429530201343,
+      "grad_norm": 90.40325164794922,
+      "kl": 0.0023139841941883788,
+      "learning_rate": 3.013156219837776e-08,
+      "loss": 0.0,
+      "reward": 0.9272007264662534,
+      "reward_std": 0.688656156140496,
+      "rewards/concensus_correctness_reward_func": 0.0,
+      "rewards/consensus_reward_func": 0.0,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.125,
-      "rewards/question_recreation_reward_func": 0.8809627071022987,
+      "rewards/final_correctness_reward_func": 0.375,
+      "rewards/question_recreation_reward_func": 0.2672007321380079,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.3125,
-      "rewards/xmlcount_reward_func": 1.2149687632918358,
+      "rewards/strict_format_reward_func": 0.015625,
+      "rewards/xmlcount_reward_func": 0.26937499875202775,
       "step": 18
     },
     {
-      "completion_length": 144.125,
-      "epoch": 1.0,
-      "grad_norm": 1659.4306640625,
-      "kl": 25.288832587655634,
-      "learning_rate": 0.0,
-      "loss": 0.0253,
-      "reward": 6.783304065465927,
-      "reward_std": 0.4975810807009111,
-      "rewards/concensus_correctness_reward_func": 1.9735000059008598,
-      "rewards/consensus_reward_func": 1.875,
+      "completion_length": 456.38461538461536,
+      "epoch": 1.0536912751677852,
+      "grad_norm": 4.295146465301514,
+      "kl": 0.00429495136352041,
+      "learning_rate": 3.4096741493194193e-09,
+      "loss": 0.0,
+      "reward": 1.0834939227654383,
+      "reward_std": 1.518596117886213,
+      "rewards/concensus_correctness_reward_func": 0.2748461594948402,
+      "rewards/consensus_reward_func": 0.23076923076923078,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.25,
-      "rewards/question_recreation_reward_func": 0.9823666401207447,
+      "rewards/final_correctness_reward_func": 0.0,
+      "rewards/question_recreation_reward_func": 0.2927246823763618,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.453125,
-      "rewards/xmlcount_reward_func": 1.2493125051259995,
+      "rewards/strict_format_reward_func": 0.0,
+      "rewards/xmlcount_reward_func": 0.2851538443221496,
       "step": 20
     },
     {
-      "epoch": 1.0,
+      "epoch": 1.0536912751677852,
       "step": 20,
       "total_flos": 0.0,
-      "train_loss": 6530814950.684069,
-      "train_runtime": 138.6461,
-      "train_samples_per_second": 2.308,
-      "train_steps_per_second": 0.144
+      "train_loss": 2.5933608412742614e-06,
+      "train_runtime": 409.99,
+      "train_samples_per_second": 0.781,
+      "train_steps_per_second": 0.049
     }
   ],
   "logging_steps": 2,
   "max_steps": 20,
   "num_input_tokens_seen": 0,
-  "num_train_epochs": 1,
+  "num_train_epochs": 2,
   "save_steps": 25,
   "stateful_callbacks": {
     "TrainerControl": {

training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:a5eba6f7842a7334cd1309a633f5c39e06b9bb4db534d4590387557c9da37a9a
-size 6008
+oid sha256:d79376453f8deca09aff7733ac31ca7be1698542060a55c79f5eadee71ed5b96
+size 5944