End of training

README.md

@@ -38,7 +38,7 @@ This model was trained with GRPO, a method introduced in [DeepSeekMath: Pushing
 ### Framework versions
 
 - TRL: 0.15.2
-- Transformers: 4.51.1
+- Transformers: 4.51.3
 - Pytorch: 2.5.1
 - Datasets: 3.5.0
 - Tokenizers: 0.21.1

all_results.json

@@ -1,8 +1,8 @@
 {
     "total_flos": 0.0,
-    "train_loss": 0.004387809181207558,
-    "train_runtime": 428.4371,
-    "train_samples": 160,
-    "train_samples_per_second": 0.747,
-    "train_steps_per_second": 0.047
+    "train_loss": 1.6415491700172423e-06,
+    "train_runtime": 480.6902,
+    "train_samples": 28,
+    "train_samples_per_second": 0.666,
+    "train_steps_per_second": 0.042
 }

config.json

@@ -21,7 +21,7 @@
   "sliding_window": 32768,
   "tie_word_embeddings": true,
   "torch_dtype": "float32",
-  "transformers_version": "4.51.1",
+  "transformers_version": "4.51.3",
   "use_cache": true,
   "use_sliding_window": false,
   "vocab_size": 151936

generation_config.json

@@ -10,5 +10,5 @@
   "temperature": 0.7,
   "top_k": 20,
   "top_p": 0.8,
-  "transformers_version": "4.51.1"
+  "transformers_version": "4.51.3"
 }

model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:8404e647722253eaa2165b9a636568c407f0f005cd3baca14089d8fd75c1cb20
+oid sha256:9230b65525d4f5ffc3a56b0d9e0ec1355d057bbca7da7fd34bb760b19bfcb6ff
 size 1976163472

train_results.json

@@ -1,8 +1,8 @@
 {
     "total_flos": 0.0,
-    "train_loss": 0.004387809181207558,
-    "train_runtime": 428.4371,
-    "train_samples": 160,
-    "train_samples_per_second": 0.747,
-    "train_steps_per_second": 0.047
+    "train_loss": 1.6415491700172423e-06,
+    "train_runtime": 480.6902,
+    "train_samples": 28,
+    "train_samples_per_second": 0.666,
+    "train_steps_per_second": 0.042
 }

trainer_state.json

@@ -2,7 +2,7 @@
   "best_global_step": null,
   "best_metric": null,
   "best_model_checkpoint": null,
-  "epoch": 1.0,
+  "epoch": 5.0,
   "eval_steps": 500,
   "global_step": 20,
   "is_hyper_param_search": false,
@@ -10,209 +10,209 @@
   "is_world_process_zero": true,
   "log_history": [
     {
-      "completion_length": 270.96875,
-      "epoch": 0.1,
-      "grad_norm": 16.205791473388672,
+      "completion_length": 345.84375,
+      "epoch": 0.5714285714285714,
+      "grad_norm": 7.687195301055908,
       "kl": 0.0,
       "learning_rate": 5e-07,
-      "loss": 0.0,
-      "reward": 3.2229582499712706,
-      "reward_std": 1.0055385334417224,
-      "rewards/concensus_correctness_reward_func": 0.8008749969303608,
-      "rewards/consensus_reward_func": 0.8125,
+      "loss": -0.0,
+      "reward": 0.3834745204076171,
+      "reward_std": 0.4209819081588648,
+      "rewards/concensus_correctness_reward_func": 0.0,
+      "rewards/consensus_reward_func": 0.0,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.125,
-      "rewards/question_recreation_reward_func": 0.5988019495271146,
-      "rewards/soft_format_reward_func": 0.015625,
-      "rewards/strict_format_reward_func": 0.171875,
-      "rewards/xmlcount_reward_func": 0.6982812527567148,
+      "rewards/final_correctness_reward_func": 0.0,
+      "rewards/question_recreation_reward_func": 0.3486932716332376,
+      "rewards/soft_format_reward_func": 0.0,
+      "rewards/strict_format_reward_func": 0.0,
+      "rewards/xmlcount_reward_func": 0.034781253430992365,
       "step": 2
     },
     {
-      "completion_length": 224.6875,
-      "epoch": 0.2,
-      "grad_norm": 34.84421920776367,
-      "kl": 0.052124221707344986,
+      "completion_length": 383.75,
+      "epoch": 1.0,
+      "grad_norm": 3.4109249114990234,
+      "kl": 0.00163125775482816,
       "learning_rate": 4.864543104251586e-07,
-      "loss": 0.0001,
-      "reward": 5.273870535194874,
-      "reward_std": 0.9794920776039362,
-      "rewards/concensus_correctness_reward_func": 1.3452499955892563,
-      "rewards/consensus_reward_func": 1.5625,
+      "loss": 0.0,
+      "reward": 0.6712911507735649,
+      "reward_std": 0.6472472019959241,
+      "rewards/concensus_correctness_reward_func": 0.0,
+      "rewards/consensus_reward_func": 0.08333333333333333,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.125,
-      "rewards/question_recreation_reward_func": 0.7844330379739404,
+      "rewards/final_correctness_reward_func": 0.25,
+      "rewards/question_recreation_reward_func": 0.2176661485185226,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.328125,
-      "rewards/xmlcount_reward_func": 1.1285625100135803,
+      "rewards/strict_format_reward_func": 0.0,
+      "rewards/xmlcount_reward_func": 0.12029167078435421,
       "step": 4
     },
     {
-      "completion_length": 289.03125,
-      "epoch": 0.3,
-      "grad_norm": 20.824169158935547,
-      "kl": 0.22483220288995653,
+      "completion_length": 416.6875,
+      "epoch": 1.5714285714285714,
+      "grad_norm": 48.837833404541016,
+      "kl": 0.0011946773374802433,
       "learning_rate": 4.472851273490984e-07,
-      "loss": 0.0002,
-      "reward": 5.732023686170578,
-      "reward_std": 2.016730991192162,
-      "rewards/concensus_correctness_reward_func": 1.8961249869316816,
-      "rewards/consensus_reward_func": 1.5625,
+      "loss": 0.0,
+      "reward": 0.444772994145751,
+      "reward_std": 0.4007917964627268,
+      "rewards/concensus_correctness_reward_func": 0.0,
+      "rewards/consensus_reward_func": 0.0,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.1875,
-      "rewards/question_recreation_reward_func": 0.7713987100869417,
+      "rewards/final_correctness_reward_func": 0.0625,
+      "rewards/question_recreation_reward_func": 0.2896167419385165,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.265625,
-      "rewards/xmlcount_reward_func": 1.0488750003278255,
+      "rewards/strict_format_reward_func": 0.0,
+      "rewards/xmlcount_reward_func": 0.09265625150874257,
       "step": 6
     },
     {
-      "completion_length": 183.6875,
-      "epoch": 0.4,
-      "grad_norm": 22.67264175415039,
-      "kl": 0.3627709628781304,
+      "completion_length": 349.5,
+      "epoch": 2.0,
+      "grad_norm": 7.226299285888672,
+      "kl": 0.0016452606602494295,
       "learning_rate": 3.867370395306068e-07,
-      "loss": 0.0004,
-      "reward": 5.419944912195206,
-      "reward_std": 1.3517259755608393,
-      "rewards/concensus_correctness_reward_func": 1.5902499887160957,
-      "rewards/consensus_reward_func": 1.4375,
+      "loss": 0.0,
+      "reward": 0.34873829782009125,
+      "reward_std": 0.5460468224579623,
+      "rewards/concensus_correctness_reward_func": 0.0,
+      "rewards/consensus_reward_func": 0.0,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.25,
-      "rewards/question_recreation_reward_func": 0.6896323460387066,
+      "rewards/final_correctness_reward_func": 0.0,
+      "rewards/question_recreation_reward_func": 0.33102997318686295,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.296875,
-      "rewards/xmlcount_reward_func": 1.1556874960660934,
+      "rewards/strict_format_reward_func": 0.0,
+      "rewards/xmlcount_reward_func": 0.01770832948386669,
       "step": 8
     },
     {
-      "completion_length": 193.625,
-      "epoch": 0.5,
-      "grad_norm": 19.43205451965332,
-      "kl": 0.8366234111599624,
+      "completion_length": 357.40625,
+      "epoch": 2.571428571428571,
+      "grad_norm": 6.055001258850098,
+      "kl": 0.001523839953733841,
       "learning_rate": 3.1137137178519977e-07,
-      "loss": 0.0008,
-      "reward": 5.874577015638351,
-      "reward_std": 1.1465824714978226,
-      "rewards/concensus_correctness_reward_func": 1.7906249836087227,
-      "rewards/consensus_reward_func": 1.625,
+      "loss": 0.0,
+      "reward": 0.4659112503286451,
+      "reward_std": 0.5484347752062604,
+      "rewards/concensus_correctness_reward_func": 0.0,
+      "rewards/consensus_reward_func": 0.0,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.25,
-      "rewards/question_recreation_reward_func": 0.7391394600272179,
+      "rewards/final_correctness_reward_func": 0.0625,
+      "rewards/question_recreation_reward_func": 0.20313000353053212,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.34375,
-      "rewards/xmlcount_reward_func": 1.126062497496605,
+      "rewards/strict_format_reward_func": 0.0,
+      "rewards/xmlcount_reward_func": 0.2002812456630636,
       "step": 10
     },
     {
-      "completion_length": 159.875,
-      "epoch": 0.6,
-      "grad_norm": 24.322616577148438,
-      "kl": 0.8972976715303957,
+      "completion_length": 401.0416666666667,
+      "epoch": 3.0,
+      "grad_norm": 11.492475509643555,
+      "kl": 0.0019178439833922312,
       "learning_rate": 2.2935516363191693e-07,
-      "loss": 0.0009,
-      "reward": 5.864399433135986,
-      "reward_std": 0.4836496456991881,
-      "rewards/concensus_correctness_reward_func": 1.7774374820291996,
-      "rewards/consensus_reward_func": 1.6875,
+      "loss": 0.0,
+      "reward": 0.4682794213294983,
+      "reward_std": 0.589628112812837,
+      "rewards/concensus_correctness_reward_func": 0.0,
+      "rewards/consensus_reward_func": 0.0,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.125,
-      "rewards/question_recreation_reward_func": 0.7427119584754109,
+      "rewards/final_correctness_reward_func": 0.08333333333333333,
+      "rewards/question_recreation_reward_func": 0.2939044243345658,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.34375,
-      "rewards/xmlcount_reward_func": 1.1880000159144402,
+      "rewards/strict_format_reward_func": 0.0,
+      "rewards/xmlcount_reward_func": 0.09104166552424431,
       "step": 12
     },
     {
-      "completion_length": 219.0,
-      "epoch": 0.7,
-      "grad_norm": 1444.1883544921875,
-      "kl": 38.506456029834226,
+      "completion_length": 354.625,
+      "epoch": 3.571428571428571,
+      "grad_norm": 17.427644729614258,
+      "kl": 0.0031623352733731735,
       "learning_rate": 1.4957614383675767e-07,
-      "loss": 0.0385,
-      "reward": 5.395884931087494,
-      "reward_std": 0.9716897597536445,
-      "rewards/concensus_correctness_reward_func": 1.5414999965578318,
-      "rewards/consensus_reward_func": 1.75,
+      "loss": 0.0,
+      "reward": 0.3975663346936926,
+      "reward_std": 0.4194677284976933,
+      "rewards/concensus_correctness_reward_func": 0.0,
+      "rewards/consensus_reward_func": 0.0,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.0,
-      "rewards/question_recreation_reward_func": 0.801947420462966,
+      "rewards/final_correctness_reward_func": 0.0625,
+      "rewards/question_recreation_reward_func": 0.24159757781308144,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.25,
-      "rewards/xmlcount_reward_func": 1.052437499165535,
+      "rewards/strict_format_reward_func": 0.0,
+      "rewards/xmlcount_reward_func": 0.09346875036135316,
       "step": 14
     },
     {
-      "completion_length": 198.96875,
-      "epoch": 0.8,
-      "grad_norm": 15.164183616638184,
-      "kl": 1.3254998605698347,
+      "completion_length": 388.3333333333333,
+      "epoch": 4.0,
+      "grad_norm": 4.762839317321777,
+      "kl": 0.0018625854621253286,
       "learning_rate": 8.067960709356478e-08,
-      "loss": 0.0013,
-      "reward": 5.65350778400898,
-      "reward_std": 1.3196950405836105,
-      "rewards/concensus_correctness_reward_func": 1.5979374945163727,
-      "rewards/consensus_reward_func": 1.5,
+      "loss": 0.0,
+      "reward": 0.4537687711417675,
+      "reward_std": 0.32351934304460883,
+      "rewards/concensus_correctness_reward_func": 0.0,
+      "rewards/consensus_reward_func": 0.0,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.3125,
-      "rewards/question_recreation_reward_func": 0.8574764824006706,
+      "rewards/final_correctness_reward_func": 0.08333333333333333,
+      "rewards/question_recreation_reward_func": 0.23481043707579374,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.28125,
-      "rewards/xmlcount_reward_func": 1.1043437533080578,
+      "rewards/strict_format_reward_func": 0.0,
+      "rewards/xmlcount_reward_func": 0.13562499731779099,
       "step": 16
     },
     {
-      "completion_length": 148.0,
-      "epoch": 0.9,
-      "grad_norm": 13.233468055725098,
-      "kl": 1.1750296542886645,
+      "completion_length": 370.96875,
+      "epoch": 4.571428571428571,
+      "grad_norm": 8.153398513793945,
+      "kl": 0.0022796935754740844,
       "learning_rate": 3.013156219837776e-08,
-      "loss": 0.0012,
-      "reward": 6.176962792873383,
-      "reward_std": 0.6192302792333066,
-      "rewards/concensus_correctness_reward_func": 1.8604374825954437,
-      "rewards/consensus_reward_func": 1.75,
+      "loss": 0.0,
+      "reward": 0.4138966519385576,
+      "reward_std": 0.555002001565299,
+      "rewards/concensus_correctness_reward_func": 0.0,
+      "rewards/consensus_reward_func": 0.0,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.1875,
-      "rewards/question_recreation_reward_func": 0.7856503538787365,
+      "rewards/final_correctness_reward_func": 0.25,
+      "rewards/question_recreation_reward_func": 0.1937091457657516,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.359375,
-      "rewards/xmlcount_reward_func": 1.2340000048279762,
+      "rewards/strict_format_reward_func": 0.0,
+      "rewards/xmlcount_reward_func": -0.02981249918229878,
       "step": 18
     },
     {
-      "completion_length": 168.34375,
-      "epoch": 1.0,
-      "grad_norm": 12.352527618408203,
-      "kl": 0.4884904745267704,
+      "completion_length": 372.8333333333333,
+      "epoch": 5.0,
+      "grad_norm": 5.878365516662598,
+      "kl": 0.002409215856459923,
       "learning_rate": 3.4096741493194193e-09,
-      "loss": 0.0005,
-      "reward": 5.992502212524414,
-      "reward_std": 0.57326033472782,
-      "rewards/concensus_correctness_reward_func": 1.8762500043958426,
-      "rewards/consensus_reward_func": 1.8125,
+      "loss": 0.0,
+      "reward": 0.3080611840511362,
+      "reward_std": 0.5041453222947894,
+      "rewards/concensus_correctness_reward_func": 0.0,
+      "rewards/consensus_reward_func": 0.0,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.0625,
-      "rewards/question_recreation_reward_func": 0.7554397168569267,
+      "rewards/final_correctness_reward_func": 0.08333333333333333,
+      "rewards/question_recreation_reward_func": 0.21501951090370616,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.3125,
-      "rewards/xmlcount_reward_func": 1.1733125075697899,
+      "rewards/strict_format_reward_func": 0.0,
+      "rewards/xmlcount_reward_func": 0.009708332208295664,
       "step": 20
     },
     {
-      "epoch": 1.0,
+      "epoch": 5.0,
       "step": 20,
       "total_flos": 0.0,
-      "train_loss": 0.004387809181207558,
-      "train_runtime": 428.4371,
-      "train_samples_per_second": 0.747,
-      "train_steps_per_second": 0.047
+      "train_loss": 1.6415491700172423e-06,
+      "train_runtime": 480.6902,
+      "train_samples_per_second": 0.666,
+      "train_steps_per_second": 0.042
     }
   ],
   "logging_steps": 2,
   "max_steps": 20,
   "num_input_tokens_seen": 0,
-  "num_train_epochs": 1,
+  "num_train_epochs": 7,
   "save_steps": 25,
   "stateful_callbacks": {
     "TrainerControl": {

training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:4a11cac0734c6fbdbde2ee200b3346f83079dca6d13f7e1a35b74040d05d9fa7
-size 5944
+oid sha256:c9f5c68d4a3219cf854e3bd1def0c74e48c084119f6f2fd5a4f6e89a3a7b77e4
+size 6008