End of training

README.md

@@ -37,9 +37,9 @@ This model was trained with GRPO, a method introduced in [DeepSeekMath: Pushing
 
 ### Framework versions
 
-- TRL: 0.18.1
+- TRL: 0.18.2
 - Transformers: 4.52.4
-- Pytorch: 2.7.0
+- Pytorch: 2.7.1
 - Datasets: 3.6.0
 - Tokenizers: 0.21.1
 

all_results.json

@@ -1,8 +1,8 @@
 {
     "total_flos": 0.0,
-    "train_loss": 0.0,
-    "train_runtime": 447.6474,
-    "train_samples": 2,
-    "train_samples_per_second": 0.089,
-    "train_steps_per_second": 0.022
+    "train_loss": 0.0028146922588348387,
+    "train_runtime": 1322.7791,
+    "train_samples": 64,
+    "train_samples_per_second": 0.03,
+    "train_steps_per_second": 0.008
 }

model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:d34cfb4097011c0b93505b900eff8279f5b9beaeb2200ef608176c2982db43be
+oid sha256:3ed74ed220f2776a51119e2ef37fe92cdb9ff0e5d2c20c8beb1405fa994bf573
 size 1976163472

train_results.json

@@ -1,8 +1,8 @@
 {
     "total_flos": 0.0,
-    "train_loss": 0.0,
-    "train_runtime": 447.6474,
-    "train_samples": 2,
-    "train_samples_per_second": 0.089,
-    "train_steps_per_second": 0.022
+    "train_loss": 0.0028146922588348387,
+    "train_runtime": 1322.7791,
+    "train_samples": 64,
+    "train_samples_per_second": 0.03,
+    "train_steps_per_second": 0.008
 }

trainer_state.json

@@ -2,7 +2,7 @@
   "best_global_step": null,
   "best_metric": null,
   "best_model_checkpoint": null,
-  "epoch": 10.0,
+  "epoch": 0.3125,
   "eval_steps": 500,
   "global_step": 10,
   "is_hyper_param_search": false,
@@ -15,38 +15,38 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": 0.0,
-      "completions/max_length": 67.0,
-      "completions/max_terminated_length": 67.0,
-      "completions/mean_length": 49.5,
-      "completions/mean_terminated_length": 49.5,
-      "completions/min_length": 32.0,
-      "completions/min_terminated_length": 32.0,
-      "epoch": 2.0,
-      "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.0,
+      "completions/clipped_ratio": 0.125,
+      "completions/max_length": 199.5,
+      "completions/max_terminated_length": 181.0,
+      "completions/mean_length": 94.75,
+      "completions/mean_terminated_length": 73.45833206176758,
+      "completions/min_length": 4.5,
+      "completions/min_terminated_length": 4.5,
+      "epoch": 0.0625,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 13.581038475036621,
       "kl": 0.0,
       "learning_rate": 5e-07,
-      "loss": 0.0,
-      "num_tokens": 1420.0,
-      "reward": 1.3853346705436707,
-      "reward_std": 0.0,
+      "loss": 0.1697,
+      "num_tokens": 1782.0,
+      "reward": 0.06285679526627064,
+      "reward_std": 0.010482232493814081,
       "rewards/concensus_correctness_reward_func/mean": 0.0,
       "rewards/concensus_correctness_reward_func/std": 0.0,
       "rewards/consensus_reward_func/mean": 0.0,
       "rewards/consensus_reward_func/std": 0.0,
       "rewards/cumulative_reward_2/mean": 0.0,
       "rewards/cumulative_reward_2/std": 0.0,
-      "rewards/final_correctness_reward_func/mean": 1.0,
-      "rewards/final_correctness_reward_func/std": 1.154700517654419,
-      "rewards/question_recreation_reward_func/mean": 0.020834745839238167,
-      "rewards/question_recreation_reward_func/std": 0.004241383750922978,
+      "rewards/final_correctness_reward_func/mean": 0.0,
+      "rewards/final_correctness_reward_func/std": 0.0,
+      "rewards/question_recreation_reward_func/mean": 0.06285679526627064,
+      "rewards/question_recreation_reward_func/std": 0.022788123576901853,
       "rewards/soft_format_reward_func/mean": 0.0,
       "rewards/soft_format_reward_func/std": 0.0,
       "rewards/strict_format_reward_func/mean": 0.0,
       "rewards/strict_format_reward_func/std": 0.0,
-      "rewards/xmlcount_reward_func/mean": 0.3645000010728836,
-      "rewards/xmlcount_reward_func/std": 0.03983715549111366,
+      "rewards/xmlcount_reward_func/mean": 0.0,
+      "rewards/xmlcount_reward_func/std": 0.0,
       "step": 2
     },
     {
@@ -55,38 +55,38 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": 0.0,
-      "completions/max_length": 67.0,
-      "completions/max_terminated_length": 67.0,
-      "completions/mean_length": 49.5,
-      "completions/mean_terminated_length": 49.5,
-      "completions/min_length": 32.0,
-      "completions/min_terminated_length": 32.0,
-      "epoch": 4.0,
-      "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.0,
-      "kl": 0.0,
+      "completions/clipped_ratio": 0.125,
+      "completions/max_length": 155.0,
+      "completions/max_terminated_length": 130.5,
+      "completions/mean_length": 73.0,
+      "completions/mean_terminated_length": 50.5,
+      "completions/min_length": 4.0,
+      "completions/min_terminated_length": 4.0,
+      "epoch": 0.125,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 38.976619720458984,
+      "kl": 0.006027122130035423,
       "learning_rate": 4.415111107797445e-07,
-      "loss": 0.0,
-      "num_tokens": 2840.0,
-      "reward": 1.3885241746902466,
-      "reward_std": 0.0,
+      "loss": -0.2687,
+      "num_tokens": 3390.0,
+      "reward": 0.022088108584284782,
+      "reward_std": 0.009733497630804777,
       "rewards/concensus_correctness_reward_func/mean": 0.0,
       "rewards/concensus_correctness_reward_func/std": 0.0,
       "rewards/consensus_reward_func/mean": 0.0,
       "rewards/consensus_reward_func/std": 0.0,
       "rewards/cumulative_reward_2/mean": 0.0,
       "rewards/cumulative_reward_2/std": 0.0,
-      "rewards/final_correctness_reward_func/mean": 1.0,
-      "rewards/final_correctness_reward_func/std": 1.154700517654419,
-      "rewards/question_recreation_reward_func/mean": 0.024024246260523796,
-      "rewards/question_recreation_reward_func/std": 0.006436373107135296,
+      "rewards/final_correctness_reward_func/mean": 0.0,
+      "rewards/final_correctness_reward_func/std": 0.0,
+      "rewards/question_recreation_reward_func/mean": 0.022088108584284782,
+      "rewards/question_recreation_reward_func/std": 0.009718202985823154,
       "rewards/soft_format_reward_func/mean": 0.0,
       "rewards/soft_format_reward_func/std": 0.0,
       "rewards/strict_format_reward_func/mean": 0.0,
       "rewards/strict_format_reward_func/std": 0.0,
-      "rewards/xmlcount_reward_func/mean": 0.3644999861717224,
-      "rewards/xmlcount_reward_func/std": 0.03983715549111366,
+      "rewards/xmlcount_reward_func/mean": 0.0,
+      "rewards/xmlcount_reward_func/std": 0.0,
       "step": 4
     },
     {
@@ -95,38 +95,38 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": 0.0,
-      "completions/max_length": 67.0,
-      "completions/max_terminated_length": 67.0,
-      "completions/mean_length": 49.5,
-      "completions/mean_terminated_length": 49.5,
-      "completions/min_length": 32.0,
-      "completions/min_terminated_length": 32.0,
-      "epoch": 6.0,
-      "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.0,
-      "kl": 0.0,
+      "completions/clipped_ratio": 0.125,
+      "completions/max_length": 211.0,
+      "completions/max_terminated_length": 187.0,
+      "completions/mean_length": 139.75,
+      "completions/mean_terminated_length": 127.83333587646484,
+      "completions/min_length": 68.0,
+      "completions/min_terminated_length": 68.0,
+      "epoch": 0.1875,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 15.524645805358887,
+      "kl": 0.0035549709282349795,
       "learning_rate": 2.934120444167326e-07,
-      "loss": 0.0,
-      "num_tokens": 4260.0,
-      "reward": 1.3853346705436707,
-      "reward_std": 0.0,
+      "loss": -0.0062,
+      "num_tokens": 5532.0,
+      "reward": 0.03519869223237038,
+      "reward_std": 0.0057324927765876055,
       "rewards/concensus_correctness_reward_func/mean": 0.0,
       "rewards/concensus_correctness_reward_func/std": 0.0,
       "rewards/consensus_reward_func/mean": 0.0,
       "rewards/consensus_reward_func/std": 0.0,
       "rewards/cumulative_reward_2/mean": 0.0,
       "rewards/cumulative_reward_2/std": 0.0,
-      "rewards/final_correctness_reward_func/mean": 1.0,
-      "rewards/final_correctness_reward_func/std": 1.154700517654419,
-      "rewards/question_recreation_reward_func/mean": 0.020834745839238167,
-      "rewards/question_recreation_reward_func/std": 0.004241383750922978,
+      "rewards/final_correctness_reward_func/mean": 0.0,
+      "rewards/final_correctness_reward_func/std": 0.0,
+      "rewards/question_recreation_reward_func/mean": 0.03519869036972523,
+      "rewards/question_recreation_reward_func/std": 0.010448201559484005,
       "rewards/soft_format_reward_func/mean": 0.0,
       "rewards/soft_format_reward_func/std": 0.0,
       "rewards/strict_format_reward_func/mean": 0.0,
       "rewards/strict_format_reward_func/std": 0.0,
-      "rewards/xmlcount_reward_func/mean": 0.3645000010728836,
-      "rewards/xmlcount_reward_func/std": 0.03983715549111366,
+      "rewards/xmlcount_reward_func/mean": 0.0,
+      "rewards/xmlcount_reward_func/std": 0.0,
       "step": 6
     },
     {
@@ -135,38 +135,38 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": 0.0,
-      "completions/max_length": 67.0,
-      "completions/max_terminated_length": 67.0,
-      "completions/mean_length": 49.5,
-      "completions/mean_terminated_length": 49.5,
-      "completions/min_length": 32.0,
-      "completions/min_terminated_length": 32.0,
-      "epoch": 8.0,
-      "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.0,
-      "kl": 0.0,
+      "completions/clipped_ratio": 0.25,
+      "completions/max_length": 180.5,
+      "completions/max_terminated_length": 63.0,
+      "completions/mean_length": 101.5,
+      "completions/mean_terminated_length": 40.75,
+      "completions/min_length": 4.5,
+      "completions/min_terminated_length": 4.5,
+      "epoch": 0.25,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 12.807693481445312,
+      "kl": 0.012600050424225628,
       "learning_rate": 1.2500000000000005e-07,
-      "loss": 0.0,
-      "num_tokens": 5680.0,
-      "reward": 1.3885241746902466,
-      "reward_std": 0.0,
+      "loss": -0.0197,
+      "num_tokens": 7368.0,
+      "reward": 0.07958310190588236,
+      "reward_std": 0.02216464071534574,
       "rewards/concensus_correctness_reward_func/mean": 0.0,
       "rewards/concensus_correctness_reward_func/std": 0.0,
       "rewards/consensus_reward_func/mean": 0.0,
       "rewards/consensus_reward_func/std": 0.0,
       "rewards/cumulative_reward_2/mean": 0.0,
       "rewards/cumulative_reward_2/std": 0.0,
-      "rewards/final_correctness_reward_func/mean": 1.0,
-      "rewards/final_correctness_reward_func/std": 1.154700517654419,
-      "rewards/question_recreation_reward_func/mean": 0.024024246260523796,
-      "rewards/question_recreation_reward_func/std": 0.006436373107135296,
+      "rewards/final_correctness_reward_func/mean": 0.0,
+      "rewards/final_correctness_reward_func/std": 0.0,
+      "rewards/question_recreation_reward_func/mean": 0.07958310190588236,
+      "rewards/question_recreation_reward_func/std": 0.020205570850521326,
       "rewards/soft_format_reward_func/mean": 0.0,
       "rewards/soft_format_reward_func/std": 0.0,
       "rewards/strict_format_reward_func/mean": 0.0,
       "rewards/strict_format_reward_func/std": 0.0,
-      "rewards/xmlcount_reward_func/mean": 0.3644999861717224,
-      "rewards/xmlcount_reward_func/std": 0.03983715549111366,
+      "rewards/xmlcount_reward_func/mean": 0.0,
+      "rewards/xmlcount_reward_func/std": 0.0,
       "step": 8
     },
     {
@@ -175,54 +175,54 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": 0.0,
-      "completions/max_length": 67.0,
-      "completions/max_terminated_length": 67.0,
-      "completions/mean_length": 49.5,
-      "completions/mean_terminated_length": 49.5,
-      "completions/min_length": 32.0,
-      "completions/min_terminated_length": 32.0,
-      "epoch": 10.0,
-      "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.0,
-      "kl": 0.0,
+      "completions/clipped_ratio": 0.125,
+      "completions/max_length": 245.0,
+      "completions/max_terminated_length": 194.5,
+      "completions/mean_length": 119.625,
+      "completions/mean_terminated_length": 96.75,
+      "completions/min_length": 33.5,
+      "completions/min_terminated_length": 33.5,
+      "epoch": 0.3125,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 13.917776107788086,
+      "kl": 0.002077412951621227,
       "learning_rate": 1.507684480352292e-08,
-      "loss": 0.0,
-      "num_tokens": 7100.0,
-      "reward": 1.3853346705436707,
-      "reward_std": 0.0,
+      "loss": 0.139,
+      "num_tokens": 9349.0,
+      "reward": 0.17312341183423996,
+      "reward_std": 0.043228439055383205,
       "rewards/concensus_correctness_reward_func/mean": 0.0,
       "rewards/concensus_correctness_reward_func/std": 0.0,
       "rewards/consensus_reward_func/mean": 0.0,
       "rewards/consensus_reward_func/std": 0.0,
       "rewards/cumulative_reward_2/mean": 0.0,
       "rewards/cumulative_reward_2/std": 0.0,
-      "rewards/final_correctness_reward_func/mean": 1.0,
-      "rewards/final_correctness_reward_func/std": 1.154700517654419,
-      "rewards/question_recreation_reward_func/mean": 0.020834745839238167,
-      "rewards/question_recreation_reward_func/std": 0.004241383750922978,
+      "rewards/final_correctness_reward_func/mean": 0.0,
+      "rewards/final_correctness_reward_func/std": 0.0,
+      "rewards/question_recreation_reward_func/mean": 0.17312341928482056,
+      "rewards/question_recreation_reward_func/std": 0.07501886645331979,
       "rewards/soft_format_reward_func/mean": 0.0,
       "rewards/soft_format_reward_func/std": 0.0,
       "rewards/strict_format_reward_func/mean": 0.0,
       "rewards/strict_format_reward_func/std": 0.0,
-      "rewards/xmlcount_reward_func/mean": 0.3645000010728836,
-      "rewards/xmlcount_reward_func/std": 0.03983715549111366,
+      "rewards/xmlcount_reward_func/mean": 0.0,
+      "rewards/xmlcount_reward_func/std": 0.0,
       "step": 10
     },
     {
-      "epoch": 10.0,
+      "epoch": 0.3125,
       "step": 10,
       "total_flos": 0.0,
-      "train_loss": 0.0,
-      "train_runtime": 447.6474,
-      "train_samples_per_second": 0.089,
-      "train_steps_per_second": 0.022
+      "train_loss": 0.0028146922588348387,
+      "train_runtime": 1322.7791,
+      "train_samples_per_second": 0.03,
+      "train_steps_per_second": 0.008
     }
   ],
   "logging_steps": 2,
   "max_steps": 10,
-  "num_input_tokens_seen": 7100,
-  "num_train_epochs": 10,
+  "num_input_tokens_seen": 9349,
+  "num_train_epochs": 1,
   "save_steps": 10,
   "stateful_callbacks": {
     "TrainerControl": {

training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:28ee1a580d965a76bcf2cf92000418dfd16e154a05afb479ec9bbddcc7bd6995
-size 6993
+oid sha256:ebd4df9671b079db4379101c5551d8648c8cf9167f6e380fb19b19f2bd109811
+size 6865