End of training

all_results.json

@@ -1,8 +1,8 @@
 {
     "total_flos": 0.0,
-    "train_loss": 0.0025116026401519774,
-    "train_runtime": 799.9489,
-    "train_samples": 2,
-    "train_samples_per_second": 0.05,
-    "train_steps_per_second": 0.013
+    "train_loss": 0.009888447250705212,
+    "train_runtime": 625.0756,
+    "train_samples": 27,
+    "train_samples_per_second": 0.064,
+    "train_steps_per_second": 0.016
 }

model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:9a3158ccaf4cbb037a69a564fb85739b375d4a19acc624a9a1c79f01fe6f025a
+oid sha256:2f8973be940c2e7e354d3b1e541c04a913d6bc99f036ae16e3fab345cb7289bf
 size 1976163472

train_results.json

@@ -1,8 +1,8 @@
 {
     "total_flos": 0.0,
-    "train_loss": 0.0025116026401519774,
-    "train_runtime": 799.9489,
-    "train_samples": 2,
-    "train_samples_per_second": 0.05,
-    "train_steps_per_second": 0.013
+    "train_loss": 0.009888447250705212,
+    "train_runtime": 625.0756,
+    "train_samples": 27,
+    "train_samples_per_second": 0.064,
+    "train_steps_per_second": 0.016
 }

trainer_state.json

@@ -2,7 +2,7 @@
   "best_global_step": null,
   "best_metric": null,
   "best_model_checkpoint": null,
-  "epoch": 10.0,
+  "epoch": 0.7407407407407407,
   "eval_steps": 500,
   "global_step": 10,
   "is_hyper_param_search": false,
@@ -16,21 +16,21 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": 0.0,
-      "completions/max_length": 69.0,
-      "completions/max_terminated_length": 69.0,
-      "completions/mean_length": 64.875,
-      "completions/mean_terminated_length": 64.875,
-      "completions/min_length": 62.0,
-      "completions/min_terminated_length": 62.0,
-      "epoch": 2.0,
+      "completions/max_length": 50.0,
+      "completions/max_terminated_length": 50.0,
+      "completions/mean_length": 21.125,
+      "completions/mean_terminated_length": 21.125,
+      "completions/min_length": 4.0,
+      "completions/min_terminated_length": 4.0,
+      "epoch": 0.14814814814814814,
       "frac_reward_zero_std": 0.5,
-      "grad_norm": 6.228816032409668,
+      "grad_norm": 12.743616104125977,
       "kl": 0.0,
       "learning_rate": 5e-07,
-      "loss": 0.0046,
-      "num_tokens": 1543.0,
-      "reward": 0.6934062242507935,
-      "reward_std": 0.06088415801059455,
+      "loss": 0.0992,
+      "num_tokens": 1193.0,
+      "reward": 0.09130313247442245,
+      "reward_std": 0.01168987131677568,
       "rewards/concensus_correctness_reward_func/mean": 0.0,
       "rewards/concensus_correctness_reward_func/std": 0.0,
       "rewards/consensus_reward_func/mean": 0.0,
@@ -39,13 +39,13 @@
       "rewards/cumulative_reward_2/std": 0.0,
       "rewards/final_correctness_reward_func/mean": 0.0,
       "rewards/final_correctness_reward_func/std": 0.0,
-      "rewards/question_recreation_reward_func/mean": 0.199406236410141,
-      "rewards/question_recreation_reward_func/std": 0.20537260174751282,
+      "rewards/question_recreation_reward_func/mean": 0.09130313247442245,
+      "rewards/question_recreation_reward_func/std": 0.013678928487934172,
       "rewards/soft_format_reward_func/mean": 0.0,
       "rewards/soft_format_reward_func/std": 0.0,
       "rewards/strict_format_reward_func/mean": 0.0,
       "rewards/strict_format_reward_func/std": 0.0,
-      "rewards/xmlcount_reward_func/mean": 0.49399998784065247,
+      "rewards/xmlcount_reward_func/mean": 0.0,
       "rewards/xmlcount_reward_func/std": 0.0,
       "step": 2
     },
@@ -56,21 +56,21 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": 0.0,
-      "completions/max_length": 95.0,
-      "completions/max_terminated_length": 95.0,
-      "completions/mean_length": 71.0,
-      "completions/mean_terminated_length": 71.0,
-      "completions/min_length": 62.0,
-      "completions/min_terminated_length": 62.0,
-      "epoch": 4.0,
+      "completions/max_length": 22.0,
+      "completions/max_terminated_length": 22.0,
+      "completions/mean_length": 12.375,
+      "completions/mean_terminated_length": 12.375,
+      "completions/min_length": 7.0,
+      "completions/min_terminated_length": 7.0,
+      "epoch": 0.2962962962962963,
       "frac_reward_zero_std": 0.5,
-      "grad_norm": 14.296950340270996,
-      "kl": 0.008060061466267143,
+      "grad_norm": 43.35768508911133,
+      "kl": 0.017325570806860924,
       "learning_rate": 4.415111107797445e-07,
-      "loss": 0.0558,
-      "num_tokens": 3135.0,
-      "reward": 0.6558665037155151,
-      "reward_std": 0.09568738762754947,
+      "loss": -0.1484,
+      "num_tokens": 2316.0,
+      "reward": 0.1280631353147328,
+      "reward_std": 0.018915515393018723,
       "rewards/concensus_correctness_reward_func/mean": 0.0,
       "rewards/concensus_correctness_reward_func/std": 0.0,
       "rewards/consensus_reward_func/mean": 0.0,
@@ -79,14 +79,14 @@
       "rewards/cumulative_reward_2/std": 0.0,
       "rewards/final_correctness_reward_func/mean": 0.0,
       "rewards/final_correctness_reward_func/std": 0.0,
-      "rewards/question_recreation_reward_func/mean": 0.22361651808023453,
-      "rewards/question_recreation_reward_func/std": 0.20930995047092438,
+      "rewards/question_recreation_reward_func/mean": 0.1280631353147328,
+      "rewards/question_recreation_reward_func/std": 0.018273787572979927,
       "rewards/soft_format_reward_func/mean": 0.0,
       "rewards/soft_format_reward_func/std": 0.0,
       "rewards/strict_format_reward_func/mean": 0.0,
       "rewards/strict_format_reward_func/std": 0.0,
-      "rewards/xmlcount_reward_func/mean": 0.4322499930858612,
-      "rewards/xmlcount_reward_func/std": 0.12349999696016312,
+      "rewards/xmlcount_reward_func/mean": 0.0,
+      "rewards/xmlcount_reward_func/std": 0.0,
       "step": 4
     },
     {
@@ -96,21 +96,21 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": 0.0,
-      "completions/max_length": 65.5,
-      "completions/max_terminated_length": 65.5,
-      "completions/mean_length": 63.25,
-      "completions/mean_terminated_length": 63.25,
-      "completions/min_length": 61.0,
-      "completions/min_terminated_length": 61.0,
-      "epoch": 6.0,
-      "frac_reward_zero_std": 0.0,
-      "grad_norm": 7.470577716827393,
-      "kl": 0.021054526325315237,
+      "completions/max_length": 19.5,
+      "completions/max_terminated_length": 19.5,
+      "completions/mean_length": 13.375,
+      "completions/mean_terminated_length": 13.375,
+      "completions/min_length": 6.5,
+      "completions/min_terminated_length": 6.5,
+      "epoch": 0.4444444444444444,
+      "frac_reward_zero_std": 0.5,
+      "grad_norm": 0.018915563821792603,
+      "kl": 0.03074789233505726,
       "learning_rate": 2.934120444167326e-07,
-      "loss": -0.0026,
-      "num_tokens": 4665.0,
-      "reward": 0.7195066809654236,
-      "reward_std": 0.007645343546755612,
+      "loss": 0.0001,
+      "num_tokens": 3447.0,
+      "reward": 0.09580376278609037,
+      "reward_std": 0.002759912982583046,
       "rewards/concensus_correctness_reward_func/mean": 0.0,
       "rewards/concensus_correctness_reward_func/std": 0.0,
       "rewards/consensus_reward_func/mean": 0.0,
@@ -119,13 +119,13 @@
       "rewards/cumulative_reward_2/std": 0.0,
       "rewards/final_correctness_reward_func/mean": 0.0,
       "rewards/final_correctness_reward_func/std": 0.0,
-      "rewards/question_recreation_reward_func/mean": 0.22550669312477112,
-      "rewards/question_recreation_reward_func/std": 0.20679625123739243,
+      "rewards/question_recreation_reward_func/mean": 0.09580376371741295,
+      "rewards/question_recreation_reward_func/std": 0.002817572792991996,
       "rewards/soft_format_reward_func/mean": 0.0,
       "rewards/soft_format_reward_func/std": 0.0,
       "rewards/strict_format_reward_func/mean": 0.0,
       "rewards/strict_format_reward_func/std": 0.0,
-      "rewards/xmlcount_reward_func/mean": 0.49399998784065247,
+      "rewards/xmlcount_reward_func/mean": 0.0,
       "rewards/xmlcount_reward_func/std": 0.0,
       "step": 6
     },
@@ -136,21 +136,21 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": 0.0,
-      "completions/max_length": 64.5,
-      "completions/max_terminated_length": 64.5,
-      "completions/mean_length": 59.75,
-      "completions/mean_terminated_length": 59.75,
-      "completions/min_length": 49.0,
-      "completions/min_terminated_length": 49.0,
-      "epoch": 8.0,
+      "completions/max_length": 61.5,
+      "completions/max_terminated_length": 61.5,
+      "completions/mean_length": 29.375,
+      "completions/mean_terminated_length": 29.375,
+      "completions/min_length": 3.0,
+      "completions/min_terminated_length": 3.0,
+      "epoch": 0.5925925925925926,
       "frac_reward_zero_std": 0.25,
-      "grad_norm": 12.293351173400879,
-      "kl": 0.05089123750803992,
+      "grad_norm": 36.02655792236328,
+      "kl": 0.03904347028583288,
       "learning_rate": 1.2500000000000005e-07,
-      "loss": -0.0458,
-      "num_tokens": 6167.0,
-      "reward": 0.6569569408893585,
-      "reward_std": 0.09472145570907742,
+      "loss": 0.0023,
+      "num_tokens": 4706.0,
+      "reward": 0.01581350341439247,
+      "reward_std": 0.008347119903191924,
       "rewards/concensus_correctness_reward_func/mean": 0.0,
       "rewards/concensus_correctness_reward_func/std": 0.0,
       "rewards/consensus_reward_func/mean": 0.0,
@@ -159,14 +159,14 @@
       "rewards/cumulative_reward_2/std": 0.0,
       "rewards/final_correctness_reward_func/mean": 0.0,
       "rewards/final_correctness_reward_func/std": 0.0,
-      "rewards/question_recreation_reward_func/mean": 0.22470694035291672,
-      "rewards/question_recreation_reward_func/std": 0.20819812268018723,
+      "rewards/question_recreation_reward_func/mean": 0.01581350341439247,
+      "rewards/question_recreation_reward_func/std": 0.012281739618629217,
       "rewards/soft_format_reward_func/mean": 0.0,
       "rewards/soft_format_reward_func/std": 0.0,
       "rewards/strict_format_reward_func/mean": 0.0,
       "rewards/strict_format_reward_func/std": 0.0,
-      "rewards/xmlcount_reward_func/mean": 0.4322499930858612,
-      "rewards/xmlcount_reward_func/std": 0.12349999696016312,
+      "rewards/xmlcount_reward_func/mean": 0.0,
+      "rewards/xmlcount_reward_func/std": 0.0,
       "step": 8
     },
     {
@@ -176,53 +176,53 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": 0.0,
-      "completions/max_length": 65.5,
-      "completions/max_terminated_length": 65.5,
-      "completions/mean_length": 63.0,
-      "completions/mean_terminated_length": 63.0,
-      "completions/min_length": 61.0,
-      "completions/min_terminated_length": 61.0,
-      "epoch": 10.0,
+      "completions/max_length": 18.0,
+      "completions/max_terminated_length": 18.0,
+      "completions/mean_length": 11.75,
+      "completions/mean_terminated_length": 11.75,
+      "completions/min_length": 4.5,
+      "completions/min_terminated_length": 4.5,
+      "epoch": 0.7407407407407407,
       "frac_reward_zero_std": 0.25,
-      "grad_norm": 14.979830741882324,
-      "kl": 0.10098545905202627,
+      "grad_norm": 89.89861297607422,
+      "kl": 0.04395892460888717,
       "learning_rate": 1.507684480352292e-08,
-      "loss": 0.0006,
-      "num_tokens": 7695.0,
-      "reward": 0.7237494587898254,
-      "reward_std": 0.002912905707489699,
-      "rewards/concensus_correctness_reward_func/mean": 0.0,
-      "rewards/concensus_correctness_reward_func/std": 0.0,
-      "rewards/consensus_reward_func/mean": 0.0,
-      "rewards/consensus_reward_func/std": 0.0,
+      "loss": 0.0962,
+      "num_tokens": 5824.0,
+      "reward": 0.5346636138856411,
+      "reward_std": 0.6897922963835299,
+      "rewards/concensus_correctness_reward_func/mean": 0.2395000010728836,
+      "rewards/concensus_correctness_reward_func/std": 0.4790000319480896,
+      "rewards/consensus_reward_func/mean": 0.25,
+      "rewards/consensus_reward_func/std": 0.5,
       "rewards/cumulative_reward_2/mean": 0.0,
       "rewards/cumulative_reward_2/std": 0.0,
       "rewards/final_correctness_reward_func/mean": 0.0,
       "rewards/final_correctness_reward_func/std": 0.0,
-      "rewards/question_recreation_reward_func/mean": 0.22974946349859238,
-      "rewards/question_recreation_reward_func/std": 0.2142425775527954,
+      "rewards/question_recreation_reward_func/mean": 0.045163657516241074,
+      "rewards/question_recreation_reward_func/std": 0.028757712803781033,
       "rewards/soft_format_reward_func/mean": 0.0,
       "rewards/soft_format_reward_func/std": 0.0,
       "rewards/strict_format_reward_func/mean": 0.0,
       "rewards/strict_format_reward_func/std": 0.0,
-      "rewards/xmlcount_reward_func/mean": 0.49399998784065247,
+      "rewards/xmlcount_reward_func/mean": 0.0,
       "rewards/xmlcount_reward_func/std": 0.0,
       "step": 10
     },
     {
-      "epoch": 10.0,
+      "epoch": 0.7407407407407407,
       "step": 10,
       "total_flos": 0.0,
-      "train_loss": 0.0025116026401519774,
-      "train_runtime": 799.9489,
-      "train_samples_per_second": 0.05,
-      "train_steps_per_second": 0.013
+      "train_loss": 0.009888447250705212,
+      "train_runtime": 625.0756,
+      "train_samples_per_second": 0.064,
+      "train_steps_per_second": 0.016
     }
   ],
   "logging_steps": 2,
   "max_steps": 10,
-  "num_input_tokens_seen": 7695,
-  "num_train_epochs": 10,
+  "num_input_tokens_seen": 5824,
+  "num_train_epochs": 1,
   "save_steps": 10,
   "stateful_callbacks": {
     "TrainerControl": {