End of training

all_results.json

@@ -1,8 +1,8 @@
 {
     "total_flos": 0.0,
-    "train_loss": -0.0020117913249123377,
-    "train_runtime": 717.7906,
-    "train_samples": 2,
-    "train_samples_per_second": 0.056,
-    "train_steps_per_second": 0.014
+    "train_loss": 0.019962620735168458,
+    "train_runtime": 743.3925,
+    "train_samples": 47,
+    "train_samples_per_second": 0.054,
+    "train_steps_per_second": 0.013
 }

model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:026cc738e8cc6bcb57abd24e589111d6d63bdb72c6a0b0430c4ce089deb5d82f
+oid sha256:888f28d08d0f21e5b10a537a21426665887b6170f976dc56c687abd925bffac0
 size 1976163472

train_results.json

@@ -1,8 +1,8 @@
 {
     "total_flos": 0.0,
-    "train_loss": -0.0020117913249123377,
-    "train_runtime": 717.7906,
-    "train_samples": 2,
-    "train_samples_per_second": 0.056,
-    "train_steps_per_second": 0.014
+    "train_loss": 0.019962620735168458,
+    "train_runtime": 743.3925,
+    "train_samples": 47,
+    "train_samples_per_second": 0.054,
+    "train_steps_per_second": 0.013
 }

trainer_state.json

@@ -2,7 +2,7 @@
   "best_global_step": null,
   "best_metric": null,
   "best_model_checkpoint": null,
-  "epoch": 10.0,
+  "epoch": 0.425531914893617,
   "eval_steps": 500,
   "global_step": 10,
   "is_hyper_param_search": false,
@@ -16,21 +16,21 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": 0.0,
-      "completions/max_length": 62.5,
-      "completions/max_terminated_length": 62.5,
-      "completions/mean_length": 61.625,
-      "completions/mean_terminated_length": 61.625,
-      "completions/min_length": 61.0,
-      "completions/min_terminated_length": 61.0,
-      "epoch": 2.0,
-      "frac_reward_zero_std": 0.75,
-      "grad_norm": 4.375613689422607,
+      "completions/max_length": 28.5,
+      "completions/max_terminated_length": 28.5,
+      "completions/mean_length": 18.0,
+      "completions/mean_terminated_length": 18.0,
+      "completions/min_length": 8.5,
+      "completions/min_terminated_length": 8.5,
+      "epoch": 0.0851063829787234,
+      "frac_reward_zero_std": 0.25,
+      "grad_norm": 47.61864471435547,
       "kl": 0.0,
       "learning_rate": 5e-07,
-      "loss": 0.0013,
-      "num_tokens": 1517.0,
-      "reward": 0.7305998206138611,
-      "reward_std": 0.00035074606421403587,
+      "loss": -0.1209,
+      "num_tokens": 1168.0,
+      "reward": 0.09952777437865734,
+      "reward_std": 0.03358310600742698,
       "rewards/concensus_correctness_reward_func/mean": 0.0,
       "rewards/concensus_correctness_reward_func/std": 0.0,
       "rewards/consensus_reward_func/mean": 0.0,
@@ -39,13 +39,13 @@
       "rewards/cumulative_reward_2/std": 0.0,
       "rewards/final_correctness_reward_func/mean": 0.0,
       "rewards/final_correctness_reward_func/std": 0.0,
-      "rewards/question_recreation_reward_func/mean": 0.236599862575531,
-      "rewards/question_recreation_reward_func/std": 0.21793056279420853,
+      "rewards/question_recreation_reward_func/mean": 0.09952777437865734,
+      "rewards/question_recreation_reward_func/std": 0.04570972081273794,
       "rewards/soft_format_reward_func/mean": 0.0,
       "rewards/soft_format_reward_func/std": 0.0,
       "rewards/strict_format_reward_func/mean": 0.0,
       "rewards/strict_format_reward_func/std": 0.0,
-      "rewards/xmlcount_reward_func/mean": 0.49399998784065247,
+      "rewards/xmlcount_reward_func/mean": 0.0,
       "rewards/xmlcount_reward_func/std": 0.0,
       "step": 2
     },
@@ -55,22 +55,22 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": 0.0,
-      "completions/max_length": 62.5,
-      "completions/max_terminated_length": 62.5,
-      "completions/mean_length": 60.875,
-      "completions/mean_terminated_length": 60.875,
-      "completions/min_length": 58.0,
-      "completions/min_terminated_length": 58.0,
-      "epoch": 4.0,
-      "frac_reward_zero_std": 0.5,
-      "grad_norm": 9.196222305297852,
-      "kl": 0.014104919039709785,
+      "completions/clipped_ratio": 0.125,
+      "completions/max_length": 230.0,
+      "completions/max_terminated_length": 164.5,
+      "completions/mean_length": 95.0,
+      "completions/mean_terminated_length": 71.75,
+      "completions/min_length": 20.0,
+      "completions/min_terminated_length": 20.0,
+      "epoch": 0.1702127659574468,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 15.737818717956543,
+      "kl": 0.0014761510828975588,
       "learning_rate": 4.415111107797445e-07,
-      "loss": -0.0091,
-      "num_tokens": 3028.0,
-      "reward": 0.7158937156200409,
-      "reward_std": 0.016261233657132834,
+      "loss": 0.0007,
+      "num_tokens": 2952.0,
+      "reward": 0.10542272962629795,
+      "reward_std": 0.017522182315587997,
       "rewards/concensus_correctness_reward_func/mean": 0.0,
       "rewards/concensus_correctness_reward_func/std": 0.0,
       "rewards/consensus_reward_func/mean": 0.0,
@@ -79,14 +79,14 @@
       "rewards/cumulative_reward_2/std": 0.0,
       "rewards/final_correctness_reward_func/mean": 0.0,
       "rewards/final_correctness_reward_func/std": 0.0,
-      "rewards/question_recreation_reward_func/mean": 0.219393752515316,
-      "rewards/question_recreation_reward_func/std": 0.1909932717680931,
+      "rewards/question_recreation_reward_func/mean": 0.10542273707687855,
+      "rewards/question_recreation_reward_func/std": 0.03612042544409633,
       "rewards/soft_format_reward_func/mean": 0.0,
       "rewards/soft_format_reward_func/std": 0.0,
       "rewards/strict_format_reward_func/mean": 0.0,
       "rewards/strict_format_reward_func/std": 0.0,
-      "rewards/xmlcount_reward_func/mean": 0.4964999854564667,
-      "rewards/xmlcount_reward_func/std": 0.0010000020265579224,
+      "rewards/xmlcount_reward_func/mean": 0.0,
+      "rewards/xmlcount_reward_func/std": 0.0,
       "step": 4
     },
     {
@@ -96,21 +96,21 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": 0.0,
-      "completions/max_length": 62.0,
-      "completions/max_terminated_length": 62.0,
-      "completions/mean_length": 61.5,
-      "completions/mean_terminated_length": 61.5,
-      "completions/min_length": 61.0,
-      "completions/min_terminated_length": 61.0,
-      "epoch": 6.0,
-      "frac_reward_zero_std": 0.75,
-      "grad_norm": 0.00028228608425706625,
-      "kl": 0.02566487668082118,
+      "completions/max_length": 83.0,
+      "completions/max_terminated_length": 83.0,
+      "completions/mean_length": 40.25,
+      "completions/mean_terminated_length": 40.25,
+      "completions/min_length": 8.5,
+      "completions/min_terminated_length": 8.5,
+      "epoch": 0.2553191489361702,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 74.227783203125,
+      "kl": 0.21264880849048495,
       "learning_rate": 2.934120444167326e-07,
-      "loss": 0.0,
-      "num_tokens": 4544.0,
-      "reward": 0.7348592281341553,
-      "reward_std": 1.6079022316262126e-05,
+      "loss": 0.1325,
+      "num_tokens": 4298.0,
+      "reward": 0.08795275818556547,
+      "reward_std": 0.032405472593382,
       "rewards/concensus_correctness_reward_func/mean": 0.0,
       "rewards/concensus_correctness_reward_func/std": 0.0,
       "rewards/consensus_reward_func/mean": 0.0,
@@ -119,13 +119,13 @@
       "rewards/cumulative_reward_2/std": 0.0,
       "rewards/final_correctness_reward_func/mean": 0.0,
       "rewards/final_correctness_reward_func/std": 0.0,
-      "rewards/question_recreation_reward_func/mean": 0.23685923963785172,
-      "rewards/question_recreation_reward_func/std": 0.2182031199336052,
+      "rewards/question_recreation_reward_func/mean": 0.08795275120064616,
+      "rewards/question_recreation_reward_func/std": 0.02911469037644565,
       "rewards/soft_format_reward_func/mean": 0.0,
       "rewards/soft_format_reward_func/std": 0.0,
       "rewards/strict_format_reward_func/mean": 0.0,
       "rewards/strict_format_reward_func/std": 0.0,
-      "rewards/xmlcount_reward_func/mean": 0.49799999594688416,
+      "rewards/xmlcount_reward_func/mean": 0.0,
       "rewards/xmlcount_reward_func/std": 0.0,
       "step": 6
     },
@@ -136,21 +136,21 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": 0.0,
-      "completions/max_length": 62.5,
-      "completions/max_terminated_length": 62.5,
-      "completions/mean_length": 61.375,
-      "completions/mean_terminated_length": 61.375,
-      "completions/min_length": 60.5,
-      "completions/min_terminated_length": 60.5,
-      "epoch": 8.0,
-      "frac_reward_zero_std": 0.5,
-      "grad_norm": 2.685593843460083,
-      "kl": 0.026662976946681738,
+      "completions/max_length": 61.5,
+      "completions/max_terminated_length": 61.5,
+      "completions/mean_length": 32.125,
+      "completions/mean_terminated_length": 32.125,
+      "completions/min_length": 13.0,
+      "completions/min_terminated_length": 13.0,
+      "epoch": 0.3404255319148936,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 45.97591018676758,
+      "kl": 0.01475359802134335,
       "learning_rate": 1.2500000000000005e-07,
-      "loss": -0.0023,
-      "num_tokens": 6059.0,
-      "reward": 0.6660745143890381,
-      "reward_std": 0.08813039663982636,
+      "loss": 0.0381,
+      "num_tokens": 5579.0,
+      "reward": 0.14321431331336498,
+      "reward_std": 0.03659438155591488,
       "rewards/concensus_correctness_reward_func/mean": 0.0,
       "rewards/concensus_correctness_reward_func/std": 0.0,
       "rewards/consensus_reward_func/mean": 0.0,
@@ -159,14 +159,14 @@
       "rewards/cumulative_reward_2/std": 0.0,
       "rewards/final_correctness_reward_func/mean": 0.0,
       "rewards/final_correctness_reward_func/std": 0.0,
-      "rewards/question_recreation_reward_func/mean": 0.23032455891370773,
-      "rewards/question_recreation_reward_func/std": 0.20196203142404556,
+      "rewards/question_recreation_reward_func/mean": 0.14321431331336498,
+      "rewards/question_recreation_reward_func/std": 0.04845181666314602,
       "rewards/soft_format_reward_func/mean": 0.0,
       "rewards/soft_format_reward_func/std": 0.0,
       "rewards/strict_format_reward_func/mean": 0.0,
       "rewards/strict_format_reward_func/std": 0.0,
-      "rewards/xmlcount_reward_func/mean": 0.43574999272823334,
-      "rewards/xmlcount_reward_func/std": 0.12449999898672104,
+      "rewards/xmlcount_reward_func/mean": 0.0,
+      "rewards/xmlcount_reward_func/std": 0.0,
       "step": 8
     },
     {
@@ -176,21 +176,21 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": 0.0,
-      "completions/max_length": 62.0,
-      "completions/max_terminated_length": 62.0,
-      "completions/mean_length": 61.5,
-      "completions/mean_terminated_length": 61.5,
-      "completions/min_length": 61.0,
-      "completions/min_terminated_length": 61.0,
-      "epoch": 10.0,
-      "frac_reward_zero_std": 1.0,
-      "grad_norm": 6.908038631081581e-05,
-      "kl": 0.02584641706198454,
+      "completions/max_length": 63.0,
+      "completions/max_terminated_length": 63.0,
+      "completions/mean_length": 28.5,
+      "completions/mean_terminated_length": 28.5,
+      "completions/min_length": 12.0,
+      "completions/min_terminated_length": 12.0,
+      "epoch": 0.425531914893617,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 58.416831970214844,
+      "kl": 0.06678577116690576,
       "learning_rate": 1.507684480352292e-08,
-      "loss": 0.0,
-      "num_tokens": 7575.0,
-      "reward": 0.7348478436470032,
-      "reward_std": 0.0,
+      "loss": 0.0494,
+      "num_tokens": 6831.0,
+      "reward": 0.14338569156825542,
+      "reward_std": 0.012190744746476412,
       "rewards/concensus_correctness_reward_func/mean": 0.0,
       "rewards/concensus_correctness_reward_func/std": 0.0,
       "rewards/consensus_reward_func/mean": 0.0,
@@ -199,30 +199,30 @@
       "rewards/cumulative_reward_2/std": 0.0,
       "rewards/final_correctness_reward_func/mean": 0.0,
       "rewards/final_correctness_reward_func/std": 0.0,
-      "rewards/question_recreation_reward_func/mean": 0.2368478775024414,
-      "rewards/question_recreation_reward_func/std": 0.21821624040603638,
+      "rewards/question_recreation_reward_func/mean": 0.14338569156825542,
+      "rewards/question_recreation_reward_func/std": 0.01375252427533269,
       "rewards/soft_format_reward_func/mean": 0.0,
       "rewards/soft_format_reward_func/std": 0.0,
       "rewards/strict_format_reward_func/mean": 0.0,
       "rewards/strict_format_reward_func/std": 0.0,
-      "rewards/xmlcount_reward_func/mean": 0.49799999594688416,
+      "rewards/xmlcount_reward_func/mean": 0.0,
       "rewards/xmlcount_reward_func/std": 0.0,
       "step": 10
     },
     {
-      "epoch": 10.0,
+      "epoch": 0.425531914893617,
       "step": 10,
       "total_flos": 0.0,
-      "train_loss": -0.0020117913249123377,
-      "train_runtime": 717.7906,
-      "train_samples_per_second": 0.056,
-      "train_steps_per_second": 0.014
+      "train_loss": 0.019962620735168458,
+      "train_runtime": 743.3925,
+      "train_samples_per_second": 0.054,
+      "train_steps_per_second": 0.013
     }
   ],
   "logging_steps": 2,
   "max_steps": 10,
-  "num_input_tokens_seen": 7575,
-  "num_train_epochs": 10,
+  "num_input_tokens_seen": 6831,
+  "num_train_epochs": 1,
   "save_steps": 10,
   "stateful_callbacks": {
     "TrainerControl": {