End of training

all_results.json

@@ -1,8 +1,8 @@
 {
     "total_flos": 0.0,
-    "train_loss": -0.016702868044376373,
-    "train_runtime": 701.409,
-    "train_samples": 40,
-    "train_samples_per_second": 0.057,
-    "train_steps_per_second": 0.014
+    "train_loss": -0.06971818171441554,
+    "train_runtime": 764.1528,
+    "train_samples": 43,
+    "train_samples_per_second": 0.052,
+    "train_steps_per_second": 0.013
 }

model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:3e7867941982071863d9c770c681b7ec53d58e421532c68e7267510fe1185ebe
+oid sha256:058b9729a6e38ab44c603af7d2ecc05460889c84fa659ec83268fcd64cf88d62
 size 1976163472

train_results.json

@@ -1,8 +1,8 @@
 {
     "total_flos": 0.0,
-    "train_loss": -0.016702868044376373,
-    "train_runtime": 701.409,
-    "train_samples": 40,
-    "train_samples_per_second": 0.057,
-    "train_steps_per_second": 0.014
+    "train_loss": -0.06971818171441554,
+    "train_runtime": 764.1528,
+    "train_samples": 43,
+    "train_samples_per_second": 0.052,
+    "train_steps_per_second": 0.013
 }

trainer_state.json

@@ -2,7 +2,7 @@
   "best_global_step": null,
   "best_metric": null,
   "best_model_checkpoint": null,
-  "epoch": 0.5,
+  "epoch": 0.46511627906976744,
   "eval_steps": 500,
   "global_step": 10,
   "is_hyper_param_search": false,
@@ -16,31 +16,31 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": 0.0,
-      "completions/max_length": 67.0,
-      "completions/max_terminated_length": 67.0,
-      "completions/mean_length": 33.0,
-      "completions/mean_terminated_length": 33.0,
-      "completions/min_length": 14.5,
-      "completions/min_terminated_length": 14.5,
-      "epoch": 0.1,
+      "completions/max_length": 14.0,
+      "completions/max_terminated_length": 14.0,
+      "completions/mean_length": 10.375,
+      "completions/mean_terminated_length": 10.375,
+      "completions/min_length": 7.0,
+      "completions/min_terminated_length": 7.0,
+      "epoch": 0.09302325581395349,
       "frac_reward_zero_std": 0.0,
-      "grad_norm": 94.07942962646484,
+      "grad_norm": 71.25790405273438,
       "kl": 0.0,
       "learning_rate": 5e-07,
-      "loss": 0.0734,
-      "num_tokens": 1288.0,
-      "reward": 0.566925510764122,
-      "reward_std": 0.6912682754918933,
-      "rewards/concensus_correctness_reward_func/mean": 0.23999999463558197,
-      "rewards/concensus_correctness_reward_func/std": 0.47999998927116394,
-      "rewards/consensus_reward_func/mean": 0.25,
-      "rewards/consensus_reward_func/std": 0.5,
+      "loss": -0.0188,
+      "num_tokens": 1107.0,
+      "reward": 0.027277667075395584,
+      "reward_std": 0.014282920397818089,
+      "rewards/concensus_correctness_reward_func/mean": 0.0,
+      "rewards/concensus_correctness_reward_func/std": 0.0,
+      "rewards/consensus_reward_func/mean": 0.0,
+      "rewards/consensus_reward_func/std": 0.0,
       "rewards/cumulative_reward_2/mean": 0.0,
       "rewards/cumulative_reward_2/std": 0.0,
       "rewards/final_correctness_reward_func/mean": 0.0,
       "rewards/final_correctness_reward_func/std": 0.0,
-      "rewards/question_recreation_reward_func/mean": 0.07692548632621765,
-      "rewards/question_recreation_reward_func/std": 0.04143594345077872,
+      "rewards/question_recreation_reward_func/mean": 0.027277665212750435,
+      "rewards/question_recreation_reward_func/std": 0.012967212125658989,
       "rewards/soft_format_reward_func/mean": 0.0,
       "rewards/soft_format_reward_func/std": 0.0,
       "rewards/strict_format_reward_func/mean": 0.0,
@@ -56,21 +56,21 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": 0.0,
-      "completions/max_length": 37.0,
-      "completions/max_terminated_length": 37.0,
-      "completions/mean_length": 14.125,
-      "completions/mean_terminated_length": 14.125,
-      "completions/min_length": 3.0,
-      "completions/min_terminated_length": 3.0,
-      "epoch": 0.2,
-      "frac_reward_zero_std": 0.25,
-      "grad_norm": 24.905315399169922,
-      "kl": 0.004891372343990952,
+      "completions/max_length": 44.0,
+      "completions/max_terminated_length": 44.0,
+      "completions/mean_length": 22.875,
+      "completions/mean_terminated_length": 22.875,
+      "completions/min_length": 6.0,
+      "completions/min_terminated_length": 6.0,
+      "epoch": 0.18604651162790697,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 35.277915954589844,
+      "kl": 0.017253971062018536,
       "learning_rate": 4.415111107797445e-07,
-      "loss": -0.13,
-      "num_tokens": 2425.0,
-      "reward": 0.03376455884426832,
-      "reward_std": 0.03677849750965834,
+      "loss": -0.0746,
+      "num_tokens": 2314.0,
+      "reward": 0.02928277850151062,
+      "reward_std": 0.0103015610948205,
       "rewards/concensus_correctness_reward_func/mean": 0.0,
       "rewards/concensus_correctness_reward_func/std": 0.0,
       "rewards/consensus_reward_func/mean": 0.0,
@@ -79,8 +79,8 @@
       "rewards/cumulative_reward_2/std": 0.0,
       "rewards/final_correctness_reward_func/mean": 0.0,
       "rewards/final_correctness_reward_func/std": 0.0,
-      "rewards/question_recreation_reward_func/mean": 0.03376455884426832,
-      "rewards/question_recreation_reward_func/std": 0.045659529976546764,
+      "rewards/question_recreation_reward_func/mean": 0.02928277850151062,
+      "rewards/question_recreation_reward_func/std": 0.014456782955676317,
       "rewards/soft_format_reward_func/mean": 0.0,
       "rewards/soft_format_reward_func/std": 0.0,
       "rewards/strict_format_reward_func/mean": 0.0,
@@ -96,21 +96,21 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": 0.0,
-      "completions/max_length": 17.0,
-      "completions/max_terminated_length": 17.0,
-      "completions/mean_length": 10.75,
-      "completions/mean_terminated_length": 10.75,
-      "completions/min_length": 4.0,
-      "completions/min_terminated_length": 4.0,
-      "epoch": 0.3,
+      "completions/max_length": 17.5,
+      "completions/max_terminated_length": 17.5,
+      "completions/mean_length": 11.25,
+      "completions/mean_terminated_length": 11.25,
+      "completions/min_length": 7.5,
+      "completions/min_terminated_length": 7.5,
+      "epoch": 0.27906976744186046,
       "frac_reward_zero_std": 0.25,
-      "grad_norm": 82.60856628417969,
-      "kl": 0.027754191542044282,
+      "grad_norm": 74.44947814941406,
+      "kl": 0.09166394639760256,
       "learning_rate": 2.934120444167326e-07,
-      "loss": -0.0586,
-      "num_tokens": 3535.0,
-      "reward": 0.100416315253824,
-      "reward_std": 0.022026267266483046,
+      "loss": -0.1635,
+      "num_tokens": 3428.0,
+      "reward": 0.08016655035316944,
+      "reward_std": 0.005623728677164763,
       "rewards/concensus_correctness_reward_func/mean": 0.0,
       "rewards/concensus_correctness_reward_func/std": 0.0,
       "rewards/consensus_reward_func/mean": 0.0,
@@ -119,8 +119,8 @@
       "rewards/cumulative_reward_2/std": 0.0,
       "rewards/final_correctness_reward_func/mean": 0.0,
       "rewards/final_correctness_reward_func/std": 0.0,
-      "rewards/question_recreation_reward_func/mean": 0.100416315253824,
-      "rewards/question_recreation_reward_func/std": 0.025629534393374342,
+      "rewards/question_recreation_reward_func/mean": 0.08016655035316944,
+      "rewards/question_recreation_reward_func/std": 0.025440931785851717,
       "rewards/soft_format_reward_func/mean": 0.0,
       "rewards/soft_format_reward_func/std": 0.0,
       "rewards/strict_format_reward_func/mean": 0.0,
@@ -136,31 +136,31 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": 0.0,
-      "completions/max_length": 87.5,
-      "completions/max_terminated_length": 87.5,
-      "completions/mean_length": 32.75,
-      "completions/mean_terminated_length": 32.75,
-      "completions/min_length": 6.5,
-      "completions/min_terminated_length": 6.5,
-      "epoch": 0.4,
-      "frac_reward_zero_std": 0.25,
-      "grad_norm": 55.12723922729492,
-      "kl": 0.03376331052277237,
+      "completions/max_length": 26.5,
+      "completions/max_terminated_length": 26.5,
+      "completions/mean_length": 13.875,
+      "completions/mean_terminated_length": 13.875,
+      "completions/min_length": 5.0,
+      "completions/min_terminated_length": 5.0,
+      "epoch": 0.37209302325581395,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 114.46356201171875,
+      "kl": 0.14744188636541367,
       "learning_rate": 1.2500000000000005e-07,
-      "loss": -0.028,
-      "num_tokens": 4821.0,
-      "reward": 0.18568507581949234,
-      "reward_std": 0.014692435972392559,
+      "loss": -0.0753,
+      "num_tokens": 4563.0,
+      "reward": 0.27879717759788036,
+      "reward_std": 0.36270917439833283,
       "rewards/concensus_correctness_reward_func/mean": 0.0,
       "rewards/concensus_correctness_reward_func/std": 0.0,
-      "rewards/consensus_reward_func/mean": 0.0,
-      "rewards/consensus_reward_func/std": 0.0,
+      "rewards/consensus_reward_func/mean": 0.25,
+      "rewards/consensus_reward_func/std": 0.5,
       "rewards/cumulative_reward_2/mean": 0.0,
       "rewards/cumulative_reward_2/std": 0.0,
       "rewards/final_correctness_reward_func/mean": 0.0,
       "rewards/final_correctness_reward_func/std": 0.0,
-      "rewards/question_recreation_reward_func/mean": 0.18568507581949234,
-      "rewards/question_recreation_reward_func/std": 0.06828897818922997,
+      "rewards/question_recreation_reward_func/mean": 0.028797149658203125,
+      "rewards/question_recreation_reward_func/std": 0.02207007259130478,
       "rewards/soft_format_reward_func/mean": 0.0,
       "rewards/soft_format_reward_func/std": 0.0,
       "rewards/strict_format_reward_func/mean": 0.0,
@@ -176,21 +176,21 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": 0.0,
-      "completions/max_length": 137.0,
-      "completions/max_terminated_length": 137.0,
-      "completions/mean_length": 49.0,
-      "completions/mean_terminated_length": 49.0,
-      "completions/min_length": 18.5,
-      "completions/min_terminated_length": 18.5,
-      "epoch": 0.5,
+      "completions/max_length": 64.5,
+      "completions/max_terminated_length": 64.5,
+      "completions/mean_length": 46.25,
+      "completions/mean_terminated_length": 46.25,
+      "completions/min_length": 24.5,
+      "completions/min_terminated_length": 24.5,
+      "epoch": 0.46511627906976744,
       "frac_reward_zero_std": 0.25,
-      "grad_norm": 83.61050415039062,
-      "kl": 0.010740740879555233,
+      "grad_norm": 31.20872688293457,
+      "kl": 0.014436870347708464,
       "learning_rate": 1.507684480352292e-08,
-      "loss": 0.0596,
-      "num_tokens": 6237.0,
-      "reward": 0.02530778758227825,
-      "reward_std": 0.007394111715257168,
+      "loss": -0.0165,
+      "num_tokens": 5957.0,
+      "reward": 0.04023433290421963,
+      "reward_std": 0.024168971925973892,
       "rewards/concensus_correctness_reward_func/mean": 0.0,
       "rewards/concensus_correctness_reward_func/std": 0.0,
       "rewards/consensus_reward_func/mean": 0.0,
@@ -199,8 +199,8 @@
       "rewards/cumulative_reward_2/std": 0.0,
       "rewards/final_correctness_reward_func/mean": 0.0,
       "rewards/final_correctness_reward_func/std": 0.0,
-      "rewards/question_recreation_reward_func/mean": 0.025307788513600826,
-      "rewards/question_recreation_reward_func/std": 0.009072140324860811,
+      "rewards/question_recreation_reward_func/mean": 0.04023433104157448,
+      "rewards/question_recreation_reward_func/std": 0.03306303359568119,
       "rewards/soft_format_reward_func/mean": 0.0,
       "rewards/soft_format_reward_func/std": 0.0,
       "rewards/strict_format_reward_func/mean": 0.0,
@@ -210,18 +210,18 @@
       "step": 10
     },
     {
-      "epoch": 0.5,
+      "epoch": 0.46511627906976744,
       "step": 10,
       "total_flos": 0.0,
-      "train_loss": -0.016702868044376373,
-      "train_runtime": 701.409,
-      "train_samples_per_second": 0.057,
-      "train_steps_per_second": 0.014
+      "train_loss": -0.06971818171441554,
+      "train_runtime": 764.1528,
+      "train_samples_per_second": 0.052,
+      "train_steps_per_second": 0.013
     }
   ],
   "logging_steps": 2,
   "max_steps": 10,
-  "num_input_tokens_seen": 6237,
+  "num_input_tokens_seen": 5957,
   "num_train_epochs": 1,
   "save_steps": 10,
   "stateful_callbacks": {