End of training

all_results.json

@@ -1,8 +1,8 @@
 {
     "total_flos": 0.0,
-    "train_loss": 0.056733327358961104,
-    "train_runtime": 778.6822,
-    "train_samples": 49,
+    "train_loss": 0.1390644297003746,
+    "train_runtime": 786.0915,
+    "train_samples": 36,
     "train_samples_per_second": 0.051,
     "train_steps_per_second": 0.013
 }

model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:fedc5f020be8194eb5cea3c7ebfea8065867f92bd11479c4f6aa5a2cc53947e5
+oid sha256:b18a9716f6f36cfe6821cae0a0917f30b3d279710ce2c87bce6aec636a933dcf
 size 1976163472

train_results.json

@@ -1,8 +1,8 @@
 {
     "total_flos": 0.0,
-    "train_loss": 0.056733327358961104,
-    "train_runtime": 778.6822,
-    "train_samples": 49,
+    "train_loss": 0.1390644297003746,
+    "train_runtime": 786.0915,
+    "train_samples": 36,
     "train_samples_per_second": 0.051,
     "train_steps_per_second": 0.013
 }

trainer_state.json

@@ -2,7 +2,7 @@
   "best_global_step": null,
   "best_metric": null,
   "best_model_checkpoint": null,
-  "epoch": 0.40816326530612246,
+  "epoch": 0.5555555555555556,
   "eval_steps": 500,
   "global_step": 10,
   "is_hyper_param_search": false,
@@ -15,22 +15,22 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": 0.125,
-      "completions/max_length": 172.0,
-      "completions/max_terminated_length": 71.0,
-      "completions/mean_length": 70.25,
-      "completions/mean_terminated_length": 43.66666603088379,
-      "completions/min_length": 29.0,
-      "completions/min_terminated_length": 29.0,
-      "epoch": 0.08163265306122448,
+      "completions/clipped_ratio": 0.0,
+      "completions/max_length": 92.0,
+      "completions/max_terminated_length": 92.0,
+      "completions/mean_length": 61.875,
+      "completions/mean_terminated_length": 61.875,
+      "completions/min_length": 21.5,
+      "completions/min_terminated_length": 21.5,
+      "epoch": 0.1111111111111111,
       "frac_reward_zero_std": 0.0,
-      "grad_norm": 41.77123260498047,
+      "grad_norm": 34.5233154296875,
       "kl": 0.0,
       "learning_rate": 5e-07,
-      "loss": 0.1625,
-      "num_tokens": 1586.0,
-      "reward": 0.29884233325719833,
-      "reward_std": 0.06489030597731471,
+      "loss": 0.1811,
+      "num_tokens": 1519.0,
+      "reward": 0.3038717731833458,
+      "reward_std": 0.12032702565193176,
       "rewards/concensus_correctness_reward_func/mean": 0.0,
       "rewards/concensus_correctness_reward_func/std": 0.0,
       "rewards/consensus_reward_func/mean": 0.0,
@@ -39,14 +39,14 @@
       "rewards/cumulative_reward_2/std": 0.0,
       "rewards/final_correctness_reward_func/mean": 0.0,
       "rewards/final_correctness_reward_func/std": 0.0,
-      "rewards/question_recreation_reward_func/mean": 0.020842326804995537,
-      "rewards/question_recreation_reward_func/std": 0.01152020413428545,
+      "rewards/question_recreation_reward_func/mean": 0.00999677274376154,
+      "rewards/question_recreation_reward_func/std": 0.00840075733140111,
       "rewards/soft_format_reward_func/mean": 0.0,
       "rewards/soft_format_reward_func/std": 0.0,
       "rewards/strict_format_reward_func/mean": 0.0,
       "rewards/strict_format_reward_func/std": 0.0,
-      "rewards/xmlcount_reward_func/mean": 0.27799999713897705,
-      "rewards/xmlcount_reward_func/std": 0.1191326342523098,
+      "rewards/xmlcount_reward_func/mean": 0.29387499392032623,
+      "rewards/xmlcount_reward_func/std": 0.1346326470375061,
       "step": 2
     },
     {
@@ -56,37 +56,37 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": 0.0,
-      "completions/max_length": 79.5,
-      "completions/max_terminated_length": 79.5,
-      "completions/mean_length": 40.75,
-      "completions/mean_terminated_length": 40.75,
-      "completions/min_length": 23.0,
-      "completions/min_terminated_length": 23.0,
-      "epoch": 0.16326530612244897,
-      "frac_reward_zero_std": 0.25,
-      "grad_norm": 31.240802764892578,
-      "kl": 0.011891998292412609,
+      "completions/max_length": 79.0,
+      "completions/max_terminated_length": 79.0,
+      "completions/mean_length": 56.625,
+      "completions/mean_terminated_length": 56.625,
+      "completions/min_length": 34.0,
+      "completions/min_terminated_length": 34.0,
+      "epoch": 0.2222222222222222,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 54.46159744262695,
+      "kl": 0.010409117210656404,
       "learning_rate": 4.415111107797445e-07,
-      "loss": -0.0471,
-      "num_tokens": 2936.0,
-      "reward": 0.8488049805164337,
-      "reward_std": 0.0757781732827425,
+      "loss": -0.0001,
+      "num_tokens": 2996.0,
+      "reward": 0.30906572937965393,
+      "reward_std": 0.09662564098834991,
       "rewards/concensus_correctness_reward_func/mean": 0.0,
       "rewards/concensus_correctness_reward_func/std": 0.0,
       "rewards/consensus_reward_func/mean": 0.0,
       "rewards/consensus_reward_func/std": 0.0,
       "rewards/cumulative_reward_2/mean": 0.0,
       "rewards/cumulative_reward_2/std": 0.0,
-      "rewards/final_correctness_reward_func/mean": 0.5,
-      "rewards/final_correctness_reward_func/std": 0.5773502588272095,
-      "rewards/question_recreation_reward_func/mean": 0.025055013597011566,
-      "rewards/question_recreation_reward_func/std": 0.02120710676535964,
+      "rewards/final_correctness_reward_func/mean": 0.0,
+      "rewards/final_correctness_reward_func/std": 0.0,
+      "rewards/question_recreation_reward_func/mean": 0.025940729305148125,
+      "rewards/question_recreation_reward_func/std": 0.012318885885179043,
       "rewards/soft_format_reward_func/mean": 0.0,
       "rewards/soft_format_reward_func/std": 0.0,
       "rewards/strict_format_reward_func/mean": 0.0,
       "rewards/strict_format_reward_func/std": 0.0,
-      "rewards/xmlcount_reward_func/mean": 0.32375000417232513,
-      "rewards/xmlcount_reward_func/std": 0.1294357106089592,
+      "rewards/xmlcount_reward_func/mean": 0.28312499821186066,
+      "rewards/xmlcount_reward_func/std": 0.08894590102136135,
       "step": 4
     },
     {
@@ -96,37 +96,37 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": 0.0,
-      "completions/max_length": 68.5,
-      "completions/max_terminated_length": 68.5,
-      "completions/mean_length": 35.125,
-      "completions/mean_terminated_length": 35.125,
-      "completions/min_length": 22.5,
-      "completions/min_terminated_length": 22.5,
-      "epoch": 0.24489795918367346,
+      "completions/max_length": 82.0,
+      "completions/max_terminated_length": 82.0,
+      "completions/mean_length": 46.375,
+      "completions/mean_terminated_length": 46.375,
+      "completions/min_length": 24.5,
+      "completions/min_terminated_length": 24.5,
+      "epoch": 0.3333333333333333,
       "frac_reward_zero_std": 0.0,
-      "grad_norm": 60.871788024902344,
-      "kl": 0.05835737707093358,
+      "grad_norm": 51.68680953979492,
+      "kl": 0.016515322029590607,
       "learning_rate": 2.934120444167326e-07,
-      "loss": 0.0565,
-      "num_tokens": 4241.0,
-      "reward": 0.888050839304924,
-      "reward_std": 0.056016623973846436,
+      "loss": 0.2082,
+      "num_tokens": 4391.0,
+      "reward": 0.2931773364543915,
+      "reward_std": 0.18366657197475433,
       "rewards/concensus_correctness_reward_func/mean": 0.0,
       "rewards/concensus_correctness_reward_func/std": 0.0,
       "rewards/consensus_reward_func/mean": 0.0,
       "rewards/consensus_reward_func/std": 0.0,
       "rewards/cumulative_reward_2/mean": 0.0,
       "rewards/cumulative_reward_2/std": 0.0,
-      "rewards/final_correctness_reward_func/mean": 0.5,
-      "rewards/final_correctness_reward_func/std": 0.5773502588272095,
-      "rewards/question_recreation_reward_func/mean": 0.024800879880785942,
-      "rewards/question_recreation_reward_func/std": 0.010365220019593835,
+      "rewards/final_correctness_reward_func/mean": 0.0,
+      "rewards/final_correctness_reward_func/std": 0.0,
+      "rewards/question_recreation_reward_func/mean": 0.01792734395712614,
+      "rewards/question_recreation_reward_func/std": 0.016361330170184374,
       "rewards/soft_format_reward_func/mean": 0.0,
       "rewards/soft_format_reward_func/std": 0.0,
       "rewards/strict_format_reward_func/mean": 0.0,
       "rewards/strict_format_reward_func/std": 0.0,
-      "rewards/xmlcount_reward_func/mean": 0.3632499873638153,
-      "rewards/xmlcount_reward_func/std": 0.07748057693243027,
+      "rewards/xmlcount_reward_func/mean": 0.27525000274181366,
+      "rewards/xmlcount_reward_func/std": 0.18220577389001846,
       "step": 6
     },
     {
@@ -136,21 +136,21 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": 0.0,
-      "completions/max_length": 81.5,
-      "completions/max_terminated_length": 81.5,
-      "completions/mean_length": 39.625,
-      "completions/mean_terminated_length": 39.625,
-      "completions/min_length": 19.0,
-      "completions/min_terminated_length": 19.0,
-      "epoch": 0.32653061224489793,
-      "frac_reward_zero_std": 0.25,
-      "grad_norm": 45.16495895385742,
-      "kl": 0.07873396738432348,
+      "completions/max_length": 74.5,
+      "completions/max_terminated_length": 74.5,
+      "completions/mean_length": 39.875,
+      "completions/mean_terminated_length": 39.875,
+      "completions/min_length": 19.5,
+      "completions/min_terminated_length": 19.5,
+      "epoch": 0.4444444444444444,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 71.04228210449219,
+      "kl": 0.027144306804984808,
       "learning_rate": 1.2500000000000005e-07,
-      "loss": 0.0947,
-      "num_tokens": 5582.0,
-      "reward": 0.4169105738401413,
-      "reward_std": 0.05461369641125202,
+      "loss": 0.1933,
+      "num_tokens": 5734.0,
+      "reward": 0.32249294221401215,
+      "reward_std": 0.12439239956438541,
       "rewards/concensus_correctness_reward_func/mean": 0.0,
       "rewards/concensus_correctness_reward_func/std": 0.0,
       "rewards/consensus_reward_func/mean": 0.0,
@@ -159,14 +159,14 @@
       "rewards/cumulative_reward_2/std": 0.0,
       "rewards/final_correctness_reward_func/mean": 0.0,
       "rewards/final_correctness_reward_func/std": 0.0,
-      "rewards/question_recreation_reward_func/mean": 0.08241057302802801,
-      "rewards/question_recreation_reward_func/std": 0.024038489558734,
+      "rewards/question_recreation_reward_func/mean": 0.014992930926382542,
+      "rewards/question_recreation_reward_func/std": 0.006889663287438452,
       "rewards/soft_format_reward_func/mean": 0.0,
       "rewards/soft_format_reward_func/std": 0.0,
       "rewards/strict_format_reward_func/mean": 0.0,
       "rewards/strict_format_reward_func/std": 0.0,
-      "rewards/xmlcount_reward_func/mean": 0.3344999849796295,
-      "rewards/xmlcount_reward_func/std": 0.0637132003903389,
+      "rewards/xmlcount_reward_func/mean": 0.3075000047683716,
+      "rewards/xmlcount_reward_func/std": 0.13611583784222603,
       "step": 8
     },
     {
@@ -176,21 +176,21 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": 0.0,
-      "completions/max_length": 51.5,
-      "completions/max_terminated_length": 51.5,
-      "completions/mean_length": 35.0,
-      "completions/mean_terminated_length": 35.0,
+      "completions/max_length": 62.5,
+      "completions/max_terminated_length": 62.5,
+      "completions/mean_length": 33.0,
+      "completions/mean_terminated_length": 33.0,
       "completions/min_length": 22.0,
       "completions/min_terminated_length": 22.0,
-      "epoch": 0.40816326530612246,
+      "epoch": 0.5555555555555556,
       "frac_reward_zero_std": 0.25,
-      "grad_norm": 39.669822692871094,
-      "kl": 0.12252630665898323,
+      "grad_norm": 23.007801055908203,
+      "kl": 0.017435546033084393,
       "learning_rate": 1.507684480352292e-08,
-      "loss": 0.017,
-      "num_tokens": 6886.0,
-      "reward": 0.5029706656932831,
-      "reward_std": 0.015668265521526337,
+      "loss": 0.1128,
+      "num_tokens": 7022.0,
+      "reward": 0.350043848156929,
+      "reward_std": 0.011826190399006009,
       "rewards/concensus_correctness_reward_func/mean": 0.0,
       "rewards/concensus_correctness_reward_func/std": 0.0,
       "rewards/consensus_reward_func/mean": 0.0,
@@ -199,29 +199,29 @@
       "rewards/cumulative_reward_2/std": 0.0,
       "rewards/final_correctness_reward_func/mean": 0.0,
       "rewards/final_correctness_reward_func/std": 0.0,
-      "rewards/question_recreation_reward_func/mean": 0.16784566640853882,
-      "rewards/question_recreation_reward_func/std": 0.038624364882707596,
+      "rewards/question_recreation_reward_func/mean": 0.010043838061392307,
+      "rewards/question_recreation_reward_func/std": 0.003311631502583623,
       "rewards/soft_format_reward_func/mean": 0.0,
       "rewards/soft_format_reward_func/std": 0.0,
       "rewards/strict_format_reward_func/mean": 0.0,
       "rewards/strict_format_reward_func/std": 0.0,
-      "rewards/xmlcount_reward_func/mean": 0.33512498438358307,
-      "rewards/xmlcount_reward_func/std": 0.07968567498028278,
+      "rewards/xmlcount_reward_func/mean": 0.3400000035762787,
+      "rewards/xmlcount_reward_func/std": 0.10695708636194468,
       "step": 10
     },
     {
-      "epoch": 0.40816326530612246,
+      "epoch": 0.5555555555555556,
       "step": 10,
       "total_flos": 0.0,
-      "train_loss": 0.056733327358961104,
-      "train_runtime": 778.6822,
+      "train_loss": 0.1390644297003746,
+      "train_runtime": 786.0915,
       "train_samples_per_second": 0.051,
       "train_steps_per_second": 0.013
     }
   ],
   "logging_steps": 2,
   "max_steps": 10,
-  "num_input_tokens_seen": 6886,
+  "num_input_tokens_seen": 7022,
   "num_train_epochs": 1,
   "save_steps": 10,
   "stateful_callbacks": {