End of training

all_results.json

@@ -1,8 +1,8 @@
 {
     "total_flos": 0.0,
-    "train_loss": -0.03826482892036438,
-    "train_runtime": 975.4406,
-    "train_samples": 35,
-    "train_samples_per_second": 0.041,
-    "train_steps_per_second": 0.01
+    "train_loss": -0.004030822962522507,
+    "train_runtime": 861.6022,
+    "train_samples": 30,
+    "train_samples_per_second": 0.046,
+    "train_steps_per_second": 0.012
 }

model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:a60deac64403d059ffffad5de3b93783a82bc2afdbd44e424a35726d183979db
+oid sha256:15c958dcb66d2fa3d2f40661baa3d0b07e3a58bd5eec2d40040825b0555b4034
 size 1976163472

train_results.json

@@ -1,8 +1,8 @@
 {
     "total_flos": 0.0,
-    "train_loss": -0.03826482892036438,
-    "train_runtime": 975.4406,
-    "train_samples": 35,
-    "train_samples_per_second": 0.041,
-    "train_steps_per_second": 0.01
+    "train_loss": -0.004030822962522507,
+    "train_runtime": 861.6022,
+    "train_samples": 30,
+    "train_samples_per_second": 0.046,
+    "train_steps_per_second": 0.012
 }

trainer_state.json

@@ -2,7 +2,7 @@
   "best_global_step": null,
   "best_metric": null,
   "best_model_checkpoint": null,
-  "epoch": 0.5714285714285714,
+  "epoch": 0.6666666666666666,
   "eval_steps": 500,
   "global_step": 10,
   "is_hyper_param_search": false,
@@ -16,21 +16,21 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": 0.0,
-      "completions/max_length": 148.5,
-      "completions/max_terminated_length": 148.5,
-      "completions/mean_length": 53.125,
-      "completions/mean_terminated_length": 53.125,
-      "completions/min_length": 12.0,
-      "completions/min_terminated_length": 12.0,
-      "epoch": 0.11428571428571428,
+      "completions/max_length": 125.5,
+      "completions/max_terminated_length": 125.5,
+      "completions/mean_length": 49.375,
+      "completions/mean_terminated_length": 49.375,
+      "completions/min_length": 10.5,
+      "completions/min_terminated_length": 10.5,
+      "epoch": 0.13333333333333333,
       "frac_reward_zero_std": 0.0,
-      "grad_norm": 83.93975830078125,
+      "grad_norm": 53.73799514770508,
       "kl": 0.0,
       "learning_rate": 5e-07,
-      "loss": -0.0043,
-      "num_tokens": 1449.0,
-      "reward": 0.04567142203450203,
-      "reward_std": 0.03146282769739628,
+      "loss": -0.0839,
+      "num_tokens": 1419.0,
+      "reward": 0.05085550621151924,
+      "reward_std": 0.023044501431286335,
       "rewards/concensus_correctness_reward_func/mean": 0.0,
       "rewards/concensus_correctness_reward_func/std": 0.0,
       "rewards/consensus_reward_func/mean": 0.0,
@@ -39,8 +39,8 @@
       "rewards/cumulative_reward_2/std": 0.0,
       "rewards/final_correctness_reward_func/mean": 0.0,
       "rewards/final_correctness_reward_func/std": 0.0,
-      "rewards/question_recreation_reward_func/mean": 0.045671421103179455,
-      "rewards/question_recreation_reward_func/std": 0.03892342001199722,
+      "rewards/question_recreation_reward_func/mean": 0.05085550807416439,
+      "rewards/question_recreation_reward_func/std": 0.03716153930872679,
       "rewards/soft_format_reward_func/mean": 0.0,
       "rewards/soft_format_reward_func/std": 0.0,
       "rewards/strict_format_reward_func/mean": 0.0,
@@ -55,22 +55,22 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": 0.25,
-      "completions/max_length": 195.5,
-      "completions/max_terminated_length": 89.5,
-      "completions/mean_length": 114.25,
-      "completions/mean_terminated_length": 59.875,
-      "completions/min_length": 23.0,
-      "completions/min_terminated_length": 23.0,
-      "epoch": 0.22857142857142856,
+      "completions/clipped_ratio": 0.0,
+      "completions/max_length": 153.0,
+      "completions/max_terminated_length": 153.0,
+      "completions/mean_length": 62.875,
+      "completions/mean_terminated_length": 62.875,
+      "completions/min_length": 21.5,
+      "completions/min_terminated_length": 21.5,
+      "epoch": 0.26666666666666666,
       "frac_reward_zero_std": 0.0,
-      "grad_norm": 13.177401542663574,
-      "kl": 0.0011668150291370694,
+      "grad_norm": 42.25807571411133,
+      "kl": 0.006594628292077687,
       "learning_rate": 4.415111107797445e-07,
-      "loss": -0.0901,
-      "num_tokens": 3387.0,
-      "reward": 0.10457494854927063,
-      "reward_std": 0.03707591432612389,
+      "loss": 0.0386,
+      "num_tokens": 2946.0,
+      "reward": 0.16623521596193314,
+      "reward_std": 0.0614180862903595,
       "rewards/concensus_correctness_reward_func/mean": 0.0,
       "rewards/concensus_correctness_reward_func/std": 0.0,
       "rewards/consensus_reward_func/mean": 0.0,
@@ -79,8 +79,8 @@
       "rewards/cumulative_reward_2/std": 0.0,
       "rewards/final_correctness_reward_func/mean": 0.0,
       "rewards/final_correctness_reward_func/std": 0.0,
-      "rewards/question_recreation_reward_func/mean": 0.10457494854927063,
-      "rewards/question_recreation_reward_func/std": 0.048114391742274165,
+      "rewards/question_recreation_reward_func/mean": 0.16623521596193314,
+      "rewards/question_recreation_reward_func/std": 0.08241075463593006,
       "rewards/soft_format_reward_func/mean": 0.0,
       "rewards/soft_format_reward_func/std": 0.0,
       "rewards/strict_format_reward_func/mean": 0.0,
@@ -96,21 +96,21 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": 0.0,
-      "completions/max_length": 95.0,
-      "completions/max_terminated_length": 95.0,
-      "completions/mean_length": 46.125,
-      "completions/mean_terminated_length": 46.125,
-      "completions/min_length": 19.0,
-      "completions/min_terminated_length": 19.0,
-      "epoch": 0.34285714285714286,
+      "completions/max_length": 106.5,
+      "completions/max_terminated_length": 106.5,
+      "completions/mean_length": 48.125,
+      "completions/mean_terminated_length": 48.125,
+      "completions/min_length": 15.0,
+      "completions/min_terminated_length": 15.0,
+      "epoch": 0.4,
       "frac_reward_zero_std": 0.0,
-      "grad_norm": 38.18785095214844,
-      "kl": 0.00460785545874387,
+      "grad_norm": 25.941099166870117,
+      "kl": 0.008665428555104882,
       "learning_rate": 2.934120444167326e-07,
-      "loss": 0.1126,
-      "num_tokens": 4780.0,
-      "reward": 0.03618481941521168,
-      "reward_std": 0.015940708108246326,
+      "loss": 0.0405,
+      "num_tokens": 4355.0,
+      "reward": 0.1510972185060382,
+      "reward_std": 0.02434489829465747,
       "rewards/concensus_correctness_reward_func/mean": 0.0,
       "rewards/concensus_correctness_reward_func/std": 0.0,
       "rewards/consensus_reward_func/mean": 0.0,
@@ -119,8 +119,8 @@
       "rewards/cumulative_reward_2/std": 0.0,
       "rewards/final_correctness_reward_func/mean": 0.0,
       "rewards/final_correctness_reward_func/std": 0.0,
-      "rewards/question_recreation_reward_func/mean": 0.0361848184838891,
-      "rewards/question_recreation_reward_func/std": 0.01561805885285139,
+      "rewards/question_recreation_reward_func/mean": 0.15109721757471561,
+      "rewards/question_recreation_reward_func/std": 0.07022390188649297,
       "rewards/soft_format_reward_func/mean": 0.0,
       "rewards/soft_format_reward_func/std": 0.0,
       "rewards/strict_format_reward_func/mean": 0.0,
@@ -135,22 +135,22 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": 0.25,
-      "completions/max_length": 230.5,
-      "completions/max_terminated_length": 158.5,
-      "completions/mean_length": 125.5,
-      "completions/mean_terminated_length": 79.75,
-      "completions/min_length": 22.5,
-      "completions/min_terminated_length": 22.5,
-      "epoch": 0.45714285714285713,
+      "completions/clipped_ratio": 0.0,
+      "completions/max_length": 107.5,
+      "completions/max_terminated_length": 107.5,
+      "completions/mean_length": 49.125,
+      "completions/mean_terminated_length": 49.125,
+      "completions/min_length": 21.0,
+      "completions/min_terminated_length": 21.0,
+      "epoch": 0.5333333333333333,
       "frac_reward_zero_std": 0.0,
-      "grad_norm": 34.29578399658203,
-      "kl": 0.009755991166457534,
+      "grad_norm": 22.325267791748047,
+      "kl": 0.038245625502895564,
       "learning_rate": 1.2500000000000005e-07,
-      "loss": -0.0313,
-      "num_tokens": 6808.0,
-      "reward": -0.022816544398665428,
-      "reward_std": 0.07706129271537066,
+      "loss": 0.0087,
+      "num_tokens": 5772.0,
+      "reward": 0.20961301028728485,
+      "reward_std": 0.010687269910704345,
       "rewards/concensus_correctness_reward_func/mean": 0.0,
       "rewards/concensus_correctness_reward_func/std": 0.0,
       "rewards/consensus_reward_func/mean": 0.0,
@@ -159,14 +159,14 @@
       "rewards/cumulative_reward_2/std": 0.0,
       "rewards/final_correctness_reward_func/mean": 0.0,
       "rewards/final_correctness_reward_func/std": 0.0,
-      "rewards/question_recreation_reward_func/mean": 0.0246834522113204,
-      "rewards/question_recreation_reward_func/std": 0.012613118626177311,
+      "rewards/question_recreation_reward_func/mean": 0.20961301028728485,
+      "rewards/question_recreation_reward_func/std": 0.04553865268826485,
       "rewards/soft_format_reward_func/mean": 0.0,
       "rewards/soft_format_reward_func/std": 0.0,
       "rewards/strict_format_reward_func/mean": 0.0,
       "rewards/strict_format_reward_func/std": 0.0,
-      "rewards/xmlcount_reward_func/mean": -0.04749999940395355,
-      "rewards/xmlcount_reward_func/std": 0.0949999988079071,
+      "rewards/xmlcount_reward_func/mean": 0.0,
+      "rewards/xmlcount_reward_func/std": 0.0,
       "step": 8
     },
     {
@@ -176,21 +176,21 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": 0.0,
-      "completions/max_length": 139.5,
-      "completions/max_terminated_length": 139.5,
-      "completions/mean_length": 48.75,
-      "completions/mean_terminated_length": 48.75,
-      "completions/min_length": 8.5,
-      "completions/min_terminated_length": 8.5,
-      "epoch": 0.5714285714285714,
-      "frac_reward_zero_std": 0.0,
-      "grad_norm": 55.47910690307617,
-      "kl": 0.030410898034460843,
+      "completions/max_length": 37.0,
+      "completions/max_terminated_length": 37.0,
+      "completions/mean_length": 23.75,
+      "completions/mean_terminated_length": 23.75,
+      "completions/min_length": 10.0,
+      "completions/min_terminated_length": 10.0,
+      "epoch": 0.6666666666666666,
+      "frac_reward_zero_std": 0.25,
+      "grad_norm": 44.80072021484375,
+      "kl": 0.03419979359023273,
       "learning_rate": 1.507684480352292e-08,
-      "loss": -0.1783,
-      "num_tokens": 8222.0,
-      "reward": 0.09751012921333313,
-      "reward_std": 0.152525519952178,
+      "loss": -0.024,
+      "num_tokens": 6986.0,
+      "reward": 0.11464355699717999,
+      "reward_std": 0.01553453115047887,
       "rewards/concensus_correctness_reward_func/mean": 0.0,
       "rewards/concensus_correctness_reward_func/std": 0.0,
       "rewards/consensus_reward_func/mean": 0.0,
@@ -199,29 +199,29 @@
       "rewards/cumulative_reward_2/std": 0.0,
       "rewards/final_correctness_reward_func/mean": 0.0,
       "rewards/final_correctness_reward_func/std": 0.0,
-      "rewards/question_recreation_reward_func/mean": 0.17151013016700745,
-      "rewards/question_recreation_reward_func/std": 0.05669265054166317,
+      "rewards/question_recreation_reward_func/mean": 0.11464355699717999,
+      "rewards/question_recreation_reward_func/std": 0.039072669111192226,
       "rewards/soft_format_reward_func/mean": 0.0,
       "rewards/soft_format_reward_func/std": 0.0,
       "rewards/strict_format_reward_func/mean": 0.0,
       "rewards/strict_format_reward_func/std": 0.0,
-      "rewards/xmlcount_reward_func/mean": -0.07400000095367432,
-      "rewards/xmlcount_reward_func/std": 0.14800001680850983,
+      "rewards/xmlcount_reward_func/mean": 0.0,
+      "rewards/xmlcount_reward_func/std": 0.0,
       "step": 10
     },
     {
-      "epoch": 0.5714285714285714,
+      "epoch": 0.6666666666666666,
       "step": 10,
       "total_flos": 0.0,
-      "train_loss": -0.03826482892036438,
-      "train_runtime": 975.4406,
-      "train_samples_per_second": 0.041,
-      "train_steps_per_second": 0.01
+      "train_loss": -0.004030822962522507,
+      "train_runtime": 861.6022,
+      "train_samples_per_second": 0.046,
+      "train_steps_per_second": 0.012
     }
   ],
   "logging_steps": 2,
   "max_steps": 10,
-  "num_input_tokens_seen": 8222,
+  "num_input_tokens_seen": 6986,
   "num_train_epochs": 1,
   "save_steps": 10,
   "stateful_callbacks": {