End of training

all_results.json

@@ -1,8 +1,8 @@
 {
     "total_flos": 0.0,
-    "train_loss": 3.0167182531126714e-06,
-    "train_runtime": 698.9124,
+    "train_loss": 2.266971819153696e-08,
+    "train_runtime": 634.082,
     "train_samples": 5,
-    "train_samples_per_second": 0.057,
-    "train_steps_per_second": 0.014
+    "train_samples_per_second": 0.063,
+    "train_steps_per_second": 0.016
 }

model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:6ac3a4c3796a58fa7809b52cae817c5ba173d2f561e63769b5606ff63be467c8
+oid sha256:b2eb00926fd309dca5c3474543ba4365b9ca1b3bfc73819477a04df4968d1772
 size 1976163472

train_results.json

@@ -1,8 +1,8 @@
 {
     "total_flos": 0.0,
-    "train_loss": 3.0167182531126714e-06,
-    "train_runtime": 698.9124,
+    "train_loss": 2.266971819153696e-08,
+    "train_runtime": 634.082,
     "train_samples": 5,
-    "train_samples_per_second": 0.057,
-    "train_steps_per_second": 0.014
+    "train_samples_per_second": 0.063,
+    "train_steps_per_second": 0.016
 }

trainer_state.json

@@ -10,95 +10,95 @@
   "is_world_process_zero": true,
   "log_history": [
     {
-      "completion_length": 81.125,
+      "completion_length": 61.5,
       "epoch": 0.8,
-      "grad_norm": 5.711555480957031,
+      "grad_norm": 0.0,
       "kl": 0.0,
       "learning_rate": 5e-07,
       "loss": -0.0,
-      "reward": 1.3288777396082878,
-      "reward_std": 0.013100428506731987,
+      "reward": 1.796084702014923,
+      "reward_std": 0.007014752831310034,
       "rewards/concensus_correctness_reward_func": 0.0,
       "rewards/consensus_reward_func": 0.0,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 1.0,
-      "rewards/question_recreation_reward_func": 0.20387776382267475,
+      "rewards/final_correctness_reward_func": 1.5,
+      "rewards/question_recreation_reward_func": 0.17108468152582645,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
       "rewards/xmlcount_reward_func": 0.125,
       "step": 2
     },
     {
-      "completion_length": 61.0,
+      "completion_length": 90.5,
       "epoch": 1.4,
-      "grad_norm": 2.062848761852365e-05,
-      "kl": 1.8187058306769661e-06,
+      "grad_norm": 6.7355570793151855,
+      "kl": 0.0006590462605042072,
       "learning_rate": 4.415111107797445e-07,
       "loss": 0.0,
-      "reward": 2.636155287424723,
-      "reward_std": 0.0,
+      "reward": 2.4892822901407876,
+      "reward_std": 0.02238767563054959,
       "rewards/concensus_correctness_reward_func": 0.0,
       "rewards/consensus_reward_func": 0.0,
       "rewards/cumulative_reward_2": 0.0,
       "rewards/final_correctness_reward_func": 2.0,
-      "rewards/question_recreation_reward_func": 0.5111553370952606,
+      "rewards/question_recreation_reward_func": 0.36428221066792804,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
       "rewards/xmlcount_reward_func": 0.125,
       "step": 4
     },
     {
-      "completion_length": 86.66666666666667,
+      "completion_length": 61.0,
       "epoch": 2.0,
-      "grad_norm": 2.228761911392212,
-      "kl": 0.007802025240380317,
+      "grad_norm": 2.9161038398742676,
+      "kl": 0.0013541329723617916,
       "learning_rate": 2.934120444167326e-07,
-      "loss": -0.0,
-      "reward": 0.9451011121273041,
-      "reward_std": 0.011130627671567103,
+      "loss": 0.0,
+      "reward": 1.5693917274475098,
+      "reward_std": 0.0041435409026841325,
       "rewards/concensus_correctness_reward_func": 0.0,
       "rewards/consensus_reward_func": 0.0,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.6666666666666666,
-      "rewards/question_recreation_reward_func": 0.15343446532885233,
+      "rewards/final_correctness_reward_func": 1.3333333333333333,
+      "rewards/question_recreation_reward_func": 0.1110583816965421,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
       "rewards/xmlcount_reward_func": 0.125,
       "step": 6
     },
     {
-      "completion_length": 76.875,
+      "completion_length": 66.5,
       "epoch": 2.8,
-      "grad_norm": 6.640476703643799,
-      "kl": 0.007669217509828741,
+      "grad_norm": 5.4519829750061035,
+      "kl": 0.0025089465470955474,
       "learning_rate": 1.2500000000000005e-07,
       "loss": 0.0,
-      "reward": 1.3868866637349129,
-      "reward_std": 0.00798460966325365,
+      "reward": 1.8168215155601501,
+      "reward_std": 0.020925488555803895,
       "rewards/concensus_correctness_reward_func": 0.0,
       "rewards/consensus_reward_func": 0.0,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 1.0,
-      "rewards/question_recreation_reward_func": 0.26188666746020317,
+      "rewards/final_correctness_reward_func": 1.5,
+      "rewards/question_recreation_reward_func": 0.19182147271931171,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
       "rewards/xmlcount_reward_func": 0.125,
       "step": 8
     },
     {
-      "completion_length": 93.0,
+      "completion_length": 54.666666666666664,
       "epoch": 3.4,
-      "grad_norm": 1.3355332612991333,
-      "kl": 0.0003495829805615358,
+      "grad_norm": 0.00030291583971120417,
+      "kl": 0.0006291301785192142,
       "learning_rate": 1.507684480352292e-08,
       "loss": 0.0,
-      "reward": 1.6964898606141408,
-      "reward_std": 0.0001928780305509766,
+      "reward": 1.649997631708781,
+      "reward_std": 0.0,
       "rewards/concensus_correctness_reward_func": 0.0,
       "rewards/consensus_reward_func": 0.0,
       "rewards/cumulative_reward_2": 0.0,
       "rewards/final_correctness_reward_func": 1.3333333333333333,
-      "rewards/question_recreation_reward_func": 0.23815655459960303,
+      "rewards/question_recreation_reward_func": 0.19166426112254462,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
       "rewards/xmlcount_reward_func": 0.125,
@@ -108,10 +108,10 @@
       "epoch": 3.4,
       "step": 10,
       "total_flos": 0.0,
-      "train_loss": 3.0167182531126714e-06,
-      "train_runtime": 698.9124,
-      "train_samples_per_second": 0.057,
-      "train_steps_per_second": 0.014
+      "train_loss": 2.266971819153696e-08,
+      "train_runtime": 634.082,
+      "train_samples_per_second": 0.063,
+      "train_steps_per_second": 0.016
     }
   ],
   "logging_steps": 2,