| { |
| "best_global_step": null, |
| "best_metric": null, |
| "best_model_checkpoint": null, |
| "epoch": 0.9989528795811519, |
| "eval_steps": 200, |
| "global_step": 477, |
| "is_hyper_param_search": false, |
| "is_local_process_zero": true, |
| "is_world_process_zero": true, |
| "log_history": [ |
| { |
| "epoch": 0.0020942408376963353, |
| "grad_norm": 37.91044616699219, |
| "learning_rate": 0.0, |
| "logits/chosen": -0.5995081663131714, |
| "logits/rejected": -0.6144353747367859, |
| "logps/chosen": -267.5272216796875, |
| "logps/ref_chosen": -267.5935363769531, |
| "logps/ref_rejected": -204.2306671142578, |
| "logps/rejected": -204.23907470703125, |
| "loss": 43.6518, |
| "r_dpo/chosen_len": 257.75, |
| "r_dpo/length_delta": 47.875, |
| "r_dpo/regularization_term": 4.787500381469727, |
| "r_dpo/rejected_len": 209.875, |
| "step": 1 |
| }, |
| { |
| "epoch": 0.020942408376963352, |
| "grad_norm": 34.83302307128906, |
| "learning_rate": 9.375e-08, |
| "logits/chosen": -0.6325575709342957, |
| "logits/rejected": -0.6374133229255676, |
| "logps/chosen": -296.6266174316406, |
| "logps/ref_chosen": -296.63226318359375, |
| "logps/ref_rejected": -258.9539489746094, |
| "logps/rejected": -259.0103454589844, |
| "loss": 41.9209, |
| "r_dpo/chosen_len": 291.8680419921875, |
| "r_dpo/length_delta": 49.76388931274414, |
| "r_dpo/regularization_term": 4.976388931274414, |
| "r_dpo/rejected_len": 242.1041717529297, |
| "step": 10 |
| }, |
| { |
| "epoch": 0.041884816753926704, |
| "grad_norm": 35.58102035522461, |
| "learning_rate": 1.9791666666666664e-07, |
| "logits/chosen": -0.59663987159729, |
| "logits/rejected": -0.627370297908783, |
| "logps/chosen": -297.972412109375, |
| "logps/ref_chosen": -297.9349365234375, |
| "logps/ref_rejected": -256.9902648925781, |
| "logps/rejected": -257.0819396972656, |
| "loss": 41.1855, |
| "r_dpo/chosen_len": 291.29998779296875, |
| "r_dpo/length_delta": 52.89374923706055, |
| "r_dpo/regularization_term": 5.289375305175781, |
| "r_dpo/rejected_len": 238.40625, |
| "step": 20 |
| }, |
| { |
| "epoch": 0.06282722513089005, |
| "grad_norm": 40.33411407470703, |
| "learning_rate": 3.020833333333333e-07, |
| "logits/chosen": -0.6156569719314575, |
| "logits/rejected": -0.6075023412704468, |
| "logps/chosen": -278.87744140625, |
| "logps/ref_chosen": -278.64752197265625, |
| "logps/ref_rejected": -249.309814453125, |
| "logps/rejected": -249.558837890625, |
| "loss": 43.5709, |
| "r_dpo/chosen_len": 270.8812561035156, |
| "r_dpo/length_delta": 25.228124618530273, |
| "r_dpo/regularization_term": 2.522813081741333, |
| "r_dpo/rejected_len": 245.6531219482422, |
| "step": 30 |
| }, |
| { |
| "epoch": 0.08376963350785341, |
| "grad_norm": 32.58840560913086, |
| "learning_rate": 4.0625e-07, |
| "logits/chosen": -0.6117347478866577, |
| "logits/rejected": -0.6369837522506714, |
| "logps/chosen": -284.4401550292969, |
| "logps/ref_chosen": -283.49981689453125, |
| "logps/ref_rejected": -265.32733154296875, |
| "logps/rejected": -266.5763244628906, |
| "loss": 41.7009, |
| "r_dpo/chosen_len": 281.43438720703125, |
| "r_dpo/length_delta": 33.34375, |
| "r_dpo/regularization_term": 3.3343749046325684, |
| "r_dpo/rejected_len": 248.0906219482422, |
| "step": 40 |
| }, |
| { |
| "epoch": 0.10471204188481675, |
| "grad_norm": 37.238525390625, |
| "learning_rate": 4.999932966293553e-07, |
| "logits/chosen": -0.6101135611534119, |
| "logits/rejected": -0.651242733001709, |
| "logps/chosen": -283.24017333984375, |
| "logps/ref_chosen": -280.224365234375, |
| "logps/ref_rejected": -274.3541259765625, |
| "logps/rejected": -278.0672302246094, |
| "loss": 43.8578, |
| "r_dpo/chosen_len": 290.32501220703125, |
| "r_dpo/length_delta": 35.11249923706055, |
| "r_dpo/regularization_term": 3.5112500190734863, |
| "r_dpo/rejected_len": 255.21249389648438, |
| "step": 50 |
| }, |
| { |
| "epoch": 0.1256544502617801, |
| "grad_norm": 44.93247604370117, |
| "learning_rate": 4.991893270335525e-07, |
| "logits/chosen": -0.6015418767929077, |
| "logits/rejected": -0.6169525384902954, |
| "logps/chosen": -289.905029296875, |
| "logps/ref_chosen": -281.12664794921875, |
| "logps/ref_rejected": -259.86456298828125, |
| "logps/rejected": -269.1077575683594, |
| "loss": 46.1321, |
| "r_dpo/chosen_len": 273.953125, |
| "r_dpo/length_delta": 29.084375381469727, |
| "r_dpo/regularization_term": 2.908437728881836, |
| "r_dpo/rejected_len": 244.86874389648438, |
| "step": 60 |
| }, |
| { |
| "epoch": 0.14659685863874344, |
| "grad_norm": 45.56925964355469, |
| "learning_rate": 4.970496218214204e-07, |
| "logits/chosen": -0.6171663403511047, |
| "logits/rejected": -0.6253775954246521, |
| "logps/chosen": -308.27239990234375, |
| "logps/ref_chosen": -287.71063232421875, |
| "logps/ref_rejected": -276.839599609375, |
| "logps/rejected": -298.0381164550781, |
| "loss": 42.0532, |
| "r_dpo/chosen_len": 267.4937438964844, |
| "r_dpo/length_delta": 14.484375, |
| "r_dpo/regularization_term": 1.4484375715255737, |
| "r_dpo/rejected_len": 253.00936889648438, |
| "step": 70 |
| }, |
| { |
| "epoch": 0.16753926701570682, |
| "grad_norm": 55.37849426269531, |
| "learning_rate": 4.935856505068998e-07, |
| "logits/chosen": -0.5272337198257446, |
| "logits/rejected": -0.5274912118911743, |
| "logps/chosen": -323.0798645019531, |
| "logps/ref_chosen": -280.123046875, |
| "logps/ref_rejected": -258.8989562988281, |
| "logps/rejected": -299.3633117675781, |
| "loss": 44.0294, |
| "r_dpo/chosen_len": 267.4781188964844, |
| "r_dpo/length_delta": 32.46562576293945, |
| "r_dpo/regularization_term": 3.2465622425079346, |
| "r_dpo/rejected_len": 235.0124969482422, |
| "step": 80 |
| }, |
| { |
| "epoch": 0.18848167539267016, |
| "grad_norm": 100.971923828125, |
| "learning_rate": 4.8881598109976e-07, |
| "logits/chosen": -0.46989256143569946, |
| "logits/rejected": -0.482027530670166, |
| "logps/chosen": -357.15936279296875, |
| "logps/ref_chosen": -278.02545166015625, |
| "logps/ref_rejected": -251.0922393798828, |
| "logps/rejected": -327.29400634765625, |
| "loss": 41.8973, |
| "r_dpo/chosen_len": 274.20623779296875, |
| "r_dpo/length_delta": 44.97187423706055, |
| "r_dpo/regularization_term": 4.497187614440918, |
| "r_dpo/rejected_len": 229.234375, |
| "step": 90 |
| }, |
| { |
| "epoch": 0.2094240837696335, |
| "grad_norm": 248.96990966796875, |
| "learning_rate": 4.827661805750437e-07, |
| "logits/chosen": -0.40429097414016724, |
| "logits/rejected": -0.41918930411338806, |
| "logps/chosen": -423.6812438964844, |
| "logps/ref_chosen": -274.0089416503906, |
| "logps/ref_rejected": -274.14447021484375, |
| "logps/rejected": -434.80133056640625, |
| "loss": 40.9219, |
| "r_dpo/chosen_len": 275.3343811035156, |
| "r_dpo/length_delta": 21.912500381469727, |
| "r_dpo/regularization_term": 2.1912498474121094, |
| "r_dpo/rejected_len": 253.421875, |
| "step": 100 |
| }, |
| { |
| "epoch": 0.23036649214659685, |
| "grad_norm": 1346.3997802734375, |
| "learning_rate": 4.75468677825789e-07, |
| "logits/chosen": -0.35518646240234375, |
| "logits/rejected": -0.3851023316383362, |
| "logps/chosen": -763.7197265625, |
| "logps/ref_chosen": -273.23333740234375, |
| "logps/ref_rejected": -263.88787841796875, |
| "logps/rejected": -708.5206909179688, |
| "loss": 31.9332, |
| "r_dpo/chosen_len": 283.43438720703125, |
| "r_dpo/length_delta": 50.34375, |
| "r_dpo/regularization_term": 5.034375190734863, |
| "r_dpo/rejected_len": 233.0906219482422, |
| "step": 110 |
| }, |
| { |
| "epoch": 0.2513089005235602, |
| "grad_norm": 7287.37548828125, |
| "learning_rate": 4.669625898336438e-07, |
| "logits/chosen": 0.0007931709405966103, |
| "logits/rejected": -0.02699187956750393, |
| "logps/chosen": -2300.340576171875, |
| "logps/ref_chosen": -269.77142333984375, |
| "logps/ref_rejected": -272.7685546875, |
| "logps/rejected": -2165.2236328125, |
| "loss": 18.6284, |
| "r_dpo/chosen_len": 264.7593688964844, |
| "r_dpo/length_delta": 13.840624809265137, |
| "r_dpo/regularization_term": 1.3840625286102295, |
| "r_dpo/rejected_len": 250.9187469482422, |
| "step": 120 |
| }, |
| { |
| "epoch": 0.27225130890052357, |
| "grad_norm": 2478.686767578125, |
| "learning_rate": 4.5729351198915705e-07, |
| "logits/chosen": -0.08189640194177628, |
| "logits/rejected": -0.0916411355137825, |
| "logps/chosen": -2413.971923828125, |
| "logps/ref_chosen": -275.03448486328125, |
| "logps/ref_rejected": -276.39862060546875, |
| "logps/rejected": -2312.552734375, |
| "loss": 13.7684, |
| "r_dpo/chosen_len": 266.625, |
| "r_dpo/length_delta": 18.668750762939453, |
| "r_dpo/regularization_term": 1.8668749332427979, |
| "r_dpo/rejected_len": 247.9562530517578, |
| "step": 130 |
| }, |
| { |
| "epoch": 0.2931937172774869, |
| "grad_norm": 1929.59423828125, |
| "learning_rate": 4.4651327368569684e-07, |
| "logits/chosen": -0.08853980153799057, |
| "logits/rejected": -0.12103728950023651, |
| "logps/chosen": -2533.98876953125, |
| "logps/ref_chosen": -276.0029602050781, |
| "logps/ref_rejected": -255.9320526123047, |
| "logps/rejected": -2369.556884765625, |
| "loss": 10.996, |
| "r_dpo/chosen_len": 261.46875, |
| "r_dpo/length_delta": 22.375, |
| "r_dpo/regularization_term": 2.237499952316284, |
| "r_dpo/rejected_len": 239.09375, |
| "step": 140 |
| }, |
| { |
| "epoch": 0.31413612565445026, |
| "grad_norm": 1273.77880859375, |
| "learning_rate": 4.346796604970912e-07, |
| "logits/chosen": -0.03744601085782051, |
| "logits/rejected": -0.0864439308643341, |
| "logps/chosen": -2928.5224609375, |
| "logps/ref_chosen": -298.2093505859375, |
| "logps/ref_rejected": -254.8907012939453, |
| "logps/rejected": -2446.001953125, |
| "loss": 8.5066, |
| "r_dpo/chosen_len": 283.84375, |
| "r_dpo/length_delta": 48.359375, |
| "r_dpo/regularization_term": 4.8359375, |
| "r_dpo/rejected_len": 235.484375, |
| "step": 150 |
| }, |
| { |
| "epoch": 0.33507853403141363, |
| "grad_norm": 637.610595703125, |
| "learning_rate": 4.218561044282098e-07, |
| "logits/chosen": -0.11696837842464447, |
| "logits/rejected": -0.14894640445709229, |
| "logps/chosen": -2785.264892578125, |
| "logps/ref_chosen": -281.94189453125, |
| "logps/ref_rejected": -255.5653533935547, |
| "logps/rejected": -2388.72412109375, |
| "loss": 7.7028, |
| "r_dpo/chosen_len": 267.828125, |
| "r_dpo/length_delta": 41.368751525878906, |
| "r_dpo/regularization_term": 4.136875152587891, |
| "r_dpo/rejected_len": 226.45938110351562, |
| "step": 160 |
| }, |
| { |
| "epoch": 0.35602094240837695, |
| "grad_norm": 1146.212646484375, |
| "learning_rate": 4.081113438988443e-07, |
| "logits/chosen": -0.19355528056621552, |
| "logits/rejected": -0.21304742991924286, |
| "logps/chosen": -2857.443359375, |
| "logps/ref_chosen": -288.2863464355469, |
| "logps/ref_rejected": -239.758056640625, |
| "logps/rejected": -2423.08349609375, |
| "loss": 6.7267, |
| "r_dpo/chosen_len": 285.203125, |
| "r_dpo/length_delta": 46.396873474121094, |
| "r_dpo/regularization_term": 4.639687538146973, |
| "r_dpo/rejected_len": 238.80624389648438, |
| "step": 170 |
| }, |
| { |
| "epoch": 0.3769633507853403, |
| "grad_norm": 608.8386840820312, |
| "learning_rate": 3.935190552834828e-07, |
| "logits/chosen": -0.3072236180305481, |
| "logits/rejected": -0.32055360078811646, |
| "logps/chosen": -2654.16943359375, |
| "logps/ref_chosen": -286.17889404296875, |
| "logps/ref_rejected": -249.9820098876953, |
| "logps/rejected": -2274.89892578125, |
| "loss": 6.9642, |
| "r_dpo/chosen_len": 266.09063720703125, |
| "r_dpo/length_delta": 40.12812423706055, |
| "r_dpo/regularization_term": 4.012812614440918, |
| "r_dpo/rejected_len": 225.96249389648438, |
| "step": 180 |
| }, |
| { |
| "epoch": 0.39790575916230364, |
| "grad_norm": 977.245361328125, |
| "learning_rate": 3.781574579820464e-07, |
| "logits/chosen": -0.33928078413009644, |
| "logits/rejected": -0.34710997343063354, |
| "logps/chosen": -2732.35009765625, |
| "logps/ref_chosen": -280.9278259277344, |
| "logps/ref_rejected": -254.3533477783203, |
| "logps/rejected": -2360.5947265625, |
| "loss": 6.7039, |
| "r_dpo/chosen_len": 276.33123779296875, |
| "r_dpo/length_delta": 41.993751525878906, |
| "r_dpo/regularization_term": 4.199374675750732, |
| "r_dpo/rejected_len": 234.33749389648438, |
| "step": 190 |
| }, |
| { |
| "epoch": 0.418848167539267, |
| "grad_norm": 616.4851684570312, |
| "learning_rate": 3.621088951385353e-07, |
| "logits/chosen": -0.2376505434513092, |
| "logits/rejected": -0.25424009561538696, |
| "logps/chosen": -2470.09521484375, |
| "logps/ref_chosen": -253.1712188720703, |
| "logps/ref_rejected": -241.90478515625, |
| "logps/rejected": -2249.572021484375, |
| "loss": 6.4185, |
| "r_dpo/chosen_len": 248.0749969482422, |
| "r_dpo/length_delta": 28.131250381469727, |
| "r_dpo/regularization_term": 2.8131251335144043, |
| "r_dpo/rejected_len": 219.94375610351562, |
| "step": 200 |
| }, |
| { |
| "epoch": 0.418848167539267, |
| "eval_logits/chosen": -0.22856707870960236, |
| "eval_logits/rejected": -0.23529842495918274, |
| "eval_logps/chosen": -2812.3984375, |
| "eval_logps/ref_chosen": -288.6414794921875, |
| "eval_logps/ref_rejected": -265.96160888671875, |
| "eval_logps/rejected": -2464.037109375, |
| "eval_loss": 0.7758205533027649, |
| "eval_r_dpo/chosen_len": 286.97601318359375, |
| "eval_r_dpo/length_delta": 40.88800048828125, |
| "eval_r_dpo/regularization_term": 4.088800430297852, |
| "eval_r_dpo/rejected_len": 246.08799743652344, |
| "eval_runtime": 209.8859, |
| "eval_samples_per_second": 9.529, |
| "eval_steps_per_second": 1.191, |
| "step": 200 |
| }, |
| { |
| "epoch": 0.4397905759162304, |
| "grad_norm": 846.9074096679688, |
| "learning_rate": 3.454593922550693e-07, |
| "logits/chosen": -0.3170786499977112, |
| "logits/rejected": -0.32814693450927734, |
| "logps/chosen": -2884.038330078125, |
| "logps/ref_chosen": -287.9228210449219, |
| "logps/ref_rejected": -263.35595703125, |
| "logps/rejected": -2537.31689453125, |
| "loss": 6.545, |
| "r_dpo/chosen_len": 280.3125, |
| "r_dpo/length_delta": 36.68437576293945, |
| "r_dpo/regularization_term": 3.6684374809265137, |
| "r_dpo/rejected_len": 243.6281280517578, |
| "step": 210 |
| }, |
| { |
| "epoch": 0.4607329842931937, |
| "grad_norm": 546.0598754882812, |
| "learning_rate": 3.2829819606729477e-07, |
| "logits/chosen": -0.3490475118160248, |
| "logits/rejected": -0.35787373781204224, |
| "logps/chosen": -2633.925048828125, |
| "logps/ref_chosen": -282.3331604003906, |
| "logps/ref_rejected": -272.5645446777344, |
| "logps/rejected": -2502.832763671875, |
| "loss": 6.0607, |
| "r_dpo/chosen_len": 261.359375, |
| "r_dpo/length_delta": 17.865625381469727, |
| "r_dpo/regularization_term": 1.7865626811981201, |
| "r_dpo/rejected_len": 243.49374389648438, |
| "step": 220 |
| }, |
| { |
| "epoch": 0.4816753926701571, |
| "grad_norm": 405.8363037109375, |
| "learning_rate": 3.1071729615293424e-07, |
| "logits/chosen": -0.36600062251091003, |
| "logits/rejected": -0.3698672950267792, |
| "logps/chosen": -2655.687255859375, |
| "logps/ref_chosen": -276.1485595703125, |
| "logps/ref_rejected": -252.81198120117188, |
| "logps/rejected": -2387.78125, |
| "loss": 5.9948, |
| "r_dpo/chosen_len": 264.43438720703125, |
| "r_dpo/length_delta": 31.256250381469727, |
| "r_dpo/regularization_term": 3.1256251335144043, |
| "r_dpo/rejected_len": 233.17813110351562, |
| "step": 230 |
| }, |
| { |
| "epoch": 0.5026178010471204, |
| "grad_norm": 502.72357177734375, |
| "learning_rate": 2.9281093183781403e-07, |
| "logits/chosen": -0.3056614398956299, |
| "logits/rejected": -0.3123835325241089, |
| "logps/chosen": -2676.06982421875, |
| "logps/ref_chosen": -270.52520751953125, |
| "logps/ref_rejected": -254.83334350585938, |
| "logps/rejected": -2375.5478515625, |
| "loss": 6.2228, |
| "r_dpo/chosen_len": 271.81561279296875, |
| "r_dpo/length_delta": 37.099998474121094, |
| "r_dpo/regularization_term": 3.7100002765655518, |
| "r_dpo/rejected_len": 234.7156219482422, |
| "step": 240 |
| }, |
| { |
| "epoch": 0.5235602094240838, |
| "grad_norm": 619.77783203125, |
| "learning_rate": 2.7467508704251135e-07, |
| "logits/chosen": -0.33823755383491516, |
| "logits/rejected": -0.33679407835006714, |
| "logps/chosen": -2791.576171875, |
| "logps/ref_chosen": -289.6054992675781, |
| "logps/ref_rejected": -265.0482482910156, |
| "logps/rejected": -2412.063720703125, |
| "loss": 6.0215, |
| "r_dpo/chosen_len": 277.50311279296875, |
| "r_dpo/length_delta": 41.103126525878906, |
| "r_dpo/regularization_term": 4.110312461853027, |
| "r_dpo/rejected_len": 236.39999389648438, |
| "step": 250 |
| }, |
| { |
| "epoch": 0.5445026178010471, |
| "grad_norm": 730.1420288085938, |
| "learning_rate": 2.5640697577740815e-07, |
| "logits/chosen": -0.3466644883155823, |
| "logits/rejected": -0.34878239035606384, |
| "logps/chosen": -2722.85693359375, |
| "logps/ref_chosen": -288.6393737792969, |
| "logps/ref_rejected": -265.315673828125, |
| "logps/rejected": -2532.46484375, |
| "loss": 5.8667, |
| "r_dpo/chosen_len": 271.48126220703125, |
| "r_dpo/length_delta": 24.390625, |
| "r_dpo/regularization_term": 2.4390625953674316, |
| "r_dpo/rejected_len": 247.0906219482422, |
| "step": 260 |
| }, |
| { |
| "epoch": 0.5654450261780105, |
| "grad_norm": 586.3690795898438, |
| "learning_rate": 2.381045210440644e-07, |
| "logits/chosen": -0.3886115252971649, |
| "logits/rejected": -0.3948022127151489, |
| "logps/chosen": -2752.974853515625, |
| "logps/ref_chosen": -280.1373596191406, |
| "logps/ref_rejected": -264.84295654296875, |
| "logps/rejected": -2619.281494140625, |
| "loss": 5.7602, |
| "r_dpo/chosen_len": 272.2875061035156, |
| "r_dpo/length_delta": 19.956249237060547, |
| "r_dpo/regularization_term": 1.9956251382827759, |
| "r_dpo/rejected_len": 252.3312530517578, |
| "step": 270 |
| }, |
| { |
| "epoch": 0.5863874345549738, |
| "grad_norm": 440.32672119140625, |
| "learning_rate": 2.1986582993616925e-07, |
| "logits/chosen": -0.38749268651008606, |
| "logits/rejected": -0.3799517750740051, |
| "logps/chosen": -2869.620849609375, |
| "logps/ref_chosen": -301.7547912597656, |
| "logps/ref_rejected": -254.6543731689453, |
| "logps/rejected": -2384.27294921875, |
| "loss": 5.5612, |
| "r_dpo/chosen_len": 285.44061279296875, |
| "r_dpo/length_delta": 52.962501525878906, |
| "r_dpo/regularization_term": 5.296250343322754, |
| "r_dpo/rejected_len": 232.47811889648438, |
| "step": 280 |
| }, |
| { |
| "epoch": 0.6073298429319371, |
| "grad_norm": 649.3321533203125, |
| "learning_rate": 2.0178866775369774e-07, |
| "logits/chosen": -0.39530712366104126, |
| "logits/rejected": -0.3784952759742737, |
| "logps/chosen": -2955.660400390625, |
| "logps/ref_chosen": -302.79217529296875, |
| "logps/ref_rejected": -292.9220275878906, |
| "logps/rejected": -2789.33447265625, |
| "loss": 5.8331, |
| "r_dpo/chosen_len": 294.90625, |
| "r_dpo/length_delta": 20.774999618530273, |
| "r_dpo/regularization_term": 2.077500104904175, |
| "r_dpo/rejected_len": 274.1312561035156, |
| "step": 290 |
| }, |
| { |
| "epoch": 0.6282722513089005, |
| "grad_norm": 495.6314697265625, |
| "learning_rate": 1.839699339491937e-07, |
| "logits/chosen": -0.3710184097290039, |
| "logits/rejected": -0.36620840430259705, |
| "logps/chosen": -2728.482421875, |
| "logps/ref_chosen": -275.8238220214844, |
| "logps/ref_rejected": -264.05743408203125, |
| "logps/rejected": -2575.94580078125, |
| "loss": 5.8295, |
| "r_dpo/chosen_len": 266.859375, |
| "r_dpo/length_delta": 20.734375, |
| "r_dpo/regularization_term": 2.073437452316284, |
| "r_dpo/rejected_len": 246.125, |
| "step": 300 |
| }, |
| { |
| "epoch": 0.6492146596858639, |
| "grad_norm": 494.1045837402344, |
| "learning_rate": 1.6650514271527465e-07, |
| "logits/chosen": -0.3966061472892761, |
| "logits/rejected": -0.3880545496940613, |
| "logps/chosen": -3014.114013671875, |
| "logps/ref_chosen": -296.6716003417969, |
| "logps/ref_rejected": -278.68426513671875, |
| "logps/rejected": -2734.437255859375, |
| "loss": 5.8626, |
| "r_dpo/chosen_len": 292.91876220703125, |
| "r_dpo/length_delta": 32.55937576293945, |
| "r_dpo/regularization_term": 3.2559375762939453, |
| "r_dpo/rejected_len": 260.359375, |
| "step": 310 |
| }, |
| { |
| "epoch": 0.6701570680628273, |
| "grad_norm": 792.8230590820312, |
| "learning_rate": 1.4948791099758052e-07, |
| "logits/chosen": -0.39138588309288025, |
| "logits/rejected": -0.3824613690376282, |
| "logps/chosen": -2841.270751953125, |
| "logps/ref_chosen": -284.1717529296875, |
| "logps/ref_rejected": -261.2606506347656, |
| "logps/rejected": -2460.640869140625, |
| "loss": 5.3257, |
| "r_dpo/chosen_len": 279.90313720703125, |
| "r_dpo/length_delta": 44.537498474121094, |
| "r_dpo/regularization_term": 4.453750133514404, |
| "r_dpo/rejected_len": 235.36563110351562, |
| "step": 320 |
| }, |
| { |
| "epoch": 0.6910994764397905, |
| "grad_norm": 460.28485107421875, |
| "learning_rate": 1.3300945667758012e-07, |
| "logits/chosen": -0.40215617418289185, |
| "logits/rejected": -0.40834659337997437, |
| "logps/chosen": -2728.718017578125, |
| "logps/ref_chosen": -283.40338134765625, |
| "logps/ref_rejected": -271.27569580078125, |
| "logps/rejected": -2660.712890625, |
| "loss": 5.4139, |
| "r_dpo/chosen_len": 267.67498779296875, |
| "r_dpo/length_delta": 13.015625, |
| "r_dpo/regularization_term": 1.3015625476837158, |
| "r_dpo/rejected_len": 254.6593780517578, |
| "step": 330 |
| }, |
| { |
| "epoch": 0.7120418848167539, |
| "grad_norm": 641.794189453125, |
| "learning_rate": 1.1715810961514072e-07, |
| "logits/chosen": -0.38855430483818054, |
| "logits/rejected": -0.37820714712142944, |
| "logps/chosen": -2629.27197265625, |
| "logps/ref_chosen": -259.7261962890625, |
| "logps/ref_rejected": -243.4088897705078, |
| "logps/rejected": -2364.59521484375, |
| "loss": 5.6332, |
| "r_dpo/chosen_len": 256.11248779296875, |
| "r_dpo/length_delta": 32.546875, |
| "r_dpo/regularization_term": 3.254687547683716, |
| "r_dpo/rejected_len": 223.5656280517578, |
| "step": 340 |
| }, |
| { |
| "epoch": 0.7329842931937173, |
| "grad_norm": 610.6142578125, |
| "learning_rate": 1.0201883817182949e-07, |
| "logits/chosen": -0.39518895745277405, |
| "logits/rejected": -0.3816191256046295, |
| "logps/chosen": -2879.398681640625, |
| "logps/ref_chosen": -298.24725341796875, |
| "logps/ref_rejected": -272.657958984375, |
| "logps/rejected": -2458.894775390625, |
| "loss": 5.4022, |
| "r_dpo/chosen_len": 281.4624938964844, |
| "r_dpo/length_delta": 45.275001525878906, |
| "r_dpo/regularization_term": 4.527500152587891, |
| "r_dpo/rejected_len": 236.1875, |
| "step": 350 |
| }, |
| { |
| "epoch": 0.7539267015706806, |
| "grad_norm": 533.6195678710938, |
| "learning_rate": 8.76727937529367e-08, |
| "logits/chosen": -0.36005398631095886, |
| "logits/rejected": -0.3565328121185303, |
| "logps/chosen": -2746.4658203125, |
| "logps/ref_chosen": -281.881103515625, |
| "logps/ref_rejected": -265.4746398925781, |
| "logps/rejected": -2505.47509765625, |
| "loss": 5.5109, |
| "r_dpo/chosen_len": 272.64373779296875, |
| "r_dpo/length_delta": 30.071874618530273, |
| "r_dpo/regularization_term": 3.0071873664855957, |
| "r_dpo/rejected_len": 242.57186889648438, |
| "step": 360 |
| }, |
| { |
| "epoch": 0.774869109947644, |
| "grad_norm": 462.19488525390625, |
| "learning_rate": 7.419687580962222e-08, |
| "logits/chosen": -0.4140191674232483, |
| "logits/rejected": -0.4048995077610016, |
| "logps/chosen": -2749.48779296875, |
| "logps/ref_chosen": -302.17822265625, |
| "logps/ref_rejected": -265.92877197265625, |
| "logps/rejected": -2479.15966796875, |
| "loss": 5.1316, |
| "r_dpo/chosen_len": 273.88751220703125, |
| "r_dpo/length_delta": 33.759376525878906, |
| "r_dpo/regularization_term": 3.3759372234344482, |
| "r_dpo/rejected_len": 240.1281280517578, |
| "step": 370 |
| }, |
| { |
| "epoch": 0.7958115183246073, |
| "grad_norm": 304.5794372558594, |
| "learning_rate": 6.166331963291519e-08, |
| "logits/chosen": -0.3827962875366211, |
| "logits/rejected": -0.3733735680580139, |
| "logps/chosen": -2883.859619140625, |
| "logps/ref_chosen": -301.2120361328125, |
| "logps/ref_rejected": -266.4872741699219, |
| "logps/rejected": -2585.99365234375, |
| "loss": 5.3235, |
| "r_dpo/chosen_len": 286.75311279296875, |
| "r_dpo/length_delta": 33.53125, |
| "r_dpo/regularization_term": 3.3531250953674316, |
| "r_dpo/rejected_len": 253.2218780517578, |
| "step": 380 |
| }, |
| { |
| "epoch": 0.8167539267015707, |
| "grad_norm": 505.1428527832031, |
| "learning_rate": 5.013930914912476e-08, |
| "logits/chosen": -0.39441806077957153, |
| "logits/rejected": -0.3891170024871826, |
| "logps/chosen": -2894.778076171875, |
| "logps/ref_chosen": -296.6472473144531, |
| "logps/ref_rejected": -278.953857421875, |
| "logps/rejected": -2663.129150390625, |
| "loss": 5.7398, |
| "r_dpo/chosen_len": 287.91876220703125, |
| "r_dpo/length_delta": 30.081249237060547, |
| "r_dpo/regularization_term": 3.008124828338623, |
| "r_dpo/rejected_len": 257.8374938964844, |
| "step": 390 |
| }, |
| { |
| "epoch": 0.837696335078534, |
| "grad_norm": 815.798828125, |
| "learning_rate": 3.968661679220467e-08, |
| "logits/chosen": -0.3783096373081207, |
| "logits/rejected": -0.37295296788215637, |
| "logps/chosen": -2797.49951171875, |
| "logps/ref_chosen": -296.6556091308594, |
| "logps/ref_rejected": -256.9266662597656, |
| "logps/rejected": -2443.60888671875, |
| "loss": 5.4191, |
| "r_dpo/chosen_len": 278.96875, |
| "r_dpo/length_delta": 39.60625076293945, |
| "r_dpo/regularization_term": 3.960625171661377, |
| "r_dpo/rejected_len": 239.3625030517578, |
| "step": 400 |
| }, |
| { |
| "epoch": 0.837696335078534, |
| "eval_logits/chosen": -0.3396916091442108, |
| "eval_logits/rejected": -0.3239772915840149, |
| "eval_logps/chosen": -2847.308349609375, |
| "eval_logps/ref_chosen": -288.6414794921875, |
| "eval_logps/ref_rejected": -265.96160888671875, |
| "eval_logps/rejected": -2499.736328125, |
| "eval_loss": 0.6649385690689087, |
| "eval_r_dpo/chosen_len": 286.97601318359375, |
| "eval_r_dpo/length_delta": 40.88800048828125, |
| "eval_r_dpo/regularization_term": 4.088800430297852, |
| "eval_r_dpo/rejected_len": 246.08799743652344, |
| "eval_runtime": 211.0607, |
| "eval_samples_per_second": 9.476, |
| "eval_steps_per_second": 1.184, |
| "step": 400 |
| }, |
| { |
| "epoch": 0.8586387434554974, |
| "grad_norm": 450.2701721191406, |
| "learning_rate": 3.036127238347164e-08, |
| "logits/chosen": -0.38624387979507446, |
| "logits/rejected": -0.3744073510169983, |
| "logps/chosen": -2840.36572265625, |
| "logps/ref_chosen": -289.9568786621094, |
| "logps/ref_rejected": -272.4674377441406, |
| "logps/rejected": -2619.307861328125, |
| "loss": 5.3372, |
| "r_dpo/chosen_len": 282.40625, |
| "r_dpo/length_delta": 26.265625, |
| "r_dpo/regularization_term": 2.6265625953674316, |
| "r_dpo/rejected_len": 256.140625, |
| "step": 410 |
| }, |
| { |
| "epoch": 0.8795811518324608, |
| "grad_norm": 457.4356384277344, |
| "learning_rate": 2.2213262793589482e-08, |
| "logits/chosen": -0.3804797828197479, |
| "logits/rejected": -0.36465829610824585, |
| "logps/chosen": -3006.153564453125, |
| "logps/ref_chosen": -307.40240478515625, |
| "logps/ref_rejected": -279.85760498046875, |
| "logps/rejected": -2696.711669921875, |
| "loss": 5.0272, |
| "r_dpo/chosen_len": 296.8343811035156, |
| "r_dpo/length_delta": 37.165626525878906, |
| "r_dpo/regularization_term": 3.7165627479553223, |
| "r_dpo/rejected_len": 259.66876220703125, |
| "step": 420 |
| }, |
| { |
| "epoch": 0.900523560209424, |
| "grad_norm": 489.8297119140625, |
| "learning_rate": 1.5286263996730026e-08, |
| "logits/chosen": -0.3834610879421234, |
| "logits/rejected": -0.37544751167297363, |
| "logps/chosen": -2919.646484375, |
| "logps/ref_chosen": -297.7133483886719, |
| "logps/ref_rejected": -266.862060546875, |
| "logps/rejected": -2492.005859375, |
| "loss": 5.2819, |
| "r_dpo/chosen_len": 290.703125, |
| "r_dpo/length_delta": 48.868751525878906, |
| "r_dpo/regularization_term": 4.886875152587891, |
| "r_dpo/rejected_len": 241.83438110351562, |
| "step": 430 |
| }, |
| { |
| "epoch": 0.9214659685863874, |
| "grad_norm": 458.5052795410156, |
| "learning_rate": 9.617406953185136e-09, |
| "logits/chosen": -0.38718608021736145, |
| "logits/rejected": -0.3853945732116699, |
| "logps/chosen": -2841.03955078125, |
| "logps/ref_chosen": -293.67095947265625, |
| "logps/ref_rejected": -289.4698791503906, |
| "logps/rejected": -2790.7744140625, |
| "loss": 5.4764, |
| "r_dpo/chosen_len": 285.3656311035156, |
| "r_dpo/length_delta": 12.868749618530273, |
| "r_dpo/regularization_term": 1.286874771118164, |
| "r_dpo/rejected_len": 272.49688720703125, |
| "step": 440 |
| }, |
| { |
| "epoch": 0.9424083769633508, |
| "grad_norm": 631.5553588867188, |
| "learning_rate": 5.2370785753763356e-09, |
| "logits/chosen": -0.39063310623168945, |
| "logits/rejected": -0.3791298270225525, |
| "logps/chosen": -2851.560546875, |
| "logps/ref_chosen": -296.9415283203125, |
| "logps/ref_rejected": -262.6710510253906, |
| "logps/rejected": -2496.45947265625, |
| "loss": 5.2111, |
| "r_dpo/chosen_len": 282.81561279296875, |
| "r_dpo/length_delta": 40.631248474121094, |
| "r_dpo/regularization_term": 4.063125133514404, |
| "r_dpo/rejected_len": 242.1843719482422, |
| "step": 450 |
| }, |
| { |
| "epoch": 0.9633507853403142, |
| "grad_norm": 656.00634765625, |
| "learning_rate": 2.168758844148272e-09, |
| "logits/chosen": -0.3994686007499695, |
| "logits/rejected": -0.3830736577510834, |
| "logps/chosen": -2940.84423828125, |
| "logps/ref_chosen": -312.42291259765625, |
| "logps/ref_rejected": -278.7356262207031, |
| "logps/rejected": -2538.37939453125, |
| "loss": 5.6979, |
| "r_dpo/chosen_len": 288.9125061035156, |
| "r_dpo/length_delta": 43.23125076293945, |
| "r_dpo/regularization_term": 4.323124885559082, |
| "r_dpo/rejected_len": 245.68124389648438, |
| "step": 460 |
| }, |
| { |
| "epoch": 0.9842931937172775, |
| "grad_norm": 382.99267578125, |
| "learning_rate": 4.288949484559934e-10, |
| "logits/chosen": -0.3687112331390381, |
| "logits/rejected": -0.35538098216056824, |
| "logps/chosen": -2719.51953125, |
| "logps/ref_chosen": -278.0654602050781, |
| "logps/ref_rejected": -256.5596618652344, |
| "logps/rejected": -2522.361083984375, |
| "loss": 5.2482, |
| "r_dpo/chosen_len": 268.8687438964844, |
| "r_dpo/length_delta": 26.262500762939453, |
| "r_dpo/regularization_term": 2.6262500286102295, |
| "r_dpo/rejected_len": 242.6062469482422, |
| "step": 470 |
| }, |
| { |
| "epoch": 0.9989528795811519, |
| "step": 477, |
| "total_flos": 0.0, |
| "train_loss": 14.700552830406204, |
| "train_runtime": 18661.0672, |
| "train_samples_per_second": 3.276, |
| "train_steps_per_second": 0.026 |
| } |
| ], |
| "logging_steps": 10, |
| "max_steps": 477, |
| "num_input_tokens_seen": 0, |
| "num_train_epochs": 1, |
| "save_steps": 50, |
| "stateful_callbacks": { |
| "TrainerControl": { |
| "args": { |
| "should_epoch_stop": false, |
| "should_evaluate": false, |
| "should_log": false, |
| "should_save": false, |
| "should_training_stop": false |
| }, |
| "attributes": {} |
| } |
| }, |
| "total_flos": 0.0, |
| "train_batch_size": 4, |
| "trial_name": null, |
| "trial_params": null |
| } |
|
|
