oluwagbotty's picture
Training in progress, step 2500, checkpoint
c9702f2 verified
Raw
History Blame
20.8 kB
{
"best_global_step": 1000,
"best_metric": 0.4849427044391632,
"best_model_checkpoint": "./openai/whisper-small-kpo-waxal/checkpoint-1000",
"epoch": 5.543840177580466,
"eval_steps": 500,
"global_step": 2500,
"is_hyper_param_search": false,
"is_local_process_zero": true,
"is_world_process_zero": true,
"log_history": [
{
"epoch": 0.05549389567147614,
"grad_norm": 23.202674865722656,
"learning_rate": 4.800000000000001e-06,
"loss": 5.858453369140625,
"step": 25
},
{
"epoch": 0.11098779134295228,
"grad_norm": 10.280803680419922,
"learning_rate": 9.800000000000001e-06,
"loss": 2.6889111328125,
"step": 50
},
{
"epoch": 0.16648168701442842,
"grad_norm": 11.007328987121582,
"learning_rate": 1.48e-05,
"loss": 1.9935995483398437,
"step": 75
},
{
"epoch": 0.22197558268590456,
"grad_norm": 8.749046325683594,
"learning_rate": 1.9800000000000004e-05,
"loss": 1.6322769165039062,
"step": 100
},
{
"epoch": 0.27746947835738067,
"grad_norm": 10.869550704956055,
"learning_rate": 2.48e-05,
"loss": 1.5226153564453124,
"step": 125
},
{
"epoch": 0.33296337402885684,
"grad_norm": 6.452018737792969,
"learning_rate": 2.98e-05,
"loss": 1.417423858642578,
"step": 150
},
{
"epoch": 0.38845726970033295,
"grad_norm": 8.565808296203613,
"learning_rate": 3.48e-05,
"loss": 1.358447265625,
"step": 175
},
{
"epoch": 0.4439511653718091,
"grad_norm": 7.219192981719971,
"learning_rate": 3.9800000000000005e-05,
"loss": 1.337242889404297,
"step": 200
},
{
"epoch": 0.49944506104328523,
"grad_norm": 8.29965591430664,
"learning_rate": 4.4800000000000005e-05,
"loss": 1.273004684448242,
"step": 225
},
{
"epoch": 0.5549389567147613,
"grad_norm": 7.17012357711792,
"learning_rate": 4.9800000000000004e-05,
"loss": 1.2329086303710937,
"step": 250
},
{
"epoch": 0.6104328523862376,
"grad_norm": 5.866935729980469,
"learning_rate": 5.4800000000000004e-05,
"loss": 1.2089253234863282,
"step": 275
},
{
"epoch": 0.6659267480577137,
"grad_norm": 6.515329360961914,
"learning_rate": 5.9800000000000003e-05,
"loss": 1.1282675170898437,
"step": 300
},
{
"epoch": 0.7214206437291898,
"grad_norm": 5.666633129119873,
"learning_rate": 6.48e-05,
"loss": 1.148940887451172,
"step": 325
},
{
"epoch": 0.7769145394006659,
"grad_norm": 6.579802513122559,
"learning_rate": 6.98e-05,
"loss": 1.1574817657470704,
"step": 350
},
{
"epoch": 0.832408435072142,
"grad_norm": 6.159289360046387,
"learning_rate": 7.48e-05,
"loss": 1.1553614807128907,
"step": 375
},
{
"epoch": 0.8879023307436182,
"grad_norm": 4.6766204833984375,
"learning_rate": 7.98e-05,
"loss": 1.1272061920166017,
"step": 400
},
{
"epoch": 0.9433962264150944,
"grad_norm": 5.207922458648682,
"learning_rate": 8.48e-05,
"loss": 1.1430768585205078,
"step": 425
},
{
"epoch": 0.9988901220865705,
"grad_norm": 5.042243480682373,
"learning_rate": 8.98e-05,
"loss": 1.1235774230957032,
"step": 450
},
{
"epoch": 1.053274139844617,
"grad_norm": 5.123244285583496,
"learning_rate": 9.48e-05,
"loss": 0.9237544250488281,
"step": 475
},
{
"epoch": 1.1087680355160932,
"grad_norm": 4.658648490905762,
"learning_rate": 9.98e-05,
"loss": 0.96513427734375,
"step": 500
},
{
"epoch": 1.1087680355160932,
"eval_cer": 0.279414692054622,
"eval_loss": 0.5450363755226135,
"eval_runtime": 211.0102,
"eval_samples_per_second": 8.341,
"eval_steps_per_second": 0.133,
"eval_wer": 0.77612952445537,
"step": 500
},
{
"epoch": 1.1642619311875695,
"grad_norm": 5.715120792388916,
"learning_rate": 9.981580966999233e-05,
"loss": 1.0203121948242186,
"step": 525
},
{
"epoch": 1.2197558268590456,
"grad_norm": 5.163341522216797,
"learning_rate": 9.9623944742901e-05,
"loss": 0.9690386962890625,
"step": 550
},
{
"epoch": 1.2752497225305217,
"grad_norm": 5.062475681304932,
"learning_rate": 9.943207981580968e-05,
"loss": 0.9559343719482422,
"step": 575
},
{
"epoch": 1.3307436182019978,
"grad_norm": 5.288034915924072,
"learning_rate": 9.924021488871836e-05,
"loss": 0.9570340728759765,
"step": 600
},
{
"epoch": 1.386237513873474,
"grad_norm": 4.448609828948975,
"learning_rate": 9.904834996162702e-05,
"loss": 0.9402964782714843,
"step": 625
},
{
"epoch": 1.44173140954495,
"grad_norm": 3.7210636138916016,
"learning_rate": 9.885648503453569e-05,
"loss": 0.9727059173583984,
"step": 650
},
{
"epoch": 1.4972253052164262,
"grad_norm": 4.641828536987305,
"learning_rate": 9.866462010744436e-05,
"loss": 0.9270778656005859,
"step": 675
},
{
"epoch": 1.5527192008879025,
"grad_norm": 4.52638053894043,
"learning_rate": 9.847275518035304e-05,
"loss": 0.9360514831542969,
"step": 700
},
{
"epoch": 1.6082130965593784,
"grad_norm": 4.332831859588623,
"learning_rate": 9.828089025326171e-05,
"loss": 0.9043086242675781,
"step": 725
},
{
"epoch": 1.6637069922308547,
"grad_norm": 4.253662109375,
"learning_rate": 9.808902532617038e-05,
"loss": 0.9130302429199219,
"step": 750
},
{
"epoch": 1.7192008879023306,
"grad_norm": 3.683013916015625,
"learning_rate": 9.789716039907906e-05,
"loss": 0.9052154541015625,
"step": 775
},
{
"epoch": 1.774694783573807,
"grad_norm": 3.542875051498413,
"learning_rate": 9.770529547198772e-05,
"loss": 0.9154473114013671,
"step": 800
},
{
"epoch": 1.830188679245283,
"grad_norm": 3.8648242950439453,
"learning_rate": 9.75134305448964e-05,
"loss": 0.8997632598876953,
"step": 825
},
{
"epoch": 1.8856825749167592,
"grad_norm": 3.9503533840179443,
"learning_rate": 9.732156561780506e-05,
"loss": 0.934085464477539,
"step": 850
},
{
"epoch": 1.9411764705882353,
"grad_norm": 3.7985453605651855,
"learning_rate": 9.712970069071374e-05,
"loss": 0.9187238311767578,
"step": 875
},
{
"epoch": 1.9966703662597114,
"grad_norm": 4.175370693206787,
"learning_rate": 9.693783576362242e-05,
"loss": 0.9017095947265625,
"step": 900
},
{
"epoch": 2.051054384017758,
"grad_norm": 3.5287599563598633,
"learning_rate": 9.674597083653109e-05,
"loss": 0.699445571899414,
"step": 925
},
{
"epoch": 2.106548279689234,
"grad_norm": 3.792794704437256,
"learning_rate": 9.655410590943976e-05,
"loss": 0.6688622283935547,
"step": 950
},
{
"epoch": 2.1620421753607104,
"grad_norm": 3.698681354522705,
"learning_rate": 9.636224098234843e-05,
"loss": 0.7054427337646484,
"step": 975
},
{
"epoch": 2.2175360710321863,
"grad_norm": 4.332374095916748,
"learning_rate": 9.61703760552571e-05,
"loss": 0.7015140533447266,
"step": 1000
},
{
"epoch": 2.2175360710321863,
"eval_cer": 0.26578814979424464,
"eval_loss": 0.4849427044391632,
"eval_runtime": 197.0927,
"eval_samples_per_second": 8.93,
"eval_steps_per_second": 0.142,
"eval_wer": 0.7392896499109943,
"step": 1000
},
{
"epoch": 2.2730299667036626,
"grad_norm": 2.7401206493377686,
"learning_rate": 9.597851112816577e-05,
"loss": 0.6754109191894532,
"step": 1025
},
{
"epoch": 2.328523862375139,
"grad_norm": 3.663332462310791,
"learning_rate": 9.578664620107444e-05,
"loss": 0.6997630310058593,
"step": 1050
},
{
"epoch": 2.384017758046615,
"grad_norm": 3.4043989181518555,
"learning_rate": 9.559478127398312e-05,
"loss": 0.6894276428222657,
"step": 1075
},
{
"epoch": 2.439511653718091,
"grad_norm": 2.929145574569702,
"learning_rate": 9.540291634689179e-05,
"loss": 0.7089242553710937,
"step": 1100
},
{
"epoch": 2.495005549389567,
"grad_norm": 4.095437526702881,
"learning_rate": 9.521105141980047e-05,
"loss": 0.7410848236083984,
"step": 1125
},
{
"epoch": 2.5504994450610434,
"grad_norm": 3.1726880073547363,
"learning_rate": 9.501918649270914e-05,
"loss": 0.7000830841064453,
"step": 1150
},
{
"epoch": 2.6059933407325193,
"grad_norm": 3.1098122596740723,
"learning_rate": 9.482732156561781e-05,
"loss": 0.704638442993164,
"step": 1175
},
{
"epoch": 2.6614872364039956,
"grad_norm": 3.6283864974975586,
"learning_rate": 9.463545663852649e-05,
"loss": 0.7068312072753906,
"step": 1200
},
{
"epoch": 2.7169811320754715,
"grad_norm": 3.307800531387329,
"learning_rate": 9.444359171143516e-05,
"loss": 0.703956527709961,
"step": 1225
},
{
"epoch": 2.772475027746948,
"grad_norm": 3.7725470066070557,
"learning_rate": 9.425172678434382e-05,
"loss": 0.7211872863769532,
"step": 1250
},
{
"epoch": 2.8279689234184238,
"grad_norm": 3.9565186500549316,
"learning_rate": 9.405986185725249e-05,
"loss": 0.7074081420898437,
"step": 1275
},
{
"epoch": 2.8834628190899,
"grad_norm": 2.8780977725982666,
"learning_rate": 9.386799693016117e-05,
"loss": 0.7226101684570313,
"step": 1300
},
{
"epoch": 2.9389567147613764,
"grad_norm": 3.7520499229431152,
"learning_rate": 9.367613200306984e-05,
"loss": 0.7472096252441406,
"step": 1325
},
{
"epoch": 2.9944506104328523,
"grad_norm": 3.8520190715789795,
"learning_rate": 9.348426707597852e-05,
"loss": 0.7333020782470703,
"step": 1350
},
{
"epoch": 3.048834628190899,
"grad_norm": 3.3807711601257324,
"learning_rate": 9.329240214888719e-05,
"loss": 0.5055583190917968,
"step": 1375
},
{
"epoch": 3.104328523862375,
"grad_norm": 2.786001205444336,
"learning_rate": 9.310053722179586e-05,
"loss": 0.5185424041748047,
"step": 1400
},
{
"epoch": 3.1598224195338513,
"grad_norm": 2.996492385864258,
"learning_rate": 9.290867229470454e-05,
"loss": 0.48687358856201174,
"step": 1425
},
{
"epoch": 3.2153163152053272,
"grad_norm": 3.1847450733184814,
"learning_rate": 9.27168073676132e-05,
"loss": 0.5263909530639649,
"step": 1450
},
{
"epoch": 3.2708102108768036,
"grad_norm": 3.4472248554229736,
"learning_rate": 9.252494244052187e-05,
"loss": 0.4982139205932617,
"step": 1475
},
{
"epoch": 3.32630410654828,
"grad_norm": 3.200753688812256,
"learning_rate": 9.233307751343055e-05,
"loss": 0.5312052917480469,
"step": 1500
},
{
"epoch": 3.32630410654828,
"eval_cer": 0.25577363653143875,
"eval_loss": 0.49039116501808167,
"eval_runtime": 196.3736,
"eval_samples_per_second": 8.963,
"eval_steps_per_second": 0.143,
"eval_wer": 0.7167754513859456,
"step": 1500
},
{
"epoch": 3.381798002219756,
"grad_norm": 3.604504108428955,
"learning_rate": 9.214121258633922e-05,
"loss": 0.5276579666137695,
"step": 1525
},
{
"epoch": 3.437291897891232,
"grad_norm": 3.206066131591797,
"learning_rate": 9.19493476592479e-05,
"loss": 0.5374650955200195,
"step": 1550
},
{
"epoch": 3.492785793562708,
"grad_norm": 3.3676679134368896,
"learning_rate": 9.175748273215656e-05,
"loss": 0.5156414031982421,
"step": 1575
},
{
"epoch": 3.5482796892341844,
"grad_norm": 2.790008068084717,
"learning_rate": 9.156561780506524e-05,
"loss": 0.5307263946533203,
"step": 1600
},
{
"epoch": 3.6037735849056602,
"grad_norm": 3.1276090145111084,
"learning_rate": 9.13737528779739e-05,
"loss": 0.5411431884765625,
"step": 1625
},
{
"epoch": 3.6592674805771366,
"grad_norm": 2.7448694705963135,
"learning_rate": 9.118188795088259e-05,
"loss": 0.5055676651000977,
"step": 1650
},
{
"epoch": 3.714761376248613,
"grad_norm": 3.283252477645874,
"learning_rate": 9.099002302379126e-05,
"loss": 0.541312484741211,
"step": 1675
},
{
"epoch": 3.770255271920089,
"grad_norm": 2.889359951019287,
"learning_rate": 9.079815809669992e-05,
"loss": 0.5509023666381836,
"step": 1700
},
{
"epoch": 3.8257491675915647,
"grad_norm": 3.8324856758117676,
"learning_rate": 9.06062931696086e-05,
"loss": 0.554903221130371,
"step": 1725
},
{
"epoch": 3.881243063263041,
"grad_norm": 2.673110008239746,
"learning_rate": 9.041442824251727e-05,
"loss": 0.5421103668212891,
"step": 1750
},
{
"epoch": 3.9367369589345174,
"grad_norm": 3.1440036296844482,
"learning_rate": 9.022256331542594e-05,
"loss": 0.5445112609863281,
"step": 1775
},
{
"epoch": 3.9922308546059933,
"grad_norm": 3.1932945251464844,
"learning_rate": 9.003069838833462e-05,
"loss": 0.5776439666748047,
"step": 1800
},
{
"epoch": 4.04661487236404,
"grad_norm": 2.3801052570343018,
"learning_rate": 8.983883346124329e-05,
"loss": 0.3385904312133789,
"step": 1825
},
{
"epoch": 4.102108768035516,
"grad_norm": 2.50673770904541,
"learning_rate": 8.964696853415197e-05,
"loss": 0.3545961380004883,
"step": 1850
},
{
"epoch": 4.157602663706992,
"grad_norm": 2.9095983505249023,
"learning_rate": 8.945510360706062e-05,
"loss": 0.35681617736816407,
"step": 1875
},
{
"epoch": 4.213096559378468,
"grad_norm": 2.819798707962036,
"learning_rate": 8.92632386799693e-05,
"loss": 0.33423614501953125,
"step": 1900
},
{
"epoch": 4.268590455049944,
"grad_norm": 3.1799139976501465,
"learning_rate": 8.907137375287797e-05,
"loss": 0.34991111755371096,
"step": 1925
},
{
"epoch": 4.324084350721421,
"grad_norm": 2.945661783218384,
"learning_rate": 8.887950882578665e-05,
"loss": 0.3595976257324219,
"step": 1950
},
{
"epoch": 4.379578246392897,
"grad_norm": 3.115243673324585,
"learning_rate": 8.868764389869532e-05,
"loss": 0.37289775848388673,
"step": 1975
},
{
"epoch": 4.435072142064373,
"grad_norm": 2.712531328201294,
"learning_rate": 8.849577897160399e-05,
"loss": 0.37404159545898436,
"step": 2000
},
{
"epoch": 4.435072142064373,
"eval_cer": 0.26078632206414065,
"eval_loss": 0.5362429022789001,
"eval_runtime": 196.8231,
"eval_samples_per_second": 8.942,
"eval_steps_per_second": 0.142,
"eval_wer": 0.7275578536916165,
"step": 2000
},
{
"epoch": 4.490566037735849,
"grad_norm": 2.464198350906372,
"learning_rate": 8.830391404451267e-05,
"loss": 0.363293342590332,
"step": 2025
},
{
"epoch": 4.546059933407325,
"grad_norm": 3.0920937061309814,
"learning_rate": 8.811204911742134e-05,
"loss": 0.3825681686401367,
"step": 2050
},
{
"epoch": 4.601553829078801,
"grad_norm": 3.222235679626465,
"learning_rate": 8.792018419033002e-05,
"loss": 0.37105091094970705,
"step": 2075
},
{
"epoch": 4.657047724750278,
"grad_norm": 2.993513584136963,
"learning_rate": 8.772831926323869e-05,
"loss": 0.3899434280395508,
"step": 2100
},
{
"epoch": 4.712541620421754,
"grad_norm": 2.9703752994537354,
"learning_rate": 8.753645433614735e-05,
"loss": 0.3792466735839844,
"step": 2125
},
{
"epoch": 4.76803551609323,
"grad_norm": 3.080484628677368,
"learning_rate": 8.734458940905604e-05,
"loss": 0.3823820877075195,
"step": 2150
},
{
"epoch": 4.823529411764706,
"grad_norm": 2.8566219806671143,
"learning_rate": 8.71527244819647e-05,
"loss": 0.3843545150756836,
"step": 2175
},
{
"epoch": 4.879023307436182,
"grad_norm": 2.913648843765259,
"learning_rate": 8.696085955487337e-05,
"loss": 0.3789427947998047,
"step": 2200
},
{
"epoch": 4.934517203107658,
"grad_norm": 3.40332293510437,
"learning_rate": 8.676899462778204e-05,
"loss": 0.4051662063598633,
"step": 2225
},
{
"epoch": 4.990011098779134,
"grad_norm": 2.9995148181915283,
"learning_rate": 8.657712970069072e-05,
"loss": 0.4019134902954102,
"step": 2250
},
{
"epoch": 5.0443951165371805,
"grad_norm": 2.564155101776123,
"learning_rate": 8.63852647735994e-05,
"loss": 0.24176448822021485,
"step": 2275
},
{
"epoch": 5.099889012208657,
"grad_norm": 2.846173048019409,
"learning_rate": 8.619339984650806e-05,
"loss": 0.21462602615356446,
"step": 2300
},
{
"epoch": 5.155382907880133,
"grad_norm": 2.316955327987671,
"learning_rate": 8.600153491941674e-05,
"loss": 0.22590169906616211,
"step": 2325
},
{
"epoch": 5.210876803551609,
"grad_norm": 2.8092174530029297,
"learning_rate": 8.58096699923254e-05,
"loss": 0.21187593460083007,
"step": 2350
},
{
"epoch": 5.266370699223086,
"grad_norm": 2.7788586616516113,
"learning_rate": 8.561780506523409e-05,
"loss": 0.2337300682067871,
"step": 2375
},
{
"epoch": 5.321864594894562,
"grad_norm": 2.503588914871216,
"learning_rate": 8.542594013814275e-05,
"loss": 0.23152971267700195,
"step": 2400
},
{
"epoch": 5.377358490566038,
"grad_norm": 2.533194065093994,
"learning_rate": 8.523407521105142e-05,
"loss": 0.23667486190795897,
"step": 2425
},
{
"epoch": 5.4328523862375135,
"grad_norm": 2.8751583099365234,
"learning_rate": 8.50422102839601e-05,
"loss": 0.22971002578735353,
"step": 2450
},
{
"epoch": 5.48834628190899,
"grad_norm": 2.6748523712158203,
"learning_rate": 8.485034535686877e-05,
"loss": 0.2416807746887207,
"step": 2475
},
{
"epoch": 5.543840177580466,
"grad_norm": 2.857468605041504,
"learning_rate": 8.465848042977744e-05,
"loss": 0.23854978561401366,
"step": 2500
},
{
"epoch": 5.543840177580466,
"eval_cer": 0.2641594794045581,
"eval_loss": 0.5750833749771118,
"eval_runtime": 198.9608,
"eval_samples_per_second": 8.846,
"eval_steps_per_second": 0.141,
"eval_wer": 0.723912859201492,
"step": 2500
}
],
"logging_steps": 25,
"max_steps": 13530,
"num_input_tokens_seen": 0,
"num_train_epochs": 30,
"save_steps": 500,
"stateful_callbacks": {
"EarlyStoppingCallback": {
"args": {
"early_stopping_patience": 3,
"early_stopping_threshold": 0.0
},
"attributes": {
"early_stopping_patience_counter": 3
}
},
"TrainerControl": {
"args": {
"should_epoch_stop": false,
"should_evaluate": false,
"should_log": false,
"should_save": true,
"should_training_stop": true
},
"attributes": {}
}
},
"total_flos": 2.30565305585664e+19,
"train_batch_size": 16,
"trial_name": null,
"trial_params": null
}