dmatekenya's picture
Training in progress, step 100
ea21e16 verified
Raw History Blame Contribute Delete
3.94 kB
Invalid JSON:Unexpected token 'N', ..."ad_norm": NaN, "... is not valid JSON
{
"best_global_step": 500,
"best_metric": 1.0,
"best_model_checkpoint": "/home/jupyter-wb344850/chichewa-asr/models/checkpoints/mms-1b-all-chichewa-1h/checkpoint-500",
"epoch": 125.0,
"eval_steps": 500,
"global_step": 500,
"is_hyper_param_search": false,
"is_local_process_zero": true,
"is_world_process_zero": true,
"log_history": [
{
"epoch": 6.25,
"grad_norm": NaN,
"learning_rate": 4.8e-05,
"loss": 70.277255859375,
"step": 25
},
{
"epoch": 12.5,
"grad_norm": NaN,
"learning_rate": 9.800000000000001e-05,
"loss": 0.0,
"step": 50
},
{
"epoch": 18.75,
"grad_norm": NaN,
"learning_rate": 0.000148,
"loss": 0.0,
"step": 75
},
{
"epoch": 25.0,
"grad_norm": NaN,
"learning_rate": 0.00019800000000000002,
"loss": 0.0,
"step": 100
},
{
"epoch": 31.25,
"grad_norm": NaN,
"learning_rate": 0.000248,
"loss": 0.0,
"step": 125
},
{
"epoch": 37.5,
"grad_norm": NaN,
"learning_rate": 0.000298,
"loss": 0.0,
"step": 150
},
{
"epoch": 43.75,
"grad_norm": NaN,
"learning_rate": 0.000348,
"loss": 0.0,
"step": 175
},
{
"epoch": 50.0,
"grad_norm": NaN,
"learning_rate": 0.000398,
"loss": 0.0,
"step": 200
},
{
"epoch": 56.25,
"grad_norm": NaN,
"learning_rate": 0.000448,
"loss": 0.0,
"step": 225
},
{
"epoch": 62.5,
"grad_norm": NaN,
"learning_rate": 0.000498,
"loss": 0.0,
"step": 250
},
{
"epoch": 68.75,
"grad_norm": NaN,
"learning_rate": 0.0005480000000000001,
"loss": 0.0,
"step": 275
},
{
"epoch": 75.0,
"grad_norm": NaN,
"learning_rate": 0.000598,
"loss": 0.0,
"step": 300
},
{
"epoch": 81.25,
"grad_norm": NaN,
"learning_rate": 0.000648,
"loss": 0.0,
"step": 325
},
{
"epoch": 87.5,
"grad_norm": NaN,
"learning_rate": 0.0006979999999999999,
"loss": 0.0,
"step": 350
},
{
"epoch": 93.75,
"grad_norm": NaN,
"learning_rate": 0.000748,
"loss": 0.0,
"step": 375
},
{
"epoch": 100.0,
"grad_norm": NaN,
"learning_rate": 0.0007980000000000001,
"loss": 0.0,
"step": 400
},
{
"epoch": 106.25,
"grad_norm": NaN,
"learning_rate": 0.000848,
"loss": 0.0,
"step": 425
},
{
"epoch": 112.5,
"grad_norm": NaN,
"learning_rate": 0.000898,
"loss": 0.0,
"step": 450
},
{
"epoch": 118.75,
"grad_norm": NaN,
"learning_rate": 0.000948,
"loss": 0.0,
"step": 475
},
{
"epoch": 125.0,
"grad_norm": NaN,
"learning_rate": 0.000998,
"loss": 0.0,
"step": 500
},
{
"epoch": 125.0,
"eval_loss": NaN,
"eval_runtime": 1.3702,
"eval_samples_per_second": 24.815,
"eval_steps_per_second": 2.19,
"eval_wer": 1.0,
"step": 500
}
],
"logging_steps": 25,
"max_steps": 5000,
"num_input_tokens_seen": 0,
"num_train_epochs": 1250,
"save_steps": 500,
"stateful_callbacks": {
"EarlyStoppingCallback": {
"args": {
"early_stopping_patience": 3,
"early_stopping_threshold": 0.0
},
"attributes": {
"early_stopping_patience_counter": 0
}
},
"TrainerControl": {
"args": {
"should_epoch_stop": false,
"should_evaluate": false,
"should_log": false,
"should_save": true,
"should_training_stop": false
},
"attributes": {}
}
},
"total_flos": 7.435379130650906e+19,
"train_batch_size": 32,
"trial_name": null,
"trial_params": null
}