Upload folder using huggingface_hub

added_tokens.json

@@ -0,0 +1,3 @@
+{
+  "<mask>": 59811
+}

checkpoint-1000/added_tokens.json

@@ -0,0 +1,3 @@
+{
+  "<mask>": 59811
+}

checkpoint-1000/optimizer.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:a4740eff7f7726a7d81146d0a540c888654d2eb3bdf282d11aa73680ccbb36dc
-size 151392779
+oid sha256:b84355664b1171ee54bd6a99361e9ba9e7be9beb3365865942cade794a976f4f
+size 396383563

checkpoint-1000/pytorch_model.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:215946acfe8af69cc186f5f2a033d67d1fc49ffaf3d6f583a9706375c1d75377
-size 199234413
+oid sha256:a7e9201f999da17c123fdeaf604f0dfa742666f1e456cc9291cd718895a517f2
+size 199236461

checkpoint-1000/rng_state.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:4d30b109bc34b215f2601d0ec75f5f8775fac00c35608a61cc209a5dcfb7edaa
+oid sha256:343d370adc10ca90f4022d1d35c813a99b720c147ec4ead44d32675cfd7f07ac
 size 14391

checkpoint-1000/special_tokens_map.json

@@ -1,5 +1,12 @@
 {
   "eos_token": "</s>",
+  "mask_token": {
+    "content": "<mask>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
   "pad_token": "<pad>",
   "unk_token": "<unk>"
 }

checkpoint-1000/tokenizer_config.json

@@ -23,11 +23,20 @@
       "rstrip": false,
       "single_word": false,
       "special": true
+    },
+    "59811": {
+      "content": "<mask>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
     }
   },
   "clean_up_tokenization_spaces": false,
   "eos_token": "</s>",
   "extra_special_tokens": {},
+  "mask_token": "<mask>",
   "model_max_length": 512,
   "pad_token": "<pad>",
   "separate_vocabs": false,

checkpoint-1000/trainer_state.json

@@ -1,6 +1,6 @@
 {
   "best_global_step": 1000,
-  "best_metric": 5.472690582275391,
+  "best_metric": 6.018856525421143,
   "best_model_checkpoint": "./nyanja_english_warmup\\checkpoint-1000",
   "epoch": 0.5035246727089627,
   "eval_steps": 500,
@@ -11,158 +11,158 @@
   "log_history": [
     {
       "epoch": 0.025176233635448138,
-      "grad_norm": 0.947960615158081,
+      "grad_norm": 1.8817049264907837,
       "learning_rate": 4.876636455186304e-05,
-      "loss": 9.5286,
+      "loss": 9.8135,
       "step": 50
     },
     {
       "epoch": 0.050352467270896276,
-      "grad_norm": 1.194724440574646,
+      "grad_norm": 1.9862349033355713,
       "learning_rate": 4.7507552870090635e-05,
-      "loss": 8.2179,
+      "loss": 8.8496,
       "step": 100
     },
     {
       "epoch": 0.0755287009063444,
-      "grad_norm": 0.9894101023674011,
+      "grad_norm": 2.0134317874908447,
       "learning_rate": 4.624874118831823e-05,
-      "loss": 7.5072,
+      "loss": 8.3485,
       "step": 150
     },
     {
       "epoch": 0.10070493454179255,
-      "grad_norm": 0.9540104269981384,
+      "grad_norm": 2.8336329460144043,
       "learning_rate": 4.498992950654582e-05,
-      "loss": 7.1644,
+      "loss": 7.9298,
       "step": 200
     },
     {
       "epoch": 0.12588116817724068,
-      "grad_norm": 1.0262048244476318,
+      "grad_norm": 1.8131136894226074,
       "learning_rate": 4.3731117824773415e-05,
-      "loss": 6.9377,
+      "loss": 7.6853,
       "step": 250
     },
     {
       "epoch": 0.1510574018126888,
-      "grad_norm": 0.916906476020813,
+      "grad_norm": 2.619373083114624,
       "learning_rate": 4.2472306143001015e-05,
-      "loss": 6.7447,
+      "loss": 7.3954,
       "step": 300
     },
     {
       "epoch": 0.17623363544813697,
-      "grad_norm": 0.9376256465911865,
+      "grad_norm": 2.30450439453125,
       "learning_rate": 4.12134944612286e-05,
-      "loss": 6.6128,
+      "loss": 7.1206,
       "step": 350
     },
     {
       "epoch": 0.2014098690835851,
-      "grad_norm": 0.8047090768814087,
+      "grad_norm": 2.5505969524383545,
       "learning_rate": 3.9954682779456194e-05,
-      "loss": 6.5074,
+      "loss": 6.9686,
       "step": 400
     },
     {
       "epoch": 0.22658610271903323,
-      "grad_norm": 0.9348500967025757,
+      "grad_norm": 2.0169050693511963,
       "learning_rate": 3.869587109768379e-05,
-      "loss": 6.4087,
+      "loss": 6.807,
       "step": 450
     },
     {
       "epoch": 0.25176233635448136,
-      "grad_norm": 0.8273019194602966,
+      "grad_norm": 2.125231981277466,
       "learning_rate": 3.743705941591138e-05,
-      "loss": 6.3114,
+      "loss": 6.7269,
       "step": 500
     },
     {
       "epoch": 0.25176233635448136,
-      "eval_loss": 5.968835353851318,
-      "eval_runtime": 41.9731,
-      "eval_samples_per_second": 39.859,
-      "eval_steps_per_second": 2.502,
+      "eval_loss": 6.588558197021484,
+      "eval_runtime": 35.6768,
+      "eval_samples_per_second": 46.893,
+      "eval_steps_per_second": 2.943,
       "step": 500
     },
     {
       "epoch": 0.2769385699899295,
-      "grad_norm": 0.7640300989151001,
+      "grad_norm": 2.285013437271118,
       "learning_rate": 3.6178247734138974e-05,
-      "loss": 6.2115,
+      "loss": 6.6117,
       "step": 550
     },
     {
       "epoch": 0.3021148036253776,
-      "grad_norm": 0.8070991635322571,
+      "grad_norm": 2.147951602935791,
       "learning_rate": 3.491943605236657e-05,
-      "loss": 6.1867,
+      "loss": 6.5024,
       "step": 600
     },
     {
       "epoch": 0.32729103726082576,
-      "grad_norm": 0.7798317074775696,
+      "grad_norm": 1.968063235282898,
       "learning_rate": 3.366062437059416e-05,
-      "loss": 6.0893,
+      "loss": 6.4696,
       "step": 650
     },
     {
       "epoch": 0.35246727089627394,
-      "grad_norm": 0.8209011554718018,
+      "grad_norm": 3.044461488723755,
       "learning_rate": 3.240181268882175e-05,
-      "loss": 6.0416,
+      "loss": 6.4161,
       "step": 700
     },
     {
       "epoch": 0.3776435045317221,
-      "grad_norm": 0.7711942195892334,
+      "grad_norm": 3.290184497833252,
       "learning_rate": 3.1143001007049346e-05,
-      "loss": 5.9784,
+      "loss": 6.275,
       "step": 750
     },
     {
       "epoch": 0.4028197381671702,
-      "grad_norm": 0.8087317943572998,
+      "grad_norm": 2.1646714210510254,
       "learning_rate": 2.988418932527694e-05,
-      "loss": 5.9354,
+      "loss": 6.3287,
       "step": 800
     },
     {
       "epoch": 0.42799597180261834,
-      "grad_norm": 0.7595328688621521,
+      "grad_norm": 2.3437819480895996,
       "learning_rate": 2.862537764350453e-05,
-      "loss": 5.8544,
+      "loss": 6.2043,
       "step": 850
     },
     {
       "epoch": 0.45317220543806647,
-      "grad_norm": 0.7424010038375854,
+      "grad_norm": 2.464350700378418,
       "learning_rate": 2.7366565961732126e-05,
-      "loss": 5.8309,
+      "loss": 6.2089,
       "step": 900
     },
     {
       "epoch": 0.4783484390735146,
-      "grad_norm": 0.7028313279151917,
+      "grad_norm": 2.730142831802368,
       "learning_rate": 2.610775427995972e-05,
-      "loss": 5.8075,
+      "loss": 6.146,
       "step": 950
     },
     {
       "epoch": 0.5035246727089627,
-      "grad_norm": 0.739640474319458,
+      "grad_norm": 2.321312189102173,
       "learning_rate": 2.4848942598187312e-05,
-      "loss": 5.7741,
+      "loss": 6.1237,
       "step": 1000
     },
     {
       "epoch": 0.5035246727089627,
-      "eval_loss": 5.472690582275391,
-      "eval_runtime": 37.0841,
-      "eval_samples_per_second": 45.114,
-      "eval_steps_per_second": 2.831,
+      "eval_loss": 6.018856525421143,
+      "eval_runtime": 33.2683,
+      "eval_samples_per_second": 50.288,
+      "eval_steps_per_second": 3.156,
       "step": 1000
     }
   ],

checkpoint-1000/training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:fd35ffac2ed4cf9159515c227a2e8bb8761c759f619e17bd96f19d5dd4c854ad
+oid sha256:00a8fef87f609a94dc07e98c9168ebeac65261d0032405c3c986af25ab8cfec3
 size 5713

checkpoint-1500/added_tokens.json

@@ -0,0 +1,3 @@
+{
+  "<mask>": 59811
+}

checkpoint-1500/optimizer.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:5da0f570703c36c705757779b20b5d3fc9b39084e216990d8f3923c9f791b6b0
-size 151392779
+oid sha256:4ac77f5975048fe6c4cf6cc0fb96e5e7d54577ab89092c4a0a717d746d5191df
+size 396383563

checkpoint-1500/pytorch_model.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:14c9b1e4723bc4f3f18cbfefb09cc3d30e23120b46f6caa05f88aa21b076609d
-size 199234413
+oid sha256:389bfa804e20a1a306e528f6f064050efbd7d2812976e75dee161a3dfb4e2f1e
+size 199236461

checkpoint-1500/rng_state.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:94c2c5693e8ea1cb95056ee4aac8a87df5457ec36d1f15b47727ec41df4454e6
+oid sha256:6f6b5d5e0b2f1b5845936c368d104ce0c8a941a010bfdecae927cf3b584a018f
 size 14391

checkpoint-1500/special_tokens_map.json

@@ -1,5 +1,12 @@
 {
   "eos_token": "</s>",
+  "mask_token": {
+    "content": "<mask>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
   "pad_token": "<pad>",
   "unk_token": "<unk>"
 }

checkpoint-1500/tokenizer_config.json

@@ -23,11 +23,20 @@
       "rstrip": false,
       "single_word": false,
       "special": true
+    },
+    "59811": {
+      "content": "<mask>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
     }
   },
   "clean_up_tokenization_spaces": false,
   "eos_token": "</s>",
   "extra_special_tokens": {},
+  "mask_token": "<mask>",
   "model_max_length": 512,
   "pad_token": "<pad>",
   "separate_vocabs": false,

checkpoint-1500/trainer_state.json

@@ -1,6 +1,6 @@
 {
   "best_global_step": 1500,
-  "best_metric": 5.239852428436279,
+  "best_metric": 5.870590686798096,
   "best_model_checkpoint": "./nyanja_english_warmup\\checkpoint-1500",
   "epoch": 0.7552870090634441,
   "eval_steps": 500,
@@ -11,236 +11,236 @@
   "log_history": [
     {
       "epoch": 0.025176233635448138,
-      "grad_norm": 0.947960615158081,
+      "grad_norm": 1.8817049264907837,
       "learning_rate": 4.876636455186304e-05,
-      "loss": 9.5286,
+      "loss": 9.8135,
       "step": 50
     },
     {
       "epoch": 0.050352467270896276,
-      "grad_norm": 1.194724440574646,
+      "grad_norm": 1.9862349033355713,
       "learning_rate": 4.7507552870090635e-05,
-      "loss": 8.2179,
+      "loss": 8.8496,
       "step": 100
     },
     {
       "epoch": 0.0755287009063444,
-      "grad_norm": 0.9894101023674011,
+      "grad_norm": 2.0134317874908447,
       "learning_rate": 4.624874118831823e-05,
-      "loss": 7.5072,
+      "loss": 8.3485,
       "step": 150
     },
     {
       "epoch": 0.10070493454179255,
-      "grad_norm": 0.9540104269981384,
+      "grad_norm": 2.8336329460144043,
       "learning_rate": 4.498992950654582e-05,
-      "loss": 7.1644,
+      "loss": 7.9298,
       "step": 200
     },
     {
       "epoch": 0.12588116817724068,
-      "grad_norm": 1.0262048244476318,
+      "grad_norm": 1.8131136894226074,
       "learning_rate": 4.3731117824773415e-05,
-      "loss": 6.9377,
+      "loss": 7.6853,
       "step": 250
     },
     {
       "epoch": 0.1510574018126888,
-      "grad_norm": 0.916906476020813,
+      "grad_norm": 2.619373083114624,
       "learning_rate": 4.2472306143001015e-05,
-      "loss": 6.7447,
+      "loss": 7.3954,
       "step": 300
     },
     {
       "epoch": 0.17623363544813697,
-      "grad_norm": 0.9376256465911865,
+      "grad_norm": 2.30450439453125,
       "learning_rate": 4.12134944612286e-05,
-      "loss": 6.6128,
+      "loss": 7.1206,
       "step": 350
     },
     {
       "epoch": 0.2014098690835851,
-      "grad_norm": 0.8047090768814087,
+      "grad_norm": 2.5505969524383545,
       "learning_rate": 3.9954682779456194e-05,
-      "loss": 6.5074,
+      "loss": 6.9686,
       "step": 400
     },
     {
       "epoch": 0.22658610271903323,
-      "grad_norm": 0.9348500967025757,
+      "grad_norm": 2.0169050693511963,
       "learning_rate": 3.869587109768379e-05,
-      "loss": 6.4087,
+      "loss": 6.807,
       "step": 450
     },
     {
       "epoch": 0.25176233635448136,
-      "grad_norm": 0.8273019194602966,
+      "grad_norm": 2.125231981277466,
       "learning_rate": 3.743705941591138e-05,
-      "loss": 6.3114,
+      "loss": 6.7269,
       "step": 500
     },
     {
       "epoch": 0.25176233635448136,
-      "eval_loss": 5.968835353851318,
-      "eval_runtime": 41.9731,
-      "eval_samples_per_second": 39.859,
-      "eval_steps_per_second": 2.502,
+      "eval_loss": 6.588558197021484,
+      "eval_runtime": 35.6768,
+      "eval_samples_per_second": 46.893,
+      "eval_steps_per_second": 2.943,
       "step": 500
     },
     {
       "epoch": 0.2769385699899295,
-      "grad_norm": 0.7640300989151001,
+      "grad_norm": 2.285013437271118,
       "learning_rate": 3.6178247734138974e-05,
-      "loss": 6.2115,
+      "loss": 6.6117,
       "step": 550
     },
     {
       "epoch": 0.3021148036253776,
-      "grad_norm": 0.8070991635322571,
+      "grad_norm": 2.147951602935791,
       "learning_rate": 3.491943605236657e-05,
-      "loss": 6.1867,
+      "loss": 6.5024,
       "step": 600
     },
     {
       "epoch": 0.32729103726082576,
-      "grad_norm": 0.7798317074775696,
+      "grad_norm": 1.968063235282898,
       "learning_rate": 3.366062437059416e-05,
-      "loss": 6.0893,
+      "loss": 6.4696,
       "step": 650
     },
     {
       "epoch": 0.35246727089627394,
-      "grad_norm": 0.8209011554718018,
+      "grad_norm": 3.044461488723755,
       "learning_rate": 3.240181268882175e-05,
-      "loss": 6.0416,
+      "loss": 6.4161,
       "step": 700
     },
     {
       "epoch": 0.3776435045317221,
-      "grad_norm": 0.7711942195892334,
+      "grad_norm": 3.290184497833252,
       "learning_rate": 3.1143001007049346e-05,
-      "loss": 5.9784,
+      "loss": 6.275,
       "step": 750
     },
     {
       "epoch": 0.4028197381671702,
-      "grad_norm": 0.8087317943572998,
+      "grad_norm": 2.1646714210510254,
       "learning_rate": 2.988418932527694e-05,
-      "loss": 5.9354,
+      "loss": 6.3287,
       "step": 800
     },
     {
       "epoch": 0.42799597180261834,
-      "grad_norm": 0.7595328688621521,
+      "grad_norm": 2.3437819480895996,
       "learning_rate": 2.862537764350453e-05,
-      "loss": 5.8544,
+      "loss": 6.2043,
       "step": 850
     },
     {
       "epoch": 0.45317220543806647,
-      "grad_norm": 0.7424010038375854,
+      "grad_norm": 2.464350700378418,
       "learning_rate": 2.7366565961732126e-05,
-      "loss": 5.8309,
+      "loss": 6.2089,
       "step": 900
     },
     {
       "epoch": 0.4783484390735146,
-      "grad_norm": 0.7028313279151917,
+      "grad_norm": 2.730142831802368,
       "learning_rate": 2.610775427995972e-05,
-      "loss": 5.8075,
+      "loss": 6.146,
       "step": 950
     },
     {
       "epoch": 0.5035246727089627,
-      "grad_norm": 0.739640474319458,
+      "grad_norm": 2.321312189102173,
       "learning_rate": 2.4848942598187312e-05,
-      "loss": 5.7741,
+      "loss": 6.1237,
       "step": 1000
     },
     {
       "epoch": 0.5035246727089627,
-      "eval_loss": 5.472690582275391,
-      "eval_runtime": 37.0841,
-      "eval_samples_per_second": 45.114,
-      "eval_steps_per_second": 2.831,
+      "eval_loss": 6.018856525421143,
+      "eval_runtime": 33.2683,
+      "eval_samples_per_second": 50.288,
+      "eval_steps_per_second": 3.156,
       "step": 1000
     },
     {
       "epoch": 0.5287009063444109,
-      "grad_norm": 0.7200352549552917,
+      "grad_norm": 2.415438652038574,
       "learning_rate": 2.3590130916414905e-05,
-      "loss": 5.7262,
+      "loss": 6.2424,
       "step": 1050
     },
     {
       "epoch": 0.553877139979859,
-      "grad_norm": 0.7275882959365845,
+      "grad_norm": 2.567075252532959,
       "learning_rate": 2.2331319234642498e-05,
-      "loss": 5.7028,
+      "loss": 6.0269,
       "step": 1100
     },
     {
       "epoch": 0.5790533736153072,
-      "grad_norm": 0.7147909998893738,
+      "grad_norm": 3.5269229412078857,
       "learning_rate": 2.107250755287009e-05,
-      "loss": 5.6455,
+      "loss": 5.98,
       "step": 1150
     },
     {
       "epoch": 0.6042296072507553,
-      "grad_norm": 0.7415097951889038,
+      "grad_norm": 3.201200246810913,
       "learning_rate": 1.9813695871097685e-05,
-      "loss": 5.6142,
+      "loss": 6.0753,
       "step": 1200
     },
     {
       "epoch": 0.6294058408862034,
-      "grad_norm": 0.7115822434425354,
+      "grad_norm": 3.0480122566223145,
       "learning_rate": 1.8554884189325278e-05,
-      "loss": 5.6208,
+      "loss": 5.9747,
       "step": 1250
     },
     {
       "epoch": 0.6545820745216515,
-      "grad_norm": 0.6943323612213135,
+      "grad_norm": 3.005802869796753,
       "learning_rate": 1.729607250755287e-05,
-      "loss": 5.5773,
+      "loss": 5.9316,
       "step": 1300
     },
     {
       "epoch": 0.6797583081570997,
-      "grad_norm": 0.709179699420929,
+      "grad_norm": 2.7067759037017822,
       "learning_rate": 1.6037260825780464e-05,
-      "loss": 5.5679,
+      "loss": 5.956,
       "step": 1350
     },
     {
       "epoch": 0.7049345417925479,
-      "grad_norm": 0.7256223559379578,
+      "grad_norm": 2.6058285236358643,
       "learning_rate": 1.4778449144008057e-05,
-      "loss": 5.5666,
+      "loss": 6.0221,
       "step": 1400
     },
     {
       "epoch": 0.730110775427996,
-      "grad_norm": 0.7237951159477234,
+      "grad_norm": 3.462085485458374,
       "learning_rate": 1.3519637462235652e-05,
-      "loss": 5.5505,
+      "loss": 5.9177,
       "step": 1450
     },
     {
       "epoch": 0.7552870090634441,
-      "grad_norm": 0.7216870784759521,
+      "grad_norm": 3.490471839904785,
       "learning_rate": 1.2260825780463244e-05,
-      "loss": 5.4866,
+      "loss": 5.9148,
       "step": 1500
     },
     {
       "epoch": 0.7552870090634441,
-      "eval_loss": 5.239852428436279,
-      "eval_runtime": 42.2008,
-      "eval_samples_per_second": 39.644,
-      "eval_steps_per_second": 2.488,
+      "eval_loss": 5.870590686798096,
+      "eval_runtime": 35.2827,
+      "eval_samples_per_second": 47.417,
+      "eval_steps_per_second": 2.976,
       "step": 1500
     }
   ],

checkpoint-1500/training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:fd35ffac2ed4cf9159515c227a2e8bb8761c759f619e17bd96f19d5dd4c854ad
+oid sha256:00a8fef87f609a94dc07e98c9168ebeac65261d0032405c3c986af25ab8cfec3
 size 5713

checkpoint-1986/added_tokens.json

@@ -0,0 +1,3 @@
+{
+  "<mask>": 59811
+}

checkpoint-1986/optimizer.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:85902f687be92bb633ac00ea98834201fb24f0e1afc041251a56d5791f5a0842
-size 151392779
+oid sha256:95c5f717e00ca12ebadf45a555b075351f97072ecc3d95daefc1d0a97527f6b2
+size 396383563

checkpoint-1986/pytorch_model.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:a66f9ba7a90adbbb6b72b315da4574f8eaa1650de9a87b3fec5f9a6417508beb
-size 199234413
+oid sha256:9c6b44f8050f47a2a6a8c6ee0e0c45377705b9fde279db687bf52251b613d8e7
+size 199236461

checkpoint-1986/rng_state.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:a5694aa24e78c35c22a02b03a246e26b2afdfa338640982ad10ec0e2268d72f2
+oid sha256:f5bbdfeabe62f0870bd5c0d42ac94377220e8e74535faeb12d4d94f9ed2cdae9
 size 14391

checkpoint-1986/special_tokens_map.json

@@ -1,5 +1,12 @@
 {
   "eos_token": "</s>",
+  "mask_token": {
+    "content": "<mask>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
   "pad_token": "<pad>",
   "unk_token": "<unk>"
 }

checkpoint-1986/tokenizer_config.json

@@ -23,11 +23,20 @@
       "rstrip": false,
       "single_word": false,
       "special": true
+    },
+    "59811": {
+      "content": "<mask>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
     }
   },
   "clean_up_tokenization_spaces": false,
   "eos_token": "</s>",
   "extra_special_tokens": {},
+  "mask_token": "<mask>",
   "model_max_length": 512,
   "pad_token": "<pad>",
   "separate_vocabs": false,

checkpoint-1986/trainer_state.json

@@ -1,6 +1,6 @@
 {
   "best_global_step": 1500,
-  "best_metric": 5.239852428436279,
+  "best_metric": 5.870590686798096,
   "best_model_checkpoint": "./nyanja_english_warmup\\checkpoint-1500",
   "epoch": 1.0,
   "eval_steps": 500,
@@ -11,299 +11,299 @@
   "log_history": [
     {
       "epoch": 0.025176233635448138,
-      "grad_norm": 0.947960615158081,
+      "grad_norm": 1.8817049264907837,
       "learning_rate": 4.876636455186304e-05,
-      "loss": 9.5286,
+      "loss": 9.8135,
       "step": 50
     },
     {
       "epoch": 0.050352467270896276,
-      "grad_norm": 1.194724440574646,
+      "grad_norm": 1.9862349033355713,
       "learning_rate": 4.7507552870090635e-05,
-      "loss": 8.2179,
+      "loss": 8.8496,
       "step": 100
     },
     {
       "epoch": 0.0755287009063444,
-      "grad_norm": 0.9894101023674011,
+      "grad_norm": 2.0134317874908447,
       "learning_rate": 4.624874118831823e-05,
-      "loss": 7.5072,
+      "loss": 8.3485,
       "step": 150
     },
     {
       "epoch": 0.10070493454179255,
-      "grad_norm": 0.9540104269981384,
+      "grad_norm": 2.8336329460144043,
       "learning_rate": 4.498992950654582e-05,
-      "loss": 7.1644,
+      "loss": 7.9298,
       "step": 200
     },
     {
       "epoch": 0.12588116817724068,
-      "grad_norm": 1.0262048244476318,
+      "grad_norm": 1.8131136894226074,
       "learning_rate": 4.3731117824773415e-05,
-      "loss": 6.9377,
+      "loss": 7.6853,
       "step": 250
     },
     {
       "epoch": 0.1510574018126888,
-      "grad_norm": 0.916906476020813,
+      "grad_norm": 2.619373083114624,
       "learning_rate": 4.2472306143001015e-05,
-      "loss": 6.7447,
+      "loss": 7.3954,
       "step": 300
     },
     {
       "epoch": 0.17623363544813697,
-      "grad_norm": 0.9376256465911865,
+      "grad_norm": 2.30450439453125,
       "learning_rate": 4.12134944612286e-05,
-      "loss": 6.6128,
+      "loss": 7.1206,
       "step": 350
     },
     {
       "epoch": 0.2014098690835851,
-      "grad_norm": 0.8047090768814087,
+      "grad_norm": 2.5505969524383545,
       "learning_rate": 3.9954682779456194e-05,
-      "loss": 6.5074,
+      "loss": 6.9686,
       "step": 400
     },
     {
       "epoch": 0.22658610271903323,
-      "grad_norm": 0.9348500967025757,
+      "grad_norm": 2.0169050693511963,
       "learning_rate": 3.869587109768379e-05,
-      "loss": 6.4087,
+      "loss": 6.807,
       "step": 450
     },
     {
       "epoch": 0.25176233635448136,
-      "grad_norm": 0.8273019194602966,
+      "grad_norm": 2.125231981277466,
       "learning_rate": 3.743705941591138e-05,
-      "loss": 6.3114,
+      "loss": 6.7269,
       "step": 500
     },
     {
       "epoch": 0.25176233635448136,
-      "eval_loss": 5.968835353851318,
-      "eval_runtime": 41.9731,
-      "eval_samples_per_second": 39.859,
-      "eval_steps_per_second": 2.502,
+      "eval_loss": 6.588558197021484,
+      "eval_runtime": 35.6768,
+      "eval_samples_per_second": 46.893,
+      "eval_steps_per_second": 2.943,
       "step": 500
     },
     {
       "epoch": 0.2769385699899295,
-      "grad_norm": 0.7640300989151001,
+      "grad_norm": 2.285013437271118,
       "learning_rate": 3.6178247734138974e-05,
-      "loss": 6.2115,
+      "loss": 6.6117,
       "step": 550
     },
     {
       "epoch": 0.3021148036253776,
-      "grad_norm": 0.8070991635322571,
+      "grad_norm": 2.147951602935791,
       "learning_rate": 3.491943605236657e-05,
-      "loss": 6.1867,
+      "loss": 6.5024,
       "step": 600
     },
     {
       "epoch": 0.32729103726082576,
-      "grad_norm": 0.7798317074775696,
+      "grad_norm": 1.968063235282898,
       "learning_rate": 3.366062437059416e-05,
-      "loss": 6.0893,
+      "loss": 6.4696,
       "step": 650
     },
     {
       "epoch": 0.35246727089627394,
-      "grad_norm": 0.8209011554718018,
+      "grad_norm": 3.044461488723755,
       "learning_rate": 3.240181268882175e-05,
-      "loss": 6.0416,
+      "loss": 6.4161,
       "step": 700
     },
     {
       "epoch": 0.3776435045317221,
-      "grad_norm": 0.7711942195892334,
+      "grad_norm": 3.290184497833252,
       "learning_rate": 3.1143001007049346e-05,
-      "loss": 5.9784,
+      "loss": 6.275,
       "step": 750
     },
     {
       "epoch": 0.4028197381671702,
-      "grad_norm": 0.8087317943572998,
+      "grad_norm": 2.1646714210510254,
       "learning_rate": 2.988418932527694e-05,
-      "loss": 5.9354,
+      "loss": 6.3287,
       "step": 800
     },
     {
       "epoch": 0.42799597180261834,
-      "grad_norm": 0.7595328688621521,
+      "grad_norm": 2.3437819480895996,
       "learning_rate": 2.862537764350453e-05,
-      "loss": 5.8544,
+      "loss": 6.2043,
       "step": 850
     },
     {
       "epoch": 0.45317220543806647,
-      "grad_norm": 0.7424010038375854,
+      "grad_norm": 2.464350700378418,
       "learning_rate": 2.7366565961732126e-05,
-      "loss": 5.8309,
+      "loss": 6.2089,
       "step": 900
     },
     {
       "epoch": 0.4783484390735146,
-      "grad_norm": 0.7028313279151917,
+      "grad_norm": 2.730142831802368,
       "learning_rate": 2.610775427995972e-05,
-      "loss": 5.8075,
+      "loss": 6.146,
       "step": 950
     },
     {
       "epoch": 0.5035246727089627,
-      "grad_norm": 0.739640474319458,
+      "grad_norm": 2.321312189102173,
       "learning_rate": 2.4848942598187312e-05,
-      "loss": 5.7741,
+      "loss": 6.1237,
       "step": 1000
     },
     {
       "epoch": 0.5035246727089627,
-      "eval_loss": 5.472690582275391,
-      "eval_runtime": 37.0841,
-      "eval_samples_per_second": 45.114,
-      "eval_steps_per_second": 2.831,
+      "eval_loss": 6.018856525421143,
+      "eval_runtime": 33.2683,
+      "eval_samples_per_second": 50.288,
+      "eval_steps_per_second": 3.156,
       "step": 1000
     },
     {
       "epoch": 0.5287009063444109,
-      "grad_norm": 0.7200352549552917,
+      "grad_norm": 2.415438652038574,
       "learning_rate": 2.3590130916414905e-05,
-      "loss": 5.7262,
+      "loss": 6.2424,
       "step": 1050
     },
     {
       "epoch": 0.553877139979859,
-      "grad_norm": 0.7275882959365845,
+      "grad_norm": 2.567075252532959,
       "learning_rate": 2.2331319234642498e-05,
-      "loss": 5.7028,
+      "loss": 6.0269,
       "step": 1100
     },
     {
       "epoch": 0.5790533736153072,
-      "grad_norm": 0.7147909998893738,
+      "grad_norm": 3.5269229412078857,
       "learning_rate": 2.107250755287009e-05,
-      "loss": 5.6455,
+      "loss": 5.98,
       "step": 1150
     },
     {
       "epoch": 0.6042296072507553,
-      "grad_norm": 0.7415097951889038,
+      "grad_norm": 3.201200246810913,
       "learning_rate": 1.9813695871097685e-05,
-      "loss": 5.6142,
+      "loss": 6.0753,
       "step": 1200
     },
     {
       "epoch": 0.6294058408862034,
-      "grad_norm": 0.7115822434425354,
+      "grad_norm": 3.0480122566223145,
       "learning_rate": 1.8554884189325278e-05,
-      "loss": 5.6208,
+      "loss": 5.9747,
       "step": 1250
     },
     {
       "epoch": 0.6545820745216515,
-      "grad_norm": 0.6943323612213135,
+      "grad_norm": 3.005802869796753,
       "learning_rate": 1.729607250755287e-05,
-      "loss": 5.5773,
+      "loss": 5.9316,
       "step": 1300
     },
     {
       "epoch": 0.6797583081570997,
-      "grad_norm": 0.709179699420929,
+      "grad_norm": 2.7067759037017822,
       "learning_rate": 1.6037260825780464e-05,
-      "loss": 5.5679,
+      "loss": 5.956,
       "step": 1350
     },
     {
       "epoch": 0.7049345417925479,
-      "grad_norm": 0.7256223559379578,
+      "grad_norm": 2.6058285236358643,
       "learning_rate": 1.4778449144008057e-05,
-      "loss": 5.5666,
+      "loss": 6.0221,
       "step": 1400
     },
     {
       "epoch": 0.730110775427996,
-      "grad_norm": 0.7237951159477234,
+      "grad_norm": 3.462085485458374,
       "learning_rate": 1.3519637462235652e-05,
-      "loss": 5.5505,
+      "loss": 5.9177,
       "step": 1450
     },
     {
       "epoch": 0.7552870090634441,
-      "grad_norm": 0.7216870784759521,
+      "grad_norm": 3.490471839904785,
       "learning_rate": 1.2260825780463244e-05,
-      "loss": 5.4866,
+      "loss": 5.9148,
       "step": 1500
     },
     {
       "epoch": 0.7552870090634441,
-      "eval_loss": 5.239852428436279,
-      "eval_runtime": 42.2008,
-      "eval_samples_per_second": 39.644,
-      "eval_steps_per_second": 2.488,
+      "eval_loss": 5.870590686798096,
+      "eval_runtime": 35.2827,
+      "eval_samples_per_second": 47.417,
+      "eval_steps_per_second": 2.976,
       "step": 1500
     },
     {
       "epoch": 0.7804632426988922,
-      "grad_norm": 0.7159505486488342,
+      "grad_norm": 3.5403401851654053,
       "learning_rate": 1.1002014098690837e-05,
-      "loss": 5.4953,
+      "loss": 5.9664,
       "step": 1550
     },
     {
       "epoch": 0.8056394763343404,
-      "grad_norm": 0.6942981481552124,
+      "grad_norm": 3.068039894104004,
       "learning_rate": 9.743202416918428e-06,
-      "loss": 5.47,
+      "loss": 5.8968,
       "step": 1600
     },
     {
       "epoch": 0.8308157099697885,
-      "grad_norm": 0.6944827437400818,
+      "grad_norm": 2.8226842880249023,
       "learning_rate": 8.484390735146023e-06,
-      "loss": 5.4747,
+      "loss": 5.911,
       "step": 1650
     },
     {
       "epoch": 0.8559919436052367,
-      "grad_norm": 0.6906715631484985,
+      "grad_norm": 4.368133068084717,
       "learning_rate": 7.225579053373615e-06,
-      "loss": 5.4678,
+      "loss": 5.9657,
       "step": 1700
     },
     {
       "epoch": 0.8811681772406847,
-      "grad_norm": 0.6964088082313538,
+      "grad_norm": 2.99777889251709,
       "learning_rate": 5.966767371601209e-06,
-      "loss": 5.4578,
+      "loss": 5.9008,
       "step": 1750
     },
     {
       "epoch": 0.9063444108761329,
-      "grad_norm": 0.6903862357139587,
+      "grad_norm": 2.882019281387329,
       "learning_rate": 4.7079556898288016e-06,
-      "loss": 5.4521,
+      "loss": 5.8568,
       "step": 1800
     },
     {
       "epoch": 0.9315206445115811,
-      "grad_norm": 0.663235604763031,
+      "grad_norm": 3.1223084926605225,
       "learning_rate": 3.449144008056395e-06,
-      "loss": 5.4358,
+      "loss": 5.8966,
       "step": 1850
     },
     {
       "epoch": 0.9566968781470292,
-      "grad_norm": 0.8237895965576172,
+      "grad_norm": 2.971224308013916,
       "learning_rate": 2.1903323262839883e-06,
-      "loss": 5.4173,
+      "loss": 5.824,
       "step": 1900
     },
     {
       "epoch": 0.9818731117824774,
-      "grad_norm": 0.6885905265808105,
+      "grad_norm": 2.8992207050323486,
       "learning_rate": 9.31520644511581e-07,
-      "loss": 5.4231,
+      "loss": 5.8318,
       "step": 1950
     }
   ],

checkpoint-1986/training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:fd35ffac2ed4cf9159515c227a2e8bb8761c759f619e17bd96f19d5dd4c854ad
+oid sha256:00a8fef87f609a94dc07e98c9168ebeac65261d0032405c3c986af25ab8cfec3
 size 5713

checkpoint-500/added_tokens.json

@@ -0,0 +1,3 @@
+{
+  "<mask>": 59811
+}

checkpoint-500/optimizer.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:9745d97b78ceff16ed4689c2fc51e5f355b5e5eb2698f609c767d75a2844f230
-size 151392779
+oid sha256:920db38bd4c2463783ef7f3336db8a1a2d8aaa539b9c9d17e0fc3ed5376cfeea
+size 396383563

checkpoint-500/pytorch_model.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:ca409ad8e52fff5c20c44e8a2f413510d9bec8f2193f0b64bd95861a7bb80ec1
-size 199234413
+oid sha256:1f541d397f94c60d4b4f722a75dd0a68d5fcba4c2c02f3f021c5c31b561cce79
+size 199236461

checkpoint-500/rng_state.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:d040fa084e8dd5f024209465e525c8f2ec894da956a0bcfb8f5f69e53ce7e10b
+oid sha256:b53687351ff03aa2e66e4de762a436528cb6e0c8c1f3a18483ad6f1ffeab4011
 size 14391

checkpoint-500/special_tokens_map.json

@@ -1,5 +1,12 @@
 {
   "eos_token": "</s>",
+  "mask_token": {
+    "content": "<mask>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
   "pad_token": "<pad>",
   "unk_token": "<unk>"
 }

checkpoint-500/tokenizer_config.json

@@ -23,11 +23,20 @@
       "rstrip": false,
       "single_word": false,
       "special": true
+    },
+    "59811": {
+      "content": "<mask>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
     }
   },
   "clean_up_tokenization_spaces": false,
   "eos_token": "</s>",
   "extra_special_tokens": {},
+  "mask_token": "<mask>",
   "model_max_length": 512,
   "pad_token": "<pad>",
   "separate_vocabs": false,

checkpoint-500/trainer_state.json

@@ -1,6 +1,6 @@
 {
   "best_global_step": 500,
-  "best_metric": 5.968835353851318,
+  "best_metric": 6.588558197021484,
   "best_model_checkpoint": "./nyanja_english_warmup\\checkpoint-500",
   "epoch": 0.25176233635448136,
   "eval_steps": 500,
@@ -11,80 +11,80 @@
   "log_history": [
     {
       "epoch": 0.025176233635448138,
-      "grad_norm": 0.947960615158081,
+      "grad_norm": 1.8817049264907837,
       "learning_rate": 4.876636455186304e-05,
-      "loss": 9.5286,
+      "loss": 9.8135,
       "step": 50
     },
     {
       "epoch": 0.050352467270896276,
-      "grad_norm": 1.194724440574646,
+      "grad_norm": 1.9862349033355713,
       "learning_rate": 4.7507552870090635e-05,
-      "loss": 8.2179,
+      "loss": 8.8496,
       "step": 100
     },
     {
       "epoch": 0.0755287009063444,
-      "grad_norm": 0.9894101023674011,
+      "grad_norm": 2.0134317874908447,
       "learning_rate": 4.624874118831823e-05,
-      "loss": 7.5072,
+      "loss": 8.3485,
       "step": 150
     },
     {
       "epoch": 0.10070493454179255,
-      "grad_norm": 0.9540104269981384,
+      "grad_norm": 2.8336329460144043,
       "learning_rate": 4.498992950654582e-05,
-      "loss": 7.1644,
+      "loss": 7.9298,
       "step": 200
     },
     {
       "epoch": 0.12588116817724068,
-      "grad_norm": 1.0262048244476318,
+      "grad_norm": 1.8131136894226074,
       "learning_rate": 4.3731117824773415e-05,
-      "loss": 6.9377,
+      "loss": 7.6853,
       "step": 250
     },
     {
       "epoch": 0.1510574018126888,
-      "grad_norm": 0.916906476020813,
+      "grad_norm": 2.619373083114624,
       "learning_rate": 4.2472306143001015e-05,
-      "loss": 6.7447,
+      "loss": 7.3954,
       "step": 300
     },
     {
       "epoch": 0.17623363544813697,
-      "grad_norm": 0.9376256465911865,
+      "grad_norm": 2.30450439453125,
       "learning_rate": 4.12134944612286e-05,
-      "loss": 6.6128,
+      "loss": 7.1206,
       "step": 350
     },
     {
       "epoch": 0.2014098690835851,
-      "grad_norm": 0.8047090768814087,
+      "grad_norm": 2.5505969524383545,
       "learning_rate": 3.9954682779456194e-05,
-      "loss": 6.5074,
+      "loss": 6.9686,
       "step": 400
     },
     {
       "epoch": 0.22658610271903323,
-      "grad_norm": 0.9348500967025757,
+      "grad_norm": 2.0169050693511963,
       "learning_rate": 3.869587109768379e-05,
-      "loss": 6.4087,
+      "loss": 6.807,
       "step": 450
     },
     {
       "epoch": 0.25176233635448136,
-      "grad_norm": 0.8273019194602966,
+      "grad_norm": 2.125231981277466,
       "learning_rate": 3.743705941591138e-05,
-      "loss": 6.3114,
+      "loss": 6.7269,
       "step": 500
     },
     {
       "epoch": 0.25176233635448136,
-      "eval_loss": 5.968835353851318,
-      "eval_runtime": 41.9731,
-      "eval_samples_per_second": 39.859,
-      "eval_steps_per_second": 2.502,
+      "eval_loss": 6.588558197021484,
+      "eval_runtime": 35.6768,
+      "eval_samples_per_second": 46.893,
+      "eval_steps_per_second": 2.943,
       "step": 500
     }
   ],

checkpoint-500/training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:fd35ffac2ed4cf9159515c227a2e8bb8761c759f619e17bd96f19d5dd4c854ad
+oid sha256:00a8fef87f609a94dc07e98c9168ebeac65261d0032405c3c986af25ab8cfec3
 size 5713

config.json

@@ -20,15 +20,15 @@
   "decoder_layerdrop": 0.0,
   "decoder_layers": 6,
   "decoder_start_token_id": 59810,
-  "decoder_vocab_size": 59811,
+  "decoder_vocab_size": 59812,
   "dropout": 0.1,
   "encoder_attention_heads": 8,
   "encoder_ffn_dim": 2048,
   "encoder_layerdrop": 0.0,
   "encoder_layers": 6,
   "eos_token_id": 0,
-  "final_training_loss": 6.0598699871266835,
-  "final_validation_loss": 5.239852428436279,
+  "final_training_loss": 6.501624530895959,
+  "final_validation_loss": 5.806530952453613,
   "forced_eos_token_id": 0,
   "id2label": {
     "0": "LABEL_0",
@@ -59,6 +59,6 @@
   "transformers_version": "4.55.4",
   "use_cache": true,
   "version": "v1",
-  "vocab_size": 59811,
+  "vocab_size": 59812,
   "warmup_type": "encoder_lm"
 }

model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:bb0a24fd315411070f2040c48bc2558a9598a57ca02ff60dc9fe44048eba942a
-size 299315212
+oid sha256:13acf2a71a4f14d4469c5cd80c3759ec81b5b75d75248a0633c061f6b1acaa45
+size 299317264

special_tokens_map.json

@@ -1,5 +1,12 @@
 {
   "eos_token": "</s>",
+  "mask_token": {
+    "content": "<mask>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
   "pad_token": "<pad>",
   "unk_token": "<unk>"
 }

tokenizer_config.json

@@ -23,11 +23,20 @@
       "rstrip": false,
       "single_word": false,
       "special": true
+    },
+    "59811": {
+      "content": "<mask>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
     }
   },
   "clean_up_tokenization_spaces": false,
   "eos_token": "</s>",
   "extra_special_tokens": {},
+  "mask_token": "<mask>",
   "model_max_length": 512,
   "pad_token": "<pad>",
   "separate_vocabs": false,