| { |
| "epoch": 3.5724508050089447, |
| "global_step": 1000, |
| "max_steps": 2232, |
| "logging_steps": 5, |
| "eval_steps": 100, |
| "save_steps": 100, |
| "train_batch_size": 16, |
| "num_train_epochs": 8, |
| "num_input_tokens_seen": 0, |
| "total_flos": 4.722006162277663e+17, |
| "log_history": [ |
| { |
| "loss": 10.1939, |
| "grad_norm": 74.8128433227539, |
| "learning_rate": 3.75e-05, |
| "epoch": 0.017889087656529516, |
| "step": 5 |
| }, |
| { |
| "loss": 3.9273, |
| "grad_norm": 12.053423881530762, |
| "learning_rate": 7.5e-05, |
| "epoch": 0.03577817531305903, |
| "step": 10 |
| }, |
| { |
| "loss": 1.4898, |
| "grad_norm": 40.130340576171875, |
| "learning_rate": 0.0001125, |
| "epoch": 0.05366726296958855, |
| "step": 15 |
| }, |
| { |
| "loss": 0.9064, |
| "grad_norm": 3.677988290786743, |
| "learning_rate": 0.00015, |
| "epoch": 0.07155635062611806, |
| "step": 20 |
| }, |
| { |
| "loss": 0.8129, |
| "grad_norm": 1.335487961769104, |
| "learning_rate": 0.00018749999999999998, |
| "epoch": 0.08944543828264759, |
| "step": 25 |
| }, |
| { |
| "loss": 1.1672, |
| "grad_norm": 29.67264175415039, |
| "learning_rate": 0.000225, |
| "epoch": 0.1073345259391771, |
| "step": 30 |
| }, |
| { |
| "loss": 0.7385, |
| "grad_norm": 1.039961576461792, |
| "learning_rate": 0.0002625, |
| "epoch": 0.1252236135957066, |
| "step": 35 |
| }, |
| { |
| "loss": 0.6884, |
| "grad_norm": 0.4314720928668976, |
| "learning_rate": 0.0003, |
| "epoch": 0.14311270125223613, |
| "step": 40 |
| }, |
| { |
| "loss": 0.6748, |
| "grad_norm": 0.40387117862701416, |
| "learning_rate": 0.0002999961486050259, |
| "epoch": 0.16100178890876565, |
| "step": 45 |
| }, |
| { |
| "loss": 0.6628, |
| "grad_norm": 0.448535293340683, |
| "learning_rate": 0.00029998459461788026, |
| "epoch": 0.17889087656529518, |
| "step": 50 |
| }, |
| { |
| "loss": 0.6659, |
| "grad_norm": 0.5291304588317871, |
| "learning_rate": 0.0002999653386318826, |
| "epoch": 0.1967799642218247, |
| "step": 55 |
| }, |
| { |
| "loss": 0.6517, |
| "grad_norm": 0.6947611570358276, |
| "learning_rate": 0.0002999383816358651, |
| "epoch": 0.2146690518783542, |
| "step": 60 |
| }, |
| { |
| "loss": 0.7263, |
| "grad_norm": 0.5037692785263062, |
| "learning_rate": 0.0002999037250141215, |
| "epoch": 0.23255813953488372, |
| "step": 65 |
| }, |
| { |
| "loss": 0.6464, |
| "grad_norm": 0.35925614833831787, |
| "learning_rate": 0.0002998613705463364, |
| "epoch": 0.2504472271914132, |
| "step": 70 |
| }, |
| { |
| "loss": 0.6531, |
| "grad_norm": 0.7011401653289795, |
| "learning_rate": 0.0002998113204074936, |
| "epoch": 0.26833631484794274, |
| "step": 75 |
| }, |
| { |
| "loss": 0.6563, |
| "grad_norm": 0.29620787501335144, |
| "learning_rate": 0.0002997535771677644, |
| "epoch": 0.28622540250447226, |
| "step": 80 |
| }, |
| { |
| "loss": 0.6555, |
| "grad_norm": 0.24026329815387726, |
| "learning_rate": 0.00029968814379237584, |
| "epoch": 0.3041144901610018, |
| "step": 85 |
| }, |
| { |
| "loss": 0.6213, |
| "grad_norm": 0.17133238911628723, |
| "learning_rate": 0.00029961502364145824, |
| "epoch": 0.3220035778175313, |
| "step": 90 |
| }, |
| { |
| "loss": 0.6216, |
| "grad_norm": 0.24976064264774323, |
| "learning_rate": 0.0002995342204698726, |
| "epoch": 0.33989266547406083, |
| "step": 95 |
| }, |
| { |
| "loss": 0.6357, |
| "grad_norm": 0.527025043964386, |
| "learning_rate": 0.00029944573842701806, |
| "epoch": 0.35778175313059035, |
| "step": 100 |
| }, |
| { |
| "eval_loss": 0.5987367630004883, |
| "eval_runtime": 20.6719, |
| "eval_samples_per_second": 162.83, |
| "eval_steps_per_second": 10.207, |
| "epoch": 0.35778175313059035, |
| "step": 100 |
| }, |
| { |
| "loss": 0.6205, |
| "grad_norm": 0.3432077467441559, |
| "learning_rate": 0.00029934958205661853, |
| "epoch": 0.3756708407871199, |
| "step": 105 |
| }, |
| { |
| "loss": 0.6071, |
| "grad_norm": 0.22968943417072296, |
| "learning_rate": 0.00029924575629648953, |
| "epoch": 0.3935599284436494, |
| "step": 110 |
| }, |
| { |
| "loss": 0.6096, |
| "grad_norm": 0.402422696352005, |
| "learning_rate": 0.0002991342664782845, |
| "epoch": 0.41144901610017887, |
| "step": 115 |
| }, |
| { |
| "loss": 0.597, |
| "grad_norm": 0.3375358283519745, |
| "learning_rate": 0.00029901511832722104, |
| "epoch": 0.4293381037567084, |
| "step": 120 |
| }, |
| { |
| "loss": 0.5821, |
| "grad_norm": 0.321890652179718, |
| "learning_rate": 0.00029888831796178714, |
| "epoch": 0.4472271914132379, |
| "step": 125 |
| }, |
| { |
| "loss": 0.5982, |
| "grad_norm": 0.20958741009235382, |
| "learning_rate": 0.0002987538718934266, |
| "epoch": 0.46511627906976744, |
| "step": 130 |
| }, |
| { |
| "loss": 0.5672, |
| "grad_norm": 0.2855433225631714, |
| "learning_rate": 0.000298611787026205, |
| "epoch": 0.48300536672629696, |
| "step": 135 |
| }, |
| { |
| "loss": 0.5756, |
| "grad_norm": 0.3264622688293457, |
| "learning_rate": 0.0002984620706564548, |
| "epoch": 0.5008944543828264, |
| "step": 140 |
| }, |
| { |
| "loss": 0.5643, |
| "grad_norm": 0.3137723505496979, |
| "learning_rate": 0.0002983047304724011, |
| "epoch": 0.518783542039356, |
| "step": 145 |
| }, |
| { |
| "loss": 0.5854, |
| "grad_norm": 0.2823493480682373, |
| "learning_rate": 0.00029813977455376634, |
| "epoch": 0.5366726296958855, |
| "step": 150 |
| }, |
| { |
| "loss": 0.5702, |
| "grad_norm": 0.24358761310577393, |
| "learning_rate": 0.00029796721137135597, |
| "epoch": 0.554561717352415, |
| "step": 155 |
| }, |
| { |
| "loss": 0.5487, |
| "grad_norm": 0.31340864300727844, |
| "learning_rate": 0.00029778704978662284, |
| "epoch": 0.5724508050089445, |
| "step": 160 |
| }, |
| { |
| "loss": 0.5478, |
| "grad_norm": 0.3177396357059479, |
| "learning_rate": 0.0002975992990512127, |
| "epoch": 0.590339892665474, |
| "step": 165 |
| }, |
| { |
| "loss": 0.5501, |
| "grad_norm": 0.3493712842464447, |
| "learning_rate": 0.0002974039688064886, |
| "epoch": 0.6082289803220036, |
| "step": 170 |
| }, |
| { |
| "loss": 0.5411, |
| "grad_norm": 0.2914203107357025, |
| "learning_rate": 0.00029720106908303625, |
| "epoch": 0.6261180679785331, |
| "step": 175 |
| }, |
| { |
| "loss": 0.5306, |
| "grad_norm": 0.29958376288414, |
| "learning_rate": 0.0002969906103001486, |
| "epoch": 0.6440071556350626, |
| "step": 180 |
| }, |
| { |
| "loss": 0.5304, |
| "grad_norm": 0.26977163553237915, |
| "learning_rate": 0.00029677260326529107, |
| "epoch": 0.6618962432915921, |
| "step": 185 |
| }, |
| { |
| "loss": 0.5231, |
| "grad_norm": 0.2629687786102295, |
| "learning_rate": 0.0002965470591735462, |
| "epoch": 0.6797853309481217, |
| "step": 190 |
| }, |
| { |
| "loss": 0.524, |
| "grad_norm": 0.3607335388660431, |
| "learning_rate": 0.0002963139896070391, |
| "epoch": 0.6976744186046512, |
| "step": 195 |
| }, |
| { |
| "loss": 0.522, |
| "grad_norm": 0.36577939987182617, |
| "learning_rate": 0.0002960734065343426, |
| "epoch": 0.7155635062611807, |
| "step": 200 |
| }, |
| { |
| "eval_loss": 0.5033101439476013, |
| "eval_runtime": 20.6749, |
| "eval_samples_per_second": 162.806, |
| "eval_steps_per_second": 10.206, |
| "epoch": 0.7155635062611807, |
| "step": 200 |
| }, |
| { |
| "loss": 0.5033, |
| "grad_norm": 0.30699774622917175, |
| "learning_rate": 0.00029582532230986244, |
| "epoch": 0.7334525939177102, |
| "step": 205 |
| }, |
| { |
| "loss": 0.5037, |
| "grad_norm": 0.27759912610054016, |
| "learning_rate": 0.0002955697496732031, |
| "epoch": 0.7513416815742398, |
| "step": 210 |
| }, |
| { |
| "loss": 0.5085, |
| "grad_norm": 0.32028377056121826, |
| "learning_rate": 0.0002953067017485136, |
| "epoch": 0.7692307692307693, |
| "step": 215 |
| }, |
| { |
| "loss": 0.493, |
| "grad_norm": 0.5791727304458618, |
| "learning_rate": 0.00029503619204381313, |
| "epoch": 0.7871198568872988, |
| "step": 220 |
| }, |
| { |
| "loss": 0.5089, |
| "grad_norm": 0.44022566080093384, |
| "learning_rate": 0.0002947582344502981, |
| "epoch": 0.8050089445438283, |
| "step": 225 |
| }, |
| { |
| "loss": 0.4986, |
| "grad_norm": 0.304227739572525, |
| "learning_rate": 0.0002944728432416282, |
| "epoch": 0.8228980322003577, |
| "step": 230 |
| }, |
| { |
| "loss": 0.4991, |
| "grad_norm": 0.2987827956676483, |
| "learning_rate": 0.0002941800330731936, |
| "epoch": 0.8407871198568873, |
| "step": 235 |
| }, |
| { |
| "loss": 0.4798, |
| "grad_norm": 0.32087433338165283, |
| "learning_rate": 0.0002938798189813625, |
| "epoch": 0.8586762075134168, |
| "step": 240 |
| }, |
| { |
| "loss": 0.4839, |
| "grad_norm": 0.354798287153244, |
| "learning_rate": 0.0002935722163827087, |
| "epoch": 0.8765652951699463, |
| "step": 245 |
| }, |
| { |
| "loss": 0.4874, |
| "grad_norm": 0.5133390426635742, |
| "learning_rate": 0.0002932572410732204, |
| "epoch": 0.8944543828264758, |
| "step": 250 |
| }, |
| { |
| "loss": 0.4749, |
| "grad_norm": 0.3183499574661255, |
| "learning_rate": 0.00029293490922748856, |
| "epoch": 0.9123434704830053, |
| "step": 255 |
| }, |
| { |
| "loss": 0.471, |
| "grad_norm": 0.3271007537841797, |
| "learning_rate": 0.0002926052373978764, |
| "epoch": 0.9302325581395349, |
| "step": 260 |
| }, |
| { |
| "loss": 0.459, |
| "grad_norm": 1.1660605669021606, |
| "learning_rate": 0.00029226824251366967, |
| "epoch": 0.9481216457960644, |
| "step": 265 |
| }, |
| { |
| "loss": 0.4369, |
| "grad_norm": 0.48370498418807983, |
| "learning_rate": 0.00029192394188020716, |
| "epoch": 0.9660107334525939, |
| "step": 270 |
| }, |
| { |
| "loss": 0.443, |
| "grad_norm": 0.3688034117221832, |
| "learning_rate": 0.0002915723531779919, |
| "epoch": 0.9838998211091234, |
| "step": 275 |
| }, |
| { |
| "loss": 0.4541, |
| "grad_norm": 0.7489350438117981, |
| "learning_rate": 0.00029121349446178333, |
| "epoch": 1.0, |
| "step": 280 |
| }, |
| { |
| "loss": 0.4349, |
| "grad_norm": 0.3898984491825104, |
| "learning_rate": 0.00029084738415967017, |
| "epoch": 1.0178890876565294, |
| "step": 285 |
| }, |
| { |
| "loss": 0.4455, |
| "grad_norm": 0.488726407289505, |
| "learning_rate": 0.0002904740410721242, |
| "epoch": 1.035778175313059, |
| "step": 290 |
| }, |
| { |
| "loss": 0.4431, |
| "grad_norm": 0.4695197641849518, |
| "learning_rate": 0.00029009348437103455, |
| "epoch": 1.0536672629695885, |
| "step": 295 |
| }, |
| { |
| "loss": 0.4223, |
| "grad_norm": 0.3967965543270111, |
| "learning_rate": 0.0002897057335987235, |
| "epoch": 1.071556350626118, |
| "step": 300 |
| }, |
| { |
| "eval_loss": 0.4258047640323639, |
| "eval_runtime": 20.6743, |
| "eval_samples_per_second": 162.811, |
| "eval_steps_per_second": 10.206, |
| "epoch": 1.071556350626118, |
| "step": 300 |
| }, |
| { |
| "loss": 0.4262, |
| "grad_norm": 0.502098023891449, |
| "learning_rate": 0.00028931080866694274, |
| "epoch": 1.0894454382826475, |
| "step": 305 |
| }, |
| { |
| "loss": 0.4155, |
| "grad_norm": 0.3177882134914398, |
| "learning_rate": 0.0002889087298558508, |
| "epoch": 1.1073345259391771, |
| "step": 310 |
| }, |
| { |
| "loss": 0.4151, |
| "grad_norm": 0.39488786458969116, |
| "learning_rate": 0.0002884995178129719, |
| "epoch": 1.1252236135957066, |
| "step": 315 |
| }, |
| { |
| "loss": 0.4025, |
| "grad_norm": 0.5109437704086304, |
| "learning_rate": 0.0002880831935521355, |
| "epoch": 1.1431127012522362, |
| "step": 320 |
| }, |
| { |
| "loss": 0.4054, |
| "grad_norm": 0.5653850436210632, |
| "learning_rate": 0.00028765977845239706, |
| "epoch": 1.1610017889087656, |
| "step": 325 |
| }, |
| { |
| "loss": 0.3982, |
| "grad_norm": 0.7002792358398438, |
| "learning_rate": 0.00028722929425694044, |
| "epoch": 1.1788908765652952, |
| "step": 330 |
| }, |
| { |
| "loss": 0.4375, |
| "grad_norm": 0.33770787715911865, |
| "learning_rate": 0.0002867917630719612, |
| "epoch": 1.1967799642218246, |
| "step": 335 |
| }, |
| { |
| "loss": 0.4053, |
| "grad_norm": 0.4120868444442749, |
| "learning_rate": 0.00028634720736553143, |
| "epoch": 1.2146690518783543, |
| "step": 340 |
| }, |
| { |
| "loss": 0.411, |
| "grad_norm": 0.4081322252750397, |
| "learning_rate": 0.00028589564996644594, |
| "epoch": 1.2325581395348837, |
| "step": 345 |
| }, |
| { |
| "loss": 0.4167, |
| "grad_norm": 0.6889946460723877, |
| "learning_rate": 0.0002854371140630501, |
| "epoch": 1.250447227191413, |
| "step": 350 |
| }, |
| { |
| "loss": 0.4196, |
| "grad_norm": 0.3247791826725006, |
| "learning_rate": 0.00028497162320204885, |
| "epoch": 1.2683363148479427, |
| "step": 355 |
| }, |
| { |
| "loss": 0.4091, |
| "grad_norm": 0.41395092010498047, |
| "learning_rate": 0.00028449920128729766, |
| "epoch": 1.2862254025044724, |
| "step": 360 |
| }, |
| { |
| "loss": 0.3958, |
| "grad_norm": 0.3653516173362732, |
| "learning_rate": 0.00028401987257857514, |
| "epoch": 1.3041144901610018, |
| "step": 365 |
| }, |
| { |
| "loss": 0.3898, |
| "grad_norm": 0.3595719337463379, |
| "learning_rate": 0.0002835336616903369, |
| "epoch": 1.3220035778175312, |
| "step": 370 |
| }, |
| { |
| "loss": 0.3604, |
| "grad_norm": 0.7938292622566223, |
| "learning_rate": 0.000283040593590452, |
| "epoch": 1.3398926654740608, |
| "step": 375 |
| }, |
| { |
| "loss": 0.382, |
| "grad_norm": 0.562586784362793, |
| "learning_rate": 0.00028254069359892034, |
| "epoch": 1.3577817531305905, |
| "step": 380 |
| }, |
| { |
| "loss": 0.385, |
| "grad_norm": 0.29362237453460693, |
| "learning_rate": 0.0002820339873865729, |
| "epoch": 1.3756708407871199, |
| "step": 385 |
| }, |
| { |
| "loss": 0.3776, |
| "grad_norm": 0.38946282863616943, |
| "learning_rate": 0.00028152050097375304, |
| "epoch": 1.3935599284436493, |
| "step": 390 |
| }, |
| { |
| "loss": 0.3756, |
| "grad_norm": 0.3675141930580139, |
| "learning_rate": 0.00028100026072898067, |
| "epoch": 1.411449016100179, |
| "step": 395 |
| }, |
| { |
| "loss": 0.3627, |
| "grad_norm": 0.4281761050224304, |
| "learning_rate": 0.00028047329336759806, |
| "epoch": 1.4293381037567083, |
| "step": 400 |
| }, |
| { |
| "eval_loss": 0.3722744286060333, |
| "eval_runtime": 20.6775, |
| "eval_samples_per_second": 162.785, |
| "eval_steps_per_second": 10.204, |
| "epoch": 1.4293381037567083, |
| "step": 400 |
| }, |
| { |
| "loss": 0.3637, |
| "grad_norm": 0.3468483090400696, |
| "learning_rate": 0.0002799396259503977, |
| "epoch": 1.447227191413238, |
| "step": 405 |
| }, |
| { |
| "loss": 0.3729, |
| "grad_norm": 0.4413630962371826, |
| "learning_rate": 0.00027939928588223314, |
| "epoch": 1.4651162790697674, |
| "step": 410 |
| }, |
| { |
| "loss": 0.3511, |
| "grad_norm": 0.41624078154563904, |
| "learning_rate": 0.00027885230091061127, |
| "epoch": 1.483005366726297, |
| "step": 415 |
| }, |
| { |
| "loss": 0.3599, |
| "grad_norm": 0.42356589436531067, |
| "learning_rate": 0.00027829869912426777, |
| "epoch": 1.5008944543828264, |
| "step": 420 |
| }, |
| { |
| "loss": 0.3586, |
| "grad_norm": 0.4348715543746948, |
| "learning_rate": 0.0002777385089517244, |
| "epoch": 1.518783542039356, |
| "step": 425 |
| }, |
| { |
| "loss": 0.3487, |
| "grad_norm": 0.5163382291793823, |
| "learning_rate": 0.00027717175915982945, |
| "epoch": 1.5366726296958855, |
| "step": 430 |
| }, |
| { |
| "loss": 0.3551, |
| "grad_norm": 0.47645968198776245, |
| "learning_rate": 0.0002765984788522801, |
| "epoch": 1.5545617173524149, |
| "step": 435 |
| }, |
| { |
| "loss": 0.3523, |
| "grad_norm": 0.5956697463989258, |
| "learning_rate": 0.0002760186974681285, |
| "epoch": 1.5724508050089445, |
| "step": 440 |
| }, |
| { |
| "loss": 0.3842, |
| "grad_norm": 0.3199741244316101, |
| "learning_rate": 0.0002754324447802693, |
| "epoch": 1.5903398926654742, |
| "step": 445 |
| }, |
| { |
| "loss": 0.363, |
| "grad_norm": 0.4985601007938385, |
| "learning_rate": 0.00027483975089391126, |
| "epoch": 1.6082289803220036, |
| "step": 450 |
| }, |
| { |
| "loss": 0.3573, |
| "grad_norm": 0.43190059065818787, |
| "learning_rate": 0.0002742406462450311, |
| "epoch": 1.626118067978533, |
| "step": 455 |
| }, |
| { |
| "loss": 0.3399, |
| "grad_norm": 0.3566244840621948, |
| "learning_rate": 0.00027363516159881066, |
| "epoch": 1.6440071556350626, |
| "step": 460 |
| }, |
| { |
| "loss": 0.3465, |
| "grad_norm": 0.4065592586994171, |
| "learning_rate": 0.0002730233280480569, |
| "epoch": 1.6618962432915922, |
| "step": 465 |
| }, |
| { |
| "loss": 0.3324, |
| "grad_norm": 0.36722782254219055, |
| "learning_rate": 0.0002724051770116052, |
| "epoch": 1.6797853309481217, |
| "step": 470 |
| }, |
| { |
| "loss": 0.325, |
| "grad_norm": 0.3565702736377716, |
| "learning_rate": 0.00027178074023270635, |
| "epoch": 1.697674418604651, |
| "step": 475 |
| }, |
| { |
| "loss": 0.3679, |
| "grad_norm": 0.5317059755325317, |
| "learning_rate": 0.00027115004977739586, |
| "epoch": 1.7155635062611807, |
| "step": 480 |
| }, |
| { |
| "loss": 0.3433, |
| "grad_norm": 0.30274462699890137, |
| "learning_rate": 0.00027051313803284774, |
| "epoch": 1.7334525939177103, |
| "step": 485 |
| }, |
| { |
| "loss": 0.3492, |
| "grad_norm": 4.536398887634277, |
| "learning_rate": 0.00026987003770571124, |
| "epoch": 1.7513416815742398, |
| "step": 490 |
| }, |
| { |
| "loss": 0.3506, |
| "grad_norm": 0.4281180202960968, |
| "learning_rate": 0.0002692207818204313, |
| "epoch": 1.7692307692307692, |
| "step": 495 |
| }, |
| { |
| "loss": 0.3417, |
| "grad_norm": 0.3851430416107178, |
| "learning_rate": 0.0002685654037175525, |
| "epoch": 1.7871198568872988, |
| "step": 500 |
| }, |
| { |
| "eval_loss": 0.3463773727416992, |
| "eval_runtime": 20.6737, |
| "eval_samples_per_second": 162.815, |
| "eval_steps_per_second": 10.206, |
| "epoch": 1.7871198568872988, |
| "step": 500 |
| }, |
| { |
| "loss": 0.3287, |
| "grad_norm": 0.41302162408828735, |
| "learning_rate": 0.0002679039370520074, |
| "epoch": 1.8050089445438284, |
| "step": 505 |
| }, |
| { |
| "loss": 0.3292, |
| "grad_norm": 0.3512372076511383, |
| "learning_rate": 0.00026723641579138777, |
| "epoch": 1.8228980322003578, |
| "step": 510 |
| }, |
| { |
| "loss": 0.3241, |
| "grad_norm": 0.4498589336872101, |
| "learning_rate": 0.0002665628742142008, |
| "epoch": 1.8407871198568873, |
| "step": 515 |
| }, |
| { |
| "loss": 0.3302, |
| "grad_norm": 0.532430112361908, |
| "learning_rate": 0.0002658833469081082, |
| "epoch": 1.8586762075134167, |
| "step": 520 |
| }, |
| { |
| "loss": 0.3254, |
| "grad_norm": 0.40190622210502625, |
| "learning_rate": 0.0002651978687681509, |
| "epoch": 1.8765652951699463, |
| "step": 525 |
| }, |
| { |
| "loss": 0.3212, |
| "grad_norm": 0.4508027732372284, |
| "learning_rate": 0.00026450647499495626, |
| "epoch": 1.894454382826476, |
| "step": 530 |
| }, |
| { |
| "loss": 0.3191, |
| "grad_norm": 0.48150140047073364, |
| "learning_rate": 0.00026380920109293104, |
| "epoch": 1.9123434704830053, |
| "step": 535 |
| }, |
| { |
| "loss": 0.3374, |
| "grad_norm": 0.34987515211105347, |
| "learning_rate": 0.00026310608286843795, |
| "epoch": 1.9302325581395348, |
| "step": 540 |
| }, |
| { |
| "loss": 0.3193, |
| "grad_norm": 0.2754597067832947, |
| "learning_rate": 0.00026239715642795684, |
| "epoch": 1.9481216457960644, |
| "step": 545 |
| }, |
| { |
| "loss": 0.3276, |
| "grad_norm": 0.35444873571395874, |
| "learning_rate": 0.00026168245817623085, |
| "epoch": 1.966010733452594, |
| "step": 550 |
| }, |
| { |
| "loss": 0.3028, |
| "grad_norm": 0.3781956434249878, |
| "learning_rate": 0.00026096202481439674, |
| "epoch": 1.9838998211091234, |
| "step": 555 |
| }, |
| { |
| "loss": 0.3163, |
| "grad_norm": 0.43553584814071655, |
| "learning_rate": 0.00026023589333810015, |
| "epoch": 2.0, |
| "step": 560 |
| }, |
| { |
| "loss": 0.3041, |
| "grad_norm": 0.3745397627353668, |
| "learning_rate": 0.00025950410103559604, |
| "epoch": 2.0178890876565294, |
| "step": 565 |
| }, |
| { |
| "loss": 0.314, |
| "grad_norm": 0.4507661759853363, |
| "learning_rate": 0.00025876668548583374, |
| "epoch": 2.035778175313059, |
| "step": 570 |
| }, |
| { |
| "loss": 0.2911, |
| "grad_norm": 0.45311978459358215, |
| "learning_rate": 0.00025802368455652704, |
| "epoch": 2.0536672629695887, |
| "step": 575 |
| }, |
| { |
| "loss": 0.2791, |
| "grad_norm": 0.3784424066543579, |
| "learning_rate": 0.00025727513640220985, |
| "epoch": 2.071556350626118, |
| "step": 580 |
| }, |
| { |
| "loss": 0.2781, |
| "grad_norm": 0.37560757994651794, |
| "learning_rate": 0.000256521079462277, |
| "epoch": 2.0894454382826475, |
| "step": 585 |
| }, |
| { |
| "loss": 0.2705, |
| "grad_norm": 0.4536455571651459, |
| "learning_rate": 0.0002557615524590097, |
| "epoch": 2.107334525939177, |
| "step": 590 |
| }, |
| { |
| "loss": 0.2845, |
| "grad_norm": 0.8856727480888367, |
| "learning_rate": 0.000254996594395588, |
| "epoch": 2.1252236135957068, |
| "step": 595 |
| }, |
| { |
| "loss": 0.2894, |
| "grad_norm": 0.5242048501968384, |
| "learning_rate": 0.00025422624455408686, |
| "epoch": 2.143112701252236, |
| "step": 600 |
| }, |
| { |
| "eval_loss": 0.33662545680999756, |
| "eval_runtime": 20.6734, |
| "eval_samples_per_second": 162.818, |
| "eval_steps_per_second": 10.206, |
| "epoch": 2.143112701252236, |
| "step": 600 |
| }, |
| { |
| "loss": 0.2955, |
| "grad_norm": 0.37391841411590576, |
| "learning_rate": 0.0002534505424934599, |
| "epoch": 2.1610017889087656, |
| "step": 605 |
| }, |
| { |
| "loss": 0.2869, |
| "grad_norm": 0.3889912962913513, |
| "learning_rate": 0.00025266952804750727, |
| "epoch": 2.178890876565295, |
| "step": 610 |
| }, |
| { |
| "loss": 0.2805, |
| "grad_norm": 0.37954917550086975, |
| "learning_rate": 0.0002518832413228304, |
| "epoch": 2.196779964221825, |
| "step": 615 |
| }, |
| { |
| "loss": 0.2834, |
| "grad_norm": 0.3385255038738251, |
| "learning_rate": 0.00025109172269677265, |
| "epoch": 2.2146690518783543, |
| "step": 620 |
| }, |
| { |
| "loss": 0.2741, |
| "grad_norm": 0.33614033460617065, |
| "learning_rate": 0.00025029501281534534, |
| "epoch": 2.2325581395348837, |
| "step": 625 |
| }, |
| { |
| "loss": 0.2803, |
| "grad_norm": 0.4882681369781494, |
| "learning_rate": 0.00024949315259114094, |
| "epoch": 2.250447227191413, |
| "step": 630 |
| }, |
| { |
| "loss": 0.305, |
| "grad_norm": 0.39836975932121277, |
| "learning_rate": 0.00024868618320123193, |
| "epoch": 2.268336314847943, |
| "step": 635 |
| }, |
| { |
| "loss": 0.2794, |
| "grad_norm": 0.30090266466140747, |
| "learning_rate": 0.00024787414608505636, |
| "epoch": 2.2862254025044724, |
| "step": 640 |
| }, |
| { |
| "loss": 0.2923, |
| "grad_norm": 0.40115466713905334, |
| "learning_rate": 0.0002470570829422898, |
| "epoch": 2.304114490161002, |
| "step": 645 |
| }, |
| { |
| "loss": 0.2752, |
| "grad_norm": 0.35384082794189453, |
| "learning_rate": 0.00024623503573070407, |
| "epoch": 2.322003577817531, |
| "step": 650 |
| }, |
| { |
| "loss": 0.2647, |
| "grad_norm": 0.5324997901916504, |
| "learning_rate": 0.00024540804666401235, |
| "epoch": 2.3398926654740606, |
| "step": 655 |
| }, |
| { |
| "loss": 0.2667, |
| "grad_norm": 0.3377424478530884, |
| "learning_rate": 0.00024457615820970194, |
| "epoch": 2.3577817531305905, |
| "step": 660 |
| }, |
| { |
| "loss": 0.273, |
| "grad_norm": 0.3497815728187561, |
| "learning_rate": 0.0002437394130868529, |
| "epoch": 2.37567084078712, |
| "step": 665 |
| }, |
| { |
| "loss": 0.2674, |
| "grad_norm": 0.4024289548397064, |
| "learning_rate": 0.00024289785426394471, |
| "epoch": 2.3935599284436493, |
| "step": 670 |
| }, |
| { |
| "loss": 0.2552, |
| "grad_norm": 0.2747831642627716, |
| "learning_rate": 0.00024205152495664965, |
| "epoch": 2.4114490161001787, |
| "step": 675 |
| }, |
| { |
| "loss": 0.2775, |
| "grad_norm": 0.5090282559394836, |
| "learning_rate": 0.00024120046862561364, |
| "epoch": 2.4293381037567086, |
| "step": 680 |
| }, |
| { |
| "loss": 0.2638, |
| "grad_norm": 0.423360675573349, |
| "learning_rate": 0.0002403447289742243, |
| "epoch": 2.447227191413238, |
| "step": 685 |
| }, |
| { |
| "loss": 0.2629, |
| "grad_norm": 0.3922378122806549, |
| "learning_rate": 0.0002394843499463669, |
| "epoch": 2.4651162790697674, |
| "step": 690 |
| }, |
| { |
| "loss": 0.2642, |
| "grad_norm": 0.381626158952713, |
| "learning_rate": 0.00023861937572416757, |
| "epoch": 2.483005366726297, |
| "step": 695 |
| }, |
| { |
| "loss": 0.2741, |
| "grad_norm": 0.41004201769828796, |
| "learning_rate": 0.0002377498507257247, |
| "epoch": 2.500894454382826, |
| "step": 700 |
| }, |
| { |
| "eval_loss": 0.3169912099838257, |
| "eval_runtime": 20.6812, |
| "eval_samples_per_second": 162.756, |
| "eval_steps_per_second": 10.202, |
| "epoch": 2.500894454382826, |
| "step": 700 |
| }, |
| { |
| "loss": 0.2669, |
| "grad_norm": 0.36008498072624207, |
| "learning_rate": 0.00023687581960282763, |
| "epoch": 2.518783542039356, |
| "step": 705 |
| }, |
| { |
| "loss": 0.2529, |
| "grad_norm": 0.47046661376953125, |
| "learning_rate": 0.00023599732723866414, |
| "epoch": 2.5366726296958855, |
| "step": 710 |
| }, |
| { |
| "loss": 0.2882, |
| "grad_norm": 0.6661579012870789, |
| "learning_rate": 0.00023511441874551512, |
| "epoch": 2.554561717352415, |
| "step": 715 |
| }, |
| { |
| "loss": 0.2805, |
| "grad_norm": 0.3578526973724365, |
| "learning_rate": 0.00023422713946243842, |
| "epoch": 2.5724508050089447, |
| "step": 720 |
| }, |
| { |
| "loss": 0.2615, |
| "grad_norm": 0.42732375860214233, |
| "learning_rate": 0.0002333355349529403, |
| "epoch": 2.590339892665474, |
| "step": 725 |
| }, |
| { |
| "loss": 0.2635, |
| "grad_norm": 0.3077392876148224, |
| "learning_rate": 0.0002324396510026358, |
| "epoch": 2.6082289803220036, |
| "step": 730 |
| }, |
| { |
| "loss": 0.2575, |
| "grad_norm": 0.3254878520965576, |
| "learning_rate": 0.00023153953361689753, |
| "epoch": 2.626118067978533, |
| "step": 735 |
| }, |
| { |
| "loss": 0.2539, |
| "grad_norm": 0.3971039056777954, |
| "learning_rate": 0.00023063522901849303, |
| "epoch": 2.6440071556350624, |
| "step": 740 |
| }, |
| { |
| "loss": 0.2675, |
| "grad_norm": 0.36017006635665894, |
| "learning_rate": 0.00022972678364521155, |
| "epoch": 2.6618962432915922, |
| "step": 745 |
| }, |
| { |
| "loss": 0.2752, |
| "grad_norm": 0.3148644268512726, |
| "learning_rate": 0.00022881424414747902, |
| "epoch": 2.6797853309481217, |
| "step": 750 |
| }, |
| { |
| "loss": 0.257, |
| "grad_norm": 0.46191850304603577, |
| "learning_rate": 0.00022789765738596258, |
| "epoch": 2.697674418604651, |
| "step": 755 |
| }, |
| { |
| "loss": 0.2602, |
| "grad_norm": 0.5142350792884827, |
| "learning_rate": 0.00022697707042916413, |
| "epoch": 2.715563506261181, |
| "step": 760 |
| }, |
| { |
| "loss": 0.2619, |
| "grad_norm": 0.4233485162258148, |
| "learning_rate": 0.00022605253055100346, |
| "epoch": 2.7334525939177103, |
| "step": 765 |
| }, |
| { |
| "loss": 0.2699, |
| "grad_norm": 0.36081498861312866, |
| "learning_rate": 0.00022512408522839034, |
| "epoch": 2.7513416815742398, |
| "step": 770 |
| }, |
| { |
| "loss": 0.2589, |
| "grad_norm": 0.3283044695854187, |
| "learning_rate": 0.0002241917821387868, |
| "epoch": 2.769230769230769, |
| "step": 775 |
| }, |
| { |
| "loss": 0.2569, |
| "grad_norm": 0.36509019136428833, |
| "learning_rate": 0.00022325566915775872, |
| "epoch": 2.7871198568872986, |
| "step": 780 |
| }, |
| { |
| "loss": 0.2631, |
| "grad_norm": 0.30786919593811035, |
| "learning_rate": 0.000222315794356517, |
| "epoch": 2.8050089445438284, |
| "step": 785 |
| }, |
| { |
| "loss": 0.2596, |
| "grad_norm": 0.26746758818626404, |
| "learning_rate": 0.0002213722059994496, |
| "epoch": 2.822898032200358, |
| "step": 790 |
| }, |
| { |
| "loss": 0.2462, |
| "grad_norm": 0.3422580659389496, |
| "learning_rate": 0.00022042495254164248, |
| "epoch": 2.8407871198568873, |
| "step": 795 |
| }, |
| { |
| "loss": 0.2604, |
| "grad_norm": 0.3908096253871918, |
| "learning_rate": 0.0002194740826263918, |
| "epoch": 2.8586762075134167, |
| "step": 800 |
| }, |
| { |
| "eval_loss": 0.31991705298423767, |
| "eval_runtime": 20.6861, |
| "eval_samples_per_second": 162.718, |
| "eval_steps_per_second": 10.2, |
| "epoch": 2.8586762075134167, |
| "step": 800 |
| }, |
| { |
| "loss": 0.259, |
| "grad_norm": 0.3868574798107147, |
| "learning_rate": 0.00021851964508270573, |
| "epoch": 2.8765652951699465, |
| "step": 805 |
| }, |
| { |
| "loss": 0.2627, |
| "grad_norm": 0.3655846416950226, |
| "learning_rate": 0.00021756168892279705, |
| "epoch": 2.894454382826476, |
| "step": 810 |
| }, |
| { |
| "loss": 0.249, |
| "grad_norm": 0.6286638379096985, |
| "learning_rate": 0.00021660026333956625, |
| "epoch": 2.9123434704830053, |
| "step": 815 |
| }, |
| { |
| "loss": 0.2517, |
| "grad_norm": 0.5203955769538879, |
| "learning_rate": 0.0002156354177040755, |
| "epoch": 2.9302325581395348, |
| "step": 820 |
| }, |
| { |
| "loss": 0.2665, |
| "grad_norm": 0.38546180725097656, |
| "learning_rate": 0.00021466720156301314, |
| "epoch": 2.948121645796064, |
| "step": 825 |
| }, |
| { |
| "loss": 0.2651, |
| "grad_norm": 0.3447847366333008, |
| "learning_rate": 0.00021369566463614968, |
| "epoch": 2.966010733452594, |
| "step": 830 |
| }, |
| { |
| "loss": 0.2494, |
| "grad_norm": 0.6365504264831543, |
| "learning_rate": 0.00021272085681378418, |
| "epoch": 2.9838998211091234, |
| "step": 835 |
| }, |
| { |
| "loss": 0.2448, |
| "grad_norm": 0.6940473914146423, |
| "learning_rate": 0.0002117428281541827, |
| "epoch": 3.0, |
| "step": 840 |
| }, |
| { |
| "loss": 0.1991, |
| "grad_norm": 0.3761572539806366, |
| "learning_rate": 0.00021076162888100734, |
| "epoch": 3.0178890876565294, |
| "step": 845 |
| }, |
| { |
| "loss": 0.1976, |
| "grad_norm": 0.3070697784423828, |
| "learning_rate": 0.00020977730938073747, |
| "epoch": 3.035778175313059, |
| "step": 850 |
| }, |
| { |
| "loss": 0.188, |
| "grad_norm": 0.5102139115333557, |
| "learning_rate": 0.0002087899202000821, |
| "epoch": 3.0536672629695887, |
| "step": 855 |
| }, |
| { |
| "loss": 0.2003, |
| "grad_norm": 0.4111884534358978, |
| "learning_rate": 0.00020779951204338422, |
| "epoch": 3.071556350626118, |
| "step": 860 |
| }, |
| { |
| "loss": 0.1914, |
| "grad_norm": 0.46625199913978577, |
| "learning_rate": 0.00020680613577001724, |
| "epoch": 3.0894454382826475, |
| "step": 865 |
| }, |
| { |
| "loss": 0.2037, |
| "grad_norm": 0.4094313383102417, |
| "learning_rate": 0.0002058098423917729, |
| "epoch": 3.107334525939177, |
| "step": 870 |
| }, |
| { |
| "loss": 0.2, |
| "grad_norm": 0.4287453591823578, |
| "learning_rate": 0.000204810683070242, |
| "epoch": 3.1252236135957068, |
| "step": 875 |
| }, |
| { |
| "loss": 0.1918, |
| "grad_norm": 0.4345555901527405, |
| "learning_rate": 0.00020380870911418709, |
| "epoch": 3.143112701252236, |
| "step": 880 |
| }, |
| { |
| "loss": 0.1911, |
| "grad_norm": 0.43231672048568726, |
| "learning_rate": 0.00020280397197690753, |
| "epoch": 3.1610017889087656, |
| "step": 885 |
| }, |
| { |
| "loss": 0.196, |
| "grad_norm": 0.4027317464351654, |
| "learning_rate": 0.00020179652325359756, |
| "epoch": 3.178890876565295, |
| "step": 890 |
| }, |
| { |
| "loss": 0.1964, |
| "grad_norm": 0.4894247353076935, |
| "learning_rate": 0.00020078641467869652, |
| "epoch": 3.196779964221825, |
| "step": 895 |
| }, |
| { |
| "loss": 0.1847, |
| "grad_norm": 0.4106491804122925, |
| "learning_rate": 0.00019977369812323214, |
| "epoch": 3.2146690518783543, |
| "step": 900 |
| }, |
| { |
| "eval_loss": 0.3646461069583893, |
| "eval_runtime": 20.6747, |
| "eval_samples_per_second": 162.808, |
| "eval_steps_per_second": 10.206, |
| "epoch": 3.2146690518783543, |
| "step": 900 |
| }, |
| { |
| "loss": 0.1918, |
| "grad_norm": 0.5217190980911255, |
| "learning_rate": 0.00019875842559215724, |
| "epoch": 3.2325581395348837, |
| "step": 905 |
| }, |
| { |
| "loss": 0.1856, |
| "grad_norm": 0.3551487922668457, |
| "learning_rate": 0.00019774064922167876, |
| "epoch": 3.250447227191413, |
| "step": 910 |
| }, |
| { |
| "loss": 0.1978, |
| "grad_norm": 0.44846031069755554, |
| "learning_rate": 0.00019672042127658062, |
| "epoch": 3.268336314847943, |
| "step": 915 |
| }, |
| { |
| "loss": 0.1912, |
| "grad_norm": 0.45646387338638306, |
| "learning_rate": 0.00019569779414753999, |
| "epoch": 3.2862254025044724, |
| "step": 920 |
| }, |
| { |
| "loss": 0.1988, |
| "grad_norm": 0.42022913694381714, |
| "learning_rate": 0.00019467282034843654, |
| "epoch": 3.304114490161002, |
| "step": 925 |
| }, |
| { |
| "loss": 0.188, |
| "grad_norm": 0.44034436345100403, |
| "learning_rate": 0.00019364555251365627, |
| "epoch": 3.322003577817531, |
| "step": 930 |
| }, |
| { |
| "loss": 0.2039, |
| "grad_norm": 0.5154224634170532, |
| "learning_rate": 0.00019261604339538809, |
| "epoch": 3.3398926654740606, |
| "step": 935 |
| }, |
| { |
| "loss": 0.2013, |
| "grad_norm": 0.6147205829620361, |
| "learning_rate": 0.0001915843458609152, |
| "epoch": 3.3577817531305905, |
| "step": 940 |
| }, |
| { |
| "loss": 0.2017, |
| "grad_norm": 0.460700660943985, |
| "learning_rate": 0.0001905505128899004, |
| "epoch": 3.37567084078712, |
| "step": 945 |
| }, |
| { |
| "loss": 0.1908, |
| "grad_norm": 0.5751897692680359, |
| "learning_rate": 0.000189514597571665, |
| "epoch": 3.3935599284436493, |
| "step": 950 |
| }, |
| { |
| "loss": 0.1965, |
| "grad_norm": 0.4614831507205963, |
| "learning_rate": 0.00018847665310246312, |
| "epoch": 3.4114490161001787, |
| "step": 955 |
| }, |
| { |
| "loss": 0.2014, |
| "grad_norm": 0.3863570988178253, |
| "learning_rate": 0.00018743673278274954, |
| "epoch": 3.4293381037567086, |
| "step": 960 |
| }, |
| { |
| "loss": 0.189, |
| "grad_norm": 0.3508825898170471, |
| "learning_rate": 0.0001863948900144428, |
| "epoch": 3.447227191413238, |
| "step": 965 |
| }, |
| { |
| "loss": 0.1793, |
| "grad_norm": 0.3714524805545807, |
| "learning_rate": 0.00018535117829818293, |
| "epoch": 3.4651162790697674, |
| "step": 970 |
| }, |
| { |
| "loss": 0.2049, |
| "grad_norm": 0.43370577692985535, |
| "learning_rate": 0.00018430565123058409, |
| "epoch": 3.483005366726297, |
| "step": 975 |
| }, |
| { |
| "loss": 0.1803, |
| "grad_norm": 0.34576115012168884, |
| "learning_rate": 0.00018325836250148215, |
| "epoch": 3.500894454382826, |
| "step": 980 |
| }, |
| { |
| "loss": 0.1891, |
| "grad_norm": 0.41390174627304077, |
| "learning_rate": 0.00018220936589117775, |
| "epoch": 3.518783542039356, |
| "step": 985 |
| }, |
| { |
| "loss": 0.1909, |
| "grad_norm": 0.5129839777946472, |
| "learning_rate": 0.00018115871526767455, |
| "epoch": 3.5366726296958855, |
| "step": 990 |
| }, |
| { |
| "loss": 0.1919, |
| "grad_norm": 0.537712812423706, |
| "learning_rate": 0.00018010646458391293, |
| "epoch": 3.554561717352415, |
| "step": 995 |
| }, |
| { |
| "loss": 0.1909, |
| "grad_norm": 0.3729874789714813, |
| "learning_rate": 0.00017905266787499947, |
| "epoch": 3.5724508050089447, |
| "step": 1000 |
| }, |
| { |
| "eval_loss": 0.36307844519615173, |
| "eval_runtime": 20.6777, |
| "eval_samples_per_second": 162.784, |
| "eval_steps_per_second": 10.204, |
| "epoch": 3.5724508050089447, |
| "step": 1000 |
| }, |
| { |
| "train_runtime": 2976.681, |
| "train_samples_per_second": 96.118, |
| "train_steps_per_second": 0.75, |
| "total_flos": 4.722006162277663e+17, |
| "train_loss": 0.444128730237484, |
| "epoch": 3.5724508050089447, |
| "step": 1000 |
| } |
| ], |
| "best_metric": 0.3169912099838257, |
| "best_model_checkpoint": "saves/adpr-llama/checkpoint-700", |
| "is_local_process_zero": true, |
| "is_world_process_zero": true, |
| "is_hyper_param_search": false, |
| "trial_name": null, |
| "trial_params": null, |
| "stateful_callbacks": { |
| "EarlyStoppingCallback": { |
| "args": { |
| "early_stopping_patience": 3, |
| "early_stopping_threshold": 0.0 |
| }, |
| "attributes": { |
| "early_stopping_patience_counter": 3 |
| } |
| }, |
| "TrainerControl": { |
| "args": { |
| "should_training_stop": true, |
| "should_epoch_stop": false, |
| "should_save": true, |
| "should_evaluate": false, |
| "should_log": false |
| }, |
| "attributes": {} |
| } |
| }, |
| "model": "PeftModelForCausalLM(\n (base_model): LoraModel(\n (model): LlamaForCausalLM(\n (model): LlamaModel(\n (embed_tokens): Embedding(32000, 4096, padding_idx=0)\n (layers): ModuleList(\n (0-31): 32 x LlamaDecoderLayer(\n (self_attn): LlamaAttention(\n (q_proj): lora.Linear(\n (base_layer): Linear(in_features=4096, out_features=4096, bias=False)\n (lora_dropout): ModuleDict(\n (default): Dropout(p=0.05, inplace=False)\n )\n (lora_A): ModuleDict(\n (default): Linear(in_features=4096, out_features=64, bias=False)\n )\n (lora_B): ModuleDict(\n (default): Linear(in_features=64, out_features=4096, bias=False)\n )\n (lora_embedding_A): ParameterDict()\n (lora_embedding_B): ParameterDict()\n (lora_magnitude_vector): ModuleDict()\n )\n (k_proj): lora.Linear(\n (base_layer): Linear(in_features=4096, out_features=4096, bias=False)\n (lora_dropout): ModuleDict(\n (default): Dropout(p=0.05, inplace=False)\n )\n (lora_A): ModuleDict(\n (default): Linear(in_features=4096, out_features=64, bias=False)\n )\n (lora_B): ModuleDict(\n (default): Linear(in_features=64, out_features=4096, bias=False)\n )\n (lora_embedding_A): ParameterDict()\n (lora_embedding_B): ParameterDict()\n (lora_magnitude_vector): ModuleDict()\n )\n (v_proj): lora.Linear(\n (base_layer): Linear(in_features=4096, out_features=4096, bias=False)\n (lora_dropout): ModuleDict(\n (default): Dropout(p=0.05, inplace=False)\n )\n (lora_A): ModuleDict(\n (default): Linear(in_features=4096, out_features=64, bias=False)\n )\n (lora_B): ModuleDict(\n (default): Linear(in_features=64, out_features=4096, bias=False)\n )\n (lora_embedding_A): ParameterDict()\n (lora_embedding_B): ParameterDict()\n (lora_magnitude_vector): ModuleDict()\n )\n (o_proj): lora.Linear(\n (base_layer): Linear(in_features=4096, out_features=4096, bias=False)\n (lora_dropout): ModuleDict(\n (default): Dropout(p=0.05, inplace=False)\n )\n (lora_A): ModuleDict(\n (default): Linear(in_features=4096, out_features=64, bias=False)\n )\n (lora_B): ModuleDict(\n (default): Linear(in_features=64, out_features=4096, bias=False)\n )\n (lora_embedding_A): ParameterDict()\n (lora_embedding_B): ParameterDict()\n (lora_magnitude_vector): ModuleDict()\n )\n )\n (mlp): LlamaMLP(\n (gate_proj): lora.Linear(\n (base_layer): Linear(in_features=4096, out_features=11008, bias=False)\n (lora_dropout): ModuleDict(\n (default): Dropout(p=0.05, inplace=False)\n )\n (lora_A): ModuleDict(\n (default): Linear(in_features=4096, out_features=64, bias=False)\n )\n (lora_B): ModuleDict(\n (default): Linear(in_features=64, out_features=11008, bias=False)\n )\n (lora_embedding_A): ParameterDict()\n (lora_embedding_B): ParameterDict()\n (lora_magnitude_vector): ModuleDict()\n )\n (up_proj): lora.Linear(\n (base_layer): Linear(in_features=4096, out_features=11008, bias=False)\n (lora_dropout): ModuleDict(\n (default): Dropout(p=0.05, inplace=False)\n )\n (lora_A): ModuleDict(\n (default): Linear(in_features=4096, out_features=64, bias=False)\n )\n (lora_B): ModuleDict(\n (default): Linear(in_features=64, out_features=11008, bias=False)\n )\n (lora_embedding_A): ParameterDict()\n (lora_embedding_B): ParameterDict()\n (lora_magnitude_vector): ModuleDict()\n )\n (down_proj): lora.Linear(\n (base_layer): Linear(in_features=11008, out_features=4096, bias=False)\n (lora_dropout): ModuleDict(\n (default): Dropout(p=0.05, inplace=False)\n )\n (lora_A): ModuleDict(\n (default): Linear(in_features=11008, out_features=64, bias=False)\n )\n (lora_B): ModuleDict(\n (default): Linear(in_features=64, out_features=4096, bias=False)\n )\n (lora_embedding_A): ParameterDict()\n (lora_embedding_B): ParameterDict()\n (lora_magnitude_vector): ModuleDict()\n )\n (act_fn): SiLU()\n )\n (input_layernorm): LlamaRMSNorm((4096,), eps=1e-05)\n (post_attention_layernorm): LlamaRMSNorm((4096,), eps=1e-05)\n )\n )\n (norm): LlamaRMSNorm((4096,), eps=1e-05)\n (rotary_emb): LlamaRotaryEmbedding()\n )\n (lm_head): Linear(in_features=4096, out_features=32000, bias=False)\n )\n )\n)", |
| "optimizer": "AcceleratedOptimizer (\nParameter Group 0\n amsgrad: False\n betas: (0.9, 0.999)\n capturable: False\n decoupled_weight_decay: True\n differentiable: False\n eps: 1e-08\n foreach: None\n fused: None\n initial_lr: 0.0003\n lr: 0.00017905266787499947\n maximize: False\n weight_decay: 0.0\n\nParameter Group 1\n amsgrad: False\n betas: (0.9, 0.999)\n capturable: False\n decoupled_weight_decay: True\n differentiable: False\n eps: 1e-08\n foreach: None\n fused: None\n initial_lr: 0.0003\n lr: 0.00017905266787499947\n maximize: False\n weight_decay: 0.0\n)", |
| "lr_scheduler": "<torch.optim.lr_scheduler.LambdaLR object at 0x7c03193e23f0>", |
| "train_dataloader": "<accelerate.data_loader.DataLoaderShard object at 0x7c03195b34d0>" |
| } |
