Upload folder using huggingface_hub

training/eval/loss_curve.json

@@ -62,137 +62,1244 @@
     60,
     61,
     62,
-    63
+    63,
+    64,
+    65,
+    66,
+    67,
+    68,
+    69,
+    70,
+    71,
+    72,
+    73,
+    74,
+    75,
+    76,
+    77,
+    78,
+    79,
+    80,
+    81,
+    82,
+    83,
+    84,
+    85,
+    86,
+    87,
+    88,
+    89,
+    90,
+    91,
+    92,
+    93,
+    94,
+    95,
+    96,
+    97,
+    98,
+    99,
+    100,
+    101,
+    102,
+    103,
+    104,
+    105,
+    106,
+    107,
+    108,
+    109,
+    110,
+    111,
+    112,
+    113,
+    114,
+    115,
+    116,
+    117,
+    118,
+    119,
+    120,
+    121,
+    122,
+    123,
+    124,
+    125,
+    126,
+    127,
+    128,
+    129,
+    130,
+    131,
+    132,
+    133,
+    134,
+    135,
+    136,
+    137,
+    138,
+    139,
+    140,
+    141,
+    142,
+    143,
+    144,
+    145,
+    146,
+    147,
+    148,
+    149,
+    150,
+    151,
+    152,
+    153,
+    154,
+    155,
+    156,
+    157,
+    158,
+    159,
+    160,
+    161,
+    162,
+    163,
+    164,
+    165,
+    166,
+    167,
+    168,
+    169,
+    170,
+    171,
+    172,
+    173,
+    174,
+    175,
+    176,
+    177,
+    178,
+    179,
+    180,
+    181,
+    182,
+    183,
+    184,
+    185,
+    186,
+    187,
+    188,
+    189,
+    190,
+    191,
+    192,
+    193,
+    194,
+    195,
+    196,
+    197,
+    198,
+    199,
+    200,
+    201,
+    202,
+    203,
+    204,
+    205,
+    206,
+    207,
+    208,
+    209,
+    210,
+    211,
+    212,
+    213,
+    214,
+    215,
+    216,
+    217,
+    218,
+    219,
+    220,
+    221,
+    222,
+    223,
+    224,
+    225,
+    226,
+    227,
+    228,
+    229,
+    230,
+    231,
+    232,
+    233,
+    234,
+    235,
+    236,
+    237,
+    238,
+    239,
+    240,
+    241,
+    242,
+    243,
+    244,
+    245,
+    246,
+    247,
+    248,
+    249,
+    250,
+    251,
+    252,
+    253,
+    254,
+    255,
+    256,
+    257,
+    258,
+    259,
+    260,
+    261,
+    262,
+    263,
+    264,
+    265,
+    266,
+    267,
+    268,
+    269,
+    270,
+    271,
+    272,
+    273,
+    274,
+    275,
+    276,
+    277,
+    278,
+    279,
+    280,
+    281,
+    282,
+    283,
+    284,
+    285,
+    286,
+    287,
+    288,
+    289,
+    290,
+    291,
+    292,
+    293,
+    294,
+    295,
+    296,
+    297,
+    298,
+    299,
+    300,
+    301,
+    302,
+    303,
+    304,
+    305,
+    306,
+    307,
+    308,
+    309,
+    310,
+    311,
+    312,
+    313,
+    314,
+    315,
+    316,
+    317,
+    318,
+    319,
+    320,
+    321,
+    322,
+    323,
+    324,
+    325,
+    326,
+    327,
+    328,
+    329,
+    330,
+    331,
+    332,
+    333,
+    334,
+    335,
+    336,
+    337,
+    338,
+    339,
+    340,
+    341,
+    342,
+    343,
+    344,
+    345,
+    346,
+    347,
+    348,
+    349,
+    350,
+    351,
+    352,
+    353,
+    354,
+    355,
+    356,
+    357,
+    358,
+    359,
+    360,
+    361,
+    362,
+    363,
+    364,
+    365,
+    366,
+    367,
+    368,
+    369,
+    370,
+    371,
+    372,
+    373,
+    374,
+    375,
+    376,
+    377,
+    378,
+    379,
+    380,
+    381,
+    382,
+    383,
+    384,
+    385,
+    386,
+    387,
+    388,
+    389,
+    390,
+    391,
+    392,
+    393,
+    394,
+    395,
+    396,
+    397,
+    398,
+    399,
+    400,
+    401,
+    402,
+    403,
+    404,
+    405,
+    406,
+    407,
+    408,
+    409,
+    410,
+    411,
+    412,
+    413,
+    414,
+    415,
+    416,
+    417,
+    418,
+    419,
+    420,
+    421,
+    422,
+    423,
+    424,
+    425,
+    426,
+    427,
+    428,
+    429,
+    430,
+    431,
+    432
   ],
   "loss": [
-    2.929511785507202,
-    2.247201919555664,
-    2.8277809619903564,
-    2.989217519760132,
-    2.9572653770446777,
-    2.567988872528076,
-    2.778838634490967,
-    3.0401687622070312,
-    2.732299327850342,
-    3.147907018661499,
-    2.766887664794922,
-    2.7122697830200195,
-    3.1168272495269775,
-    2.4730920791625977,
-    2.7186336517333984,
-    2.8276560306549072,
-    3.050647735595703,
-    2.3382976055145264,
-    2.9696552753448486,
-    2.906693696975708,
-    2.7200937271118164,
-    2.2900822162628174,
-    3.231929302215576,
-    3.1112563610076904,
-    2.561981439590454,
-    2.362828254699707,
-    2.390101671218872,
-    2.8416247367858887,
-    2.1005444526672363,
-    2.302135467529297,
-    2.0826117992401123,
-    2.4838976860046387,
-    2.144624948501587,
-    2.113643169403076,
-    2.270902633666992,
-    2.0940098762512207,
-    2.092400312423706,
-    2.1469905376434326,
-    2.428116798400879,
-    2.335599899291992,
-    2.409480094909668,
-    2.306694269180298,
-    1.7273067235946655,
-    2.1605944633483887,
-    1.9863046407699585,
-    1.6670739650726318,
-    1.9110578298568726,
-    2.1242356300354004,
-    2.232588052749634,
-    2.3207507133483887,
-    2.6646008491516113,
-    1.7286312580108643,
-    1.836452603340149,
-    2.011589288711548,
-    2.1369283199310303,
-    1.8148353099822998,
-    2.4181647300720215,
-    1.809765100479126,
-    1.986395239830017,
-    1.6849592924118042,
-    2.13490629196167,
-    1.7944599390029907,
-    2.70202374458313
+    2.1868221759796143,
+    1.8832122087478638,
+    2.74792742729187,
+    2.466888427734375,
+    2.378871440887451,
+    2.976832866668701,
+    2.992896795272827,
+    3.661931037902832,
+    3.0851643085479736,
+    2.459578037261963,
+    2.6869261264801025,
+    2.444610834121704,
+    2.187511682510376,
+    2.189753770828247,
+    3.067650079727173,
+    2.0504701137542725,
+    1.7403210401535034,
+    3.0753378868103027,
+    2.989851951599121,
+    2.31941819190979,
+    2.687067985534668,
+    3.692761182785034,
+    1.9659239053726196,
+    2.625602960586548,
+    2.905078887939453,
+    3.7031500339508057,
+    2.465721607208252,
+    2.029315710067749,
+    1.9020980596542358,
+    2.1143009662628174,
+    3.058995008468628,
+    2.0514791011810303,
+    2.775183916091919,
+    2.0651276111602783,
+    2.062998056411743,
+    2.709225654602051,
+    2.990910768508911,
+    2.4446091651916504,
+    1.851803183555603,
+    2.3031060695648193,
+    1.938597559928894,
+    1.7308727502822876,
+    3.1031439304351807,
+    2.8817687034606934,
+    1.3779128789901733,
+    1.417384386062622,
+    2.4849507808685303,
+    2.2591872215270996,
+    1.399527907371521,
+    1.8651618957519531,
+    1.8742029666900635,
+    1.781115174293518,
+    2.127704381942749,
+    1.2853440046310425,
+    1.752669095993042,
+    2.1292359828948975,
+    1.7802091836929321,
+    1.8441812992095947,
+    0.9849231839179993,
+    2.00901198387146,
+    1.312231421470642,
+    1.5197911262512207,
+    1.1018126010894775,
+    1.4974277019500732,
+    1.271100640296936,
+    0.9362401962280273,
+    1.1917356252670288,
+    1.520263433456421,
+    0.9220766425132751,
+    1.3558319807052612,
+    1.2839670181274414,
+    1.0550564527511597,
+    1.3950097560882568,
+    1.0342011451721191,
+    1.5444986820220947,
+    1.214152455329895,
+    1.2319684028625488,
+    1.2454372644424438,
+    1.3274402618408203,
+    0.7429965138435364,
+    1.4318790435791016,
+    1.0145909786224365,
+    1.2779990434646606,
+    0.9261456727981567,
+    1.3646568059921265,
+    0.9034634828567505,
+    1.0444238185882568,
+    0.6867676377296448,
+    1.0076674222946167,
+    0.8004315495491028,
+    0.7038512229919434,
+    0.7358322739601135,
+    0.7483066320419312,
+    0.6646082997322083,
+    0.8452937006950378,
+    0.5372214317321777,
+    0.8221765160560608,
+    0.4908658564090729,
+    0.6595877408981323,
+    0.5910767316818237,
+    0.7960184812545776,
+    1.1173912286758423,
+    0.7301482558250427,
+    0.7841973304748535,
+    0.4997318387031555,
+    0.9121840000152588,
+    0.4693012237548828,
+    0.646824836730957,
+    0.6363754272460938,
+    0.8195663094520569,
+    0.7982276082038879,
+    0.790673017501831,
+    0.5768085718154907,
+    0.47824233770370483,
+    0.43671277165412903,
+    0.32966217398643494,
+    0.5077826976776123,
+    0.5701857209205627,
+    0.6318569779396057,
+    0.4258284866809845,
+    0.561607837677002,
+    0.3217211067676544,
+    0.7238078713417053,
+    0.47419074177742004,
+    0.2767798900604248,
+    0.34714624285697937,
+    0.3780307173728943,
+    0.4380798637866974,
+    0.28320565819740295,
+    0.33708804845809937,
+    0.31937864422798157,
+    0.31313425302505493,
+    0.2263900488615036,
+    0.20638689398765564,
+    0.2329217791557312,
+    0.18222281336784363,
+    0.28628602623939514,
+    0.2572261393070221,
+    0.23541218042373657,
+    0.12017600983381271,
+    0.22665250301361084,
+    0.2642861604690552,
+    0.3743697702884674,
+    0.05140205845236778,
+    0.22477872669696808,
+    0.08610273897647858,
+    0.1152583658695221,
+    0.09971025586128235,
+    0.19319814443588257,
+    0.17943711578845978,
+    0.2125009447336197,
+    0.13632674515247345,
+    0.03438631445169449,
+    0.1440068632364273,
+    0.08364047110080719,
+    0.1448456346988678,
+    0.09101057052612305,
+    0.25706419348716736,
+    0.11460493505001068,
+    0.08179458230733871,
+    0.1108948141336441,
+    0.10725291818380356,
+    0.17279917001724243,
+    0.07994291186332703,
+    0.18839159607887268,
+    0.07633119821548462,
+    0.24155637621879578,
+    0.2127387672662735,
+    0.07558262348175049,
+    0.23579266667366028,
+    0.06715679913759232,
+    0.09585361182689667,
+    0.08734259754419327,
+    0.0595259815454483,
+    0.08579694479703903,
+    0.12705564498901367,
+    0.19376082718372345,
+    0.18897753953933716,
+    0.12125109881162643,
+    0.10712607204914093,
+    0.09171106666326523,
+    0.08163383603096008,
+    0.08653677254915237,
+    0.1647498607635498,
+    0.17448236048221588,
+    0.030835004523396492,
+    0.14000149071216583,
+    0.05882776528596878,
+    0.0566595159471035,
+    0.04071396589279175,
+    0.12734399735927582,
+    0.05481724068522453,
+    0.08446623384952545,
+    0.0658019408583641,
+    0.040359292179346085,
+    0.11326637119054794,
+    0.12553784251213074,
+    0.20116768777370453,
+    0.03213895112276077,
+    0.028803518041968346,
+    0.09090777486562729,
+    0.04714714363217354,
+    0.04253566637635231,
+    0.11862995475530624,
+    0.024751504883170128,
+    0.10534296184778214,
+    0.05181167647242546,
+    0.09807577729225159,
+    0.061464644968509674,
+    0.04382835701107979,
+    0.06480970978736877,
+    0.04080868884921074,
+    0.0872291699051857,
+    0.10696477442979813,
+    0.03661800175905228,
+    0.04979350417852402,
+    0.03905259072780609,
+    0.10408294945955276,
+    0.027889195829629898,
+    0.114476777613163,
+    0.051795843988657,
+    0.056143928319215775,
+    0.11293105036020279,
+    0.0537584014236927,
+    0.06011063605546951,
+    0.10228238999843597,
+    0.035928282886743546,
+    0.03800540789961815,
+    0.08192744106054306,
+    0.1576160043478012,
+    0.021977996453642845,
+    0.03995497524738312,
+    0.0348578579723835,
+    0.04426475241780281,
+    0.2049272060394287,
+    0.0383828803896904,
+    0.03449924290180206,
+    0.08406080305576324,
+    0.05599617958068848,
+    0.033643074333667755,
+    0.07033411413431168,
+    0.15729187428951263,
+    0.13229504227638245,
+    0.07126174122095108,
+    0.06950366497039795,
+    0.08713675290346146,
+    0.043588027358055115,
+    0.0350881963968277,
+    0.02048255130648613,
+    0.03595464304089546,
+    0.0626595988869667,
+    0.03857016935944557,
+    0.09025806933641434,
+    0.024599924683570862,
+    0.031761568039655685,
+    0.08141270279884338,
+    0.05745274946093559,
+    0.04628489911556244,
+    0.045103780925273895,
+    0.06380065530538559,
+    0.0335506796836853,
+    0.0789261907339096,
+    0.13149335980415344,
+    0.056063808500766754,
+    0.04866420850157738,
+    0.029578950256109238,
+    0.025290166959166527,
+    0.01872171461582184,
+    0.09680814296007156,
+    0.06373206526041031,
+    0.04284566640853882,
+    0.021252375096082687,
+    0.04372170940041542,
+    0.025219587609171867,
+    0.07590273767709732,
+    0.05064677819609642,
+    0.02392454445362091,
+    0.0703398659825325,
+    0.07361158728599548,
+    0.0311483982950449,
+    0.0434940829873085,
+    0.05944303795695305,
+    0.022419409826397896,
+    0.04728378355503082,
+    0.03046773374080658,
+    0.02224596217274666,
+    0.04721511900424957,
+    0.2853592038154602,
+    0.05080932378768921,
+    0.04493005573749542,
+    0.028672179207205772,
+    0.038387756794691086,
+    0.008835444226861,
+    0.08745629340410233,
+    0.030413512140512466,
+    0.0215326976031065,
+    0.059723690152168274,
+    0.026869062334299088,
+    0.09244062751531601,
+    0.050737522542476654,
+    0.06389229744672775,
+    0.05108116939663887,
+    0.013370499014854431,
+    0.046124354004859924,
+    0.05524074286222458,
+    0.07775624096393585,
+    0.019194353371858597,
+    0.015412382781505585,
+    0.03348630666732788,
+    0.0736440047621727,
+    0.05221782624721527,
+    0.01482109073549509,
+    0.06728484481573105,
+    0.07350276410579681,
+    0.021377986297011375,
+    0.0244090985506773,
+    0.02625943161547184,
+    0.042253926396369934,
+    0.05598453804850578,
+    0.030156061053276062,
+    0.05984398350119591,
+    0.05524429306387901,
+    0.04773629456758499,
+    0.08369806408882141,
+    0.04128924757242203,
+    0.11620330810546875,
+    0.04122908413410187,
+    0.04114435240626335,
+    0.057129234075546265,
+    0.07486903667449951,
+    0.06950649619102478,
+    0.10660420358181,
+    0.06830599904060364,
+    0.052171748131513596,
+    0.04475213959813118,
+    0.025314627215266228,
+    0.04686501622200012,
+    0.026790857315063477,
+    0.036465834826231,
+    0.03383910283446312,
+    0.02571028843522072,
+    0.0349169559776783,
+    0.04037737473845482,
+    0.040855783969163895,
+    0.044299956411123276,
+    0.03438091278076172,
+    0.03904359042644501,
+    0.03698223456740379,
+    0.04931587725877762,
+    0.06975017488002777,
+    0.06531553715467453,
+    0.03872193768620491,
+    0.0347418375313282,
+    0.04271344095468521,
+    0.019307807087898254,
+    0.03517657518386841,
+    0.02759283222258091,
+    0.031846512109041214,
+    0.034291353076696396,
+    0.05099368467926979,
+    0.033949777483940125,
+    0.06098097190260887,
+    0.029235392808914185,
+    0.008116567507386208,
+    0.054314855486154556,
+    0.04422992467880249,
+    0.021239448338747025,
+    0.02742270566523075,
+    0.11845722794532776,
+    0.021434683352708817,
+    0.036361224949359894,
+    0.048877276480197906,
+    0.035936783999204636,
+    0.04833187162876129,
+    0.05008803308010101,
+    0.07015033066272736,
+    0.04083660617470741,
+    0.026741312816739082,
+    0.013898066245019436,
+    0.030615374445915222,
+    0.020529018715023994,
+    0.02677951566874981,
+    0.022005200386047363,
+    0.02057856135070324,
+    0.025452110916376114,
+    0.03068704903125763,
+    0.02286878228187561,
+    0.02546192891895771,
+    0.048926085233688354,
+    0.03712352365255356,
+    0.10778707265853882,
+    0.05378232151269913,
+    0.02750377357006073,
+    0.027333255857229233,
+    0.03805176541209221,
+    0.025970155373215675,
+    0.0352700836956501,
+    0.04533504322171211,
+    0.021136652678251266,
+    0.0373135469853878,
+    0.038868822157382965,
+    0.060746271163225174,
+    0.03975040093064308,
+    0.015389256179332733,
+    0.059138789772987366,
+    0.03004509024322033,
+    0.017793692648410797,
+    0.02942165918648243,
+    0.033082515001297,
+    0.04292313754558563,
+    0.03316476568579674,
+    0.020032936707139015,
+    0.022493427619338036,
+    0.02955540269613266,
+    0.05007736012339592,
+    0.030021416023373604,
+    0.021669035777449608,
+    0.05724608525633812,
+    0.08293737471103668,
+    0.06380705535411835,
+    0.028525589033961296,
+    0.02092256210744381,
+    0.02828337252140045,
+    0.027102960273623466,
+    0.02546386793255806,
+    0.05688481032848358,
+    0.05016172677278519,
+    0.04136890545487404,
+    0.024178320541977882,
+    0.048685427755117416,
+    0.051612742245197296,
+    0.001959565095603466
   ],
   "grad_norm": [
-    0.016382873058319092,
-    0.01044374331831932,
-    0.0145557327196002,
-    0.01770804077386856,
-    0.02307666651904583,
-    0.02299576811492443,
-    0.030841359868645668,
-    0.04295551776885986,
-    0.05395013093948364,
-    0.06203395873308182,
-    0.09240083396434784,
-    0.09567058831453323,
-    0.13628879189491272,
-    0.12498781830072403,
-    0.12038960307836533,
-    0.1502494513988495,
-    0.21029499173164368,
-    0.09396432340145111,
-    0.1293041855096817,
-    0.14531083405017853,
-    0.11825048923492432,
-    0.08734378963708878,
-    0.1376802921295166,
-    0.14673109352588654,
-    0.11715734004974365,
-    0.11244065314531326,
-    0.12498508393764496,
-    0.16375483572483063,
-    0.10912039875984192,
-    0.13031816482543945,
-    0.12367933988571167,
-    0.16138437390327454,
-    0.14064301550388336,
-    0.14102506637573242,
-    0.17388373613357544,
-    0.15909229218959808,
-    0.17481838166713715,
-    0.173412024974823,
-    0.21484504640102386,
-    0.21365772187709808,
-    0.22236712276935577,
-    0.23586925864219666,
-    0.14186467230319977,
-    0.20375414192676544,
-    0.16824188828468323,
-    0.16498133540153503,
-    0.18045908212661743,
-    0.20699940621852875,
-    0.24143847823143005,
-    0.2376975119113922,
-    0.3076815605163574,
-    0.14694075286388397,
-    0.17607715725898743,
-    0.19781576097011566,
-    0.23790021240711212,
-    0.1699458807706833,
-    0.28625208139419556,
-    0.16722449660301208,
-    0.18187613785266876,
-    0.14591267704963684,
-    0.21220159530639648,
-    0.17583312094211578,
-    0.34388935565948486
+    0.0076949396170675755,
+    0.006594177335500717,
+    0.009377734735608101,
+    0.008589445613324642,
+    0.00799858383834362,
+    0.011955071240663528,
+    0.011000607162714005,
+    0.020188629627227783,
+    0.018263965845108032,
+    0.010343211703002453,
+    0.012613879516720772,
+    0.016117630526423454,
+    0.012199142947793007,
+    0.01421026699244976,
+    0.02615666575729847,
+    0.01980031281709671,
+    0.015442154370248318,
+    0.031070800498127937,
+    0.031223593279719353,
+    0.026247547939419746,
+    0.031783849000930786,
+    0.0513211265206337,
+    0.027629079297184944,
+    0.04290119186043739,
+    0.07088887691497803,
+    0.09999432414770126,
+    0.06483375281095505,
+    0.049069229513406754,
+    0.042897339910268784,
+    0.04957382753491402,
+    0.0893547534942627,
+    0.07541681081056595,
+    0.097597137093544,
+    0.07455836236476898,
+    0.085120789706707,
+    0.11521550267934799,
+    0.13470634818077087,
+    0.10531394928693771,
+    0.09221255779266357,
+    0.13410773873329163,
+    0.09980085492134094,
+    0.10402636229991913,
+    0.1748587191104889,
+    0.18696419894695282,
+    0.10656597465276718,
+    0.11789694428443909,
+    0.1899501532316208,
+    0.17800557613372803,
+    0.13259239494800568,
+    0.17156220972537994,
+    0.18881964683532715,
+    0.16628162562847137,
+    0.19077593088150024,
+    0.1424628347158432,
+    0.14783352613449097,
+    0.15748907625675201,
+    0.13087883591651917,
+    0.1455100029706955,
+    0.08878733217716217,
+    0.15038780868053436,
+    0.09907542914152145,
+    0.11735238879919052,
+    0.09152618795633316,
+    0.11173691600561142,
+    0.09824828803539276,
+    0.085940882563591,
+    0.10098903626203537,
+    0.12762703001499176,
+    0.08911759406328201,
+    0.12125050276517868,
+    0.1123146340250969,
+    0.09719366580247879,
+    0.12218249589204788,
+    0.10656294971704483,
+    0.14769215881824493,
+    0.12346755713224411,
+    0.11060963571071625,
+    0.12222012132406235,
+    0.13087722659111023,
+    0.08241765946149826,
+    0.15565797686576843,
+    0.11180328577756882,
+    0.14479899406433105,
+    0.11080848425626755,
+    0.17476730048656464,
+    0.11177902668714523,
+    0.13410885632038116,
+    0.09709858894348145,
+    0.17227135598659515,
+    0.11489490419626236,
+    0.10744050145149231,
+    0.11097412556409836,
+    0.11108928173780441,
+    0.10148708522319794,
+    0.12204466015100479,
+    0.09771820157766342,
+    0.12123334407806396,
+    0.08476655185222626,
+    0.10870066285133362,
+    0.10581999272108078,
+    0.12385259568691254,
+    0.21214503049850464,
+    0.12593984603881836,
+    0.1239934116601944,
+    0.10253363102674484,
+    0.16617585718631744,
+    0.09640590101480484,
+    0.13547973334789276,
+    0.12478470802307129,
+    0.19010059535503387,
+    0.20280708372592926,
+    0.1604999601840973,
+    0.1250085085630417,
+    0.12070450186729431,
+    0.10591929405927658,
+    0.10009104758501053,
+    0.13747617602348328,
+    0.14619451761245728,
+    0.1434018909931183,
+    0.12144937366247177,
+    0.17137911915779114,
+    0.11768746376037598,
+    0.20892943441867828,
+    0.14127720892429352,
+    0.10477142035961151,
+    0.1317528337240219,
+    0.14182144403457642,
+    0.14403733611106873,
+    0.11800187826156616,
+    0.1537708342075348,
+    0.13129201531410217,
+    0.12122059613466263,
+    0.105872243642807,
+    0.09957372397184372,
+    0.10845856368541718,
+    0.10875073820352554,
+    0.13273923099040985,
+    0.1393309384584427,
+    0.11792351305484772,
+    0.06682579964399338,
+    0.11375295370817184,
+    0.14054830372333527,
+    0.15167202055454254,
+    0.07225672155618668,
+    0.16073709726333618,
+    0.056013837456703186,
+    0.08070174604654312,
+    0.06995324790477753,
+    0.1185460016131401,
+    0.15586988627910614,
+    0.12325572967529297,
+    0.09859131276607513,
+    0.03663024306297302,
+    0.07878220081329346,
+    0.06468985974788666,
+    0.08314953744411469,
+    0.07002438604831696,
+    0.13579510152339935,
+    0.07060762494802475,
+    0.0790892019867897,
+    0.09651272743940353,
+    0.0770317018032074,
+    0.1355382800102234,
+    0.09308040142059326,
+    0.14823825657367706,
+    0.08155263215303421,
+    0.13393718004226685,
+    0.14737831056118011,
+    0.1098741888999939,
+    0.17045724391937256,
+    0.08132632821798325,
+    0.0826469361782074,
+    0.09726084023714066,
+    0.057246219366788864,
+    0.11157628148794174,
+    0.12022493034601212,
+    0.15590807795524597,
+    0.11823689937591553,
+    0.1327953338623047,
+    0.0977955311536789,
+    0.07793338596820831,
+    0.08580821752548218,
+    0.08491761982440948,
+    0.10599955916404724,
+    0.14335739612579346,
+    0.07008443772792816,
+    0.15662521123886108,
+    0.0602765753865242,
+    0.09357666224241257,
+    0.062147025018930435,
+    0.12242279201745987,
+    0.07418089359998703,
+    0.10358873754739761,
+    0.07359012216329575,
+    0.0519556924700737,
+    0.1042555645108223,
+    0.13447637856006622,
+    0.17256833612918854,
+    0.05497027561068535,
+    0.04173656180500984,
+    0.08534984290599823,
+    0.07170568406581879,
+    0.07198602706193924,
+    0.11679284274578094,
+    0.048340629786252975,
+    0.19447995722293854,
+    0.07507354766130447,
+    0.11548162251710892,
+    0.08529048413038254,
+    0.07457166910171509,
+    0.06320083141326904,
+    0.06339902430772781,
+    0.12174341827630997,
+    0.12399957329034805,
+    0.068714939057827,
+    0.056455325335264206,
+    0.050612255930900574,
+    0.13071991503238678,
+    0.03307435289025307,
+    0.13784416019916534,
+    0.10135186463594437,
+    0.09920874238014221,
+    0.13444030284881592,
+    0.061538826674222946,
+    0.0547422431409359,
+    0.1446034461259842,
+    0.04926638677716255,
+    0.04877897724509239,
+    0.09127748012542725,
+    0.1500283032655716,
+    0.04419546201825142,
+    0.06664355844259262,
+    0.0857829600572586,
+    0.08369336277246475,
+    0.166261225938797,
+    0.08680809289216995,
+    0.07622619718313217,
+    0.10996700823307037,
+    0.08527426421642303,
+    0.10883290320634842,
+    0.1137668639421463,
+    0.19040368497371674,
+    0.12500418722629547,
+    0.21917997300624847,
+    0.09373047202825546,
+    0.115310437977314,
+    0.062020331621170044,
+    0.05954181030392647,
+    0.029808200895786285,
+    0.0568767786026001,
+    0.11015237122774124,
+    0.05474976450204849,
+    0.1491897851228714,
+    0.0617690235376358,
+    0.06003709137439728,
+    0.10421556234359741,
+    0.096063032746315,
+    0.07777556777000427,
+    0.09218036383390427,
+    0.08230383694171906,
+    0.0767148807644844,
+    0.14433541893959045,
+    0.15919332206249237,
+    0.11021993309259415,
+    0.1294475644826889,
+    0.04417555779218674,
+    0.061295147985219955,
+    0.051739294081926346,
+    0.09733899682760239,
+    0.07189490646123886,
+    0.08853131532669067,
+    0.05203741416335106,
+    0.09113649278879166,
+    0.05584145337343216,
+    0.11569365859031677,
+    0.08057525008916855,
+    0.05737221986055374,
+    0.0958796814084053,
+    0.08356361091136932,
+    0.09096954017877579,
+    0.10716217756271362,
+    0.07482625544071198,
+    0.05911727249622345,
+    0.06664453446865082,
+    0.05361577123403549,
+    0.04416531324386597,
+    0.09031284600496292,
+    1.5009806156158447,
+    0.09160251170396805,
+    0.12212435901165009,
+    0.04882851243019104,
+    0.07065180689096451,
+    0.019306981936097145,
+    0.10127201676368713,
+    0.08436867594718933,
+    0.06640994548797607,
+    0.10789379477500916,
+    0.05553349852561951,
+    0.11580894887447357,
+    0.10262706130743027,
+    0.14019900560379028,
+    0.13831448554992676,
+    0.0712503045797348,
+    0.09519438445568085,
+    0.13820020854473114,
+    0.17915113270282745,
+    0.06591162830591202,
+    0.056267302483320236,
+    0.048587020486593246,
+    0.10055860877037048,
+    0.08325883001089096,
+    0.052128084003925323,
+    0.1364588588476181,
+    0.1235118880867958,
+    0.06310161203145981,
+    0.04321255162358284,
+    0.05049276724457741,
+    0.0864456295967102,
+    0.1588626503944397,
+    0.07595802843570709,
+    0.18927976489067078,
+    0.06601539999246597,
+    0.08920346200466156,
+    0.19322074949741364,
+    0.08737342059612274,
+    0.12210743874311447,
+    0.07811716943979263,
+    0.13561144471168518,
+    0.10362031310796738,
+    0.09930214285850525,
+    0.11724870651960373,
+    0.14630524814128876,
+    0.12264080345630646,
+    0.1265028417110443,
+    0.1135849580168724,
+    0.06668857485055923,
+    0.09173810482025146,
+    0.0968220978975296,
+    0.12610825896263123,
+    0.14428986608982086,
+    0.06458335369825363,
+    0.07151847332715988,
+    0.08952357620000839,
+    0.07944576442241669,
+    0.12419721484184265,
+    0.0789700374007225,
+    0.06483232975006104,
+    0.10574886947870255,
+    0.08509401232004166,
+    0.1544674187898636,
+    0.09705117344856262,
+    0.07944180071353912,
+    0.07753515988588333,
+    0.0747690200805664,
+    0.05712605267763138,
+    0.06657753884792328,
+    0.060331664979457855,
+    0.09460993856191635,
+    0.060675546526908875,
+    0.114589624106884,
+    0.08393321931362152,
+    0.10751429945230484,
+    0.08020557463169098,
+    0.033069733530282974,
+    0.16630196571350098,
+    0.08266974985599518,
+    0.03755073994398117,
+    0.04478418454527855,
+    0.18494436144828796,
+    0.05650821700692177,
+    0.08012785762548447,
+    0.10531586408615112,
+    0.07290879637002945,
+    0.09117095917463303,
+    0.10888297855854034,
+    0.13270482420921326,
+    0.09928689152002335,
+    0.08380257338285446,
+    0.05985996127128601,
+    0.06112183257937431,
+    0.05544551461935043,
+    0.07464675605297089,
+    0.05754541978240013,
+    0.0796816349029541,
+    0.07248039543628693,
+    0.09227784723043442,
+    0.07590407878160477,
+    0.058416660875082016,
+    0.13287417590618134,
+    0.07496257871389389,
+    0.16476300358772278,
+    0.08258198201656342,
+    0.07838060706853867,
+    0.08748514205217361,
+    0.09477583318948746,
+    0.06408924609422684,
+    0.12438032031059265,
+    0.08364567160606384,
+    0.07156810909509659,
+    0.1006389856338501,
+    0.06504259258508682,
+    0.13034126162528992,
+    0.07353894412517548,
+    0.05378017574548721,
+    0.0991799384355545,
+    0.08791416138410568,
+    0.042945291846990585,
+    0.11117517203092575,
+    0.06654345244169235,
+    0.1068139597773552,
+    0.07217641174793243,
+    0.05371885746717453,
+    0.08958594501018524,
+    0.05896255001425743,
+    0.08083942532539368,
+    0.0591212622821331,
+    0.08475339412689209,
+    0.11193077266216278,
+    0.1608777791261673,
+    0.159384623169899,
+    0.09755388647317886,
+    0.07181630283594131,
+    0.09422580152750015,
+    0.04159745201468468,
+    0.05818576738238335,
+    0.1406513899564743,
+    0.11675318330526352,
+    0.12026544660329819,
+    0.08556979149580002,
+    0.09717367589473724,
+    0.08194345980882645,
+    0.011058744974434376
   ],
   "epoch": [
     1,
@@ -216,6 +1323,192 @@
     1,
     1,
     1,
+    1,
+    1,
+    1,
+    1,
+    1,
+    1,
+    1,
+    1,
+    1,
+    1,
+    1,
+    1,
+    1,
+    1,
+    1,
+    1,
+    1,
+    1,
+    1,
+    1,
+    1,
+    1,
+    1,
+    1,
+    1,
+    1,
+    1,
+    1,
+    1,
+    1,
+    1,
+    1,
+    1,
+    1,
+    1,
+    1,
+    1,
+    1,
+    1,
+    1,
+    1,
+    1,
+    1,
+    1,
+    1,
+    1,
+    1,
+    1,
+    1,
+    1,
+    1,
+    1,
+    1,
+    1,
+    1,
+    1,
+    1,
+    1,
+    1,
+    1,
+    1,
+    1,
+    1,
+    1,
+    1,
+    1,
+    1,
+    1,
+    1,
+    1,
+    1,
+    1,
+    1,
+    1,
+    1,
+    1,
+    1,
+    1,
+    1,
+    1,
+    1,
+    1,
+    1,
+    1,
+    1,
+    1,
+    1,
+    1,
+    1,
+    1,
+    1,
+    1,
+    1,
+    1,
+    1,
+    1,
+    1,
+    1,
+    1,
+    1,
+    1,
+    1,
+    1,
+    1,
+    1,
+    1,
+    1,
+    1,
+    1,
+    1,
+    1,
+    1,
+    1,
+    1,
+    1,
+    1,
+    1,
+    1,
+    1,
+    1,
+    1,
+    1,
+    1,
+    2,
+    2,
+    2,
+    2,
+    2,
+    2,
+    2,
+    2,
+    2,
+    2,
+    2,
+    2,
+    2,
+    2,
+    2,
+    2,
+    2,
+    2,
+    2,
+    2,
+    2,
+    2,
+    2,
+    2,
+    2,
+    2,
+    2,
+    2,
+    2,
+    2,
+    2,
+    2,
+    2,
+    2,
+    2,
+    2,
+    2,
+    2,
+    2,
+    2,
+    2,
+    2,
+    2,
+    2,
+    2,
+    2,
+    2,
+    2,
+    2,
+    2,
+    2,
+    2,
+    2,
+    2,
+    2,
+    2,
+    2,
+    2,
+    2,
+    2,
+    2,
+    2,
+    2,
     2,
     2,
     2,
@@ -237,6 +1530,189 @@
     2,
     2,
     2,
+    2,
+    2,
+    2,
+    2,
+    2,
+    2,
+    2,
+    2,
+    2,
+    2,
+    2,
+    2,
+    2,
+    2,
+    2,
+    2,
+    2,
+    2,
+    2,
+    2,
+    2,
+    2,
+    2,
+    2,
+    2,
+    2,
+    2,
+    2,
+    2,
+    2,
+    2,
+    2,
+    2,
+    2,
+    2,
+    2,
+    2,
+    2,
+    2,
+    2,
+    2,
+    2,
+    2,
+    2,
+    2,
+    2,
+    2,
+    2,
+    2,
+    2,
+    2,
+    2,
+    2,
+    2,
+    2,
+    2,
+    2,
+    2,
+    2,
+    2,
+    3,
+    3,
+    3,
+    3,
+    3,
+    3,
+    3,
+    3,
+    3,
+    3,
+    3,
+    3,
+    3,
+    3,
+    3,
+    3,
+    3,
+    3,
+    3,
+    3,
+    3,
+    3,
+    3,
+    3,
+    3,
+    3,
+    3,
+    3,
+    3,
+    3,
+    3,
+    3,
+    3,
+    3,
+    3,
+    3,
+    3,
+    3,
+    3,
+    3,
+    3,
+    3,
+    3,
+    3,
+    3,
+    3,
+    3,
+    3,
+    3,
+    3,
+    3,
+    3,
+    3,
+    3,
+    3,
+    3,
+    3,
+    3,
+    3,
+    3,
+    3,
+    3,
+    3,
+    3,
+    3,
+    3,
+    3,
+    3,
+    3,
+    3,
+    3,
+    3,
+    3,
+    3,
+    3,
+    3,
+    3,
+    3,
+    3,
+    3,
+    3,
+    3,
+    3,
+    3,
+    3,
+    3,
+    3,
+    3,
+    3,
+    3,
+    3,
+    3,
+    3,
+    3,
+    3,
+    3,
+    3,
+    3,
+    3,
+    3,
+    3,
+    3,
+    3,
+    3,
+    3,
+    3,
+    3,
+    3,
+    3,
+    3,
+    3,
+    3,
+    3,
+    3,
+    3,
+    3,
+    3,
+    3,
+    3,
+    3,
+    3,
+    3,
+    3,
     3,
     3,
     3,
@@ -260,14 +1736,14 @@
     3
   ],
   "eval_step": [
-    21,
-    42,
-    63
+    144,
+    288,
+    432
   ],
   "eval_loss": [
-    2.5687661170959473,
-    2.1139297485351562,
-    1.918800711631775
+    0.18269625306129456,
+    0.04497125372290611,
+    0.036886125802993774
   ],
   "skipped_nonfinite": 0,
   "config": {
@@ -282,7 +1758,7 @@
     "seed": 3407,
     "warmup_frac": 0.1
   },
-  "wall_seconds": 46.4,
-  "final_loss": 2.70202374458313,
-  "first_loss": 2.929511785507202
+  "wall_seconds": 323.7,
+  "final_loss": 0.001959565095603466,
+  "first_loss": 2.1868221759796143
 }

training/eval/lr_sweep.json

@@ -1,35 +1,23 @@
 {
   "results": {
-    "3e-05": {
-      "first_loss": 2.929511785507202,
-      "final_loss": 3.1568758487701416,
-      "eval_loss": [
-        2.855921506881714,
-        2.8544867038726807
-      ],
-      "best_eval": 2.8544867038726807,
-      "skipped_nonfinite": 0
-    },
-    "0.0001": {
-      "first_loss": 2.929511785507202,
-      "final_loss": 3.1012957096099854,
+    "0.0003": {
+      "first_loss": 2.186704635620117,
+      "final_loss": 0.44588127732276917,
       "eval_loss": [
-        2.8395700454711914,
-        2.8022356033325195
+        0.789882481098175
       ],
-      "best_eval": 2.8022356033325195,
+      "best_eval": 0.789882481098175,
       "skipped_nonfinite": 0
     },
-    "0.0003": {
-      "first_loss": 2.929511785507202,
-      "final_loss": 2.6298987865448,
+    "0.001": {
+      "first_loss": 2.186704635620117,
+      "final_loss": 0.0013384957564994693,
       "eval_loss": [
-        2.585228681564331,
-        2.4067068099975586
+        0.04626748710870743
       ],
-      "best_eval": 2.4067068099975586,
+      "best_eval": 0.04626748710870743,
       "skipped_nonfinite": 0
     }
   },
-  "best_lr": 0.0003
+  "best_lr": 0.001
 }

training/eval/quant_delta.json

@@ -1,12 +1,24 @@
 {
   "tensors_compared": 404,
   "tensors_changed": 135,
-  "elements_changed": 13908650,
+  "elements_changed": 25473408,
   "elements_total": 43634423,
-  "frac_elements_changed": 0.31875407175660375,
-  "max_rel_l2": 0.10790533817679403,
+  "frac_elements_changed": 0.5837915629135282,
+  "max_rel_l2": 0.2211545791610293,
   "survived_quantization": true,
   "per_tensor": [
+    {
+      "index": 371,
+      "name": "layer26.gate_proj",
+      "shape": [
+        512,
+        512
+      ],
+      "l2_delta": 82.86674499511719,
+      "rel_l2": 0.2211545791610293,
+      "elements_changed": 252451,
+      "elements": 262144
+    },
     {
       "index": 326,
       "name": "layer23.v_proj",
@@ -14,333 +26,345 @@
         256,
         512
       ],
-      "l2_delta": 42.710548400878906,
-      "rel_l2": 0.10790533817679403,
-      "elements_changed": 94492,
+      "l2_delta": 86.51488494873047,
+      "rel_l2": 0.21857405880386876,
+      "elements_changed": 126291,
       "elements": 131072
     },
     {
-      "index": 330,
-      "name": "layer23.out_proj",
+      "index": 357,
+      "name": "layer25.gate_proj",
       "shape": [
         512,
         512
       ],
-      "l2_delta": 47.22929000854492,
-      "rel_l2": 0.09371299430943461,
-      "elements_changed": 166624,
+      "l2_delta": 81.38810729980469,
+      "rel_l2": 0.21411586424498366,
+      "elements_changed": 252820,
       "elements": 262144
     },
     {
-      "index": 312,
-      "name": "layer22.v_proj",
+      "index": 330,
+      "name": "layer23.out_proj",
       "shape": [
-        256,
+        512,
         512
       ],
-      "l2_delta": 40.94161605834961,
-      "rel_l2": 0.08936908183539394,
-      "elements_changed": 79197,
-      "elements": 131072
+      "l2_delta": 106.61084747314453,
+      "rel_l2": 0.21153868162674502,
+      "elements_changed": 255121,
+      "elements": 262144
     },
     {
-      "index": 316,
-      "name": "layer22.out_proj",
+      "index": 287,
+      "name": "layer20.gate_proj",
       "shape": [
         512,
         512
       ],
-      "l2_delta": 37.57916259765625,
-      "rel_l2": 0.08215282822196236,
-      "elements_changed": 143793,
+      "l2_delta": 73.9795150756836,
+      "rel_l2": 0.20416839322381067,
+      "elements_changed": 253652,
       "elements": 262144
     },
     {
-      "index": 368,
-      "name": "layer26.v_proj",
+      "index": 372,
+      "name": "layer26.out_proj",
       "shape": [
-        256,
+        512,
         512
       ],
-      "l2_delta": 27.882591247558594,
-      "rel_l2": 0.0790704562625433,
-      "elements_changed": 76344,
-      "elements": 131072
+      "l2_delta": 86.22384643554688,
+      "rel_l2": 0.1990471407953846,
+      "elements_changed": 259024,
+      "elements": 262144
     },
     {
-      "index": 371,
-      "name": "layer26.gate_proj",
+      "index": 343,
+      "name": "layer24.gate_proj",
       "shape": [
         512,
         512
       ],
-      "l2_delta": 29.047800064086914,
-      "rel_l2": 0.07752270225052751,
-      "elements_changed": 157672,
+      "l2_delta": 71.4694595336914,
+      "rel_l2": 0.19725806375534627,
+      "elements_changed": 251637,
       "elements": 262144
     },
     {
-      "index": 288,
-      "name": "layer20.out_proj",
+      "index": 312,
+      "name": "layer22.v_proj",
+      "shape": [
+        256,
+        512
+      ],
+      "l2_delta": 89.95745849609375,
+      "rel_l2": 0.1963629246726303,
+      "elements_changed": 127551,
+      "elements": 131072
+    },
+    {
+      "index": 301,
+      "name": "layer21.gate_proj",
       "shape": [
         512,
         512
       ],
-      "l2_delta": 31.738048553466797,
-      "rel_l2": 0.0755003861074496,
-      "elements_changed": 146134,
+      "l2_delta": 71.2203598022461,
+      "rel_l2": 0.19504317612615085,
+      "elements_changed": 245126,
       "elements": 262144
     },
     {
-      "index": 298,
-      "name": "layer21.v_proj",
+      "index": 344,
+      "name": "layer24.out_proj",
       "shape": [
-        256,
+        512,
         512
       ],
-      "l2_delta": 26.3834285736084,
-      "rel_l2": 0.07548833436498338,
-      "elements_changed": 75578,
-      "elements": 131072
+      "l2_delta": 83.65033721923828,
+      "rel_l2": 0.1946801626616094,
+      "elements_changed": 255356,
+      "elements": 262144
     },
     {
-      "index": 354,
-      "name": "layer25.v_proj",
+      "index": 368,
+      "name": "layer26.v_proj",
       "shape": [
         256,
         512
       ],
-      "l2_delta": 27.52884292602539,
-      "rel_l2": 0.07506288350973363,
-      "elements_changed": 75981,
+      "l2_delta": 68.56392669677734,
+      "rel_l2": 0.19443605219226012,
+      "elements_changed": 124756,
       "elements": 131072
     },
     {
-      "index": 302,
-      "name": "layer21.out_proj",
+      "index": 316,
+      "name": "layer22.out_proj",
       "shape": [
         512,
         512
       ],
-      "l2_delta": 35.056644439697266,
-      "rel_l2": 0.07407784622936468,
-      "elements_changed": 146409,
+      "l2_delta": 88.35701751708984,
+      "rel_l2": 0.19315967628132083,
+      "elements_changed": 254799,
       "elements": 262144
     },
     {
-      "index": 340,
-      "name": "layer24.v_proj",
+      "index": 214,
+      "name": "layer15.v_proj",
       "shape": [
         256,
         512
       ],
-      "l2_delta": 26.069698333740234,
-      "rel_l2": 0.07392395130765277,
-      "elements_changed": 75575,
+      "l2_delta": 60.01744079589844,
+      "rel_l2": 0.1927138228615754,
+      "elements_changed": 126408,
       "elements": 131072
     },
     {
-      "index": 274,
-      "name": "layer19.out_proj",
+      "index": 329,
+      "name": "layer23.gate_proj",
       "shape": [
         512,
         512
       ],
-      "l2_delta": 27.664209365844727,
-      "rel_l2": 0.07366255352617818,
-      "elements_changed": 147237,
+      "l2_delta": 71.6612548828125,
+      "rel_l2": 0.19236075345710513,
+      "elements_changed": 256361,
       "elements": 262144
     },
     {
-      "index": 343,
-      "name": "layer24.gate_proj",
+      "index": 217,
+      "name": "layer15.gate_proj",
       "shape": [
         512,
         512
       ],
-      "l2_delta": 26.439563751220703,
-      "rel_l2": 0.07297406733072098,
-      "elements_changed": 152768,
+      "l2_delta": 67.80730438232422,
+      "rel_l2": 0.1917143714308789,
+      "elements_changed": 249006,
       "elements": 262144
     },
     {
-      "index": 256,
-      "name": "layer18.v_proj",
+      "index": 354,
+      "name": "layer25.v_proj",
       "shape": [
         256,
         512
       ],
-      "l2_delta": 26.23402214050293,
-      "rel_l2": 0.072267736874874,
-      "elements_changed": 75054,
+      "l2_delta": 70.13273620605469,
+      "rel_l2": 0.1912308999764426,
+      "elements_changed": 126670,
       "elements": 131072
     },
     {
-      "index": 284,
-      "name": "layer20.v_proj",
+      "index": 298,
+      "name": "layer21.v_proj",
       "shape": [
         256,
         512
       ],
-      "l2_delta": 29.92852783203125,
-      "rel_l2": 0.07205717973339744,
-      "elements_changed": 72998,
+      "l2_delta": 66.46131896972656,
+      "rel_l2": 0.19015929846749105,
+      "elements_changed": 124510,
       "elements": 131072
     },
     {
-      "index": 246,
-      "name": "layer17.out_proj",
+      "index": 231,
+      "name": "layer16.gate_proj",
       "shape": [
         512,
         512
       ],
-      "l2_delta": 28.66230583190918,
-      "rel_l2": 0.07185160339594422,
-      "elements_changed": 142615,
+      "l2_delta": 62.224788665771484,
+      "rel_l2": 0.18793213138038253,
+      "elements_changed": 251981,
       "elements": 262144
     },
     {
-      "index": 344,
-      "name": "layer24.out_proj",
+      "index": 358,
+      "name": "layer25.out_proj",
       "shape": [
         512,
         512
       ],
-      "l2_delta": 30.749006271362305,
-      "rel_l2": 0.07156243168395647,
-      "elements_changed": 136359,
+      "l2_delta": 83.08171081542969,
+      "rel_l2": 0.18736332320337823,
+      "elements_changed": 256708,
       "elements": 262144
     },
     {
-      "index": 260,
-      "name": "layer18.out_proj",
+      "index": 147,
+      "name": "layer10.gate_proj",
       "shape": [
         512,
         512
       ],
-      "l2_delta": 27.700838088989258,
-      "rel_l2": 0.07146887237809658,
-      "elements_changed": 156852,
+      "l2_delta": 58.84019470214844,
+      "rel_l2": 0.18642623587390691,
+      "elements_changed": 248778,
       "elements": 262144
     },
     {
-      "index": 301,
-      "name": "layer21.gate_proj",
+      "index": 340,
+      "name": "layer24.v_proj",
       "shape": [
-        512,
+        256,
         512
       ],
-      "l2_delta": 25.907243728637695,
-      "rel_l2": 0.07094924984285801,
-      "elements_changed": 148962,
-      "elements": 262144
+      "l2_delta": 65.23371124267578,
+      "rel_l2": 0.18497850001124974,
+      "elements_changed": 123412,
+      "elements": 131072
     },
     {
-      "index": 287,
-      "name": "layer20.gate_proj",
+      "index": 315,
+      "name": "layer22.gate_proj",
       "shape": [
         512,
         512
       ],
-      "l2_delta": 25.67437171936035,
-      "rel_l2": 0.07085603650699802,
-      "elements_changed": 150153,
+      "l2_delta": 64.64527130126953,
+      "rel_l2": 0.1840523999839433,
+      "elements_changed": 249727,
       "elements": 262144
     },
     {
-      "index": 315,
-      "name": "layer22.gate_proj",
+      "index": 284,
+      "name": "layer20.v_proj",
       "shape": [
-        512,
+        256,
         512
       ],
-      "l2_delta": 24.833499908447266,
-      "rel_l2": 0.07070378337264391,
-      "elements_changed": 149937,
-      "elements": 262144
+      "l2_delta": 75.571044921875,
+      "rel_l2": 0.18194801953299475,
+      "elements_changed": 122216,
+      "elements": 131072
     },
     {
-      "index": 357,
-      "name": "layer25.gate_proj",
+      "index": 161,
+      "name": "layer11.gate_proj",
       "shape": [
         512,
         512
       ],
-      "l2_delta": 26.454133987426758,
-      "rel_l2": 0.06959554595249899,
-      "elements_changed": 132430,
+      "l2_delta": 56.16728973388672,
+      "rel_l2": 0.18117371195650783,
+      "elements_changed": 251285,
       "elements": 262144
     },
     {
-      "index": 46,
-      "name": "layer03.v_proj",
+      "index": 203,
+      "name": "layer14.gate_proj",
       "shape": [
-        256,
+        512,
         512
       ],
-      "l2_delta": 18.333171844482422,
-      "rel_l2": 0.06918459301773189,
-      "elements_changed": 72362,
-      "elements": 131072
+      "l2_delta": 58.19242858886719,
+      "rel_l2": 0.17774989464201638,
+      "elements_changed": 245945,
+      "elements": 262144
     },
     {
-      "index": 358,
-      "name": "layer25.out_proj",
+      "index": 245,
+      "name": "layer17.gate_proj",
       "shape": [
         512,
         512
       ],
-      "l2_delta": 30.208126068115234,
-      "rel_l2": 0.06812443836697608,
-      "elements_changed": 136408,
+      "l2_delta": 61.24203872680664,
+      "rel_l2": 0.17641460615258053,
+      "elements_changed": 248150,
       "elements": 262144
     },
     {
-      "index": 242,
-      "name": "layer17.v_proj",
+      "index": 189,
+      "name": "layer13.gate_proj",
       "shape": [
-        256,
+        512,
         512
       ],
-      "l2_delta": 23.941869735717773,
-      "rel_l2": 0.06723200618495882,
-      "elements_changed": 70428,
-      "elements": 131072
+      "l2_delta": 55.636661529541016,
+      "rel_l2": 0.1729518731213408,
+      "elements_changed": 245536,
+      "elements": 262144
     },
     {
-      "index": 130,
-      "name": "layer09.v_proj",
+      "index": 297,
+      "name": "layer21.k_proj",
       "shape": [
         256,
         512
       ],
-      "l2_delta": 19.131349563598633,
-      "rel_l2": 0.06697352061465177,
-      "elements_changed": 70656,
+      "l2_delta": 42.96588897705078,
+      "rel_l2": 0.17131077960294988,
+      "elements_changed": 124399,
       "elements": 131072
     },
     {
-      "index": 259,
-      "name": "layer18.gate_proj",
+      "index": 218,
+      "name": "layer15.out_proj",
       "shape": [
         512,
         512
       ],
-      "l2_delta": 22.746126174926758,
-      "rel_l2": 0.0660536707021693,
-      "elements_changed": 155108,
+      "l2_delta": 68.41730499267578,
+      "rel_l2": 0.17051787457544484,
+      "elements_changed": 254925,
       "elements": 262144
     },
     {
-      "index": 273,
-      "name": "layer19.gate_proj",
+      "index": 119,
+      "name": "layer08.gate_proj",
       "shape": [
         512,
         512
       ],
-      "l2_delta": 23.085159301757812,
-      "rel_l2": 0.06563402901779904,
-      "elements_changed": 147263,
+      "l2_delta": 53.99991989135742,
+      "rel_l2": 0.17020874764017047,
+      "elements_changed": 249800,
       "elements": 262144
     },
     {
@@ -350,225 +374,189 @@
         512,
         512
       ],
-      "l2_delta": 21.216899871826172,
-      "rel_l2": 0.06509856049798989,
-      "elements_changed": 150027,
+      "l2_delta": 55.266292572021484,
+      "rel_l2": 0.16957030066757275,
+      "elements_changed": 252428,
       "elements": 262144
     },
     {
-      "index": 228,
-      "name": "layer16.v_proj",
+      "index": 325,
+      "name": "layer23.k_proj",
       "shape": [
         256,
         512
       ],
-      "l2_delta": 21.619653701782227,
-      "rel_l2": 0.06507947821426656,
-      "elements_changed": 69381,
+      "l2_delta": 45.33278274536133,
+      "rel_l2": 0.16903329824761038,
+      "elements_changed": 124478,
       "elements": 131072
     },
     {
-      "index": 232,
-      "name": "layer16.out_proj",
+      "index": 288,
+      "name": "layer20.out_proj",
       "shape": [
         512,
         512
       ],
-      "l2_delta": 24.660661697387695,
-      "rel_l2": 0.06475604330817823,
-      "elements_changed": 133164,
+      "l2_delta": 70.94759368896484,
+      "rel_l2": 0.16877441938143967,
+      "elements_changed": 243414,
       "elements": 262144
     },
     {
-      "index": 245,
-      "name": "layer17.gate_proj",
+      "index": 302,
+      "name": "layer21.out_proj",
       "shape": [
         512,
         512
       ],
-      "l2_delta": 22.429218292236328,
-      "rel_l2": 0.06460989532020842,
-      "elements_changed": 144742,
+      "l2_delta": 79.3014907836914,
+      "rel_l2": 0.16757119039555177,
+      "elements_changed": 253023,
       "elements": 262144
     },
     {
-      "index": 214,
-      "name": "layer15.v_proj",
-      "shape": [
-        256,
-        512
-      ],
-      "l2_delta": 20.080358505249023,
-      "rel_l2": 0.06447730194190397,
-      "elements_changed": 61519,
-      "elements": 131072
-    },
-    {
-      "index": 372,
-      "name": "layer26.out_proj",
+      "index": 105,
+      "name": "layer07.gate_proj",
       "shape": [
         512,
         512
       ],
-      "l2_delta": 27.703821182250977,
-      "rel_l2": 0.06395407562286969,
-      "elements_changed": 122880,
+      "l2_delta": 52.6637077331543,
+      "rel_l2": 0.16735901981831144,
+      "elements_changed": 246375,
       "elements": 262144
     },
     {
-      "index": 218,
-      "name": "layer15.out_proj",
+      "index": 162,
+      "name": "layer11.out_proj",
       "shape": [
         512,
         512
       ],
-      "l2_delta": 25.607032775878906,
-      "rel_l2": 0.06382094126031511,
-      "elements_changed": 123432,
+      "l2_delta": 61.89963150024414,
+      "rel_l2": 0.16689410343362973,
+      "elements_changed": 248631,
       "elements": 262144
     },
     {
-      "index": 172,
-      "name": "layer12.v_proj",
+      "index": 246,
+      "name": "layer17.out_proj",
       "shape": [
-        256,
+        512,
         512
       ],
-      "l2_delta": 18.66848373413086,
-      "rel_l2": 0.06330385346734449,
-      "elements_changed": 68343,
-      "elements": 131072
+      "l2_delta": 66.2942886352539,
+      "rel_l2": 0.16618868566860281,
+      "elements_changed": 241730,
+      "elements": 262144
     },
     {
-      "index": 186,
-      "name": "layer13.v_proj",
+      "index": 228,
+      "name": "layer16.v_proj",
       "shape": [
         256,
         512
       ],
-      "l2_delta": 19.474767684936523,
-      "rel_l2": 0.06291022566773993,
-      "elements_changed": 63921,
+      "l2_delta": 54.29331970214844,
+      "rel_l2": 0.16343374253237472,
+      "elements_changed": 120166,
       "elements": 131072
     },
     {
-      "index": 240,
-      "name": "layer17.q_proj",
+      "index": 133,
+      "name": "layer09.gate_proj",
       "shape": [
         512,
         512
       ],
-      "l2_delta": 22.345680236816406,
-      "rel_l2": 0.0628242570808463,
-      "elements_changed": 142328,
+      "l2_delta": 51.62324523925781,
+      "rel_l2": 0.16272147911538504,
+      "elements_changed": 243241,
       "elements": 262144
     },
     {
-      "index": 102,
-      "name": "layer07.v_proj",
+      "index": 226,
+      "name": "layer16.q_proj",
       "shape": [
-        256,
+        512,
         512
       ],
-      "l2_delta": 17.382795333862305,
-      "rel_l2": 0.06263170153545683,
-      "elements_changed": 62373,
-      "elements": 131072
+      "l2_delta": 52.6801872253418,
+      "rel_l2": 0.1622030489081987,
+      "elements_changed": 240238,
+      "elements": 262144
     },
     {
-      "index": 297,
-      "name": "layer21.k_proj",
+      "index": 158,
+      "name": "layer11.v_proj",
       "shape": [
         256,
         512
       ],
-      "l2_delta": 15.62189769744873,
-      "rel_l2": 0.06228660775195176,
-      "elements_changed": 68693,
+      "l2_delta": 49.18385314941406,
+      "rel_l2": 0.1620142808028453,
+      "elements_changed": 120325,
       "elements": 131072
     },
     {
-      "index": 77,
-      "name": "layer05.gate_proj",
-      "shape": [
-        512,
-        512
-      ],
-      "l2_delta": 19.22298812866211,
-      "rel_l2": 0.06224692070386283,
-      "elements_changed": 157786,
-      "elements": 262144
-    },
-    {
-      "index": 268,
-      "name": "layer19.q_proj",
+      "index": 175,
+      "name": "layer12.gate_proj",
       "shape": [
         512,
         512
       ],
-      "l2_delta": 20.993209838867188,
-      "rel_l2": 0.06220001661908948,
-      "elements_changed": 144139,
+      "l2_delta": 51.028018951416016,
+      "rel_l2": 0.1599833179098988,
+      "elements_changed": 245263,
       "elements": 262144
     },
     {
-      "index": 44,
-      "name": "layer03.q_proj",
+      "index": 35,
+      "name": "layer02.gate_proj",
       "shape": [
         512,
         512
       ],
-      "l2_delta": 20.09811782836914,
-      "rel_l2": 0.06172652065877976,
-      "elements_changed": 160355,
+      "l2_delta": 51.190494537353516,
+      "rel_l2": 0.15982231237117656,
+      "elements_changed": 240880,
       "elements": 262144
     },
     {
-      "index": 270,
-      "name": "layer19.v_proj",
-      "shape": [
-        256,
-        512
-      ],
-      "l2_delta": 21.656429290771484,
-      "rel_l2": 0.06157875963776336,
-      "elements_changed": 65648,
-      "elements": 131072
-    },
-    {
-      "index": 158,
-      "name": "layer11.v_proj",
+      "index": 242,
+      "name": "layer17.v_proj",
       "shape": [
         256,
         512
       ],
-      "l2_delta": 18.491987228393555,
-      "rel_l2": 0.06091360923517384,
-      "elements_changed": 63126,
+      "l2_delta": 56.631103515625,
+      "rel_l2": 0.1590277928938618,
+      "elements_changed": 121613,
       "elements": 131072
     },
     {
-      "index": 50,
-      "name": "layer03.out_proj",
+      "index": 274,
+      "name": "layer19.out_proj",
       "shape": [
         512,
         512
       ],
-      "l2_delta": 21.070030212402344,
-      "rel_l2": 0.06089377455757774,
-      "elements_changed": 126353,
+      "l2_delta": 59.30603790283203,
+      "rel_l2": 0.15791646649539157,
+      "elements_changed": 245652,
       "elements": 262144
     },
     {
-      "index": 282,
-      "name": "layer20.q_proj",
+      "index": 259,
+      "name": "layer18.gate_proj",
       "shape": [
         512,
         512
       ],
-      "l2_delta": 23.53261375427246,
-      "rel_l2": 0.060190255751143854,
-      "elements_changed": 135494,
+      "l2_delta": 54.33344268798828,
+      "rel_l2": 0.1577817384739414,
+      "elements_changed": 244339,
       "elements": 262144
     },
     {
@@ -578,58 +566,46 @@
         256,
         512
       ],
-      "l2_delta": 15.313697814941406,
-      "rel_l2": 0.060050094874026246,
-      "elements_changed": 61247,
+      "l2_delta": 40.2072868347168,
+      "rel_l2": 0.15766612468323468,
+      "elements_changed": 116826,
       "elements": 131072
     },
     {
-      "index": 329,
-      "name": "layer23.gate_proj",
-      "shape": [
-        512,
-        512
-      ],
-      "l2_delta": 22.3406925201416,
-      "rel_l2": 0.059969260278174015,
-      "elements_changed": 128604,
-      "elements": 262144
-    },
-    {
-      "index": 325,
-      "name": "layer23.k_proj",
+      "index": 46,
+      "name": "layer03.v_proj",
       "shape": [
         256,
         512
       ],
-      "l2_delta": 16.021921157836914,
-      "rel_l2": 0.059741273611743285,
-      "elements_changed": 49599,
+      "l2_delta": 41.757694244384766,
+      "rel_l2": 0.15758261069952934,
+      "elements_changed": 118004,
       "elements": 131072
     },
     {
-      "index": 176,
-      "name": "layer12.out_proj",
+      "index": 204,
+      "name": "layer14.out_proj",
       "shape": [
         512,
         512
       ],
-      "l2_delta": 21.808856964111328,
-      "rel_l2": 0.059712479488872476,
-      "elements_changed": 119079,
+      "l2_delta": 61.04585266113281,
+      "rel_l2": 0.15709891193510062,
+      "elements_changed": 240691,
       "elements": 262144
     },
     {
-      "index": 60,
-      "name": "layer04.v_proj",
+      "index": 352,
+      "name": "layer25.q_proj",
       "shape": [
-        256,
+        512,
         512
       ],
-      "l2_delta": 16.008520126342773,
-      "rel_l2": 0.05918895314527836,
-      "elements_changed": 62425,
-      "elements": 131072
+      "l2_delta": 57.957374572753906,
+      "rel_l2": 0.15608588726426406,
+      "elements_changed": 245929,
+      "elements": 262144
     },
     {
       "index": 200,
@@ -638,189 +614,177 @@
         256,
         512
       ],
-      "l2_delta": 19.11776351928711,
-      "rel_l2": 0.059110174909070286,
-      "elements_changed": 60419,
+      "l2_delta": 50.31721115112305,
+      "rel_l2": 0.15557568483776357,
+      "elements_changed": 118482,
       "elements": 131072
     },
     {
-      "index": 45,
-      "name": "layer03.k_proj",
-      "shape": [
-        256,
-        512
-      ],
-      "l2_delta": 14.018465995788574,
-      "rel_l2": 0.058790946570816786,
-      "elements_changed": 75441,
-      "elements": 131072
-    },
-    {
-      "index": 74,
-      "name": "layer05.v_proj",
+      "index": 186,
+      "name": "layer13.v_proj",
       "shape": [
         256,
         512
       ],
-      "l2_delta": 16.1820125579834,
-      "rel_l2": 0.05863619220958507,
-      "elements_changed": 65461,
+      "l2_delta": 47.755062103271484,
+      "rel_l2": 0.15426534386941707,
+      "elements_changed": 116046,
       "elements": 131072
     },
     {
-      "index": 162,
-      "name": "layer11.out_proj",
+      "index": 184,
+      "name": "layer13.q_proj",
       "shape": [
         512,
         512
       ],
-      "l2_delta": 21.648418426513672,
-      "rel_l2": 0.05836857662124884,
-      "elements_changed": 107345,
+      "l2_delta": 49.666107177734375,
+      "rel_l2": 0.15406160158050833,
+      "elements_changed": 247390,
       "elements": 262144
     },
     {
-      "index": 78,
-      "name": "layer05.out_proj",
+      "index": 338,
+      "name": "layer24.q_proj",
       "shape": [
         512,
         512
       ],
-      "l2_delta": 19.40888786315918,
-      "rel_l2": 0.058281551549707766,
-      "elements_changed": 133403,
+      "l2_delta": 60.1658935546875,
+      "rel_l2": 0.15308882442015215,
+      "elements_changed": 239331,
       "elements": 262144
     },
     {
-      "index": 296,
-      "name": "layer21.q_proj",
+      "index": 91,
+      "name": "layer06.gate_proj",
       "shape": [
         512,
         512
       ],
-      "l2_delta": 21.102941513061523,
-      "rel_l2": 0.058270301730098326,
-      "elements_changed": 128922,
+      "l2_delta": 48.499847412109375,
+      "rel_l2": 0.15253503450277534,
+      "elements_changed": 242657,
       "elements": 262144
     },
     {
-      "index": 269,
-      "name": "layer19.k_proj",
+      "index": 116,
+      "name": "layer08.v_proj",
       "shape": [
         256,
         512
       ],
-      "l2_delta": 13.79250431060791,
-      "rel_l2": 0.05826202230946683,
-      "elements_changed": 65678,
+      "l2_delta": 44.808414459228516,
+      "rel_l2": 0.15208531517250495,
+      "elements_changed": 115199,
       "elements": 131072
     },
     {
-      "index": 254,
-      "name": "layer18.q_proj",
+      "index": 102,
+      "name": "layer07.v_proj",
       "shape": [
-        512,
+        256,
         512
       ],
-      "l2_delta": 20.748870849609375,
-      "rel_l2": 0.05818409392239331,
-      "elements_changed": 148178,
-      "elements": 262144
+      "l2_delta": 42.1906852722168,
+      "rel_l2": 0.15201665536487347,
+      "elements_changed": 115218,
+      "elements": 131072
     },
     {
-      "index": 116,
-      "name": "layer08.v_proj",
+      "index": 353,
+      "name": "layer25.k_proj",
       "shape": [
         256,
         512
       ],
-      "l2_delta": 17.13956069946289,
-      "rel_l2": 0.058173794416848314,
-      "elements_changed": 59909,
+      "l2_delta": 38.59225082397461,
+      "rel_l2": 0.1514911249890463,
+      "elements_changed": 124336,
       "elements": 131072
     },
     {
-      "index": 283,
-      "name": "layer20.k_proj",
+      "index": 148,
+      "name": "layer10.out_proj",
       "shape": [
-        256,
+        512,
         512
       ],
-      "l2_delta": 16.008220672607422,
-      "rel_l2": 0.05817136496459642,
-      "elements_changed": 58874,
-      "elements": 131072
+      "l2_delta": 56.39916229248047,
+      "rel_l2": 0.1514664998319441,
+      "elements_changed": 247212,
+      "elements": 262144
     },
     {
-      "index": 311,
-      "name": "layer22.k_proj",
+      "index": 36,
+      "name": "layer02.out_proj",
       "shape": [
-        256,
+        512,
         512
       ],
-      "l2_delta": 17.985631942749023,
-      "rel_l2": 0.05788171525178631,
-      "elements_changed": 58951,
-      "elements": 131072
+      "l2_delta": 48.39224624633789,
+      "rel_l2": 0.15114477562959072,
+      "elements_changed": 252329,
+      "elements": 262144
     },
     {
-      "index": 105,
-      "name": "layer07.gate_proj",
+      "index": 273,
+      "name": "layer19.gate_proj",
       "shape": [
         512,
         512
       ],
-      "l2_delta": 18.168752670288086,
-      "rel_l2": 0.057738142054636905,
-      "elements_changed": 145057,
+      "l2_delta": 52.73903274536133,
+      "rel_l2": 0.14994374352513679,
+      "elements_changed": 235542,
       "elements": 262144
     },
     {
-      "index": 203,
-      "name": "layer14.gate_proj",
+      "index": 256,
+      "name": "layer18.v_proj",
       "shape": [
-        512,
+        256,
         512
       ],
-      "l2_delta": 18.90132713317871,
-      "rel_l2": 0.05773446800533683,
-      "elements_changed": 145692,
-      "elements": 262144
+      "l2_delta": 54.05394744873047,
+      "rel_l2": 0.14890421416707061,
+      "elements_changed": 117365,
+      "elements": 131072
     },
     {
-      "index": 49,
-      "name": "layer03.gate_proj",
+      "index": 270,
+      "name": "layer19.v_proj",
       "shape": [
-        512,
+        256,
         512
       ],
-      "l2_delta": 19.131404876708984,
-      "rel_l2": 0.05759906148768893,
-      "elements_changed": 141173,
-      "elements": 262144
+      "l2_delta": 52.218238830566406,
+      "rel_l2": 0.1484794346510768,
+      "elements_changed": 117094,
+      "elements": 131072
     },
     {
-      "index": 133,
-      "name": "layer09.gate_proj",
+      "index": 283,
+      "name": "layer20.k_proj",
       "shape": [
-        512,
+        256,
         512
       ],
-      "l2_delta": 18.251859664916992,
-      "rel_l2": 0.05753163303695528,
-      "elements_changed": 144869,
-      "elements": 262144
+      "l2_delta": 40.59066390991211,
+      "rel_l2": 0.1475001108960959,
+      "elements_changed": 118417,
+      "elements": 131072
     },
     {
-      "index": 72,
-      "name": "layer05.q_proj",
+      "index": 190,
+      "name": "layer13.out_proj",
       "shape": [
         512,
         512
       ],
-      "l2_delta": 17.537029266357422,
-      "rel_l2": 0.057488712410466096,
-      "elements_changed": 150900,
+      "l2_delta": 56.75508499145508,
+      "rel_l2": 0.14623698876537022,
+      "elements_changed": 236684,
       "elements": 262144
     },
     {
@@ -830,693 +794,705 @@
         256,
         512
       ],
-      "l2_delta": 17.8068790435791,
-      "rel_l2": 0.05746422420130803,
-      "elements_changed": 58411,
+      "l2_delta": 45.08926010131836,
+      "rel_l2": 0.14550665196254783,
+      "elements_changed": 118784,
       "elements": 131072
     },
     {
-      "index": 231,
-      "name": "layer16.gate_proj",
+      "index": 77,
+      "name": "layer05.gate_proj",
       "shape": [
         512,
         512
       ],
-      "l2_delta": 19.006389617919922,
-      "rel_l2": 0.057403349811720565,
-      "elements_changed": 137763,
+      "l2_delta": 43.93071746826172,
+      "rel_l2": 0.14225425664355387,
+      "elements_changed": 236994,
       "elements": 262144
     },
     {
-      "index": 217,
-      "name": "layer15.gate_proj",
+      "index": 232,
+      "name": "layer16.out_proj",
       "shape": [
         512,
         512
       ],
-      "l2_delta": 20.298234939575195,
-      "rel_l2": 0.057390031767895734,
-      "elements_changed": 119455,
+      "l2_delta": 54.10078048706055,
+      "rel_l2": 0.14206238775001914,
+      "elements_changed": 233294,
       "elements": 262144
     },
     {
-      "index": 161,
-      "name": "layer11.gate_proj",
+      "index": 296,
+      "name": "layer21.q_proj",
       "shape": [
         512,
         512
       ],
-      "l2_delta": 17.785181045532227,
-      "rel_l2": 0.05736803899750185,
-      "elements_changed": 144824,
+      "l2_delta": 51.06755828857422,
+      "rel_l2": 0.14100982217350128,
+      "elements_changed": 237228,
       "elements": 262144
     },
     {
-      "index": 189,
-      "name": "layer13.gate_proj",
+      "index": 49,
+      "name": "layer03.gate_proj",
       "shape": [
         512,
         512
       ],
-      "l2_delta": 18.438119888305664,
-      "rel_l2": 0.0573166556700229,
-      "elements_changed": 143850,
+      "l2_delta": 46.82871627807617,
+      "rel_l2": 0.1409875608024057,
+      "elements_changed": 238610,
       "elements": 262144
     },
     {
-      "index": 147,
-      "name": "layer10.gate_proj",
+      "index": 92,
+      "name": "layer06.out_proj",
       "shape": [
         512,
         512
       ],
-      "l2_delta": 18.047332763671875,
-      "rel_l2": 0.05718023762032748,
-      "elements_changed": 146988,
+      "l2_delta": 47.38945388793945,
+      "rel_l2": 0.14043225674708634,
+      "elements_changed": 232726,
       "elements": 262144
     },
     {
-      "index": 134,
-      "name": "layer09.out_proj",
+      "index": 74,
+      "name": "layer05.v_proj",
       "shape": [
-        512,
+        256,
         512
       ],
-      "l2_delta": 20.410003662109375,
-      "rel_l2": 0.05712638632673821,
-      "elements_changed": 115645,
-      "elements": 262144
+      "l2_delta": 38.35731887817383,
+      "rel_l2": 0.13898933240385722,
+      "elements_changed": 116532,
+      "elements": 131072
     },
     {
-      "index": 352,
-      "name": "layer25.q_proj",
+      "index": 176,
+      "name": "layer12.out_proj",
       "shape": [
         512,
         512
       ],
-      "l2_delta": 21.13182258605957,
-      "rel_l2": 0.056910432920243254,
-      "elements_changed": 120245,
+      "l2_delta": 50.575836181640625,
+      "rel_l2": 0.13847624318865123,
+      "elements_changed": 235572,
       "elements": 262144
     },
     {
-      "index": 92,
-      "name": "layer06.out_proj",
+      "index": 268,
+      "name": "layer19.q_proj",
       "shape": [
         512,
         512
       ],
-      "l2_delta": 19.178056716918945,
-      "rel_l2": 0.05683158516975367,
-      "elements_changed": 130369,
+      "l2_delta": 46.417274475097656,
+      "rel_l2": 0.13752805149494496,
+      "elements_changed": 228025,
       "elements": 262144
     },
     {
-      "index": 338,
-      "name": "layer24.q_proj",
+      "index": 282,
+      "name": "layer20.q_proj",
       "shape": [
         512,
         512
       ],
-      "l2_delta": 22.287878036499023,
-      "rel_l2": 0.05671028627416662,
-      "elements_changed": 129442,
+      "l2_delta": 53.76548767089844,
+      "rel_l2": 0.1375180201947949,
+      "elements_changed": 227688,
       "elements": 262144
     },
     {
-      "index": 310,
-      "name": "layer22.q_proj",
+      "index": 240,
+      "name": "layer17.q_proj",
       "shape": [
         512,
         512
       ],
-      "l2_delta": 21.00835418701172,
-      "rel_l2": 0.05665616011958008,
-      "elements_changed": 119810,
+      "l2_delta": 48.61473083496094,
+      "rel_l2": 0.13667896056525966,
+      "elements_changed": 234141,
       "elements": 262144
     },
     {
-      "index": 226,
-      "name": "layer16.q_proj",
+      "index": 78,
+      "name": "layer05.out_proj",
       "shape": [
         512,
         512
       ],
-      "l2_delta": 18.35666847229004,
-      "rel_l2": 0.056520444418050986,
-      "elements_changed": 137396,
+      "l2_delta": 45.46884536743164,
+      "rel_l2": 0.13653512111931587,
+      "elements_changed": 240666,
       "elements": 262144
     },
     {
-      "index": 204,
-      "name": "layer14.out_proj",
+      "index": 7,
+      "name": "layer00.gate_proj",
       "shape": [
         512,
         512
       ],
-      "l2_delta": 21.87782859802246,
-      "rel_l2": 0.05630166372367248,
-      "elements_changed": 105706,
+      "l2_delta": 41.555110931396484,
+      "rel_l2": 0.13604044004407473,
+      "elements_changed": 237404,
       "elements": 262144
     },
     {
-      "index": 190,
-      "name": "layer13.out_proj",
+      "index": 88,
+      "name": "layer06.v_proj",
       "shape": [
-        512,
+        256,
         512
       ],
-      "l2_delta": 21.837644577026367,
-      "rel_l2": 0.05626758175330994,
-      "elements_changed": 111131,
-      "elements": 262144
+      "l2_delta": 38.24935531616211,
+      "rel_l2": 0.1360178979475252,
+      "elements_changed": 114096,
+      "elements": 131072
     },
     {
-      "index": 119,
-      "name": "layer08.gate_proj",
+      "index": 21,
+      "name": "layer01.gate_proj",
       "shape": [
         512,
         512
       ],
-      "l2_delta": 17.689348220825195,
-      "rel_l2": 0.05575715321976653,
-      "elements_changed": 152448,
+      "l2_delta": 41.87150192260742,
+      "rel_l2": 0.13393652643592144,
+      "elements_changed": 233605,
       "elements": 262144
     },
     {
-      "index": 91,
-      "name": "layer06.gate_proj",
+      "index": 156,
+      "name": "layer11.q_proj",
       "shape": [
         512,
         512
       ],
-      "l2_delta": 17.620288848876953,
-      "rel_l2": 0.05541690357651001,
-      "elements_changed": 141145,
+      "l2_delta": 43.62898635864258,
+      "rel_l2": 0.13232472651068536,
+      "elements_changed": 230426,
       "elements": 262144
     },
     {
-      "index": 35,
-      "name": "layer02.gate_proj",
+      "index": 172,
+      "name": "layer12.v_proj",
+      "shape": [
+        256,
+        512
+      ],
+      "l2_delta": 39.006839752197266,
+      "rel_l2": 0.132270156648169,
+      "elements_changed": 108954,
+      "elements": 131072
+    },
+    {
+      "index": 260,
+      "name": "layer18.out_proj",
       "shape": [
         512,
         512
       ],
-      "l2_delta": 17.662721633911133,
-      "rel_l2": 0.05514494516633797,
-      "elements_changed": 136201,
+      "l2_delta": 50.74163818359375,
+      "rel_l2": 0.1309147272710237,
+      "elements_changed": 226620,
       "elements": 262144
     },
     {
-      "index": 64,
-      "name": "layer04.out_proj",
+      "index": 60,
+      "name": "layer04.v_proj",
       "shape": [
-        512,
+        256,
         512
       ],
-      "l2_delta": 17.29135513305664,
-      "rel_l2": 0.05507002777298561,
-      "elements_changed": 132519,
-      "elements": 262144
+      "l2_delta": 35.262001037597656,
+      "rel_l2": 0.13037563189795845,
+      "elements_changed": 112075,
+      "elements": 131072
     },
     {
-      "index": 88,
-      "name": "layer06.v_proj",
+      "index": 130,
+      "name": "layer09.v_proj",
       "shape": [
         256,
         512
       ],
-      "l2_delta": 15.372090339660645,
-      "rel_l2": 0.05466443545982086,
-      "elements_changed": 56795,
+      "l2_delta": 36.883121490478516,
+      "rel_l2": 0.12911752457732115,
+      "elements_changed": 113136,
       "elements": 131072
     },
     {
-      "index": 366,
-      "name": "layer26.q_proj",
+      "index": 134,
+      "name": "layer09.out_proj",
       "shape": [
         512,
         512
       ],
-      "l2_delta": 17.822195053100586,
-      "rel_l2": 0.053229167283802364,
-      "elements_changed": 132670,
+      "l2_delta": 45.91088104248047,
+      "rel_l2": 0.128501825401564,
+      "elements_changed": 230691,
       "elements": 262144
     },
     {
-      "index": 86,
-      "name": "layer06.q_proj",
+      "index": 2,
+      "name": "layer00.q_proj",
       "shape": [
         512,
         512
       ],
-      "l2_delta": 16.65826988220215,
-      "rel_l2": 0.052807688813799505,
-      "elements_changed": 128699,
+      "l2_delta": 40.08195877075195,
+      "rel_l2": 0.12775174763039226,
+      "elements_changed": 228836,
       "elements": 262144
     },
     {
-      "index": 100,
-      "name": "layer07.q_proj",
+      "index": 198,
+      "name": "layer14.q_proj",
       "shape": [
         512,
         512
       ],
-      "l2_delta": 16.822599411010742,
-      "rel_l2": 0.05271572795447827,
-      "elements_changed": 127980,
+      "l2_delta": 41.794525146484375,
+      "rel_l2": 0.12742569982935756,
+      "elements_changed": 236641,
       "elements": 262144
     },
     {
-      "index": 175,
-      "name": "layer12.gate_proj",
+      "index": 63,
+      "name": "layer04.gate_proj",
       "shape": [
         512,
         512
       ],
-      "l2_delta": 16.753116607666016,
-      "rel_l2": 0.05252446078256899,
-      "elements_changed": 133796,
+      "l2_delta": 39.12565231323242,
+      "rel_l2": 0.12694873588378225,
+      "elements_changed": 229349,
       "elements": 262144
     },
     {
-      "index": 73,
-      "name": "layer05.k_proj",
+      "index": 241,
+      "name": "layer17.k_proj",
       "shape": [
         256,
         512
       ],
-      "l2_delta": 12.190053939819336,
-      "rel_l2": 0.05211851339654197,
-      "elements_changed": 65076,
+      "l2_delta": 32.07200241088867,
+      "rel_l2": 0.12459707972012186,
+      "elements_changed": 121255,
       "elements": 131072
     },
     {
-      "index": 18,
-      "name": "layer01.v_proj",
+      "index": 4,
+      "name": "layer00.v_proj",
       "shape": [
         256,
         512
       ],
-      "l2_delta": 13.014585494995117,
-      "rel_l2": 0.05208991258740191,
-      "elements_changed": 60617,
+      "l2_delta": 32.07552719116211,
+      "rel_l2": 0.12389845050738145,
+      "elements_changed": 111519,
       "elements": 131072
     },
     {
-      "index": 148,
-      "name": "layer10.out_proj",
+      "index": 366,
+      "name": "layer26.q_proj",
       "shape": [
         512,
         512
       ],
-      "l2_delta": 19.1829776763916,
-      "rel_l2": 0.05151811422179134,
-      "elements_changed": 99105,
+      "l2_delta": 41.47237777709961,
+      "rel_l2": 0.12386466020470524,
+      "elements_changed": 227735,
       "elements": 262144
     },
     {
-      "index": 184,
-      "name": "layer13.q_proj",
+      "index": 170,
+      "name": "layer12.q_proj",
       "shape": [
         512,
         512
       ],
-      "l2_delta": 16.585750579833984,
-      "rel_l2": 0.051448109041442903,
-      "elements_changed": 124979,
+      "l2_delta": 38.825801849365234,
+      "rel_l2": 0.12321986510398557,
+      "elements_changed": 232882,
       "elements": 262144
     },
     {
-      "index": 255,
-      "name": "layer18.k_proj",
+      "index": 367,
+      "name": "layer26.k_proj",
       "shape": [
         256,
         512
       ],
-      "l2_delta": 13.718951225280762,
-      "rel_l2": 0.051418835966121,
-      "elements_changed": 60796,
+      "l2_delta": 31.134965896606445,
+      "rel_l2": 0.12287986047134011,
+      "elements_changed": 117511,
       "elements": 131072
     },
     {
-      "index": 367,
-      "name": "layer26.k_proj",
+      "index": 311,
+      "name": "layer22.k_proj",
       "shape": [
         256,
         512
       ],
-      "l2_delta": 12.984603881835938,
-      "rel_l2": 0.051246123685317016,
-      "elements_changed": 53647,
+      "l2_delta": 38.102996826171875,
+      "rel_l2": 0.12262381547407014,
+      "elements_changed": 116724,
       "elements": 131072
     },
     {
-      "index": 2,
-      "name": "layer00.q_proj",
+      "index": 50,
+      "name": "layer03.out_proj",
       "shape": [
         512,
         512
       ],
-      "l2_delta": 16.005855560302734,
-      "rel_l2": 0.05101487259750367,
-      "elements_changed": 126541,
+      "l2_delta": 42.404380798339844,
+      "rel_l2": 0.12255145239743698,
+      "elements_changed": 238182,
       "elements": 262144
     },
     {
-      "index": 241,
-      "name": "layer17.k_proj",
+      "index": 18,
+      "name": "layer01.v_proj",
       "shape": [
         256,
         512
       ],
-      "l2_delta": 13.100255966186523,
-      "rel_l2": 0.050893412143757595,
-      "elements_changed": 47036,
+      "l2_delta": 30.478450775146484,
+      "rel_l2": 0.12198773731882177,
+      "elements_changed": 110952,
       "elements": 131072
     },
     {
-      "index": 353,
-      "name": "layer25.k_proj",
+      "index": 106,
+      "name": "layer07.out_proj",
       "shape": [
-        256,
+        512,
         512
       ],
-      "l2_delta": 12.919740676879883,
-      "rel_l2": 0.05071551951282901,
-      "elements_changed": 53233,
-      "elements": 131072
+      "l2_delta": 41.00419998168945,
+      "rel_l2": 0.12155581830628641,
+      "elements_changed": 232637,
+      "elements": 262144
     },
     {
-      "index": 170,
-      "name": "layer12.q_proj",
+      "index": 64,
+      "name": "layer04.out_proj",
       "shape": [
         512,
         512
       ],
-      "l2_delta": 15.96741771697998,
-      "rel_l2": 0.05067514290570742,
-      "elements_changed": 126488,
+      "l2_delta": 38.067256927490234,
+      "rel_l2": 0.12123774453226939,
+      "elements_changed": 229303,
       "elements": 262144
     },
     {
-      "index": 120,
-      "name": "layer08.out_proj",
+      "index": 310,
+      "name": "layer22.q_proj",
       "shape": [
         512,
         512
       ],
-      "l2_delta": 17.927276611328125,
-      "rel_l2": 0.0503821105732096,
-      "elements_changed": 99889,
+      "l2_delta": 44.94417190551758,
+      "rel_l2": 0.12120721962576249,
+      "elements_changed": 223072,
       "elements": 262144
     },
     {
-      "index": 63,
-      "name": "layer04.gate_proj",
+      "index": 128,
+      "name": "layer09.q_proj",
       "shape": [
         512,
         512
       ],
-      "l2_delta": 15.438628196716309,
-      "rel_l2": 0.05009282190778971,
-      "elements_changed": 123479,
+      "l2_delta": 38.18143844604492,
+      "rel_l2": 0.12095480170877637,
+      "elements_changed": 226568,
       "elements": 262144
     },
     {
-      "index": 21,
-      "name": "layer01.gate_proj",
+      "index": 72,
+      "name": "layer05.q_proj",
       "shape": [
         512,
         512
       ],
-      "l2_delta": 15.602681159973145,
-      "rel_l2": 0.04990909859208276,
-      "elements_changed": 124265,
+      "l2_delta": 36.8487548828125,
+      "rel_l2": 0.12079511529387849,
+      "elements_changed": 223037,
       "elements": 262144
     },
     {
-      "index": 227,
-      "name": "layer16.k_proj",
+      "index": 339,
+      "name": "layer24.k_proj",
       "shape": [
         256,
         512
       ],
-      "l2_delta": 11.47036075592041,
-      "rel_l2": 0.049544209804384086,
-      "elements_changed": 54664,
+      "l2_delta": 30.36958885192871,
+      "rel_l2": 0.12059316886794642,
+      "elements_changed": 118011,
       "elements": 131072
     },
     {
-      "index": 156,
-      "name": "layer11.q_proj",
+      "index": 86,
+      "name": "layer06.q_proj",
       "shape": [
         512,
         512
       ],
-      "l2_delta": 16.31541633605957,
-      "rel_l2": 0.04948391390141436,
-      "elements_changed": 121134,
+      "l2_delta": 37.44362258911133,
+      "rel_l2": 0.11869847131362199,
+      "elements_changed": 226594,
       "elements": 262144
     },
     {
-      "index": 212,
-      "name": "layer15.q_proj",
+      "index": 120,
+      "name": "layer08.out_proj",
       "shape": [
         512,
         512
       ],
-      "l2_delta": 15.739275932312012,
-      "rel_l2": 0.04907852205725618,
-      "elements_changed": 124953,
+      "l2_delta": 41.837158203125,
+      "rel_l2": 0.11757749804154716,
+      "elements_changed": 231750,
       "elements": 262144
     },
     {
-      "index": 16,
-      "name": "layer01.q_proj",
+      "index": 100,
+      "name": "layer07.q_proj",
       "shape": [
         512,
         512
       ],
-      "l2_delta": 15.214017868041992,
-      "rel_l2": 0.04905283487466476,
-      "elements_changed": 125072,
+      "l2_delta": 37.50966262817383,
+      "rel_l2": 0.11754123857201615,
+      "elements_changed": 218997,
       "elements": 262144
     },
     {
-      "index": 106,
-      "name": "layer07.out_proj",
+      "index": 212,
+      "name": "layer15.q_proj",
       "shape": [
         512,
         512
       ],
-      "l2_delta": 16.359912872314453,
-      "rel_l2": 0.048498509847814765,
-      "elements_changed": 99630,
+      "l2_delta": 37.065826416015625,
+      "rel_l2": 0.11557939432234293,
+      "elements_changed": 227390,
       "elements": 262144
     },
     {
-      "index": 198,
-      "name": "layer14.q_proj",
+      "index": 254,
+      "name": "layer18.q_proj",
       "shape": [
         512,
         512
       ],
-      "l2_delta": 15.799996376037598,
-      "rel_l2": 0.04817199354368688,
-      "elements_changed": 107457,
+      "l2_delta": 40.88981628417969,
+      "rel_l2": 0.11466344016464437,
+      "elements_changed": 226287,
       "elements": 262144
     },
     {
-      "index": 4,
-      "name": "layer00.v_proj",
+      "index": 114,
+      "name": "layer08.q_proj",
       "shape": [
-        256,
+        512,
         512
       ],
-      "l2_delta": 12.356016159057617,
-      "rel_l2": 0.04772770366103968,
-      "elements_changed": 52836,
-      "elements": 131072
+      "l2_delta": 36.712154388427734,
+      "rel_l2": 0.11264809490134592,
+      "elements_changed": 223923,
+      "elements": 262144
     },
     {
-      "index": 185,
-      "name": "layer13.k_proj",
+      "index": 73,
+      "name": "layer05.k_proj",
       "shape": [
         256,
         512
       ],
-      "l2_delta": 11.608853340148926,
-      "rel_l2": 0.04761647277460257,
-      "elements_changed": 51466,
+      "l2_delta": 26.100017547607422,
+      "rel_l2": 0.11159049179934297,
+      "elements_changed": 113410,
       "elements": 131072
     },
     {
-      "index": 142,
-      "name": "layer10.q_proj",
+      "index": 199,
+      "name": "layer14.k_proj",
       "shape": [
-        512,
+        256,
         512
       ],
-      "l2_delta": 15.606034278869629,
-      "rel_l2": 0.0473754511348583,
-      "elements_changed": 111895,
-      "elements": 262144
+      "l2_delta": 27.815847396850586,
+      "rel_l2": 0.11142185834614146,
+      "elements_changed": 117156,
+      "elements": 131072
     },
     {
-      "index": 128,
-      "name": "layer09.q_proj",
+      "index": 22,
+      "name": "layer01.out_proj",
       "shape": [
         512,
         512
       ],
-      "l2_delta": 14.942874908447266,
-      "rel_l2": 0.04733746409435007,
-      "elements_changed": 113337,
+      "l2_delta": 35.027244567871094,
+      "rel_l2": 0.11123347654733093,
+      "elements_changed": 229254,
       "elements": 262144
     },
     {
-      "index": 114,
-      "name": "layer08.q_proj",
+      "index": 44,
+      "name": "layer03.q_proj",
       "shape": [
         512,
         512
       ],
-      "l2_delta": 15.396636962890625,
-      "rel_l2": 0.047243259096338544,
-      "elements_changed": 115657,
+      "l2_delta": 36.15559005737305,
+      "rel_l2": 0.11104317308044687,
+      "elements_changed": 225912,
       "elements": 262144
     },
     {
-      "index": 7,
-      "name": "layer00.gate_proj",
+      "index": 269,
+      "name": "layer19.k_proj",
       "shape": [
-        512,
+        256,
         512
       ],
-      "l2_delta": 14.171582221984863,
-      "rel_l2": 0.04639401119112399,
-      "elements_changed": 120002,
-      "elements": 262144
+      "l2_delta": 26.220874786376953,
+      "rel_l2": 0.11076169761300565,
+      "elements_changed": 106884,
+      "elements": 131072
     },
     {
-      "index": 22,
-      "name": "layer01.out_proj",
+      "index": 213,
+      "name": "layer15.k_proj",
       "shape": [
-        512,
+        256,
         512
       ],
-      "l2_delta": 14.496060371398926,
-      "rel_l2": 0.04603408607337959,
-      "elements_changed": 99259,
-      "elements": 262144
+      "l2_delta": 24.490400314331055,
+      "rel_l2": 0.10713777800117577,
+      "elements_changed": 116331,
+      "elements": 131072
     },
     {
-      "index": 36,
-      "name": "layer02.out_proj",
+      "index": 227,
+      "name": "layer16.k_proj",
       "shape": [
-        512,
+        256,
         512
       ],
-      "l2_delta": 14.47131633758545,
-      "rel_l2": 0.04519864297629511,
-      "elements_changed": 99642,
-      "elements": 262144
+      "l2_delta": 24.69317626953125,
+      "rel_l2": 0.10665784031272395,
+      "elements_changed": 115242,
+      "elements": 131072
     },
     {
-      "index": 339,
-      "name": "layer24.k_proj",
+      "index": 185,
+      "name": "layer13.k_proj",
       "shape": [
         256,
         512
       ],
-      "l2_delta": 11.315240859985352,
-      "rel_l2": 0.04493115657451654,
-      "elements_changed": 49381,
+      "l2_delta": 25.714191436767578,
+      "rel_l2": 0.10547287148810292,
+      "elements_changed": 117612,
       "elements": 131072
     },
     {
-      "index": 171,
-      "name": "layer12.k_proj",
+      "index": 45,
+      "name": "layer03.k_proj",
       "shape": [
         256,
         512
       ],
-      "l2_delta": 10.25149917602539,
-      "rel_l2": 0.044672997004544086,
-      "elements_changed": 49840,
+      "l2_delta": 25.14592933654785,
+      "rel_l2": 0.10545754353883269,
+      "elements_changed": 112453,
       "elements": 131072
     },
     {
-      "index": 30,
-      "name": "layer02.q_proj",
+      "index": 255,
+      "name": "layer18.k_proj",
       "shape": [
-        512,
+        256,
         512
       ],
-      "l2_delta": 14.142129898071289,
-      "rel_l2": 0.044275975043145274,
-      "elements_changed": 99378,
-      "elements": 262144
+      "l2_delta": 28.132455825805664,
+      "rel_l2": 0.1054408684510534,
+      "elements_changed": 117860,
+      "elements": 131072
     },
     {
-      "index": 58,
-      "name": "layer04.q_proj",
+      "index": 142,
+      "name": "layer10.q_proj",
       "shape": [
         512,
         512
       ],
-      "l2_delta": 13.479544639587402,
-      "rel_l2": 0.04392790483100051,
-      "elements_changed": 109113,
+      "l2_delta": 33.040523529052734,
+      "rel_l2": 0.10030156796721786,
+      "elements_changed": 206438,
       "elements": 262144
     },
     {
-      "index": 59,
-      "name": "layer04.k_proj",
+      "index": 171,
+      "name": "layer12.k_proj",
       "shape": [
         256,
         512
       ],
-      "l2_delta": 10.233344078063965,
-      "rel_l2": 0.04372801666959533,
-      "elements_changed": 50597,
+      "l2_delta": 22.79279899597168,
+      "rel_l2": 0.0993242669963315,
+      "elements_changed": 114190,
       "elements": 131072
     },
     {
-      "index": 199,
-      "name": "layer14.k_proj",
+      "index": 8,
+      "name": "layer00.out_proj",
       "shape": [
-        256,
+        512,
         512
       ],
-      "l2_delta": 10.908309936523438,
-      "rel_l2": 0.04369538512354424,
-      "elements_changed": 42555,
-      "elements": 131072
+      "l2_delta": 29.878782272338867,
+      "rel_l2": 0.09549170352693005,
+      "elements_changed": 225142,
+      "elements": 262144
     },
     {
-      "index": 8,
-      "name": "layer00.out_proj",
+      "index": 157,
+      "name": "layer11.k_proj",
       "shape": [
-        512,
+        256,
         512
       ],
-      "l2_delta": 13.663572311401367,
-      "rel_l2": 0.043668372572433334,
-      "elements_changed": 102523,
-      "elements": 262144
+      "l2_delta": 22.1978759765625,
+      "rel_l2": 0.0936927124283696,
+      "elements_changed": 109036,
+      "elements": 131072
     },
     {
-      "index": 3,
-      "name": "layer00.k_proj",
+      "index": 101,
+      "name": "layer07.k_proj",
       "shape": [
         256,
         512
       ],
-      "l2_delta": 10.122297286987305,
-      "rel_l2": 0.04310717659653027,
-      "elements_changed": 51021,
+      "l2_delta": 21.17139434814453,
+      "rel_l2": 0.09098053767550045,
+      "elements_changed": 108857,
       "elements": 131072
     },
     {
@@ -1526,59 +1502,83 @@
         256,
         512
       ],
-      "l2_delta": 9.9559965133667,
-      "rel_l2": 0.04191098369921191,
-      "elements_changed": 45160,
+      "l2_delta": 21.236209869384766,
+      "rel_l2": 0.08939641998406618,
+      "elements_changed": 104579,
       "elements": 131072
     },
     {
-      "index": 101,
-      "name": "layer07.k_proj",
+      "index": 129,
+      "name": "layer09.k_proj",
       "shape": [
         256,
         512
       ],
-      "l2_delta": 9.705802917480469,
-      "rel_l2": 0.04170906995940038,
-      "elements_changed": 45337,
+      "l2_delta": 20.95911979675293,
+      "rel_l2": 0.08935737299212503,
+      "elements_changed": 106314,
       "elements": 131072
     },
     {
-      "index": 157,
-      "name": "layer11.k_proj",
+      "index": 16,
+      "name": "layer01.q_proj",
       "shape": [
-        256,
+        512,
         512
       ],
-      "l2_delta": 9.879606246948242,
-      "rel_l2": 0.04169980533174367,
-      "elements_changed": 45916,
-      "elements": 131072
+      "l2_delta": 27.632654190063477,
+      "rel_l2": 0.08909283759822095,
+      "elements_changed": 205732,
+      "elements": 262144
     },
     {
-      "index": 143,
-      "name": "layer10.k_proj",
+      "index": 30,
+      "name": "layer02.q_proj",
+      "shape": [
+        512,
+        512
+      ],
+      "l2_delta": 27.292007446289062,
+      "rel_l2": 0.08544542082971739,
+      "elements_changed": 190989,
+      "elements": 262144
+    },
+    {
+      "index": 3,
+      "name": "layer00.k_proj",
       "shape": [
         256,
         512
       ],
-      "l2_delta": 9.901769638061523,
-      "rel_l2": 0.04155204071178388,
-      "elements_changed": 39818,
+      "l2_delta": 19.55088233947754,
+      "rel_l2": 0.08326008550541993,
+      "elements_changed": 98387,
       "elements": 131072
     },
     {
-      "index": 213,
-      "name": "layer15.k_proj",
+      "index": 17,
+      "name": "layer01.k_proj",
       "shape": [
         256,
         512
       ],
-      "l2_delta": 9.396194458007812,
-      "rel_l2": 0.04110538753867717,
-      "elements_changed": 47113,
+      "l2_delta": 19.00261116027832,
+      "rel_l2": 0.08206246638951739,
+      "elements_changed": 104364,
       "elements": 131072
     },
+    {
+      "index": 58,
+      "name": "layer04.q_proj",
+      "shape": [
+        512,
+        512
+      ],
+      "l2_delta": 25.040836334228516,
+      "rel_l2": 0.08160449813327816,
+      "elements_changed": 199296,
+      "elements": 262144
+    },
     {
       "index": 115,
       "name": "layer08.k_proj",
@@ -1586,45 +1586,45 @@
         256,
         512
       ],
-      "l2_delta": 9.189550399780273,
-      "rel_l2": 0.039187450401305335,
-      "elements_changed": 49167,
+      "l2_delta": 18.997591018676758,
+      "rel_l2": 0.08101235897313101,
+      "elements_changed": 95869,
       "elements": 131072
     },
     {
-      "index": 17,
-      "name": "layer01.k_proj",
+      "index": 59,
+      "name": "layer04.k_proj",
       "shape": [
         256,
         512
       ],
-      "l2_delta": 8.998276710510254,
-      "rel_l2": 0.03885891122496713,
-      "elements_changed": 44275,
+      "l2_delta": 17.163497924804688,
+      "rel_l2": 0.07334119889247537,
+      "elements_changed": 91159,
       "elements": 131072
     },
     {
-      "index": 31,
-      "name": "layer02.k_proj",
+      "index": 143,
+      "name": "layer10.k_proj",
       "shape": [
         256,
         512
       ],
-      "l2_delta": 9.045960426330566,
-      "rel_l2": 0.03871419375314476,
-      "elements_changed": 39429,
+      "l2_delta": 16.693031311035156,
+      "rel_l2": 0.07005106581887803,
+      "elements_changed": 87370,
       "elements": 131072
     },
     {
-      "index": 129,
-      "name": "layer09.k_proj",
+      "index": 31,
+      "name": "layer02.k_proj",
       "shape": [
         256,
         512
       ],
-      "l2_delta": 8.910950660705566,
-      "rel_l2": 0.037991058289883424,
-      "elements_changed": 42265,
+      "l2_delta": 14.048103332519531,
+      "rel_l2": 0.060121973637681875,
+      "elements_changed": 76201,
       "elements": 131072
     },
     {

training/eval/sweep_lr0.0003.json

@@ -41,95 +41,401 @@
     39,
     40,
     41,
-    42
+    42,
+    43,
+    44,
+    45,
+    46,
+    47,
+    48,
+    49,
+    50,
+    51,
+    52,
+    53,
+    54,
+    55,
+    56,
+    57,
+    58,
+    59,
+    60,
+    61,
+    62,
+    63,
+    64,
+    65,
+    66,
+    67,
+    68,
+    69,
+    70,
+    71,
+    72,
+    73,
+    74,
+    75,
+    76,
+    77,
+    78,
+    79,
+    80,
+    81,
+    82,
+    83,
+    84,
+    85,
+    86,
+    87,
+    88,
+    89,
+    90,
+    91,
+    92,
+    93,
+    94,
+    95,
+    96,
+    97,
+    98,
+    99,
+    100,
+    101,
+    102,
+    103,
+    104,
+    105,
+    106,
+    107,
+    108,
+    109,
+    110,
+    111,
+    112,
+    113,
+    114,
+    115,
+    116,
+    117,
+    118,
+    119,
+    120,
+    121,
+    122,
+    123,
+    124,
+    125,
+    126,
+    127,
+    128,
+    129,
+    130,
+    131,
+    132,
+    133,
+    134,
+    135,
+    136,
+    137,
+    138,
+    139,
+    140,
+    141,
+    142,
+    143,
+    144
   ],
   "loss": [
-    2.929511785507202,
-    2.247201919555664,
-    2.8279354572296143,
-    2.9886436462402344,
-    2.9571781158447266,
-    2.5674026012420654,
-    2.777461051940918,
-    3.0373775959014893,
-    2.7279069423675537,
-    3.1416196823120117,
-    2.7582950592041016,
-    2.7006595134735107,
-    3.1009507179260254,
-    2.4599664211273193,
-    2.7058029174804688,
-    2.8157052993774414,
-    3.03840708732605,
-    2.3351850509643555,
-    2.9690005779266357,
-    2.912125587463379,
-    2.730013370513916,
-    2.3012330532073975,
-    3.256481885910034,
-    3.1451406478881836,
-    2.594045639038086,
-    2.398125410079956,
-    2.4386231899261475,
-    2.9154207706451416,
-    2.1565053462982178,
-    2.3790969848632812,
-    2.1627516746520996,
-    2.6049036979675293,
-    2.256344795227051,
-    2.234454393386841,
-    2.4338934421539307,
-    2.2532193660736084,
-    2.2719876766204834,
-    2.345329999923706,
-    2.684453010559082,
-    2.6082634925842285,
-    2.7198550701141357,
-    2.6298987865448
+    2.186704635620117,
+    1.883162260055542,
+    2.748201370239258,
+    2.467050075531006,
+    2.378779172897339,
+    2.976663112640381,
+    2.992650032043457,
+    3.6616547107696533,
+    3.0847907066345215,
+    2.4588916301727295,
+    2.685457468032837,
+    2.4418461322784424,
+    2.183880567550659,
+    2.1833832263946533,
+    3.0518643856048584,
+    2.033839464187622,
+    1.7243285179138184,
+    3.028388261795044,
+    2.9296529293060303,
+    2.2583603858947754,
+    2.6058058738708496,
+    3.5526580810546875,
+    1.8763458728790283,
+    2.475576162338257,
+    2.6972968578338623,
+    3.4062695503234863,
+    2.2567076683044434,
+    1.8545160293579102,
+    1.7336345911026,
+    1.9339534044265747,
+    2.759056568145752,
+    1.820198893547058,
+    2.4605908393859863,
+    1.7973570823669434,
+    1.7877330780029297,
+    2.3408303260803223,
+    2.5842158794403076,
+    2.0886354446411133,
+    1.5313191413879395,
+    1.922637701034546,
+    1.5941201448440552,
+    1.4148709774017334,
+    2.5699918270111084,
+    2.386748790740967,
+    1.112070083618164,
+    1.14950430393219,
+    2.041619062423706,
+    1.9064078330993652,
+    1.1485358476638794,
+    1.5758014917373657,
+    1.591287612915039,
+    1.5382776260375977,
+    1.8633322715759277,
+    1.1190390586853027,
+    1.5539438724517822,
+    1.931726098060608,
+    1.6130378246307373,
+    1.67075514793396,
+    0.8929879069328308,
+    1.8361151218414307,
+    1.204717993736267,
+    1.404712200164795,
+    1.0190882682800293,
+    1.399904727935791,
+    1.192184567451477,
+    0.8895714282989502,
+    1.1295701265335083,
+    1.4481754302978516,
+    0.8810505867004395,
+    1.307167649269104,
+    1.2419353723526,
+    1.0253962278366089,
+    1.3730403184890747,
+    1.022551417350769,
+    1.527650237083435,
+    1.2122101783752441,
+    1.2403998374938965,
+    1.2700071334838867,
+    1.3587489128112793,
+    0.7688336372375488,
+    1.4823236465454102,
+    1.0703743696212769,
+    1.3534530401229858,
+    0.9952245950698853,
+    1.4585095643997192,
+    0.9907617568969727,
+    1.15060555934906,
+    0.7762649059295654,
+    1.177674651145935,
+    0.9260612726211548,
+    0.8299102783203125,
+    0.8821147680282593,
+    0.8932853937149048,
+    0.8035864233970642,
+    1.0306886434555054,
+    0.6965221166610718,
+    1.0181723833084106,
+    0.654222846031189,
+    0.8802157044410706,
+    0.8032260537147522,
+    1.0697497129440308,
+    1.5103988647460938,
+    1.0312896966934204,
+    1.1030751466751099,
+    0.7566086053848267,
+    1.2659751176834106,
+    0.6971560716629028,
+    0.9575833678245544,
+    0.96876460313797,
+    1.3060075044631958,
+    1.3291523456573486,
+    1.2842150926589966,
+    0.9220280647277832,
+    0.8584076166152954,
+    0.7654063105583191,
+    0.6525526642799377,
+    0.9988993406295776,
+    1.082180380821228,
+    1.0873360633850098,
+    0.7878360748291016,
+    1.1192176342010498,
+    0.7204289436340332,
+    1.4367657899856567,
+    1.0086344480514526,
+    0.703212320804596,
+    0.7965445518493652,
+    0.9995136857032776,
+    1.0909080505371094,
+    0.7841006517410278,
+    0.8603678941726685,
+    0.8610115051269531,
+    0.8927115797996521,
+    0.6957674026489258,
+    0.7486045360565186,
+    0.7375907301902771,
+    0.7676178216934204,
+    1.003730058670044,
+    0.9070913195610046,
+    0.9035248160362244,
+    0.6334217190742493,
+    0.8439401388168335,
+    1.0656341314315796,
+    1.3582684993743896,
+    0.44588127732276917
   ],
   "grad_norm": [
-    0.016382873058319092,
-    0.01044374331831932,
-    0.014581313356757164,
-    0.019003715366125107,
-    0.02694617584347725,
-    0.02906985394656658,
-    0.03875892236828804,
-    0.051806092262268066,
-    0.06310596317052841,
-    0.07135758548974991,
-    0.10482443869113922,
-    0.10832356661558151,
-    0.15019597113132477,
-    0.13573578000068665,
-    0.12627138197422028,
-    0.15686063468456268,
-    0.21380013227462769,
-    0.09234306216239929,
-    0.12907597422599792,
-    0.148917555809021,
-    0.12556204199790955,
-    0.08786641061306,
-    0.1341717541217804,
-    0.1436973512172699,
-    0.10974177718162537,
-    0.10140179842710495,
-    0.11574269086122513,
-    0.1426159143447876,
-    0.09566441923379898,
-    0.10903562605381012,
-    0.10174928605556488,
-    0.1311177909374237,
-    0.11360231041908264,
-    0.1114194393157959,
-    0.13377518951892853,
-    0.12109410017728806,
-    0.1319703459739685,
-    0.126362144947052,
-    0.15615588426589966,
-    0.15437155961990356,
-    0.16725720465183258,
-    0.16275674104690552
+    0.007697182707488537,
+    0.006591176148504019,
+    0.009374779649078846,
+    0.00879797525703907,
+    0.00858077872544527,
+    0.013686838559806347,
+    0.013488121330738068,
+    0.028339505195617676,
+    0.02803395316004753,
+    0.016524825245141983,
+    0.021972868591547012,
+    0.03226516395807266,
+    0.025491131469607353,
+    0.030927162617444992,
+    0.06004085764288902,
+    0.048798877745866776,
+    0.0372982956469059,
+    0.07970226556062698,
+    0.08211911469697952,
+    0.06958767771720886,
+    0.08860500156879425,
+    0.12372057139873505,
+    0.06893937289714813,
+    0.10168690234422684,
+    0.12031954526901245,
+    0.16483980417251587,
+    0.10167452692985535,
+    0.08443624526262283,
+    0.09491752833127975,
+    0.10911590605974197,
+    0.14407660067081451,
+    0.12247786670923233,
+    0.16354046761989594,
+    0.13414815068244934,
+    0.1312817484140396,
+    0.17796407639980316,
+    0.20327840745449066,
+    0.17143777012825012,
+    0.15170809626579285,
+    0.19115859270095825,
+    0.1492016762495041,
+    0.1395121067762375,
+    0.23521490395069122,
+    0.21002896130084991,
+    0.11470843851566315,
+    0.11491400003433228,
+    0.15364284813404083,
+    0.13125018775463104,
+    0.10506079345941544,
+    0.11934610456228256,
+    0.1245480477809906,
+    0.11354843527078629,
+    0.1361245959997177,
+    0.089741550385952,
+    0.11588369309902191,
+    0.1479126363992691,
+    0.12335845082998276,
+    0.14498183131217957,
+    0.07999479025602341,
+    0.14764580130577087,
+    0.09854836761951447,
+    0.11640656739473343,
+    0.08927378058433533,
+    0.11217103153467178,
+    0.09801290929317474,
+    0.08770257234573364,
+    0.10023589432239532,
+    0.1262200027704239,
+    0.089995376765728,
+    0.1187262162566185,
+    0.11250575631856918,
+    0.09785343706607819,
+    0.12236326932907104,
+    0.10739341378211975,
+    0.14591622352600098,
+    0.12155532836914062,
+    0.10962728410959244,
+    0.12190016359090805,
+    0.12980800867080688,
+    0.08024413138628006,
+    0.15067395567893982,
+    0.11113952100276947,
+    0.14148364961147308,
+    0.10851052403450012,
+    0.16500495374202728,
+    0.11044300347566605,
+    0.12667730450630188,
+    0.09310226142406464,
+    0.1707594096660614,
+    0.11097094416618347,
+    0.10086149722337723,
+    0.10504462569952011,
+    0.10616394877433777,
+    0.09521491825580597,
+    0.11324740201234818,
+    0.10025249421596527,
+    0.11523750424385071,
+    0.09159111231565475,
+    0.10806699097156525,
+    0.11230102926492691,
+    0.13171078264713287,
+    0.19787003099918365,
+    0.13186782598495483,
+    0.13199909031391144,
+    0.10982373356819153,
+    0.15635566413402557,
+    0.09592770785093307,
+    0.13705973327159882,
+    0.1327674835920334,
+    0.18176007270812988,
+    0.1943148821592331,
+    0.17516621947288513,
+    0.12861448526382446,
+    0.12530305981636047,
+    0.10954950004816055,
+    0.1122756376862526,
+    0.14607051014900208,
+    0.1565093845129013,
+    0.14510004222393036,
+    0.11518478393554688,
+    0.17200490832328796,
+    0.11564990878105164,
+    0.20338574051856995,
+    0.14326445758342743,
+    0.11406676471233368,
+    0.12304076552391052,
+    0.14822599291801453,
+    0.1604507863521576,
+    0.11749894171953201,
+    0.14413923025131226,
+    0.12644687294960022,
+    0.12683425843715668,
+    0.10894184559583664,
+    0.12332484126091003,
+    0.11594563722610474,
+    0.13957937061786652,
+    0.1577451527118683,
+    0.13786892592906952,
+    0.14430369436740875,
+    0.10614383220672607,
+    0.13644066452980042,
+    0.18304558098316193,
+    0.2012057900428772,
+    0.12607848644256592
   ],
   "epoch": [
     1,
@@ -153,40 +459,140 @@
     1,
     1,
     1,
-    2,
-    2,
-    2,
-    2,
-    2,
-    2,
-    2,
-    2,
-    2,
-    2,
-    2,
-    2,
-    2,
-    2,
-    2,
-    2,
-    2,
-    2,
-    2,
-    2,
-    2
+    1,
+    1,
+    1,
+    1,
+    1,
+    1,
+    1,
+    1,
+    1,
+    1,
+    1,
+    1,
+    1,
+    1,
+    1,
+    1,
+    1,
+    1,
+    1,
+    1,
+    1,
+    1,
+    1,
+    1,
+    1,
+    1,
+    1,
+    1,
+    1,
+    1,
+    1,
+    1,
+    1,
+    1,
+    1,
+    1,
+    1,
+    1,
+    1,
+    1,
+    1,
+    1,
+    1,
+    1,
+    1,
+    1,
+    1,
+    1,
+    1,
+    1,
+    1,
+    1,
+    1,
+    1,
+    1,
+    1,
+    1,
+    1,
+    1,
+    1,
+    1,
+    1,
+    1,
+    1,
+    1,
+    1,
+    1,
+    1,
+    1,
+    1,
+    1,
+    1,
+    1,
+    1,
+    1,
+    1,
+    1,
+    1,
+    1,
+    1,
+    1,
+    1,
+    1,
+    1,
+    1,
+    1,
+    1,
+    1,
+    1,
+    1,
+    1,
+    1,
+    1,
+    1,
+    1,
+    1,
+    1,
+    1,
+    1,
+    1,
+    1,
+    1,
+    1,
+    1,
+    1,
+    1,
+    1,
+    1,
+    1,
+    1,
+    1,
+    1,
+    1,
+    1,
+    1,
+    1,
+    1,
+    1,
+    1,
+    1,
+    1,
+    1,
+    1
   ],
   "eval_step": [
-    21,
-    42
+    144
   ],
   "eval_loss": [
-    2.585228681564331,
-    2.4067068099975586
+    0.789882481098175
   ],
   "skipped_nonfinite": 0,
   "config": {
     "lr": 0.0003,
-    "epochs": 2,
+    "epochs": 1,
     "batch_size": 8,
     "lora_rank": 16,
     "lora_alpha": 32,
@@ -196,7 +602,7 @@
     "seed": 3407,
     "warmup_frac": 0.1
   },
-  "wall_seconds": 40.1,
-  "final_loss": 2.6298987865448,
-  "first_loss": 2.929511785507202
+  "wall_seconds": 238.0,
+  "final_loss": 0.44588127732276917,
+  "first_loss": 2.186704635620117
 }

training/eval/sweep_lr0.001.json

@@ -0,0 +1,608 @@
+{
+  "step": [
+    1,
+    2,
+    3,
+    4,
+    5,
+    6,
+    7,
+    8,
+    9,
+    10,
+    11,
+    12,
+    13,
+    14,
+    15,
+    16,
+    17,
+    18,
+    19,
+    20,
+    21,
+    22,
+    23,
+    24,
+    25,
+    26,
+    27,
+    28,
+    29,
+    30,
+    31,
+    32,
+    33,
+    34,
+    35,
+    36,
+    37,
+    38,
+    39,
+    40,
+    41,
+    42,
+    43,
+    44,
+    45,
+    46,
+    47,
+    48,
+    49,
+    50,
+    51,
+    52,
+    53,
+    54,
+    55,
+    56,
+    57,
+    58,
+    59,
+    60,
+    61,
+    62,
+    63,
+    64,
+    65,
+    66,
+    67,
+    68,
+    69,
+    70,
+    71,
+    72,
+    73,
+    74,
+    75,
+    76,
+    77,
+    78,
+    79,
+    80,
+    81,
+    82,
+    83,
+    84,
+    85,
+    86,
+    87,
+    88,
+    89,
+    90,
+    91,
+    92,
+    93,
+    94,
+    95,
+    96,
+    97,
+    98,
+    99,
+    100,
+    101,
+    102,
+    103,
+    104,
+    105,
+    106,
+    107,
+    108,
+    109,
+    110,
+    111,
+    112,
+    113,
+    114,
+    115,
+    116,
+    117,
+    118,
+    119,
+    120,
+    121,
+    122,
+    123,
+    124,
+    125,
+    126,
+    127,
+    128,
+    129,
+    130,
+    131,
+    132,
+    133,
+    134,
+    135,
+    136,
+    137,
+    138,
+    139,
+    140,
+    141,
+    142,
+    143,
+    144
+  ],
+  "loss": [
+    2.186704635620117,
+    1.883162260055542,
+    2.74814510345459,
+    2.466845989227295,
+    2.3788509368896484,
+    2.975734233856201,
+    2.9905762672424316,
+    3.654111385345459,
+    3.0711445808410645,
+    2.4414923191070557,
+    2.6454920768737793,
+    2.364337921142578,
+    2.0904364585876465,
+    2.0475728511810303,
+    2.7865400314331055,
+    1.7993525266647339,
+    1.4746829271316528,
+    2.5417492389678955,
+    2.321533441543579,
+    1.7003793716430664,
+    1.888978123664856,
+    2.5126349925994873,
+    1.2252076864242554,
+    1.5984466075897217,
+    1.6967116594314575,
+    2.1224136352539062,
+    1.3066543340682983,
+    1.1011003255844116,
+    0.9361790418624878,
+    1.0468217134475708,
+    1.5412594079971313,
+    0.9191675186157227,
+    1.2560166120529175,
+    0.8764936327934265,
+    0.8498842120170593,
+    1.0467239618301392,
+    1.2317255735397339,
+    0.9136865139007568,
+    0.6303170323371887,
+    0.7960976958274841,
+    0.6028057932853699,
+    0.5116087198257446,
+    1.028873085975647,
+    0.9757534265518188,
+    0.3805548846721649,
+    0.38887566328048706,
+    0.6246833801269531,
+    0.5998170971870422,
+    0.2699912190437317,
+    0.35834720730781555,
+    0.34884750843048096,
+    0.2955508232116699,
+    0.33022114634513855,
+    0.17820875346660614,
+    0.2592273950576782,
+    0.4250122010707855,
+    0.25173136591911316,
+    0.2432330995798111,
+    0.11214160919189453,
+    0.20630334317684174,
+    0.16084854304790497,
+    0.17087283730506897,
+    0.08054812997579575,
+    0.16278403997421265,
+    0.1395648568868637,
+    0.11121319234371185,
+    0.13235296308994293,
+    0.19356468319892883,
+    0.033520184457302094,
+    0.1071089506149292,
+    0.09159337729215622,
+    0.06580451875925064,
+    0.14314305782318115,
+    0.05499166250228882,
+    0.15369239449501038,
+    0.06662755459547043,
+    0.10243431478738785,
+    0.13850469887256622,
+    0.14262966811656952,
+    0.05796191096305847,
+    0.12378686666488647,
+    0.07255993783473969,
+    0.1127157211303711,
+    0.05874176323413849,
+    0.12458953261375427,
+    0.04156043007969856,
+    0.054499849677085876,
+    0.05232254043221474,
+    0.0723092183470726,
+    0.0509815588593483,
+    0.05762720853090286,
+    0.04377233237028122,
+    0.06369898468255997,
+    0.048773881047964096,
+    0.06131862476468086,
+    0.0160438884049654,
+    0.08374661207199097,
+    0.033671338111162186,
+    0.02858029119670391,
+    0.03577163815498352,
+    0.091847263276577,
+    0.13571976125240326,
+    0.07209137082099915,
+    0.09624846279621124,
+    0.038384925574064255,
+    0.05720242112874985,
+    0.05769316852092743,
+    0.07771953195333481,
+    0.04193703457713127,
+    0.09051437675952911,
+    0.08769053220748901,
+    0.08581390231847763,
+    0.09552276879549026,
+    0.036670003086328506,
+    0.05020442605018616,
+    0.018213853240013123,
+    0.04456958919763565,
+    0.06055033206939697,
+    0.10573932528495789,
+    0.0470181480050087,
+    0.09163397550582886,
+    0.02687552571296692,
+    0.07665979117155075,
+    0.049437541514635086,
+    0.05302508547902107,
+    0.06494462490081787,
+    0.07063387334346771,
+    0.08954703062772751,
+    0.02619367651641369,
+    0.046229876577854156,
+    0.05161109194159508,
+    0.03903370723128319,
+    0.045858919620513916,
+    0.0331801138818264,
+    0.08467432111501694,
+    0.03939277306199074,
+    0.04427666217088699,
+    0.033651553094387054,
+    0.09268827736377716,
+    0.01843259669840336,
+    0.06518390029668808,
+    0.06839370727539062,
+    0.08438707143068314,
+    0.0013384957564994693
+  ],
+  "grad_norm": [
+    0.007697182707488537,
+    0.006591176148504019,
+    0.009401866234838963,
+    0.009703319519758224,
+    0.011120510287582874,
+    0.021190090104937553,
+    0.024285579100251198,
+    0.06156115606427193,
+    0.06622587889432907,
+    0.041893120855093,
+    0.060033902525901794,
+    0.09933964163064957,
+    0.07785724103450775,
+    0.08121656626462936,
+    0.11960174888372421,
+    0.10720126330852509,
+    0.11621396988630295,
+    0.18052898347377777,
+    0.1914040595293045,
+    0.15056905150413513,
+    0.18656770884990692,
+    0.18351945281028748,
+    0.10182812064886093,
+    0.12158484011888504,
+    0.14395610988140106,
+    0.17389456927776337,
+    0.1148117259144783,
+    0.1054484024643898,
+    0.09071443974971771,
+    0.10577467083930969,
+    0.15791375935077667,
+    0.11229434609413147,
+    0.14645855128765106,
+    0.11103765666484833,
+    0.11816119402647018,
+    0.15448927879333496,
+    0.18822108209133148,
+    0.14451083540916443,
+    0.11646274477243423,
+    0.16750699281692505,
+    0.11721494048833847,
+    0.10051786154508591,
+    0.1804969608783722,
+    0.22421590983867645,
+    0.10155627876520157,
+    0.10060790926218033,
+    0.1647266149520874,
+    0.1522713303565979,
+    0.09706219285726547,
+    0.13451211154460907,
+    0.1339847594499588,
+    0.1166047602891922,
+    0.12492746114730835,
+    0.10209975391626358,
+    0.10415752977132797,
+    0.1561277061700821,
+    0.12280881404876709,
+    0.1371864229440689,
+    0.0648735761642456,
+    0.13667267560958862,
+    0.09626387804746628,
+    0.11199527978897095,
+    0.05472167208790779,
+    0.09406712651252747,
+    0.06885150074958801,
+    0.09396475553512573,
+    0.09136505424976349,
+    0.08790481835603714,
+    0.047599270939826965,
+    0.07677532732486725,
+    0.06523379683494568,
+    0.08178972452878952,
+    0.09766194969415665,
+    0.06418506801128387,
+    0.22500546276569366,
+    0.06222788617014885,
+    0.07834617793560028,
+    0.08243827521800995,
+    0.1330053061246872,
+    0.047072622925043106,
+    0.13396283984184265,
+    0.10260838270187378,
+    0.08647335320711136,
+    0.06956706196069717,
+    0.14067068696022034,
+    0.0706518217921257,
+    0.06656435877084732,
+    0.055245932191610336,
+    0.18999306857585907,
+    0.07734040915966034,
+    0.07759352028369904,
+    0.06273593008518219,
+    0.059440549463033676,
+    0.07261243462562561,
+    0.06288521736860275,
+    0.03500049188733101,
+    0.09681231528520584,
+    0.047959692776203156,
+    0.0441308356821537,
+    0.07678006589412689,
+    0.1159924566745758,
+    0.13018791377544403,
+    0.08089562505483627,
+    0.08631563186645508,
+    0.04344863072037697,
+    0.09920323640108109,
+    0.06217660754919052,
+    0.09193172305822372,
+    0.07680899649858475,
+    0.13016067445278168,
+    0.15259546041488647,
+    0.10346519201993942,
+    0.10286461561918259,
+    0.06691890209913254,
+    0.07234055548906326,
+    0.030533060431480408,
+    0.0731254294514656,
+    0.10405300557613373,
+    0.10680389404296875,
+    0.07811491936445236,
+    0.13516882061958313,
+    0.0701913982629776,
+    0.13999493420124054,
+    0.10206928849220276,
+    0.07351543754339218,
+    0.10105010122060776,
+    0.1009650006890297,
+    0.14674152433872223,
+    0.04721233621239662,
+    0.1040613204240799,
+    0.08361753821372986,
+    0.06652839481830597,
+    0.06213260069489479,
+    0.047044869512319565,
+    0.1662881076335907,
+    0.09059280902147293,
+    0.06861348450183868,
+    0.07727435231208801,
+    0.15771834552288055,
+    0.03932024911046028,
+    0.12314016371965408,
+    0.0896899625658989,
+    0.16253338754177094,
+    0.004918188787996769
+  ],
+  "epoch": [
+    1,
+    1,
+    1,
+    1,
+    1,
+    1,
+    1,
+    1,
+    1,
+    1,
+    1,
+    1,
+    1,
+    1,
+    1,
+    1,
+    1,
+    1,
+    1,
+    1,
+    1,
+    1,
+    1,
+    1,
+    1,
+    1,
+    1,
+    1,
+    1,
+    1,
+    1,
+    1,
+    1,
+    1,
+    1,
+    1,
+    1,
+    1,
+    1,
+    1,
+    1,
+    1,
+    1,
+    1,
+    1,
+    1,
+    1,
+    1,
+    1,
+    1,
+    1,
+    1,
+    1,
+    1,
+    1,
+    1,
+    1,
+    1,
+    1,
+    1,
+    1,
+    1,
+    1,
+    1,
+    1,
+    1,
+    1,
+    1,
+    1,
+    1,
+    1,
+    1,
+    1,
+    1,
+    1,
+    1,
+    1,
+    1,
+    1,
+    1,
+    1,
+    1,
+    1,
+    1,
+    1,
+    1,
+    1,
+    1,
+    1,
+    1,
+    1,
+    1,
+    1,
+    1,
+    1,
+    1,
+    1,
+    1,
+    1,
+    1,
+    1,
+    1,
+    1,
+    1,
+    1,
+    1,
+    1,
+    1,
+    1,
+    1,
+    1,
+    1,
+    1,
+    1,
+    1,
+    1,
+    1,
+    1,
+    1,
+    1,
+    1,
+    1,
+    1,
+    1,
+    1,
+    1,
+    1,
+    1,
+    1,
+    1,
+    1,
+    1,
+    1,
+    1,
+    1,
+    1,
+    1,
+    1,
+    1,
+    1,
+    1,
+    1,
+    1,
+    1
+  ],
+  "eval_step": [
+    144
+  ],
+  "eval_loss": [
+    0.04626748710870743
+  ],
+  "skipped_nonfinite": 0,
+  "config": {
+    "lr": 0.001,
+    "epochs": 1,
+    "batch_size": 8,
+    "lora_rank": 16,
+    "lora_alpha": 32,
+    "max_len": 512,
+    "grad_clip": 1.0,
+    "weight_decay": 0.0,
+    "seed": 3407,
+    "warmup_frac": 0.1
+  },
+  "wall_seconds": 88.1,
+  "final_loss": 0.0013384957564994693,
+  "first_loss": 2.186704635620117
+}