{ "method": "randomized SVD of canonical mixed-Q8 minus decoded original ternary weights", "seed": 1337, "svd_q": 40, "svd_power_iterations": 4, "ranks": [ 8, 16, 32 ], "modules": 126, "source_revision": "3f926b415992eaa2ae9dd7b573706494d6bbf787", "reference_revision": "61231c9a2feed6457f47d1d50a691b8dc96b2613", "gguf_adapter_input_basis": "canonical, cyclic GDN columns", "mlx_adapter_input_basis": "canonical, grouped GDN columns", "seconds": 10.651134014129639, "rows": [ { "module": "model.layers.63.mlp.down_proj", "shape": [ 5120, 17408 ], "delta_relative_norm": 0.023493614047765732, "ranks": { "8": { "residual_relative_to_target": 0.006203466095030308, "delta_energy_captured": 0.9302781224250793, "probe_relative_error": 0.00619496637955308 }, "16": { "residual_relative_to_target": 0.006184915546327829, "delta_energy_captured": 0.9306944608688354, "probe_relative_error": 0.006179067771881819 }, "32": { "residual_relative_to_target": 0.0061549535021185875, "delta_energy_captured": 0.9313643574714661, "probe_relative_error": 0.0061479369178414345 } } }, { "module": "model.layers.63.self_attn.o_proj", "shape": [ 5120, 6144 ], "delta_relative_norm": 0.025529770180583, "ranks": { "8": { "residual_relative_to_target": 0.006384628359228373, "delta_energy_captured": 0.9374572038650513, "probe_relative_error": 0.006409766618162394 }, "16": { "residual_relative_to_target": 0.006359532941132784, "delta_energy_captured": 0.9379478693008423, "probe_relative_error": 0.0063870251178741455 }, "32": { "residual_relative_to_target": 0.006322126369923353, "delta_energy_captured": 0.9386757016181946, "probe_relative_error": 0.006350215524435043 } } }, { "module": "model.layers.62.mlp.down_proj", "shape": [ 5120, 17408 ], "delta_relative_norm": 0.02229161560535431, "ranks": { "8": { "residual_relative_to_target": 0.006088209804147482, "delta_energy_captured": 0.925407350063324, "probe_relative_error": 0.006114326883107424 }, "16": { "residual_relative_to_target": 0.006071844138205051, "delta_energy_captured": 0.9258078336715698, "probe_relative_error": 0.0061011542566120625 }, "32": { "residual_relative_to_target": 0.006048388779163361, "delta_energy_captured": 0.926379919052124, "probe_relative_error": 0.006077746860682964 } } }, { "module": "model.layers.62.linear_attn.out_proj", "shape": [ 5120, 6144 ], "delta_relative_norm": 0.022418532520532608, "ranks": { "8": { "residual_relative_to_target": 0.0061709885485470295, "delta_energy_captured": 0.9242303967475891, "probe_relative_error": 0.006205723155289888 }, "16": { "residual_relative_to_target": 0.006148629356175661, "delta_energy_captured": 0.9247784614562988, "probe_relative_error": 0.0061827413737773895 }, "32": { "residual_relative_to_target": 0.0061127119697630405, "delta_energy_captured": 0.9256547093391418, "probe_relative_error": 0.006148936226963997 } } }, { "module": "model.layers.61.mlp.down_proj", "shape": [ 5120, 17408 ], "delta_relative_norm": 0.02129940316081047, "ranks": { "8": { "residual_relative_to_target": 0.006037276703864336, "delta_energy_captured": 0.9196571707725525, "probe_relative_error": 0.006025936454534531 }, "16": { "residual_relative_to_target": 0.00602235272526741, "delta_energy_captured": 0.9200538396835327, "probe_relative_error": 0.006011681165546179 }, "32": { "residual_relative_to_target": 0.006001652218401432, "delta_energy_captured": 0.9206025004386902, "probe_relative_error": 0.005993095226585865 } } }, { "module": "model.layers.61.linear_attn.out_proj", "shape": [ 5120, 6144 ], "delta_relative_norm": 0.02114199846982956, "ranks": { "8": { "residual_relative_to_target": 0.006076216232031584, "delta_energy_captured": 0.9174010753631592, "probe_relative_error": 0.006095276679843664 }, "16": { "residual_relative_to_target": 0.0060568880289793015, "delta_energy_captured": 0.9179257154464722, "probe_relative_error": 0.006078938953578472 }, "32": { "residual_relative_to_target": 0.006024396046996117, "delta_energy_captured": 0.9188039302825928, "probe_relative_error": 0.006048615090548992 } } }, { "module": "model.layers.60.mlp.down_proj", "shape": [ 5120, 17408 ], "delta_relative_norm": 0.020835502073168755, "ranks": { "8": { "residual_relative_to_target": 0.0060300398617982864, "delta_energy_captured": 0.9162408113479614, "probe_relative_error": 0.0060409135185182095 }, "16": { "residual_relative_to_target": 0.006015256978571415, "delta_energy_captured": 0.9166510105133057, "probe_relative_error": 0.006025105714797974 }, "32": { "residual_relative_to_target": 0.00599451782181859, "delta_energy_captured": 0.9172247648239136, "probe_relative_error": 0.006004229187965393 } } }, { "module": "model.layers.60.linear_attn.out_proj", "shape": [ 5120, 6144 ], "delta_relative_norm": 0.021040836349129677, "ranks": { "8": { "residual_relative_to_target": 0.006096669472754002, "delta_energy_captured": 0.9160425662994385, "probe_relative_error": 0.006109232548624277 }, "16": { "residual_relative_to_target": 0.006076797377318144, "delta_energy_captured": 0.9165889620780945, "probe_relative_error": 0.006089756730943918 }, "32": { "residual_relative_to_target": 0.006043015979230404, "delta_energy_captured": 0.9175137281417847, "probe_relative_error": 0.006061606574803591 } } }, { "module": "model.layers.59.mlp.down_proj", "shape": [ 5120, 17408 ], "delta_relative_norm": 0.020492179319262505, "ranks": { "8": { "residual_relative_to_target": 0.006026110146194696, "delta_energy_captured": 0.9135235548019409, "probe_relative_error": 0.006006068084388971 }, "16": { "residual_relative_to_target": 0.006011185701936483, "delta_energy_captured": 0.9139513373374939, "probe_relative_error": 0.005989533849060535 }, "32": { "residual_relative_to_target": 0.005990565754473209, "delta_energy_captured": 0.9145406484603882, "probe_relative_error": 0.005969911813735962 } } }, { "module": "model.layers.59.self_attn.o_proj", "shape": [ 5120, 6144 ], "delta_relative_norm": 0.020459972321987152, "ranks": { "8": { "residual_relative_to_target": 0.006094166077673435, "delta_energy_captured": 0.911280632019043, "probe_relative_error": 0.006112969480454922 }, "16": { "residual_relative_to_target": 0.006076589692384005, "delta_energy_captured": 0.9117916822433472, "probe_relative_error": 0.006095355376601219 }, "32": { "residual_relative_to_target": 0.006045016460120678, "delta_energy_captured": 0.9127058982849121, "probe_relative_error": 0.006063488312065601 } } }, { "module": "model.layers.58.mlp.down_proj", "shape": [ 5120, 17408 ], "delta_relative_norm": 0.020321514457464218, "ranks": { "8": { "residual_relative_to_target": 0.006021411623805761, "delta_energy_captured": 0.912202000617981, "probe_relative_error": 0.005999132990837097 }, "16": { "residual_relative_to_target": 0.006006334908306599, "delta_energy_captured": 0.912641167640686, "probe_relative_error": 0.005983144510537386 }, "32": { "residual_relative_to_target": 0.005985502153635025, "delta_energy_captured": 0.9132460951805115, "probe_relative_error": 0.005962659139186144 } } }, { "module": "model.layers.58.linear_attn.out_proj", "shape": [ 5120, 6144 ], "delta_relative_norm": 0.020883062854409218, "ranks": { "8": { "residual_relative_to_target": 0.006070531439036131, "delta_energy_captured": 0.9154983758926392, "probe_relative_error": 0.006043680943548679 }, "16": { "residual_relative_to_target": 0.00605217507109046, "delta_energy_captured": 0.9160086512565613, "probe_relative_error": 0.006030766759067774 }, "32": { "residual_relative_to_target": 0.006020673550665379, "delta_energy_captured": 0.91688072681427, "probe_relative_error": 0.006002210546284914 } } }, { "module": "model.layers.57.mlp.down_proj", "shape": [ 5120, 17408 ], "delta_relative_norm": 0.020097721368074417, "ranks": { "8": { "residual_relative_to_target": 0.006018276326358318, "delta_energy_captured": 0.910329282283783, "probe_relative_error": 0.006053621880710125 }, "16": { "residual_relative_to_target": 0.00600338215008378, "delta_energy_captured": 0.9107725620269775, "probe_relative_error": 0.00603972002863884 }, "32": { "residual_relative_to_target": 0.0059826611541211605, "delta_energy_captured": 0.9113874435424805, "probe_relative_error": 0.006016792729496956 } } }, { "module": "model.layers.57.linear_attn.out_proj", "shape": [ 5120, 6144 ], "delta_relative_norm": 0.020595118403434753, "ranks": { "8": { "residual_relative_to_target": 0.006130394525825977, "delta_energy_captured": 0.9113970398902893, "probe_relative_error": 0.006177142262458801 }, "16": { "residual_relative_to_target": 0.006112022791057825, "delta_energy_captured": 0.9119272828102112, "probe_relative_error": 0.006158243864774704 }, "32": { "residual_relative_to_target": 0.006079487036913633, "delta_energy_captured": 0.9128624796867371, "probe_relative_error": 0.006125116255134344 } } }, { "module": "model.layers.56.mlp.down_proj", "shape": [ 5120, 17408 ], "delta_relative_norm": 0.020840104669332504, "ranks": { "8": { "residual_relative_to_target": 0.006023420952260494, "delta_energy_captured": 0.9164614677429199, "probe_relative_error": 0.006023530848324299 }, "16": { "residual_relative_to_target": 0.006008591037243605, "delta_energy_captured": 0.9168723225593567, "probe_relative_error": 0.006012072321027517 }, "32": { "residual_relative_to_target": 0.005988001357764006, "delta_energy_captured": 0.9174410700798035, "probe_relative_error": 0.0059908549301326275 } } }, { "module": "model.layers.56.linear_attn.out_proj", "shape": [ 5120, 6144 ], "delta_relative_norm": 0.021443670615553856, "ranks": { "8": { "residual_relative_to_target": 0.006094329059123993, "delta_energy_captured": 0.9192293286323547, "probe_relative_error": 0.006112502422183752 }, "16": { "residual_relative_to_target": 0.006074394099414349, "delta_energy_captured": 0.9197568893432617, "probe_relative_error": 0.006086415611207485 }, "32": { "residual_relative_to_target": 0.006041317712515593, "delta_energy_captured": 0.9206284284591675, "probe_relative_error": 0.006052073091268539 } } }, { "module": "model.layers.55.mlp.down_proj", "shape": [ 5120, 17408 ], "delta_relative_norm": 0.020723003894090652, "ranks": { "8": { "residual_relative_to_target": 0.006033357698470354, "delta_energy_captured": 0.9152357578277588, "probe_relative_error": 0.0060050650499761105 }, "16": { "residual_relative_to_target": 0.006018562242388725, "delta_energy_captured": 0.9156509637832642, "probe_relative_error": 0.005991325713694096 }, "32": { "residual_relative_to_target": 0.00599807919934392, "delta_energy_captured": 0.9162241220474243, "probe_relative_error": 0.005970488302409649 } } }, { "module": "model.layers.55.self_attn.o_proj", "shape": [ 5120, 6144 ], "delta_relative_norm": 0.02028011530637741, "ranks": { "8": { "residual_relative_to_target": 0.006079335231333971, "delta_energy_captured": 0.9101389646530151, "probe_relative_error": 0.0060809580609202385 }, "16": { "residual_relative_to_target": 0.006061325315386057, "delta_energy_captured": 0.9106706380844116, "probe_relative_error": 0.0060623278841376305 }, "32": { "residual_relative_to_target": 0.006028881296515465, "delta_energy_captured": 0.9116243720054626, "probe_relative_error": 0.006032067351043224 } } }, { "module": "model.layers.54.mlp.down_proj", "shape": [ 5120, 17408 ], "delta_relative_norm": 0.021479183807969093, "ranks": { "8": { "residual_relative_to_target": 0.00607015285640955, "delta_energy_captured": 0.920133650302887, "probe_relative_error": 0.0060868519358336926 }, "16": { "residual_relative_to_target": 0.006055823992937803, "delta_energy_captured": 0.9205102920532227, "probe_relative_error": 0.006073798518627882 }, "32": { "residual_relative_to_target": 0.00603545643389225, "delta_energy_captured": 0.9210440516471863, "probe_relative_error": 0.0060530672781169415 } } }, { "module": "model.layers.54.linear_attn.out_proj", "shape": [ 5120, 6144 ], "delta_relative_norm": 0.022341154515743256, "ranks": { "8": { "residual_relative_to_target": 0.0060975272208452225, "delta_energy_captured": 0.9255102872848511, "probe_relative_error": 0.006094307638704777 }, "16": { "residual_relative_to_target": 0.006079318933188915, "delta_energy_captured": 0.9259545207023621, "probe_relative_error": 0.006073355209082365 }, "32": { "residual_relative_to_target": 0.006047883071005344, "delta_energy_captured": 0.9267182946205139, "probe_relative_error": 0.006038699299097061 } } }, { "module": "model.layers.53.mlp.down_proj", "shape": [ 5120, 17408 ], "delta_relative_norm": 0.021105965599417686, "ranks": { "8": { "residual_relative_to_target": 0.006058567203581333, "delta_energy_captured": 0.917599618434906, "probe_relative_error": 0.006060657557100058 }, "16": { "residual_relative_to_target": 0.006044069305062294, "delta_energy_captured": 0.9179934859275818, "probe_relative_error": 0.006045852787792683 }, "32": { "residual_relative_to_target": 0.006023644004017115, "delta_energy_captured": 0.9185467958450317, "probe_relative_error": 0.006026324350386858 } } }, { "module": "model.layers.53.linear_attn.out_proj", "shape": [ 5120, 6144 ], "delta_relative_norm": 0.02132597006857395, "ranks": { "8": { "residual_relative_to_target": 0.006103483960032463, "delta_energy_captured": 0.9180898070335388, "probe_relative_error": 0.006080629304051399 }, "16": { "residual_relative_to_target": 0.006084129214286804, "delta_energy_captured": 0.9186084270477295, "probe_relative_error": 0.0060598887503147125 }, "32": { "residual_relative_to_target": 0.00605209544301033, "delta_energy_captured": 0.9194632768630981, "probe_relative_error": 0.006029986310750246 } } }, { "module": "model.layers.52.mlp.down_proj", "shape": [ 5120, 17408 ], "delta_relative_norm": 0.02121596783399582, "ranks": { "8": { "residual_relative_to_target": 0.006060957908630371, "delta_energy_captured": 0.9183874726295471, "probe_relative_error": 0.006080988794565201 }, "16": { "residual_relative_to_target": 0.00604634452611208, "delta_energy_captured": 0.9187805652618408, "probe_relative_error": 0.006065780762583017 }, "32": { "residual_relative_to_target": 0.006025712005794048, "delta_energy_captured": 0.9193339347839355, "probe_relative_error": 0.006043375004082918 } } }, { "module": "model.layers.52.linear_attn.out_proj", "shape": [ 5120, 6144 ], "delta_relative_norm": 0.02138073556125164, "ranks": { "8": { "residual_relative_to_target": 0.006117842625826597, "delta_energy_captured": 0.9181250333786011, "probe_relative_error": 0.006146178115159273 }, "16": { "residual_relative_to_target": 0.006097991019487381, "delta_energy_captured": 0.9186554551124573, "probe_relative_error": 0.006124954670667648 }, "32": { "residual_relative_to_target": 0.006065126042813063, "delta_energy_captured": 0.919529914855957, "probe_relative_error": 0.006093480624258518 } } }, { "module": "model.layers.51.mlp.down_proj", "shape": [ 5120, 17408 ], "delta_relative_norm": 0.020931726321578026, "ranks": { "8": { "residual_relative_to_target": 0.006090490147471428, "delta_energy_captured": 0.9153368473052979, "probe_relative_error": 0.006093429401516914 }, "16": { "residual_relative_to_target": 0.00607492309063673, "delta_energy_captured": 0.915769100189209, "probe_relative_error": 0.006077541504055262 }, "32": { "residual_relative_to_target": 0.006054316647350788, "delta_energy_captured": 0.9163395166397095, "probe_relative_error": 0.006055254489183426 } } }, { "module": "model.layers.51.self_attn.o_proj", "shape": [ 5120, 6144 ], "delta_relative_norm": 0.021571796387434006, "ranks": { "8": { "residual_relative_to_target": 0.006139351055026054, "delta_energy_captured": 0.91900235414505, "probe_relative_error": 0.006092521827667952 }, "16": { "residual_relative_to_target": 0.006121189333498478, "delta_energy_captured": 0.9194808602333069, "probe_relative_error": 0.006075660698115826 }, "32": { "residual_relative_to_target": 0.0060895271599292755, "delta_energy_captured": 0.920311689376831, "probe_relative_error": 0.006044131703674793 } } }, { "module": "model.layers.50.mlp.down_proj", "shape": [ 5120, 17408 ], "delta_relative_norm": 0.021893208846449852, "ranks": { "8": { "residual_relative_to_target": 0.006137816235423088, "delta_energy_captured": 0.9214024543762207, "probe_relative_error": 0.0061620427295565605 }, "16": { "residual_relative_to_target": 0.006123154424130917, "delta_energy_captured": 0.9217774868011475, "probe_relative_error": 0.006145530845969915 }, "32": { "residual_relative_to_target": 0.006102458573877811, "delta_energy_captured": 0.9223054051399231, "probe_relative_error": 0.006125468760728836 } } }, { "module": "model.layers.50.linear_attn.out_proj", "shape": [ 5120, 6144 ], "delta_relative_norm": 0.024059979245066643, "ranks": { "8": { "residual_relative_to_target": 0.006143790669739246, "delta_energy_captured": 0.9347947835922241, "probe_relative_error": 0.006125996354967356 }, "16": { "residual_relative_to_target": 0.006122578866779804, "delta_energy_captured": 0.9352442622184753, "probe_relative_error": 0.006099299527704716 }, "32": { "residual_relative_to_target": 0.006086510140448809, "delta_energy_captured": 0.9360049962997437, "probe_relative_error": 0.006064813118427992 } } }, { "module": "model.layers.49.mlp.down_proj", "shape": [ 5120, 17408 ], "delta_relative_norm": 0.022697126492857933, "ranks": { "8": { "residual_relative_to_target": 0.0060973260551691055, "delta_energy_captured": 0.9278332591056824, "probe_relative_error": 0.006105566862970591 }, "16": { "residual_relative_to_target": 0.006082397419959307, "delta_energy_captured": 0.9281861782073975, "probe_relative_error": 0.006091735325753689 }, "32": { "residual_relative_to_target": 0.006061552558094263, "delta_energy_captured": 0.9286775588989258, "probe_relative_error": 0.006069166120141745 } } }, { "module": "model.layers.49.linear_attn.out_proj", "shape": [ 5120, 6144 ], "delta_relative_norm": 0.02313533052802086, "ranks": { "8": { "residual_relative_to_target": 0.006113432347774506, "delta_energy_captured": 0.9301737546920776, "probe_relative_error": 0.006142680533230305 }, "16": { "residual_relative_to_target": 0.006094715557992458, "delta_energy_captured": 0.930600643157959, "probe_relative_error": 0.006125420331954956 }, "32": { "residual_relative_to_target": 0.006062833592295647, "delta_energy_captured": 0.9313247799873352, "probe_relative_error": 0.0060928696766495705 } } }, { "module": "model.layers.48.mlp.down_proj", "shape": [ 5120, 17408 ], "delta_relative_norm": 0.02249142900109291, "ranks": { "8": { "residual_relative_to_target": 0.006094043608754873, "delta_energy_captured": 0.9265863299369812, "probe_relative_error": 0.0061579071916639805 }, "16": { "residual_relative_to_target": 0.006078573875129223, "delta_energy_captured": 0.9269585609436035, "probe_relative_error": 0.0061442130245268345 }, "32": { "residual_relative_to_target": 0.006057319696992636, "delta_energy_captured": 0.9274684190750122, "probe_relative_error": 0.0061202202923595905 } } }, { "module": "model.layers.48.linear_attn.out_proj", "shape": [ 5120, 6144 ], "delta_relative_norm": 0.024167608469724655, "ranks": { "8": { "residual_relative_to_target": 0.006104980129748583, "delta_energy_captured": 0.9361881613731384, "probe_relative_error": 0.006097967736423016 }, "16": { "residual_relative_to_target": 0.006086121313273907, "delta_energy_captured": 0.9365817904472351, "probe_relative_error": 0.006079885642975569 }, "32": { "residual_relative_to_target": 0.00605367636308074, "delta_energy_captured": 0.9372561573982239, "probe_relative_error": 0.006046242546290159 } } }, { "module": "model.layers.47.mlp.down_proj", "shape": [ 5120, 17408 ], "delta_relative_norm": 0.023318033665418625, "ranks": { "8": { "residual_relative_to_target": 0.006081343162804842, "delta_energy_captured": 0.9319833517074585, "probe_relative_error": 0.006071115843951702 }, "16": { "residual_relative_to_target": 0.006066549103707075, "delta_energy_captured": 0.9323139190673828, "probe_relative_error": 0.006056347396224737 }, "32": { "residual_relative_to_target": 0.006044955458492041, "delta_energy_captured": 0.9327948689460754, "probe_relative_error": 0.0060320631600916386 } } }, { "module": "model.layers.47.self_attn.o_proj", "shape": [ 5120, 6144 ], "delta_relative_norm": 0.023469405248761177, "ranks": { "8": { "residual_relative_to_target": 0.006134840659797192, "delta_energy_captured": 0.9316714406013489, "probe_relative_error": 0.006136295851320028 }, "16": { "residual_relative_to_target": 0.006115900352597237, "delta_energy_captured": 0.9320926666259766, "probe_relative_error": 0.0061173150315880775 }, "32": { "residual_relative_to_target": 0.00608331011608243, "delta_energy_captured": 0.9328144788742065, "probe_relative_error": 0.006087667308747768 } } }, { "module": "model.layers.46.mlp.down_proj", "shape": [ 5120, 17408 ], "delta_relative_norm": 0.02287241257727146, "ranks": { "8": { "residual_relative_to_target": 0.006067721173167229, "delta_energy_captured": 0.929623544216156, "probe_relative_error": 0.006093467120081186 }, "16": { "residual_relative_to_target": 0.00605345843359828, "delta_energy_captured": 0.9299539923667908, "probe_relative_error": 0.0060799019411206245 }, "32": { "residual_relative_to_target": 0.0060324957594275475, "delta_energy_captured": 0.9304382801055908, "probe_relative_error": 0.006062073167413473 } } }, { "module": "model.layers.46.linear_attn.out_proj", "shape": [ 5120, 6144 ], "delta_relative_norm": 0.022646324709057808, "ranks": { "8": { "residual_relative_to_target": 0.006105736363679171, "delta_energy_captured": 0.9273090362548828, "probe_relative_error": 0.006053128745406866 }, "16": { "residual_relative_to_target": 0.006086830981075764, "delta_energy_captured": 0.9277584552764893, "probe_relative_error": 0.006032545119524002 }, "32": { "residual_relative_to_target": 0.0060533578507602215, "delta_energy_captured": 0.9285508394241333, "probe_relative_error": 0.00600312789902091 } } }, { "module": "model.layers.45.mlp.down_proj", "shape": [ 5120, 17408 ], "delta_relative_norm": 0.023377450183033943, "ranks": { "8": { "residual_relative_to_target": 0.006068831775337458, "delta_energy_captured": 0.9326068162918091, "probe_relative_error": 0.006056040525436401 }, "16": { "residual_relative_to_target": 0.006054762285202742, "delta_energy_captured": 0.932918906211853, "probe_relative_error": 0.006046064663678408 }, "32": { "residual_relative_to_target": 0.006033135578036308, "delta_energy_captured": 0.9333972930908203, "probe_relative_error": 0.006024109665304422 } } }, { "module": "model.layers.45.linear_attn.out_proj", "shape": [ 5120, 6144 ], "delta_relative_norm": 0.023702578619122505, "ranks": { "8": { "residual_relative_to_target": 0.006082818377763033, "delta_energy_captured": 0.9341405034065247, "probe_relative_error": 0.006086487788707018 }, "16": { "residual_relative_to_target": 0.006064688786864281, "delta_energy_captured": 0.9345324635505676, "probe_relative_error": 0.006067697424441576 }, "32": { "residual_relative_to_target": 0.006032311357557774, "delta_energy_captured": 0.9352296590805054, "probe_relative_error": 0.006036863196641207 } } }, { "module": "model.layers.44.mlp.down_proj", "shape": [ 5120, 17408 ], "delta_relative_norm": 0.023573627695441246, "ranks": { "8": { "residual_relative_to_target": 0.006071622483432293, "delta_energy_captured": 0.9336628913879395, "probe_relative_error": 0.006076062098145485 }, "16": { "residual_relative_to_target": 0.00605763541534543, "delta_energy_captured": 0.933968186378479, "probe_relative_error": 0.006062559317797422 }, "32": { "residual_relative_to_target": 0.006036465056240559, "delta_energy_captured": 0.934428870677948, "probe_relative_error": 0.006040880456566811 } } }, { "module": "model.layers.44.linear_attn.out_proj", "shape": [ 5120, 6144 ], "delta_relative_norm": 0.02365094982087612, "ranks": { "8": { "residual_relative_to_target": 0.006102156825363636, "delta_energy_captured": 0.9334313869476318, "probe_relative_error": 0.006059129256755114 }, "16": { "residual_relative_to_target": 0.006084059830754995, "delta_energy_captured": 0.9338256120681763, "probe_relative_error": 0.006042174994945526 }, "32": { "residual_relative_to_target": 0.006051745265722275, "delta_energy_captured": 0.9345267415046692, "probe_relative_error": 0.006016638595610857 } } }, { "module": "model.layers.43.mlp.down_proj", "shape": [ 5120, 17408 ], "delta_relative_norm": 0.022710220888257027, "ranks": { "8": { "residual_relative_to_target": 0.006059695966541767, "delta_energy_captured": 0.9288034439086914, "probe_relative_error": 0.006042527500540018 }, "16": { "residual_relative_to_target": 0.006045423448085785, "delta_energy_captured": 0.9291384220123291, "probe_relative_error": 0.00602593831717968 }, "32": { "residual_relative_to_target": 0.006024426780641079, "delta_energy_captured": 0.9296298027038574, "probe_relative_error": 0.006004304625093937 } } }, { "module": "model.layers.43.self_attn.o_proj", "shape": [ 5120, 6144 ], "delta_relative_norm": 0.024399735033512115, "ranks": { "8": { "residual_relative_to_target": 0.0061387596651911736, "delta_energy_captured": 0.9367018342018127, "probe_relative_error": 0.006134283263236284 }, "16": { "residual_relative_to_target": 0.006119549740105867, "delta_energy_captured": 0.9370973706245422, "probe_relative_error": 0.006112574599683285 }, "32": { "residual_relative_to_target": 0.006086132489144802, "delta_energy_captured": 0.9377824664115906, "probe_relative_error": 0.006079253740608692 } } }, { "module": "model.layers.42.mlp.down_proj", "shape": [ 5120, 17408 ], "delta_relative_norm": 0.022289995104074478, "ranks": { "8": { "residual_relative_to_target": 0.006064626853913069, "delta_energy_captured": 0.9259733557701111, "probe_relative_error": 0.006071004550904036 }, "16": { "residual_relative_to_target": 0.006050636991858482, "delta_energy_captured": 0.9263144731521606, "probe_relative_error": 0.006055327598005533 }, "32": { "residual_relative_to_target": 0.0060300338082015514, "delta_energy_captured": 0.9268154501914978, "probe_relative_error": 0.006037381011992693 } } }, { "module": "model.layers.42.linear_attn.out_proj", "shape": [ 5120, 6144 ], "delta_relative_norm": 0.02273198962211609, "ranks": { "8": { "residual_relative_to_target": 0.006101517006754875, "delta_energy_captured": 0.9279555082321167, "probe_relative_error": 0.006098469253629446 }, "16": { "residual_relative_to_target": 0.006080562248826027, "delta_energy_captured": 0.9284495115280151, "probe_relative_error": 0.006075063720345497 }, "32": { "residual_relative_to_target": 0.006045809481292963, "delta_energy_captured": 0.9292650818824768, "probe_relative_error": 0.006045033689588308 } } }, { "module": "model.layers.41.mlp.down_proj", "shape": [ 5120, 17408 ], "delta_relative_norm": 0.021448547020554543, "ranks": { "8": { "residual_relative_to_target": 0.006042253226041794, "delta_energy_captured": 0.9206399321556091, "probe_relative_error": 0.006040771026164293 }, "16": { "residual_relative_to_target": 0.006028073374181986, "delta_energy_captured": 0.9210119843482971, "probe_relative_error": 0.0060270256362855434 }, "32": { "residual_relative_to_target": 0.00600754888728261, "delta_energy_captured": 0.9215489625930786, "probe_relative_error": 0.006009232718497515 } } }, { "module": "model.layers.41.linear_attn.out_proj", "shape": [ 5120, 6144 ], "delta_relative_norm": 0.021758820861577988, "ranks": { "8": { "residual_relative_to_target": 0.006061953026801348, "delta_energy_captured": 0.9223834276199341, "probe_relative_error": 0.006062312051653862 }, "16": { "residual_relative_to_target": 0.006044324953109026, "delta_energy_captured": 0.9228341579437256, "probe_relative_error": 0.006046661175787449 }, "32": { "residual_relative_to_target": 0.006013399921357632, "delta_energy_captured": 0.9236217737197876, "probe_relative_error": 0.006019572261720896 } } }, { "module": "model.layers.40.mlp.down_proj", "shape": [ 5120, 17408 ], "delta_relative_norm": 0.020844709128141403, "ranks": { "8": { "residual_relative_to_target": 0.006030261982232332, "delta_energy_captured": 0.9163086414337158, "probe_relative_error": 0.006053948774933815 }, "16": { "residual_relative_to_target": 0.006015774793922901, "delta_energy_captured": 0.9167102575302124, "probe_relative_error": 0.006038118619471788 }, "32": { "residual_relative_to_target": 0.005995276849716902, "delta_energy_captured": 0.917276918888092, "probe_relative_error": 0.006015749182552099 } } }, { "module": "model.layers.40.linear_attn.out_proj", "shape": [ 5120, 6144 ], "delta_relative_norm": 0.020884139463305473, "ranks": { "8": { "residual_relative_to_target": 0.006066871806979179, "delta_energy_captured": 0.9156089425086975, "probe_relative_error": 0.006084297318011522 }, "16": { "residual_relative_to_target": 0.006048972252756357, "delta_energy_captured": 0.9161061644554138, "probe_relative_error": 0.00606860127300024 }, "32": { "residual_relative_to_target": 0.006017731502652168, "delta_energy_captured": 0.9169704914093018, "probe_relative_error": 0.006036028731614351 } } }, { "module": "model.layers.39.mlp.down_proj", "shape": [ 5120, 17408 ], "delta_relative_norm": 0.020970268175005913, "ranks": { "8": { "residual_relative_to_target": 0.0060237399302423, "delta_energy_captured": 0.9174866080284119, "probe_relative_error": 0.006015862803906202 }, "16": { "residual_relative_to_target": 0.006009472534060478, "delta_energy_captured": 0.9178770184516907, "probe_relative_error": 0.006003722548484802 }, "32": { "residual_relative_to_target": 0.005988978315144777, "delta_energy_captured": 0.9184361696243286, "probe_relative_error": 0.005982957314699888 } } }, { "module": "model.layers.39.self_attn.o_proj", "shape": [ 5120, 6144 ], "delta_relative_norm": 0.02168138138949871, "ranks": { "8": { "residual_relative_to_target": 0.006081074010580778, "delta_energy_captured": 0.9213340878486633, "probe_relative_error": 0.006080102175474167 }, "16": { "residual_relative_to_target": 0.006062882021069527, "delta_energy_captured": 0.9218040704727173, "probe_relative_error": 0.00605847267434001 }, "32": { "residual_relative_to_target": 0.006031112279742956, "delta_energy_captured": 0.9226213693618774, "probe_relative_error": 0.006028905510902405 } } }, { "module": "model.layers.38.mlp.down_proj", "shape": [ 5120, 17408 ], "delta_relative_norm": 0.020847443491220474, "ranks": { "8": { "residual_relative_to_target": 0.006040703505277634, "delta_energy_captured": 0.9160405397415161, "probe_relative_error": 0.006071120500564575 }, "16": { "residual_relative_to_target": 0.006026363465934992, "delta_energy_captured": 0.9164386987686157, "probe_relative_error": 0.006058972328901291 }, "32": { "residual_relative_to_target": 0.006005906034260988, "delta_energy_captured": 0.9170050621032715, "probe_relative_error": 0.006041419226676226 } } }, { "module": "model.layers.38.linear_attn.out_proj", "shape": [ 5120, 6144 ], "delta_relative_norm": 0.021166682243347168, "ranks": { "8": { "residual_relative_to_target": 0.006069451104849577, "delta_energy_captured": 0.9177770018577576, "probe_relative_error": 0.006059565115720034 }, "16": { "residual_relative_to_target": 0.006051459349691868, "delta_energy_captured": 0.9182637333869934, "probe_relative_error": 0.006041659973561764 }, "32": { "residual_relative_to_target": 0.006020289845764637, "delta_energy_captured": 0.9191035628318787, "probe_relative_error": 0.00600909860804677 } } }, { "module": "model.layers.37.mlp.down_proj", "shape": [ 5120, 17408 ], "delta_relative_norm": 0.021743137389421463, "ranks": { "8": { "residual_relative_to_target": 0.006035120692104101, "delta_energy_captured": 0.9229580163955688, "probe_relative_error": 0.006035762373358011 }, "16": { "residual_relative_to_target": 0.006020473316311836, "delta_energy_captured": 0.9233314990997314, "probe_relative_error": 0.006022346206009388 }, "32": { "residual_relative_to_target": 0.00599979842081666, "delta_energy_captured": 0.9238572120666504, "probe_relative_error": 0.0060012806206941605 } } }, { "module": "model.layers.37.linear_attn.out_proj", "shape": [ 5120, 6144 ], "delta_relative_norm": 0.02178158611059189, "ranks": { "8": { "residual_relative_to_target": 0.006120794452726841, "delta_energy_captured": 0.9210346341133118, "probe_relative_error": 0.006112252827733755 }, "16": { "residual_relative_to_target": 0.006102846935391426, "delta_energy_captured": 0.9214970469474792, "probe_relative_error": 0.006095332559198141 }, "32": { "residual_relative_to_target": 0.006070833187550306, "delta_energy_captured": 0.9223185181617737, "probe_relative_error": 0.006068515125662088 } } }, { "module": "model.layers.36.mlp.down_proj", "shape": [ 5120, 17408 ], "delta_relative_norm": 0.021916626021265984, "ranks": { "8": { "residual_relative_to_target": 0.006045699119567871, "delta_energy_captured": 0.9239068031311035, "probe_relative_error": 0.006041909102350473 }, "16": { "residual_relative_to_target": 0.006031061057001352, "delta_energy_captured": 0.9242748618125916, "probe_relative_error": 0.006029164418578148 }, "32": { "residual_relative_to_target": 0.006010203622281551, "delta_energy_captured": 0.9247977137565613, "probe_relative_error": 0.006008479744195938 } } }, { "module": "model.layers.36.linear_attn.out_proj", "shape": [ 5120, 6144 ], "delta_relative_norm": 0.022344063967466354, "ranks": { "8": { "residual_relative_to_target": 0.0060781389474868774, "delta_energy_captured": 0.9260025024414062, "probe_relative_error": 0.006062383763492107 }, "16": { "residual_relative_to_target": 0.006060427986085415, "delta_energy_captured": 0.9264331459999084, "probe_relative_error": 0.006043384317308664 }, "32": { "residual_relative_to_target": 0.006029218435287476, "delta_energy_captured": 0.9271888732910156, "probe_relative_error": 0.0060142274014651775 } } }, { "module": "model.layers.35.mlp.down_proj", "shape": [ 5120, 17408 ], "delta_relative_norm": 0.022243203595280647, "ranks": { "8": { "residual_relative_to_target": 0.006073181051760912, "delta_energy_captured": 0.9254516959190369, "probe_relative_error": 0.006040290463715792 }, "16": { "residual_relative_to_target": 0.006058766972273588, "delta_energy_captured": 0.9258051514625549, "probe_relative_error": 0.006026617716997862 }, "32": { "residual_relative_to_target": 0.0060376450419425964, "delta_energy_captured": 0.9263215661048889, "probe_relative_error": 0.006009394768625498 } } }, { "module": "model.layers.35.self_attn.o_proj", "shape": [ 5120, 6144 ], "delta_relative_norm": 0.023746883496642113, "ranks": { "8": { "residual_relative_to_target": 0.006093916483223438, "delta_energy_captured": 0.9341463446617126, "probe_relative_error": 0.0060934810899198055 }, "16": { "residual_relative_to_target": 0.006076579913496971, "delta_energy_captured": 0.9345205426216125, "probe_relative_error": 0.006073717959225178 }, "32": { "residual_relative_to_target": 0.006044890731573105, "delta_energy_captured": 0.9352017045021057, "probe_relative_error": 0.006036730948835611 } } }, { "module": "model.layers.34.mlp.down_proj", "shape": [ 5120, 17408 ], "delta_relative_norm": 0.022315077483654022, "ranks": { "8": { "residual_relative_to_target": 0.006146028637886047, "delta_energy_captured": 0.9241435527801514, "probe_relative_error": 0.006127975415438414 }, "16": { "residual_relative_to_target": 0.006132046692073345, "delta_energy_captured": 0.9244883060455322, "probe_relative_error": 0.006115989293903112 }, "32": { "residual_relative_to_target": 0.0061110700480639935, "delta_energy_captured": 0.9250040650367737, "probe_relative_error": 0.006096362601965666 } } }, { "module": "model.layers.34.linear_attn.out_proj", "shape": [ 5120, 6144 ], "delta_relative_norm": 0.02435901388525963, "ranks": { "8": { "residual_relative_to_target": 0.006124407984316349, "delta_energy_captured": 0.9367866516113281, "probe_relative_error": 0.00614601094275713 }, "16": { "residual_relative_to_target": 0.006103230640292168, "delta_energy_captured": 0.9372230768203735, "probe_relative_error": 0.00612616166472435 }, "32": { "residual_relative_to_target": 0.006068452727049589, "delta_energy_captured": 0.9379364848136902, "probe_relative_error": 0.006096133030951023 } } }, { "module": "model.layers.33.mlp.down_proj", "shape": [ 5120, 17408 ], "delta_relative_norm": 0.022657053545117378, "ranks": { "8": { "residual_relative_to_target": 0.006092540454119444, "delta_energy_captured": 0.9276914000511169, "probe_relative_error": 0.00605214387178421 }, "16": { "residual_relative_to_target": 0.0060781617648899555, "delta_energy_captured": 0.9280322790145874, "probe_relative_error": 0.0060340771451592445 }, "32": { "residual_relative_to_target": 0.006056721787899733, "delta_energy_captured": 0.9285390973091125, "probe_relative_error": 0.006012708880007267 } } }, { "module": "model.layers.33.linear_attn.out_proj", "shape": [ 5120, 6144 ], "delta_relative_norm": 0.025525521486997604, "ranks": { "8": { "residual_relative_to_target": 0.006090549286454916, "delta_energy_captured": 0.9430670738220215, "probe_relative_error": 0.0061125364154577255 }, "16": { "residual_relative_to_target": 0.006072657648473978, "delta_energy_captured": 0.943401038646698, "probe_relative_error": 0.0060964045114815235 }, "32": { "residual_relative_to_target": 0.006040295120328665, "delta_energy_captured": 0.9440026879310608, "probe_relative_error": 0.006064408924430609 } } }, { "module": "model.layers.32.mlp.down_proj", "shape": [ 5120, 17408 ], "delta_relative_norm": 0.022199278697371483, "ranks": { "8": { "residual_relative_to_target": 0.006080122664570808, "delta_energy_captured": 0.9249851703643799, "probe_relative_error": 0.006090234033763409 }, "16": { "residual_relative_to_target": 0.00606575608253479, "delta_energy_captured": 0.9253392815589905, "probe_relative_error": 0.006073952652513981 }, "32": { "residual_relative_to_target": 0.006044203415513039, "delta_energy_captured": 0.9258689284324646, "probe_relative_error": 0.006049816962331533 } } }, { "module": "model.layers.32.linear_attn.out_proj", "shape": [ 5120, 6144 ], "delta_relative_norm": 0.023357966914772987, "ranks": { "8": { "residual_relative_to_target": 0.006065764930099249, "delta_energy_captured": 0.9325625896453857, "probe_relative_error": 0.0060863494873046875 }, "16": { "residual_relative_to_target": 0.00604749983176589, "delta_energy_captured": 0.9329680800437927, "probe_relative_error": 0.006069990806281567 }, "32": { "residual_relative_to_target": 0.006014317274093628, "delta_energy_captured": 0.9337016940116882, "probe_relative_error": 0.006035847123712301 } } }, { "module": "model.layers.31.mlp.down_proj", "shape": [ 5120, 17408 ], "delta_relative_norm": 0.02377847209572792, "ranks": { "8": { "residual_relative_to_target": 0.006074395962059498, "delta_energy_captured": 0.9347413182258606, "probe_relative_error": 0.006080507766455412 }, "16": { "residual_relative_to_target": 0.006060002837330103, "delta_energy_captured": 0.9350502490997314, "probe_relative_error": 0.006071282085031271 }, "32": { "residual_relative_to_target": 0.0060380371287465096, "delta_energy_captured": 0.9355202317237854, "probe_relative_error": 0.006048315204679966 } } }, { "module": "model.layers.31.self_attn.o_proj", "shape": [ 5120, 6144 ], "delta_relative_norm": 0.023346781730651855, "ranks": { "8": { "residual_relative_to_target": 0.006091511342674494, "delta_energy_captured": 0.9319236874580383, "probe_relative_error": 0.0060823941603302956 }, "16": { "residual_relative_to_target": 0.006072774063795805, "delta_energy_captured": 0.9323418140411377, "probe_relative_error": 0.006064068991690874 }, "32": { "residual_relative_to_target": 0.00603942945599556, "delta_energy_captured": 0.9330828189849854, "probe_relative_error": 0.006024840287864208 } } }, { "module": "model.layers.30.mlp.down_proj", "shape": [ 5120, 17408 ], "delta_relative_norm": 0.02418408915400505, "ranks": { "8": { "residual_relative_to_target": 0.006072863470762968, "delta_energy_captured": 0.9369438290596008, "probe_relative_error": 0.006057096179574728 }, "16": { "residual_relative_to_target": 0.006058792117983103, "delta_energy_captured": 0.9372357130050659, "probe_relative_error": 0.006040608510375023 }, "32": { "residual_relative_to_target": 0.006037280894815922, "delta_energy_captured": 0.9376806020736694, "probe_relative_error": 0.006020132452249527 } } }, { "module": "model.layers.30.linear_attn.out_proj", "shape": [ 5120, 6144 ], "delta_relative_norm": 0.023017892614006996, "ranks": { "8": { "residual_relative_to_target": 0.006092648953199387, "delta_energy_captured": 0.9299381971359253, "probe_relative_error": 0.006104538217186928 }, "16": { "residual_relative_to_target": 0.006072753109037876, "delta_energy_captured": 0.9303950667381287, "probe_relative_error": 0.006088663823902607 }, "32": { "residual_relative_to_target": 0.006038023624569178, "delta_energy_captured": 0.9311888813972473, "probe_relative_error": 0.006058224011212587 } } }, { "module": "model.layers.29.mlp.down_proj", "shape": [ 5120, 17408 ], "delta_relative_norm": 0.024167638272047043, "ranks": { "8": { "residual_relative_to_target": 0.0060697998851537704, "delta_energy_captured": 0.9369216561317444, "probe_relative_error": 0.006075077224522829 }, "16": { "residual_relative_to_target": 0.006055579520761967, "delta_energy_captured": 0.9372168779373169, "probe_relative_error": 0.006063246633857489 }, "32": { "residual_relative_to_target": 0.006033545825630426, "delta_energy_captured": 0.9376729130744934, "probe_relative_error": 0.006041319575160742 } } }, { "module": "model.layers.29.linear_attn.out_proj", "shape": [ 5120, 6144 ], "delta_relative_norm": 0.024435250088572502, "ranks": { "8": { "residual_relative_to_target": 0.006068781483918428, "delta_energy_captured": 0.9383164644241333, "probe_relative_error": 0.0060287355445325375 }, "16": { "residual_relative_to_target": 0.006050553172826767, "delta_energy_captured": 0.9386864304542542, "probe_relative_error": 0.006010016892105341 }, "32": { "residual_relative_to_target": 0.006017674691975117, "delta_energy_captured": 0.939350962638855, "probe_relative_error": 0.005975026171654463 } } }, { "module": "model.layers.28.mlp.down_proj", "shape": [ 5120, 17408 ], "delta_relative_norm": 0.024680564180016518, "ranks": { "8": { "residual_relative_to_target": 0.006069253198802471, "delta_energy_captured": 0.939527153968811, "probe_relative_error": 0.006070159375667572 }, "16": { "residual_relative_to_target": 0.006055272184312344, "delta_energy_captured": 0.9398054480552673, "probe_relative_error": 0.006056688725948334 }, "32": { "residual_relative_to_target": 0.0060339197516441345, "delta_energy_captured": 0.9402292370796204, "probe_relative_error": 0.006036709062755108 } } }, { "module": "model.layers.28.linear_attn.out_proj", "shape": [ 5120, 6144 ], "delta_relative_norm": 0.024254897609353065, "ranks": { "8": { "residual_relative_to_target": 0.006076928228139877, "delta_energy_captured": 0.9372274875640869, "probe_relative_error": 0.006075568031519651 }, "16": { "residual_relative_to_target": 0.006058897357434034, "delta_energy_captured": 0.9375994801521301, "probe_relative_error": 0.006057745311409235 }, "32": { "residual_relative_to_target": 0.006026200018823147, "delta_energy_captured": 0.938271164894104, "probe_relative_error": 0.006025613751262426 } } }, { "module": "model.layers.27.mlp.down_proj", "shape": [ 5120, 17408 ], "delta_relative_norm": 0.024300511926412582, "ranks": { "8": { "residual_relative_to_target": 0.0060642254538834095, "delta_energy_captured": 0.9377241134643555, "probe_relative_error": 0.006078042555600405 }, "16": { "residual_relative_to_target": 0.00604989705607295, "delta_energy_captured": 0.9380180835723877, "probe_relative_error": 0.006064364220947027 }, "32": { "residual_relative_to_target": 0.006028610281646252, "delta_energy_captured": 0.9384534955024719, "probe_relative_error": 0.006044549401849508 } } }, { "module": "model.layers.27.self_attn.o_proj", "shape": [ 5120, 6144 ], "delta_relative_norm": 0.024306191131472588, "ranks": { "8": { "residual_relative_to_target": 0.006086335983127356, "delta_energy_captured": 0.9372984766960144, "probe_relative_error": 0.006093572359532118 }, "16": { "residual_relative_to_target": 0.006067522335797548, "delta_energy_captured": 0.9376855492591858, "probe_relative_error": 0.006069639697670937 }, "32": { "residual_relative_to_target": 0.0060343570075929165, "delta_energy_captured": 0.9383648633956909, "probe_relative_error": 0.006034309044480324 } } }, { "module": "model.layers.26.mlp.down_proj", "shape": [ 5120, 17408 ], "delta_relative_norm": 0.024375613778829575, "ranks": { "8": { "residual_relative_to_target": 0.006068732123821974, "delta_energy_captured": 0.9380152821540833, "probe_relative_error": 0.0060697924345731735 }, "16": { "residual_relative_to_target": 0.006054791621863842, "delta_energy_captured": 0.9382997155189514, "probe_relative_error": 0.006054124794900417 }, "32": { "residual_relative_to_target": 0.006034021731466055, "delta_energy_captured": 0.9387222528457642, "probe_relative_error": 0.0060351537540555 } } }, { "module": "model.layers.26.linear_attn.out_proj", "shape": [ 5120, 6144 ], "delta_relative_norm": 0.024777619168162346, "ranks": { "8": { "residual_relative_to_target": 0.006096011959016323, "delta_energy_captured": 0.9394697546958923, "probe_relative_error": 0.006109443958848715 }, "16": { "residual_relative_to_target": 0.006076372694224119, "delta_energy_captured": 0.93985915184021, "probe_relative_error": 0.006087914574891329 }, "32": { "residual_relative_to_target": 0.006042290944606066, "delta_energy_captured": 0.9405319094657898, "probe_relative_error": 0.006055893842130899 } } }, { "module": "model.layers.25.mlp.down_proj", "shape": [ 5120, 17408 ], "delta_relative_norm": 0.024253619834780693, "ranks": { "8": { "residual_relative_to_target": 0.006057672202587128, "delta_energy_captured": 0.9376181364059448, "probe_relative_error": 0.006024549249559641 }, "16": { "residual_relative_to_target": 0.006043728906661272, "delta_energy_captured": 0.937904953956604, "probe_relative_error": 0.00601200619712472 }, "32": { "residual_relative_to_target": 0.006023138761520386, "delta_energy_captured": 0.9383273720741272, "probe_relative_error": 0.005992107558995485 } } }, { "module": "model.layers.25.linear_attn.out_proj", "shape": [ 5120, 6144 ], "delta_relative_norm": 0.024274000898003578, "ranks": { "8": { "residual_relative_to_target": 0.006065714173018932, "delta_energy_captured": 0.9375573992729187, "probe_relative_error": 0.00602920725941658 }, "16": { "residual_relative_to_target": 0.006047368049621582, "delta_energy_captured": 0.9379345178604126, "probe_relative_error": 0.006012463476508856 }, "32": { "residual_relative_to_target": 0.006015493534505367, "delta_energy_captured": 0.9385870695114136, "probe_relative_error": 0.00597974332049489 } } }, { "module": "model.layers.24.mlp.down_proj", "shape": [ 5120, 17408 ], "delta_relative_norm": 0.023362569510936737, "ranks": { "8": { "residual_relative_to_target": 0.006049581803381443, "delta_energy_captured": 0.9329483509063721, "probe_relative_error": 0.006024221424013376 }, "16": { "residual_relative_to_target": 0.006035591010004282, "delta_energy_captured": 0.9332581162452698, "probe_relative_error": 0.006011801306158304 }, "32": { "residual_relative_to_target": 0.006015092600136995, "delta_energy_captured": 0.9337106943130493, "probe_relative_error": 0.005993457976728678 } } }, { "module": "model.layers.24.linear_attn.out_proj", "shape": [ 5120, 6144 ], "delta_relative_norm": 0.024083241820335388, "ranks": { "8": { "residual_relative_to_target": 0.006072258111089468, "delta_energy_captured": 0.9364273548126221, "probe_relative_error": 0.006066437810659409 }, "16": { "residual_relative_to_target": 0.0060536982491612434, "delta_energy_captured": 0.9368153214454651, "probe_relative_error": 0.006046985741704702 }, "32": { "residual_relative_to_target": 0.006021730601787567, "delta_energy_captured": 0.9374809265136719, "probe_relative_error": 0.006014283746480942 } } }, { "module": "model.layers.23.mlp.down_proj", "shape": [ 5120, 17408 ], "delta_relative_norm": 0.021976280957460403, "ranks": { "8": { "residual_relative_to_target": 0.006038030609488487, "delta_energy_captured": 0.9245112538337708, "probe_relative_error": 0.00608322536572814 }, "16": { "residual_relative_to_target": 0.006023796740919352, "delta_energy_captured": 0.9248667359352112, "probe_relative_error": 0.006067873910069466 }, "32": { "residual_relative_to_target": 0.006003255490213633, "delta_energy_captured": 0.9253782629966736, "probe_relative_error": 0.00604618526995182 } } }, { "module": "model.layers.23.self_attn.o_proj", "shape": [ 5120, 6144 ], "delta_relative_norm": 0.02459094114601612, "ranks": { "8": { "residual_relative_to_target": 0.006089568603783846, "delta_energy_captured": 0.9386770725250244, "probe_relative_error": 0.006102755665779114 }, "16": { "residual_relative_to_target": 0.0060713873244822025, "delta_energy_captured": 0.9390427470207214, "probe_relative_error": 0.006082585081458092 }, "32": { "residual_relative_to_target": 0.0060394457541406155, "delta_energy_captured": 0.9396824240684509, "probe_relative_error": 0.0060536605305969715 } } }, { "module": "model.layers.22.mlp.down_proj", "shape": [ 5120, 17408 ], "delta_relative_norm": 0.021504782140254974, "ranks": { "8": { "residual_relative_to_target": 0.006072333548218012, "delta_energy_captured": 0.9202664494514465, "probe_relative_error": 0.006064536981284618 }, "16": { "residual_relative_to_target": 0.006050500087440014, "delta_energy_captured": 0.9208387732505798, "probe_relative_error": 0.006038529332727194 }, "32": { "residual_relative_to_target": 0.006023169029504061, "delta_energy_captured": 0.9215523600578308, "probe_relative_error": 0.006012582685798407 } } }, { "module": "model.layers.22.linear_attn.out_proj", "shape": [ 5120, 6144 ], "delta_relative_norm": 0.022083798423409462, "ranks": { "8": { "residual_relative_to_target": 0.006067238748073578, "delta_energy_captured": 0.9245195388793945, "probe_relative_error": 0.00604527723044157 }, "16": { "residual_relative_to_target": 0.0060494886711239815, "delta_energy_captured": 0.9249604940414429, "probe_relative_error": 0.006027002353221178 }, "32": { "residual_relative_to_target": 0.006018282845616341, "delta_energy_captured": 0.9257327318191528, "probe_relative_error": 0.005999112967401743 } } }, { "module": "model.layers.21.mlp.down_proj", "shape": [ 5120, 17408 ], "delta_relative_norm": 0.021827833727002144, "ranks": { "8": { "residual_relative_to_target": 0.006038367282599211, "delta_energy_captured": 0.9234724640846252, "probe_relative_error": 0.005995467305183411 }, "16": { "residual_relative_to_target": 0.006024343892931938, "delta_energy_captured": 0.9238275289535522, "probe_relative_error": 0.005979957990348339 }, "32": { "residual_relative_to_target": 0.006004046648740768, "delta_energy_captured": 0.9243398904800415, "probe_relative_error": 0.005958807189017534 } } }, { "module": "model.layers.21.linear_attn.out_proj", "shape": [ 5120, 6144 ], "delta_relative_norm": 0.02285146340727806, "ranks": { "8": { "residual_relative_to_target": 0.00608475599437952, "delta_energy_captured": 0.9290980100631714, "probe_relative_error": 0.006055457983165979 }, "16": { "residual_relative_to_target": 0.006066649686545134, "delta_energy_captured": 0.9295193552970886, "probe_relative_error": 0.006038536783307791 }, "32": { "residual_relative_to_target": 0.006035485304892063, "delta_energy_captured": 0.9302416443824768, "probe_relative_error": 0.006008730735629797 } } }, { "module": "model.layers.20.mlp.down_proj", "shape": [ 5120, 17408 ], "delta_relative_norm": 0.021813180297613144, "ranks": { "8": { "residual_relative_to_target": 0.006039663217961788, "delta_energy_captured": 0.9233367443084717, "probe_relative_error": 0.006016040686517954 }, "16": { "residual_relative_to_target": 0.006025310605764389, "delta_energy_captured": 0.9237006306648254, "probe_relative_error": 0.0059998370707035065 }, "32": { "residual_relative_to_target": 0.006004769820719957, "delta_energy_captured": 0.9242199659347534, "probe_relative_error": 0.005978855304419994 } } }, { "module": "model.layers.20.linear_attn.out_proj", "shape": [ 5120, 6144 ], "delta_relative_norm": 0.0237997155636549, "ranks": { "8": { "residual_relative_to_target": 0.00606476841494441, "delta_energy_captured": 0.9350640773773193, "probe_relative_error": 0.006062047556042671 }, "16": { "residual_relative_to_target": 0.006047225557267666, "delta_energy_captured": 0.9354392290115356, "probe_relative_error": 0.006042941939085722 }, "32": { "residual_relative_to_target": 0.006016072351485491, "delta_energy_captured": 0.9361026883125305, "probe_relative_error": 0.0060104308649897575 } } }, { "module": "model.layers.19.mlp.down_proj", "shape": [ 5120, 17408 ], "delta_relative_norm": 0.022161902859807014, "ranks": { "8": { "residual_relative_to_target": 0.006053135730326176, "delta_energy_captured": 0.9253986477851868, "probe_relative_error": 0.006019380409270525 }, "16": { "residual_relative_to_target": 0.00603879988193512, "delta_energy_captured": 0.9257515668869019, "probe_relative_error": 0.006007079500705004 }, "32": { "residual_relative_to_target": 0.006018429063260555, "delta_energy_captured": 0.9262516498565674, "probe_relative_error": 0.0059866951778531075 } } }, { "module": "model.layers.19.self_attn.o_proj", "shape": [ 5120, 6144 ], "delta_relative_norm": 0.02385675348341465, "ranks": { "8": { "residual_relative_to_target": 0.0061067258939146996, "delta_energy_captured": 0.934476912021637, "probe_relative_error": 0.006082329899072647 }, "16": { "residual_relative_to_target": 0.006087568122893572, "delta_energy_captured": 0.9348874092102051, "probe_relative_error": 0.006062052678316832 }, "32": { "residual_relative_to_target": 0.00605359673500061, "delta_energy_captured": 0.9356120824813843, "probe_relative_error": 0.00603104941546917 } } }, { "module": "model.layers.18.mlp.down_proj", "shape": [ 5120, 17408 ], "delta_relative_norm": 0.023070180788636208, "ranks": { "8": { "residual_relative_to_target": 0.006130789872258902, "delta_energy_captured": 0.9293794631958008, "probe_relative_error": 0.0060980236157774925 }, "16": { "residual_relative_to_target": 0.006104371510446072, "delta_energy_captured": 0.929986834526062, "probe_relative_error": 0.006068575661629438 }, "32": { "residual_relative_to_target": 0.006074471864849329, "delta_energy_captured": 0.930670976638794, "probe_relative_error": 0.006038994528353214 } } }, { "module": "model.layers.18.linear_attn.out_proj", "shape": [ 5120, 6144 ], "delta_relative_norm": 0.025940420106053352, "ranks": { "8": { "residual_relative_to_target": 0.006097191479057074, "delta_energy_captured": 0.9447534084320068, "probe_relative_error": 0.006080218590795994 }, "16": { "residual_relative_to_target": 0.006074738688766956, "delta_energy_captured": 0.9451595544815063, "probe_relative_error": 0.006060102954506874 }, "32": { "residual_relative_to_target": 0.006038224324584007, "delta_energy_captured": 0.9458168148994446, "probe_relative_error": 0.006025283597409725 } } }, { "module": "model.layers.17.mlp.down_proj", "shape": [ 5120, 17408 ], "delta_relative_norm": 0.024629604071378708, "ranks": { "8": { "residual_relative_to_target": 0.006067629903554916, "delta_energy_captured": 0.9393091201782227, "probe_relative_error": 0.0060760448686778545 }, "16": { "residual_relative_to_target": 0.006053771357983351, "delta_energy_captured": 0.9395860433578491, "probe_relative_error": 0.0060633281245827675 }, "32": { "residual_relative_to_target": 0.006033207289874554, "delta_energy_captured": 0.9399958252906799, "probe_relative_error": 0.006042406428605318 } } }, { "module": "model.layers.17.linear_attn.out_proj", "shape": [ 5120, 6144 ], "delta_relative_norm": 0.029034972190856934, "ranks": { "8": { "residual_relative_to_target": 0.006091829389333725, "delta_energy_captured": 0.9559797048568726, "probe_relative_error": 0.0060930377803742886 }, "16": { "residual_relative_to_target": 0.006073659285902977, "delta_energy_captured": 0.9562419652938843, "probe_relative_error": 0.006077329162508249 }, "32": { "residual_relative_to_target": 0.006041647866368294, "delta_energy_captured": 0.9567019939422607, "probe_relative_error": 0.0060445102863013744 } } }, { "module": "model.layers.16.mlp.down_proj", "shape": [ 5120, 17408 ], "delta_relative_norm": 0.02343214862048626, "ranks": { "8": { "residual_relative_to_target": 0.006054579745978117, "delta_energy_captured": 0.933235764503479, "probe_relative_error": 0.006049185525625944 }, "16": { "residual_relative_to_target": 0.006040811073035002, "delta_energy_captured": 0.933539092540741, "probe_relative_error": 0.006034540943801403 }, "32": { "residual_relative_to_target": 0.006020474247634411, "delta_energy_captured": 0.9339858293533325, "probe_relative_error": 0.006015355233103037 } } }, { "module": "model.layers.16.linear_attn.out_proj", "shape": [ 5120, 6144 ], "delta_relative_norm": 0.02820233814418316, "ranks": { "8": { "residual_relative_to_target": 0.006078017875552177, "delta_energy_captured": 0.9535534381866455, "probe_relative_error": 0.006061699707061052 }, "16": { "residual_relative_to_target": 0.0060597974807024, "delta_energy_captured": 0.9538314342498779, "probe_relative_error": 0.006042744964361191 }, "32": { "residual_relative_to_target": 0.006027461960911751, "delta_energy_captured": 0.954322874546051, "probe_relative_error": 0.0060088844038546085 } } }, { "module": "model.layers.15.mlp.down_proj", "shape": [ 5120, 17408 ], "delta_relative_norm": 0.022821994498372078, "ranks": { "8": { "residual_relative_to_target": 0.00604657270014286, "delta_energy_captured": 0.9298041462898254, "probe_relative_error": 0.006050425581634045 }, "16": { "residual_relative_to_target": 0.006032528821378946, "delta_energy_captured": 0.9301298260688782, "probe_relative_error": 0.006037232466042042 }, "32": { "residual_relative_to_target": 0.006012138910591602, "delta_energy_captured": 0.9306013584136963, "probe_relative_error": 0.0060179997235536575 } } }, { "module": "model.layers.15.self_attn.o_proj", "shape": [ 5120, 6144 ], "delta_relative_norm": 0.02703053131699562, "ranks": { "8": { "residual_relative_to_target": 0.00609366362914443, "delta_energy_captured": 0.9491784572601318, "probe_relative_error": 0.006042139139026403 }, "16": { "residual_relative_to_target": 0.006074550561606884, "delta_energy_captured": 0.9494967460632324, "probe_relative_error": 0.0060224770568311214 }, "32": { "residual_relative_to_target": 0.006040977314114571, "delta_energy_captured": 0.9500534534454346, "probe_relative_error": 0.005992259830236435 } } }, { "module": "model.layers.14.mlp.down_proj", "shape": [ 5120, 17408 ], "delta_relative_norm": 0.022100793197751045, "ranks": { "8": { "residual_relative_to_target": 0.006036078091710806, "delta_energy_captured": 0.9254077076911926, "probe_relative_error": 0.0060438006184995174 }, "16": { "residual_relative_to_target": 0.006021819543093443, "delta_energy_captured": 0.9257596731185913, "probe_relative_error": 0.006028554402291775 }, "32": { "residual_relative_to_target": 0.006001460365951061, "delta_energy_captured": 0.9262608289718628, "probe_relative_error": 0.006006680894643068 } } }, { "module": "model.layers.14.linear_attn.out_proj", "shape": [ 5120, 6144 ], "delta_relative_norm": 0.023013804107904434, "ranks": { "8": { "residual_relative_to_target": 0.0060501997359097, "delta_energy_captured": 0.9308865666389465, "probe_relative_error": 0.006103380583226681 }, "16": { "residual_relative_to_target": 0.00603087292984128, "delta_energy_captured": 0.9313274025917053, "probe_relative_error": 0.006081442814320326 }, "32": { "residual_relative_to_target": 0.005998046603053808, "delta_energy_captured": 0.9320729374885559, "probe_relative_error": 0.006047853268682957 } } }, { "module": "model.layers.13.mlp.down_proj", "shape": [ 5120, 17408 ], "delta_relative_norm": 0.022425832226872444, "ranks": { "8": { "residual_relative_to_target": 0.006038348190486431, "delta_energy_captured": 0.9274997711181641, "probe_relative_error": 0.0060749780386686325 }, "16": { "residual_relative_to_target": 0.0060240947641432285, "delta_energy_captured": 0.9278416633605957, "probe_relative_error": 0.006057634949684143 }, "32": { "residual_relative_to_target": 0.006003596819937229, "delta_energy_captured": 0.9283319115638733, "probe_relative_error": 0.006034675519913435 } } }, { "module": "model.layers.13.linear_attn.out_proj", "shape": [ 5120, 6144 ], "delta_relative_norm": 0.025106720626354218, "ranks": { "8": { "residual_relative_to_target": 0.006043462548404932, "delta_energy_captured": 0.9420582056045532, "probe_relative_error": 0.005998039152473211 }, "16": { "residual_relative_to_target": 0.006025892682373524, "delta_energy_captured": 0.9423946738243103, "probe_relative_error": 0.005979448091238737 }, "32": { "residual_relative_to_target": 0.005994727369397879, "delta_energy_captured": 0.9429889917373657, "probe_relative_error": 0.00594943854957819 } } }, { "module": "model.layers.12.mlp.down_proj", "shape": [ 5120, 17408 ], "delta_relative_norm": 0.022239768877625465, "ranks": { "8": { "residual_relative_to_target": 0.006037112325429916, "delta_energy_captured": 0.9263117909431458, "probe_relative_error": 0.006053021643310785 }, "16": { "residual_relative_to_target": 0.006022987887263298, "delta_energy_captured": 0.926656186580658, "probe_relative_error": 0.006039791274815798 }, "32": { "residual_relative_to_target": 0.00600282521918416, "delta_energy_captured": 0.9271464347839355, "probe_relative_error": 0.006018165498971939 } } }, { "module": "model.layers.12.linear_attn.out_proj", "shape": [ 5120, 6144 ], "delta_relative_norm": 0.024097878485918045, "ranks": { "8": { "residual_relative_to_target": 0.006034597754478455, "delta_energy_captured": 0.9372896552085876, "probe_relative_error": 0.006013261619955301 }, "16": { "residual_relative_to_target": 0.006016909144818783, "delta_energy_captured": 0.9376567602157593, "probe_relative_error": 0.0059937983751297 }, "32": { "residual_relative_to_target": 0.005985576193779707, "delta_energy_captured": 0.9383043646812439, "probe_relative_error": 0.00596481841057539 } } }, { "module": "model.layers.11.mlp.down_proj", "shape": [ 5120, 17408 ], "delta_relative_norm": 0.02259434014558792, "ranks": { "8": { "residual_relative_to_target": 0.006034224294126034, "delta_energy_captured": 0.9286746978759766, "probe_relative_error": 0.00604576850309968 }, "16": { "residual_relative_to_target": 0.00602027727290988, "delta_energy_captured": 0.9290040135383606, "probe_relative_error": 0.006031317170709372 }, "32": { "residual_relative_to_target": 0.006000080611556768, "delta_energy_captured": 0.9294795989990234, "probe_relative_error": 0.006006903015077114 } } }, { "module": "model.layers.11.self_attn.o_proj", "shape": [ 5120, 6144 ], "delta_relative_norm": 0.025072524324059486, "ranks": { "8": { "residual_relative_to_target": 0.006019134074449539, "delta_energy_captured": 0.942366898059845, "probe_relative_error": 0.005995700601488352 }, "16": { "residual_relative_to_target": 0.0059996191412210464, "delta_energy_captured": 0.9427400231361389, "probe_relative_error": 0.005978513974696398 }, "32": { "residual_relative_to_target": 0.005966710392385721, "delta_energy_captured": 0.9433664679527283, "probe_relative_error": 0.005948569625616074 } } }, { "module": "model.layers.10.mlp.down_proj", "shape": [ 5120, 17408 ], "delta_relative_norm": 0.021959984675049782, "ranks": { "8": { "residual_relative_to_target": 0.006046056281775236, "delta_energy_captured": 0.9241980314254761, "probe_relative_error": 0.0060799033381044865 }, "16": { "residual_relative_to_target": 0.006032118108123541, "delta_energy_captured": 0.9245471358299255, "probe_relative_error": 0.006066825240850449 }, "32": { "residual_relative_to_target": 0.006012010388076305, "delta_energy_captured": 0.925049364566803, "probe_relative_error": 0.0060454779304564 } } }, { "module": "model.layers.10.linear_attn.out_proj", "shape": [ 5120, 6144 ], "delta_relative_norm": 0.025852782651782036, "ranks": { "8": { "residual_relative_to_target": 0.006047027185559273, "delta_energy_captured": 0.945289671421051, "probe_relative_error": 0.0060429624281823635 }, "16": { "residual_relative_to_target": 0.006027723662555218, "delta_energy_captured": 0.9456384181976318, "probe_relative_error": 0.006024111993610859 }, "32": { "residual_relative_to_target": 0.005995101761072874, "delta_energy_captured": 0.9462252259254456, "probe_relative_error": 0.005992739927023649 } } }, { "module": "model.layers.9.mlp.down_proj", "shape": [ 5120, 17408 ], "delta_relative_norm": 0.022012993693351746, "ranks": { "8": { "residual_relative_to_target": 0.006033116020262241, "delta_energy_captured": 0.9248852729797363, "probe_relative_error": 0.006004780996590853 }, "16": { "residual_relative_to_target": 0.006018947809934616, "delta_energy_captured": 0.9252376556396484, "probe_relative_error": 0.005990602076053619 }, "32": { "residual_relative_to_target": 0.005998850800096989, "delta_energy_captured": 0.9257360696792603, "probe_relative_error": 0.005967405159026384 } } }, { "module": "model.layers.9.linear_attn.out_proj", "shape": [ 5120, 6144 ], "delta_relative_norm": 0.02678707055747509, "ranks": { "8": { "residual_relative_to_target": 0.006034271325916052, "delta_energy_captured": 0.9492542743682861, "probe_relative_error": 0.0060495235957205296 }, "16": { "residual_relative_to_target": 0.006015921011567116, "delta_energy_captured": 0.9495624899864197, "probe_relative_error": 0.006031757220625877 }, "32": { "residual_relative_to_target": 0.005984538234770298, "delta_energy_captured": 0.950087308883667, "probe_relative_error": 0.005997255444526672 } } }, { "module": "model.layers.8.mlp.down_proj", "shape": [ 5120, 17408 ], "delta_relative_norm": 0.02180212363600731, "ranks": { "8": { "residual_relative_to_target": 0.0060295783914625645, "delta_energy_captured": 0.9235149621963501, "probe_relative_error": 0.006019883789122105 }, "16": { "residual_relative_to_target": 0.00601540133357048, "delta_energy_captured": 0.9238742589950562, "probe_relative_error": 0.00600812491029501 }, "32": { "residual_relative_to_target": 0.005995131563395262, "delta_energy_captured": 0.9243863821029663, "probe_relative_error": 0.005985751282423735 } } }, { "module": "model.layers.8.linear_attn.out_proj", "shape": [ 5120, 6144 ], "delta_relative_norm": 0.024273570626974106, "ranks": { "8": { "residual_relative_to_target": 0.0060166832990944386, "delta_energy_captured": 0.9385606050491333, "probe_relative_error": 0.006043833680450916 }, "16": { "residual_relative_to_target": 0.005998707842081785, "delta_energy_captured": 0.938927173614502, "probe_relative_error": 0.006026110146194696 }, "32": { "residual_relative_to_target": 0.005967574659734964, "delta_energy_captured": 0.9395594596862793, "probe_relative_error": 0.005996168591082096 } } }, { "module": "model.layers.7.mlp.down_proj", "shape": [ 5120, 17408 ], "delta_relative_norm": 0.02162010408937931, "ranks": { "8": { "residual_relative_to_target": 0.006035885773599148, "delta_energy_captured": 0.9220588803291321, "probe_relative_error": 0.006062371656298637 }, "16": { "residual_relative_to_target": 0.00602191174402833, "delta_energy_captured": 0.9224193692207336, "probe_relative_error": 0.006047965493053198 }, "32": { "residual_relative_to_target": 0.006001768168061972, "delta_energy_captured": 0.9229375123977661, "probe_relative_error": 0.00602865731343627 } } }, { "module": "model.layers.7.self_attn.o_proj", "shape": [ 5120, 6144 ], "delta_relative_norm": 0.02745869569480419, "ranks": { "8": { "residual_relative_to_target": 0.006018606945872307, "delta_energy_captured": 0.9519567489624023, "probe_relative_error": 0.005995569750666618 }, "16": { "residual_relative_to_target": 0.005999917164444923, "delta_energy_captured": 0.9522547125816345, "probe_relative_error": 0.005973567720502615 }, "32": { "residual_relative_to_target": 0.00596750108525157, "delta_energy_captured": 0.9527692198753357, "probe_relative_error": 0.00593875115737319 } } }, { "module": "model.layers.6.mlp.down_proj", "shape": [ 5120, 17408 ], "delta_relative_norm": 0.02127215638756752, "ranks": { "8": { "residual_relative_to_target": 0.006108672358095646, "delta_energy_captured": 0.9175348281860352, "probe_relative_error": 0.00613442761823535 }, "16": { "residual_relative_to_target": 0.0060856821946799755, "delta_energy_captured": 0.9181543588638306, "probe_relative_error": 0.006111704744398594 }, "32": { "residual_relative_to_target": 0.006046954542398453, "delta_energy_captured": 0.9191927313804626, "probe_relative_error": 0.00607213145121932 } } }, { "module": "model.layers.6.linear_attn.out_proj", "shape": [ 5120, 6144 ], "delta_relative_norm": 0.023792579770088196, "ranks": { "8": { "residual_relative_to_target": 0.006038430146872997, "delta_energy_captured": 0.9355882406234741, "probe_relative_error": 0.006044465582817793 }, "16": { "residual_relative_to_target": 0.006018117535859346, "delta_energy_captured": 0.9360209107398987, "probe_relative_error": 0.006026023533195257 }, "32": { "residual_relative_to_target": 0.005984195042401552, "delta_energy_captured": 0.9367401003837585, "probe_relative_error": 0.005987845826894045 } } }, { "module": "model.layers.5.mlp.down_proj", "shape": [ 5120, 17408 ], "delta_relative_norm": 0.021345321089029312, "ranks": { "8": { "residual_relative_to_target": 0.00603437889367342, "delta_energy_captured": 0.920079231262207, "probe_relative_error": 0.005967453587800264 }, "16": { "residual_relative_to_target": 0.006020332220941782, "delta_energy_captured": 0.9204508662223816, "probe_relative_error": 0.005954425781965256 }, "32": { "residual_relative_to_target": 0.006000148132443428, "delta_energy_captured": 0.9209833741188049, "probe_relative_error": 0.005935054272413254 } } }, { "module": "model.layers.5.linear_attn.out_proj", "shape": [ 5120, 6144 ], "delta_relative_norm": 0.023078523576259613, "ranks": { "8": { "residual_relative_to_target": 0.006015125662088394, "delta_energy_captured": 0.9320681691169739, "probe_relative_error": 0.006044980604201555 }, "16": { "residual_relative_to_target": 0.005995744373649359, "delta_energy_captured": 0.9325052499771118, "probe_relative_error": 0.006025515031069517 }, "32": { "residual_relative_to_target": 0.005963634233921766, "delta_energy_captured": 0.9332262277603149, "probe_relative_error": 0.005991884507238865 } } }, { "module": "model.layers.4.mlp.down_proj", "shape": [ 5120, 17408 ], "delta_relative_norm": 0.021551694720983505, "ranks": { "8": { "residual_relative_to_target": 0.0060305059887468815, "delta_energy_captured": 0.9217031002044678, "probe_relative_error": 0.006035069469362497 }, "16": { "residual_relative_to_target": 0.006016145925968885, "delta_energy_captured": 0.9220755100250244, "probe_relative_error": 0.0060176122933626175 }, "32": { "residual_relative_to_target": 0.005995978135615587, "delta_energy_captured": 0.9225971102714539, "probe_relative_error": 0.005999614950269461 } } }, { "module": "model.layers.4.linear_attn.out_proj", "shape": [ 5120, 6144 ], "delta_relative_norm": 0.02401619590818882, "ranks": { "8": { "residual_relative_to_target": 0.006006218492984772, "delta_energy_captured": 0.9374548196792603, "probe_relative_error": 0.0059804245829582214 }, "16": { "residual_relative_to_target": 0.005987741053104401, "delta_energy_captured": 0.9378390312194824, "probe_relative_error": 0.005964599549770355 }, "32": { "residual_relative_to_target": 0.0059563880786299706, "delta_energy_captured": 0.9384883046150208, "probe_relative_error": 0.0059371329843997955 } } }, { "module": "model.layers.3.mlp.down_proj", "shape": [ 5120, 17408 ], "delta_relative_norm": 0.022607041522860527, "ranks": { "8": { "residual_relative_to_target": 0.006031069438904524, "delta_energy_captured": 0.9288293123245239, "probe_relative_error": 0.006065687630325556 }, "16": { "residual_relative_to_target": 0.006016932427883148, "delta_energy_captured": 0.9291625618934631, "probe_relative_error": 0.006053399294614792 }, "32": { "residual_relative_to_target": 0.00599654670804739, "delta_energy_captured": 0.9296417236328125, "probe_relative_error": 0.006030156277120113 } } }, { "module": "model.layers.3.self_attn.o_proj", "shape": [ 5120, 6144 ], "delta_relative_norm": 0.028329668566584587, "ranks": { "8": { "residual_relative_to_target": 0.00593116320669651, "delta_energy_captured": 0.956167459487915, "probe_relative_error": 0.0058668553829193115 }, "16": { "residual_relative_to_target": 0.005911922547966242, "delta_energy_captured": 0.9564513564109802, "probe_relative_error": 0.005846976302564144 }, "32": { "residual_relative_to_target": 0.005879995413124561, "delta_energy_captured": 0.9569204449653625, "probe_relative_error": 0.005814167205244303 } } }, { "module": "model.layers.2.mlp.down_proj", "shape": [ 5120, 17408 ], "delta_relative_norm": 0.021558906883001328, "ranks": { "8": { "residual_relative_to_target": 0.00602675648406148, "delta_energy_captured": 0.921852707862854, "probe_relative_error": 0.006030671298503876 }, "16": { "residual_relative_to_target": 0.006012590602040291, "delta_energy_captured": 0.9222196340560913, "probe_relative_error": 0.0060173142701387405 }, "32": { "residual_relative_to_target": 0.0059923515655100346, "delta_energy_captured": 0.9227424263954163, "probe_relative_error": 0.005995551124215126 } } }, { "module": "model.layers.2.linear_attn.out_proj", "shape": [ 5120, 6144 ], "delta_relative_norm": 0.02806072309613228, "ranks": { "8": { "residual_relative_to_target": 0.006055373232811689, "delta_energy_captured": 0.9534323811531067, "probe_relative_error": 0.006050335709005594 }, "16": { "residual_relative_to_target": 0.006034280173480511, "delta_energy_captured": 0.9537562131881714, "probe_relative_error": 0.006029642652720213 }, "32": { "residual_relative_to_target": 0.005999798886477947, "delta_energy_captured": 0.9542832374572754, "probe_relative_error": 0.005996998865157366 } } }, { "module": "model.layers.1.mlp.down_proj", "shape": [ 5120, 17408 ], "delta_relative_norm": 0.01705504208803177, "ranks": { "8": { "residual_relative_to_target": 0.0060407547280192375, "delta_energy_captured": 0.8745481967926025, "probe_relative_error": 0.006024964619427919 }, "16": { "residual_relative_to_target": 0.006017753388732672, "delta_energy_captured": 0.8755017518997192, "probe_relative_error": 0.006006648764014244 }, "32": { "residual_relative_to_target": 0.005986127071082592, "delta_energy_captured": 0.876806914806366, "probe_relative_error": 0.005975868087261915 } } }, { "module": "model.layers.1.linear_attn.out_proj", "shape": [ 5120, 6144 ], "delta_relative_norm": 0.021867001429200172, "ranks": { "8": { "residual_relative_to_target": 0.005905060563236475, "delta_energy_captured": 0.9270760416984558, "probe_relative_error": 0.005854501388967037 }, "16": { "residual_relative_to_target": 0.005885052494704723, "delta_energy_captured": 0.9275693893432617, "probe_relative_error": 0.0058358353562653065 }, "32": { "residual_relative_to_target": 0.0058518461883068085, "delta_energy_captured": 0.9283844232559204, "probe_relative_error": 0.005808894522488117 } } } ] }