| { |
| "method": "randomized SVD of canonical mixed-Q8 minus decoded original ternary weights", |
| "seed": 1337, |
| "svd_q": 40, |
| "svd_power_iterations": 4, |
| "ranks": [ |
| 8, |
| 16, |
| 32 |
| ], |
| "modules": 126, |
| "source_revision": "3f926b415992eaa2ae9dd7b573706494d6bbf787", |
| "reference_revision": "61231c9a2feed6457f47d1d50a691b8dc96b2613", |
| "gguf_adapter_input_basis": "canonical, cyclic GDN columns", |
| "mlx_adapter_input_basis": "canonical, grouped GDN columns", |
| "seconds": 10.651134014129639, |
| "rows": [ |
| { |
| "module": "model.layers.63.mlp.down_proj", |
| "shape": [ |
| 5120, |
| 17408 |
| ], |
| "delta_relative_norm": 0.023493614047765732, |
| "ranks": { |
| "8": { |
| "residual_relative_to_target": 0.006203466095030308, |
| "delta_energy_captured": 0.9302781224250793, |
| "probe_relative_error": 0.00619496637955308 |
| }, |
| "16": { |
| "residual_relative_to_target": 0.006184915546327829, |
| "delta_energy_captured": 0.9306944608688354, |
| "probe_relative_error": 0.006179067771881819 |
| }, |
| "32": { |
| "residual_relative_to_target": 0.0061549535021185875, |
| "delta_energy_captured": 0.9313643574714661, |
| "probe_relative_error": 0.0061479369178414345 |
| } |
| } |
| }, |
| { |
| "module": "model.layers.63.self_attn.o_proj", |
| "shape": [ |
| 5120, |
| 6144 |
| ], |
| "delta_relative_norm": 0.025529770180583, |
| "ranks": { |
| "8": { |
| "residual_relative_to_target": 0.006384628359228373, |
| "delta_energy_captured": 0.9374572038650513, |
| "probe_relative_error": 0.006409766618162394 |
| }, |
| "16": { |
| "residual_relative_to_target": 0.006359532941132784, |
| "delta_energy_captured": 0.9379478693008423, |
| "probe_relative_error": 0.0063870251178741455 |
| }, |
| "32": { |
| "residual_relative_to_target": 0.006322126369923353, |
| "delta_energy_captured": 0.9386757016181946, |
| "probe_relative_error": 0.006350215524435043 |
| } |
| } |
| }, |
| { |
| "module": "model.layers.62.mlp.down_proj", |
| "shape": [ |
| 5120, |
| 17408 |
| ], |
| "delta_relative_norm": 0.02229161560535431, |
| "ranks": { |
| "8": { |
| "residual_relative_to_target": 0.006088209804147482, |
| "delta_energy_captured": 0.925407350063324, |
| "probe_relative_error": 0.006114326883107424 |
| }, |
| "16": { |
| "residual_relative_to_target": 0.006071844138205051, |
| "delta_energy_captured": 0.9258078336715698, |
| "probe_relative_error": 0.0061011542566120625 |
| }, |
| "32": { |
| "residual_relative_to_target": 0.006048388779163361, |
| "delta_energy_captured": 0.926379919052124, |
| "probe_relative_error": 0.006077746860682964 |
| } |
| } |
| }, |
| { |
| "module": "model.layers.62.linear_attn.out_proj", |
| "shape": [ |
| 5120, |
| 6144 |
| ], |
| "delta_relative_norm": 0.022418532520532608, |
| "ranks": { |
| "8": { |
| "residual_relative_to_target": 0.0061709885485470295, |
| "delta_energy_captured": 0.9242303967475891, |
| "probe_relative_error": 0.006205723155289888 |
| }, |
| "16": { |
| "residual_relative_to_target": 0.006148629356175661, |
| "delta_energy_captured": 0.9247784614562988, |
| "probe_relative_error": 0.0061827413737773895 |
| }, |
| "32": { |
| "residual_relative_to_target": 0.0061127119697630405, |
| "delta_energy_captured": 0.9256547093391418, |
| "probe_relative_error": 0.006148936226963997 |
| } |
| } |
| }, |
| { |
| "module": "model.layers.61.mlp.down_proj", |
| "shape": [ |
| 5120, |
| 17408 |
| ], |
| "delta_relative_norm": 0.02129940316081047, |
| "ranks": { |
| "8": { |
| "residual_relative_to_target": 0.006037276703864336, |
| "delta_energy_captured": 0.9196571707725525, |
| "probe_relative_error": 0.006025936454534531 |
| }, |
| "16": { |
| "residual_relative_to_target": 0.00602235272526741, |
| "delta_energy_captured": 0.9200538396835327, |
| "probe_relative_error": 0.006011681165546179 |
| }, |
| "32": { |
| "residual_relative_to_target": 0.006001652218401432, |
| "delta_energy_captured": 0.9206025004386902, |
| "probe_relative_error": 0.005993095226585865 |
| } |
| } |
| }, |
| { |
| "module": "model.layers.61.linear_attn.out_proj", |
| "shape": [ |
| 5120, |
| 6144 |
| ], |
| "delta_relative_norm": 0.02114199846982956, |
| "ranks": { |
| "8": { |
| "residual_relative_to_target": 0.006076216232031584, |
| "delta_energy_captured": 0.9174010753631592, |
| "probe_relative_error": 0.006095276679843664 |
| }, |
| "16": { |
| "residual_relative_to_target": 0.0060568880289793015, |
| "delta_energy_captured": 0.9179257154464722, |
| "probe_relative_error": 0.006078938953578472 |
| }, |
| "32": { |
| "residual_relative_to_target": 0.006024396046996117, |
| "delta_energy_captured": 0.9188039302825928, |
| "probe_relative_error": 0.006048615090548992 |
| } |
| } |
| }, |
| { |
| "module": "model.layers.60.mlp.down_proj", |
| "shape": [ |
| 5120, |
| 17408 |
| ], |
| "delta_relative_norm": 0.020835502073168755, |
| "ranks": { |
| "8": { |
| "residual_relative_to_target": 0.0060300398617982864, |
| "delta_energy_captured": 0.9162408113479614, |
| "probe_relative_error": 0.0060409135185182095 |
| }, |
| "16": { |
| "residual_relative_to_target": 0.006015256978571415, |
| "delta_energy_captured": 0.9166510105133057, |
| "probe_relative_error": 0.006025105714797974 |
| }, |
| "32": { |
| "residual_relative_to_target": 0.00599451782181859, |
| "delta_energy_captured": 0.9172247648239136, |
| "probe_relative_error": 0.006004229187965393 |
| } |
| } |
| }, |
| { |
| "module": "model.layers.60.linear_attn.out_proj", |
| "shape": [ |
| 5120, |
| 6144 |
| ], |
| "delta_relative_norm": 0.021040836349129677, |
| "ranks": { |
| "8": { |
| "residual_relative_to_target": 0.006096669472754002, |
| "delta_energy_captured": 0.9160425662994385, |
| "probe_relative_error": 0.006109232548624277 |
| }, |
| "16": { |
| "residual_relative_to_target": 0.006076797377318144, |
| "delta_energy_captured": 0.9165889620780945, |
| "probe_relative_error": 0.006089756730943918 |
| }, |
| "32": { |
| "residual_relative_to_target": 0.006043015979230404, |
| "delta_energy_captured": 0.9175137281417847, |
| "probe_relative_error": 0.006061606574803591 |
| } |
| } |
| }, |
| { |
| "module": "model.layers.59.mlp.down_proj", |
| "shape": [ |
| 5120, |
| 17408 |
| ], |
| "delta_relative_norm": 0.020492179319262505, |
| "ranks": { |
| "8": { |
| "residual_relative_to_target": 0.006026110146194696, |
| "delta_energy_captured": 0.9135235548019409, |
| "probe_relative_error": 0.006006068084388971 |
| }, |
| "16": { |
| "residual_relative_to_target": 0.006011185701936483, |
| "delta_energy_captured": 0.9139513373374939, |
| "probe_relative_error": 0.005989533849060535 |
| }, |
| "32": { |
| "residual_relative_to_target": 0.005990565754473209, |
| "delta_energy_captured": 0.9145406484603882, |
| "probe_relative_error": 0.005969911813735962 |
| } |
| } |
| }, |
| { |
| "module": "model.layers.59.self_attn.o_proj", |
| "shape": [ |
| 5120, |
| 6144 |
| ], |
| "delta_relative_norm": 0.020459972321987152, |
| "ranks": { |
| "8": { |
| "residual_relative_to_target": 0.006094166077673435, |
| "delta_energy_captured": 0.911280632019043, |
| "probe_relative_error": 0.006112969480454922 |
| }, |
| "16": { |
| "residual_relative_to_target": 0.006076589692384005, |
| "delta_energy_captured": 0.9117916822433472, |
| "probe_relative_error": 0.006095355376601219 |
| }, |
| "32": { |
| "residual_relative_to_target": 0.006045016460120678, |
| "delta_energy_captured": 0.9127058982849121, |
| "probe_relative_error": 0.006063488312065601 |
| } |
| } |
| }, |
| { |
| "module": "model.layers.58.mlp.down_proj", |
| "shape": [ |
| 5120, |
| 17408 |
| ], |
| "delta_relative_norm": 0.020321514457464218, |
| "ranks": { |
| "8": { |
| "residual_relative_to_target": 0.006021411623805761, |
| "delta_energy_captured": 0.912202000617981, |
| "probe_relative_error": 0.005999132990837097 |
| }, |
| "16": { |
| "residual_relative_to_target": 0.006006334908306599, |
| "delta_energy_captured": 0.912641167640686, |
| "probe_relative_error": 0.005983144510537386 |
| }, |
| "32": { |
| "residual_relative_to_target": 0.005985502153635025, |
| "delta_energy_captured": 0.9132460951805115, |
| "probe_relative_error": 0.005962659139186144 |
| } |
| } |
| }, |
| { |
| "module": "model.layers.58.linear_attn.out_proj", |
| "shape": [ |
| 5120, |
| 6144 |
| ], |
| "delta_relative_norm": 0.020883062854409218, |
| "ranks": { |
| "8": { |
| "residual_relative_to_target": 0.006070531439036131, |
| "delta_energy_captured": 0.9154983758926392, |
| "probe_relative_error": 0.006043680943548679 |
| }, |
| "16": { |
| "residual_relative_to_target": 0.00605217507109046, |
| "delta_energy_captured": 0.9160086512565613, |
| "probe_relative_error": 0.006030766759067774 |
| }, |
| "32": { |
| "residual_relative_to_target": 0.006020673550665379, |
| "delta_energy_captured": 0.91688072681427, |
| "probe_relative_error": 0.006002210546284914 |
| } |
| } |
| }, |
| { |
| "module": "model.layers.57.mlp.down_proj", |
| "shape": [ |
| 5120, |
| 17408 |
| ], |
| "delta_relative_norm": 0.020097721368074417, |
| "ranks": { |
| "8": { |
| "residual_relative_to_target": 0.006018276326358318, |
| "delta_energy_captured": 0.910329282283783, |
| "probe_relative_error": 0.006053621880710125 |
| }, |
| "16": { |
| "residual_relative_to_target": 0.00600338215008378, |
| "delta_energy_captured": 0.9107725620269775, |
| "probe_relative_error": 0.00603972002863884 |
| }, |
| "32": { |
| "residual_relative_to_target": 0.0059826611541211605, |
| "delta_energy_captured": 0.9113874435424805, |
| "probe_relative_error": 0.006016792729496956 |
| } |
| } |
| }, |
| { |
| "module": "model.layers.57.linear_attn.out_proj", |
| "shape": [ |
| 5120, |
| 6144 |
| ], |
| "delta_relative_norm": 0.020595118403434753, |
| "ranks": { |
| "8": { |
| "residual_relative_to_target": 0.006130394525825977, |
| "delta_energy_captured": 0.9113970398902893, |
| "probe_relative_error": 0.006177142262458801 |
| }, |
| "16": { |
| "residual_relative_to_target": 0.006112022791057825, |
| "delta_energy_captured": 0.9119272828102112, |
| "probe_relative_error": 0.006158243864774704 |
| }, |
| "32": { |
| "residual_relative_to_target": 0.006079487036913633, |
| "delta_energy_captured": 0.9128624796867371, |
| "probe_relative_error": 0.006125116255134344 |
| } |
| } |
| }, |
| { |
| "module": "model.layers.56.mlp.down_proj", |
| "shape": [ |
| 5120, |
| 17408 |
| ], |
| "delta_relative_norm": 0.020840104669332504, |
| "ranks": { |
| "8": { |
| "residual_relative_to_target": 0.006023420952260494, |
| "delta_energy_captured": 0.9164614677429199, |
| "probe_relative_error": 0.006023530848324299 |
| }, |
| "16": { |
| "residual_relative_to_target": 0.006008591037243605, |
| "delta_energy_captured": 0.9168723225593567, |
| "probe_relative_error": 0.006012072321027517 |
| }, |
| "32": { |
| "residual_relative_to_target": 0.005988001357764006, |
| "delta_energy_captured": 0.9174410700798035, |
| "probe_relative_error": 0.0059908549301326275 |
| } |
| } |
| }, |
| { |
| "module": "model.layers.56.linear_attn.out_proj", |
| "shape": [ |
| 5120, |
| 6144 |
| ], |
| "delta_relative_norm": 0.021443670615553856, |
| "ranks": { |
| "8": { |
| "residual_relative_to_target": 0.006094329059123993, |
| "delta_energy_captured": 0.9192293286323547, |
| "probe_relative_error": 0.006112502422183752 |
| }, |
| "16": { |
| "residual_relative_to_target": 0.006074394099414349, |
| "delta_energy_captured": 0.9197568893432617, |
| "probe_relative_error": 0.006086415611207485 |
| }, |
| "32": { |
| "residual_relative_to_target": 0.006041317712515593, |
| "delta_energy_captured": 0.9206284284591675, |
| "probe_relative_error": 0.006052073091268539 |
| } |
| } |
| }, |
| { |
| "module": "model.layers.55.mlp.down_proj", |
| "shape": [ |
| 5120, |
| 17408 |
| ], |
| "delta_relative_norm": 0.020723003894090652, |
| "ranks": { |
| "8": { |
| "residual_relative_to_target": 0.006033357698470354, |
| "delta_energy_captured": 0.9152357578277588, |
| "probe_relative_error": 0.0060050650499761105 |
| }, |
| "16": { |
| "residual_relative_to_target": 0.006018562242388725, |
| "delta_energy_captured": 0.9156509637832642, |
| "probe_relative_error": 0.005991325713694096 |
| }, |
| "32": { |
| "residual_relative_to_target": 0.00599807919934392, |
| "delta_energy_captured": 0.9162241220474243, |
| "probe_relative_error": 0.005970488302409649 |
| } |
| } |
| }, |
| { |
| "module": "model.layers.55.self_attn.o_proj", |
| "shape": [ |
| 5120, |
| 6144 |
| ], |
| "delta_relative_norm": 0.02028011530637741, |
| "ranks": { |
| "8": { |
| "residual_relative_to_target": 0.006079335231333971, |
| "delta_energy_captured": 0.9101389646530151, |
| "probe_relative_error": 0.0060809580609202385 |
| }, |
| "16": { |
| "residual_relative_to_target": 0.006061325315386057, |
| "delta_energy_captured": 0.9106706380844116, |
| "probe_relative_error": 0.0060623278841376305 |
| }, |
| "32": { |
| "residual_relative_to_target": 0.006028881296515465, |
| "delta_energy_captured": 0.9116243720054626, |
| "probe_relative_error": 0.006032067351043224 |
| } |
| } |
| }, |
| { |
| "module": "model.layers.54.mlp.down_proj", |
| "shape": [ |
| 5120, |
| 17408 |
| ], |
| "delta_relative_norm": 0.021479183807969093, |
| "ranks": { |
| "8": { |
| "residual_relative_to_target": 0.00607015285640955, |
| "delta_energy_captured": 0.920133650302887, |
| "probe_relative_error": 0.0060868519358336926 |
| }, |
| "16": { |
| "residual_relative_to_target": 0.006055823992937803, |
| "delta_energy_captured": 0.9205102920532227, |
| "probe_relative_error": 0.006073798518627882 |
| }, |
| "32": { |
| "residual_relative_to_target": 0.00603545643389225, |
| "delta_energy_captured": 0.9210440516471863, |
| "probe_relative_error": 0.0060530672781169415 |
| } |
| } |
| }, |
| { |
| "module": "model.layers.54.linear_attn.out_proj", |
| "shape": [ |
| 5120, |
| 6144 |
| ], |
| "delta_relative_norm": 0.022341154515743256, |
| "ranks": { |
| "8": { |
| "residual_relative_to_target": 0.0060975272208452225, |
| "delta_energy_captured": 0.9255102872848511, |
| "probe_relative_error": 0.006094307638704777 |
| }, |
| "16": { |
| "residual_relative_to_target": 0.006079318933188915, |
| "delta_energy_captured": 0.9259545207023621, |
| "probe_relative_error": 0.006073355209082365 |
| }, |
| "32": { |
| "residual_relative_to_target": 0.006047883071005344, |
| "delta_energy_captured": 0.9267182946205139, |
| "probe_relative_error": 0.006038699299097061 |
| } |
| } |
| }, |
| { |
| "module": "model.layers.53.mlp.down_proj", |
| "shape": [ |
| 5120, |
| 17408 |
| ], |
| "delta_relative_norm": 0.021105965599417686, |
| "ranks": { |
| "8": { |
| "residual_relative_to_target": 0.006058567203581333, |
| "delta_energy_captured": 0.917599618434906, |
| "probe_relative_error": 0.006060657557100058 |
| }, |
| "16": { |
| "residual_relative_to_target": 0.006044069305062294, |
| "delta_energy_captured": 0.9179934859275818, |
| "probe_relative_error": 0.006045852787792683 |
| }, |
| "32": { |
| "residual_relative_to_target": 0.006023644004017115, |
| "delta_energy_captured": 0.9185467958450317, |
| "probe_relative_error": 0.006026324350386858 |
| } |
| } |
| }, |
| { |
| "module": "model.layers.53.linear_attn.out_proj", |
| "shape": [ |
| 5120, |
| 6144 |
| ], |
| "delta_relative_norm": 0.02132597006857395, |
| "ranks": { |
| "8": { |
| "residual_relative_to_target": 0.006103483960032463, |
| "delta_energy_captured": 0.9180898070335388, |
| "probe_relative_error": 0.006080629304051399 |
| }, |
| "16": { |
| "residual_relative_to_target": 0.006084129214286804, |
| "delta_energy_captured": 0.9186084270477295, |
| "probe_relative_error": 0.0060598887503147125 |
| }, |
| "32": { |
| "residual_relative_to_target": 0.00605209544301033, |
| "delta_energy_captured": 0.9194632768630981, |
| "probe_relative_error": 0.006029986310750246 |
| } |
| } |
| }, |
| { |
| "module": "model.layers.52.mlp.down_proj", |
| "shape": [ |
| 5120, |
| 17408 |
| ], |
| "delta_relative_norm": 0.02121596783399582, |
| "ranks": { |
| "8": { |
| "residual_relative_to_target": 0.006060957908630371, |
| "delta_energy_captured": 0.9183874726295471, |
| "probe_relative_error": 0.006080988794565201 |
| }, |
| "16": { |
| "residual_relative_to_target": 0.00604634452611208, |
| "delta_energy_captured": 0.9187805652618408, |
| "probe_relative_error": 0.006065780762583017 |
| }, |
| "32": { |
| "residual_relative_to_target": 0.006025712005794048, |
| "delta_energy_captured": 0.9193339347839355, |
| "probe_relative_error": 0.006043375004082918 |
| } |
| } |
| }, |
| { |
| "module": "model.layers.52.linear_attn.out_proj", |
| "shape": [ |
| 5120, |
| 6144 |
| ], |
| "delta_relative_norm": 0.02138073556125164, |
| "ranks": { |
| "8": { |
| "residual_relative_to_target": 0.006117842625826597, |
| "delta_energy_captured": 0.9181250333786011, |
| "probe_relative_error": 0.006146178115159273 |
| }, |
| "16": { |
| "residual_relative_to_target": 0.006097991019487381, |
| "delta_energy_captured": 0.9186554551124573, |
| "probe_relative_error": 0.006124954670667648 |
| }, |
| "32": { |
| "residual_relative_to_target": 0.006065126042813063, |
| "delta_energy_captured": 0.919529914855957, |
| "probe_relative_error": 0.006093480624258518 |
| } |
| } |
| }, |
| { |
| "module": "model.layers.51.mlp.down_proj", |
| "shape": [ |
| 5120, |
| 17408 |
| ], |
| "delta_relative_norm": 0.020931726321578026, |
| "ranks": { |
| "8": { |
| "residual_relative_to_target": 0.006090490147471428, |
| "delta_energy_captured": 0.9153368473052979, |
| "probe_relative_error": 0.006093429401516914 |
| }, |
| "16": { |
| "residual_relative_to_target": 0.00607492309063673, |
| "delta_energy_captured": 0.915769100189209, |
| "probe_relative_error": 0.006077541504055262 |
| }, |
| "32": { |
| "residual_relative_to_target": 0.006054316647350788, |
| "delta_energy_captured": 0.9163395166397095, |
| "probe_relative_error": 0.006055254489183426 |
| } |
| } |
| }, |
| { |
| "module": "model.layers.51.self_attn.o_proj", |
| "shape": [ |
| 5120, |
| 6144 |
| ], |
| "delta_relative_norm": 0.021571796387434006, |
| "ranks": { |
| "8": { |
| "residual_relative_to_target": 0.006139351055026054, |
| "delta_energy_captured": 0.91900235414505, |
| "probe_relative_error": 0.006092521827667952 |
| }, |
| "16": { |
| "residual_relative_to_target": 0.006121189333498478, |
| "delta_energy_captured": 0.9194808602333069, |
| "probe_relative_error": 0.006075660698115826 |
| }, |
| "32": { |
| "residual_relative_to_target": 0.0060895271599292755, |
| "delta_energy_captured": 0.920311689376831, |
| "probe_relative_error": 0.006044131703674793 |
| } |
| } |
| }, |
| { |
| "module": "model.layers.50.mlp.down_proj", |
| "shape": [ |
| 5120, |
| 17408 |
| ], |
| "delta_relative_norm": 0.021893208846449852, |
| "ranks": { |
| "8": { |
| "residual_relative_to_target": 0.006137816235423088, |
| "delta_energy_captured": 0.9214024543762207, |
| "probe_relative_error": 0.0061620427295565605 |
| }, |
| "16": { |
| "residual_relative_to_target": 0.006123154424130917, |
| "delta_energy_captured": 0.9217774868011475, |
| "probe_relative_error": 0.006145530845969915 |
| }, |
| "32": { |
| "residual_relative_to_target": 0.006102458573877811, |
| "delta_energy_captured": 0.9223054051399231, |
| "probe_relative_error": 0.006125468760728836 |
| } |
| } |
| }, |
| { |
| "module": "model.layers.50.linear_attn.out_proj", |
| "shape": [ |
| 5120, |
| 6144 |
| ], |
| "delta_relative_norm": 0.024059979245066643, |
| "ranks": { |
| "8": { |
| "residual_relative_to_target": 0.006143790669739246, |
| "delta_energy_captured": 0.9347947835922241, |
| "probe_relative_error": 0.006125996354967356 |
| }, |
| "16": { |
| "residual_relative_to_target": 0.006122578866779804, |
| "delta_energy_captured": 0.9352442622184753, |
| "probe_relative_error": 0.006099299527704716 |
| }, |
| "32": { |
| "residual_relative_to_target": 0.006086510140448809, |
| "delta_energy_captured": 0.9360049962997437, |
| "probe_relative_error": 0.006064813118427992 |
| } |
| } |
| }, |
| { |
| "module": "model.layers.49.mlp.down_proj", |
| "shape": [ |
| 5120, |
| 17408 |
| ], |
| "delta_relative_norm": 0.022697126492857933, |
| "ranks": { |
| "8": { |
| "residual_relative_to_target": 0.0060973260551691055, |
| "delta_energy_captured": 0.9278332591056824, |
| "probe_relative_error": 0.006105566862970591 |
| }, |
| "16": { |
| "residual_relative_to_target": 0.006082397419959307, |
| "delta_energy_captured": 0.9281861782073975, |
| "probe_relative_error": 0.006091735325753689 |
| }, |
| "32": { |
| "residual_relative_to_target": 0.006061552558094263, |
| "delta_energy_captured": 0.9286775588989258, |
| "probe_relative_error": 0.006069166120141745 |
| } |
| } |
| }, |
| { |
| "module": "model.layers.49.linear_attn.out_proj", |
| "shape": [ |
| 5120, |
| 6144 |
| ], |
| "delta_relative_norm": 0.02313533052802086, |
| "ranks": { |
| "8": { |
| "residual_relative_to_target": 0.006113432347774506, |
| "delta_energy_captured": 0.9301737546920776, |
| "probe_relative_error": 0.006142680533230305 |
| }, |
| "16": { |
| "residual_relative_to_target": 0.006094715557992458, |
| "delta_energy_captured": 0.930600643157959, |
| "probe_relative_error": 0.006125420331954956 |
| }, |
| "32": { |
| "residual_relative_to_target": 0.006062833592295647, |
| "delta_energy_captured": 0.9313247799873352, |
| "probe_relative_error": 0.0060928696766495705 |
| } |
| } |
| }, |
| { |
| "module": "model.layers.48.mlp.down_proj", |
| "shape": [ |
| 5120, |
| 17408 |
| ], |
| "delta_relative_norm": 0.02249142900109291, |
| "ranks": { |
| "8": { |
| "residual_relative_to_target": 0.006094043608754873, |
| "delta_energy_captured": 0.9265863299369812, |
| "probe_relative_error": 0.0061579071916639805 |
| }, |
| "16": { |
| "residual_relative_to_target": 0.006078573875129223, |
| "delta_energy_captured": 0.9269585609436035, |
| "probe_relative_error": 0.0061442130245268345 |
| }, |
| "32": { |
| "residual_relative_to_target": 0.006057319696992636, |
| "delta_energy_captured": 0.9274684190750122, |
| "probe_relative_error": 0.0061202202923595905 |
| } |
| } |
| }, |
| { |
| "module": "model.layers.48.linear_attn.out_proj", |
| "shape": [ |
| 5120, |
| 6144 |
| ], |
| "delta_relative_norm": 0.024167608469724655, |
| "ranks": { |
| "8": { |
| "residual_relative_to_target": 0.006104980129748583, |
| "delta_energy_captured": 0.9361881613731384, |
| "probe_relative_error": 0.006097967736423016 |
| }, |
| "16": { |
| "residual_relative_to_target": 0.006086121313273907, |
| "delta_energy_captured": 0.9365817904472351, |
| "probe_relative_error": 0.006079885642975569 |
| }, |
| "32": { |
| "residual_relative_to_target": 0.00605367636308074, |
| "delta_energy_captured": 0.9372561573982239, |
| "probe_relative_error": 0.006046242546290159 |
| } |
| } |
| }, |
| { |
| "module": "model.layers.47.mlp.down_proj", |
| "shape": [ |
| 5120, |
| 17408 |
| ], |
| "delta_relative_norm": 0.023318033665418625, |
| "ranks": { |
| "8": { |
| "residual_relative_to_target": 0.006081343162804842, |
| "delta_energy_captured": 0.9319833517074585, |
| "probe_relative_error": 0.006071115843951702 |
| }, |
| "16": { |
| "residual_relative_to_target": 0.006066549103707075, |
| "delta_energy_captured": 0.9323139190673828, |
| "probe_relative_error": 0.006056347396224737 |
| }, |
| "32": { |
| "residual_relative_to_target": 0.006044955458492041, |
| "delta_energy_captured": 0.9327948689460754, |
| "probe_relative_error": 0.0060320631600916386 |
| } |
| } |
| }, |
| { |
| "module": "model.layers.47.self_attn.o_proj", |
| "shape": [ |
| 5120, |
| 6144 |
| ], |
| "delta_relative_norm": 0.023469405248761177, |
| "ranks": { |
| "8": { |
| "residual_relative_to_target": 0.006134840659797192, |
| "delta_energy_captured": 0.9316714406013489, |
| "probe_relative_error": 0.006136295851320028 |
| }, |
| "16": { |
| "residual_relative_to_target": 0.006115900352597237, |
| "delta_energy_captured": 0.9320926666259766, |
| "probe_relative_error": 0.0061173150315880775 |
| }, |
| "32": { |
| "residual_relative_to_target": 0.00608331011608243, |
| "delta_energy_captured": 0.9328144788742065, |
| "probe_relative_error": 0.006087667308747768 |
| } |
| } |
| }, |
| { |
| "module": "model.layers.46.mlp.down_proj", |
| "shape": [ |
| 5120, |
| 17408 |
| ], |
| "delta_relative_norm": 0.02287241257727146, |
| "ranks": { |
| "8": { |
| "residual_relative_to_target": 0.006067721173167229, |
| "delta_energy_captured": 0.929623544216156, |
| "probe_relative_error": 0.006093467120081186 |
| }, |
| "16": { |
| "residual_relative_to_target": 0.00605345843359828, |
| "delta_energy_captured": 0.9299539923667908, |
| "probe_relative_error": 0.0060799019411206245 |
| }, |
| "32": { |
| "residual_relative_to_target": 0.0060324957594275475, |
| "delta_energy_captured": 0.9304382801055908, |
| "probe_relative_error": 0.006062073167413473 |
| } |
| } |
| }, |
| { |
| "module": "model.layers.46.linear_attn.out_proj", |
| "shape": [ |
| 5120, |
| 6144 |
| ], |
| "delta_relative_norm": 0.022646324709057808, |
| "ranks": { |
| "8": { |
| "residual_relative_to_target": 0.006105736363679171, |
| "delta_energy_captured": 0.9273090362548828, |
| "probe_relative_error": 0.006053128745406866 |
| }, |
| "16": { |
| "residual_relative_to_target": 0.006086830981075764, |
| "delta_energy_captured": 0.9277584552764893, |
| "probe_relative_error": 0.006032545119524002 |
| }, |
| "32": { |
| "residual_relative_to_target": 0.0060533578507602215, |
| "delta_energy_captured": 0.9285508394241333, |
| "probe_relative_error": 0.00600312789902091 |
| } |
| } |
| }, |
| { |
| "module": "model.layers.45.mlp.down_proj", |
| "shape": [ |
| 5120, |
| 17408 |
| ], |
| "delta_relative_norm": 0.023377450183033943, |
| "ranks": { |
| "8": { |
| "residual_relative_to_target": 0.006068831775337458, |
| "delta_energy_captured": 0.9326068162918091, |
| "probe_relative_error": 0.006056040525436401 |
| }, |
| "16": { |
| "residual_relative_to_target": 0.006054762285202742, |
| "delta_energy_captured": 0.932918906211853, |
| "probe_relative_error": 0.006046064663678408 |
| }, |
| "32": { |
| "residual_relative_to_target": 0.006033135578036308, |
| "delta_energy_captured": 0.9333972930908203, |
| "probe_relative_error": 0.006024109665304422 |
| } |
| } |
| }, |
| { |
| "module": "model.layers.45.linear_attn.out_proj", |
| "shape": [ |
| 5120, |
| 6144 |
| ], |
| "delta_relative_norm": 0.023702578619122505, |
| "ranks": { |
| "8": { |
| "residual_relative_to_target": 0.006082818377763033, |
| "delta_energy_captured": 0.9341405034065247, |
| "probe_relative_error": 0.006086487788707018 |
| }, |
| "16": { |
| "residual_relative_to_target": 0.006064688786864281, |
| "delta_energy_captured": 0.9345324635505676, |
| "probe_relative_error": 0.006067697424441576 |
| }, |
| "32": { |
| "residual_relative_to_target": 0.006032311357557774, |
| "delta_energy_captured": 0.9352296590805054, |
| "probe_relative_error": 0.006036863196641207 |
| } |
| } |
| }, |
| { |
| "module": "model.layers.44.mlp.down_proj", |
| "shape": [ |
| 5120, |
| 17408 |
| ], |
| "delta_relative_norm": 0.023573627695441246, |
| "ranks": { |
| "8": { |
| "residual_relative_to_target": 0.006071622483432293, |
| "delta_energy_captured": 0.9336628913879395, |
| "probe_relative_error": 0.006076062098145485 |
| }, |
| "16": { |
| "residual_relative_to_target": 0.00605763541534543, |
| "delta_energy_captured": 0.933968186378479, |
| "probe_relative_error": 0.006062559317797422 |
| }, |
| "32": { |
| "residual_relative_to_target": 0.006036465056240559, |
| "delta_energy_captured": 0.934428870677948, |
| "probe_relative_error": 0.006040880456566811 |
| } |
| } |
| }, |
| { |
| "module": "model.layers.44.linear_attn.out_proj", |
| "shape": [ |
| 5120, |
| 6144 |
| ], |
| "delta_relative_norm": 0.02365094982087612, |
| "ranks": { |
| "8": { |
| "residual_relative_to_target": 0.006102156825363636, |
| "delta_energy_captured": 0.9334313869476318, |
| "probe_relative_error": 0.006059129256755114 |
| }, |
| "16": { |
| "residual_relative_to_target": 0.006084059830754995, |
| "delta_energy_captured": 0.9338256120681763, |
| "probe_relative_error": 0.006042174994945526 |
| }, |
| "32": { |
| "residual_relative_to_target": 0.006051745265722275, |
| "delta_energy_captured": 0.9345267415046692, |
| "probe_relative_error": 0.006016638595610857 |
| } |
| } |
| }, |
| { |
| "module": "model.layers.43.mlp.down_proj", |
| "shape": [ |
| 5120, |
| 17408 |
| ], |
| "delta_relative_norm": 0.022710220888257027, |
| "ranks": { |
| "8": { |
| "residual_relative_to_target": 0.006059695966541767, |
| "delta_energy_captured": 0.9288034439086914, |
| "probe_relative_error": 0.006042527500540018 |
| }, |
| "16": { |
| "residual_relative_to_target": 0.006045423448085785, |
| "delta_energy_captured": 0.9291384220123291, |
| "probe_relative_error": 0.00602593831717968 |
| }, |
| "32": { |
| "residual_relative_to_target": 0.006024426780641079, |
| "delta_energy_captured": 0.9296298027038574, |
| "probe_relative_error": 0.006004304625093937 |
| } |
| } |
| }, |
| { |
| "module": "model.layers.43.self_attn.o_proj", |
| "shape": [ |
| 5120, |
| 6144 |
| ], |
| "delta_relative_norm": 0.024399735033512115, |
| "ranks": { |
| "8": { |
| "residual_relative_to_target": 0.0061387596651911736, |
| "delta_energy_captured": 0.9367018342018127, |
| "probe_relative_error": 0.006134283263236284 |
| }, |
| "16": { |
| "residual_relative_to_target": 0.006119549740105867, |
| "delta_energy_captured": 0.9370973706245422, |
| "probe_relative_error": 0.006112574599683285 |
| }, |
| "32": { |
| "residual_relative_to_target": 0.006086132489144802, |
| "delta_energy_captured": 0.9377824664115906, |
| "probe_relative_error": 0.006079253740608692 |
| } |
| } |
| }, |
| { |
| "module": "model.layers.42.mlp.down_proj", |
| "shape": [ |
| 5120, |
| 17408 |
| ], |
| "delta_relative_norm": 0.022289995104074478, |
| "ranks": { |
| "8": { |
| "residual_relative_to_target": 0.006064626853913069, |
| "delta_energy_captured": 0.9259733557701111, |
| "probe_relative_error": 0.006071004550904036 |
| }, |
| "16": { |
| "residual_relative_to_target": 0.006050636991858482, |
| "delta_energy_captured": 0.9263144731521606, |
| "probe_relative_error": 0.006055327598005533 |
| }, |
| "32": { |
| "residual_relative_to_target": 0.0060300338082015514, |
| "delta_energy_captured": 0.9268154501914978, |
| "probe_relative_error": 0.006037381011992693 |
| } |
| } |
| }, |
| { |
| "module": "model.layers.42.linear_attn.out_proj", |
| "shape": [ |
| 5120, |
| 6144 |
| ], |
| "delta_relative_norm": 0.02273198962211609, |
| "ranks": { |
| "8": { |
| "residual_relative_to_target": 0.006101517006754875, |
| "delta_energy_captured": 0.9279555082321167, |
| "probe_relative_error": 0.006098469253629446 |
| }, |
| "16": { |
| "residual_relative_to_target": 0.006080562248826027, |
| "delta_energy_captured": 0.9284495115280151, |
| "probe_relative_error": 0.006075063720345497 |
| }, |
| "32": { |
| "residual_relative_to_target": 0.006045809481292963, |
| "delta_energy_captured": 0.9292650818824768, |
| "probe_relative_error": 0.006045033689588308 |
| } |
| } |
| }, |
| { |
| "module": "model.layers.41.mlp.down_proj", |
| "shape": [ |
| 5120, |
| 17408 |
| ], |
| "delta_relative_norm": 0.021448547020554543, |
| "ranks": { |
| "8": { |
| "residual_relative_to_target": 0.006042253226041794, |
| "delta_energy_captured": 0.9206399321556091, |
| "probe_relative_error": 0.006040771026164293 |
| }, |
| "16": { |
| "residual_relative_to_target": 0.006028073374181986, |
| "delta_energy_captured": 0.9210119843482971, |
| "probe_relative_error": 0.0060270256362855434 |
| }, |
| "32": { |
| "residual_relative_to_target": 0.00600754888728261, |
| "delta_energy_captured": 0.9215489625930786, |
| "probe_relative_error": 0.006009232718497515 |
| } |
| } |
| }, |
| { |
| "module": "model.layers.41.linear_attn.out_proj", |
| "shape": [ |
| 5120, |
| 6144 |
| ], |
| "delta_relative_norm": 0.021758820861577988, |
| "ranks": { |
| "8": { |
| "residual_relative_to_target": 0.006061953026801348, |
| "delta_energy_captured": 0.9223834276199341, |
| "probe_relative_error": 0.006062312051653862 |
| }, |
| "16": { |
| "residual_relative_to_target": 0.006044324953109026, |
| "delta_energy_captured": 0.9228341579437256, |
| "probe_relative_error": 0.006046661175787449 |
| }, |
| "32": { |
| "residual_relative_to_target": 0.006013399921357632, |
| "delta_energy_captured": 0.9236217737197876, |
| "probe_relative_error": 0.006019572261720896 |
| } |
| } |
| }, |
| { |
| "module": "model.layers.40.mlp.down_proj", |
| "shape": [ |
| 5120, |
| 17408 |
| ], |
| "delta_relative_norm": 0.020844709128141403, |
| "ranks": { |
| "8": { |
| "residual_relative_to_target": 0.006030261982232332, |
| "delta_energy_captured": 0.9163086414337158, |
| "probe_relative_error": 0.006053948774933815 |
| }, |
| "16": { |
| "residual_relative_to_target": 0.006015774793922901, |
| "delta_energy_captured": 0.9167102575302124, |
| "probe_relative_error": 0.006038118619471788 |
| }, |
| "32": { |
| "residual_relative_to_target": 0.005995276849716902, |
| "delta_energy_captured": 0.917276918888092, |
| "probe_relative_error": 0.006015749182552099 |
| } |
| } |
| }, |
| { |
| "module": "model.layers.40.linear_attn.out_proj", |
| "shape": [ |
| 5120, |
| 6144 |
| ], |
| "delta_relative_norm": 0.020884139463305473, |
| "ranks": { |
| "8": { |
| "residual_relative_to_target": 0.006066871806979179, |
| "delta_energy_captured": 0.9156089425086975, |
| "probe_relative_error": 0.006084297318011522 |
| }, |
| "16": { |
| "residual_relative_to_target": 0.006048972252756357, |
| "delta_energy_captured": 0.9161061644554138, |
| "probe_relative_error": 0.00606860127300024 |
| }, |
| "32": { |
| "residual_relative_to_target": 0.006017731502652168, |
| "delta_energy_captured": 0.9169704914093018, |
| "probe_relative_error": 0.006036028731614351 |
| } |
| } |
| }, |
| { |
| "module": "model.layers.39.mlp.down_proj", |
| "shape": [ |
| 5120, |
| 17408 |
| ], |
| "delta_relative_norm": 0.020970268175005913, |
| "ranks": { |
| "8": { |
| "residual_relative_to_target": 0.0060237399302423, |
| "delta_energy_captured": 0.9174866080284119, |
| "probe_relative_error": 0.006015862803906202 |
| }, |
| "16": { |
| "residual_relative_to_target": 0.006009472534060478, |
| "delta_energy_captured": 0.9178770184516907, |
| "probe_relative_error": 0.006003722548484802 |
| }, |
| "32": { |
| "residual_relative_to_target": 0.005988978315144777, |
| "delta_energy_captured": 0.9184361696243286, |
| "probe_relative_error": 0.005982957314699888 |
| } |
| } |
| }, |
| { |
| "module": "model.layers.39.self_attn.o_proj", |
| "shape": [ |
| 5120, |
| 6144 |
| ], |
| "delta_relative_norm": 0.02168138138949871, |
| "ranks": { |
| "8": { |
| "residual_relative_to_target": 0.006081074010580778, |
| "delta_energy_captured": 0.9213340878486633, |
| "probe_relative_error": 0.006080102175474167 |
| }, |
| "16": { |
| "residual_relative_to_target": 0.006062882021069527, |
| "delta_energy_captured": 0.9218040704727173, |
| "probe_relative_error": 0.00605847267434001 |
| }, |
| "32": { |
| "residual_relative_to_target": 0.006031112279742956, |
| "delta_energy_captured": 0.9226213693618774, |
| "probe_relative_error": 0.006028905510902405 |
| } |
| } |
| }, |
| { |
| "module": "model.layers.38.mlp.down_proj", |
| "shape": [ |
| 5120, |
| 17408 |
| ], |
| "delta_relative_norm": 0.020847443491220474, |
| "ranks": { |
| "8": { |
| "residual_relative_to_target": 0.006040703505277634, |
| "delta_energy_captured": 0.9160405397415161, |
| "probe_relative_error": 0.006071120500564575 |
| }, |
| "16": { |
| "residual_relative_to_target": 0.006026363465934992, |
| "delta_energy_captured": 0.9164386987686157, |
| "probe_relative_error": 0.006058972328901291 |
| }, |
| "32": { |
| "residual_relative_to_target": 0.006005906034260988, |
| "delta_energy_captured": 0.9170050621032715, |
| "probe_relative_error": 0.006041419226676226 |
| } |
| } |
| }, |
| { |
| "module": "model.layers.38.linear_attn.out_proj", |
| "shape": [ |
| 5120, |
| 6144 |
| ], |
| "delta_relative_norm": 0.021166682243347168, |
| "ranks": { |
| "8": { |
| "residual_relative_to_target": 0.006069451104849577, |
| "delta_energy_captured": 0.9177770018577576, |
| "probe_relative_error": 0.006059565115720034 |
| }, |
| "16": { |
| "residual_relative_to_target": 0.006051459349691868, |
| "delta_energy_captured": 0.9182637333869934, |
| "probe_relative_error": 0.006041659973561764 |
| }, |
| "32": { |
| "residual_relative_to_target": 0.006020289845764637, |
| "delta_energy_captured": 0.9191035628318787, |
| "probe_relative_error": 0.00600909860804677 |
| } |
| } |
| }, |
| { |
| "module": "model.layers.37.mlp.down_proj", |
| "shape": [ |
| 5120, |
| 17408 |
| ], |
| "delta_relative_norm": 0.021743137389421463, |
| "ranks": { |
| "8": { |
| "residual_relative_to_target": 0.006035120692104101, |
| "delta_energy_captured": 0.9229580163955688, |
| "probe_relative_error": 0.006035762373358011 |
| }, |
| "16": { |
| "residual_relative_to_target": 0.006020473316311836, |
| "delta_energy_captured": 0.9233314990997314, |
| "probe_relative_error": 0.006022346206009388 |
| }, |
| "32": { |
| "residual_relative_to_target": 0.00599979842081666, |
| "delta_energy_captured": 0.9238572120666504, |
| "probe_relative_error": 0.0060012806206941605 |
| } |
| } |
| }, |
| { |
| "module": "model.layers.37.linear_attn.out_proj", |
| "shape": [ |
| 5120, |
| 6144 |
| ], |
| "delta_relative_norm": 0.02178158611059189, |
| "ranks": { |
| "8": { |
| "residual_relative_to_target": 0.006120794452726841, |
| "delta_energy_captured": 0.9210346341133118, |
| "probe_relative_error": 0.006112252827733755 |
| }, |
| "16": { |
| "residual_relative_to_target": 0.006102846935391426, |
| "delta_energy_captured": 0.9214970469474792, |
| "probe_relative_error": 0.006095332559198141 |
| }, |
| "32": { |
| "residual_relative_to_target": 0.006070833187550306, |
| "delta_energy_captured": 0.9223185181617737, |
| "probe_relative_error": 0.006068515125662088 |
| } |
| } |
| }, |
| { |
| "module": "model.layers.36.mlp.down_proj", |
| "shape": [ |
| 5120, |
| 17408 |
| ], |
| "delta_relative_norm": 0.021916626021265984, |
| "ranks": { |
| "8": { |
| "residual_relative_to_target": 0.006045699119567871, |
| "delta_energy_captured": 0.9239068031311035, |
| "probe_relative_error": 0.006041909102350473 |
| }, |
| "16": { |
| "residual_relative_to_target": 0.006031061057001352, |
| "delta_energy_captured": 0.9242748618125916, |
| "probe_relative_error": 0.006029164418578148 |
| }, |
| "32": { |
| "residual_relative_to_target": 0.006010203622281551, |
| "delta_energy_captured": 0.9247977137565613, |
| "probe_relative_error": 0.006008479744195938 |
| } |
| } |
| }, |
| { |
| "module": "model.layers.36.linear_attn.out_proj", |
| "shape": [ |
| 5120, |
| 6144 |
| ], |
| "delta_relative_norm": 0.022344063967466354, |
| "ranks": { |
| "8": { |
| "residual_relative_to_target": 0.0060781389474868774, |
| "delta_energy_captured": 0.9260025024414062, |
| "probe_relative_error": 0.006062383763492107 |
| }, |
| "16": { |
| "residual_relative_to_target": 0.006060427986085415, |
| "delta_energy_captured": 0.9264331459999084, |
| "probe_relative_error": 0.006043384317308664 |
| }, |
| "32": { |
| "residual_relative_to_target": 0.006029218435287476, |
| "delta_energy_captured": 0.9271888732910156, |
| "probe_relative_error": 0.0060142274014651775 |
| } |
| } |
| }, |
| { |
| "module": "model.layers.35.mlp.down_proj", |
| "shape": [ |
| 5120, |
| 17408 |
| ], |
| "delta_relative_norm": 0.022243203595280647, |
| "ranks": { |
| "8": { |
| "residual_relative_to_target": 0.006073181051760912, |
| "delta_energy_captured": 0.9254516959190369, |
| "probe_relative_error": 0.006040290463715792 |
| }, |
| "16": { |
| "residual_relative_to_target": 0.006058766972273588, |
| "delta_energy_captured": 0.9258051514625549, |
| "probe_relative_error": 0.006026617716997862 |
| }, |
| "32": { |
| "residual_relative_to_target": 0.0060376450419425964, |
| "delta_energy_captured": 0.9263215661048889, |
| "probe_relative_error": 0.006009394768625498 |
| } |
| } |
| }, |
| { |
| "module": "model.layers.35.self_attn.o_proj", |
| "shape": [ |
| 5120, |
| 6144 |
| ], |
| "delta_relative_norm": 0.023746883496642113, |
| "ranks": { |
| "8": { |
| "residual_relative_to_target": 0.006093916483223438, |
| "delta_energy_captured": 0.9341463446617126, |
| "probe_relative_error": 0.0060934810899198055 |
| }, |
| "16": { |
| "residual_relative_to_target": 0.006076579913496971, |
| "delta_energy_captured": 0.9345205426216125, |
| "probe_relative_error": 0.006073717959225178 |
| }, |
| "32": { |
| "residual_relative_to_target": 0.006044890731573105, |
| "delta_energy_captured": 0.9352017045021057, |
| "probe_relative_error": 0.006036730948835611 |
| } |
| } |
| }, |
| { |
| "module": "model.layers.34.mlp.down_proj", |
| "shape": [ |
| 5120, |
| 17408 |
| ], |
| "delta_relative_norm": 0.022315077483654022, |
| "ranks": { |
| "8": { |
| "residual_relative_to_target": 0.006146028637886047, |
| "delta_energy_captured": 0.9241435527801514, |
| "probe_relative_error": 0.006127975415438414 |
| }, |
| "16": { |
| "residual_relative_to_target": 0.006132046692073345, |
| "delta_energy_captured": 0.9244883060455322, |
| "probe_relative_error": 0.006115989293903112 |
| }, |
| "32": { |
| "residual_relative_to_target": 0.0061110700480639935, |
| "delta_energy_captured": 0.9250040650367737, |
| "probe_relative_error": 0.006096362601965666 |
| } |
| } |
| }, |
| { |
| "module": "model.layers.34.linear_attn.out_proj", |
| "shape": [ |
| 5120, |
| 6144 |
| ], |
| "delta_relative_norm": 0.02435901388525963, |
| "ranks": { |
| "8": { |
| "residual_relative_to_target": 0.006124407984316349, |
| "delta_energy_captured": 0.9367866516113281, |
| "probe_relative_error": 0.00614601094275713 |
| }, |
| "16": { |
| "residual_relative_to_target": 0.006103230640292168, |
| "delta_energy_captured": 0.9372230768203735, |
| "probe_relative_error": 0.00612616166472435 |
| }, |
| "32": { |
| "residual_relative_to_target": 0.006068452727049589, |
| "delta_energy_captured": 0.9379364848136902, |
| "probe_relative_error": 0.006096133030951023 |
| } |
| } |
| }, |
| { |
| "module": "model.layers.33.mlp.down_proj", |
| "shape": [ |
| 5120, |
| 17408 |
| ], |
| "delta_relative_norm": 0.022657053545117378, |
| "ranks": { |
| "8": { |
| "residual_relative_to_target": 0.006092540454119444, |
| "delta_energy_captured": 0.9276914000511169, |
| "probe_relative_error": 0.00605214387178421 |
| }, |
| "16": { |
| "residual_relative_to_target": 0.0060781617648899555, |
| "delta_energy_captured": 0.9280322790145874, |
| "probe_relative_error": 0.0060340771451592445 |
| }, |
| "32": { |
| "residual_relative_to_target": 0.006056721787899733, |
| "delta_energy_captured": 0.9285390973091125, |
| "probe_relative_error": 0.006012708880007267 |
| } |
| } |
| }, |
| { |
| "module": "model.layers.33.linear_attn.out_proj", |
| "shape": [ |
| 5120, |
| 6144 |
| ], |
| "delta_relative_norm": 0.025525521486997604, |
| "ranks": { |
| "8": { |
| "residual_relative_to_target": 0.006090549286454916, |
| "delta_energy_captured": 0.9430670738220215, |
| "probe_relative_error": 0.0061125364154577255 |
| }, |
| "16": { |
| "residual_relative_to_target": 0.006072657648473978, |
| "delta_energy_captured": 0.943401038646698, |
| "probe_relative_error": 0.0060964045114815235 |
| }, |
| "32": { |
| "residual_relative_to_target": 0.006040295120328665, |
| "delta_energy_captured": 0.9440026879310608, |
| "probe_relative_error": 0.006064408924430609 |
| } |
| } |
| }, |
| { |
| "module": "model.layers.32.mlp.down_proj", |
| "shape": [ |
| 5120, |
| 17408 |
| ], |
| "delta_relative_norm": 0.022199278697371483, |
| "ranks": { |
| "8": { |
| "residual_relative_to_target": 0.006080122664570808, |
| "delta_energy_captured": 0.9249851703643799, |
| "probe_relative_error": 0.006090234033763409 |
| }, |
| "16": { |
| "residual_relative_to_target": 0.00606575608253479, |
| "delta_energy_captured": 0.9253392815589905, |
| "probe_relative_error": 0.006073952652513981 |
| }, |
| "32": { |
| "residual_relative_to_target": 0.006044203415513039, |
| "delta_energy_captured": 0.9258689284324646, |
| "probe_relative_error": 0.006049816962331533 |
| } |
| } |
| }, |
| { |
| "module": "model.layers.32.linear_attn.out_proj", |
| "shape": [ |
| 5120, |
| 6144 |
| ], |
| "delta_relative_norm": 0.023357966914772987, |
| "ranks": { |
| "8": { |
| "residual_relative_to_target": 0.006065764930099249, |
| "delta_energy_captured": 0.9325625896453857, |
| "probe_relative_error": 0.0060863494873046875 |
| }, |
| "16": { |
| "residual_relative_to_target": 0.00604749983176589, |
| "delta_energy_captured": 0.9329680800437927, |
| "probe_relative_error": 0.006069990806281567 |
| }, |
| "32": { |
| "residual_relative_to_target": 0.006014317274093628, |
| "delta_energy_captured": 0.9337016940116882, |
| "probe_relative_error": 0.006035847123712301 |
| } |
| } |
| }, |
| { |
| "module": "model.layers.31.mlp.down_proj", |
| "shape": [ |
| 5120, |
| 17408 |
| ], |
| "delta_relative_norm": 0.02377847209572792, |
| "ranks": { |
| "8": { |
| "residual_relative_to_target": 0.006074395962059498, |
| "delta_energy_captured": 0.9347413182258606, |
| "probe_relative_error": 0.006080507766455412 |
| }, |
| "16": { |
| "residual_relative_to_target": 0.006060002837330103, |
| "delta_energy_captured": 0.9350502490997314, |
| "probe_relative_error": 0.006071282085031271 |
| }, |
| "32": { |
| "residual_relative_to_target": 0.0060380371287465096, |
| "delta_energy_captured": 0.9355202317237854, |
| "probe_relative_error": 0.006048315204679966 |
| } |
| } |
| }, |
| { |
| "module": "model.layers.31.self_attn.o_proj", |
| "shape": [ |
| 5120, |
| 6144 |
| ], |
| "delta_relative_norm": 0.023346781730651855, |
| "ranks": { |
| "8": { |
| "residual_relative_to_target": 0.006091511342674494, |
| "delta_energy_captured": 0.9319236874580383, |
| "probe_relative_error": 0.0060823941603302956 |
| }, |
| "16": { |
| "residual_relative_to_target": 0.006072774063795805, |
| "delta_energy_captured": 0.9323418140411377, |
| "probe_relative_error": 0.006064068991690874 |
| }, |
| "32": { |
| "residual_relative_to_target": 0.00603942945599556, |
| "delta_energy_captured": 0.9330828189849854, |
| "probe_relative_error": 0.006024840287864208 |
| } |
| } |
| }, |
| { |
| "module": "model.layers.30.mlp.down_proj", |
| "shape": [ |
| 5120, |
| 17408 |
| ], |
| "delta_relative_norm": 0.02418408915400505, |
| "ranks": { |
| "8": { |
| "residual_relative_to_target": 0.006072863470762968, |
| "delta_energy_captured": 0.9369438290596008, |
| "probe_relative_error": 0.006057096179574728 |
| }, |
| "16": { |
| "residual_relative_to_target": 0.006058792117983103, |
| "delta_energy_captured": 0.9372357130050659, |
| "probe_relative_error": 0.006040608510375023 |
| }, |
| "32": { |
| "residual_relative_to_target": 0.006037280894815922, |
| "delta_energy_captured": 0.9376806020736694, |
| "probe_relative_error": 0.006020132452249527 |
| } |
| } |
| }, |
| { |
| "module": "model.layers.30.linear_attn.out_proj", |
| "shape": [ |
| 5120, |
| 6144 |
| ], |
| "delta_relative_norm": 0.023017892614006996, |
| "ranks": { |
| "8": { |
| "residual_relative_to_target": 0.006092648953199387, |
| "delta_energy_captured": 0.9299381971359253, |
| "probe_relative_error": 0.006104538217186928 |
| }, |
| "16": { |
| "residual_relative_to_target": 0.006072753109037876, |
| "delta_energy_captured": 0.9303950667381287, |
| "probe_relative_error": 0.006088663823902607 |
| }, |
| "32": { |
| "residual_relative_to_target": 0.006038023624569178, |
| "delta_energy_captured": 0.9311888813972473, |
| "probe_relative_error": 0.006058224011212587 |
| } |
| } |
| }, |
| { |
| "module": "model.layers.29.mlp.down_proj", |
| "shape": [ |
| 5120, |
| 17408 |
| ], |
| "delta_relative_norm": 0.024167638272047043, |
| "ranks": { |
| "8": { |
| "residual_relative_to_target": 0.0060697998851537704, |
| "delta_energy_captured": 0.9369216561317444, |
| "probe_relative_error": 0.006075077224522829 |
| }, |
| "16": { |
| "residual_relative_to_target": 0.006055579520761967, |
| "delta_energy_captured": 0.9372168779373169, |
| "probe_relative_error": 0.006063246633857489 |
| }, |
| "32": { |
| "residual_relative_to_target": 0.006033545825630426, |
| "delta_energy_captured": 0.9376729130744934, |
| "probe_relative_error": 0.006041319575160742 |
| } |
| } |
| }, |
| { |
| "module": "model.layers.29.linear_attn.out_proj", |
| "shape": [ |
| 5120, |
| 6144 |
| ], |
| "delta_relative_norm": 0.024435250088572502, |
| "ranks": { |
| "8": { |
| "residual_relative_to_target": 0.006068781483918428, |
| "delta_energy_captured": 0.9383164644241333, |
| "probe_relative_error": 0.0060287355445325375 |
| }, |
| "16": { |
| "residual_relative_to_target": 0.006050553172826767, |
| "delta_energy_captured": 0.9386864304542542, |
| "probe_relative_error": 0.006010016892105341 |
| }, |
| "32": { |
| "residual_relative_to_target": 0.006017674691975117, |
| "delta_energy_captured": 0.939350962638855, |
| "probe_relative_error": 0.005975026171654463 |
| } |
| } |
| }, |
| { |
| "module": "model.layers.28.mlp.down_proj", |
| "shape": [ |
| 5120, |
| 17408 |
| ], |
| "delta_relative_norm": 0.024680564180016518, |
| "ranks": { |
| "8": { |
| "residual_relative_to_target": 0.006069253198802471, |
| "delta_energy_captured": 0.939527153968811, |
| "probe_relative_error": 0.006070159375667572 |
| }, |
| "16": { |
| "residual_relative_to_target": 0.006055272184312344, |
| "delta_energy_captured": 0.9398054480552673, |
| "probe_relative_error": 0.006056688725948334 |
| }, |
| "32": { |
| "residual_relative_to_target": 0.0060339197516441345, |
| "delta_energy_captured": 0.9402292370796204, |
| "probe_relative_error": 0.006036709062755108 |
| } |
| } |
| }, |
| { |
| "module": "model.layers.28.linear_attn.out_proj", |
| "shape": [ |
| 5120, |
| 6144 |
| ], |
| "delta_relative_norm": 0.024254897609353065, |
| "ranks": { |
| "8": { |
| "residual_relative_to_target": 0.006076928228139877, |
| "delta_energy_captured": 0.9372274875640869, |
| "probe_relative_error": 0.006075568031519651 |
| }, |
| "16": { |
| "residual_relative_to_target": 0.006058897357434034, |
| "delta_energy_captured": 0.9375994801521301, |
| "probe_relative_error": 0.006057745311409235 |
| }, |
| "32": { |
| "residual_relative_to_target": 0.006026200018823147, |
| "delta_energy_captured": 0.938271164894104, |
| "probe_relative_error": 0.006025613751262426 |
| } |
| } |
| }, |
| { |
| "module": "model.layers.27.mlp.down_proj", |
| "shape": [ |
| 5120, |
| 17408 |
| ], |
| "delta_relative_norm": 0.024300511926412582, |
| "ranks": { |
| "8": { |
| "residual_relative_to_target": 0.0060642254538834095, |
| "delta_energy_captured": 0.9377241134643555, |
| "probe_relative_error": 0.006078042555600405 |
| }, |
| "16": { |
| "residual_relative_to_target": 0.00604989705607295, |
| "delta_energy_captured": 0.9380180835723877, |
| "probe_relative_error": 0.006064364220947027 |
| }, |
| "32": { |
| "residual_relative_to_target": 0.006028610281646252, |
| "delta_energy_captured": 0.9384534955024719, |
| "probe_relative_error": 0.006044549401849508 |
| } |
| } |
| }, |
| { |
| "module": "model.layers.27.self_attn.o_proj", |
| "shape": [ |
| 5120, |
| 6144 |
| ], |
| "delta_relative_norm": 0.024306191131472588, |
| "ranks": { |
| "8": { |
| "residual_relative_to_target": 0.006086335983127356, |
| "delta_energy_captured": 0.9372984766960144, |
| "probe_relative_error": 0.006093572359532118 |
| }, |
| "16": { |
| "residual_relative_to_target": 0.006067522335797548, |
| "delta_energy_captured": 0.9376855492591858, |
| "probe_relative_error": 0.006069639697670937 |
| }, |
| "32": { |
| "residual_relative_to_target": 0.0060343570075929165, |
| "delta_energy_captured": 0.9383648633956909, |
| "probe_relative_error": 0.006034309044480324 |
| } |
| } |
| }, |
| { |
| "module": "model.layers.26.mlp.down_proj", |
| "shape": [ |
| 5120, |
| 17408 |
| ], |
| "delta_relative_norm": 0.024375613778829575, |
| "ranks": { |
| "8": { |
| "residual_relative_to_target": 0.006068732123821974, |
| "delta_energy_captured": 0.9380152821540833, |
| "probe_relative_error": 0.0060697924345731735 |
| }, |
| "16": { |
| "residual_relative_to_target": 0.006054791621863842, |
| "delta_energy_captured": 0.9382997155189514, |
| "probe_relative_error": 0.006054124794900417 |
| }, |
| "32": { |
| "residual_relative_to_target": 0.006034021731466055, |
| "delta_energy_captured": 0.9387222528457642, |
| "probe_relative_error": 0.0060351537540555 |
| } |
| } |
| }, |
| { |
| "module": "model.layers.26.linear_attn.out_proj", |
| "shape": [ |
| 5120, |
| 6144 |
| ], |
| "delta_relative_norm": 0.024777619168162346, |
| "ranks": { |
| "8": { |
| "residual_relative_to_target": 0.006096011959016323, |
| "delta_energy_captured": 0.9394697546958923, |
| "probe_relative_error": 0.006109443958848715 |
| }, |
| "16": { |
| "residual_relative_to_target": 0.006076372694224119, |
| "delta_energy_captured": 0.93985915184021, |
| "probe_relative_error": 0.006087914574891329 |
| }, |
| "32": { |
| "residual_relative_to_target": 0.006042290944606066, |
| "delta_energy_captured": 0.9405319094657898, |
| "probe_relative_error": 0.006055893842130899 |
| } |
| } |
| }, |
| { |
| "module": "model.layers.25.mlp.down_proj", |
| "shape": [ |
| 5120, |
| 17408 |
| ], |
| "delta_relative_norm": 0.024253619834780693, |
| "ranks": { |
| "8": { |
| "residual_relative_to_target": 0.006057672202587128, |
| "delta_energy_captured": 0.9376181364059448, |
| "probe_relative_error": 0.006024549249559641 |
| }, |
| "16": { |
| "residual_relative_to_target": 0.006043728906661272, |
| "delta_energy_captured": 0.937904953956604, |
| "probe_relative_error": 0.00601200619712472 |
| }, |
| "32": { |
| "residual_relative_to_target": 0.006023138761520386, |
| "delta_energy_captured": 0.9383273720741272, |
| "probe_relative_error": 0.005992107558995485 |
| } |
| } |
| }, |
| { |
| "module": "model.layers.25.linear_attn.out_proj", |
| "shape": [ |
| 5120, |
| 6144 |
| ], |
| "delta_relative_norm": 0.024274000898003578, |
| "ranks": { |
| "8": { |
| "residual_relative_to_target": 0.006065714173018932, |
| "delta_energy_captured": 0.9375573992729187, |
| "probe_relative_error": 0.00602920725941658 |
| }, |
| "16": { |
| "residual_relative_to_target": 0.006047368049621582, |
| "delta_energy_captured": 0.9379345178604126, |
| "probe_relative_error": 0.006012463476508856 |
| }, |
| "32": { |
| "residual_relative_to_target": 0.006015493534505367, |
| "delta_energy_captured": 0.9385870695114136, |
| "probe_relative_error": 0.00597974332049489 |
| } |
| } |
| }, |
| { |
| "module": "model.layers.24.mlp.down_proj", |
| "shape": [ |
| 5120, |
| 17408 |
| ], |
| "delta_relative_norm": 0.023362569510936737, |
| "ranks": { |
| "8": { |
| "residual_relative_to_target": 0.006049581803381443, |
| "delta_energy_captured": 0.9329483509063721, |
| "probe_relative_error": 0.006024221424013376 |
| }, |
| "16": { |
| "residual_relative_to_target": 0.006035591010004282, |
| "delta_energy_captured": 0.9332581162452698, |
| "probe_relative_error": 0.006011801306158304 |
| }, |
| "32": { |
| "residual_relative_to_target": 0.006015092600136995, |
| "delta_energy_captured": 0.9337106943130493, |
| "probe_relative_error": 0.005993457976728678 |
| } |
| } |
| }, |
| { |
| "module": "model.layers.24.linear_attn.out_proj", |
| "shape": [ |
| 5120, |
| 6144 |
| ], |
| "delta_relative_norm": 0.024083241820335388, |
| "ranks": { |
| "8": { |
| "residual_relative_to_target": 0.006072258111089468, |
| "delta_energy_captured": 0.9364273548126221, |
| "probe_relative_error": 0.006066437810659409 |
| }, |
| "16": { |
| "residual_relative_to_target": 0.0060536982491612434, |
| "delta_energy_captured": 0.9368153214454651, |
| "probe_relative_error": 0.006046985741704702 |
| }, |
| "32": { |
| "residual_relative_to_target": 0.006021730601787567, |
| "delta_energy_captured": 0.9374809265136719, |
| "probe_relative_error": 0.006014283746480942 |
| } |
| } |
| }, |
| { |
| "module": "model.layers.23.mlp.down_proj", |
| "shape": [ |
| 5120, |
| 17408 |
| ], |
| "delta_relative_norm": 0.021976280957460403, |
| "ranks": { |
| "8": { |
| "residual_relative_to_target": 0.006038030609488487, |
| "delta_energy_captured": 0.9245112538337708, |
| "probe_relative_error": 0.00608322536572814 |
| }, |
| "16": { |
| "residual_relative_to_target": 0.006023796740919352, |
| "delta_energy_captured": 0.9248667359352112, |
| "probe_relative_error": 0.006067873910069466 |
| }, |
| "32": { |
| "residual_relative_to_target": 0.006003255490213633, |
| "delta_energy_captured": 0.9253782629966736, |
| "probe_relative_error": 0.00604618526995182 |
| } |
| } |
| }, |
| { |
| "module": "model.layers.23.self_attn.o_proj", |
| "shape": [ |
| 5120, |
| 6144 |
| ], |
| "delta_relative_norm": 0.02459094114601612, |
| "ranks": { |
| "8": { |
| "residual_relative_to_target": 0.006089568603783846, |
| "delta_energy_captured": 0.9386770725250244, |
| "probe_relative_error": 0.006102755665779114 |
| }, |
| "16": { |
| "residual_relative_to_target": 0.0060713873244822025, |
| "delta_energy_captured": 0.9390427470207214, |
| "probe_relative_error": 0.006082585081458092 |
| }, |
| "32": { |
| "residual_relative_to_target": 0.0060394457541406155, |
| "delta_energy_captured": 0.9396824240684509, |
| "probe_relative_error": 0.0060536605305969715 |
| } |
| } |
| }, |
| { |
| "module": "model.layers.22.mlp.down_proj", |
| "shape": [ |
| 5120, |
| 17408 |
| ], |
| "delta_relative_norm": 0.021504782140254974, |
| "ranks": { |
| "8": { |
| "residual_relative_to_target": 0.006072333548218012, |
| "delta_energy_captured": 0.9202664494514465, |
| "probe_relative_error": 0.006064536981284618 |
| }, |
| "16": { |
| "residual_relative_to_target": 0.006050500087440014, |
| "delta_energy_captured": 0.9208387732505798, |
| "probe_relative_error": 0.006038529332727194 |
| }, |
| "32": { |
| "residual_relative_to_target": 0.006023169029504061, |
| "delta_energy_captured": 0.9215523600578308, |
| "probe_relative_error": 0.006012582685798407 |
| } |
| } |
| }, |
| { |
| "module": "model.layers.22.linear_attn.out_proj", |
| "shape": [ |
| 5120, |
| 6144 |
| ], |
| "delta_relative_norm": 0.022083798423409462, |
| "ranks": { |
| "8": { |
| "residual_relative_to_target": 0.006067238748073578, |
| "delta_energy_captured": 0.9245195388793945, |
| "probe_relative_error": 0.00604527723044157 |
| }, |
| "16": { |
| "residual_relative_to_target": 0.0060494886711239815, |
| "delta_energy_captured": 0.9249604940414429, |
| "probe_relative_error": 0.006027002353221178 |
| }, |
| "32": { |
| "residual_relative_to_target": 0.006018282845616341, |
| "delta_energy_captured": 0.9257327318191528, |
| "probe_relative_error": 0.005999112967401743 |
| } |
| } |
| }, |
| { |
| "module": "model.layers.21.mlp.down_proj", |
| "shape": [ |
| 5120, |
| 17408 |
| ], |
| "delta_relative_norm": 0.021827833727002144, |
| "ranks": { |
| "8": { |
| "residual_relative_to_target": 0.006038367282599211, |
| "delta_energy_captured": 0.9234724640846252, |
| "probe_relative_error": 0.005995467305183411 |
| }, |
| "16": { |
| "residual_relative_to_target": 0.006024343892931938, |
| "delta_energy_captured": 0.9238275289535522, |
| "probe_relative_error": 0.005979957990348339 |
| }, |
| "32": { |
| "residual_relative_to_target": 0.006004046648740768, |
| "delta_energy_captured": 0.9243398904800415, |
| "probe_relative_error": 0.005958807189017534 |
| } |
| } |
| }, |
| { |
| "module": "model.layers.21.linear_attn.out_proj", |
| "shape": [ |
| 5120, |
| 6144 |
| ], |
| "delta_relative_norm": 0.02285146340727806, |
| "ranks": { |
| "8": { |
| "residual_relative_to_target": 0.00608475599437952, |
| "delta_energy_captured": 0.9290980100631714, |
| "probe_relative_error": 0.006055457983165979 |
| }, |
| "16": { |
| "residual_relative_to_target": 0.006066649686545134, |
| "delta_energy_captured": 0.9295193552970886, |
| "probe_relative_error": 0.006038536783307791 |
| }, |
| "32": { |
| "residual_relative_to_target": 0.006035485304892063, |
| "delta_energy_captured": 0.9302416443824768, |
| "probe_relative_error": 0.006008730735629797 |
| } |
| } |
| }, |
| { |
| "module": "model.layers.20.mlp.down_proj", |
| "shape": [ |
| 5120, |
| 17408 |
| ], |
| "delta_relative_norm": 0.021813180297613144, |
| "ranks": { |
| "8": { |
| "residual_relative_to_target": 0.006039663217961788, |
| "delta_energy_captured": 0.9233367443084717, |
| "probe_relative_error": 0.006016040686517954 |
| }, |
| "16": { |
| "residual_relative_to_target": 0.006025310605764389, |
| "delta_energy_captured": 0.9237006306648254, |
| "probe_relative_error": 0.0059998370707035065 |
| }, |
| "32": { |
| "residual_relative_to_target": 0.006004769820719957, |
| "delta_energy_captured": 0.9242199659347534, |
| "probe_relative_error": 0.005978855304419994 |
| } |
| } |
| }, |
| { |
| "module": "model.layers.20.linear_attn.out_proj", |
| "shape": [ |
| 5120, |
| 6144 |
| ], |
| "delta_relative_norm": 0.0237997155636549, |
| "ranks": { |
| "8": { |
| "residual_relative_to_target": 0.00606476841494441, |
| "delta_energy_captured": 0.9350640773773193, |
| "probe_relative_error": 0.006062047556042671 |
| }, |
| "16": { |
| "residual_relative_to_target": 0.006047225557267666, |
| "delta_energy_captured": 0.9354392290115356, |
| "probe_relative_error": 0.006042941939085722 |
| }, |
| "32": { |
| "residual_relative_to_target": 0.006016072351485491, |
| "delta_energy_captured": 0.9361026883125305, |
| "probe_relative_error": 0.0060104308649897575 |
| } |
| } |
| }, |
| { |
| "module": "model.layers.19.mlp.down_proj", |
| "shape": [ |
| 5120, |
| 17408 |
| ], |
| "delta_relative_norm": 0.022161902859807014, |
| "ranks": { |
| "8": { |
| "residual_relative_to_target": 0.006053135730326176, |
| "delta_energy_captured": 0.9253986477851868, |
| "probe_relative_error": 0.006019380409270525 |
| }, |
| "16": { |
| "residual_relative_to_target": 0.00603879988193512, |
| "delta_energy_captured": 0.9257515668869019, |
| "probe_relative_error": 0.006007079500705004 |
| }, |
| "32": { |
| "residual_relative_to_target": 0.006018429063260555, |
| "delta_energy_captured": 0.9262516498565674, |
| "probe_relative_error": 0.0059866951778531075 |
| } |
| } |
| }, |
| { |
| "module": "model.layers.19.self_attn.o_proj", |
| "shape": [ |
| 5120, |
| 6144 |
| ], |
| "delta_relative_norm": 0.02385675348341465, |
| "ranks": { |
| "8": { |
| "residual_relative_to_target": 0.0061067258939146996, |
| "delta_energy_captured": 0.934476912021637, |
| "probe_relative_error": 0.006082329899072647 |
| }, |
| "16": { |
| "residual_relative_to_target": 0.006087568122893572, |
| "delta_energy_captured": 0.9348874092102051, |
| "probe_relative_error": 0.006062052678316832 |
| }, |
| "32": { |
| "residual_relative_to_target": 0.00605359673500061, |
| "delta_energy_captured": 0.9356120824813843, |
| "probe_relative_error": 0.00603104941546917 |
| } |
| } |
| }, |
| { |
| "module": "model.layers.18.mlp.down_proj", |
| "shape": [ |
| 5120, |
| 17408 |
| ], |
| "delta_relative_norm": 0.023070180788636208, |
| "ranks": { |
| "8": { |
| "residual_relative_to_target": 0.006130789872258902, |
| "delta_energy_captured": 0.9293794631958008, |
| "probe_relative_error": 0.0060980236157774925 |
| }, |
| "16": { |
| "residual_relative_to_target": 0.006104371510446072, |
| "delta_energy_captured": 0.929986834526062, |
| "probe_relative_error": 0.006068575661629438 |
| }, |
| "32": { |
| "residual_relative_to_target": 0.006074471864849329, |
| "delta_energy_captured": 0.930670976638794, |
| "probe_relative_error": 0.006038994528353214 |
| } |
| } |
| }, |
| { |
| "module": "model.layers.18.linear_attn.out_proj", |
| "shape": [ |
| 5120, |
| 6144 |
| ], |
| "delta_relative_norm": 0.025940420106053352, |
| "ranks": { |
| "8": { |
| "residual_relative_to_target": 0.006097191479057074, |
| "delta_energy_captured": 0.9447534084320068, |
| "probe_relative_error": 0.006080218590795994 |
| }, |
| "16": { |
| "residual_relative_to_target": 0.006074738688766956, |
| "delta_energy_captured": 0.9451595544815063, |
| "probe_relative_error": 0.006060102954506874 |
| }, |
| "32": { |
| "residual_relative_to_target": 0.006038224324584007, |
| "delta_energy_captured": 0.9458168148994446, |
| "probe_relative_error": 0.006025283597409725 |
| } |
| } |
| }, |
| { |
| "module": "model.layers.17.mlp.down_proj", |
| "shape": [ |
| 5120, |
| 17408 |
| ], |
| "delta_relative_norm": 0.024629604071378708, |
| "ranks": { |
| "8": { |
| "residual_relative_to_target": 0.006067629903554916, |
| "delta_energy_captured": 0.9393091201782227, |
| "probe_relative_error": 0.0060760448686778545 |
| }, |
| "16": { |
| "residual_relative_to_target": 0.006053771357983351, |
| "delta_energy_captured": 0.9395860433578491, |
| "probe_relative_error": 0.0060633281245827675 |
| }, |
| "32": { |
| "residual_relative_to_target": 0.006033207289874554, |
| "delta_energy_captured": 0.9399958252906799, |
| "probe_relative_error": 0.006042406428605318 |
| } |
| } |
| }, |
| { |
| "module": "model.layers.17.linear_attn.out_proj", |
| "shape": [ |
| 5120, |
| 6144 |
| ], |
| "delta_relative_norm": 0.029034972190856934, |
| "ranks": { |
| "8": { |
| "residual_relative_to_target": 0.006091829389333725, |
| "delta_energy_captured": 0.9559797048568726, |
| "probe_relative_error": 0.0060930377803742886 |
| }, |
| "16": { |
| "residual_relative_to_target": 0.006073659285902977, |
| "delta_energy_captured": 0.9562419652938843, |
| "probe_relative_error": 0.006077329162508249 |
| }, |
| "32": { |
| "residual_relative_to_target": 0.006041647866368294, |
| "delta_energy_captured": 0.9567019939422607, |
| "probe_relative_error": 0.0060445102863013744 |
| } |
| } |
| }, |
| { |
| "module": "model.layers.16.mlp.down_proj", |
| "shape": [ |
| 5120, |
| 17408 |
| ], |
| "delta_relative_norm": 0.02343214862048626, |
| "ranks": { |
| "8": { |
| "residual_relative_to_target": 0.006054579745978117, |
| "delta_energy_captured": 0.933235764503479, |
| "probe_relative_error": 0.006049185525625944 |
| }, |
| "16": { |
| "residual_relative_to_target": 0.006040811073035002, |
| "delta_energy_captured": 0.933539092540741, |
| "probe_relative_error": 0.006034540943801403 |
| }, |
| "32": { |
| "residual_relative_to_target": 0.006020474247634411, |
| "delta_energy_captured": 0.9339858293533325, |
| "probe_relative_error": 0.006015355233103037 |
| } |
| } |
| }, |
| { |
| "module": "model.layers.16.linear_attn.out_proj", |
| "shape": [ |
| 5120, |
| 6144 |
| ], |
| "delta_relative_norm": 0.02820233814418316, |
| "ranks": { |
| "8": { |
| "residual_relative_to_target": 0.006078017875552177, |
| "delta_energy_captured": 0.9535534381866455, |
| "probe_relative_error": 0.006061699707061052 |
| }, |
| "16": { |
| "residual_relative_to_target": 0.0060597974807024, |
| "delta_energy_captured": 0.9538314342498779, |
| "probe_relative_error": 0.006042744964361191 |
| }, |
| "32": { |
| "residual_relative_to_target": 0.006027461960911751, |
| "delta_energy_captured": 0.954322874546051, |
| "probe_relative_error": 0.0060088844038546085 |
| } |
| } |
| }, |
| { |
| "module": "model.layers.15.mlp.down_proj", |
| "shape": [ |
| 5120, |
| 17408 |
| ], |
| "delta_relative_norm": 0.022821994498372078, |
| "ranks": { |
| "8": { |
| "residual_relative_to_target": 0.00604657270014286, |
| "delta_energy_captured": 0.9298041462898254, |
| "probe_relative_error": 0.006050425581634045 |
| }, |
| "16": { |
| "residual_relative_to_target": 0.006032528821378946, |
| "delta_energy_captured": 0.9301298260688782, |
| "probe_relative_error": 0.006037232466042042 |
| }, |
| "32": { |
| "residual_relative_to_target": 0.006012138910591602, |
| "delta_energy_captured": 0.9306013584136963, |
| "probe_relative_error": 0.0060179997235536575 |
| } |
| } |
| }, |
| { |
| "module": "model.layers.15.self_attn.o_proj", |
| "shape": [ |
| 5120, |
| 6144 |
| ], |
| "delta_relative_norm": 0.02703053131699562, |
| "ranks": { |
| "8": { |
| "residual_relative_to_target": 0.00609366362914443, |
| "delta_energy_captured": 0.9491784572601318, |
| "probe_relative_error": 0.006042139139026403 |
| }, |
| "16": { |
| "residual_relative_to_target": 0.006074550561606884, |
| "delta_energy_captured": 0.9494967460632324, |
| "probe_relative_error": 0.0060224770568311214 |
| }, |
| "32": { |
| "residual_relative_to_target": 0.006040977314114571, |
| "delta_energy_captured": 0.9500534534454346, |
| "probe_relative_error": 0.005992259830236435 |
| } |
| } |
| }, |
| { |
| "module": "model.layers.14.mlp.down_proj", |
| "shape": [ |
| 5120, |
| 17408 |
| ], |
| "delta_relative_norm": 0.022100793197751045, |
| "ranks": { |
| "8": { |
| "residual_relative_to_target": 0.006036078091710806, |
| "delta_energy_captured": 0.9254077076911926, |
| "probe_relative_error": 0.0060438006184995174 |
| }, |
| "16": { |
| "residual_relative_to_target": 0.006021819543093443, |
| "delta_energy_captured": 0.9257596731185913, |
| "probe_relative_error": 0.006028554402291775 |
| }, |
| "32": { |
| "residual_relative_to_target": 0.006001460365951061, |
| "delta_energy_captured": 0.9262608289718628, |
| "probe_relative_error": 0.006006680894643068 |
| } |
| } |
| }, |
| { |
| "module": "model.layers.14.linear_attn.out_proj", |
| "shape": [ |
| 5120, |
| 6144 |
| ], |
| "delta_relative_norm": 0.023013804107904434, |
| "ranks": { |
| "8": { |
| "residual_relative_to_target": 0.0060501997359097, |
| "delta_energy_captured": 0.9308865666389465, |
| "probe_relative_error": 0.006103380583226681 |
| }, |
| "16": { |
| "residual_relative_to_target": 0.00603087292984128, |
| "delta_energy_captured": 0.9313274025917053, |
| "probe_relative_error": 0.006081442814320326 |
| }, |
| "32": { |
| "residual_relative_to_target": 0.005998046603053808, |
| "delta_energy_captured": 0.9320729374885559, |
| "probe_relative_error": 0.006047853268682957 |
| } |
| } |
| }, |
| { |
| "module": "model.layers.13.mlp.down_proj", |
| "shape": [ |
| 5120, |
| 17408 |
| ], |
| "delta_relative_norm": 0.022425832226872444, |
| "ranks": { |
| "8": { |
| "residual_relative_to_target": 0.006038348190486431, |
| "delta_energy_captured": 0.9274997711181641, |
| "probe_relative_error": 0.0060749780386686325 |
| }, |
| "16": { |
| "residual_relative_to_target": 0.0060240947641432285, |
| "delta_energy_captured": 0.9278416633605957, |
| "probe_relative_error": 0.006057634949684143 |
| }, |
| "32": { |
| "residual_relative_to_target": 0.006003596819937229, |
| "delta_energy_captured": 0.9283319115638733, |
| "probe_relative_error": 0.006034675519913435 |
| } |
| } |
| }, |
| { |
| "module": "model.layers.13.linear_attn.out_proj", |
| "shape": [ |
| 5120, |
| 6144 |
| ], |
| "delta_relative_norm": 0.025106720626354218, |
| "ranks": { |
| "8": { |
| "residual_relative_to_target": 0.006043462548404932, |
| "delta_energy_captured": 0.9420582056045532, |
| "probe_relative_error": 0.005998039152473211 |
| }, |
| "16": { |
| "residual_relative_to_target": 0.006025892682373524, |
| "delta_energy_captured": 0.9423946738243103, |
| "probe_relative_error": 0.005979448091238737 |
| }, |
| "32": { |
| "residual_relative_to_target": 0.005994727369397879, |
| "delta_energy_captured": 0.9429889917373657, |
| "probe_relative_error": 0.00594943854957819 |
| } |
| } |
| }, |
| { |
| "module": "model.layers.12.mlp.down_proj", |
| "shape": [ |
| 5120, |
| 17408 |
| ], |
| "delta_relative_norm": 0.022239768877625465, |
| "ranks": { |
| "8": { |
| "residual_relative_to_target": 0.006037112325429916, |
| "delta_energy_captured": 0.9263117909431458, |
| "probe_relative_error": 0.006053021643310785 |
| }, |
| "16": { |
| "residual_relative_to_target": 0.006022987887263298, |
| "delta_energy_captured": 0.926656186580658, |
| "probe_relative_error": 0.006039791274815798 |
| }, |
| "32": { |
| "residual_relative_to_target": 0.00600282521918416, |
| "delta_energy_captured": 0.9271464347839355, |
| "probe_relative_error": 0.006018165498971939 |
| } |
| } |
| }, |
| { |
| "module": "model.layers.12.linear_attn.out_proj", |
| "shape": [ |
| 5120, |
| 6144 |
| ], |
| "delta_relative_norm": 0.024097878485918045, |
| "ranks": { |
| "8": { |
| "residual_relative_to_target": 0.006034597754478455, |
| "delta_energy_captured": 0.9372896552085876, |
| "probe_relative_error": 0.006013261619955301 |
| }, |
| "16": { |
| "residual_relative_to_target": 0.006016909144818783, |
| "delta_energy_captured": 0.9376567602157593, |
| "probe_relative_error": 0.0059937983751297 |
| }, |
| "32": { |
| "residual_relative_to_target": 0.005985576193779707, |
| "delta_energy_captured": 0.9383043646812439, |
| "probe_relative_error": 0.00596481841057539 |
| } |
| } |
| }, |
| { |
| "module": "model.layers.11.mlp.down_proj", |
| "shape": [ |
| 5120, |
| 17408 |
| ], |
| "delta_relative_norm": 0.02259434014558792, |
| "ranks": { |
| "8": { |
| "residual_relative_to_target": 0.006034224294126034, |
| "delta_energy_captured": 0.9286746978759766, |
| "probe_relative_error": 0.00604576850309968 |
| }, |
| "16": { |
| "residual_relative_to_target": 0.00602027727290988, |
| "delta_energy_captured": 0.9290040135383606, |
| "probe_relative_error": 0.006031317170709372 |
| }, |
| "32": { |
| "residual_relative_to_target": 0.006000080611556768, |
| "delta_energy_captured": 0.9294795989990234, |
| "probe_relative_error": 0.006006903015077114 |
| } |
| } |
| }, |
| { |
| "module": "model.layers.11.self_attn.o_proj", |
| "shape": [ |
| 5120, |
| 6144 |
| ], |
| "delta_relative_norm": 0.025072524324059486, |
| "ranks": { |
| "8": { |
| "residual_relative_to_target": 0.006019134074449539, |
| "delta_energy_captured": 0.942366898059845, |
| "probe_relative_error": 0.005995700601488352 |
| }, |
| "16": { |
| "residual_relative_to_target": 0.0059996191412210464, |
| "delta_energy_captured": 0.9427400231361389, |
| "probe_relative_error": 0.005978513974696398 |
| }, |
| "32": { |
| "residual_relative_to_target": 0.005966710392385721, |
| "delta_energy_captured": 0.9433664679527283, |
| "probe_relative_error": 0.005948569625616074 |
| } |
| } |
| }, |
| { |
| "module": "model.layers.10.mlp.down_proj", |
| "shape": [ |
| 5120, |
| 17408 |
| ], |
| "delta_relative_norm": 0.021959984675049782, |
| "ranks": { |
| "8": { |
| "residual_relative_to_target": 0.006046056281775236, |
| "delta_energy_captured": 0.9241980314254761, |
| "probe_relative_error": 0.0060799033381044865 |
| }, |
| "16": { |
| "residual_relative_to_target": 0.006032118108123541, |
| "delta_energy_captured": 0.9245471358299255, |
| "probe_relative_error": 0.006066825240850449 |
| }, |
| "32": { |
| "residual_relative_to_target": 0.006012010388076305, |
| "delta_energy_captured": 0.925049364566803, |
| "probe_relative_error": 0.0060454779304564 |
| } |
| } |
| }, |
| { |
| "module": "model.layers.10.linear_attn.out_proj", |
| "shape": [ |
| 5120, |
| 6144 |
| ], |
| "delta_relative_norm": 0.025852782651782036, |
| "ranks": { |
| "8": { |
| "residual_relative_to_target": 0.006047027185559273, |
| "delta_energy_captured": 0.945289671421051, |
| "probe_relative_error": 0.0060429624281823635 |
| }, |
| "16": { |
| "residual_relative_to_target": 0.006027723662555218, |
| "delta_energy_captured": 0.9456384181976318, |
| "probe_relative_error": 0.006024111993610859 |
| }, |
| "32": { |
| "residual_relative_to_target": 0.005995101761072874, |
| "delta_energy_captured": 0.9462252259254456, |
| "probe_relative_error": 0.005992739927023649 |
| } |
| } |
| }, |
| { |
| "module": "model.layers.9.mlp.down_proj", |
| "shape": [ |
| 5120, |
| 17408 |
| ], |
| "delta_relative_norm": 0.022012993693351746, |
| "ranks": { |
| "8": { |
| "residual_relative_to_target": 0.006033116020262241, |
| "delta_energy_captured": 0.9248852729797363, |
| "probe_relative_error": 0.006004780996590853 |
| }, |
| "16": { |
| "residual_relative_to_target": 0.006018947809934616, |
| "delta_energy_captured": 0.9252376556396484, |
| "probe_relative_error": 0.005990602076053619 |
| }, |
| "32": { |
| "residual_relative_to_target": 0.005998850800096989, |
| "delta_energy_captured": 0.9257360696792603, |
| "probe_relative_error": 0.005967405159026384 |
| } |
| } |
| }, |
| { |
| "module": "model.layers.9.linear_attn.out_proj", |
| "shape": [ |
| 5120, |
| 6144 |
| ], |
| "delta_relative_norm": 0.02678707055747509, |
| "ranks": { |
| "8": { |
| "residual_relative_to_target": 0.006034271325916052, |
| "delta_energy_captured": 0.9492542743682861, |
| "probe_relative_error": 0.0060495235957205296 |
| }, |
| "16": { |
| "residual_relative_to_target": 0.006015921011567116, |
| "delta_energy_captured": 0.9495624899864197, |
| "probe_relative_error": 0.006031757220625877 |
| }, |
| "32": { |
| "residual_relative_to_target": 0.005984538234770298, |
| "delta_energy_captured": 0.950087308883667, |
| "probe_relative_error": 0.005997255444526672 |
| } |
| } |
| }, |
| { |
| "module": "model.layers.8.mlp.down_proj", |
| "shape": [ |
| 5120, |
| 17408 |
| ], |
| "delta_relative_norm": 0.02180212363600731, |
| "ranks": { |
| "8": { |
| "residual_relative_to_target": 0.0060295783914625645, |
| "delta_energy_captured": 0.9235149621963501, |
| "probe_relative_error": 0.006019883789122105 |
| }, |
| "16": { |
| "residual_relative_to_target": 0.00601540133357048, |
| "delta_energy_captured": 0.9238742589950562, |
| "probe_relative_error": 0.00600812491029501 |
| }, |
| "32": { |
| "residual_relative_to_target": 0.005995131563395262, |
| "delta_energy_captured": 0.9243863821029663, |
| "probe_relative_error": 0.005985751282423735 |
| } |
| } |
| }, |
| { |
| "module": "model.layers.8.linear_attn.out_proj", |
| "shape": [ |
| 5120, |
| 6144 |
| ], |
| "delta_relative_norm": 0.024273570626974106, |
| "ranks": { |
| "8": { |
| "residual_relative_to_target": 0.0060166832990944386, |
| "delta_energy_captured": 0.9385606050491333, |
| "probe_relative_error": 0.006043833680450916 |
| }, |
| "16": { |
| "residual_relative_to_target": 0.005998707842081785, |
| "delta_energy_captured": 0.938927173614502, |
| "probe_relative_error": 0.006026110146194696 |
| }, |
| "32": { |
| "residual_relative_to_target": 0.005967574659734964, |
| "delta_energy_captured": 0.9395594596862793, |
| "probe_relative_error": 0.005996168591082096 |
| } |
| } |
| }, |
| { |
| "module": "model.layers.7.mlp.down_proj", |
| "shape": [ |
| 5120, |
| 17408 |
| ], |
| "delta_relative_norm": 0.02162010408937931, |
| "ranks": { |
| "8": { |
| "residual_relative_to_target": 0.006035885773599148, |
| "delta_energy_captured": 0.9220588803291321, |
| "probe_relative_error": 0.006062371656298637 |
| }, |
| "16": { |
| "residual_relative_to_target": 0.00602191174402833, |
| "delta_energy_captured": 0.9224193692207336, |
| "probe_relative_error": 0.006047965493053198 |
| }, |
| "32": { |
| "residual_relative_to_target": 0.006001768168061972, |
| "delta_energy_captured": 0.9229375123977661, |
| "probe_relative_error": 0.00602865731343627 |
| } |
| } |
| }, |
| { |
| "module": "model.layers.7.self_attn.o_proj", |
| "shape": [ |
| 5120, |
| 6144 |
| ], |
| "delta_relative_norm": 0.02745869569480419, |
| "ranks": { |
| "8": { |
| "residual_relative_to_target": 0.006018606945872307, |
| "delta_energy_captured": 0.9519567489624023, |
| "probe_relative_error": 0.005995569750666618 |
| }, |
| "16": { |
| "residual_relative_to_target": 0.005999917164444923, |
| "delta_energy_captured": 0.9522547125816345, |
| "probe_relative_error": 0.005973567720502615 |
| }, |
| "32": { |
| "residual_relative_to_target": 0.00596750108525157, |
| "delta_energy_captured": 0.9527692198753357, |
| "probe_relative_error": 0.00593875115737319 |
| } |
| } |
| }, |
| { |
| "module": "model.layers.6.mlp.down_proj", |
| "shape": [ |
| 5120, |
| 17408 |
| ], |
| "delta_relative_norm": 0.02127215638756752, |
| "ranks": { |
| "8": { |
| "residual_relative_to_target": 0.006108672358095646, |
| "delta_energy_captured": 0.9175348281860352, |
| "probe_relative_error": 0.00613442761823535 |
| }, |
| "16": { |
| "residual_relative_to_target": 0.0060856821946799755, |
| "delta_energy_captured": 0.9181543588638306, |
| "probe_relative_error": 0.006111704744398594 |
| }, |
| "32": { |
| "residual_relative_to_target": 0.006046954542398453, |
| "delta_energy_captured": 0.9191927313804626, |
| "probe_relative_error": 0.00607213145121932 |
| } |
| } |
| }, |
| { |
| "module": "model.layers.6.linear_attn.out_proj", |
| "shape": [ |
| 5120, |
| 6144 |
| ], |
| "delta_relative_norm": 0.023792579770088196, |
| "ranks": { |
| "8": { |
| "residual_relative_to_target": 0.006038430146872997, |
| "delta_energy_captured": 0.9355882406234741, |
| "probe_relative_error": 0.006044465582817793 |
| }, |
| "16": { |
| "residual_relative_to_target": 0.006018117535859346, |
| "delta_energy_captured": 0.9360209107398987, |
| "probe_relative_error": 0.006026023533195257 |
| }, |
| "32": { |
| "residual_relative_to_target": 0.005984195042401552, |
| "delta_energy_captured": 0.9367401003837585, |
| "probe_relative_error": 0.005987845826894045 |
| } |
| } |
| }, |
| { |
| "module": "model.layers.5.mlp.down_proj", |
| "shape": [ |
| 5120, |
| 17408 |
| ], |
| "delta_relative_norm": 0.021345321089029312, |
| "ranks": { |
| "8": { |
| "residual_relative_to_target": 0.00603437889367342, |
| "delta_energy_captured": 0.920079231262207, |
| "probe_relative_error": 0.005967453587800264 |
| }, |
| "16": { |
| "residual_relative_to_target": 0.006020332220941782, |
| "delta_energy_captured": 0.9204508662223816, |
| "probe_relative_error": 0.005954425781965256 |
| }, |
| "32": { |
| "residual_relative_to_target": 0.006000148132443428, |
| "delta_energy_captured": 0.9209833741188049, |
| "probe_relative_error": 0.005935054272413254 |
| } |
| } |
| }, |
| { |
| "module": "model.layers.5.linear_attn.out_proj", |
| "shape": [ |
| 5120, |
| 6144 |
| ], |
| "delta_relative_norm": 0.023078523576259613, |
| "ranks": { |
| "8": { |
| "residual_relative_to_target": 0.006015125662088394, |
| "delta_energy_captured": 0.9320681691169739, |
| "probe_relative_error": 0.006044980604201555 |
| }, |
| "16": { |
| "residual_relative_to_target": 0.005995744373649359, |
| "delta_energy_captured": 0.9325052499771118, |
| "probe_relative_error": 0.006025515031069517 |
| }, |
| "32": { |
| "residual_relative_to_target": 0.005963634233921766, |
| "delta_energy_captured": 0.9332262277603149, |
| "probe_relative_error": 0.005991884507238865 |
| } |
| } |
| }, |
| { |
| "module": "model.layers.4.mlp.down_proj", |
| "shape": [ |
| 5120, |
| 17408 |
| ], |
| "delta_relative_norm": 0.021551694720983505, |
| "ranks": { |
| "8": { |
| "residual_relative_to_target": 0.0060305059887468815, |
| "delta_energy_captured": 0.9217031002044678, |
| "probe_relative_error": 0.006035069469362497 |
| }, |
| "16": { |
| "residual_relative_to_target": 0.006016145925968885, |
| "delta_energy_captured": 0.9220755100250244, |
| "probe_relative_error": 0.0060176122933626175 |
| }, |
| "32": { |
| "residual_relative_to_target": 0.005995978135615587, |
| "delta_energy_captured": 0.9225971102714539, |
| "probe_relative_error": 0.005999614950269461 |
| } |
| } |
| }, |
| { |
| "module": "model.layers.4.linear_attn.out_proj", |
| "shape": [ |
| 5120, |
| 6144 |
| ], |
| "delta_relative_norm": 0.02401619590818882, |
| "ranks": { |
| "8": { |
| "residual_relative_to_target": 0.006006218492984772, |
| "delta_energy_captured": 0.9374548196792603, |
| "probe_relative_error": 0.0059804245829582214 |
| }, |
| "16": { |
| "residual_relative_to_target": 0.005987741053104401, |
| "delta_energy_captured": 0.9378390312194824, |
| "probe_relative_error": 0.005964599549770355 |
| }, |
| "32": { |
| "residual_relative_to_target": 0.0059563880786299706, |
| "delta_energy_captured": 0.9384883046150208, |
| "probe_relative_error": 0.0059371329843997955 |
| } |
| } |
| }, |
| { |
| "module": "model.layers.3.mlp.down_proj", |
| "shape": [ |
| 5120, |
| 17408 |
| ], |
| "delta_relative_norm": 0.022607041522860527, |
| "ranks": { |
| "8": { |
| "residual_relative_to_target": 0.006031069438904524, |
| "delta_energy_captured": 0.9288293123245239, |
| "probe_relative_error": 0.006065687630325556 |
| }, |
| "16": { |
| "residual_relative_to_target": 0.006016932427883148, |
| "delta_energy_captured": 0.9291625618934631, |
| "probe_relative_error": 0.006053399294614792 |
| }, |
| "32": { |
| "residual_relative_to_target": 0.00599654670804739, |
| "delta_energy_captured": 0.9296417236328125, |
| "probe_relative_error": 0.006030156277120113 |
| } |
| } |
| }, |
| { |
| "module": "model.layers.3.self_attn.o_proj", |
| "shape": [ |
| 5120, |
| 6144 |
| ], |
| "delta_relative_norm": 0.028329668566584587, |
| "ranks": { |
| "8": { |
| "residual_relative_to_target": 0.00593116320669651, |
| "delta_energy_captured": 0.956167459487915, |
| "probe_relative_error": 0.0058668553829193115 |
| }, |
| "16": { |
| "residual_relative_to_target": 0.005911922547966242, |
| "delta_energy_captured": 0.9564513564109802, |
| "probe_relative_error": 0.005846976302564144 |
| }, |
| "32": { |
| "residual_relative_to_target": 0.005879995413124561, |
| "delta_energy_captured": 0.9569204449653625, |
| "probe_relative_error": 0.005814167205244303 |
| } |
| } |
| }, |
| { |
| "module": "model.layers.2.mlp.down_proj", |
| "shape": [ |
| 5120, |
| 17408 |
| ], |
| "delta_relative_norm": 0.021558906883001328, |
| "ranks": { |
| "8": { |
| "residual_relative_to_target": 0.00602675648406148, |
| "delta_energy_captured": 0.921852707862854, |
| "probe_relative_error": 0.006030671298503876 |
| }, |
| "16": { |
| "residual_relative_to_target": 0.006012590602040291, |
| "delta_energy_captured": 0.9222196340560913, |
| "probe_relative_error": 0.0060173142701387405 |
| }, |
| "32": { |
| "residual_relative_to_target": 0.0059923515655100346, |
| "delta_energy_captured": 0.9227424263954163, |
| "probe_relative_error": 0.005995551124215126 |
| } |
| } |
| }, |
| { |
| "module": "model.layers.2.linear_attn.out_proj", |
| "shape": [ |
| 5120, |
| 6144 |
| ], |
| "delta_relative_norm": 0.02806072309613228, |
| "ranks": { |
| "8": { |
| "residual_relative_to_target": 0.006055373232811689, |
| "delta_energy_captured": 0.9534323811531067, |
| "probe_relative_error": 0.006050335709005594 |
| }, |
| "16": { |
| "residual_relative_to_target": 0.006034280173480511, |
| "delta_energy_captured": 0.9537562131881714, |
| "probe_relative_error": 0.006029642652720213 |
| }, |
| "32": { |
| "residual_relative_to_target": 0.005999798886477947, |
| "delta_energy_captured": 0.9542832374572754, |
| "probe_relative_error": 0.005996998865157366 |
| } |
| } |
| }, |
| { |
| "module": "model.layers.1.mlp.down_proj", |
| "shape": [ |
| 5120, |
| 17408 |
| ], |
| "delta_relative_norm": 0.01705504208803177, |
| "ranks": { |
| "8": { |
| "residual_relative_to_target": 0.0060407547280192375, |
| "delta_energy_captured": 0.8745481967926025, |
| "probe_relative_error": 0.006024964619427919 |
| }, |
| "16": { |
| "residual_relative_to_target": 0.006017753388732672, |
| "delta_energy_captured": 0.8755017518997192, |
| "probe_relative_error": 0.006006648764014244 |
| }, |
| "32": { |
| "residual_relative_to_target": 0.005986127071082592, |
| "delta_energy_captured": 0.876806914806366, |
| "probe_relative_error": 0.005975868087261915 |
| } |
| } |
| }, |
| { |
| "module": "model.layers.1.linear_attn.out_proj", |
| "shape": [ |
| 5120, |
| 6144 |
| ], |
| "delta_relative_norm": 0.021867001429200172, |
| "ranks": { |
| "8": { |
| "residual_relative_to_target": 0.005905060563236475, |
| "delta_energy_captured": 0.9270760416984558, |
| "probe_relative_error": 0.005854501388967037 |
| }, |
| "16": { |
| "residual_relative_to_target": 0.005885052494704723, |
| "delta_energy_captured": 0.9275693893432617, |
| "probe_relative_error": 0.0058358353562653065 |
| }, |
| "32": { |
| "residual_relative_to_target": 0.0058518461883068085, |
| "delta_energy_captured": 0.9283844232559204, |
| "probe_relative_error": 0.005808894522488117 |
| } |
| } |
| } |
| ] |
| } |
