diff --git "a/layer-15-part-tail-1.json" "b/layer-15-part-tail-1.json" new file mode 100644--- /dev/null +++ "b/layer-15-part-tail-1.json" @@ -0,0 +1,26086 @@ +{ + "bytes": 922818568, + "expert_reports": [ + { + "expert": 1, + "hessian": { + "down_diagonal_mean": 84.22611999511719, + "gate_up_diagonal_mean": 951.50244140625, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10602406606698199, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.024670736491873176, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10600803468692657, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.02480462562133166, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9836374453866401, + "packed_nmse_vs_bf16": 0.10619664695791371, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.024731792640755357, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10605571796727561, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.024053943731481592, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0129076729074602, + "packed_nmse_vs_bf16": 0.10580858001200108, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02815910179883072, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10595007489043493, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02920037804415417, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10599190247649169, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02854204651313757, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10598281725635471, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.028454129595776874, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197949, + "packed_nmse_vs_bf16": 0.07780773137223991, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.056515956141463174, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.07758554011188483, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.055913607671190815, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197949, + "packed_nmse_vs_bf16": 0.07792344934918484, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.0553907718222461, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.07803152443476113, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05506637385315083, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 3, + "hessian": { + "down_diagonal_mean": 51.149112701416016, + "gate_up_diagonal_mean": 926.723388671875, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10557495699995803, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.024263270528053705, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9836374453866401, + "packed_nmse_vs_bf16": 0.10585485989461646, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.024391652619063856, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10576485692753075, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.02480672254206133, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10576597205767789, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.024211823873340626, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10570410126537354, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.03080649540962875, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10586773663167603, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.030732384795057643, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10580265324346118, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.03072415779774019, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10587557190100143, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.03089099305640266, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.07596659675221401, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05580507496534697, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.07581863452746661, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05611515393652289, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.0758864085351336, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05656567390545526, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.99827255914705, + "packed_nmse_vs_bf16": 0.07552254230943822, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.056882034903571084, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 5, + "hessian": { + "down_diagonal_mean": 54.3804931640625, + "gate_up_diagonal_mean": 930.244873046875, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 1.0129076729074602, + "packed_nmse_vs_bf16": 0.1048810089683971, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.02251288862351431, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10476493483254266, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.022529637953863764, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10479733182091137, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.02282216341010157, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9836374453866401, + "packed_nmse_vs_bf16": 0.10516052898690599, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.022844748516099085, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.1050062856357066, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02911050566215861, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.1047133730232499, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.028871646429702105, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9836374453866401, + "packed_nmse_vs_bf16": 0.10505300683799643, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.028988932117945086, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.1048212794515583, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.029280821146354802, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.07600685063432229, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05543235209227889, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9836374453866401, + "packed_nmse_vs_bf16": 0.07611369424947872, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05479596290883478, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.07585635328398957, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05623461918779352, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.07625518984150795, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.055158847318902374, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 7, + "hessian": { + "down_diagonal_mean": 109.7191390991211, + "gate_up_diagonal_mean": 913.9913330078125, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10561061629446086, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.017169807153804978, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0129076729074602, + "packed_nmse_vs_bf16": 0.10533967346162551, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.017237524111156987, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10547481868180948, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.017384542459429497, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10535297313610556, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.016979918972488173, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10553785313522247, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.01769162601275053, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10519359380655535, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.017562674566858488, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10546377680677428, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.017022641629682818, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10552389852542322, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.017460624259008302, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.07966801217125798, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05042786368813729, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.07968028138925157, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05028011773000937, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.07957596881400929, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.051399592383208084, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.07940274911481493, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05198900924661994, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 9, + "hessian": { + "down_diagonal_mean": 87.315673828125, + "gate_up_diagonal_mean": 938.443359375, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10520238647047624, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.021457223988447204, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10515636516584755, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.021645138738521117, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10519383707647553, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.02248962307921455, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10532060816094213, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.021485551586646716, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.1051582542861522, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.025876529299038113, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10529389734651576, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.025470031289989682, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9836374453866401, + "packed_nmse_vs_bf16": 0.10526344561963505, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.025788664554951633, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10515758423433665, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02580271215967654, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.0785891003943206, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.0550843393417788, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.07783065374075275, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.055169255554393754, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.0773238414023449, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.056745558225511764, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9780473293593848, + "packed_nmse_vs_bf16": 0.0780961852776859, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05381864347976364, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 11, + "hessian": { + "down_diagonal_mean": 116.2898178100586, + "gate_up_diagonal_mean": 912.72509765625, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 0.9982725591470503, + "packed_nmse_vs_bf16": 0.111369023678178, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.014463489781566998, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.11179598987458031, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.014618220663919577, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9836374453866401, + "packed_nmse_vs_bf16": 0.1115572211920005, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.014827784926272979, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.1114895490621615, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.014704038631780497, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.11146123441526182, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.01702663675894874, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.11148966764312812, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.017191435285202412, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.11132711421714545, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.017351455000321956, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.11119396286218634, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.01694830055421733, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.08530453777553527, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.04300976118660203, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.08501609511204518, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.0457383197661579, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.08458119715827213, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.04659093645509082, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.08476252334053991, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.04600260632905431, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 13, + "hessian": { + "down_diagonal_mean": 42.07303237915039, + "gate_up_diagonal_mean": 904.2318115234375, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 0.99827255914705, + "packed_nmse_vs_bf16": 0.10711507816390535, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.022245681404750802, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10712553903153275, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.02142963681580675, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10715556211256955, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.022735626183948004, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.99827255914705, + "packed_nmse_vs_bf16": 0.10713496106737821, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.0231035776102565, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10715185401613965, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.028115922095587397, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10724330369344845, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.026977293467531697, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.1072027585892608, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.027980872520270792, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10715689833634225, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02863891829124628, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.07605268353166472, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05640755213212834, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.07572628552577747, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.056448952228183184, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.07548031671107645, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05863842075071977, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.0753804502851721, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05893716309204991, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 15, + "hessian": { + "down_diagonal_mean": 92.430419921875, + "gate_up_diagonal_mean": 923.5859375, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10745645225031066, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.016592436625473304, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.99827255914705, + "packed_nmse_vs_bf16": 0.10757695009081643, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.016620257772650967, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10779674689587705, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.016557335204306325, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.99827255914705, + "packed_nmse_vs_bf16": 0.10758650914021678, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.01670341962878998, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10740896899778217, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.020002209522144702, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10741803251404669, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.020440289310855328, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.1073854795410736, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02055974586381829, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10735912227979172, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.019872139955639842, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.08359488330739877, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.044311850353086116, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.08432643536667539, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.03961522795836682, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9836374453866401, + "packed_nmse_vs_bf16": 0.08367576506095838, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.04567988638684091, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.08363163114869472, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.03982053128524669, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 17, + "hessian": { + "down_diagonal_mean": 96.26081848144531, + "gate_up_diagonal_mean": 951.8980712890625, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10401023069952997, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.021180478937148237, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10381741180673215, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.020489658126325448, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10379543028883172, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.02104937280541845, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10389353020813653, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.02105152721746141, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10375039485694328, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.025135939195735875, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10379021383885746, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.024010061180628935, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10385953016948232, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.0244276841994679, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10401096165622523, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.024236082456393866, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.07670763905110434, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.055281714131877435, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.07792813654447533, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05215460284049791, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.07788548184403048, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05103690751638893, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9836374453866401, + "packed_nmse_vs_bf16": 0.07748617739216378, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.04961457745181715, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 19, + "hessian": { + "down_diagonal_mean": 86.55142211914062, + "gate_up_diagonal_mean": 941.9090576171875, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10417605356106331, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.0233793138962372, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10438053466727425, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.022919174944516247, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0073175568802046, + "packed_nmse_vs_bf16": 0.10407763545873926, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.02294735920847166, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10412370888886838, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.023179682906736828, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10413300200966745, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02754795104656457, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10423903340107593, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.027583188247162018, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10416907299524711, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.027852796614772368, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.1041113443385466, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.027568290724665243, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.0764413634263438, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05688010759997177, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.0764513068095501, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05667897455274532, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.07657459603896201, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.055838649658914746, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470503, + "packed_nmse_vs_bf16": 0.07735309523520202, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05308737491093799, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 21, + "hessian": { + "down_diagonal_mean": 87.39973449707031, + "gate_up_diagonal_mean": 936.2584228515625, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10551019042735953, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.021356168150796807, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.007317556880205, + "packed_nmse_vs_bf16": 0.10552746716368938, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.021320543387606163, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10561199042737675, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.02162833526965943, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.1056759310107399, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.021650736401990358, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0073175568802046, + "packed_nmse_vs_bf16": 0.10568348388341324, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.025849859140391007, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10563342888514547, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.025342515767638404, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10562840704568961, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.026275710296688133, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10584127819256105, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.026485781168973023, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.07963605636332455, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05276861917307375, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.07977313606337032, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05142326576374311, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.07929781079263454, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05134025415276002, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9836374453866401, + "packed_nmse_vs_bf16": 0.07928285405054009, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05315520312431824, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 23, + "hessian": { + "down_diagonal_mean": 74.96806335449219, + "gate_up_diagonal_mean": 950.7344360351562, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10685811940648604, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.020459941677859955, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10687768785755035, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.020698629816436345, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10689842751683483, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.020578912578637157, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10694187903701584, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.020916665392416434, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9836374453866401, + "packed_nmse_vs_bf16": 0.1069068791878389, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02422147552501484, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10675274766477212, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02468209122565348, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10693316451869803, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02448944654297607, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0129076729074602, + "packed_nmse_vs_bf16": 0.1067638399136586, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.024495841652014474, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9780473293593848, + "packed_nmse_vs_bf16": 0.07924215609997856, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.0513803240339618, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.07813743122897146, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.054204596382516186, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.0788255455969369, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05359744347177222, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.07902168133926239, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.049978709218970206, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 25, + "hessian": { + "down_diagonal_mean": 72.15957641601562, + "gate_up_diagonal_mean": 948.2243041992188, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.1071495973969084, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.0225989774396785, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10704850801193061, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.022325377173771876, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10708145489547771, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.02254636684178221, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10704567665791577, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.02235646883890691, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10687608714335275, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.025984093420644942, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10705728580130558, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.025723358593353216, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10703074218451178, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.025487509824771895, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.106816123394744, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.025344314607592042, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529497, + "packed_nmse_vs_bf16": 0.07739358033333549, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.056224391661247076, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.0777827328502238, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.055083020124493634, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.07813136997330598, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.053752790867211575, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.07856648705252152, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05249834127423538, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 27, + "hessian": { + "down_diagonal_mean": 98.08692932128906, + "gate_up_diagonal_mean": 933.209228515625, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.1056307623603625, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.02148313697766903, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197949, + "packed_nmse_vs_bf16": 0.10570414954994765, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.020989383946726444, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.99827255914705, + "packed_nmse_vs_bf16": 0.1055668343861527, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.02187894999281783, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10564781285419615, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.021181778898703606, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10567797700479928, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.024312560435417907, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10572100637737362, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02430365407720079, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10566170661360981, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02478467601784352, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10567883321187463, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.024769573753095565, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.07889485329998439, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05429908581117749, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.07928368146268233, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05291545863046598, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.07852292699379115, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05466543825593274, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0129076729074602, + "packed_nmse_vs_bf16": 0.0785833783661031, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05447447689266789, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 29, + "hessian": { + "down_diagonal_mean": 91.60997009277344, + "gate_up_diagonal_mean": 951.9683837890625, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10647338005257233, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.021217207664171126, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10641665117176424, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.021345814874684125, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10633148366569205, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.021473360176949962, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10654257403192689, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.02154031874766708, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9836374453866401, + "packed_nmse_vs_bf16": 0.10645792882542976, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.025072616373789824, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10623856455669989, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.025955940754026227, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9836374453866401, + "packed_nmse_vs_bf16": 0.10657191209438709, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.025468727207796655, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10636023346735539, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02558617037145537, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.07819835384415087, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05501895443730448, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925397, + "packed_nmse_vs_bf16": 0.07743389530779157, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05679695135612007, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.07783370619891625, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05583968827331129, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.07759557279665323, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.056398504678281246, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 31, + "hessian": { + "down_diagonal_mean": 67.52337646484375, + "gate_up_diagonal_mean": 945.2642822265625, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10556802346460419, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.025388750059887104, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10569750635106571, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.0258372875890532, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0129076729074602, + "packed_nmse_vs_bf16": 0.10565766851135626, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.026079970080554273, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10565320887795274, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.025694524970503944, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10563645659478357, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.03008757880205765, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10547097869719443, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.030211440243234258, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10555978401650663, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.03023371581106912, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9836374453866401, + "packed_nmse_vs_bf16": 0.10585071874498257, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02979451348607123, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.07635424469348454, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05724377452122303, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.07662483488811168, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05541561229691034, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.07621118497941178, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.058556691548472796, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.07691734088798945, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05552288741429613, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 33, + "hessian": { + "down_diagonal_mean": 74.23225402832031, + "gate_up_diagonal_mean": 911.247802734375, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10650005659871406, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.015157936958468069, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9836374453866401, + "packed_nmse_vs_bf16": 0.10640426731393407, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.015583345327663497, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.1065002325736221, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.014949095762869902, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10640567484569047, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.015438676069034355, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.99827255914705, + "packed_nmse_vs_bf16": 0.10642999815882823, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.021371564426407495, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10635726971268636, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02127024174671132, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9690023316262302, + "packed_nmse_vs_bf16": 0.10651485710981501, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.021069870241372367, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0073175568802046, + "packed_nmse_vs_bf16": 0.10627239145904024, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02153289858269788, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.08071644954774941, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.04893799785670014, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.08175540364852422, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.0427605083171864, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.08109795065407781, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.04831924093958392, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.08117724233575335, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.04836689300737747, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 35, + "hessian": { + "down_diagonal_mean": 85.55239868164062, + "gate_up_diagonal_mean": 934.489990234375, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10716714684169772, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.021811554778516956, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10717150900712381, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.021619311910342984, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.007317556880205, + "packed_nmse_vs_bf16": 0.10735148265391078, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.021562483218123138, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10733665122393471, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.021695886358388893, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.99827255914705, + "packed_nmse_vs_bf16": 0.10704593812844478, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.025801762826096687, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10684825666239178, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.025891551703734278, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10728203429469058, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.024905880392912062, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9836374453866401, + "packed_nmse_vs_bf16": 0.10710380943632454, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.025781041368609703, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.08003679445047387, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.052845869546470976, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.07968713457219045, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.052736301525012, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.0807789815442581, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.051123879443110115, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.08038217168969146, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05155638238047087, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 37, + "hessian": { + "down_diagonal_mean": 73.53243255615234, + "gate_up_diagonal_mean": 950.7189331054688, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10637595500238356, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.023084044660405093, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10648381400681575, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.023401806788353224, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10659722277018228, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.02333250655478044, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10652913813524986, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.0229122105939154, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9780473293593848, + "packed_nmse_vs_bf16": 0.10659512542925986, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02779642430460299, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.1065281093256204, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.027643082641291894, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10642558692117776, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.027861983112493712, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10658291404344893, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.027829074670357726, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.076948386398478, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05543031586129588, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.07612037755626383, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05756228155076423, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.07631134245941433, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05558847279706409, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.07704898307713759, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.052612932413035024, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 39, + "hessian": { + "down_diagonal_mean": 72.26597595214844, + "gate_up_diagonal_mean": 951.5761108398438, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 1.0073175568802046, + "packed_nmse_vs_bf16": 0.1061099736508339, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.025209019486856776, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10609574225009198, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.024568394360526494, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10609047200759351, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.025335649757284667, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10597875391254771, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.025008678914023964, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10595202403145244, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.029466231033190477, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.1061848994835648, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02902161054756026, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10605581655153146, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.0293638571524165, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.1059938210890103, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02922282866924731, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.07651171342817611, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05265871914264502, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.07671565370849123, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05753429934512201, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.0759123405648053, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05944951569786132, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.07650924073987708, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05764977839744361, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 41, + "hessian": { + "down_diagonal_mean": 87.18681335449219, + "gate_up_diagonal_mean": 943.2326049804688, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 1.0073175568802049, + "packed_nmse_vs_bf16": 0.10560081631003111, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.025881650706275572, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0073175568802046, + "packed_nmse_vs_bf16": 0.10553200361458787, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.025803912195836505, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10571643026019827, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.025762594432965503, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10555893282458215, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.025902237526485537, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.1055615134713171, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02814739873352015, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10557342443613864, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.028280309311102102, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.1055713621995232, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02844563150188993, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10528973564436209, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.028378825160929624, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.07657503927748624, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.058733925667208635, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.07663098750684237, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.058297861738117934, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.007317556880205, + "packed_nmse_vs_bf16": 0.07654338441729183, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.058545948063895153, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470503, + "packed_nmse_vs_bf16": 0.07681769951449824, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05816817487189266, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 43, + "hessian": { + "down_diagonal_mean": 69.42498016357422, + "gate_up_diagonal_mean": 950.2781982421875, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10428157500615189, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.018863254506375885, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10415233146157618, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.018626664453474738, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.1041412982703654, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.019010706125970417, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.007317556880205, + "packed_nmse_vs_bf16": 0.10418385707226177, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.01945741885446273, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10417227542911185, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.021942055468533582, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10414334074833406, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.021383368810297773, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10415806044448639, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02229950783333088, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.99827255914705, + "packed_nmse_vs_bf16": 0.10422899817500979, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.022059669564012632, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.99827255914705, + "packed_nmse_vs_bf16": 0.08034911052586449, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.042346327635915094, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.08102983889492488, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.04135431903785423, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.0798709477342832, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.04730781746521278, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.08025852033278619, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.04275588794184716, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 45, + "hessian": { + "down_diagonal_mean": 99.34165954589844, + "gate_up_diagonal_mean": 949.7115478515625, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10550912936541171, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.020657272111030363, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10548134790098175, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.020991905211142612, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10548816942686473, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.020882490082701606, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10512546574752747, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.021044389504800615, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.1054382902665314, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02457806993413919, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10530484578132898, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.024243861239465425, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10552417169628206, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.024574026517172446, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0073175568802046, + "packed_nmse_vs_bf16": 0.10546364192290505, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02449207685849112, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9836374453866401, + "packed_nmse_vs_bf16": 0.07834173305936612, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05536947765368335, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.07907719627520134, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05452111480189819, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.07879035610450362, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.054917494348829005, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.07864048791201655, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.055882108951395885, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 47, + "hessian": { + "down_diagonal_mean": 87.90596008300781, + "gate_up_diagonal_mean": 945.296630859375, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10842727074956823, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.02032693517333162, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.99827255914705, + "packed_nmse_vs_bf16": 0.10868150969979, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.020090136301779696, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10834498201633291, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.019947978061849204, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10838946938266052, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.020057952430458804, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10865474510223584, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.023560058428421668, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10840897537864293, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.022699575810385543, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10866949467923902, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02253081106428224, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10860295064320355, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02282733000432255, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.07900792073134764, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.054699112762077624, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.07907695084810704, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05365916607719808, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.07940520195709425, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05301904057893197, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.07898410218915168, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.0537664321668367, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 49, + "hessian": { + "down_diagonal_mean": 79.86009216308594, + "gate_up_diagonal_mean": 940.7034912109375, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10712764394548131, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.02034623814451598, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10723908070618104, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.020326974658472347, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10705869895227425, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.020052797148750752, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.1070095895844449, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.02014793566168176, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10689337426399542, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02320854055130763, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9836374453866401, + "packed_nmse_vs_bf16": 0.10721915853391524, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02287516496266251, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10699380974140685, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02319866863706637, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.1070534965771055, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02300346764117236, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.07912242881164477, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05063193366023748, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.08027676441060733, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05034398234634855, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.08007484231539945, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.04991378413586064, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.0803063753486394, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.04584260040391645, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 51, + "hessian": { + "down_diagonal_mean": 65.08294677734375, + "gate_up_diagonal_mean": 940.4212646484375, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10428781251851628, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.025962046047192704, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10437814580744237, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.025624953923945717, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9836374453866401, + "packed_nmse_vs_bf16": 0.10450488675825217, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.02555535277576615, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.1043739355718745, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.025908716013841964, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.1044235897229549, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.029511396627188734, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10441968568594744, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02960585248911896, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10435685566682251, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.029665688563027596, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10434073405956375, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.029925617435162533, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.07652012046707622, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05592335394305539, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.99827255914705, + "packed_nmse_vs_bf16": 0.07627729081399133, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.056031453863061055, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.07652786722856612, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05594074027181558, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.07590055983463749, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.058253546528776325, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 53, + "hessian": { + "down_diagonal_mean": 86.4132308959961, + "gate_up_diagonal_mean": 950.9166259765625, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10420799360443764, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.022061336304831373, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10397151823170475, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.021659238663097727, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.1039654940028953, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.022180069893957235, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10397160425411446, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.0213593491695391, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10402348166437048, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.025817813206419245, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10399418085745335, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.025440450090081425, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10383255899356397, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02595093644730605, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.1039308851342694, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.025588949653650606, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.07596831709459441, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05806288575738229, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.07641002124312339, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.056662613861756535, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.99827255914705, + "packed_nmse_vs_bf16": 0.07586065746009556, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05799459150653948, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.07583418699627248, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.058142650463156725, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 55, + "hessian": { + "down_diagonal_mean": 117.18891143798828, + "gate_up_diagonal_mean": 920.350341796875, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.1091694151166399, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.014144598460005224, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10909275000220978, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.014685061781512576, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.108942888259853, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.014363756227947423, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10903294849813563, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.014210383594153683, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.1090691419798354, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.015715241569959845, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10910791429109855, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.01677063323378429, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10880800490565265, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.01661753452087971, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.99827255914705, + "packed_nmse_vs_bf16": 0.10897833060554116, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.016991683948275927, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.08578179227518452, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.04505626244727175, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0073175568802046, + "packed_nmse_vs_bf16": 0.08618593421557161, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.04521253378239526, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9836374453866401, + "packed_nmse_vs_bf16": 0.08523889542224332, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.04586007312688832, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.08623953332469428, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.04618328000927776, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 57, + "hessian": { + "down_diagonal_mean": 81.76382446289062, + "gate_up_diagonal_mean": 944.5604858398438, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 0.99827255914705, + "packed_nmse_vs_bf16": 0.10838350334643192, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.01864401468457287, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10830599552139299, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.018097601550835146, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10840416930634393, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.018543543842765773, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10822930078116574, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.018360448635811565, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9780473293593848, + "packed_nmse_vs_bf16": 0.10850125966401075, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02222592970726732, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470503, + "packed_nmse_vs_bf16": 0.10841003179608054, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02238235894057982, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0129076729074602, + "packed_nmse_vs_bf16": 0.108282371288825, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.022422156753134727, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10821505201420381, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.022092466421425302, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470503, + "packed_nmse_vs_bf16": 0.08117679320740019, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.048793425624658916, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.07972805022239317, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.0512898238004253, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925397, + "packed_nmse_vs_bf16": 0.08143985036556517, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.04710552566172831, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.08073161769255276, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.049167638019874914, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 59, + "hessian": { + "down_diagonal_mean": 92.65156555175781, + "gate_up_diagonal_mean": 935.1510620117188, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10703201240659681, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.02017046792497494, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10721268242211446, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.020682674785927037, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10721835824272871, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.02043814966626905, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10732452506556328, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.020014316196724915, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10698464485419508, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02454420676392346, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10703330533930816, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02474361479295943, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470503, + "packed_nmse_vs_bf16": 0.10701665158008417, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.0247081481658113, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10717369513822106, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.024323782763410535, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.99827255914705, + "packed_nmse_vs_bf16": 0.08023612252544063, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05004623092765611, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.0806925168097492, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.050143263095420255, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.99827255914705, + "packed_nmse_vs_bf16": 0.08084899764528561, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.04792017119361804, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0073175568802049, + "packed_nmse_vs_bf16": 0.08117154683667666, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.048481702980534544, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 61, + "hessian": { + "down_diagonal_mean": 89.29368591308594, + "gate_up_diagonal_mean": 950.848876953125, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10552957094704693, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.024255982644833227, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10563678731536513, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.02429770870000524, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10573709496522875, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.024101489020471014, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10580488947318843, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.02434153690048621, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9745924476534854, + "packed_nmse_vs_bf16": 0.10578704911583861, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.027036721208257546, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9836374453866401, + "packed_nmse_vs_bf16": 0.10574361775228272, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02692781021425638, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10568094558926376, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02677666478855673, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10598340843945729, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02727114138770498, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.0757212156817326, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.06014959577661451, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.07608769246612548, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05969227898024485, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0129076729074602, + "packed_nmse_vs_bf16": 0.07605202886497242, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.059421583307160644, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.07558560950703487, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.06024857534241098, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 63, + "hessian": { + "down_diagonal_mean": 95.2347412109375, + "gate_up_diagonal_mean": 943.56103515625, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10653074065361344, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.024176460821417448, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10661760402271848, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.02418745880664133, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10662999628458326, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.024030002205649176, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10681113791779039, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.02411307057057937, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10655537467945352, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02651402329366102, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10662356558384885, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.026478360306754146, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10675678228603719, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.026685085382390943, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10669112814202716, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.026413302418844944, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0073175568802049, + "packed_nmse_vs_bf16": 0.07759157505101207, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05674454801558731, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.07765588902198249, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.056688815647164034, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470503, + "packed_nmse_vs_bf16": 0.07754376462114562, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05732580509239054, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.07804457793998472, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.050734146477172275, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 65, + "hessian": { + "down_diagonal_mean": 69.3033447265625, + "gate_up_diagonal_mean": 933.0684814453125, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10455793037861405, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.023710218837405265, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9836374453866401, + "packed_nmse_vs_bf16": 0.10495834450971847, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.023325145854462468, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10472797394106058, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.023267851209391052, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0129076729074602, + "packed_nmse_vs_bf16": 0.10470120030313675, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.02291451461335723, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0073175568802046, + "packed_nmse_vs_bf16": 0.10461711831539741, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.029821510644658128, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10457081215466171, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.029720170644489063, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10474644099436022, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.029308628648576796, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10451640247411094, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.029093165411242666, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.0780025433377655, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05445827558082346, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.07745410129334646, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05459234913687816, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.07800325352288241, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.0543124067711754, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9836374453866401, + "packed_nmse_vs_bf16": 0.07963353255951763, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.049001307379430564, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 67, + "hessian": { + "down_diagonal_mean": 70.53272247314453, + "gate_up_diagonal_mean": 947.4720458984375, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10995475880032805, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.02160729495491362, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.11000219127866857, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.02129935373377871, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.1097231565218042, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.02149371297237979, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10991988834729301, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.021464121194375033, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0129076729074602, + "packed_nmse_vs_bf16": 0.10995365222004444, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.025109905906043634, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10984834584411032, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02499024409388874, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10997833424642539, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02498779460078304, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10977728447339172, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.025041397069686132, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.0774614775101385, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.055577476287508885, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.07766536419492376, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05636734862680506, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.0773222152138923, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.057467811443005855, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.07715805475947074, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05750262255267711, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 69, + "hessian": { + "down_diagonal_mean": 108.68627166748047, + "gate_up_diagonal_mean": 943.2998046875, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10591024743194771, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.012296023731622788, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9836374453866401, + "packed_nmse_vs_bf16": 0.10590709209051781, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.012327667177915646, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.99827255914705, + "packed_nmse_vs_bf16": 0.1057819762580136, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.012535049897111263, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10588749301212452, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.012501163686271466, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10586668608742439, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.01666085587959348, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.1059139772169578, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.01689969105966149, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10582624176435118, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.016347500559860582, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9836374453866401, + "packed_nmse_vs_bf16": 0.1058044724060411, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.01631905785380173, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.08560055159280087, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.04150623333480139, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9836374453866401, + "packed_nmse_vs_bf16": 0.08502073078758747, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.04049039635763608, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.08576008800054066, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.04329384448211741, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.08511447898053688, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.040818159329369094, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 71, + "hessian": { + "down_diagonal_mean": 45.59801483154297, + "gate_up_diagonal_mean": 918.9627075195312, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10386072816001925, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.026706590431311, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10375259293093266, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.02691964112353585, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10379434167275206, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.02691907503563795, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10378242616636468, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.026624376669978765, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10388477862500997, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.03225257023172521, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.103909515942103, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.03176975071186906, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10394234856832524, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.03245663433558973, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10396955977139583, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.03221976539809751, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.99827255914705, + "packed_nmse_vs_bf16": 0.07409449533748998, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.06017218356359337, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.07416576140890238, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.06071198596764763, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.99827255914705, + "packed_nmse_vs_bf16": 0.07409983077976384, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.06075325394111211, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.07404054563603346, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.06076006069170132, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 73, + "hessian": { + "down_diagonal_mean": 52.83305740356445, + "gate_up_diagonal_mean": 925.0621337890625, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10452413251239902, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.024106228416314553, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0129076729074602, + "packed_nmse_vs_bf16": 0.10449579376408245, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.024721889571133943, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10449797382107029, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.024642830913826266, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10469398570128015, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.024626454403827498, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10421656077612179, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.030637678175161207, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10428759458756177, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.030579888503313134, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.1043493463347261, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.030675078792939917, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10450153667460914, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.03085705384024378, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.07442541861615928, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.058947859211213434, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9745924476534854, + "packed_nmse_vs_bf16": 0.07484490865015021, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.055585711513230975, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.07450006262704623, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05910878974904524, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.07451431861653567, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.06029392898698832, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 75, + "hessian": { + "down_diagonal_mean": 92.43296813964844, + "gate_up_diagonal_mean": 934.6260986328125, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 0.9836374453866401, + "packed_nmse_vs_bf16": 0.10506502238751568, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.022315491574536522, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10500164761503031, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.02262108898567572, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10499552880142887, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.022166521292947782, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197949, + "packed_nmse_vs_bf16": 0.10485760405820994, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.02252039948116589, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10477322801897618, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02733202041485831, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10474153863874638, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.027284226687429563, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10489078012910845, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02715321159405773, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10483885742760028, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.027124096589095672, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.0777087027143792, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.054165264455056636, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.07721741240076817, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.056299162955425516, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.07804535856573172, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05349595271243051, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.07765066820183598, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05630706510925328, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 77, + "hessian": { + "down_diagonal_mean": 79.07394409179688, + "gate_up_diagonal_mean": 937.4542846679688, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 1.0129076729074602, + "packed_nmse_vs_bf16": 0.10612180103403475, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.023328665754944584, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.1061553315358707, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.02287261046804203, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10628920404753127, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.0230473435893898, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10587195321989556, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.023323947222834943, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10607690858128369, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02856838634397638, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9836374453866401, + "packed_nmse_vs_bf16": 0.10616443737444564, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02807535574007142, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10599917091532973, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.027992884421272297, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10599932501356384, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.028267568157368937, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.0773890138016903, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.0558869102101412, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.99827255914705, + "packed_nmse_vs_bf16": 0.07774343916267244, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05031257355392974, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9836374453866401, + "packed_nmse_vs_bf16": 0.07726299715468862, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05310032081959466, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.07744795704267202, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05414416314864944, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 79, + "hessian": { + "down_diagonal_mean": 74.83848571777344, + "gate_up_diagonal_mean": 951.3592529296875, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10796802141136805, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.0223057849236731, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0073175568802046, + "packed_nmse_vs_bf16": 0.1079673737687931, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.022683723892790008, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10802185227092649, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.022601879216374327, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10776435809669288, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.022186748000302882, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0073175568802046, + "packed_nmse_vs_bf16": 0.10792952803883925, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.025537478390973117, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10802357844503951, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02540520170805928, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10810155950026631, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.025575991013125483, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10803606622036895, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02562126889957852, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0129076729074602, + "packed_nmse_vs_bf16": 0.07620167249039837, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05936642115317557, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.07664265552238228, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.057188193889825656, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.0761138657849886, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05894846967439715, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470503, + "packed_nmse_vs_bf16": 0.0764493955536555, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.058072776399685354, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 81, + "hessian": { + "down_diagonal_mean": 86.07308197021484, + "gate_up_diagonal_mean": 931.4693603515625, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10535986294906316, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.017054881824089606, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10530752683815356, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.016964961487847218, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0073175568802046, + "packed_nmse_vs_bf16": 0.10529705047734363, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.01706959819632355, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10532203459280605, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.017032100596363033, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10545075073660086, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.021475413576693563, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.99827255914705, + "packed_nmse_vs_bf16": 0.10529527142793622, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.021564936299202482, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10525966105381702, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02064180865243526, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10548442210296562, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.020865796778802344, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.08132655446968123, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.03830865286570169, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.08164616530490806, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.0374231587981276, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.08173500078244049, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.03842673606796281, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.08149930929941201, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.040571726745913556, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 83, + "hessian": { + "down_diagonal_mean": 91.6432876586914, + "gate_up_diagonal_mean": 937.1392822265625, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10571717250774501, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.020360441281259044, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10600393097842473, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.01990610857289676, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.1058213382594791, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.020383857902872102, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529497, + "packed_nmse_vs_bf16": 0.10581630079878823, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.019849103635407105, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10572391617408423, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.023295592753492564, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10566300126917769, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.023546267997691404, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9745924476534854, + "packed_nmse_vs_bf16": 0.10581549554897124, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.023618409235945624, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10585978158167518, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.023170171830047603, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.07879567586869089, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05426916941290065, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.07884019570447622, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.054043194767329016, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9836374453866401, + "packed_nmse_vs_bf16": 0.07868552245770015, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05418305805641152, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.07880308585311703, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.054702264737449005, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 85, + "hessian": { + "down_diagonal_mean": 135.14845275878906, + "gate_up_diagonal_mean": 929.3579711914062, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 0.9836374453866401, + "packed_nmse_vs_bf16": 0.10817906995846961, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.019131309042230208, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10791537638291913, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.018728312189559268, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.99827255914705, + "packed_nmse_vs_bf16": 0.10800156085069139, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.018584428603588384, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10809772088321642, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.019345060113295388, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10792898878607528, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02082753962205973, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10807810498642022, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.020683940657612553, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10800614401106856, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02042222088558859, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10801801288947886, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02125123739711812, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.0825202901592011, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05092834902222285, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.08358995417832096, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.04964053380064736, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9836374453866401, + "packed_nmse_vs_bf16": 0.0832146954062423, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.04978191280296033, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.08331270696564493, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.04977082551922719, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 87, + "hessian": { + "down_diagonal_mean": 91.91087341308594, + "gate_up_diagonal_mean": 947.3469848632812, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.1108262345728235, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.020298256254295877, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.11071182431555361, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.021111963417384968, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.1107610400036761, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.020511690140836935, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.11065649183376905, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.02093565353011212, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.11080410950985069, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02326705019776313, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.11063158219271811, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02377230988098322, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.11067405711260407, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.023520055033809426, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.11071988009309645, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.023592093899962797, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.08082707443754787, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05094710546781185, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.08010029646978553, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.04714955181621255, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.080069993255533, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05034864245467878, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.0806723461294212, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05029740065072458, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 89, + "hessian": { + "down_diagonal_mean": 88.91563415527344, + "gate_up_diagonal_mean": 935.3355712890625, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10431334020223997, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.021046851891167007, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10408004153202371, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.021557881485020268, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10418313117649229, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.021962313908381152, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10428353598847627, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.021488053997282774, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10428907699155958, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.023786290301666745, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10431224915286515, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02471465913333, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.1041736106129102, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.024732868578187454, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10420992255662144, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.025060029262058525, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.0771410786461219, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.051601595636253085, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.07712023418428228, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05564894952958168, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.07748009833457478, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05259128251933859, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.07691200424195083, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.056172061172717516, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 91, + "hessian": { + "down_diagonal_mean": 83.82562255859375, + "gate_up_diagonal_mean": 946.0072021484375, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.1050882351012035, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.023233074223540724, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10521297222931046, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.023359738712101733, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10522561339199753, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.023584170424300402, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10515541573753447, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.023380910797308804, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10516381540298343, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02682298606814015, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10508350512205221, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.027251362574388666, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.1053463621084386, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02677083707716796, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10503393188673975, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02706451970534436, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.07745670912232949, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.0518466329169374, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.0770398363169111, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.056812659372768505, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.07742909123629244, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05557200839315375, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.07807742859957367, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05467074993108543, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 93, + "hessian": { + "down_diagonal_mean": 51.989444732666016, + "gate_up_diagonal_mean": 932.2490844726562, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 1.0073175568802046, + "packed_nmse_vs_bf16": 0.10322646187689073, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.019966346215384338, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10358166728269373, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.01948165504279849, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0073175568802049, + "packed_nmse_vs_bf16": 0.10335901956671799, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.020166492573096065, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10344165970005499, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.020516866699987507, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.1033180068447582, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.026902703021762887, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9780473293593848, + "packed_nmse_vs_bf16": 0.10344328838460919, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02789525154809726, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10320705835683557, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02648291776268505, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10341046762694381, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.027018570428404037, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9836374453866403, + "packed_nmse_vs_bf16": 0.07649971796732638, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.053146044894856075, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.07653208216295658, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05395144194808487, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.07710381171168512, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.04711524378405919, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.07648079771395716, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05264592954816622, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 95, + "hessian": { + "down_diagonal_mean": 121.43452453613281, + "gate_up_diagonal_mean": 922.412353515625, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10630447314185502, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.01645608032838327, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10631081249564948, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.016473900941940896, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10624770549093017, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.01620136389825035, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10622904126951417, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.016831035683776717, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10618720453805414, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.01899228566842633, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10639506179593682, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.019142405134707285, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10622803141024112, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.018324357759644208, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10606342882437025, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.019939771645799974, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.08585081498175277, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.04499431477842429, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.08527488849400806, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.04563003695998057, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9836374453866401, + "packed_nmse_vs_bf16": 0.08458790404431131, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.04234830491216647, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.08476147816217443, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.04672837001977494, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 97, + "hessian": { + "down_diagonal_mean": 103.44087219238281, + "gate_up_diagonal_mean": 942.656005859375, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 0.9780473293593848, + "packed_nmse_vs_bf16": 0.10781143997671014, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.02324533460247317, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.99827255914705, + "packed_nmse_vs_bf16": 0.10759360783225075, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.022699093162790153, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10749919222840891, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.02323681433204413, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10758593946787474, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.02350781641669189, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197949, + "packed_nmse_vs_bf16": 0.10746267077496206, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02583944744932819, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9836374453866401, + "packed_nmse_vs_bf16": 0.1076581384169898, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.025390324862464234, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10755838235438984, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.025639235448066795, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.1073820025310589, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.025765846889411498, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.99827255914705, + "packed_nmse_vs_bf16": 0.07858444308933392, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05611721579477566, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.07847819201835873, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.055994720094766165, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.07821861653920835, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.056304069697320845, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.07857186945648251, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05530250031271698, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 99, + "hessian": { + "down_diagonal_mean": 72.65846252441406, + "gate_up_diagonal_mean": 953.2794799804688, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10650806670110612, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.024416998539975435, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10631554398815288, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.02395959916068871, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10641179848119853, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.02413124794314219, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0073175568802049, + "packed_nmse_vs_bf16": 0.10625249923750024, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.023903510152733072, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10648453581523945, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.028055388383178597, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10633235904616149, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.027744569637314604, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10623119159800291, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02762604602681814, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10652837345631558, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.027699496158905264, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.07637651381689221, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05739232408513708, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.07647291208737354, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05618409322437558, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.07678235622914209, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.053395207026396316, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.07656321219667825, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.056557402014224246, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 101, + "hessian": { + "down_diagonal_mean": 99.66897583007812, + "gate_up_diagonal_mean": 949.1566162109375, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10423475449346732, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.021888328649239223, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10454849849328587, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.022512761808082857, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10456239434923392, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.022372074539615292, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.1044456809169856, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.02261511122368995, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10449966381283672, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.025014416435327168, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10446914895306468, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.025134078026972814, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10447279658153626, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.024956804891915686, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10441580308890486, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02535820427944957, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.07717518034535756, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05679269678742674, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.07665058262694145, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.057067653192219465, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.07650022500415163, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.0567637627364363, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.07658873079805427, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05795229232181779, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 103, + "hessian": { + "down_diagonal_mean": 74.80037689208984, + "gate_up_diagonal_mean": 935.9478149414062, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10508450076567055, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.02389354021026076, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10502924417453612, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.02374533211542616, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10498487302348294, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.023527058060238863, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10514257783131549, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.02395085220656484, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10500944197413503, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.028714389585853012, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.99827255914705, + "packed_nmse_vs_bf16": 0.10493695162394695, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02824575858874348, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10515020116375494, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02846295938884152, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10501528688551781, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02855076721794831, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.07756086923257408, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05593403837155389, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.07756299449676989, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.054724315524242347, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.07678579405501783, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.057285465236151384, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9836374453866401, + "packed_nmse_vs_bf16": 0.07752710926799626, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05199162498335452, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 105, + "hessian": { + "down_diagonal_mean": 55.735050201416016, + "gate_up_diagonal_mean": 939.073486328125, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10460117421664418, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.02552937159424764, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.1046747355874145, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.025145840909430756, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10465412869724769, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.025501604323051315, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10459475072549386, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.024943816120471897, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.1045832214747891, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.03327720517821298, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10446951983204716, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.03400886325054018, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.1043763831501205, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.033154848576784246, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.1044038582824257, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.032719990931385994, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.07552714791783366, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05673210348394743, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.07465540331334135, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05678318944560518, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.07492245388927173, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05830938504867522, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.07474357059381148, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.0593038176251321, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 107, + "hessian": { + "down_diagonal_mean": 81.09294891357422, + "gate_up_diagonal_mean": 945.8760986328125, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10981839981972506, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.02281726392711986, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10955522099374879, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.02243870512262275, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10964496595290418, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.022656541271839678, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10970285632594459, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.022729159180833975, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10959688635531435, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.025378131762852207, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10985025605845607, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.025449025877450335, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0073175568802049, + "packed_nmse_vs_bf16": 0.10971251950780195, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.0256052872034064, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9836374453866401, + "packed_nmse_vs_bf16": 0.10976075545212674, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.025121113711717106, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0073175568802046, + "packed_nmse_vs_bf16": 0.07881852922226242, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05379198284464972, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.07804754459509475, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.056474816127432784, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.07808956855156643, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05619415285430911, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.07805290577472832, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.055727379741948815, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 109, + "hessian": { + "down_diagonal_mean": 74.31835174560547, + "gate_up_diagonal_mean": 941.337158203125, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10421401301693509, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.02540439431337385, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10422341626687594, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.025558148145444014, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10411709424888885, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.026053376184982057, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.1039776236118938, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.025732788047949193, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10429068173187213, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.029144591546649955, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925397, + "packed_nmse_vs_bf16": 0.10430066969576986, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02898424953967136, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.1041300080324175, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.029543097518328603, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10410132105218821, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.029374675121068294, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.07757503225667468, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.053052817675061516, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.07830444498333726, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.044679129680303284, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.07659918466094226, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.056961480601247526, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.0772506223512907, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05378328370424904, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 111, + "hessian": { + "down_diagonal_mean": 80.30865478515625, + "gate_up_diagonal_mean": 721.0648193359375, + "selected_natural_routes": 6353 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.1124975370150545, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.019650651416898035, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0073175568802046, + "packed_nmse_vs_bf16": 0.11266590286365331, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.01957000550385144, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.11278866443999129, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.019805294439801217, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.11255273993037035, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.019647183274741874, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.11265849900729905, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.024147202027967204, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470503, + "packed_nmse_vs_bf16": 0.1127420336275041, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.023714831160627278, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.11253183799152808, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.023755951641890552, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9780473293593848, + "packed_nmse_vs_bf16": 0.1128402140059426, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.023606363151411867, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.0810957534531455, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.051862026800729426, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.08236370197606761, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.049049936167490815, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.08238990285013402, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.048351635039668925, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.08202363182779218, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.04711422367281361, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 113, + "hessian": { + "down_diagonal_mean": 68.33041381835938, + "gate_up_diagonal_mean": 938.9124755859375, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10468891298816141, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.025016799365042725, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9836374453866401, + "packed_nmse_vs_bf16": 0.10475906166929932, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.02544894371358563, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10487346826185401, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.025986864397878513, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10470387665271383, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.025370241887304044, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10467696439543836, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.029716772549181, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.1048068623169203, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.029411565226747715, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0073175568802046, + "packed_nmse_vs_bf16": 0.10461072387088759, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02977504456811704, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10471402428681767, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02923409906072601, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0129076729074602, + "packed_nmse_vs_bf16": 0.0760786883442314, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.057587478057265266, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.07589036694491268, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.058094227257717374, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.07619375267052145, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05757878459046431, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.07579797162249896, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.058412210203471523, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 115, + "hessian": { + "down_diagonal_mean": 66.62374877929688, + "gate_up_diagonal_mean": 921.5578002929688, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10568326953584223, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.019011401553744447, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10555695650399295, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.018569493392244205, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9836374453866401, + "packed_nmse_vs_bf16": 0.10548442794022138, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.0181305378289061, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10548896686405033, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.019374364818101758, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10545318402481677, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.028768294119133772, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10570568394537291, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02835588531680593, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10574163513651695, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.027427952976229793, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.99827255914705, + "packed_nmse_vs_bf16": 0.10553183041653288, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02886307655128119, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.07830871719012635, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.049390088972290185, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.0790123072894239, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.0494490047739989, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.0788750144723928, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.051125573563369206, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.07871998665793074, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.046931301400077875, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 117, + "hessian": { + "down_diagonal_mean": 94.34171295166016, + "gate_up_diagonal_mean": 939.2135620117188, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10560076593590208, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.02082657579657664, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.1053976360889118, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.0206481654438, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10539628141486915, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.020810936279770986, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10540438745581691, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.02060631893847531, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.1055360609150606, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.023918158828777968, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.105412257816344, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02374583927378306, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10562900491665513, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02380348677850997, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9836374453866401, + "packed_nmse_vs_bf16": 0.10556064206175084, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02321950593980033, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.07799167888686225, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.053696548704008064, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.07831998290686627, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05438720459051813, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9836374453866403, + "packed_nmse_vs_bf16": 0.0781967162179462, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.053846726749496684, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.07874394122812783, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.0523161825406488, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 119, + "hessian": { + "down_diagonal_mean": 139.97561645507812, + "gate_up_diagonal_mean": 938.8812255859375, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.11089792463292933, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.015579235541291194, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.11105108566501903, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.015221510111933931, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.11102109043143227, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.015590466881681177, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.11085615441217374, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.014973667913707984, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.11079277912728365, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.01764752210840552, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.11084440042672905, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.017500877773547408, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.1108978169939834, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.018238528477249785, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.1107208807558147, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.017752262262815045, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.08774381006903867, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.04238295991777871, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9836374453866401, + "packed_nmse_vs_bf16": 0.08792272569386772, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.04175499573860472, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.08710067615854618, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.04331358385081263, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.08682816574236038, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.04045937904365413, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 121, + "hessian": { + "down_diagonal_mean": 114.59164428710938, + "gate_up_diagonal_mean": 912.14404296875, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10665504519391691, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.01620629133245158, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10674140656208476, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.01635111679542839, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0073175568802046, + "packed_nmse_vs_bf16": 0.10665639655105488, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.016301700015025396, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10683253961990333, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.01584039918756232, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10657214673196337, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.01675830704912821, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.1065977158397165, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.017653778771029326, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10671222120840859, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.017644356115118165, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.1066776139266216, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.016121174807701096, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.08126587675548871, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.048069762195570874, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.08132388478114322, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.0501065493431586, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.08079499524913647, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.04974478629911576, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.0823747289563548, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.03974470322443219, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 123, + "hessian": { + "down_diagonal_mean": 62.87162780761719, + "gate_up_diagonal_mean": 953.8812255859375, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10662194608426093, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.020304206157221574, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0073175568802049, + "packed_nmse_vs_bf16": 0.10652389038235442, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.020346091094302403, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0129076729074602, + "packed_nmse_vs_bf16": 0.10659135179758904, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.02016191563185548, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10676189721843085, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.019892539947734388, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10668034062107784, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02388979898334485, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9836374453866401, + "packed_nmse_vs_bf16": 0.10689776090185975, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02407600065741695, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10674316701594398, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02406904512472086, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10668198861572886, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02347248261705434, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.07813408176886205, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05367453485352487, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.07733411080635955, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.055888979491679966, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.07734673933579135, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05526579035772186, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.07766610932881052, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05341760721673741, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 125, + "hessian": { + "down_diagonal_mean": 78.15823364257812, + "gate_up_diagonal_mean": 947.3997802734375, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10406284809571335, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.023053177935846696, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10419797741738249, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.022529777021385106, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10390671566420867, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.022891562117860483, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.1039772683000335, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.022292728388419725, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10390404730241914, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.0275947234531519, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0073175568802046, + "packed_nmse_vs_bf16": 0.10386374540930707, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.027766906419203326, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9836374453866401, + "packed_nmse_vs_bf16": 0.10400244403958099, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.027953747951442672, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10389602730520325, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.028169014193662027, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.07653508366417752, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.04945587432325901, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.0773004743854114, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.04996235608659259, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.07587475703744546, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.04616149108856004, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.07688590003496906, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05335160808159146, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 127, + "hessian": { + "down_diagonal_mean": 56.92906951904297, + "gate_up_diagonal_mean": 935.6255493164062, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10548500797579814, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.022870185758371703, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10539162833527374, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.022578049328543973, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10562402413719016, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.02266801220910454, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10560517733130541, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.02274524008656709, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10551878174985797, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.027609011520047435, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10553782049536418, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02779106748980051, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.1055519731850345, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.027822247209045935, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0129076729074602, + "packed_nmse_vs_bf16": 0.10556842661238912, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.027705750273097487, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.07595838853930693, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.058386669091402, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.07590539989161234, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.058639225334129484, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.07578598106152024, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.059032423933965884, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470503, + "packed_nmse_vs_bf16": 0.07576267336112205, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05884079568408422, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 129, + "hessian": { + "down_diagonal_mean": 83.77416229248047, + "gate_up_diagonal_mean": 945.5359497070312, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10438582324530297, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.023076041877960477, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10464883544335292, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.02321992069207702, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10461662073873432, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.022946496307777273, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10454003121106585, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.022986475657857466, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10450034684913073, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.027277839583099997, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.1045296628440758, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.027005123513719365, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529497, + "packed_nmse_vs_bf16": 0.10448984651839373, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02720255411745386, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.1045851676866514, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.027053612647772445, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.07552771822996193, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.058888377491818474, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.07615660551021176, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.0587501884874673, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.07535246504940034, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.06010560439496925, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.07558649500214462, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05947337631821139, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 131, + "hessian": { + "down_diagonal_mean": 100.32260131835938, + "gate_up_diagonal_mean": 937.8876953125, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10583164833283715, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.02134143538103637, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470503, + "packed_nmse_vs_bf16": 0.10569064807835007, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.021666544619031555, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925397, + "packed_nmse_vs_bf16": 0.10563919151436597, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.021832183341455366, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10585701726415485, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.021717757301485268, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10573268435414503, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02374622198815142, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.1057457968577941, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02407661526457436, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10574994125114785, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.024267362463124793, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10587151674933636, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.024470493012142235, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9836374453866401, + "packed_nmse_vs_bf16": 0.07738618662826756, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.055018257737688016, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9836374453866401, + "packed_nmse_vs_bf16": 0.07826644787690762, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05417934821849707, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.07789996378449755, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05249459692487839, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925396, + "packed_nmse_vs_bf16": 0.07786073969272715, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05404838719082843, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 133, + "hessian": { + "down_diagonal_mean": 67.54437255859375, + "gate_up_diagonal_mean": 946.190185546875, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10373324676007627, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.020624256127157868, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0073175568802046, + "packed_nmse_vs_bf16": 0.10386172129005151, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.02088561881211231, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470503, + "packed_nmse_vs_bf16": 0.10412130240010681, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.02054682577264103, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10386881224649275, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.020627153468131577, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10367673470333405, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.024085015018889714, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.1038308547911805, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.024367789955634947, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10384079125730437, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.024303511341078988, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10367591440094477, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.025083592431901676, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.07934400871929403, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.04852145754559625, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.0784715062667778, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.050586351997864716, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.07874166431701761, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05109896642442524, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.07820122440749992, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.0506652951147821, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 135, + "hessian": { + "down_diagonal_mean": 88.32881927490234, + "gate_up_diagonal_mean": 950.1121826171875, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10514294659957799, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.023556726182809386, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10537834155548163, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.023662488332335908, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10504492283029249, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.02386339024128049, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10515403524613652, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.023545459465340996, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10517744022860422, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.026752735813589203, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.1050698160842284, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.027232331120731784, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10511755381160609, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.027172327179007718, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10494862222265613, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.0270010482176513, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.07642728913548606, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05872052128118881, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.07637494205726121, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05772715693365283, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.07604128771302147, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05900957383748531, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.007317556880205, + "packed_nmse_vs_bf16": 0.07632511408054775, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05858461210257617, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 137, + "hessian": { + "down_diagonal_mean": 74.02272033691406, + "gate_up_diagonal_mean": 941.197998046875, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.1044713279193591, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.024082020298460745, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10424493862612738, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.023968721525396385, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10437723071875105, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.024176793455661316, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10458214681690553, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.023940890683204482, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10432875664723058, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.028019899089182447, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10451653398510416, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02804366226879768, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10445268883704126, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.028130471089209595, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.1042661386417137, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.028023399926733274, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.0769513348176501, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.056035018323482434, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.07676895066692792, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05705559251909399, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.07718379755757211, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05532891085536154, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9745924476534854, + "packed_nmse_vs_bf16": 0.07695522071299045, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.055513149598004984, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 139, + "hessian": { + "down_diagonal_mean": 56.96836853027344, + "gate_up_diagonal_mean": 933.9132690429688, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10372201250612681, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.025330158123200285, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10369394780584189, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.025666901839779818, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9836374453866401, + "packed_nmse_vs_bf16": 0.10398532482136853, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.02569986257717333, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10360462826708308, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.025433169783103952, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10356831631849264, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.030896444817884196, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10363641431578388, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.0304941852862931, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.1036578188257568, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.030149412106881, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10365663106988592, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.03059528453478678, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.07442801135303859, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.06058508835587081, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.0746273752150832, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.06041683733733071, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.07493730939155306, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05908275379067859, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197949, + "packed_nmse_vs_bf16": 0.07438049529309991, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.06065736544962769, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 141, + "hessian": { + "down_diagonal_mean": 82.94514465332031, + "gate_up_diagonal_mean": 942.5476684570312, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10551935381787332, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.02203511857703121, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9836374453866401, + "packed_nmse_vs_bf16": 0.10548474007778466, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.02193252061939003, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9745924476534854, + "packed_nmse_vs_bf16": 0.10586431144902503, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.021775901610755276, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9836374453866401, + "packed_nmse_vs_bf16": 0.10557382385922087, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.021637116072234674, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10553748189377093, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.026887989141608646, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0073175568802046, + "packed_nmse_vs_bf16": 0.10528674785571211, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.026297929121680438, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10548880775512298, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.026191807823725734, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10544150279387197, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.026127736473146916, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.078574286007283, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.050899441804950055, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.99827255914705, + "packed_nmse_vs_bf16": 0.07930640161832218, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.04941657568448418, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.07917520182780946, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.04953469446727862, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.07962549291701598, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.04726436144035633, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 143, + "hessian": { + "down_diagonal_mean": 106.52062225341797, + "gate_up_diagonal_mean": 938.8920288085938, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10551707507824931, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.019443914473551916, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10542254893796756, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.019533697223871688, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10557391408183057, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.019435855775426603, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10550388155188876, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.019497851171464676, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10559441851003579, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.023352402891863404, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.1052781139908108, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02269737693429882, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9836374453866401, + "packed_nmse_vs_bf16": 0.10549880348979945, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02287393854131087, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10560853859166532, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.023264710495134774, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9745924476534854, + "packed_nmse_vs_bf16": 0.08009416777406382, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05217775740352611, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.08004579967312812, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.051801617632778116, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.99827255914705, + "packed_nmse_vs_bf16": 0.08027049432079168, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05073542320556934, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9780473293593848, + "packed_nmse_vs_bf16": 0.08039196358815599, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05095285200311379, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 145, + "hessian": { + "down_diagonal_mean": 84.63243103027344, + "gate_up_diagonal_mean": 949.4058837890625, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 0.9780473293593848, + "packed_nmse_vs_bf16": 0.10506897583711433, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.023398522475255517, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10504430414334442, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.023995469849129314, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9836374453866401, + "packed_nmse_vs_bf16": 0.10500539665179304, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.023481370212424125, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10493659534073978, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.02335965100416379, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10481807142860831, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.027255776026774613, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0073175568802049, + "packed_nmse_vs_bf16": 0.10489161125041081, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.027259797132554883, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10494069283012196, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.026923590008740575, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10484087474268684, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.027277675796251545, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.07659529181299714, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05752801232090175, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.07667138541101766, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.057019111632579604, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.07664150725768351, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.0577256605328007, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.07655705758253172, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05306452944882336, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 147, + "hessian": { + "down_diagonal_mean": 80.81845092773438, + "gate_up_diagonal_mean": 934.4053955078125, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 1.0073175568802049, + "packed_nmse_vs_bf16": 0.11158567199840831, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.020467746810923653, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.11157212300774537, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.01997158315079386, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.11158138665247244, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.020486315333257628, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.11188689024479986, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.020654559436902552, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470503, + "packed_nmse_vs_bf16": 0.11149691910262066, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.022253932613166796, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.11159369316095057, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.021810321983574646, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.11160032954424812, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.021936713348141245, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.11150072135261613, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02147512621765248, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.07942497777655502, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05451662540222747, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.08118205036847258, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05099734032874148, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.07967243987207513, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05409219309631303, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.08055306301699995, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.052256288591849186, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 149, + "hessian": { + "down_diagonal_mean": 83.73948669433594, + "gate_up_diagonal_mean": 940.955078125, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10415372755986663, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.021872563066975062, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10437700867912733, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.021714347699167517, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10435690000675985, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.021590223987931652, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10435442941293063, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.02195943984410369, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0129076729074602, + "packed_nmse_vs_bf16": 0.1041675706536902, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.026929394970282513, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.1040312653821096, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02641990193012521, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10408572646695388, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02689757614592416, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10437741875147256, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02678898570300774, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.07805427830211925, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.052429051781132316, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.07787271434268991, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.052215905954445314, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.07810397409566097, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.053778766636158735, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.07832652965182241, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.0510795235681364, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 151, + "hessian": { + "down_diagonal_mean": 101.26060485839844, + "gate_up_diagonal_mean": 930.9091796875, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10850545654780874, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.017593042211198225, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10837858840721819, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.017383888309540806, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10833862992439276, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.017686656930522234, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0073175568802049, + "packed_nmse_vs_bf16": 0.10840394434733766, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.017708989323291957, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10822693311711072, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.021035144470484407, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10841324518929431, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.021193568962807424, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10848210737713725, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.021211865582177275, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470504, + "packed_nmse_vs_bf16": 0.10828085556310917, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.020379200770758752, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9836374453866401, + "packed_nmse_vs_bf16": 0.0813698783186788, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.050692376002881574, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.08153672385819966, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05005573010133932, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.08137289130802539, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.04895382191131543, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.08193005539353003, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.04577239945504455, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 153, + "hessian": { + "down_diagonal_mean": 84.84527587890625, + "gate_up_diagonal_mean": 944.5126342773438, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 0.9836374453866401, + "packed_nmse_vs_bf16": 0.10504704081398346, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.022391751493096152, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10493218270226175, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.0226396095784896, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10521731358584147, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.022924556104009917, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10478190242891879, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.022783238404202624, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10469983962661983, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.027382467237760773, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10497989186675188, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02753390562470373, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9836374453866401, + "packed_nmse_vs_bf16": 0.10507573524290877, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.027972196271960723, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.1049769003914541, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.027414383929783556, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9836374453866401, + "packed_nmse_vs_bf16": 0.07822686433537152, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.052007958135271926, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.07869271778123023, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.04866348579799788, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.07824014533217338, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.04909584269246479, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.07727439441311033, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.054081710368873506, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 155, + "hessian": { + "down_diagonal_mean": 88.04729461669922, + "gate_up_diagonal_mean": 923.1803588867188, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10539766518691256, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.020724480569372806, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.007317556880205, + "packed_nmse_vs_bf16": 0.10530687455687067, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.020835515761274865, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.1054023435077251, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.021035745442005826, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10526241330339076, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.021035806415175672, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10544207011203828, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.026114442965964522, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10527898015316176, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.026252779796805476, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10535070240569644, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.0252285075201088, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10524408886366028, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.025878114299953146, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.07967357262641984, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.052301313559094294, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.08020735144405604, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05233257984773123, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.08042033459383383, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.050636703347696946, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.0795599547328424, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05148919249655827, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 157, + "hessian": { + "down_diagonal_mean": 73.06014251708984, + "gate_up_diagonal_mean": 936.161865234375, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10401981273777, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.024721187388835247, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.1038693490677701, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.024593897217553604, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10394416880209588, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.024406333301134692, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0073175568802046, + "packed_nmse_vs_bf16": 0.1038522046675524, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.024986239666781103, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.1040556574360331, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.029248649290758086, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10389704653841282, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02917349336139931, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10397748478107544, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02901071274713955, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10387049709196347, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.029361229056679202, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.07699474267202208, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.054681569320952764, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.07676960554101793, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.051192128085169916, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.0764530475222944, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05723187260975603, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9836374453866401, + "packed_nmse_vs_bf16": 0.07612482309854314, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.057629393466224886, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 159, + "hessian": { + "down_diagonal_mean": 97.10018920898438, + "gate_up_diagonal_mean": 947.5048217773438, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10659409933048505, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.02104703070118265, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.99827255914705, + "packed_nmse_vs_bf16": 0.10651137791201315, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.02115652162503874, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9780473293593848, + "packed_nmse_vs_bf16": 0.1067327543102447, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.021225661657773183, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10645916382646872, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.020911732848482956, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10640720200260566, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.025063094518114462, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10668041336660404, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.025274890487671255, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10632427735025284, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.025378121500931477, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10644469229306719, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.025092296793945628, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.99827255914705, + "packed_nmse_vs_bf16": 0.07860489306593356, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05346382908891639, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.07835221600442875, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05506358064291573, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.07920859031725426, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05363397048933945, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0073175568802046, + "packed_nmse_vs_bf16": 0.07924072814768249, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.053674496875843615, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 161, + "hessian": { + "down_diagonal_mean": 106.15055847167969, + "gate_up_diagonal_mean": 930.5347900390625, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.11114752140876051, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.016836430526346792, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9836374453866401, + "packed_nmse_vs_bf16": 0.11081940417535922, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.01652316101497666, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197949, + "packed_nmse_vs_bf16": 0.11094462833059852, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.01633848058000649, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.1109569533965721, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.01628608901642737, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.11079590018932722, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.020155283279517242, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.11101364206208604, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.019249967059952557, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.11091057713131626, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.019620889799352177, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.111120742355702, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.01961966686869326, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.08308618676238287, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.04302510550680449, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.08313487200492288, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.045159150537988274, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.08277567578916435, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.04595633484561297, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.08420728579826502, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.04500397273692333, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 163, + "hessian": { + "down_diagonal_mean": 80.89985656738281, + "gate_up_diagonal_mean": 945.6981811523438, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10428745109571644, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.02394920212898513, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10430020086559832, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.0240417064451536, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10441920687199693, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.023467782016481944, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10435828869120499, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.02419878353945722, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.99827255914705, + "packed_nmse_vs_bf16": 0.10412681107336354, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02830414282611712, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10427014885597835, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02858955442503191, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10440727567135132, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.027614735728011545, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10425607083208592, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.0291621235948711, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.07820509148236268, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.053442682621194684, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.07768704510070319, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.054243096606485686, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.07867468724086252, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.051690638854040884, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.07751551019751078, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05525181657279618, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 165, + "hessian": { + "down_diagonal_mean": 83.82040405273438, + "gate_up_diagonal_mean": 949.0816040039062, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 1.0073175568802046, + "packed_nmse_vs_bf16": 0.1096046005460577, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.02032674851357664, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10952325649390245, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.020630062371696418, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10953708238087168, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.019612120629947185, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10981576597031244, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.02004168628295714, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9836374453866401, + "packed_nmse_vs_bf16": 0.10958699857331633, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02373514506542727, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10936631082454011, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.024187283570938998, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10957681358256337, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.023488795363673175, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10969707709977539, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02376450925720536, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.07907837674395678, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.054226014005284554, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.07908129733696159, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05368392928465949, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.08015937190615072, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05137288165465571, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0129076729074602, + "packed_nmse_vs_bf16": 0.07973812873605861, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.052682809490275054, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 167, + "hessian": { + "down_diagonal_mean": 148.1298065185547, + "gate_up_diagonal_mean": 935.7196044921875, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 0.9745924476534854, + "packed_nmse_vs_bf16": 0.10797702651424701, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.01966458145971877, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9836374453866401, + "packed_nmse_vs_bf16": 0.10784051134308038, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.019551823093724198, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10762984160528635, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.01991107382327331, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470503, + "packed_nmse_vs_bf16": 0.1076726033791044, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.019765855849005125, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10750012717353204, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.021372040997938976, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10768115715483001, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.021578497473111427, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10771377595317258, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.021741582158615266, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10760434431404357, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02156429786491237, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.08428113904265803, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.04824124163668092, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.08375993119867109, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.04853687824210019, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.08258373483521213, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05038227599442301, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.0832396476001331, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.049467510753560685, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 169, + "hessian": { + "down_diagonal_mean": 75.14503479003906, + "gate_up_diagonal_mean": 942.1983642578125, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.1052516289737779, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.023961923082052635, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10548147252944559, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.02430572379270321, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10536173586080894, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.023910662848877075, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9836374453866401, + "packed_nmse_vs_bf16": 0.10545355552862158, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.023796176104331785, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10519737058940792, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.027877608701249168, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10524455750449814, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.028465254343119136, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10544620538221512, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.028066776691093767, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10504512456925863, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.027898540565266205, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.07647181394928787, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.056774635875537015, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470503, + "packed_nmse_vs_bf16": 0.077093964269281, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.054579725254285685, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.0768945847149451, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05695389342714964, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.07582097494436014, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05817205134313489, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 171, + "hessian": { + "down_diagonal_mean": 129.28321838378906, + "gate_up_diagonal_mean": 933.504150390625, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10502146896456235, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.020473262223117652, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9836374453866401, + "packed_nmse_vs_bf16": 0.10502107622964105, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.02032711497669498, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.1048559708194162, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.020438571258371497, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0073175568802046, + "packed_nmse_vs_bf16": 0.10489100888279686, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.020575868295529383, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10484246403579871, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02390779126881192, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10479234973236144, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.023902146433983612, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9836374453866401, + "packed_nmse_vs_bf16": 0.1049572750150896, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.023984540179002187, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.1048164847763171, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.023457032928762965, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.07799422498312474, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05619715755582652, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.07787354197412188, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05649940615520608, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470503, + "packed_nmse_vs_bf16": 0.07787848238555929, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.056577004259793354, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.07818453273283396, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.055697806576880504, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 173, + "hessian": { + "down_diagonal_mean": 86.9715347290039, + "gate_up_diagonal_mean": 940.7193603515625, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10482408182063373, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.024034039311163786, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10470467876464004, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.02424004053753438, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10477109531801695, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.023947628819881838, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10474111946536047, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.0240399828203511, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10490645279822193, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02676957127865662, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0073175568802049, + "packed_nmse_vs_bf16": 0.10467089927578675, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.026925431338296602, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10463805290439912, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.026605497695071513, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10460954264400849, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02671430063777588, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.99827255914705, + "packed_nmse_vs_bf16": 0.07554065433861654, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05979600688856511, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.07531841410254402, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.06011930027214802, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.07542065800035538, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05989817939696053, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.99827255914705, + "packed_nmse_vs_bf16": 0.07533531972608455, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05998310941613753, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 175, + "hessian": { + "down_diagonal_mean": 75.95941162109375, + "gate_up_diagonal_mean": 952.2274169921875, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 1.0129076729074602, + "packed_nmse_vs_bf16": 0.1060159982835411, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.024439405171872097, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10607657462155004, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.024110282038341205, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9836374453866401, + "packed_nmse_vs_bf16": 0.10628466520057742, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.02415286506873902, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.1061288416268208, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.023903585713063, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10632555226174997, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.028252380898896066, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10635150751700347, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.028311861285668156, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10611999922372913, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02803329051813505, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10625907871294825, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.027394416966707046, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.99827255914705, + "packed_nmse_vs_bf16": 0.07602667458641965, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05687623702303849, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0163625546133597, + "packed_nmse_vs_bf16": 0.07621169189096237, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05814866732154216, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.07653184763608945, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05500365070465377, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.07665040896702235, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05637577742491027, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 177, + "hessian": { + "down_diagonal_mean": 94.7817611694336, + "gate_up_diagonal_mean": 937.0667724609375, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10348771398956758, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.024012573409512065, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10364599244213923, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.024328966298085387, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0073175568802046, + "packed_nmse_vs_bf16": 0.1037795741328075, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.024059746682492052, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10372768510526607, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.024494249337968642, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10338177472958127, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.027776563635021505, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.99827255914705, + "packed_nmse_vs_bf16": 0.1035663941564658, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.028340401217488764, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10343335665032612, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.027849383648120566, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0129076729074602, + "packed_nmse_vs_bf16": 0.10347977040624924, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.028198319789289928, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529497, + "packed_nmse_vs_bf16": 0.07611140692045595, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.058083453055956004, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9836374453866401, + "packed_nmse_vs_bf16": 0.07574639480451037, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05758237305280906, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.07625248094925877, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.058505240566979436, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.07629054232419809, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05800230913093408, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 179, + "hessian": { + "down_diagonal_mean": 83.51907348632812, + "gate_up_diagonal_mean": 947.6002197265625, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10434841472193807, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.02148083690042695, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9836374453866401, + "packed_nmse_vs_bf16": 0.10444376342958916, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.021564909343380182, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10455550670192518, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.02178689733255146, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10426956152839677, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.02193501700942578, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.104410680975421, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.026041091925340043, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10427124455049959, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02584699586250424, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.1043822577940415, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.025790045409283445, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10444307797258526, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.0256885634626529, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.07826324020050351, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05409566100120027, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.07768667488706864, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05562400568705091, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.07711506659690666, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.0557502031142534, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.0775229737514481, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.056511457302383335, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 181, + "hessian": { + "down_diagonal_mean": 74.70431518554688, + "gate_up_diagonal_mean": 945.7603759765625, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 1.0129076729074602, + "packed_nmse_vs_bf16": 0.10603996984867155, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.02500641694473579, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.1062256231595507, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.025135860860977837, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10618197113533147, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.025132652882955974, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9836374453866401, + "packed_nmse_vs_bf16": 0.10611677483078735, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.025071704138065504, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10595626686536751, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.0285932926143631, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10604585905941062, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.028411976989180927, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10597918537462456, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.028597936593554692, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.1061067135711932, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.028468649393344616, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.07616061743980458, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05775639683960128, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.07573933804514778, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05923025971750589, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.0764237598625784, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.057894057225196756, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.07633436596650421, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.055828127914200984, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 183, + "hessian": { + "down_diagonal_mean": 74.25640106201172, + "gate_up_diagonal_mean": 944.1392211914062, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10547113384048346, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.02315124848244847, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10529401954892664, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.023400025209417797, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.007317556880205, + "packed_nmse_vs_bf16": 0.1054488584321096, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.023136417637543837, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10538266929743345, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.022871477876816777, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10537465619209593, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02646045324282083, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10526437960165302, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02670025496664578, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10526840538122804, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.026452649916300554, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9745924476534854, + "packed_nmse_vs_bf16": 0.10551232384003062, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.026147759913618843, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0073175568802046, + "packed_nmse_vs_bf16": 0.07692582413085927, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.057356379104578845, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925397, + "packed_nmse_vs_bf16": 0.07700114566834845, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05703578573261198, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.007317556880205, + "packed_nmse_vs_bf16": 0.07752219441008677, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05636763710720191, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.0772365633243456, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.056182408766390635, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 185, + "hessian": { + "down_diagonal_mean": 92.49586486816406, + "gate_up_diagonal_mean": 946.2374877929688, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10578221983502997, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.020934719175976415, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10523155191330437, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.02180876548961802, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10548797865417671, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.02156813713647443, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10539639576467173, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.021553259117859824, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10534831450156124, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.024028661798557373, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10551680183315762, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.024611980022553402, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10550217630983041, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.023482964882227605, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925396, + "packed_nmse_vs_bf16": 0.10540363843342183, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02429654071229437, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.07687464796893109, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05730981455327968, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.07658029897938927, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05717193153428786, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.07666994332085057, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.0573545396110244, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.07699303135695576, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.056558745781053785, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 187, + "hessian": { + "down_diagonal_mean": 89.28246307373047, + "gate_up_diagonal_mean": 941.6725463867188, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10520987474401741, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.021533705561818016, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10523178633214655, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.02099042950694384, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0129076729074602, + "packed_nmse_vs_bf16": 0.10536206870267938, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.02104561402804165, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10544718800547774, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.020517826249678486, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10529255254591592, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.025611587141398204, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10529950701879333, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02560010644422362, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10510203168135233, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.025926144825249454, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10512885457892399, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.025806746322276978, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.07867776368246118, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05410640848359066, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.07905486967206192, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05402929956449529, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.07902260078106971, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05440713970367059, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.078504439154458, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05346546673161358, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 189, + "hessian": { + "down_diagonal_mean": 112.58472442626953, + "gate_up_diagonal_mean": 936.23291015625, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10517161307756816, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.018967982557482613, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9836374453866401, + "packed_nmse_vs_bf16": 0.1052269288998316, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.018802995181324583, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10519312320439334, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.019159460347150145, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10532444655401785, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.01893420647835079, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10516634274050429, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.021265024298155865, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.007317556880205, + "packed_nmse_vs_bf16": 0.10534214221073145, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.022962925174730378, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10502805559778289, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02265601021525754, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10528006189464502, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.021860218198296307, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.07860206367684855, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.04836171919995801, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.07883809487487077, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05292488932673327, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.0785362978091779, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05322398109226171, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.07825274044427423, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05321224486059737, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 191, + "hessian": { + "down_diagonal_mean": 81.50218200683594, + "gate_up_diagonal_mean": 948.71240234375, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10772468247155201, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.021947163376613506, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10781201314053485, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.021919591185478156, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10776637907878721, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.02217054391136711, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10768736358460687, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.021949897681872905, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10763334745939761, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.026039375565725695, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10763055459108027, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02610586698493309, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10758184529694495, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02617708705201673, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.1074226505211125, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.0259922696539146, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.0782734865544681, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.055860141519470816, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0129076729074602, + "packed_nmse_vs_bf16": 0.07804277264506762, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05613954102048171, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.07806102336570736, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.056080785670599334, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.07882317510553669, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.054809429584802616, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 193, + "hessian": { + "down_diagonal_mean": 79.93345642089844, + "gate_up_diagonal_mean": 941.540283203125, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10457139556394028, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.019773516973961763, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0073175568802046, + "packed_nmse_vs_bf16": 0.10476534186750724, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.020322805376198478, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.1046073499075289, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.019903565164321277, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9836374453866401, + "packed_nmse_vs_bf16": 0.10476538111324987, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.020257806822688684, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10463199093921785, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.028160747619197304, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10464656491275638, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.028111626287905866, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.1045405812315216, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.028675038706324155, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10455439889670984, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.027161299803372216, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.99827255914705, + "packed_nmse_vs_bf16": 0.07734896224828917, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.0553997137442118, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.07696303986234569, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.056027288476571055, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.07668402637558726, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05589718710993651, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.07734702038236446, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05351685342218979, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 195, + "hessian": { + "down_diagonal_mean": 94.65736389160156, + "gate_up_diagonal_mean": 947.2274169921875, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 0.9836374453866401, + "packed_nmse_vs_bf16": 0.10447043732782782, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.02279816869702327, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10452843017199255, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.022596131742563995, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10437065116910733, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.022586839884646426, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10442824571224338, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.02279642880013992, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.1044730414208692, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.027969766177172752, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10441015595257908, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.027093555191208603, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10427038303925025, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02761031112560629, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9836374453866401, + "packed_nmse_vs_bf16": 0.1045908083188793, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.028264468246743016, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.07835859942831276, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.052219410174255854, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.07988679653896393, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.04237270570157361, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.07900169380405277, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.04652327408909718, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.07913673405000785, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.04649359154233167, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 197, + "hessian": { + "down_diagonal_mean": 81.20466613769531, + "gate_up_diagonal_mean": 946.4832763671875, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10697239461466729, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.02012430476971192, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10678345872831335, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.020010751792132284, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10687522992010942, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.0197708232700916, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10672742032195973, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.01973408990164604, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10689872047605617, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.023760660899656613, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.1068682875446123, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02313675262368281, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10690584033218394, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.023421018133458503, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.1067676190443299, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.023466809910931263, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.07888600523463715, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.052354820818851386, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.07984539772697928, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.0490537294345555, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9836374453866401, + "packed_nmse_vs_bf16": 0.07958481716750929, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.051261371518001496, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.07925599930696638, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.052915719828578976, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 199, + "hessian": { + "down_diagonal_mean": 107.64997863769531, + "gate_up_diagonal_mean": 935.6430053710938, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.107640425309977, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.020728100365380687, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9836374453866401, + "packed_nmse_vs_bf16": 0.10768405680609107, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.020028140947950867, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10763247332243589, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.020685742848482035, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10755343915163819, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.02007864745847235, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10751032781352308, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02331274470698781, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9836374453866401, + "packed_nmse_vs_bf16": 0.10771499396598115, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02318293974550464, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10778522299811828, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.023418113693317515, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10748636115148046, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.023083116828684132, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9836374453866401, + "packed_nmse_vs_bf16": 0.07965414524232092, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.053705835788482564, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.0799023177090675, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.0543524482554576, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.07910636790092208, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05522326514150245, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.08005129880240312, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05276689599759546, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 201, + "hessian": { + "down_diagonal_mean": 73.09011840820312, + "gate_up_diagonal_mean": 948.092529296875, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 1.007317556880205, + "packed_nmse_vs_bf16": 0.10404143323040843, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.025228125062475182, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10401608940730202, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.02479071469111803, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10402801000226836, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.024412810808844897, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10395414455544585, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.025012049282607505, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10391522446213795, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.029963059932252567, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10390845960696882, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02960317106973807, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10378383653834561, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.028973911794778196, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10391647172987066, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.03013874149816724, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.07588450559839768, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05752151746000487, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.07525344351647495, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05894256582926305, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.0758218220508318, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05697172945401384, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.07533167636403278, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05921348328231112, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 203, + "hessian": { + "down_diagonal_mean": 94.70033264160156, + "gate_up_diagonal_mean": 935.4718017578125, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10394238906572585, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.02179460744193791, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10394169545254925, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.02056129847180245, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10386641347407898, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.020617984908244458, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9836374453866401, + "packed_nmse_vs_bf16": 0.10413860082553968, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.021258923964758394, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10420258782226482, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.024204127800339637, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10414816221466539, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.023855813087421806, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10401494050239352, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.022818672797056152, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10396383688993276, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.023777205783294075, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.07691573970108284, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.056200030510550045, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9836374453866401, + "packed_nmse_vs_bf16": 0.07663632720470202, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.056512392245979336, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.07667359506236082, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05521650170976813, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.0768302244850636, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.056569265640922, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 205, + "hessian": { + "down_diagonal_mean": 96.3082504272461, + "gate_up_diagonal_mean": 906.7227172851562, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10670987449850483, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.01711517471948974, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470503, + "packed_nmse_vs_bf16": 0.10676194124957238, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.017997944305606643, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.1069005399463754, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.017744737734220906, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9836374453866403, + "packed_nmse_vs_bf16": 0.10714725692477527, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.018270538679592482, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.1067783691144811, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.019667567349302945, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0073175568802046, + "packed_nmse_vs_bf16": 0.10677481828991008, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.0216644971874293, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10687223351113143, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02010395508931711, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.1066724057552224, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02147915077375363, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470503, + "packed_nmse_vs_bf16": 0.08134784442555164, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.035297156510771414, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.07971821816183994, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05177786639648908, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.08156709228591916, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.04672177890142433, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9836374453866401, + "packed_nmse_vs_bf16": 0.08123352156564699, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.048368686134845136, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 207, + "hessian": { + "down_diagonal_mean": 71.20513916015625, + "gate_up_diagonal_mean": 946.705810546875, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10435292002487898, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.026790528081672597, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10436632322803639, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.026762509952081955, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10431072383659641, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.026709160277913358, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.1041866066382386, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.027003741578612742, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10427064921442242, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.031046497192782078, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10442765741446398, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.03113543771543304, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10436626356756122, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.030956054207515654, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10432380886307269, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.03105111092352279, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470503, + "packed_nmse_vs_bf16": 0.07458943501053629, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.06154056838515304, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.07462727551091533, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.06146207409964902, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.07466061574789536, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.06140396540867034, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.07494622199057514, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.060751775268363284, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 209, + "hessian": { + "down_diagonal_mean": 66.89022064208984, + "gate_up_diagonal_mean": 939.89794921875, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10453444728853063, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.024842997051305694, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10463986742741178, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.024836860423794148, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10452322032260969, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.02503549326488477, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10466993361110417, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.02464646473585217, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10452388237647939, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.029505774271016395, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10448591654872164, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.03022135771190329, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10449776309797595, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.029612226548969257, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9836374453866401, + "packed_nmse_vs_bf16": 0.1046631289177764, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.030272895711328417, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.0760851103488041, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05687775129093123, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.07628887106975389, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05694002594259779, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.0765776060503075, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05452960064920447, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.0762875762630173, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05752128166815479, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 211, + "hessian": { + "down_diagonal_mean": 94.23512268066406, + "gate_up_diagonal_mean": 934.9424438476562, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 1.0073175568802046, + "packed_nmse_vs_bf16": 0.10439661042059349, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.02202297003892881, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10431655558235428, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.021586949832206912, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10448347242108187, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.021976266583540276, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10459192781480205, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.021522233851896822, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10457957275067731, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.025265205630353584, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10456858162345903, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02578635146382057, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10455858236595439, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02514798067904235, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10448838716033104, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02565886642393206, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.0778608930393067, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05493498486773123, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.07732674388328185, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.055714875910018016, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.078034414156062, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05157474956163498, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.07748075537580304, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.056133245341290876, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 213, + "hessian": { + "down_diagonal_mean": 99.53715515136719, + "gate_up_diagonal_mean": 945.5868530273438, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10449500360729481, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.020858414184349135, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10465998048224945, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.02074848445588729, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10467302602587303, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.02061366929643744, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10467796614178598, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.021119771975149595, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10472252780084951, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.024323735415061807, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10467351548402533, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02434701215200477, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10470953510223711, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02409266504202902, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0073175568802046, + "packed_nmse_vs_bf16": 0.10447746208197652, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02391137730724157, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9836374453866401, + "packed_nmse_vs_bf16": 0.076317697787603, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05778923462910356, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.07684005163991443, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05658217737494972, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.07624460572613709, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05795321403714873, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.07652739162711739, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.057162941119795335, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 215, + "hessian": { + "down_diagonal_mean": 110.00015258789062, + "gate_up_diagonal_mean": 937.4291381835938, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 0.99827255914705, + "packed_nmse_vs_bf16": 0.1109897166709166, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.017931407395643813, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.1109777610051344, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.017986567602692526, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.11087983376556554, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.017855913440695505, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.110776200947197, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.01777544254831331, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.11109575253569363, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02094142088619758, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.1108506484270039, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.020500414247205456, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.11099478261693531, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02033365623783738, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.11110975147143064, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02063805176653517, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.08050140376684306, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.051059966093116374, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.08208671864935152, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.048621906716804, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.08187477278108622, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.04835114156867501, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.08162901653121919, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.050538886757710934, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 217, + "hessian": { + "down_diagonal_mean": 55.02124786376953, + "gate_up_diagonal_mean": 926.82177734375, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.1044806895646261, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.025669304714201428, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10438508757918502, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.02536035847469395, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10465952618683357, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.025456691064276366, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10429712263695579, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.02569908754487424, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.1044815007535902, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.030437309196148624, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10450207366007647, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.03042084587167146, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10446948378758235, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.030569193931288655, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.1044851929038997, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.030210588144887, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9836374453866401, + "packed_nmse_vs_bf16": 0.07438524243270034, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.06107327717686867, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.07463716789090635, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.060260396126213374, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.07434070168492266, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.06103317173736905, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.0744251569408392, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.060869867232213146, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 219, + "hessian": { + "down_diagonal_mean": 105.17453002929688, + "gate_up_diagonal_mean": 933.610595703125, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10429328692261607, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.025554130525630225, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9836374453866401, + "packed_nmse_vs_bf16": 0.10434086498812933, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.025236274401608173, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.99827255914705, + "packed_nmse_vs_bf16": 0.10437021392187888, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.025234134723425172, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.1044623788815413, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.025382272670385213, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10467218587833614, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.028284631340558183, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10452316770458477, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02784639791307985, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10437722644152156, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02867285314063113, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10438982243888208, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02781992967883483, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.08034898986956826, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.04445424279655979, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.08123919736819593, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.04566538697570136, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.08049266300028662, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.045526850901866346, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.08012569484336528, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.04325817763517498, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 221, + "hessian": { + "down_diagonal_mean": 70.50418090820312, + "gate_up_diagonal_mean": 952.4591674804688, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10686099662230635, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.022951619158609594, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10695905373766171, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.022904122372570057, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10688191665026597, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.023042612131821724, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.1069160290344351, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.02350087585976171, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0073175568802046, + "packed_nmse_vs_bf16": 0.1068680802344385, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02713376051231051, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10709419198422124, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.026803943869530684, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10704969590953273, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02688423900755247, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10694477392122961, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02707446381846998, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.07639725872953332, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05867211624178082, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.07633033334095787, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05894724709142428, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0073175568802046, + "packed_nmse_vs_bf16": 0.07634311136625749, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.058956338573212703, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0073175568802046, + "packed_nmse_vs_bf16": 0.07644379299280771, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.059219860015138896, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 223, + "hessian": { + "down_diagonal_mean": 95.10885620117188, + "gate_up_diagonal_mean": 948.2474365234375, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10406558468536765, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.02152490471737746, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10404219516145777, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.021320066195381307, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9836374453866401, + "packed_nmse_vs_bf16": 0.10405576895181738, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.02114072125972345, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.1042051671026223, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.020952808232731348, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.007317556880205, + "packed_nmse_vs_bf16": 0.10397270126583859, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.026031108657496586, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.1041957596342295, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.025645405934464124, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10422049539320181, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.025682796726507865, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0129076729074602, + "packed_nmse_vs_bf16": 0.10410623192557389, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.025523858201901038, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.07736979735955363, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.0562760181507748, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.07738432022397447, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05679581797407754, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0129076729074602, + "packed_nmse_vs_bf16": 0.07734997200066666, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05654779614837849, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.07789849679523948, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.054925028235353016, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 225, + "hessian": { + "down_diagonal_mean": 58.47718811035156, + "gate_up_diagonal_mean": 932.5465698242188, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 1.0073175568802046, + "packed_nmse_vs_bf16": 0.10403505222591283, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.024793137345736675, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.1042271051417301, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.0248250510738598, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9836374453866401, + "packed_nmse_vs_bf16": 0.10424582367657176, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.024324090263230888, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10408934654039072, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.02485896259702867, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10411293282705728, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02974749138439708, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9745924476534854, + "packed_nmse_vs_bf16": 0.10447129353377473, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.029719273932443765, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9836374453866401, + "packed_nmse_vs_bf16": 0.104311682206715, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.029386065797829947, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10423158936309654, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02972371666442763, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.07528485432723318, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.058082973759101646, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.07536141635071507, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.057631803566095235, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.07554742890806095, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05837323193934774, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.07562551071539429, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.055981321973950886, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 227, + "hessian": { + "down_diagonal_mean": 131.5096435546875, + "gate_up_diagonal_mean": 933.5337524414062, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 1.0017274408529497, + "packed_nmse_vs_bf16": 0.10833181976490594, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.01847494012502119, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9836374453866401, + "packed_nmse_vs_bf16": 0.10847548830096693, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.01910650202077185, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.1083199502123699, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.018497219596303842, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10816301554965516, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.019284285644823783, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10839187512932433, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02077330516457742, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10818572951560151, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02084356207051069, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10826911283085444, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02007510979511536, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10825916136639557, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.021036820488765328, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.08756830677874956, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.0427420570967909, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9836374453866401, + "packed_nmse_vs_bf16": 0.08720547133691797, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.04474441142250667, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9836374453866401, + "packed_nmse_vs_bf16": 0.08904022138053246, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.042040422947379676, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.086842311307696, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.04549669184113125, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 229, + "hessian": { + "down_diagonal_mean": 78.31498718261719, + "gate_up_diagonal_mean": 910.7775268554688, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 0.99827255914705, + "packed_nmse_vs_bf16": 0.11193460009145038, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.01539328958522437, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.11223779031352528, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.015836201111548365, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.11193560831473805, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.01613878947483169, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.11217354909465446, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.01595930705564999, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.11213138896654717, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.018515955361570396, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.11213416367376901, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.018771331163245017, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0129076729074602, + "packed_nmse_vs_bf16": 0.1116534514981457, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.018492680586471678, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.1118040015469153, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.018862061100660705, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.08365576744906617, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.045219275637038626, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.08433805092815636, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.0471286563130764, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.08492727564511283, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.04526160135424327, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529497, + "packed_nmse_vs_bf16": 0.08352173977156739, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.04746623317889324, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 231, + "hessian": { + "down_diagonal_mean": 70.42726135253906, + "gate_up_diagonal_mean": 954.2379150390625, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10644298517364077, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.024827641075014492, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10625667431462556, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.024193198276608097, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10647469412467057, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.02429759947635802, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.1064131704734336, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.024025118715602194, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9836374453866401, + "packed_nmse_vs_bf16": 0.10652015370219813, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.028825104222884708, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10632521592361253, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.028489074895844513, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10625879521513287, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.028562142988731186, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10640391214139552, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02850427161520746, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9836374453866401, + "packed_nmse_vs_bf16": 0.07567387096953493, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05925059150438775, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0073175568802046, + "packed_nmse_vs_bf16": 0.07557298598211898, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.058686248283249434, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.07554209730511151, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.058793407248643174, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9836374453866401, + "packed_nmse_vs_bf16": 0.07594274378440945, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.058881093467813304, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 233, + "hessian": { + "down_diagonal_mean": 94.74556732177734, + "gate_up_diagonal_mean": 926.2992553710938, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.1056633081521684, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.021538529320945417, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10572315406156524, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.021608622189847014, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10569214110983176, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.02140033066478881, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.99827255914705, + "packed_nmse_vs_bf16": 0.10542882257946679, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.021755921625682104, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10558684038585651, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02374158930331727, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10554124254265734, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02444317778431294, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925396, + "packed_nmse_vs_bf16": 0.10563897799989992, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02431070907915365, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10557785201758899, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02424205848757153, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.07942309545076674, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.053482032306700227, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.07944759560574864, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05391224389412825, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.07947339192827667, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.053714031857497645, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9836374453866401, + "packed_nmse_vs_bf16": 0.07989413508915087, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.052822996102424385, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 235, + "hessian": { + "down_diagonal_mean": 112.8840560913086, + "gate_up_diagonal_mean": 922.3222045898438, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10602684735586104, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.01738603282444616, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10624905056545587, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.017846849596056175, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10635183862900623, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.01740261694265297, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.1060819926688714, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.016436472502414253, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10623032375560625, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02130088583611393, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10596913640701529, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.01955409285210467, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10629038379463056, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02092211257166776, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10590949036408333, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.020040993732763798, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.08045944842975754, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.048202042364369, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.08027049104778504, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.04413235704320105, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.07985636997239778, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05041291176532543, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.08070348409780152, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.04130600529300392, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 237, + "hessian": { + "down_diagonal_mean": 97.82521057128906, + "gate_up_diagonal_mean": 944.810302734375, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 0.9836374453866401, + "packed_nmse_vs_bf16": 0.1072136271548208, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.021108566308355444, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10711912076060995, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.02068673238205854, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10718396358545505, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.020700826582749708, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10707772131867259, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.020619947905604917, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.99827255914705, + "packed_nmse_vs_bf16": 0.10705191702030165, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.023600396104650246, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10713734964588754, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.023279231084679608, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10696326101280501, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.023189317191074123, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9780473293593848, + "packed_nmse_vs_bf16": 0.10726743722444283, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.023191992548063357, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.07869336910977484, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05482998754323481, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.07933484245298779, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05400713850180766, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.07952418577684542, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05360813222053482, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.99827255914705, + "packed_nmse_vs_bf16": 0.07986173818321546, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05408057666102057, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 239, + "hessian": { + "down_diagonal_mean": 94.2193603515625, + "gate_up_diagonal_mean": 945.4796142578125, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10528278230206324, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.023677355909703785, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.007317556880205, + "packed_nmse_vs_bf16": 0.10536497135968702, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.02276395468542001, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10528272468898263, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.023303688930273424, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10514077913284524, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.02345025441691709, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10506071923477503, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.026838439146403596, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10522819166220052, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.025860576549100227, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9836374453866401, + "packed_nmse_vs_bf16": 0.10520101817505434, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02713483010555814, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10492232489262324, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02653939407127401, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.07599342194792646, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05865855994514745, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0129076729074602, + "packed_nmse_vs_bf16": 0.07667437483670882, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05709599081988971, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.0755420897341887, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.059630302908092105, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.07582542290531716, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05907605309826964, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 241, + "hessian": { + "down_diagonal_mean": 96.91939544677734, + "gate_up_diagonal_mean": 945.76953125, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10459079637346298, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.022694699313064916, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10477112615185724, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.023037678108986406, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9780473293593848, + "packed_nmse_vs_bf16": 0.10474731959208405, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.023385099727675943, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10474264196551773, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.023913231432497288, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10450793947104822, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.026448887728290994, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10475239364925949, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.0263621511712873, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10463734073949195, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.027296272760026326, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10467467894423758, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.027469052801294917, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.0799013807275447, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.04800151610564054, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.07899818685258458, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.049891677273290455, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.07811359368166428, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05491453100571064, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.07765205683611767, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05563825093404307, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 243, + "hessian": { + "down_diagonal_mean": 58.421913146972656, + "gate_up_diagonal_mean": 927.6131591796875, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10517152109517325, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.026744817762557587, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10532401126966247, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.026711572147485662, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0073175568802049, + "packed_nmse_vs_bf16": 0.1051251306188194, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.02649871366860313, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10512328170729036, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.0263948324971559, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10519990933046111, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.031041677952723102, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.1051854429528729, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.03116061921839399, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10503630112711485, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.030588297542765083, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10525733067661813, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.030654715091840302, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.07483679468182962, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.0606666773793723, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.07458793073042042, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.060567922807332544, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.07493657907354422, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.059868290584945305, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.07512466056960931, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05927562133231745, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 245, + "hessian": { + "down_diagonal_mean": 79.88057708740234, + "gate_up_diagonal_mean": 951.3792114257812, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10458594059117694, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.023309396209503105, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.99827255914705, + "packed_nmse_vs_bf16": 0.10449938939182252, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.023611633540591728, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9836374453866401, + "packed_nmse_vs_bf16": 0.1045821641469391, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.02340871775773153, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10438684028281887, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.023456272091778706, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10436619523141864, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.027752867062091038, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10444926969256534, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02786503829812468, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10443207969187458, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.027873947633065603, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9745924476534854, + "packed_nmse_vs_bf16": 0.10469527635991897, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.027929826464151913, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.07638512421124187, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05774302355063835, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.07624408671049601, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05772557839172501, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.07596953474662162, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05852616583227449, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.07601860354724001, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05873645020892155, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 247, + "hessian": { + "down_diagonal_mean": 91.5374526977539, + "gate_up_diagonal_mean": 944.7692260742188, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10869919643531689, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.025347864806368928, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10886031913095154, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.02499235697585264, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10885803901880656, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.025416536611290997, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10874791986270764, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.025184464918022544, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10876527798309267, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.027343899422439987, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10875599440854533, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.027127142650508303, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9836374453866401, + "packed_nmse_vs_bf16": 0.10877594435705781, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.027337271782384133, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10856458965608248, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02711324238099717, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.07872407504998614, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05288544791629232, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9780473293593848, + "packed_nmse_vs_bf16": 0.07937050961718663, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05137988889715058, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.07825370457069722, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.054342934585185486, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.07811157812909747, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.054321740569152385, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 249, + "hessian": { + "down_diagonal_mean": 99.3135757446289, + "gate_up_diagonal_mean": 952.9154052734375, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 0.9836374453866401, + "packed_nmse_vs_bf16": 0.10594040230198842, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.021214199169318337, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10614061340978698, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.022041204543229803, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10574316587053104, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.021852322497566763, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0129076729074602, + "packed_nmse_vs_bf16": 0.10578286635430822, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.021529895704476908, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10585191954910453, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02508823035580001, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10591971009495073, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.025972042175150787, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.1056636611853669, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.025026538470387102, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0073175568802046, + "packed_nmse_vs_bf16": 0.10576335626359254, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.025152465392957412, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.07840628775417705, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05191028623304855, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0129076729074602, + "packed_nmse_vs_bf16": 0.07779433783305852, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.053988163397246366, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.07774028192705473, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05460348420814106, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.07744901922016154, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05539569004291127, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 251, + "hessian": { + "down_diagonal_mean": 75.53700256347656, + "gate_up_diagonal_mean": 949.337890625, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10628653967329023, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.022575240995644643, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10639594668616609, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.023324164861256023, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10628521707401076, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.022864423433093868, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10629360019263111, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.022745668574482594, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9836374453866401, + "packed_nmse_vs_bf16": 0.10649357727218958, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02676713270762818, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.99827255914705, + "packed_nmse_vs_bf16": 0.10624042799626238, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.027059166151068605, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10627789680247991, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.027358081267508525, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.1064067840070524, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02689052490483932, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.07784645308655025, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.0548472448849788, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.07710271474516707, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.056287666220446064, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.07752251766658798, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.054621237741832425, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.07749871610494667, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05401553163101072, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 253, + "hessian": { + "down_diagonal_mean": 81.1227798461914, + "gate_up_diagonal_mean": 940.59228515625, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10954365235256337, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.019294553586519908, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10959251137640645, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.01935878387086749, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10931134660312902, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.01906754839895452, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10930205707496686, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.018876353233116865, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470503, + "packed_nmse_vs_bf16": 0.10923763374711973, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.023173820048758998, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9836374453866401, + "packed_nmse_vs_bf16": 0.10954804849569925, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.023386398735082733, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9836374453866401, + "packed_nmse_vs_bf16": 0.10959006842552328, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.023124680529914496, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9836374453866401, + "packed_nmse_vs_bf16": 0.1094465550584606, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02274813212528935, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.07930561422828764, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05189969012641585, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.07890682127726462, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05321655670212049, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.07915346255263454, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05231268578384501, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.07929198647796162, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.052514979659711206, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 255, + "hessian": { + "down_diagonal_mean": 96.21380615234375, + "gate_up_diagonal_mean": 944.84912109375, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10544748169516725, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.021532685338778865, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10537818328991043, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.022014436425910926, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10533203688658521, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.02217315052404395, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10517415243475386, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.021791658812550557, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10512103913463346, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.026248324493213397, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10525125425577049, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02642632330412044, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.1052059369523965, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02693063183874052, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10515399458510781, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.026585192553774842, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.07885630460614165, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05415225353383018, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.07824840009599116, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05644757423774672, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.0787930642817744, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05461684298720816, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.07892579008566868, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05468466142507412, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 257, + "hessian": { + "down_diagonal_mean": 70.38592529296875, + "gate_up_diagonal_mean": 952.8441162109375, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10881594053844973, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.023082108276979277, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10877646890649978, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.023682651908266166, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10877532936094106, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.023326735932998225, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10876143551197238, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.023215867244835302, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10860502883066618, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.027364939566131823, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.1087668442216228, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.027615567687663763, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.1087091750313295, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.027285179983062586, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10874489298948621, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.027270484771345475, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.07703213574270718, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.0575610700252747, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.99827255914705, + "packed_nmse_vs_bf16": 0.07688913318941157, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.056522591749452615, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.07747586059668217, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05474997239434085, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9836374453866401, + "packed_nmse_vs_bf16": 0.0768907552059752, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05697943436767942, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 259, + "hessian": { + "down_diagonal_mean": 76.80902099609375, + "gate_up_diagonal_mean": 954.2657470703125, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 1.007317556880205, + "packed_nmse_vs_bf16": 0.10681605509120906, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.022654270455620792, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10665048016917619, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.022463352610061854, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10675485798078742, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.0224228766273496, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.007317556880205, + "packed_nmse_vs_bf16": 0.10683604241952517, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.02263058954175947, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.99827255914705, + "packed_nmse_vs_bf16": 0.10686877628136539, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.0264218335462261, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.99827255914705, + "packed_nmse_vs_bf16": 0.10677310139469762, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.0259323559666183, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.106768644138328, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.026601601371024214, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10676750704276688, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.026366611403664156, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.07740101635886279, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.056207630348444455, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.07750312376475271, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05577155294214804, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0073175568802046, + "packed_nmse_vs_bf16": 0.07713484259398819, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.057065883175210975, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.07739252369315296, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05663234611974188, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 261, + "hessian": { + "down_diagonal_mean": 68.23966979980469, + "gate_up_diagonal_mean": 937.74365234375, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10406465494690739, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.025298147218660555, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10419900107390698, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.025103333446122596, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10405246580011376, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.025276617616566476, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0129076729074602, + "packed_nmse_vs_bf16": 0.10404335222189671, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.025376238650061375, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10409322138237255, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.030148864279556337, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10394294246027408, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.03021518055687475, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9836374453866401, + "packed_nmse_vs_bf16": 0.10410818443208493, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.030029452453160663, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10421875398871273, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.03024789030814875, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.07521752211458209, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05970617342653149, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.07522379126314252, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05983047521315656, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.07489257766278065, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.06037314384654226, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.07559211302669445, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.057287940202734484, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 263, + "hessian": { + "down_diagonal_mean": 77.89266204833984, + "gate_up_diagonal_mean": 947.8944091796875, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10581650915410112, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.02318018031213736, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10574478319389286, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.023358386248325307, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197949, + "packed_nmse_vs_bf16": 0.10591449387977314, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.023609001330623575, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10588676187139508, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.023455055089072443, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.1059019572228993, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02706664713176822, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10589080790193603, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.026859119078960103, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10573404041462602, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02713754813195328, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.1058133541862076, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.027135565622296565, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.07595635564524378, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05865365742905966, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0073175568802049, + "packed_nmse_vs_bf16": 0.07660966149352613, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05764603457515163, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.07616850566718664, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.058520437809604246, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.07693737785706542, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05598501870660874, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 265, + "hessian": { + "down_diagonal_mean": 86.71302795410156, + "gate_up_diagonal_mean": 941.0321044921875, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10492272337686119, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.022454918577324942, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10500039840040537, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.02257028218589337, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10496766672600345, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.022872526519824975, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10489339296286343, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.022802851836727367, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10513662651179513, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02659738189075094, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10502415411442145, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02733908319830868, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10487899012653008, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.027773801584935524, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10490679006162297, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.027656971987606135, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.99827255914705, + "packed_nmse_vs_bf16": 0.07872646985395004, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05140121279209023, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.07839457344456631, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05425248034177933, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.07796603020490074, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05498182394175486, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9836374453866401, + "packed_nmse_vs_bf16": 0.07810994738733137, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05520567898994301, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 267, + "hessian": { + "down_diagonal_mean": 75.40734100341797, + "gate_up_diagonal_mean": 943.6512451171875, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10632937287179686, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.022537199122104792, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10630548696764124, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.022845746398480533, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10606007433612878, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.022632542323935596, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10631774857209228, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.02245660442650979, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10632964171143965, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.027540077781880876, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9745924476534854, + "packed_nmse_vs_bf16": 0.10620878777863221, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.027307471910430706, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9780473293593848, + "packed_nmse_vs_bf16": 0.10622154058656834, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.027409230396320088, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10605523558187414, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02758735251392753, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.07691061986953265, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05759110797153488, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.07692134210479572, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.057463323701332954, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.07730347988241432, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.056907611920173434, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.07649313007359118, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05842923125353281, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 269, + "hessian": { + "down_diagonal_mean": 73.02525329589844, + "gate_up_diagonal_mean": 926.529052734375, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10736576765275324, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.022693563727723774, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10725605468579713, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.023131461294067537, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10727195959934534, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.023322475966367304, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.1071872511374409, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.022805359323322567, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10738781471121045, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02608707084400846, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.1071559561490384, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02614949668493346, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10708238341267419, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.026498267464621743, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.99827255914705, + "packed_nmse_vs_bf16": 0.10733788185363581, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.026153226540500805, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9836374453866401, + "packed_nmse_vs_bf16": 0.07952201282792194, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05248760496615878, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.07901573236608034, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05275111264164452, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.07805704450064001, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05534545618583693, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0073175568802046, + "packed_nmse_vs_bf16": 0.07934711381034654, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05368671104378548, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 271, + "hessian": { + "down_diagonal_mean": 119.22032165527344, + "gate_up_diagonal_mean": 923.1927490234375, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10646554980638888, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.019029997839231292, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10639785669847644, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.018758105409490654, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10650820586766173, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.019533316494168, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0129076729074602, + "packed_nmse_vs_bf16": 0.10640016472183284, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.019236458067071877, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10639988574623153, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.020252940335650817, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10638972899769493, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.021117582738864167, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10626418777435141, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.021032261880688383, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197949, + "packed_nmse_vs_bf16": 0.10634151850310333, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02014662377048191, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529497, + "packed_nmse_vs_bf16": 0.08009072672257041, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.052012882736487416, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.08115883144207839, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05155241671170151, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9836374453866403, + "packed_nmse_vs_bf16": 0.08105720931077029, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05190247057553482, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.08022322640584195, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05132871322561685, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 273, + "hessian": { + "down_diagonal_mean": 58.46415710449219, + "gate_up_diagonal_mean": 943.923583984375, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.1050050192630179, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.02226435392630147, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10494102325552088, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.02294670894265496, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10500131088362431, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.022673916299476994, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10504713769416041, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.022463345465814778, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10495583539083103, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.025754016317412292, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9836374453866401, + "packed_nmse_vs_bf16": 0.10508003497528953, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02558308908671297, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.1050038539616585, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.0264975728155599, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10503681696107865, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.024823581151707096, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0073175568802046, + "packed_nmse_vs_bf16": 0.07668409480425843, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.0543478639553903, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.07681449441407452, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05410986716781687, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0073175568802046, + "packed_nmse_vs_bf16": 0.07693734467071123, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05367575790348308, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.07696501916544582, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05297615708096819, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 275, + "hessian": { + "down_diagonal_mean": 89.48149108886719, + "gate_up_diagonal_mean": 948.4571533203125, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10778886927008864, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.023569234037430664, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10795298954850527, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.02413664820940458, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10808316130233293, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.02390570445694012, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10781568819101738, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.023375500570117693, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10783225817615409, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.026094167254125004, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.1078843746013981, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.026910550230367983, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10792206675584741, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.026268295870419754, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10776490282859488, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.025984149913292343, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.07918904950850239, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.051968927871193525, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.07941303346236718, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.0524963603633542, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.07916486509476083, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05307510251248222, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.07925092358806905, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05169555831138166, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 277, + "hessian": { + "down_diagonal_mean": 86.99629974365234, + "gate_up_diagonal_mean": 922.2021484375, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10450000739170674, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.023613164668487133, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10455331586541332, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.022494075261149254, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.1044909732457921, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.02365445524748488, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10474772488812731, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.0236011870911027, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10453531778972226, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.028174694135149718, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.99827255914705, + "packed_nmse_vs_bf16": 0.10441376158550522, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.026468521703086396, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.1047723525290812, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.027586213804671005, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9836374453866403, + "packed_nmse_vs_bf16": 0.104491860359967, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02743832989760122, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.07838328184131861, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05597534204990468, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.07911491646946761, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.04637363617557258, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.07784450735660406, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05640335334398972, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.07828001250696112, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05601065263907804, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 279, + "hessian": { + "down_diagonal_mean": 58.344154357910156, + "gate_up_diagonal_mean": 928.1624145507812, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 0.9836374453866401, + "packed_nmse_vs_bf16": 0.10637189151283145, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.025139623365957958, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10596797587656698, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.02550995849260082, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10602584135206097, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.025657846041292905, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.1061442025650928, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.02552541867751888, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10622150321802988, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.030346922035716335, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10613770249695682, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.03039399075882999, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10593928461141926, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.030346195004183934, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.1061792224965588, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.030335143622132348, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.07578086650384354, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05897682539291622, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.07575873275710929, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05907796938235641, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9836374453866401, + "packed_nmse_vs_bf16": 0.07526697310972465, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05984352153667864, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.07609912446836747, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05768515366088409, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 281, + "hessian": { + "down_diagonal_mean": 98.19932556152344, + "gate_up_diagonal_mean": 942.0164794921875, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10754179224040027, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.02389837099802098, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10745611497706567, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.024043900844505795, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9836374453866401, + "packed_nmse_vs_bf16": 0.10751328396912388, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.023772727312984007, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10754520927413846, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.023935328361453484, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0073175568802046, + "packed_nmse_vs_bf16": 0.10740368535779082, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.027181553428420737, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0163625546133597, + "packed_nmse_vs_bf16": 0.10734874673208401, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02694804141280349, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10746195766961618, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.026792645983337894, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10734909859497228, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.026896945761376106, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.07893116477970426, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.054316533452994545, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.07856940089048216, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05552070902013673, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.07839609561043324, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.055937424537039045, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.07943080916076668, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05453095135740494, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 283, + "hessian": { + "down_diagonal_mean": 75.02057647705078, + "gate_up_diagonal_mean": 936.66357421875, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10319891283843338, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.023510563756677216, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10298841764987168, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.02346057873772739, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9836374453866401, + "packed_nmse_vs_bf16": 0.1033583954897611, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.024143863391563658, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.103412428096957, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.02359061807271487, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.1034352572481439, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02880094652865889, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.1032857280615407, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.029223559066185675, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10316813413449323, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.029115105351517383, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10338402642647364, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.029217530205113074, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.07664076924713388, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05679959190819732, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.07668710878523333, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05600040185851605, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.07626341865781638, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05759618797482277, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.07648649387148587, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05721672819557579, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 285, + "hessian": { + "down_diagonal_mean": 74.20526123046875, + "gate_up_diagonal_mean": 950.2546997070312, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10928861197858075, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.020711124851657683, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197949, + "packed_nmse_vs_bf16": 0.10926731393403817, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.020568512090196812, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10949286576311998, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.020283722461629516, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10959040022852343, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.020320316913285317, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10933622079271522, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.023325729393563706, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10944176107329452, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.0236202061890872, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10926616462481173, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02346166329944417, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9836374453866401, + "packed_nmse_vs_bf16": 0.10973382122332787, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.022896506353482306, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.07864895893952001, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05159952372950992, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.99827255914705, + "packed_nmse_vs_bf16": 0.07817721911392625, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.055229833027353174, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9780473293593848, + "packed_nmse_vs_bf16": 0.07828897896395409, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05527982997589563, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.07911076982695676, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.04899798743067455, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 287, + "hessian": { + "down_diagonal_mean": 77.55085754394531, + "gate_up_diagonal_mean": 939.0325927734375, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10530585420961336, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.023774929949099927, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10536787976536015, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.023550931689872313, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10528731479461091, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.023528845371287365, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10532294467860549, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.02377811310337733, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.1055380779857381, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02783723320820938, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10555689690761201, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.027718793640790784, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10518049056168231, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02741302725053839, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10536373179428181, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02802206043307695, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.077831059905616, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05499200852707065, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.07780724948797516, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05515043057837583, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.07823485224430728, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05450518319042272, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.0771621414365662, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.055853057711893556, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + } + ], + "experts": [ + 1, + 3, + 5, + 7, + 9, + 11, + 13, + 15, + 17, + 19, + 21, + 23, + 25, + 27, + 29, + 31, + 33, + 35, + 37, + 39, + 41, + 43, + 45, + 47, + 49, + 51, + 53, + 55, + 57, + 59, + 61, + 63, + 65, + 67, + 69, + 71, + 73, + 75, + 77, + 79, + 81, + 83, + 85, + 87, + 89, + 91, + 93, + 95, + 97, + 99, + 101, + 103, + 105, + 107, + 109, + 111, + 113, + 115, + 117, + 119, + 121, + 123, + 125, + 127, + 129, + 131, + 133, + 135, + 137, + 139, + 141, + 143, + 145, + 147, + 149, + 151, + 153, + 155, + 157, + 159, + 161, + 163, + 165, + 167, + 169, + 171, + 173, + 175, + 177, + 179, + 181, + 183, + 185, + 187, + 189, + 191, + 193, + 195, + 197, + 199, + 201, + 203, + 205, + 207, + 209, + 211, + 213, + 215, + 217, + 219, + 221, + 223, + 225, + 227, + 229, + 231, + 233, + 235, + 237, + 239, + 241, + 243, + 245, + 247, + 249, + 251, + 253, + 255, + 257, + 259, + 261, + 263, + 265, + 267, + 269, + 271, + 273, + 275, + 277, + 279, + 281, + 283, + 285, + 287 + ], + "gpu": 1, + "layer": 15, + "physical_experts": [ + 1, + 3, + 5, + 7, + 9, + 11, + 13, + 15, + 17, + 19, + 21, + 23, + 25, + 27, + 29, + 31, + 33, + 35, + 37, + 39, + 41, + 43, + 45, + 47, + 49, + 51, + 53, + 55, + 57, + 59, + 61, + 63, + 65, + 67, + 69, + 71, + 73, + 75, + 77, + 79, + 81, + 83, + 85, + 87, + 89, + 91, + 93, + 95, + 97, + 99, + 101, + 103, + 105, + 107, + 109, + 111, + 113, + 115, + 117, + 119, + 121, + 123, + 125, + 127, + 129, + 131, + 133, + 135, + 137, + 139, + 141, + 143, + 145, + 147, + 149, + 151, + 153, + 155, + 157, + 159, + 161, + 163, + 165, + 167, + 169, + 171, + 173, + 175, + 177, + 179, + 181, + 183, + 185, + 187, + 189, + 191, + 193, + 195, + 197, + 199, + 201, + 203, + 205, + 207, + 209, + 211, + 213, + 215, + 217, + 219, + 221, + 223, + 225, + 227, + 229, + 231, + 233, + 235, + 237, + 239, + 241, + 243, + 245, + 247, + 249, + 251, + 253, + 255, + 257, + 259, + 261, + 263, + 265, + 267, + 269, + 271, + 273, + 275, + 277, + 279, + 281, + 283, + 285, + 287 + ], + "physical_tensor_count": 6912, + "schema": "glm53-brandon-style-low-bpw-tail-part-v1", + "selected_k": { + "1": 2, + "101": 2, + "103": 2, + "105": 2, + "107": 2, + "109": 2, + "11": 2, + "111": 2, + "113": 2, + "115": 2, + "117": 2, + "119": 2, + "121": 2, + "123": 2, + "125": 2, + "127": 2, + "129": 2, + "13": 2, + "131": 2, + "133": 2, + "135": 2, + "137": 2, + "139": 2, + "141": 2, + "143": 2, + "145": 2, + "147": 2, + "149": 2, + "15": 2, + "151": 2, + "153": 2, + "155": 2, + "157": 2, + "159": 2, + "161": 2, + "163": 2, + "165": 2, + "167": 2, + "169": 2, + "17": 2, + "171": 2, + "173": 2, + "175": 2, + "177": 2, + "179": 2, + "181": 2, + "183": 2, + "185": 2, + "187": 2, + "189": 2, + "19": 2, + "191": 2, + "193": 2, + "195": 2, + "197": 2, + "199": 2, + "201": 2, + "203": 2, + "205": 2, + "207": 2, + "209": 2, + "21": 2, + "211": 2, + "213": 2, + "215": 2, + "217": 2, + "219": 2, + "221": 2, + "223": 2, + "225": 2, + "227": 2, + "229": 2, + "23": 2, + "231": 2, + "233": 2, + "235": 2, + "237": 2, + "239": 2, + "241": 2, + "243": 2, + "245": 2, + "247": 2, + "249": 2, + "25": 2, + "251": 2, + "253": 2, + "255": 2, + "257": 2, + "259": 2, + "261": 2, + "263": 2, + "265": 2, + "267": 2, + "269": 2, + "27": 2, + "271": 2, + "273": 2, + "275": 2, + "277": 2, + "279": 2, + "281": 2, + "283": 2, + "285": 2, + "287": 2, + "29": 2, + "3": 2, + "31": 2, + "33": 2, + "35": 2, + "37": 2, + "39": 2, + "41": 2, + "43": 2, + "45": 2, + "47": 2, + "49": 2, + "5": 2, + "51": 2, + "53": 2, + "55": 2, + "57": 2, + "59": 2, + "61": 2, + "63": 2, + "65": 2, + "67": 2, + "69": 2, + "7": 2, + "71": 2, + "73": 2, + "75": 2, + "77": 2, + "79": 2, + "81": 2, + "83": 2, + "85": 2, + "87": 2, + "89": 2, + "9": 2, + "91": 2, + "93": 2, + "95": 2, + "97": 2, + "99": 2 + }, + "sha256": "f48ebc44a7743bfb5fcc54a711b676ed74f767d4b150f03d6b9afc0810a4663f", + "shadow_experts_ignored": [], + "source_sidecar_sha256": "69696c12c5e8faa530e55f8a69d8f3ba09ece9a95a66b43c25614331782b12bf", + "state": "COMPLETE", + "tensor_count": 6912, + "tier": "k2-k3-tail", + "variant": "2.0" +}