diff --git "a/layer-17-part-tail-0.json" "b/layer-17-part-tail-0.json" new file mode 100644--- /dev/null +++ "b/layer-17-part-tail-0.json" @@ -0,0 +1,26086 @@ +{ + "bytes": 922818568, + "expert_reports": [ + { + "expert": 0, + "hessian": { + "down_diagonal_mean": 32.362144470214844, + "gate_up_diagonal_mean": 641.0211181640625, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10396964952633694, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.023503997020566818, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10410503436157026, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.023546394675658548, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.99827255914705, + "packed_nmse_vs_bf16": 0.1040006994810764, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.02362046914088555, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.99827255914705, + "packed_nmse_vs_bf16": 0.10409870874442483, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.023681951057606614, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470503, + "packed_nmse_vs_bf16": 0.1039956427843029, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.027745001239977973, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10399473311089955, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.028850811006812426, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10408877215640214, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.028761349802716746, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9836374453866401, + "packed_nmse_vs_bf16": 0.10414754529254244, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.027783314140407137, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.07514271325547496, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05454436434268111, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.0751117661066113, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05264434908432734, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.99827255914705, + "packed_nmse_vs_bf16": 0.07477750797473465, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05502391356470568, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.0754792147488866, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.054213640048012886, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 2, + "hessian": { + "down_diagonal_mean": 59.7083625793457, + "gate_up_diagonal_mean": 637.8576049804688, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10630405878191289, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.01912623328560819, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10654000114486711, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.019462235353479018, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.1066472846414755, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.01924635888733792, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10634146405813563, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.019494013793653425, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10654647473345819, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.021583406682863685, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10642622464414687, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.022030403805149434, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10647192520065138, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.021654172537230735, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10654615825821111, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.022081974147424084, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.0794952876434069, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.03601570511354114, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.0786571752029531, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05216161058355145, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.07975016084982656, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.043671713151930476, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.99827255914705, + "packed_nmse_vs_bf16": 0.07881534483293354, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05210457973323367, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 4, + "hessian": { + "down_diagonal_mean": 58.099205017089844, + "gate_up_diagonal_mean": 640.2081298828125, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 0.99827255914705, + "packed_nmse_vs_bf16": 0.10565727574775836, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.01964931361680253, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10552172840060851, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.019381614168123472, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10562759394448025, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.01950779034865108, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.1056738092202713, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.019081802902067, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10564422200774319, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02349965365445024, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.1057413535160238, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.0230335217534235, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10577291115451674, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.023650499949188542, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0073175568802049, + "packed_nmse_vs_bf16": 0.10556170591301446, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02332441242206245, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.07766573276911395, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05613965590026065, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0073175568802046, + "packed_nmse_vs_bf16": 0.07783750129651812, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05579753723861404, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.07760499045914451, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.056537913337571244, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.0783786378494944, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05456660860624161, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 6, + "hessian": { + "down_diagonal_mean": 61.213653564453125, + "gate_up_diagonal_mean": 639.4310302734375, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.1050235429083662, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.018738093261679006, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10501563926721465, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.018455989158420224, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10496957666188864, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.01882839345108991, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10497318852989973, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.018369515480390498, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10488146598715757, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02191964650538112, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.1048584821348655, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.022082813908914246, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9836374453866401, + "packed_nmse_vs_bf16": 0.10505884967251146, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.022567815192786188, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10506624171957665, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02216122130103057, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.078933637512747, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05305674534187399, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.07961617723155492, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05270388065229602, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.07955191789366242, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05140085810770641, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.07884762579959893, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.053526589890811105, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 8, + "hessian": { + "down_diagonal_mean": 50.30316925048828, + "gate_up_diagonal_mean": 643.1956787109375, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10564892690247867, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.020735892991683602, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10585266931138723, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.020788630847300774, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10570987704683281, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.02019784170424252, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.1058269726945115, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.0204339276411914, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.1054815040459223, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.023523665720442274, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10565500299782983, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02355913901583359, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10582791828500256, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.023338817520398875, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.1055364031610715, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02345027389998939, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.07668225766297454, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05771310072164414, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0073175568802046, + "packed_nmse_vs_bf16": 0.0764083302963069, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05818984706492785, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.07693456609828273, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.0565908545742676, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9836374453866401, + "packed_nmse_vs_bf16": 0.07642956671805282, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05836429123571525, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 10, + "hessian": { + "down_diagonal_mean": 44.61397933959961, + "gate_up_diagonal_mean": 641.7138671875, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10524677306146678, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.02212376180252083, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10515354070496517, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.02157327791686539, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.105093957136519, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.02199511660385759, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10503864520224607, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.021742577439917733, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0129076729074602, + "packed_nmse_vs_bf16": 0.10499819592593053, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.025455202150539678, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10493492344890896, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.024734117150675618, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9836374453866401, + "packed_nmse_vs_bf16": 0.10504293105040306, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.025246731841604593, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10502125612977702, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.024463347419171493, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.07558641588123534, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05963446387840287, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.07523273699466534, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05974249329638815, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.07569717466474116, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.059227090691870536, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.07528014516740186, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05943782118935133, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 12, + "hessian": { + "down_diagonal_mean": 77.62517547607422, + "gate_up_diagonal_mean": 627.6060791015625, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.11123251941359155, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.013813870139134767, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.11135851619222022, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.01339603774470301, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.1114213727537574, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.01328812221087916, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.11116155150749123, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.013572502281372845, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.11131773779476292, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.01364034187717094, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.11117429095244535, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.01354314051788434, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.11117513735784162, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.013734056907736372, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.11124898793533436, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.014162190248297185, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.08436048746717027, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.04829276529481323, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.0852878427910536, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.04759981176567693, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.08495613240127554, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.046951663679598736, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.0856173237826618, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.047417903655724916, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 14, + "hessian": { + "down_diagonal_mean": 23.425031661987305, + "gate_up_diagonal_mean": 361.1800537109375, + "selected_natural_routes": 4684 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 1.0017274408529497, + "packed_nmse_vs_bf16": 0.12499700432803555, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.016878181443273847, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0129076729074602, + "packed_nmse_vs_bf16": 0.12502997355136816, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.016630651486388787, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.12529732016007836, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.016867252244140023, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0129076729074602, + "packed_nmse_vs_bf16": 0.12505755429572676, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.016789768116919554, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.12520767426179455, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02055930822124827, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.12499913336938229, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02027029403548261, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.12489325510117848, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.021062122744948703, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.12497944088299605, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02058247083411917, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.08052948651888496, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.052402821046261365, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.08103215019111595, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.050508608781170465, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.08015205893338265, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.052922088364414414, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.08044568894177069, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05230132358544005, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 16, + "hessian": { + "down_diagonal_mean": 48.8353385925293, + "gate_up_diagonal_mean": 638.2579956054688, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.1059950734353342, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.019355331766165617, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9836374453866401, + "packed_nmse_vs_bf16": 0.1059460336567264, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.018787211992519447, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10591411548646502, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.019171059069981726, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10600351506215182, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.018432410347083202, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.99827255914705, + "packed_nmse_vs_bf16": 0.10579746536352852, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02292437317520951, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.1058333959599169, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02277306809075943, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10566722871840294, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.022970135838206517, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9780473293593848, + "packed_nmse_vs_bf16": 0.10586021802269455, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.022936869653855392, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.07869784416686222, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05265680036351211, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.07833777575414896, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05387253188886947, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.07868603414696015, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05300199567684046, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.07848939081959574, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.053119846129660024, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 18, + "hessian": { + "down_diagonal_mean": 60.52018356323242, + "gate_up_diagonal_mean": 643.724365234375, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10538634780828007, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.020157576399356766, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10544079290144012, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.019657129515516205, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.1054447124620161, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.019787762601089446, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10548315054075583, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.01955223054031546, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10526822361329484, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.022761920313114782, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10539974233766415, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.023175103407488257, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10524286644667488, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.023067476503390787, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0073175568802046, + "packed_nmse_vs_bf16": 0.10533096835533835, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02317398364557054, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.07859248652147081, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05490186597450193, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.07881547778351854, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05394526923224469, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9780473293593848, + "packed_nmse_vs_bf16": 0.07844602502895381, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.055668318527839725, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.07875179993861899, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.055233451383051385, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 20, + "hessian": { + "down_diagonal_mean": 51.62102508544922, + "gate_up_diagonal_mean": 641.8621826171875, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10571518496922405, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.022699921535557124, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9836374453866401, + "packed_nmse_vs_bf16": 0.10561674695023163, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.022432452587327997, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.1057212443280596, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.02217079302579218, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10568998416537762, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.022246758798639275, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10534355806494879, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.024773071509380445, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10542668725025289, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02530766597701015, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10571169049576289, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02491617171811236, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10551926774080339, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02486784133579195, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.07548421519496544, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.059270275816649155, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.07552229889236041, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05948186291666202, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9836374453866401, + "packed_nmse_vs_bf16": 0.07592796712628727, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05904329529423749, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.07586953841757473, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05895542663537872, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 22, + "hessian": { + "down_diagonal_mean": 77.99102783203125, + "gate_up_diagonal_mean": 624.7127685546875, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10764915313149268, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.016473387800947075, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10779679701638287, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.016365417499915855, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10798126230331859, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.016192203893668624, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10783101281775584, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.016603735831080792, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10774066702132604, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.017955761422752695, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10789795879282855, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.018112884564403846, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10775085964987575, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.01842058301614992, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10768735167840277, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.018589210766694543, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.99827255914705, + "packed_nmse_vs_bf16": 0.07935196104911556, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05402698946087743, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.0799174985458393, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05249455681279133, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.0801215788953199, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05353276552033701, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.07976330956270862, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05347070323569461, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 24, + "hessian": { + "down_diagonal_mean": 43.19456481933594, + "gate_up_diagonal_mean": 640.5572509765625, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.1049974426103057, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.022754531881478104, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.1053442451050626, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.022926693532464446, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470503, + "packed_nmse_vs_bf16": 0.10514824260348939, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.022697846063687344, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.1051002398223919, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.022866249718199987, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10504902861804592, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02698711166358323, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10506597720819737, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.026637148285774126, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10503279819571179, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02598659657537808, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10504730651290471, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.026390732933416764, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0129076729074602, + "packed_nmse_vs_bf16": 0.07693098675652636, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05614808109319422, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.07735109702400239, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05586644469563624, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.07715040091303325, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05561779066498855, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.07754135233294021, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05587210902184855, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 26, + "hessian": { + "down_diagonal_mean": 210.85560607910156, + "gate_up_diagonal_mean": 643.6107177734375, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10940905140700719, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.019184849622065864, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.99827255914705, + "packed_nmse_vs_bf16": 0.10922884403328778, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.019152972221603497, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10907441128077391, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.01908890059016075, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10916653381269063, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.019380749229406472, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9836374453866401, + "packed_nmse_vs_bf16": 0.10914965558912994, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.021851565776796623, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10890437255581348, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.021678663782073806, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10903522305129572, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02229461895530631, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10913143642255727, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.022114061696858475, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197949, + "packed_nmse_vs_bf16": 0.0767865069712575, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.005922829164591838, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.07788953539404365, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.013623277464539976, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.07751855722987733, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.01155321426516852, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.07692256233365949, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.006138150438957026, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 28, + "hessian": { + "down_diagonal_mean": 31.943635940551758, + "gate_up_diagonal_mean": 635.86279296875, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10410517783190334, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.02578829123509278, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10387801161750052, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.026322790973750263, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0073175568802046, + "packed_nmse_vs_bf16": 0.10396002448394999, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.026110015543517107, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10382206046999042, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.02658551352213088, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10393583728998251, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.03131906877370362, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9836374453866401, + "packed_nmse_vs_bf16": 0.10405451593851751, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.031661999377763005, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10424448666179395, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.03125973519410281, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10385147293817423, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.03154614238506223, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.07561565872429904, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.057271383013327654, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.07574717840184936, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05777731649964714, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.07572045292245141, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.058566836979839984, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470503, + "packed_nmse_vs_bf16": 0.07560783718434827, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05784175316293455, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 30, + "hessian": { + "down_diagonal_mean": 44.13196563720703, + "gate_up_diagonal_mean": 636.0759887695312, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 0.9745924476534854, + "packed_nmse_vs_bf16": 0.10567829726279974, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.01879988548056437, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.105527404468932, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.01907122783325775, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10542391512667054, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.019322548515180824, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10544206339564863, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.018831286835831216, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.1054848180987405, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.023309580458147936, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10543502479018697, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.023128340386681143, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10559652383743044, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.023092749369840303, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10552034439373231, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02344513382113486, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.07680133966534451, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05721747143053673, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.07692751490986102, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05707453382957763, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9836374453866401, + "packed_nmse_vs_bf16": 0.077371154566334, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.0571813986487743, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.07753399730835094, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.056129977533920856, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 32, + "hessian": { + "down_diagonal_mean": 48.67224884033203, + "gate_up_diagonal_mean": 637.619873046875, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10668444614180785, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.01812528063368859, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10668902054289506, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.017628326415516492, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10704247602770621, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.017779752527144446, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10685924461258607, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.017833569539560577, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10672298359813213, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.019061455450885733, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470503, + "packed_nmse_vs_bf16": 0.10653093889157404, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.018542104683360757, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10673486828861291, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.01816036187353763, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10672776730562228, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.01872408257746266, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.08197349086501324, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.04700742490675869, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.08226353176535167, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.04475080506185399, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.08274710784944778, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.0389471602072227, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9745924476534854, + "packed_nmse_vs_bf16": 0.0822587860589954, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.04575353738440274, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 34, + "hessian": { + "down_diagonal_mean": 59.31035614013672, + "gate_up_diagonal_mean": 641.0382690429688, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10603201169586286, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.01929686980840472, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10598404092071971, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.019776961168057123, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0073175568802046, + "packed_nmse_vs_bf16": 0.10591137556174911, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.01958400455016628, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10593522822984694, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.019438337048822633, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10579562992534061, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02186461374598209, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10585997059293717, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.021798550959294352, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.1058571321582774, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.021810522878762326, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10590328531261024, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.021391648870247578, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.07634952406895947, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05805517426439057, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.07623441743760238, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.0576515592223307, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.07626480370386654, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05808296559477919, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.0768919326978028, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.056604043312683254, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 36, + "hessian": { + "down_diagonal_mean": 49.663063049316406, + "gate_up_diagonal_mean": 645.548828125, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10370871178052253, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.01717731008525089, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9836374453866401, + "packed_nmse_vs_bf16": 0.10393030667764243, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.01755151148172804, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0073175568802046, + "packed_nmse_vs_bf16": 0.10384435303220262, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.01775815562624879, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10369151404117931, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.017725010680707948, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10386192379060172, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02006413284419972, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0073175568802046, + "packed_nmse_vs_bf16": 0.1039233564765917, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.019911087715761604, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10367864540719704, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.01962619331256067, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10389207242902324, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.019815053686551874, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.07905219986806868, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.04901195370237946, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.07844312549087665, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05104847382072556, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197949, + "packed_nmse_vs_bf16": 0.07853833896153467, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.04999621640666504, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.07883695074969335, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.049574830230507155, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 38, + "hessian": { + "down_diagonal_mean": 49.659400939941406, + "gate_up_diagonal_mean": 630.33154296875, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10666299506934836, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.018517117288892306, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10643481201543485, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.018631693782387094, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10642889307257303, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.01894762997635424, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10676167063774127, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.01826107359558202, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10668988208447015, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.020382785242661218, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.1069233455867947, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.020401710774612856, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10672635581979177, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02087729470660613, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10661164440019341, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.019866780337163666, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.0821392890033039, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.04852389334429432, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.08183874043765503, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.04474247291481655, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197949, + "packed_nmse_vs_bf16": 0.08067512615412181, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05109879133374458, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.99827255914705, + "packed_nmse_vs_bf16": 0.082714662691142, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.044936671493082045, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 40, + "hessian": { + "down_diagonal_mean": 48.7020263671875, + "gate_up_diagonal_mean": 642.7711181640625, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10511550887070692, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.022437312828350788, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.1051191362604928, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.02224673924352227, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10504572057918377, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.021830074086062537, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10500443687817418, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.021854881262807227, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10522740944699467, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.023911233369445232, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.99827255914705, + "packed_nmse_vs_bf16": 0.10519043570742823, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.023968331930127613, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470503, + "packed_nmse_vs_bf16": 0.10511254222469671, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.024032482016712223, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10521170331776024, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02367025971158417, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9836374453866401, + "packed_nmse_vs_bf16": 0.07466391411797561, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.06044759929266823, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.07454478892599582, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.06063732630914243, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.0746731222061397, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.060276183349190694, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.07456004563227953, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.06060924673292886, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 42, + "hessian": { + "down_diagonal_mean": 45.41716766357422, + "gate_up_diagonal_mean": 628.8907470703125, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.1074310724753324, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.016807527688923133, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10750383768426192, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.017009617765621196, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10765850872047626, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.016708087164991715, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10771140159795349, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.01611962813537588, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10752257761707855, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.019112921288017382, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10766520508004214, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.019081460122085128, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10751508756851492, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.01915547262405486, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10748855993493266, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.018847535143997333, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.007317556880205, + "packed_nmse_vs_bf16": 0.0791058010042438, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.052183470654763926, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.07886764824087049, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05278327900334568, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.07853935946398641, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05402020839280147, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.07876587941679636, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05337657875625921, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 44, + "hessian": { + "down_diagonal_mean": 50.061405181884766, + "gate_up_diagonal_mean": 642.7662353515625, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 0.9836374453866401, + "packed_nmse_vs_bf16": 0.10517895707766682, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.020707560229689143, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.1050862857180415, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.02079632015614214, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10513879614652405, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.020766365495140025, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.105076710636976, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.020505802189184245, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10497258821928779, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.023893928484294856, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10487081406837055, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.024244611722842146, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10484923860367497, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02386279603239228, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10495377705270159, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.024373641144018885, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9836374453866401, + "packed_nmse_vs_bf16": 0.07592831908978796, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05862084965664662, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.07552038888708956, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05902591513434678, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9836374453866401, + "packed_nmse_vs_bf16": 0.0760925253299169, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05825511935475247, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.07594897571430644, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.058686742537891905, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 46, + "hessian": { + "down_diagonal_mean": 47.26289367675781, + "gate_up_diagonal_mean": 645.2223510742188, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10605559992867976, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.020778440609078852, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10592912764721492, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.020821494839995524, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10606062214029012, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.020776241569762897, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.1058362538130603, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.020852151014628954, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.99827255914705, + "packed_nmse_vs_bf16": 0.10598081056730362, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.022555456038385897, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10579827582525024, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.022239638610560098, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10577732228683781, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.023158843884755875, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10596374421833485, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.022675427381317958, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.07483365758346638, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05998578371362856, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9836374453866401, + "packed_nmse_vs_bf16": 0.07485595675509565, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.059963402318086674, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.07475325192319557, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.06028046963566964, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.07504096939292666, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05970619108754393, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 48, + "hessian": { + "down_diagonal_mean": 28.174489974975586, + "gate_up_diagonal_mean": 630.8387451171875, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 0.9836374453866401, + "packed_nmse_vs_bf16": 0.1049966163597572, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.023243601697097088, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470503, + "packed_nmse_vs_bf16": 0.1049436347658758, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.02283074309676882, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10496474043957213, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.023153954677580642, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.104868975268615, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.022827977570316023, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10492933997368273, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.028312221316913463, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10481732724667847, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.0282521578191166, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.1047625663999063, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.028984912773321596, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10484545936523669, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02854880345702109, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.07455506095217602, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.059472273451853734, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470504, + "packed_nmse_vs_bf16": 0.07484940361816707, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05879819951013686, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.007317556880205, + "packed_nmse_vs_bf16": 0.07466002701983661, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.059560186729307844, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.07486995983453015, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05879150231605915, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 50, + "hessian": { + "down_diagonal_mean": 60.62130355834961, + "gate_up_diagonal_mean": 642.5724487304688, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10438447959422087, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.020925704146272487, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10444388617916248, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.02093731495785757, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10443263493722799, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.02155289185552488, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10421009620541848, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.02094932544553854, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10452558330446074, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02282402946414196, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10428371696156977, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02296859725609888, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10443705351009851, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02334198871267678, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10445404844135123, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.023295058431199993, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.07699443253948017, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05641963700522659, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9745924476534854, + "packed_nmse_vs_bf16": 0.07656403174405164, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05833842404369446, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.07690210905908076, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05755103248750805, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.07657035284631174, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05795347152116976, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 52, + "hessian": { + "down_diagonal_mean": 44.722591400146484, + "gate_up_diagonal_mean": 642.2494506835938, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.1048763735463982, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.02202505682605284, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.1049909735203292, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.02198000051918066, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10495249315252038, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.02229780346199132, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10495756174869993, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.022473343307634144, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0073175568802049, + "packed_nmse_vs_bf16": 0.10499563735142266, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02516262964143046, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10501896408955698, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.024544318911457072, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0073175568802046, + "packed_nmse_vs_bf16": 0.10488704430064942, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.024953182833252232, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10494642116524756, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02518613809983145, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.07620919957230426, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05854561032756257, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.07624181869740213, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.0583178545227147, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.07615924735935596, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05845562305429153, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.07585834700769668, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05872267196371624, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 54, + "hessian": { + "down_diagonal_mean": 44.751502990722656, + "gate_up_diagonal_mean": 641.96533203125, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10402330713247687, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.02287657045770808, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10429765486660758, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.02313090090814873, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9836374453866401, + "packed_nmse_vs_bf16": 0.10403358999264622, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.023060203216121237, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10421631989139418, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.022819655357857193, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9836374453866401, + "packed_nmse_vs_bf16": 0.10403233205718147, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.026767532919256702, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10396808257642637, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.027243831912528414, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10395556694717918, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.027424625595553403, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10417398790418178, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.0271234806667963, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.07831143133986355, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.050606424010817094, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.07776768860037568, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.04831033946936332, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.0778441549950558, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.051644693327087446, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.07875924479118981, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.03524933547397281, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 56, + "hessian": { + "down_diagonal_mean": 49.35003662109375, + "gate_up_diagonal_mean": 641.8841552734375, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10676843343501899, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.01949788784644143, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10639291190228027, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.019494799151485803, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10669203575847949, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.01931652922028344, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10647297139679697, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.01974346448560916, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197949, + "packed_nmse_vs_bf16": 0.10652256588886556, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.023477557255504853, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10633819781675664, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02378726260820394, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10622622290798737, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02313104978430627, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10641900855727915, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.022976505614452455, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.07837809994291536, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.053695296218379075, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.07829048832084835, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05244635558735012, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.07893010819319615, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05160255088506133, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0073175568802049, + "packed_nmse_vs_bf16": 0.0787099766778813, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05209241363320722, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 58, + "hessian": { + "down_diagonal_mean": 45.20283126831055, + "gate_up_diagonal_mean": 636.7786254882812, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10636138266788962, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.022063958124071862, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10632513555685852, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.022112010099195015, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10619423070158235, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.021885791794991265, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10610335146788397, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.021801754100048525, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10625252725554254, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.024023914414994567, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10620877186553301, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02406583860627847, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10615396152040446, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.024005652080512134, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470503, + "packed_nmse_vs_bf16": 0.10624203700947812, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.023825541220869027, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.07542486062791744, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05985356205854167, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9836374453866401, + "packed_nmse_vs_bf16": 0.07530799869153347, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.06029847476965352, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.07530677615785085, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.06010161428330559, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9836374453866401, + "packed_nmse_vs_bf16": 0.07553650688610787, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.06024381798504173, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 60, + "hessian": { + "down_diagonal_mean": 57.499427795410156, + "gate_up_diagonal_mean": 638.8958129882812, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10680709229715875, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.018971758118246945, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10660284605558397, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.018691296528348764, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10686557950914283, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.018594845780976928, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10667201709965819, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.018160230080816273, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9836374453866401, + "packed_nmse_vs_bf16": 0.10684198073112153, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02151905451170158, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10671033821411227, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.0219386798666374, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10676114558247585, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02180987762590662, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10661007454906117, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02133198601996713, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.08081899040702888, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05309558519742135, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.08146696800543281, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.04993814735889462, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.08191152186226656, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.051049456549142856, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.0816060824802149, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05122567892716637, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 62, + "hessian": { + "down_diagonal_mean": 55.069602966308594, + "gate_up_diagonal_mean": 644.467529296875, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 1.0017274408529497, + "packed_nmse_vs_bf16": 0.10578829888822881, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.020068449742385582, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10587020677361132, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.020874154492294086, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10573149364938587, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.020671160719106057, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10555176572796778, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.020845303670826806, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10567521077941025, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.023011939975363223, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.1055394143647541, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.023082243052507508, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197949, + "packed_nmse_vs_bf16": 0.10582881386334513, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02319429680688421, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10550675382912156, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.023607011011420594, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.07625933615442573, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05882093306094658, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.07593805383614664, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.059289702522152564, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0163625546133597, + "packed_nmse_vs_bf16": 0.07599448869425396, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05910424926722454, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.99827255914705, + "packed_nmse_vs_bf16": 0.07592072224461272, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05890656735867515, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 64, + "hessian": { + "down_diagonal_mean": 28.976078033447266, + "gate_up_diagonal_mean": 545.3255615234375, + "selected_natural_routes": 7081 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 0.9926824431197949, + "packed_nmse_vs_bf16": 0.11034633399635664, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.02060423346580059, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9836374453866401, + "packed_nmse_vs_bf16": 0.11058851350977504, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.021050943712077524, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.11028606504506488, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.02031181390576645, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.1102761849181443, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.021175903934800025, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.11036448979676117, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.024829628699461582, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.11038549730025582, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.025197331970293023, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.11023880060933477, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.024593436436438104, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.11020389366791007, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.025340557337809926, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.07794368758293942, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05556343061909954, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9836374453866401, + "packed_nmse_vs_bf16": 0.07743474564591603, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05663130345704791, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9836374453866401, + "packed_nmse_vs_bf16": 0.07762519598818701, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05466807140121925, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.0777868200178963, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05559425662975461, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 66, + "hessian": { + "down_diagonal_mean": 52.11198425292969, + "gate_up_diagonal_mean": 643.93359375, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.1106495472213296, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.020189418170058447, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.11072730216234168, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.020295401143162136, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9836374453866401, + "packed_nmse_vs_bf16": 0.11084681763753126, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.020534009294689563, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.11075786952155682, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.02027749617211772, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.11044326200367865, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.022673031201161032, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.11072494345569663, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02254755333277921, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.110523215128865, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.022917160301994248, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9745924476534854, + "packed_nmse_vs_bf16": 0.11081289524266047, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.022231784065905185, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.07776720676688427, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.056898851309558234, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9836374453866401, + "packed_nmse_vs_bf16": 0.07759357791665261, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05679611258313357, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.07730198808825192, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05773037353735289, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0073175568802046, + "packed_nmse_vs_bf16": 0.07766705790158594, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05659270514575957, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 68, + "hessian": { + "down_diagonal_mean": 59.543296813964844, + "gate_up_diagonal_mean": 636.8530883789062, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 0.9870923270925396, + "packed_nmse_vs_bf16": 0.10923243346678893, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.0177942938970502, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10914832911459431, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.01790997158533369, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10896310951121606, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.018152158463273827, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10882815915961182, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.017808295367601253, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10899644085202402, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.018704338284763644, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.1088917300772814, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.01881574137945302, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10892076010290883, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.01925301845907999, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10880412828495611, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.019345592743065864, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.07696064714155064, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.056315654826939975, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.07645012856282607, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05785859053495421, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.07682680983054657, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.0571990120722773, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.07658701579498246, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05641962164773114, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 70, + "hessian": { + "down_diagonal_mean": 51.14374542236328, + "gate_up_diagonal_mean": 642.0706787109375, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10687495472136375, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.02142906461507555, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10693242794115336, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.021721785994119985, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10698149665566421, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.021603274829554906, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9690023316262302, + "packed_nmse_vs_bf16": 0.1072591933195453, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.021539247858640042, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10685982984401284, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.024436298252506667, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10690445375131578, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.0246707165141766, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9836374453866401, + "packed_nmse_vs_bf16": 0.1070074633606469, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.024815873249177903, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10687687347295652, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.024486777811624578, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0073175568802049, + "packed_nmse_vs_bf16": 0.07686774613960791, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.057175814075606006, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.07694668665553413, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.057316727507458184, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.07704854572613212, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05731567786638257, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.07736109563218867, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05689731578793492, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 72, + "hessian": { + "down_diagonal_mean": 53.42240524291992, + "gate_up_diagonal_mean": 644.4412841796875, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.1059321412419449, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.020213069338936268, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.99827255914705, + "packed_nmse_vs_bf16": 0.10604954252542376, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.020350631921797712, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0073175568802046, + "packed_nmse_vs_bf16": 0.10590400390675891, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.020370714595354555, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10608827969648887, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.020199434371592252, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.1060593343790459, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.022713064505203627, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10599499702744508, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.022782669082650017, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.99827255914705, + "packed_nmse_vs_bf16": 0.10601578100774652, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02269967580676395, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.99827255914705, + "packed_nmse_vs_bf16": 0.10612208899181534, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02220474660299456, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.07725193036351224, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.056613920829943175, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.0775382817156292, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.056450719104153224, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9836374453866401, + "packed_nmse_vs_bf16": 0.07752016805834933, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05587539497300785, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.07715747307338272, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.057060106019151846, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 74, + "hessian": { + "down_diagonal_mean": 135.96939086914062, + "gate_up_diagonal_mean": 623.3191528320312, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10674803515529126, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.008133669743900345, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10695951268984236, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.008111103992880904, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.1067312482703752, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.008392615128522794, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10684660545198522, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.008348706162527421, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10674880401578712, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.010035307075267239, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10696505656947998, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.009293459227442968, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10692319960214089, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.00987682518709686, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9836374453866401, + "packed_nmse_vs_bf16": 0.1067576751576914, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.009762150840916107, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0073175568802046, + "packed_nmse_vs_bf16": 0.0911112527410721, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.03327416344054707, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.09035669815429803, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.03083534315882386, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.09011494849193723, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.03263198176229768, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.0913389090430515, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.03318993038331806, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 76, + "hessian": { + "down_diagonal_mean": 86.40695190429688, + "gate_up_diagonal_mean": 625.043212890625, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10588845650409023, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.013274553070799342, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10582178675130166, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.01262654247038211, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.1058226969815869, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.013647042097510177, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10578759675848667, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.013204387816157747, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10559465160703725, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.014077280877519499, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10576913536748585, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.014845045482572095, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10566391048895159, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.014903669313863482, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10565566030364532, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.014487504249285375, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9836374453866401, + "packed_nmse_vs_bf16": 0.08982521740040628, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.03465971117071828, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.0896960567096918, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.03437570495641947, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.99827255914705, + "packed_nmse_vs_bf16": 0.08927962672084062, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.037223421863415235, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.08992682739540506, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.034262195268284105, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 78, + "hessian": { + "down_diagonal_mean": 61.78431701660156, + "gate_up_diagonal_mean": 648.421875, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10880647963263436, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.016736629313648308, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10883335923425032, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.017363792601177493, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10883946609008345, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.01659489781992022, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10869695769526654, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.01619587871663982, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9836374453866401, + "packed_nmse_vs_bf16": 0.10881133766778518, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.01773814259656085, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10866706141383717, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.017766460784214223, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9836374453866401, + "packed_nmse_vs_bf16": 0.10897384009891348, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.01802133221915572, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10883337561288497, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.01753645304255254, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.08038884959661398, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.051598250984918624, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.08034022823738782, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05203774500941022, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.08121233394482007, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.04910226160773745, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.08108909952510572, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.050567508784885994, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 80, + "hessian": { + "down_diagonal_mean": 40.8306770324707, + "gate_up_diagonal_mean": 639.020263671875, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 0.9745924476534854, + "packed_nmse_vs_bf16": 0.10478391269386235, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.021563200946540365, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10454091714435222, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.02134245432815653, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.1043260614800932, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.02170966034498007, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10469232062291065, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.021317055156186815, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10461784992368288, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.025815517842123145, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10440096632118417, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02540580361337676, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9836374453866401, + "packed_nmse_vs_bf16": 0.1045145197185365, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02588081295376609, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10463000399554391, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.025293807914308273, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.07707346257893737, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05591794392947816, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0073175568802049, + "packed_nmse_vs_bf16": 0.07757006007388995, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05569887977006421, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.07760660531041741, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05584842278857915, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.07789127615934963, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.055009614556311065, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 82, + "hessian": { + "down_diagonal_mean": 53.47895812988281, + "gate_up_diagonal_mean": 651.1846313476562, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10768035847377012, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.018993488926515704, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.107711084052305, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.019076132218881374, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10772803689357563, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.019014253985940626, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.1078892386582653, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.01906304626271621, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10763367175194287, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.01865377345526681, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10773906340543239, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.019351703951275296, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10780303706324196, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.019261696507024167, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10776917557071772, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.01945096908991978, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9836374453866401, + "packed_nmse_vs_bf16": 0.07807429107835819, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.052802828990944815, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470503, + "packed_nmse_vs_bf16": 0.07758628907140273, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05442280228397954, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.07783940181432802, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.0541389917892781, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.07744180773304794, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.054298498223532214, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 84, + "hessian": { + "down_diagonal_mean": 54.443885803222656, + "gate_up_diagonal_mean": 646.5548095703125, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10702646525593809, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.017514359712527676, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9836374453866401, + "packed_nmse_vs_bf16": 0.10686250990778544, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.017740211184086935, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10709415660266706, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.01709366767574041, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.1069806964713333, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.018064726661234317, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10704240033555316, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.020318500072280764, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10698602015635467, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.019603122893158516, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10694465700259895, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.019598541706647533, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10706568022666921, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.01979648543063898, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.0824677238566792, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.046888437904488926, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.08136776682535705, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.04646685050680157, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.08222499580651396, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.04728548930484697, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.081852481841987, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.04727021140940506, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 86, + "hessian": { + "down_diagonal_mean": 60.79225158691406, + "gate_up_diagonal_mean": 641.0740356445312, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10759378317407779, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.017723447642598263, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9836374453866401, + "packed_nmse_vs_bf16": 0.10759603563774413, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.017659367835675264, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10749960734942043, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.01774410223491159, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10770064803369768, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.017794798096813357, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9836374453866401, + "packed_nmse_vs_bf16": 0.10754125224344216, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02040205403660177, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10749726499940271, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.021242003236326596, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10758275703762624, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.020677708515624216, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10743596850895985, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.020956283598738516, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.07704116969060854, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05699712407797631, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.07679451941318778, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05769057206743424, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.07693192362074927, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.057696071537374376, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470503, + "packed_nmse_vs_bf16": 0.07698909645361207, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05744998912162028, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 88, + "hessian": { + "down_diagonal_mean": 55.61518096923828, + "gate_up_diagonal_mean": 641.3394165039062, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10512517147429533, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.021050480378315424, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10520794373629674, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.020522559469970923, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9836374453866401, + "packed_nmse_vs_bf16": 0.10511407522520429, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.02073231995689078, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10512345805038861, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.020985762611155007, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10515159799721371, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02252679136444815, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10492079442092006, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.022039548451413246, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10520066794233725, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02135286342231532, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10526806745460379, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.022014305245404823, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.07549510884765787, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05889851037302784, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.07550269194917168, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.058489851431721675, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.07586105885500549, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.057905842216083456, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.07572400765541921, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05842747206137586, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 90, + "hessian": { + "down_diagonal_mean": 51.963096618652344, + "gate_up_diagonal_mean": 641.82470703125, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10671481429649485, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.022361274652047177, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.1066850471727627, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.021917563116812106, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10662873047348576, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.02236814819884326, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10682838718562171, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.02216002701241958, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10693438079979647, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02516419032121035, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10670088255345014, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.025284753861640672, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10676569447458761, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.025249865748228652, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10671651390388755, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.024761811859904614, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.07655594347919902, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05670784050851218, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.07630808179472617, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05729903334394408, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.007317556880205, + "packed_nmse_vs_bf16": 0.07684784525156757, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05615751335752355, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.07662963443031039, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05514947848295693, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 92, + "hessian": { + "down_diagonal_mean": 45.729393005371094, + "gate_up_diagonal_mean": 642.2188720703125, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 1.0073175568802046, + "packed_nmse_vs_bf16": 0.1047462558858404, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.020776186806547237, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10488389130529507, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.021618577861230847, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10480933279693054, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.02169120199852221, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10480005261685102, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.021615681900981065, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529497, + "packed_nmse_vs_bf16": 0.10453017994645476, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.024563416889652658, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10468457884192843, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.025095957749061096, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10467051436682163, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.025042321165943283, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.99827255914705, + "packed_nmse_vs_bf16": 0.10463231476015984, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.025257201144865286, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.07836099087489784, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.052406477163783126, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.07797105544398115, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05405893771243878, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.07811878503548561, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05348788885559585, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.07801487331480914, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05497225840195526, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 94, + "hessian": { + "down_diagonal_mean": 55.48527145385742, + "gate_up_diagonal_mean": 641.685791015625, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 0.9926824431197949, + "packed_nmse_vs_bf16": 0.10573790096646578, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.019821763891554045, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.1056653335123122, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.020074573809039663, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10579708512367525, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.020348816793075153, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.007317556880205, + "packed_nmse_vs_bf16": 0.1055967934020244, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.020576065048528526, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10560677857948583, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.022553380233021888, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10557707660191396, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02249512694753587, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10564698233127133, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.022631970532858006, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10576614116274334, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02254708633307606, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9780473293593848, + "packed_nmse_vs_bf16": 0.07605702644754755, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05839229025915154, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.07572677162436, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05872031707224492, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.07617689907852994, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.058256638713813765, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.07600725526959401, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.058469545590834435, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 96, + "hessian": { + "down_diagonal_mean": 78.72541046142578, + "gate_up_diagonal_mean": 626.2255249023438, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10878777513374209, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.015291216870838259, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10867608472023968, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.015129486741964498, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10873767059238044, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.015435059776153464, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.1087500265933223, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.014984955982023826, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197949, + "packed_nmse_vs_bf16": 0.10864712126545074, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.01598994387386542, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10866189990080374, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.015500714002628748, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10859447433263458, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.015204049196248114, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10837762788256963, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.015325430535959217, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.0809128302042327, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05330403311494611, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.0809499612993305, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.0535103883580904, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.08053345756187892, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.0537138296776528, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197949, + "packed_nmse_vs_bf16": 0.08081099174456029, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05290501956823513, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 98, + "hessian": { + "down_diagonal_mean": 59.595985412597656, + "gate_up_diagonal_mean": 643.0946044921875, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10744523078815463, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.019928491188605568, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10765333636419752, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.02026387592563973, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10764491304971328, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.01999581451467542, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10762968049331205, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.0204984929715058, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10737347275467936, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.022939189765259178, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.1074594920320305, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.023321210010836858, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10750890844965, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.023327508627187205, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.99827255914705, + "packed_nmse_vs_bf16": 0.1075785266252205, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.023526213774169594, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197949, + "packed_nmse_vs_bf16": 0.07690306639376498, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05761543141739514, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470503, + "packed_nmse_vs_bf16": 0.07694739725963196, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.058454140869222376, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.0764045077919489, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.0587959185369235, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.07668594891512369, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.058790754684015935, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 100, + "hessian": { + "down_diagonal_mean": 53.75645065307617, + "gate_up_diagonal_mean": 635.3321533203125, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10641241716827603, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.019694418741025253, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10615887557213001, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.01903009485528178, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10640552665039953, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.019555152817633283, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10628766226250323, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.01941093309643103, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10613134707860394, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.021604102913437805, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470503, + "packed_nmse_vs_bf16": 0.10620095486234865, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.021327306056528505, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10623033089981984, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.021716374372259505, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10619177455318246, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.021372480101092797, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.07553300520744738, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.058727825942703066, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925396, + "packed_nmse_vs_bf16": 0.07581521337354336, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.0580327107618946, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.07561838870519062, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05876488590975009, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.07584191529644425, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.058806426666705304, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 102, + "hessian": { + "down_diagonal_mean": 62.99961471557617, + "gate_up_diagonal_mean": 635.1765747070312, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 1.0073175568802046, + "packed_nmse_vs_bf16": 0.10634137168963577, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.018035701979577285, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10639453746356033, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.018266896693078117, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10668581212161209, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.018175549874115057, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10657320088202864, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.018209460950584965, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10649807396322031, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.0196180461036444, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0073175568802046, + "packed_nmse_vs_bf16": 0.10652903176263097, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.019587360090997655, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197949, + "packed_nmse_vs_bf16": 0.10645453906734854, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.01997641444042516, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10668045261125395, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02008852707634543, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.0777369017394887, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05608063125387036, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.07742937391066122, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.056500734764897774, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.07741153320415955, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05661531372086823, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9836374453866401, + "packed_nmse_vs_bf16": 0.07726177035762553, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.056710767123907393, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 104, + "hessian": { + "down_diagonal_mean": 100.4558334350586, + "gate_up_diagonal_mean": 628.3604125976562, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10816766734639374, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.013187856474072862, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9836374453866403, + "packed_nmse_vs_bf16": 0.10803204970525929, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.013095690541802672, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10807354968728061, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.012975074385094128, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10790771648712458, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.012840223834412356, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10811063522391953, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.014122948552700972, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10819529175976861, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.01364612014339815, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10792725217650309, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.014118186303910173, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10808422070524236, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.013683483812749904, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.08478054980807763, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.04400679725560668, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.08610757013289407, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.04326290095490379, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.08484171175438969, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.04538617445539271, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.08629957739746069, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.04327965522786631, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 106, + "hessian": { + "down_diagonal_mean": 56.95550537109375, + "gate_up_diagonal_mean": 646.4797973632812, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10626006891479761, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.01808011536664162, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9780473293593848, + "packed_nmse_vs_bf16": 0.10620457819736503, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.018130136602695388, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10624491242152244, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.01774016321549224, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10599032478377308, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.01786365262740355, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.99827255914705, + "packed_nmse_vs_bf16": 0.1058136434576403, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02120977723325099, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10588921441783891, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.021115895817102385, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.1060484258225919, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02113776801193907, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.1059176271064803, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02161235282272467, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.0787232810529741, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05301412462720638, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.07889582262870988, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05263033736394954, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.07889240492625978, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.052034370240981606, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0073175568802046, + "packed_nmse_vs_bf16": 0.07781584875218621, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05398239749233666, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 108, + "hessian": { + "down_diagonal_mean": 25.47459602355957, + "gate_up_diagonal_mean": 625.839599609375, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10441466889897862, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.025987739147010476, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10445417591381218, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.025739937588775068, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10440268502585774, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.026205783107389843, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9836374453866401, + "packed_nmse_vs_bf16": 0.1043897207036415, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.025550430199260624, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0073175568802046, + "packed_nmse_vs_bf16": 0.10418478858025107, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.030309175609029567, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470503, + "packed_nmse_vs_bf16": 0.10440353600158926, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.030572675138357187, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0073175568802049, + "packed_nmse_vs_bf16": 0.10435665495794955, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.03032665645031807, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10434908255020592, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.029919092187402494, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.07364438068873655, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.06148491950288537, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.07348159620995157, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.06188272947839079, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9836374453866401, + "packed_nmse_vs_bf16": 0.07358904482644275, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.061777401556623035, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.07389977738296345, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.060066743400283444, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 110, + "hessian": { + "down_diagonal_mean": 52.221588134765625, + "gate_up_diagonal_mean": 645.3934326171875, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 0.99827255914705, + "packed_nmse_vs_bf16": 0.10659279346411207, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.018102989669042155, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.1066511219641112, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.018445118460620684, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.1066616203584092, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.01883955041947341, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.1066762266958078, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.019200271753147872, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10669306377904313, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.021306264840189777, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10660280696274563, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.021746918143314854, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10666674151329368, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.021968567718387552, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10663283837686487, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02166604918514636, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.07860103630651685, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.053002235474030816, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.0786258663310038, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.052827611412651285, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.07891841001725551, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.051689845332167604, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.07900399111389506, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05409895567904001, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 112, + "hessian": { + "down_diagonal_mean": 62.28364562988281, + "gate_up_diagonal_mean": 640.6631469726562, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10749353318816496, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.017137759413322542, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10752552583358828, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.016921766703614644, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10747000924943377, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.01719519772913477, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10761895203206882, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.017053223069590423, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10763136886834114, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.01996891793215648, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10744497491946711, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02002050397850501, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10733054082391094, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.01980046853024783, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.1074225904721609, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02025179229065296, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.07734445910119726, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.056880672287350116, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.07689157701825185, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.057145468866964325, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.07711357265934765, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05704545204773788, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.0771944309542953, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.057206604947404704, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 114, + "hessian": { + "down_diagonal_mean": 57.71485137939453, + "gate_up_diagonal_mean": 640.679931640625, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10630370218700617, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.0202575458334435, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10620658205957938, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.019894008733419035, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.1062568168969787, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.019833830650079664, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10622307722119133, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.0198970113670239, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10594439792820018, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.021963486020294937, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10627682882143405, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.022137268972692466, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10607882775922028, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02206593096746733, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0073175568802046, + "packed_nmse_vs_bf16": 0.10610571422038113, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.0222279112363947, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.0783548798963317, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05345312102729118, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.07806001714839886, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05485512071746249, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.0783250044619096, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05483464101285692, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.07848687411371129, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.053949928867157645, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 116, + "hessian": { + "down_diagonal_mean": 43.26089859008789, + "gate_up_diagonal_mean": 638.5762329101562, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10603788616202772, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.024187472162739906, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10642248068903475, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.024256668682805956, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10608960874497182, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.024314925277228915, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0073175568802046, + "packed_nmse_vs_bf16": 0.10604598994364632, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.02424131390602397, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10612336754412421, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.025795190957678543, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10624983101264986, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.025826886744843125, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10610848356931167, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02606037407125561, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0073175568802049, + "packed_nmse_vs_bf16": 0.1058998774725969, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02596140015182818, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.0774920947040265, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.057148151586443416, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.07728659470218388, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05718773297144397, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.07707264844625085, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05742081601007394, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.07739939137703544, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05594461215890692, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 118, + "hessian": { + "down_diagonal_mean": 44.822654724121094, + "gate_up_diagonal_mean": 644.765869140625, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10965674825946867, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.01988744446200372, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10957632332516282, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.020285381187720147, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10955540696994243, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.01987377934965371, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.1096994565875783, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.020534563336895753, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10963375074267158, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.022319838215032105, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.1096169228022706, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.022494114742592688, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10966937032515799, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.022798348910513924, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10984097071394441, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.022362613190019927, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.07599546471896426, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.0589207207180077, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.07609498181916423, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05905756608870944, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.07602395398707777, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05873986131993058, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9836374453866401, + "packed_nmse_vs_bf16": 0.07656432234725068, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05866655964880741, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 120, + "hessian": { + "down_diagonal_mean": 40.41440963745117, + "gate_up_diagonal_mean": 645.24365234375, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 0.9836374453866401, + "packed_nmse_vs_bf16": 0.10543983874988039, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.0232143180285204, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470503, + "packed_nmse_vs_bf16": 0.10513137215632473, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.022860470016970284, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.1052602217227208, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.022884001151545083, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10541099578740595, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.022685960601155648, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10522781250541428, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02597742585736732, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9836374453866401, + "packed_nmse_vs_bf16": 0.10526830045139975, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.026154768014851153, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10513671501082865, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02615437860696946, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10526673391659862, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.025714603997983292, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.07635029449948827, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.057852108512362166, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9836374453866401, + "packed_nmse_vs_bf16": 0.07678852877092479, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05751182908215791, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470503, + "packed_nmse_vs_bf16": 0.0765866261976691, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05671422413894766, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.07681401707529262, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05621557997233825, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 122, + "hessian": { + "down_diagonal_mean": 59.60932159423828, + "gate_up_diagonal_mean": 637.0971069335938, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10842120993688333, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.016409041448322264, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10855203472959127, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.016745295381281933, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10853028845039593, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.01632455957681098, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10857782759669328, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.01705079110618876, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9836374453866401, + "packed_nmse_vs_bf16": 0.10852079432448099, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.018592719649093196, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10843765373265035, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.018886289246966618, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10848142460098015, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.018720918043010445, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10851108829461648, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.019302186985741248, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0129076729074602, + "packed_nmse_vs_bf16": 0.07795165261224078, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05583123147806342, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.07781359319095947, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.056473980953262774, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.07785839995006276, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05671520507072874, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.0774684946386109, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05718949716982081, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 124, + "hessian": { + "down_diagonal_mean": 43.24055480957031, + "gate_up_diagonal_mean": 643.798828125, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10870922398983034, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.021011510813189478, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10869221417034863, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.02101873644357324, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9836374453866401, + "packed_nmse_vs_bf16": 0.10877872135914551, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.02108898534696588, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9836374453866401, + "packed_nmse_vs_bf16": 0.10873405127974835, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.021365727162088184, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10857070583924301, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.023701756412277338, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10869265216147088, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.023961534323511027, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10873801264144234, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.023769285408164382, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9836374453866401, + "packed_nmse_vs_bf16": 0.10859667027378547, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02412369589685312, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.07730875327563638, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05724888286508478, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.07748981373025499, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.056979200357125635, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.07750588601998329, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05602928813361648, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.07686888220561894, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05722832675087006, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 126, + "hessian": { + "down_diagonal_mean": 83.25260925292969, + "gate_up_diagonal_mean": 633.0789794921875, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10362933128017432, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.012658940765079047, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10379709127287645, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.013258318756405377, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470503, + "packed_nmse_vs_bf16": 0.10354334388421943, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.012765708229619267, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.1038213677669985, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.01307033835263732, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10368383790201433, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.014273136354751617, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10361099400979147, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.014583246914310689, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10357688406556588, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.014414795245845755, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.1037050639753995, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.014646804461855203, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9836374453866401, + "packed_nmse_vs_bf16": 0.0879696540306493, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.03820229015695948, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.08693551784066089, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.04126802754280016, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.08714051273891575, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.041458051455650166, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.08530078257264594, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.04170144684761617, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 128, + "hessian": { + "down_diagonal_mean": 50.92727279663086, + "gate_up_diagonal_mean": 642.0431518554688, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10546388260992035, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.020876259179918734, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10524790230043087, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.020456940782451968, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10531929688035985, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.020189686793719033, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10567966421307687, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.020721764391532768, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10533297029453194, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.025627388632742415, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10528941126909402, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02528589583173854, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.1055107914933566, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.024912800455265696, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10549363067512013, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.025224586202346345, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9836374453866401, + "packed_nmse_vs_bf16": 0.07877069400460585, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.053547064761161514, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.0786532250810568, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05372304609127515, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.0790962102216997, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.052514633134179214, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.07978228610235796, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05118558555924948, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 130, + "hessian": { + "down_diagonal_mean": 50.2174186706543, + "gate_up_diagonal_mean": 645.3516845703125, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.1052801346394798, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.022332177261036392, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10520520455944139, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.022291187058657017, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10514278168234083, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.022307342300681713, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9836374453866401, + "packed_nmse_vs_bf16": 0.1052512960031357, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.022034385068292577, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9836374453866401, + "packed_nmse_vs_bf16": 0.10519846381090264, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.024561172507217233, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10523156384133442, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02454969648087191, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.1053583205390633, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.024659950853098262, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0073175568802046, + "packed_nmse_vs_bf16": 0.10496758991554524, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.024794739839022514, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.077094686320681, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05727228230133674, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.07709770439749687, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05679902652435452, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.07734648029277108, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05672794435000708, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.07696682753084656, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05710980393845675, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 132, + "hessian": { + "down_diagonal_mean": 52.22130584716797, + "gate_up_diagonal_mean": 643.6414184570312, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10669805558667997, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.01910606431348032, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10693258052715841, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.019004876964772308, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10680514744734458, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.0194280188618097, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10680830134969828, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.019399448590290563, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10692850933837895, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02238124875939571, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.1068750743749951, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.022071973130504283, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10680268280476785, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02261845849475125, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10692862117493185, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02278499739375858, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.07866915512755501, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.054319944260725984, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.07876806950161414, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05498209545664242, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.07879356735609416, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.054870611934450926, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.079211811035362, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05409731706860746, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 134, + "hessian": { + "down_diagonal_mean": 33.63249969482422, + "gate_up_diagonal_mean": 642.6016235351562, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 1.0129076729074602, + "packed_nmse_vs_bf16": 0.10461304477531014, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.025711092080722447, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0129076729074602, + "packed_nmse_vs_bf16": 0.10459424469484428, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.025288153710235746, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10480520644175917, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.025667983849419894, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10438161770810371, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.025438392646161465, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10468156494976855, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.028949591896912315, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0073175568802046, + "packed_nmse_vs_bf16": 0.10458257311634032, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.028451431482023278, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9836374453866401, + "packed_nmse_vs_bf16": 0.10463096891130606, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.028418706614015278, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.1045982357607017, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.028784571045011746, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9836374453866401, + "packed_nmse_vs_bf16": 0.07530977675169032, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05953651514994854, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.07535543073034087, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05920578907146269, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.07514272939371396, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05985034942531323, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0073175568802046, + "packed_nmse_vs_bf16": 0.07528397780136248, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.059632592410785626, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 136, + "hessian": { + "down_diagonal_mean": 61.44130325317383, + "gate_up_diagonal_mean": 630.940185546875, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 0.9926824431197949, + "packed_nmse_vs_bf16": 0.10821766606138486, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.018991819326560008, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10829477137797724, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.01898892218223446, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.1082367533323124, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.019373160083079925, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10807477410266421, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.018943257968104418, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9836374453866401, + "packed_nmse_vs_bf16": 0.10800581310953718, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.020321904338665413, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10826274249888863, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02062181151891204, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10805529109108662, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.020526473523572706, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0129076729074602, + "packed_nmse_vs_bf16": 0.10816010062974221, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02094396083816246, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.08030857713515244, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05207366110404158, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.080930000875326, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.051074460851710496, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.08098134825003965, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05104105529761806, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.08015983081602009, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.051926243095237, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 138, + "hessian": { + "down_diagonal_mean": 59.768165588378906, + "gate_up_diagonal_mean": 637.6700439453125, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10774073843108763, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.019766499314263372, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10740941460471942, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.019965483838459138, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.1075809342045858, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.020017793787720197, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9836374453866401, + "packed_nmse_vs_bf16": 0.10777019718939775, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.02014723563247195, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.1074915671646348, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.021541307902922455, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10738747427847954, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02198841738952625, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10764081012603188, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.021888913635557786, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470503, + "packed_nmse_vs_bf16": 0.10758570250790615, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.022255405230659973, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470503, + "packed_nmse_vs_bf16": 0.07720280778132182, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05696995769781819, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.07719746991660588, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.057087055616219504, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0073175568802049, + "packed_nmse_vs_bf16": 0.07725981289564715, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05656646486634351, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529497, + "packed_nmse_vs_bf16": 0.07704000867019604, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05770793541466305, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 140, + "hessian": { + "down_diagonal_mean": 64.61366271972656, + "gate_up_diagonal_mean": 639.3590087890625, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10630844508686099, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.01989448480104, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10649874584762405, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.020073541998409102, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.1065262653267312, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.020403946550779938, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10621806063369517, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.020214610309639416, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10624972125584073, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02291032528795633, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10639791596524686, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02307408811251378, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10641395941116762, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.023084715140797088, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10631348799991525, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.023491487674772604, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.07971578319321035, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05489952767703927, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.07925690848545257, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.055348443765098024, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.07964444602716513, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05469889631402699, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.07914847976207778, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05455792357242112, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 142, + "hessian": { + "down_diagonal_mean": 52.42053985595703, + "gate_up_diagonal_mean": 644.5433349609375, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 0.9982725591470503, + "packed_nmse_vs_bf16": 0.10692631484555044, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.019521492769760514, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.007317556880205, + "packed_nmse_vs_bf16": 0.10684620711732071, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.01959178900654392, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10693822322656148, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.019484743202116166, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10700066528587432, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.018858421168463823, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10701822036689512, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.021992960876923106, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.1070908468519687, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.022179285671028358, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10672521953428707, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.022658684221889654, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10689926470756336, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.021989602507194183, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.07622586308064272, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05759034917363314, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.07639988688006491, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.058328882065773524, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.07599697609470386, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.058358430451091195, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.07625094482567193, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05762728404556906, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 144, + "hessian": { + "down_diagonal_mean": 57.19956970214844, + "gate_up_diagonal_mean": 639.9132690429688, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10637180924633906, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.020119451627911036, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.99827255914705, + "packed_nmse_vs_bf16": 0.10640482453766666, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.019620235493577227, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10648594387103032, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.019752247498932663, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10644426411168016, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.020241424791966608, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10640594068045739, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.025012591609296107, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10633281126737637, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02478946861375902, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10642585105082202, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.024733178873764132, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10631561305328599, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02467917453703321, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.0801106817822721, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.04765497480632525, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.08047942442174733, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.04838797648615064, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.99827255914705, + "packed_nmse_vs_bf16": 0.08004138134310126, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.04997203821051546, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.07998654344974535, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05150606183446295, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 146, + "hessian": { + "down_diagonal_mean": 70.24685668945312, + "gate_up_diagonal_mean": 645.3232421875, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10548005657748626, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.01578957988358105, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10557329236021125, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.015595431741948626, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10545536676351824, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.015277353460619057, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10551390042712927, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.015327583289921924, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10555075641731874, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.018525893322590213, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10570520854772796, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.018596343504628514, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10544156934375416, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.018706480705586632, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10531862168573905, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.018546714199220028, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.08036149544122814, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.0524639839243956, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.07974391463433132, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05308535528986543, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.08046027150698731, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05265395773398778, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.0803071492395032, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.053152890305906975, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 148, + "hessian": { + "down_diagonal_mean": 44.907447814941406, + "gate_up_diagonal_mean": 642.7547607421875, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 0.9836374453866401, + "packed_nmse_vs_bf16": 0.10699860893775927, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.021920422896968095, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10686218804136674, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.021535763346036532, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10678005863139474, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.021681211163960754, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470503, + "packed_nmse_vs_bf16": 0.10676955039990445, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.02158610804935956, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10696171636476047, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.024621975972676926, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.1068087890880493, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02463868599787189, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.1069617273138932, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02460920871490354, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10696485990944743, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.024501475570398022, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.0762417164135939, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.0587624727541639, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.07591785820328326, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05927622418767571, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.07575773966365547, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05974616195181102, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.07578814697093383, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05945858297350771, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 150, + "hessian": { + "down_diagonal_mean": 62.275230407714844, + "gate_up_diagonal_mean": 638.6527709960938, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10623604560773293, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.02092816136258701, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10634421837134718, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.02101945817508736, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10626190791389237, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.020669037300975374, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10633457528534367, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.021026564679131676, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10619745226278256, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02391053789261716, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10627153636484082, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.023718751355462205, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10619198810150703, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.023500114590647613, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10642241199302704, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02343438347790941, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.07690091224986134, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05845830874040876, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9836374453866401, + "packed_nmse_vs_bf16": 0.07719436188085682, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05443214717054115, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9745924476534854, + "packed_nmse_vs_bf16": 0.07740865751782085, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.057644962430985096, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.07674556311327839, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.058514994993385944, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 152, + "hessian": { + "down_diagonal_mean": 45.80024337768555, + "gate_up_diagonal_mean": 642.5732421875, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10829882845099326, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.021439376821344187, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10842269401008076, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.021467587125384494, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9836374453866401, + "packed_nmse_vs_bf16": 0.10847284569983277, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.02119759024245134, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10854324032267466, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.021336415520742235, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.108257023476808, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02559742626883533, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9836374453866401, + "packed_nmse_vs_bf16": 0.108261912240027, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.025243154444811025, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10821090434874989, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.025166513045903446, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10826884703455043, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.025322744187477413, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.07752707073003526, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05614425089309207, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.07750720024853816, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.0560103557867078, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.07751742418062901, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05515960211859281, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.0777311037065679, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.055499484270615664, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 154, + "hessian": { + "down_diagonal_mean": 50.30732727050781, + "gate_up_diagonal_mean": 645.647216796875, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 0.9982725591470503, + "packed_nmse_vs_bf16": 0.10652044608812868, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.02129070232311047, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10653649746069604, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.02116738074192918, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10660538077011464, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.021498517673971665, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.1065658853189854, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.021196175055706427, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10662369021106781, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.023200647021903845, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.99827255914705, + "packed_nmse_vs_bf16": 0.10670298430215511, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02342256241743019, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9836374453866401, + "packed_nmse_vs_bf16": 0.10662332946124427, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02293847606375433, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10664636846378779, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.023306547299399422, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0073175568802046, + "packed_nmse_vs_bf16": 0.07457614096342974, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.060511502073818474, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.0745631688542201, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.060383038611986184, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.07478295225062369, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.060612353487288655, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.07470265990957381, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.06063146420693428, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 156, + "hessian": { + "down_diagonal_mean": 61.52935028076172, + "gate_up_diagonal_mean": 643.775634765625, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10666146110374018, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.018832429974491492, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10659964397678275, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.018964584709526264, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10665942179245766, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.01866380559244038, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.99827255914705, + "packed_nmse_vs_bf16": 0.10674977303081953, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.01895615275706937, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10671044937833496, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.0215293663417977, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10645412349927337, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.022143517135918205, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10659459952788797, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02146553294086557, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10648178898459115, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.021147015714834957, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470503, + "packed_nmse_vs_bf16": 0.07607198988135987, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05900771250402663, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.07591154920207974, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.059223001212596715, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.0760786604146931, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05882359759696068, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.07607071110095807, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05847017145094288, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 158, + "hessian": { + "down_diagonal_mean": 69.45677947998047, + "gate_up_diagonal_mean": 639.9490966796875, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10954836100873336, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.016774889152057274, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10943460599737488, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.016202154015467707, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10929138231433888, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.01624868328018742, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10951912537933353, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.016645860216761933, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10962223393612139, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.01808270682866164, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10949864308371643, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.018408885448410826, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10941695033441283, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.018105878361975462, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10925195366286042, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.018743215046963146, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.99827255914705, + "packed_nmse_vs_bf16": 0.08013606513764727, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05473333805184121, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.08019776713668147, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.054826250973403934, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9836374453866401, + "packed_nmse_vs_bf16": 0.08002610326883323, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05409137268997254, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.99827255914705, + "packed_nmse_vs_bf16": 0.08020018597085588, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05507317447090224, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 160, + "hessian": { + "down_diagonal_mean": 52.44517517089844, + "gate_up_diagonal_mean": 642.0303955078125, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10502718693351944, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.020770144709072312, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10493248666327683, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.020424294477716983, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10485673670483026, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.020889224225152957, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9836374453866401, + "packed_nmse_vs_bf16": 0.10490456271946776, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.020458099562902295, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10478801989054266, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02528278904936138, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10472730028676272, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02527787752347133, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10480870894159251, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.025040957092589893, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10467678391685985, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.024986225300180732, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.07819680194932725, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05507236800534724, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.07884309307938663, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05345526562456685, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925397, + "packed_nmse_vs_bf16": 0.07851779678406066, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05461795984230228, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.0780742146802901, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.055081774294812036, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 162, + "hessian": { + "down_diagonal_mean": 54.279136657714844, + "gate_up_diagonal_mean": 643.4781494140625, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10542691108985167, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.01842849163911656, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9836374453866401, + "packed_nmse_vs_bf16": 0.10551897791431578, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.018026149201709373, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.1051331662135604, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.018292569652669257, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10514422136845641, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.018391714444604437, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10536583001336976, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02307593647972925, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9836374453866401, + "packed_nmse_vs_bf16": 0.10530069655243722, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.022984134542109087, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9836374453866401, + "packed_nmse_vs_bf16": 0.10543068556122125, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.022668672787895642, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10543158286998257, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02309674348019171, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.07686722012846227, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.056712785058940764, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.07718378887645529, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05634969363903996, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.07739497183100875, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05638927114108258, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.0768342017979843, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05628515903527406, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 164, + "hessian": { + "down_diagonal_mean": 49.99810028076172, + "gate_up_diagonal_mean": 643.7659301757812, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 1.007317556880205, + "packed_nmse_vs_bf16": 0.10673696562321063, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.02098000014764815, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10690160927955579, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.021487032420815507, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10677396283609822, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.020968378681998634, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10680213424770883, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.0214479196705898, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10693267539905996, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02495200226826761, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9836374453866401, + "packed_nmse_vs_bf16": 0.10673223514676729, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02505603285845299, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10677977112173362, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.025291290444434702, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.1068624168750505, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.024962314701178167, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.07612997277188095, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05881454947648831, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.07609602035742423, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05923965684239657, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.978047329359385, + "packed_nmse_vs_bf16": 0.07599616899052315, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05919885389965345, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.0760975976836939, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05906540099309405, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 166, + "hessian": { + "down_diagonal_mean": 74.0488510131836, + "gate_up_diagonal_mean": 630.76953125, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 1.0129076729074602, + "packed_nmse_vs_bf16": 0.1079099588984489, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.017867252075467268, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10783703314318581, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.017517488320935578, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.1077267967365463, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.017603483132770843, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9836374453866401, + "packed_nmse_vs_bf16": 0.10777075364498942, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.017651742764466453, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10746482480760054, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.019190146776513737, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10760879880386234, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.01888869859538018, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.1077268785123771, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.018622544630610086, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0129076729074602, + "packed_nmse_vs_bf16": 0.10790011074342958, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.019268133990609546, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.08286535453478404, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05130703102515884, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.08241674958791659, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05257346306905623, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.08265043745127576, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.0520753063422034, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.08176081663851861, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05300520333089317, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 168, + "hessian": { + "down_diagonal_mean": 40.69237518310547, + "gate_up_diagonal_mean": 643.1610107421875, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 0.9836374453866401, + "packed_nmse_vs_bf16": 0.10516976915081774, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.024425513656106627, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0073175568802046, + "packed_nmse_vs_bf16": 0.10494829967288011, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.024193838558523, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10509288725127759, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.024161771457842648, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9836374453866401, + "packed_nmse_vs_bf16": 0.1052324865219653, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.023971449424422645, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9836374453866401, + "packed_nmse_vs_bf16": 0.10506821892033112, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02697800711160781, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10492629257104735, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02717688596676302, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10504143070024773, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02697642533331656, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.1051671974761342, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.026777334338273336, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.0753521799785554, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05952421388432986, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0129076729074602, + "packed_nmse_vs_bf16": 0.07528314312690193, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.059771475902789147, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9836374453866401, + "packed_nmse_vs_bf16": 0.0754641128101121, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.0596493037371872, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.07577724731114847, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.0580674440447118, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 170, + "hessian": { + "down_diagonal_mean": 50.13011169433594, + "gate_up_diagonal_mean": 642.0795288085938, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10613026818851123, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.020651433868816755, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10590829610771382, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.021451200949459525, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10603139002470163, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.021285082987094334, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10596717640276263, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.021214197921695214, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10586332224502433, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.024733972345020335, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10601427350484699, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.024741738093297917, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10590944907779216, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.024113527175780355, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10623793776277433, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02416989962317817, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470503, + "packed_nmse_vs_bf16": 0.0749923113154955, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.06055027248941758, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.07531295720216985, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05973598468553153, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.0756994738654024, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.059522763980550474, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.07541146833934129, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.059912265832028536, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 172, + "hessian": { + "down_diagonal_mean": 53.742088317871094, + "gate_up_diagonal_mean": 635.4959716796875, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10501637161543988, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.020979099616025326, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10494699586189682, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.020973521640550292, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10483861984936686, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.02092028347140977, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.99827255914705, + "packed_nmse_vs_bf16": 0.10478756364615346, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.02096847881720212, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10478476621081657, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.025296308059693573, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10465453751511068, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.025016526054022416, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10466024361253298, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.025337664624788005, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.1047743631241307, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.024537679089197992, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9745924476534854, + "packed_nmse_vs_bf16": 0.07988343257663726, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.051509103613801795, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470504, + "packed_nmse_vs_bf16": 0.07952052276098313, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.052990679288926576, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9745924476534854, + "packed_nmse_vs_bf16": 0.07965431076878896, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05290478985774291, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0073175568802046, + "packed_nmse_vs_bf16": 0.08075905005062328, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.048659832220176086, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 174, + "hessian": { + "down_diagonal_mean": 60.194786071777344, + "gate_up_diagonal_mean": 641.6227416992188, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 0.9870923270925397, + "packed_nmse_vs_bf16": 0.10517292648521846, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.023079838574457067, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.1051382933833049, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.023168821954000027, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10509588841368918, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.023039302693109092, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10496442264965707, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.023169111900240427, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10498904309916746, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.024409766978979727, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0073175568802049, + "packed_nmse_vs_bf16": 0.10513666827681672, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02431742642375671, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10519461717150365, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02395293386829632, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10498575840831355, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.024325615203004, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.99827255914705, + "packed_nmse_vs_bf16": 0.07644916826937698, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.058633146433751226, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.07653773174314596, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.058430921637778105, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.07702782682278858, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.057090011080495676, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.07665843249962713, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.058527556357075475, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 176, + "hessian": { + "down_diagonal_mean": 72.00360107421875, + "gate_up_diagonal_mean": 637.191162109375, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10717629120851517, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.017872509102109597, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10711447113635873, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.018226169415375394, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10731015280776994, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.01829680420195519, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10730372882383087, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.01838484180500118, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10707031518701333, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.020134878081827658, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470503, + "packed_nmse_vs_bf16": 0.10695468241294781, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02022284324497928, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9836374453866401, + "packed_nmse_vs_bf16": 0.10729188479730484, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.019790616046772402, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10710272754441326, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02040552834519838, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9836374453866401, + "packed_nmse_vs_bf16": 0.08006084513824217, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.054693450913167366, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.07919165966143808, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05594914643557424, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.0795658617816464, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.055003617254135655, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.0791012219666281, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05589542207572925, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 178, + "hessian": { + "down_diagonal_mean": 68.09086608886719, + "gate_up_diagonal_mean": 641.6583862304688, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10358963367487954, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.019344391538921635, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0073175568802046, + "packed_nmse_vs_bf16": 0.1037521481758826, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.019336236686677867, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10375713165970747, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.019149999533427185, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.1036762321681953, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.019245478573831066, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10394880451352743, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.022050801722808492, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10367237589477699, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.021527980512231135, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10362295303897989, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02237872153184101, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10350160107290528, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02197004360908399, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.08186004296050056, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.04801083834468478, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9836374453866401, + "packed_nmse_vs_bf16": 0.08269514674110663, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.0453585091046818, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.08222906610597445, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.047715099961539026, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.08140964946022078, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.04834975426519871, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 180, + "hessian": { + "down_diagonal_mean": 57.31684494018555, + "gate_up_diagonal_mean": 638.9610595703125, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10670310618699053, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.015631662373833833, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10669635369122973, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.015866797920432118, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10665268568480496, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.016190775623186042, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10666056094570163, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.015752955384928997, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10659103288443966, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.017911797649764825, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.1064707170947767, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.017384121962983234, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.1066499898200935, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.016869014190943082, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10660977404475504, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.016981840032033867, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.08233354095649201, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.04659989126081481, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.08213222236136944, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.04894743071112109, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.08242356149520659, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.04579072631432856, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.99827255914705, + "packed_nmse_vs_bf16": 0.08249635345300513, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.0462459768842624, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 182, + "hessian": { + "down_diagonal_mean": 42.99714660644531, + "gate_up_diagonal_mean": 637.6536865234375, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10450443113113642, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.0201597967186019, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10455460420768602, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.020296479644455304, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10451005601910154, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.020284114277658197, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10452096300714933, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.019973101308993345, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10432597608742726, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.024968605822903105, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10440717159548418, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02476943263725449, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10454713586290117, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.024402099845391677, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10446731127267997, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.024721995256693628, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.0763848668112722, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05726377200799022, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.07638729139523522, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05483399051741398, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.07629234300007509, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.057300059272678575, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.07648582779418535, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05609237661891838, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 184, + "hessian": { + "down_diagonal_mean": 58.1275634765625, + "gate_up_diagonal_mean": 638.2576904296875, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 0.9982725591470503, + "packed_nmse_vs_bf16": 0.11002667843422262, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.01801244576817737, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10986841256402274, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.018643382497077882, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10986657076676913, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.018223357342947027, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.11013241307996975, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.018425758796918445, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10995850266457721, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.020238367613796517, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.11001071331083041, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.021167070853801633, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10960424505455024, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02071903465007079, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10989433899238099, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02098726433232432, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.07951015022272141, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.053436717460563554, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.07925095629388963, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.054679065506504046, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.0796751039944561, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05326471636980301, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9780473293593848, + "packed_nmse_vs_bf16": 0.07879394523026552, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05435296482740207, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 186, + "hessian": { + "down_diagonal_mean": 62.61451721191406, + "gate_up_diagonal_mean": 642.2891845703125, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10572245976858276, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.021971866517493926, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10600234009165015, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.022425814075158035, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.99827255914705, + "packed_nmse_vs_bf16": 0.10592467574759497, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.02217610910052926, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10593393001927905, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.022463028742320434, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0073175568802046, + "packed_nmse_vs_bf16": 0.10585814209140638, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02376072199997402, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10584067457938755, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.023538948196450252, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10589688832039554, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.023559591168830506, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.1058582654827956, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.023842041294863357, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.07704747727997335, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05828563770878252, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.07721066953198712, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05810212765925359, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.07693619058900682, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05819810762094825, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.07737586366133814, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05783629133355928, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 188, + "hessian": { + "down_diagonal_mean": 59.23967361450195, + "gate_up_diagonal_mean": 640.96044921875, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.1082509486541282, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.018425314083068103, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0129076729074602, + "packed_nmse_vs_bf16": 0.10839673973921016, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.018402133415557943, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10829277712070653, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.01802411823906408, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10834638694606627, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.018644916899446314, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10827844561815334, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02168835592298199, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9836374453866401, + "packed_nmse_vs_bf16": 0.10831559167344278, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.021506565253580794, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9836374453866401, + "packed_nmse_vs_bf16": 0.10836482146143844, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.021392991133319522, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10831482117976393, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.021812010259220343, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.0776707771160867, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05583157438776786, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0073175568802049, + "packed_nmse_vs_bf16": 0.07784010520649219, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.0558950429745498, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9836374453866401, + "packed_nmse_vs_bf16": 0.07743170742585195, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05645267789007703, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.07769546041422712, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05634365415751542, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 190, + "hessian": { + "down_diagonal_mean": 47.44281768798828, + "gate_up_diagonal_mean": 636.593505859375, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10550934704587113, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.02109204239844815, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9836374453866403, + "packed_nmse_vs_bf16": 0.10554750810304564, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.020732353129578617, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10551986729006474, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.0206394499160223, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10551875086456423, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.021010102448645915, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10561366491715224, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.024421522220033094, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10568586458545032, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02498385062207295, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10529833187738487, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.024125614642914486, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10561989663735853, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02488367622167183, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.978047329359385, + "packed_nmse_vs_bf16": 0.07943282916757645, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.04905903710521648, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.07876540371450125, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.051280869429689215, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.07992675320157844, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.04275835333490697, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.07939544126647705, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.04222007597001209, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 192, + "hessian": { + "down_diagonal_mean": 113.45757293701172, + "gate_up_diagonal_mean": 631.9871826171875, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10565170152716569, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.011400117148858609, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10556342108940181, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.011634485443666711, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10548140872855355, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.011106652723669218, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0073175568802049, + "packed_nmse_vs_bf16": 0.10555005570881074, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.011484806306721964, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9836374453866401, + "packed_nmse_vs_bf16": 0.10560124788246437, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.012103954387337865, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9780473293593848, + "packed_nmse_vs_bf16": 0.10564530703693002, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.012643046904992234, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.1055597581302714, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.012203110774410903, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10538354152556152, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.012820579529753821, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.09004227746550636, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.030713716006703615, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9836374453866401, + "packed_nmse_vs_bf16": 0.08825820174471341, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.03488765497900282, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.99827255914705, + "packed_nmse_vs_bf16": 0.09027324460730074, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.02827545092786056, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.09000094730985832, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.03169814876980241, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 194, + "hessian": { + "down_diagonal_mean": 55.88294982910156, + "gate_up_diagonal_mean": 641.3719482421875, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 1.0073175568802046, + "packed_nmse_vs_bf16": 0.10898556212795997, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.018045384350862666, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10898201482836196, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.017750727688807824, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10876548666172921, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.017870132711244355, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10893492842919819, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.01778716080115613, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10884132641936832, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02092521311586742, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.99827255914705, + "packed_nmse_vs_bf16": 0.10862408902513769, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.020642465802013243, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10874965003138218, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.020732722995427704, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10895874253556491, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02069468258513749, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.07685554599301156, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.0576176963708312, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.07668531410749642, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05766070239219354, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.07663291712210177, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05786270332417654, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.07693602881289785, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05799821230046578, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 196, + "hessian": { + "down_diagonal_mean": 49.10383605957031, + "gate_up_diagonal_mean": 644.9825439453125, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10542024946535171, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.020949639644247058, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10556211506904963, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.020886943417315923, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0129076729074602, + "packed_nmse_vs_bf16": 0.10561407926161866, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.021306764632657443, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9836374453866401, + "packed_nmse_vs_bf16": 0.10551331439932986, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.020920241713609088, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10521577888997824, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02439424999645832, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10536344835340009, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02433208209969945, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10545943632617168, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02471425736895921, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9780473293593848, + "packed_nmse_vs_bf16": 0.10558800406260423, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.025066084572859717, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.07566389496963873, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05837823079080004, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.07590226381014903, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05716570592059065, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9836374453866401, + "packed_nmse_vs_bf16": 0.07594465744323772, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05791478822802987, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.07605002305218907, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.057812642654374594, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 198, + "hessian": { + "down_diagonal_mean": 52.265045166015625, + "gate_up_diagonal_mean": 639.9756469726562, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.105976337721951, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.01989802069398645, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10585492348306401, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.01975061284050113, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9836374453866401, + "packed_nmse_vs_bf16": 0.10613399946698604, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.019615261838822468, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10587056221522652, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.01958249208054874, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10591626899878614, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.022777219243852054, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10579417110611208, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.022788693929204986, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.1057573313727981, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.023267697822881194, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10583139422296713, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.022891012800726827, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.007317556880205, + "packed_nmse_vs_bf16": 0.07740798619725045, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05528723308840731, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.07725692441688713, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.0554127828975675, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.99827255914705, + "packed_nmse_vs_bf16": 0.07695394280808229, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05682734330650631, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.07720108753299136, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05568924853887096, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 200, + "hessian": { + "down_diagonal_mean": 50.38663864135742, + "gate_up_diagonal_mean": 645.4812622070312, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.1049524055818781, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.02083931885931651, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10484107169539893, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.02084246599130206, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10499331558065414, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.020446905784611395, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10494799491321066, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.021198687191601583, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10481991494847367, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.025827557130783142, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10487390496877635, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02514150656036837, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10479966474961579, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02482661225722535, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10489445484902667, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.025673691131942452, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.07761802430073639, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05712144243577734, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.99827255914705, + "packed_nmse_vs_bf16": 0.07780091270839497, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.0565409012843616, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.07824596754438619, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.054045502934254884, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.07827612148460338, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05605780108512057, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 202, + "hessian": { + "down_diagonal_mean": 48.45643997192383, + "gate_up_diagonal_mean": 644.211669921875, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10500794700311981, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.019872853606118065, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10518023300259152, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.020235749270650943, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10522056194157536, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.019963069892488527, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10493514904338594, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.02010521497235305, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10495015948191257, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.024765193282690837, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10481818497901642, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02496958496463414, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10503077930404357, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.024742833024837735, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10510041767485614, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.024555125856325886, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197949, + "packed_nmse_vs_bf16": 0.0761708167276033, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05719032256697476, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.0763472492381126, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05662253590331754, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.07628864039349356, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05716870808340353, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.07657934762281464, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05662712802080513, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 204, + "hessian": { + "down_diagonal_mean": 36.65999984741211, + "gate_up_diagonal_mean": 642.390869140625, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10483373383840863, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.019020500111155598, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.99827255914705, + "packed_nmse_vs_bf16": 0.10485509578113439, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.019432826475851813, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9836374453866401, + "packed_nmse_vs_bf16": 0.10505398038087278, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.019846781450991856, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10494849829919853, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.020418700774573487, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.1049679330269502, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02371022088453683, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10483324184539655, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.023243935243903755, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10480352799853178, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.023260087647547993, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9836374453866401, + "packed_nmse_vs_bf16": 0.10487039042552636, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.023210222995466483, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0073175568802046, + "packed_nmse_vs_bf16": 0.07916383993259178, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05024376114042183, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.07924186461546412, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.047592437501560086, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.07812525573445629, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.0523894316708142, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.07945363601705606, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05065144168993005, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 206, + "hessian": { + "down_diagonal_mean": 54.516387939453125, + "gate_up_diagonal_mean": 644.9046630859375, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10709948283728657, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.020756874416491943, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9836374453866401, + "packed_nmse_vs_bf16": 0.10701498063739037, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.020903413459900502, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10691268938527526, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.021162045740539672, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10699110397243883, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.020738333336713596, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10687057372491973, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02376153779635602, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9836374453866401, + "packed_nmse_vs_bf16": 0.10677477723349815, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02359890235576237, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10697757113330321, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.023678451532194715, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10666627307990029, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02370243958843067, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.99827255914705, + "packed_nmse_vs_bf16": 0.07562299621018577, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.059122352984020875, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.07575767767961136, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05915907123970544, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.07562193081877513, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05946186019660096, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.0757540003646092, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05921311141376734, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 208, + "hessian": { + "down_diagonal_mean": 95.12528228759766, + "gate_up_diagonal_mean": 634.6514892578125, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 1.0129076729074602, + "packed_nmse_vs_bf16": 0.1071156854190089, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.011492264841593861, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10699986264990936, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.011646889605433055, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10681527855585679, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.011326330527082247, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10705752614417582, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.011480464500501504, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10711258424842722, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.012326343780156741, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10699517826514052, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.011854188485357117, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.1070595228856647, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.01264307164196924, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10685371583507058, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.012488047544916794, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.99827255914705, + "packed_nmse_vs_bf16": 0.08610721650768582, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.04220869823567624, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.08571608060801846, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.04032162160885096, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.08647731602296048, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.03328104223908402, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.08598654935573899, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.04091960963848083, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 210, + "hessian": { + "down_diagonal_mean": 51.072837829589844, + "gate_up_diagonal_mean": 643.8092651367188, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10468924306919715, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.01958457669047897, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9836374453866401, + "packed_nmse_vs_bf16": 0.10455580372295532, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.020229026437493577, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10449369841196796, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.01972370793364787, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10440839156579187, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.019880833279553294, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10449731268516288, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.022176055309787262, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.1046877781334402, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.021719757335444543, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9745924476534854, + "packed_nmse_vs_bf16": 0.10485831767722346, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02172203451224472, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0073175568802046, + "packed_nmse_vs_bf16": 0.10445904867567393, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02156526170337557, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.07861903086088803, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05069816214791475, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.99827255914705, + "packed_nmse_vs_bf16": 0.0784342992492067, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05077816932233975, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.07821070970800426, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.049342728222355296, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.0789828715712604, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.049707220534867855, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 212, + "hessian": { + "down_diagonal_mean": 64.65414428710938, + "gate_up_diagonal_mean": 632.2099609375, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 1.0073175568802046, + "packed_nmse_vs_bf16": 0.10645993896612402, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.0192016859964643, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9836374453866403, + "packed_nmse_vs_bf16": 0.10643550077373855, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.019177714898673787, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10640655332051656, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.018946997902685007, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197949, + "packed_nmse_vs_bf16": 0.10638302139669747, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.018964088319326097, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10612321190580755, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.019923202162661715, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10623347699724177, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.019928095673406813, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10636745485755904, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.0194008666442852, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10629641073033179, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.019952079868144974, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0129076729074602, + "packed_nmse_vs_bf16": 0.07849412765270415, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.055681857335257, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.07870181819540169, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.055089206646930095, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9836374453866401, + "packed_nmse_vs_bf16": 0.07850125250954253, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05473730862715963, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.07831254388517186, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05540396625582084, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 214, + "hessian": { + "down_diagonal_mean": 49.215431213378906, + "gate_up_diagonal_mean": 643.884765625, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10553740361706984, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.020944239207392867, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10584928787763298, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.02112887300110053, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10578490773793316, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.020788525907719627, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10590553110844896, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.02154261073153752, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10560640530037306, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.024185916243189283, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10554960483176046, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.024358656286386296, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10563449759410985, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.024024565939670832, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10558215480537053, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.023975725238009563, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.07656270780899398, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.0568623480023218, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.0764664947144058, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.057766839472239984, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.07623392766422353, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05807121353189673, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470503, + "packed_nmse_vs_bf16": 0.07606599909805, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05801352037100968, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 216, + "hessian": { + "down_diagonal_mean": 74.31973266601562, + "gate_up_diagonal_mean": 626.9261474609375, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 0.9836374453866401, + "packed_nmse_vs_bf16": 0.10826958427631955, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.01264809461961723, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10806475319653872, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.012498751539738857, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10798619307281718, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.013214086405047395, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9836374453866401, + "packed_nmse_vs_bf16": 0.10801579922291124, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.0124464429516717, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10827963834492302, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.01386312298155114, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10809295876266727, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.013968561098790116, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197949, + "packed_nmse_vs_bf16": 0.10834742935402232, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.013784840141326386, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10798512585211185, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.013923876783046626, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.08686012955307608, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.041733242995234635, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.08546907782388638, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.044055057498562346, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.08616845777254152, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.04297851785663728, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.08643499368534843, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.04332296704505041, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 218, + "hessian": { + "down_diagonal_mean": 26.542644500732422, + "gate_up_diagonal_mean": 634.0450439453125, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10382979133948635, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.024781645838157788, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10383567114948744, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.024823635941738677, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0073175568802046, + "packed_nmse_vs_bf16": 0.10391631035694023, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.024818478236763555, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10370339536103819, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.024776137847224485, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10381466803925743, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.029755764229657568, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10384137043196362, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02961597986581036, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10396707432630539, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.029748277515020637, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10355499918807948, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.029780424756896306, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.07401994714866403, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05975257648992187, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0129076729074602, + "packed_nmse_vs_bf16": 0.07401991293978454, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05939520536161338, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.0741027971421688, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05867628712342487, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.07378175431485164, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05869188337327468, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 220, + "hessian": { + "down_diagonal_mean": 38.265594482421875, + "gate_up_diagonal_mean": 639.3521118164062, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10369643161674386, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.025439129494873043, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10353780031491602, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.02440221269276168, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.1036382423579333, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.02554475972896984, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10359095441495869, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.024756803985254875, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10363975127634564, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.028967051323841118, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10353250194821575, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02875447385773112, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10349103771497287, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.029192514664538503, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10370387579477801, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02855942705858934, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.07440976924974242, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.06120808577143375, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.07411832273861757, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.061434449241187725, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.0741606155818362, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.06141537380965166, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.07442345271597496, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.06097639530485149, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 222, + "hessian": { + "down_diagonal_mean": 54.26656723022461, + "gate_up_diagonal_mean": 635.5930786132812, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10544084651716626, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.02034888451715154, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10572523600164825, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.01976538634321653, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10583077702556908, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.019911660385256193, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10576871817190489, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.02005008714696429, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10569916567976002, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02188413846272605, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10572591015485289, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02217767728494535, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.1055805778211418, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.022332563619046134, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9836374453866401, + "packed_nmse_vs_bf16": 0.10565773458390855, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02189925133703292, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.07942044078269939, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05297736504396663, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.07987585316564393, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05309516997839182, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.07922845749350092, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.054286933079852016, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.07959874030165982, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05371556052188698, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 224, + "hessian": { + "down_diagonal_mean": 53.834835052490234, + "gate_up_diagonal_mean": 644.735595703125, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10856849581329579, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.01748403416490172, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10854177842915662, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.01762059390755409, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10853878069939528, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.01749325801239987, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10881031542894867, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.017341733149841932, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9836374453866401, + "packed_nmse_vs_bf16": 0.10859859303965687, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.019725077928987898, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10862803484248469, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.01951467369838852, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.1084404785911429, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.019534807079647627, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10831117769395997, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.019787013279981076, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.07915576238831198, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05284262659080706, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.07910326957103986, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05406892204052224, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0073175568802046, + "packed_nmse_vs_bf16": 0.07922982472675615, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05373246861916735, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.07854249624985508, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.055077882031944435, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 226, + "hessian": { + "down_diagonal_mean": 55.29933547973633, + "gate_up_diagonal_mean": 650.7825317382812, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.1085906083518779, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.018662386203609133, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10870425269982548, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.01848879444362876, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0073175568802046, + "packed_nmse_vs_bf16": 0.10863449062486796, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.018597030897503964, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10865666679966203, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.018461362524677512, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10862005354818766, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.022732094337278737, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9836374453866401, + "packed_nmse_vs_bf16": 0.10870245338832932, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.022821898972408657, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10848705158930957, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02225692511298633, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10868347803962297, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02234799539897444, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.07653823843829628, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05823724342260575, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9836374453866401, + "packed_nmse_vs_bf16": 0.07641544659516421, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05840119365818364, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0073175568802046, + "packed_nmse_vs_bf16": 0.07668804044338252, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.0580748941984696, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.07641417058402007, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05791574761197548, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 228, + "hessian": { + "down_diagonal_mean": 50.27877426147461, + "gate_up_diagonal_mean": 640.3712158203125, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.1044108142819226, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.02119538499371228, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10449285121808921, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.02109205940591823, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10458282553137559, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.020361020296411524, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10430936688464472, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.021064614935164643, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10436659859392113, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.025536342303349356, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10426601898378518, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.025206489645185515, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10446420955705436, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.024503729443019925, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10436062509587588, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02536305168294987, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.99827255914705, + "packed_nmse_vs_bf16": 0.07784643849123628, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.053770170567047984, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.0776148459414232, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05527010828685226, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.07886372002076798, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05271761388945772, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.07788248051287716, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05379966433816415, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 230, + "hessian": { + "down_diagonal_mean": 66.69532775878906, + "gate_up_diagonal_mean": 640.34716796875, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10544931347618038, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.018608705095624702, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10566254641567022, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.018259952089602786, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10550780326256634, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.018628579473324292, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.105521119820091, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.01867807876447745, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9780473293593848, + "packed_nmse_vs_bf16": 0.10569115874915277, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02057968588332809, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10560708252163944, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.020437509828611256, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10553964806297389, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02082576703156194, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10548697417597057, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.021042258116340185, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.07796201496897606, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05542492986867204, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.07756419445334312, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.055605405716219566, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.07763390392202106, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05513904688371537, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.07776232048132316, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05599286352340409, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 232, + "hessian": { + "down_diagonal_mean": 61.520606994628906, + "gate_up_diagonal_mean": 641.6863403320312, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10854962123132997, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.017689413734719182, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10821580609610729, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.017891087171303614, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10831188926002577, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.017588650636492273, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9836374453866401, + "packed_nmse_vs_bf16": 0.1085234252272015, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.0180412195606374, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9836374453866401, + "packed_nmse_vs_bf16": 0.1083311394798156, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.019761886721444116, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10856470700036815, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.01889807794837657, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470503, + "packed_nmse_vs_bf16": 0.1081766277099737, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.019313528756219417, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10838786451539495, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.019708651049700324, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.99827255914705, + "packed_nmse_vs_bf16": 0.07764319284390495, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05634396600575642, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.07786362228745947, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05577200300591328, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.07830304931387604, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05571897380657015, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.07769151894624315, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.056407258378530466, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 234, + "hessian": { + "down_diagonal_mean": 58.943214416503906, + "gate_up_diagonal_mean": 640.1375732421875, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10667343833764488, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.018196531827341694, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9836374453866401, + "packed_nmse_vs_bf16": 0.1067235153320775, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.018542521154703947, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10672132092872687, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.01856070117774633, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0073175568802046, + "packed_nmse_vs_bf16": 0.10658239065886661, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.018295900757769653, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10652793158302624, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.022078649838282362, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10662699686020968, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.022437958731356494, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10643373752779894, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02196887999197884, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10647379262314126, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.022161085027492665, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.07748718840371588, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.056436830239695236, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0073175568802046, + "packed_nmse_vs_bf16": 0.0773229882646823, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05676545966447996, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0073175568802046, + "packed_nmse_vs_bf16": 0.07756302731972817, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.056208867715524934, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.07712640955006858, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05721270603443351, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 236, + "hessian": { + "down_diagonal_mean": 87.45854187011719, + "gate_up_diagonal_mean": 630.7471923828125, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10962763405942907, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.009712965501045511, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.1094793456268466, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.010375477086359528, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0129076729074602, + "packed_nmse_vs_bf16": 0.10968910142554952, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.009707927724121225, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.1098302430968119, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.009405517397906939, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10980595685630164, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.011091599961774926, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10967737204068641, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.010939105000351496, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10974079389199225, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.011434974695660101, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.99827255914705, + "packed_nmse_vs_bf16": 0.10965622428994884, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.010950515127057787, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.08474279018278451, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.04404988573239324, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.08342049497469764, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.04500076685454651, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925396, + "packed_nmse_vs_bf16": 0.08491102500138109, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.04468908372342661, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.08402490040920188, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.04265757207614693, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 238, + "hessian": { + "down_diagonal_mean": 71.45802307128906, + "gate_up_diagonal_mean": 632.00146484375, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 0.9836374453866401, + "packed_nmse_vs_bf16": 0.10842442661761986, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.013763004666698452, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10817086349071875, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.014098414127924874, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10840788930983064, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.014417314282003615, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10844309113415788, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.014297773150547807, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.99827255914705, + "packed_nmse_vs_bf16": 0.10819049980359116, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.014927972216377934, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10847119964389274, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.015045455933449239, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10848635908351308, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.014757271707339602, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.1084555277690282, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.014622959467123569, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.07742593443390001, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05419315489079832, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.07720557646960428, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05570786741871639, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.07768322883846852, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05340621753598397, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.07754763907136593, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05416847478608643, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 240, + "hessian": { + "down_diagonal_mean": 52.87041473388672, + "gate_up_diagonal_mean": 638.474853515625, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10440018939673917, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.02032144732132892, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470503, + "packed_nmse_vs_bf16": 0.10444690910806652, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.019862697870783784, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10453050221346133, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.019994682080678044, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10456446826129512, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.020407935618379003, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10455380785768521, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02354052412349831, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0163625546133594, + "packed_nmse_vs_bf16": 0.1045426884765585, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.022994712312573475, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10477333342870539, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02357783637992173, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470503, + "packed_nmse_vs_bf16": 0.10458434848438468, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.023683924430304648, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.07652516265272943, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05742633231146778, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.07636148190905648, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05693309564485488, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.07568612768037686, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.059013614154849875, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.07623568141319188, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.058162597008505304, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 242, + "hessian": { + "down_diagonal_mean": 48.5770263671875, + "gate_up_diagonal_mean": 640.0913696289062, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10602366660138464, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.021861014475595117, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.007317556880205, + "packed_nmse_vs_bf16": 0.10584122935559886, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.021545789282981634, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10611959218009903, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.021912355360796436, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9836374453866401, + "packed_nmse_vs_bf16": 0.10600427642923968, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.02211010901229713, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10595197763040615, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.023671832785015404, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925397, + "packed_nmse_vs_bf16": 0.10616186871496693, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.023625598873486784, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10614352828325205, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02383865987711227, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10594055837036419, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.024434141632330555, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.99827255914705, + "packed_nmse_vs_bf16": 0.07571095915729667, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05934322287932391, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.07554649074999127, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.059260175968528304, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.07574433349228427, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05922399138687378, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.07574155659031759, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05943786424432325, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 244, + "hessian": { + "down_diagonal_mean": 75.360107421875, + "gate_up_diagonal_mean": 641.4688720703125, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.1061216880154315, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.013585672034152096, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10600475904560708, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.014497330881097564, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10584530765216081, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.01402158896701454, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10585980146131613, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.013839546918635075, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10592793224046902, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.01705371597829206, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10583917633727265, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.017727271400671575, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10580641243169893, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.017550657694171568, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.99827255914705, + "packed_nmse_vs_bf16": 0.10597192167710598, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.017737822086083065, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.08206602995803099, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.048870326546174035, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.0826052748010255, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.04776630701674857, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.08215291686555885, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.03519872815792773, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9836374453866401, + "packed_nmse_vs_bf16": 0.08151095590209126, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.04937371484359088, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 246, + "hessian": { + "down_diagonal_mean": 45.167991638183594, + "gate_up_diagonal_mean": 643.4032592773438, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10433558193535879, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.023140290926374898, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10439701956546345, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.024008128668696133, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10437434213132746, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.023417011156837923, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10436622970355085, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.023995854106221837, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0129076729074602, + "packed_nmse_vs_bf16": 0.10429618839705383, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.0268262814412584, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10439597519078432, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.026948185081188562, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9745924476534854, + "packed_nmse_vs_bf16": 0.10460543830495371, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.026559988507849516, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10434629251367429, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02703488940082861, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9780473293593848, + "packed_nmse_vs_bf16": 0.07513098871102855, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.06007537549066292, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.07528930517725453, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05997335132471467, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.07516867942429029, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05999917304316289, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.07502424382530973, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.060233361206178164, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 248, + "hessian": { + "down_diagonal_mean": 56.113059997558594, + "gate_up_diagonal_mean": 640.0931396484375, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10587498190566542, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.018114440587791197, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10574290776698386, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.018158036173915502, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10565858156465599, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.018472369966909312, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.1058557864108433, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.018308705203650768, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10572137576439028, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.021835328510471378, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10569040930670322, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.0215819771144926, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.1055679344671088, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02156836356600391, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10563177802449475, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02115158458211964, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.07837496612102848, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05535319076341426, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.07797336464393019, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05589664612595641, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.07817022580922026, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05456047507720775, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.07827250723548093, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.0547225302866636, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 250, + "hessian": { + "down_diagonal_mean": 45.76283264160156, + "gate_up_diagonal_mean": 642.10009765625, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 0.9982725591470503, + "packed_nmse_vs_bf16": 0.10567470517848977, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.020372457313490334, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10572471869286634, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.02041458997064731, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9836374453866401, + "packed_nmse_vs_bf16": 0.10587798810541284, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.020650974157112548, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10571451606269958, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.02060820588883002, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0073175568802046, + "packed_nmse_vs_bf16": 0.1055618536653088, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.023684826883022585, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10541059156607345, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.023361072210544842, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9836374453866401, + "packed_nmse_vs_bf16": 0.10563030438551627, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.022944107850360735, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.1055415704931428, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02309804006587066, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.07781725018230609, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05400163163319988, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.0776666230151958, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.054091518074480316, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9836374453866401, + "packed_nmse_vs_bf16": 0.07774438977782436, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05198506328489693, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.0774923848090484, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.0554956810894957, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 252, + "hessian": { + "down_diagonal_mean": 48.56764221191406, + "gate_up_diagonal_mean": 642.6263427734375, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10682006111033505, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.01956559022026109, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10665119775366452, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.020373018881379757, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10663813948017647, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.019830975019522103, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.1065753447332836, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.019615167011954867, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10663943222374293, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.022816535298295155, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10672644307288509, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02335287540603187, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.1067633414989451, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.023033647047758246, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10671973303586775, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.0229092028291075, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.07631445388603422, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05820808291554819, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.07631479283371434, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05848266871640467, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.99827255914705, + "packed_nmse_vs_bf16": 0.07619601048086205, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.0583133831563974, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.07652492224134069, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.0577333945862822, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 254, + "hessian": { + "down_diagonal_mean": 59.98556900024414, + "gate_up_diagonal_mean": 640.1806640625, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.11104471505245975, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.01731533827077909, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.11102774998528366, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.01794159132998394, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.11099214325793302, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.017477863851000076, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.11117482750560222, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.01745184017049048, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.11119439880813294, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.019733715919693064, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.11127320274769528, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.01988581567164146, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.11112405260535507, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.019492231107056796, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.1109204310121036, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.019640016206598755, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.08080096958603938, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05098235284767963, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.08130842479534468, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.048939184547482674, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9836374453866401, + "packed_nmse_vs_bf16": 0.08027180356248437, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05071615884334781, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.08086062002256099, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.051722426374257106, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 256, + "hessian": { + "down_diagonal_mean": 38.98259353637695, + "gate_up_diagonal_mean": 645.4507446289062, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10701306414201271, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.02648497455118211, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10683222186731016, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.02600327736759772, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0073175568802046, + "packed_nmse_vs_bf16": 0.10708826829507853, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.025735226199440596, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10720111954563298, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.026439794743874322, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.1069658134767728, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.028245461246619925, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10688018794983685, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02824817658570891, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0129076729074602, + "packed_nmse_vs_bf16": 0.10693459453769068, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.028271436085787652, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.1070511221603433, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.028504920769098597, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.07636761720055103, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05773254513878335, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.07702302765737128, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.056802705799990603, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.07728940625478165, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.054478232995747014, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0073175568802046, + "packed_nmse_vs_bf16": 0.07641740434029054, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05142443189314658, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 258, + "hessian": { + "down_diagonal_mean": 63.83624267578125, + "gate_up_diagonal_mean": 642.2185668945312, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10652156630247989, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.01919184825777081, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.106240680134794, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.019004727145741945, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9836374453866401, + "packed_nmse_vs_bf16": 0.10634935461496241, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.018851872300663087, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10652817412534354, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.01957752768942377, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10632031380997511, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.022416636108068176, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10640892278376084, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.021937453783120456, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.1062887404806346, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02224020626687575, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9836374453866401, + "packed_nmse_vs_bf16": 0.10653699680236595, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02205122145165008, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.07687258360512017, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05767938958116966, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.07734339202673195, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05707169251040208, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.07745891177477955, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05709144877746724, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.99827255914705, + "packed_nmse_vs_bf16": 0.07723339779751598, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05678787043115941, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 260, + "hessian": { + "down_diagonal_mean": 41.25267028808594, + "gate_up_diagonal_mean": 640.4270629882812, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10539202766653434, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.023875447715914048, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10552718571470218, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.023418260681628705, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10524542480622791, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.023157679363525693, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10528017464593742, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.02372967324544957, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10511027529511278, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.027414053216305603, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10538079036796578, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.027170295971180317, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9836374453866401, + "packed_nmse_vs_bf16": 0.10538155501944924, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.026797115038022422, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.105211548417187, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.027381708412180485, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.07565605990237588, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05930461518304859, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.07560595274365435, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05920192570091694, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9836374453866401, + "packed_nmse_vs_bf16": 0.07597438484912604, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.058537257459400784, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.07552857143005937, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.0596491093006227, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 262, + "hessian": { + "down_diagonal_mean": 61.40776062011719, + "gate_up_diagonal_mean": 632.5823974609375, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10430439675001722, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.01987295869892413, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10426260670430754, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.019765456980118667, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10394010773383053, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.019742211513856685, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9836374453866401, + "packed_nmse_vs_bf16": 0.10437618051646297, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.019290729188411845, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10428813001266607, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.021204846377022912, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10430517439076421, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.020971219033584815, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.1041258212462448, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.020988986733456394, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10437051497849058, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.021132987260913224, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.07914582566717897, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05086317481566821, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9836374453866401, + "packed_nmse_vs_bf16": 0.07878154342327393, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05215584423144677, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.07943380377410902, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.04947591874565034, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.0797264763431005, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.049145986948970606, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 264, + "hessian": { + "down_diagonal_mean": 37.07640075683594, + "gate_up_diagonal_mean": 636.2662353515625, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.1063645126458417, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.02250140628861555, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10636897260276462, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.022640824341752942, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10655048389441456, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.02248067210807606, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10634980677792842, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.022077122310203704, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10634665359822154, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.025866668470031006, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.1063067991849263, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02609306147478966, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.1063394723935146, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02572930697942045, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10644741076777817, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.026573341955690985, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9836374453866401, + "packed_nmse_vs_bf16": 0.07630578878799371, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05858711345772526, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.99827255914705, + "packed_nmse_vs_bf16": 0.07635702940614136, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05853550115852495, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.99827255914705, + "packed_nmse_vs_bf16": 0.07673781885410066, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05754134822160429, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.07614573175257282, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05866885669948049, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 266, + "hessian": { + "down_diagonal_mean": 50.724708557128906, + "gate_up_diagonal_mean": 644.0693359375, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10681064788524466, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.020335439283743494, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10688967006584646, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.01992028469679085, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10677625537408206, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.020036779993139738, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10684146178791327, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.020654072283843063, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10672793442981404, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.022219543356301236, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925397, + "packed_nmse_vs_bf16": 0.10683473724617085, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.022239924537381488, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10677901320578334, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.021831157594173648, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10676503325215059, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02213987527423148, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9836374453866401, + "packed_nmse_vs_bf16": 0.0758966948025743, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05842724210995746, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.07556868818285739, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.058861421135764314, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.0758196445971395, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.059118359350823865, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.07525754902015858, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.059754376340670635, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 268, + "hessian": { + "down_diagonal_mean": 37.72359085083008, + "gate_up_diagonal_mean": 638.9014282226562, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10507882586488669, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.024323448933090987, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.1050000099426619, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.02410284072334303, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.1049459837297438, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.024241103276644443, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10503953139814609, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.024517267870142887, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10513353524855425, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02867801495839421, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.104968919768033, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.028147076400975815, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0073175568802046, + "packed_nmse_vs_bf16": 0.10512788755574298, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.028346108773322727, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10501355853183658, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.028738837370357775, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.07668700808670822, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05786092959578992, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.07691825671487598, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05732075963231106, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.07655131194377916, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.057737093904831574, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.07690208751875034, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05737221916787963, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 270, + "hessian": { + "down_diagonal_mean": 48.3609619140625, + "gate_up_diagonal_mean": 640.8698120117188, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.1099383385498087, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.018295610643006856, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.11003291622264888, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.017883553681994853, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10998905535740285, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.01820504334915848, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9836374453866401, + "packed_nmse_vs_bf16": 0.11008097587630997, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.018025947620594982, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0073175568802046, + "packed_nmse_vs_bf16": 0.11001361233942207, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.022300873406505363, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.11005946005812213, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02230130370835996, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0073175568802046, + "packed_nmse_vs_bf16": 0.10999177611190143, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02197004543818572, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10997282347599327, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.022183631200152033, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.07840722425539967, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.054483968962192035, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.07822797926656852, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05510070875345937, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0073175568802046, + "packed_nmse_vs_bf16": 0.07902157888015572, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05364996821656223, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.07852489964769346, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.054817384830813054, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 272, + "hessian": { + "down_diagonal_mean": 54.262977600097656, + "gate_up_diagonal_mean": 645.841796875, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10545891056290733, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.020901725641107236, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.1051346829503923, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.020355750592204194, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.99827255914705, + "packed_nmse_vs_bf16": 0.10528882479356563, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.020001022727995137, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10528229627589451, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.02010510738618756, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10517529340447183, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.022608944341816612, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10526216575144916, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.021665450776126586, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10536068134332778, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.022078964760671265, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.99827255914705, + "packed_nmse_vs_bf16": 0.10509777263346859, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.021756760997736442, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9836374453866401, + "packed_nmse_vs_bf16": 0.07693155640832353, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05762908017184958, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.07701438286227011, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05709792612158053, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.07695132233673259, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05695195721834083, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.07700510577010841, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.056806393792492686, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 274, + "hessian": { + "down_diagonal_mean": 51.42156982421875, + "gate_up_diagonal_mean": 639.4749145507812, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 0.99827255914705, + "packed_nmse_vs_bf16": 0.10646020747457864, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.022755817599746108, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10644402915947322, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.02277635494609325, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10670623057949608, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.023230122599280607, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9780473293593848, + "packed_nmse_vs_bf16": 0.10672551229795815, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.022726021261963023, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10648922410736036, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02582415861184594, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10656138896740354, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.026092101226729097, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10638128177105968, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02617619078296966, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10652596694764291, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.026182523172503764, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.07933620784370242, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05382089883570807, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.07863123086517622, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05440386313717387, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.07781450935749694, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05671486453941531, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.07794550950593507, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05682930881527298, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 276, + "hessian": { + "down_diagonal_mean": 49.27996063232422, + "gate_up_diagonal_mean": 640.5877685546875, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10480216066901622, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.02034646098407959, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10503287527321777, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.020781132367519155, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10484874898648679, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.02041713243455755, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10502924496416019, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.020655798185286466, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9836374453866401, + "packed_nmse_vs_bf16": 0.10501619884420445, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.023575313619890372, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10476338220719628, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.023879162970651718, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9836374453866401, + "packed_nmse_vs_bf16": 0.10478581395040053, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.023627489325350363, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9836374453866401, + "packed_nmse_vs_bf16": 0.10498813851265568, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.023750650777073726, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.0774068660518775, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.0551332294260949, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.07724257193589634, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05578868570720263, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.07721296930851686, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.056113477117045424, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.07708261459767098, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05645548103643735, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 278, + "hessian": { + "down_diagonal_mean": 47.635108947753906, + "gate_up_diagonal_mean": 643.049560546875, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10803065346077133, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.022285520128390978, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.1079512111289486, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.022060190978886007, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10804594023835191, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.021586315309529387, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10798031583884407, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.022413498983307443, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10786787733698537, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02457664987280362, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.1078493731253095, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.024740963641868334, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.007317556880205, + "packed_nmse_vs_bf16": 0.10809947210259936, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.024529511235875098, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.1080130293039082, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.024704463458764387, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.07671728937590258, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05773122597032418, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.0768098932226688, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05805143037291439, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.07697578745153914, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.0578504957427475, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9836374453866401, + "packed_nmse_vs_bf16": 0.07707070414452513, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05688221666301441, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 280, + "hessian": { + "down_diagonal_mean": 50.15987014770508, + "gate_up_diagonal_mean": 640.2251586914062, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 1.0073175568802046, + "packed_nmse_vs_bf16": 0.10631142078436943, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.01995562207222319, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10632546465771171, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.019990780407886155, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10611746589754135, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.019492534789619674, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9836374453866401, + "packed_nmse_vs_bf16": 0.1063611852500969, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.019907641026291844, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0073175568802046, + "packed_nmse_vs_bf16": 0.10614131909204186, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02294809482991576, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10630763609807044, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02293100609042141, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0129076729074602, + "packed_nmse_vs_bf16": 0.1063171734126325, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02258242822758616, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.978047329359385, + "packed_nmse_vs_bf16": 0.10638120619853454, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.022964530990497067, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.07560829371227254, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05944517117389316, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0129076729074602, + "packed_nmse_vs_bf16": 0.07524271053238152, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05977134450197598, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470503, + "packed_nmse_vs_bf16": 0.07562005503680086, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05913392397589997, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.07528608774789622, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05966224443220136, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 282, + "hessian": { + "down_diagonal_mean": 33.887664794921875, + "gate_up_diagonal_mean": 635.9260864257812, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10516640231274776, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.02477496439706102, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10506422681221322, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.024589190787095254, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10498470945836014, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.02430467362487842, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0129076729074602, + "packed_nmse_vs_bf16": 0.10507110836177407, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.024080664274903688, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9836374453866401, + "packed_nmse_vs_bf16": 0.10518087592411778, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.030675070979300164, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10489443063783287, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.03061103710661348, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10508436511660232, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.03034528276912647, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.1051049292334023, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.029106107050103726, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.07689307279914032, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05289422140019438, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197949, + "packed_nmse_vs_bf16": 0.07628026398053284, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05683061225482976, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.07641233991606383, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.056435679502845414, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0073175568802046, + "packed_nmse_vs_bf16": 0.07740606330793277, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05316600688717019, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 284, + "hessian": { + "down_diagonal_mean": 59.753257751464844, + "gate_up_diagonal_mean": 645.0802001953125, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10680976025179743, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.018374113234207994, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9836374453866401, + "packed_nmse_vs_bf16": 0.10692854869723634, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.018646635252862045, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10692388249032764, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.019331444245410088, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10676325712676196, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.0188528693294297, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10679279045912073, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.022836361953618867, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10690009056802337, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02302638892100538, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10669241805055513, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.023131647797163656, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10697228744469628, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.023321198352313027, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.07745629046123163, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05570059855736929, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.07693116543222843, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.056648805497804676, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.07660377167616114, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.057620718103103506, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0073175568802046, + "packed_nmse_vs_bf16": 0.07638222135091699, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05766035100576169, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 286, + "hessian": { + "down_diagonal_mean": 59.958839416503906, + "gate_up_diagonal_mean": 643.93115234375, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 0.9836374453866401, + "packed_nmse_vs_bf16": 0.1061278772776662, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.01942979320214802, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.106202622236407, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.019655893046119824, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10617293364856648, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.019765324629239474, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10599993631712598, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.019654570907011698, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.105990792912141, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.022980545023095243, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10597701360370682, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02317544935301799, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10607406030010338, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.022784422422587106, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10618149992937437, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.022754912414304378, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.07638137290871473, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.057967818974425725, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.0764301262153568, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.057955067130878495, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.07692411363719963, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05714224759500689, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9836374453866401, + "packed_nmse_vs_bf16": 0.07649132646833642, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05745753770651661, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + } + ], + "experts": [ + 0, + 2, + 4, + 6, + 8, + 10, + 12, + 14, + 16, + 18, + 20, + 22, + 24, + 26, + 28, + 30, + 32, + 34, + 36, + 38, + 40, + 42, + 44, + 46, + 48, + 50, + 52, + 54, + 56, + 58, + 60, + 62, + 64, + 66, + 68, + 70, + 72, + 74, + 76, + 78, + 80, + 82, + 84, + 86, + 88, + 90, + 92, + 94, + 96, + 98, + 100, + 102, + 104, + 106, + 108, + 110, + 112, + 114, + 116, + 118, + 120, + 122, + 124, + 126, + 128, + 130, + 132, + 134, + 136, + 138, + 140, + 142, + 144, + 146, + 148, + 150, + 152, + 154, + 156, + 158, + 160, + 162, + 164, + 166, + 168, + 170, + 172, + 174, + 176, + 178, + 180, + 182, + 184, + 186, + 188, + 190, + 192, + 194, + 196, + 198, + 200, + 202, + 204, + 206, + 208, + 210, + 212, + 214, + 216, + 218, + 220, + 222, + 224, + 226, + 228, + 230, + 232, + 234, + 236, + 238, + 240, + 242, + 244, + 246, + 248, + 250, + 252, + 254, + 256, + 258, + 260, + 262, + 264, + 266, + 268, + 270, + 272, + 274, + 276, + 278, + 280, + 282, + 284, + 286 + ], + "gpu": 0, + "layer": 17, + "physical_experts": [ + 0, + 2, + 4, + 6, + 8, + 10, + 12, + 14, + 16, + 18, + 20, + 22, + 24, + 26, + 28, + 30, + 32, + 34, + 36, + 38, + 40, + 42, + 44, + 46, + 48, + 50, + 52, + 54, + 56, + 58, + 60, + 62, + 64, + 66, + 68, + 70, + 72, + 74, + 76, + 78, + 80, + 82, + 84, + 86, + 88, + 90, + 92, + 94, + 96, + 98, + 100, + 102, + 104, + 106, + 108, + 110, + 112, + 114, + 116, + 118, + 120, + 122, + 124, + 126, + 128, + 130, + 132, + 134, + 136, + 138, + 140, + 142, + 144, + 146, + 148, + 150, + 152, + 154, + 156, + 158, + 160, + 162, + 164, + 166, + 168, + 170, + 172, + 174, + 176, + 178, + 180, + 182, + 184, + 186, + 188, + 190, + 192, + 194, + 196, + 198, + 200, + 202, + 204, + 206, + 208, + 210, + 212, + 214, + 216, + 218, + 220, + 222, + 224, + 226, + 228, + 230, + 232, + 234, + 236, + 238, + 240, + 242, + 244, + 246, + 248, + 250, + 252, + 254, + 256, + 258, + 260, + 262, + 264, + 266, + 268, + 270, + 272, + 274, + 276, + 278, + 280, + 282, + 284, + 286 + ], + "physical_tensor_count": 6912, + "schema": "glm53-brandon-style-low-bpw-tail-part-v1", + "selected_k": { + "0": 2, + "10": 2, + "100": 2, + "102": 2, + "104": 2, + "106": 2, + "108": 2, + "110": 2, + "112": 2, + "114": 2, + "116": 2, + "118": 2, + "12": 2, + "120": 2, + "122": 2, + "124": 2, + "126": 2, + "128": 2, + "130": 2, + "132": 2, + "134": 2, + "136": 2, + "138": 2, + "14": 2, + "140": 2, + "142": 2, + "144": 2, + "146": 2, + "148": 2, + "150": 2, + "152": 2, + "154": 2, + "156": 2, + "158": 2, + "16": 2, + "160": 2, + "162": 2, + "164": 2, + "166": 2, + "168": 2, + "170": 2, + "172": 2, + "174": 2, + "176": 2, + "178": 2, + "18": 2, + "180": 2, + "182": 2, + "184": 2, + "186": 2, + "188": 2, + "190": 2, + "192": 2, + "194": 2, + "196": 2, + "198": 2, + "2": 2, + "20": 2, + "200": 2, + "202": 2, + "204": 2, + "206": 2, + "208": 2, + "210": 2, + "212": 2, + "214": 2, + "216": 2, + "218": 2, + "22": 2, + "220": 2, + "222": 2, + "224": 2, + "226": 2, + "228": 2, + "230": 2, + "232": 2, + "234": 2, + "236": 2, + "238": 2, + "24": 2, + "240": 2, + "242": 2, + "244": 2, + "246": 2, + "248": 2, + "250": 2, + "252": 2, + "254": 2, + "256": 2, + "258": 2, + "26": 2, + "260": 2, + "262": 2, + "264": 2, + "266": 2, + "268": 2, + "270": 2, + "272": 2, + "274": 2, + "276": 2, + "278": 2, + "28": 2, + "280": 2, + "282": 2, + "284": 2, + "286": 2, + "30": 2, + "32": 2, + "34": 2, + "36": 2, + "38": 2, + "4": 2, + "40": 2, + "42": 2, + "44": 2, + "46": 2, + "48": 2, + "50": 2, + "52": 2, + "54": 2, + "56": 2, + "58": 2, + "6": 2, + "60": 2, + "62": 2, + "64": 2, + "66": 2, + "68": 2, + "70": 2, + "72": 2, + "74": 2, + "76": 2, + "78": 2, + "8": 2, + "80": 2, + "82": 2, + "84": 2, + "86": 2, + "88": 2, + "90": 2, + "92": 2, + "94": 2, + "96": 2, + "98": 2 + }, + "sha256": "4d7253c853bdbb4d161646084922d26c4bd18bc57ac139ee42973e828f1e8add", + "shadow_experts_ignored": [], + "source_sidecar_sha256": "1bec420ffc821f38c9877dd487bbce394b0dd68183733bfa58d18c5d3d9849fd", + "state": "COMPLETE", + "tensor_count": 6912, + "tier": "k2-k3-tail", + "variant": "2.0" +}