diff --git "a/layer-23-part-tail-0.json" "b/layer-23-part-tail-0.json" new file mode 100644--- /dev/null +++ "b/layer-23-part-tail-0.json" @@ -0,0 +1,26086 @@ +{ + "bytes": 922818568, + "expert_reports": [ + { + "expert": 0, + "hessian": { + "down_diagonal_mean": 70.87406921386719, + "gate_up_diagonal_mean": 622.8673706054688, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10525672726377629, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.017306581350266026, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10529044275183681, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.018101238761373633, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10547398684506648, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.0176806321860404, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.978047329359385, + "packed_nmse_vs_bf16": 0.10544492742470395, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.01769317268807462, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10509106260262381, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02013095531673611, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10517986409039648, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.01957863344569434, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10532118156545872, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.019986725264434457, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10532208037277578, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.01978122126678481, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.07964343566374782, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.054751803190186694, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.07941046412922231, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05551050015568364, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.0794688376305105, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05528163543333183, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.07924794900910788, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05529581958483912, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 2, + "hessian": { + "down_diagonal_mean": 68.49745178222656, + "gate_up_diagonal_mean": 623.7247314453125, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.1042576970054764, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.01727329086341596, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10454282226816725, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.017427435368577663, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.1044436922919538, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.017498630426801953, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10457609614338385, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.017946460111280586, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0073175568802049, + "packed_nmse_vs_bf16": 0.10432185515178442, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.01976838164078957, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10464697991548726, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.019781752607248033, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9836374453866401, + "packed_nmse_vs_bf16": 0.10440229579652632, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.020256718919040156, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10456562244823649, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.020299332581949013, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.99827255914705, + "packed_nmse_vs_bf16": 0.07783281086953438, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.0547154312457472, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.07765449505385229, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05538180029683119, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.07696924345519164, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.056203845844099314, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.07750997976791618, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05567161777832396, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 4, + "hessian": { + "down_diagonal_mean": 68.28947448730469, + "gate_up_diagonal_mean": 623.25537109375, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10408097988474757, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.014869021550438712, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10414392674191872, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.01501336372125661, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10423508440942157, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.014647493474856178, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10405850432395729, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.01462372830285295, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.1041606216224991, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.017054598894263574, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10428265418927132, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.01693986345004076, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9780473293593848, + "packed_nmse_vs_bf16": 0.10426019118033983, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.016459070385264476, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10419436208260932, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.017032009820623223, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.08502851096948161, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.04306685102937739, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9745924476534854, + "packed_nmse_vs_bf16": 0.0842425624603299, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.043429790340846756, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.08510398308398234, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.04222891439024946, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.0844236188021, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.042301130350876015, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 6, + "hessian": { + "down_diagonal_mean": 22.479969024658203, + "gate_up_diagonal_mean": 623.42041015625, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10213493980046151, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.0313261062662583, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10220923720246292, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.031094986608423913, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.1020194400380051, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.03071551918385672, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197949, + "packed_nmse_vs_bf16": 0.10212769594658684, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.030783153796561817, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.102010663109838, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.036926228752949944, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10210604314058493, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.03669457181604387, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10195056145987419, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.03674518390926817, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10225142413432387, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.03638837937065914, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9836374453866401, + "packed_nmse_vs_bf16": 0.07326638776803687, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.061009084969270766, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.07317738649357332, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.061975626576159716, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.07368559872788412, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.06149274621611513, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9836374453866401, + "packed_nmse_vs_bf16": 0.07315773268578399, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.06193624940145067, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 8, + "hessian": { + "down_diagonal_mean": 39.11426544189453, + "gate_up_diagonal_mean": 623.7344970703125, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10247398890633057, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.02436710370877992, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10239921282945476, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.024543803734042285, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10280489488626093, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.024170188295113693, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10276848722955376, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.024338674415547223, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10254838542041451, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.027124017970378576, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10267761288769438, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.027172598004102284, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10265013837258512, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.026756796748118917, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10273698765657742, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.026809749030575192, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.07485923900996673, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.060493324419570636, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.07474419506829094, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.06089863039736919, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.0748632117106316, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.060452733145698156, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.07539514873302494, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.059885162285308505, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 10, + "hessian": { + "down_diagonal_mean": 39.39421463012695, + "gate_up_diagonal_mean": 623.8229370117188, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10426371563243667, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.024334470036288875, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10419285708239275, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.024523247741379553, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10405257655279503, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.025072755002956592, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10394514999524154, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.024718096917902204, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10412976197989272, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.026903118015643275, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.1040196560854105, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.026981201398339382, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10400072972976594, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.026200970298159477, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.1041177329626113, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.026924160712969398, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.07488404986398188, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.06059759481710251, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.07447413982762513, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.06118690744581488, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9836374453866401, + "packed_nmse_vs_bf16": 0.07507580487987423, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.060012222875115886, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.07469317401047026, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.06032313598053849, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 12, + "hessian": { + "down_diagonal_mean": 33.07805633544922, + "gate_up_diagonal_mean": 622.6138916015625, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10467147293507349, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.024695686107095088, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10449018452830128, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.024796816693364938, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10441164351455132, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.025039182903333946, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10452374753744267, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.024566082671404945, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10449727827477519, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02683220538154027, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10424365753526367, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.026703997696323103, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10442891186517264, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.026897915630268603, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10435035856906015, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02701835164158967, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.07546516455286852, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.059726483352810125, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.0754961271449028, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05973491889967017, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470503, + "packed_nmse_vs_bf16": 0.07553212511285405, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05964662895596798, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.07580480057398083, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05936764424291133, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 14, + "hessian": { + "down_diagonal_mean": 46.10777282714844, + "gate_up_diagonal_mean": 623.7891845703125, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10244254342020366, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.02281977253094492, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.1024643597189159, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.02213080553112622, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10239101539000281, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.022896068872023202, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10232182947661275, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.022987884547141235, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10239149623498131, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.025233850988254574, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10232893772082782, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.024092529446130405, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10230508346768159, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.024658803193012398, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10217802391183253, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02496368517662206, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.0750442433308559, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.059485978831370724, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.07537282398893948, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.057924481606116116, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.07537135698299409, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05853073018339075, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.07509777565828661, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.0586886277473189, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 16, + "hessian": { + "down_diagonal_mean": 45.912879943847656, + "gate_up_diagonal_mean": 623.7052612304688, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10254941217179524, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.022609535190401382, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10262906775903553, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.022481802616237303, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.1028139499303008, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.022687569826716426, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.1026728570603684, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.023075806887558673, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10266881593733262, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.026150591276886077, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10259834218031037, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.026141181639319937, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0129076729074602, + "packed_nmse_vs_bf16": 0.10260745811892709, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02572007556207989, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10275085184676519, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.025887363150964404, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.07642005675689503, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05734338672712079, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9836374453866401, + "packed_nmse_vs_bf16": 0.07653045625012948, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.0571762989499097, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.07621664799662863, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.057977154726452734, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.07646517204468077, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05696027638555295, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 18, + "hessian": { + "down_diagonal_mean": 53.462528228759766, + "gate_up_diagonal_mean": 624.0899658203125, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.103174132892156, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.020907794614156656, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10319475548542077, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.02037062945877895, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10313484233761615, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.0212310464769312, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10319727010867341, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.020675680899457906, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10311382931146441, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.022709549008198857, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10307170351828358, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.022177874104912242, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10307011619540947, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02288403395219377, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10300814792487922, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02287478610210427, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.07546274660794601, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05803124975055619, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.07632770365602502, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05512027363103316, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.07543819994915454, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.058620616508953026, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.07572059180499992, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05786694899219802, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 20, + "hessian": { + "down_diagonal_mean": 36.30607986450195, + "gate_up_diagonal_mean": 624.0987548828125, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10286733442993641, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.023803951778387438, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10290575942421884, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.023433881383639322, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.1028836287593967, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.024144623217980724, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10312243877378631, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.024505282140911144, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10289033091758015, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.025822796343689787, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0073175568802049, + "packed_nmse_vs_bf16": 0.10287720964073785, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02590865935460509, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10287754276867343, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02596730252715946, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10292237032343687, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.026471180369821597, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.07537704029310804, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05973408849224175, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9836374453866401, + "packed_nmse_vs_bf16": 0.07561736061993238, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.058250026348007154, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.07558757463546877, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05934833407523833, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.07536283532636955, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05973863625270091, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 22, + "hessian": { + "down_diagonal_mean": 60.58787536621094, + "gate_up_diagonal_mean": 622.4157104492188, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 0.9745924476534854, + "packed_nmse_vs_bf16": 0.10419839410442616, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.02006328221568289, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10423709110140311, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.02019952273609258, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10404407855920346, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.019989322699216428, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10399640353706742, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.02016997249742839, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10403727745039461, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02175080972372545, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10418821666899956, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.021511081194918142, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10392590980555057, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02117425220476222, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.1041008198133613, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.021786016221608098, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.07652516280651568, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.0582401214937993, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.07648346995357076, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05850682659826424, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.0765107121024887, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.0583609165441313, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.076923444258629, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.057969579279606276, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 24, + "hessian": { + "down_diagonal_mean": 42.242286682128906, + "gate_up_diagonal_mean": 623.6864624023438, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10251707510877123, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.025206608755300834, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10273042300089145, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.024896667223627773, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.1026325538485515, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.025109543953714942, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0073175568802046, + "packed_nmse_vs_bf16": 0.10263483550898206, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.024972531554546307, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10259495986950191, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02659240423124327, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10248130055759429, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02645606928870013, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10256203690392872, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.026783128496948444, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9836374453866401, + "packed_nmse_vs_bf16": 0.10267871211027568, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02666427987610708, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.07415166171689234, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.0613522730698081, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.07407701567883267, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.061113217849457856, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.07393698330784097, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.061737075816484716, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.0740873109350215, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.06149664695488096, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 26, + "hessian": { + "down_diagonal_mean": 58.68699645996094, + "gate_up_diagonal_mean": 622.8535766601562, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10360167942115199, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.02287441870237748, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.1035149074334155, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.022835470539099373, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10353968820813955, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.02278441610550062, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10350189156594584, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.022577907912713958, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10321221782990452, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.025520927867681124, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470503, + "packed_nmse_vs_bf16": 0.10327609129421106, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.024850025399785106, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10355296691566025, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.025066186126036165, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10345215957987144, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02528522268978076, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.0763978457329266, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05503715479107974, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.07681295909106908, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.0528017979250395, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9836374453866401, + "packed_nmse_vs_bf16": 0.07723109918728964, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.04851572295280608, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.07640020809540894, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.056426216233585014, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 28, + "hessian": { + "down_diagonal_mean": 62.966339111328125, + "gate_up_diagonal_mean": 623.7869873046875, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10573867565023698, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.017081325290958148, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0073175568802049, + "packed_nmse_vs_bf16": 0.10562258431621768, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.017169548738002673, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10575623381307904, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.017303035251275186, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10593670563259415, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.01789539859519594, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10581515951448978, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.0177427330598718, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.1059391615768816, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.018902463898180632, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.99827255914705, + "packed_nmse_vs_bf16": 0.10594190061836078, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.018245212186171936, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10581849912556894, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.018119704257612813, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.0777777614427797, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05558281677424908, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.07763550069735398, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05595549339692146, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.07746357905488874, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.056043778878303425, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470503, + "packed_nmse_vs_bf16": 0.07746219255381538, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05568237172073985, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 30, + "hessian": { + "down_diagonal_mean": 55.03443145751953, + "gate_up_diagonal_mean": 623.3670654296875, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.1021392982065263, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.02092155587284411, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470503, + "packed_nmse_vs_bf16": 0.10205156317948115, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.020077664734547668, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.99827255914705, + "packed_nmse_vs_bf16": 0.10204123001492374, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.020391505513126115, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10210640013456407, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.020471991993510446, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10194121668316163, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.021741368731525025, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10212812533143892, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.0220124361314486, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10215241081186206, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.021798633231483743, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10176314615507216, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.021360351014203728, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.07604756077646414, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05631472970966225, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.07690559326379633, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05556061560395113, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.07649217410826323, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05441042620559614, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.07621551195265865, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05563835090467536, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 32, + "hessian": { + "down_diagonal_mean": 45.98277282714844, + "gate_up_diagonal_mean": 623.5048828125, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10372966754291246, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.019412110837193384, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9836374453866401, + "packed_nmse_vs_bf16": 0.10400667357105463, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.01953002556492762, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10383819712427564, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.01940682193821897, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10369791315640674, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.019828648226496933, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.1036712280709073, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.022836933474539314, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10374521973432795, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.022510411586783904, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0073175568802046, + "packed_nmse_vs_bf16": 0.10381888961705504, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02191534587231511, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10368770880354462, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.022798708588770304, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.08055338147274017, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.04647877734971486, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.07956597639276733, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.047693720530823096, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.0799661313095323, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.04590611969066649, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.07972315063863991, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.04866491482190054, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 34, + "hessian": { + "down_diagonal_mean": 28.241535186767578, + "gate_up_diagonal_mean": 623.56982421875, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10124811994514844, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.029636539705161398, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9836374453866401, + "packed_nmse_vs_bf16": 0.10137545165429449, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.029631709166343004, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10119303073840824, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.029282968508748442, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10142563750769076, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.029639564752156906, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.101279514659529, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.03438077368049901, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10133320196126352, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.03369400606509069, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10123409693324689, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.03427453713128065, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10134200352401683, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.0339872537712705, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.07405343334666847, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.06035255737316836, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.07446382540532995, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05787314876164444, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.07438913831800728, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05976765640189607, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.0742638597468211, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05892732216009254, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 36, + "hessian": { + "down_diagonal_mean": 32.97858428955078, + "gate_up_diagonal_mean": 624.1282958984375, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10164352260830509, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.031439507156514744, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10191121259471157, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.030562189989845202, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.99827255914705, + "packed_nmse_vs_bf16": 0.10171417199354293, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.031336762220749645, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10174340150206167, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.031762727930591476, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.1017012160337604, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.03332983378769106, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10177262994361111, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.032636309007025445, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10163438522042369, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.033598096230585926, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10175413620110682, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.033693670609521234, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529497, + "packed_nmse_vs_bf16": 0.07460670604222366, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05993461245379757, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.07539014062706727, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.054946486083204314, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.07464082173216761, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05936766439513143, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.07455710201050647, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.058886151588948915, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 38, + "hessian": { + "down_diagonal_mean": 42.31538391113281, + "gate_up_diagonal_mean": 623.706787109375, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10388277821495427, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.023363344295105167, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10407017743597695, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.023505538143326333, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10384357184941154, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.023640833790070183, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10399723374335788, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.023467020259630562, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10377191297378796, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.025842865521011858, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10384342660481545, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.025858540300560357, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10376188027069506, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02593140205381198, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10390147329043047, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.026050030074279536, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.07486320110415923, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.060973219625979555, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.07489336402499981, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.060528754254607284, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.07488118672048548, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.060454686682458375, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.07464190107343117, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.06101079852255636, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 40, + "hessian": { + "down_diagonal_mean": 58.50035095214844, + "gate_up_diagonal_mean": 624.0458984375, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 1.0017274408529497, + "packed_nmse_vs_bf16": 0.10270382311066327, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.01426369762956658, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10276059513989028, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.013904564847624415, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10281335071208635, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.013648506126359317, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9836374453866401, + "packed_nmse_vs_bf16": 0.10292753726315607, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.014102573043308942, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10283676814234524, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.019785021121945853, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10268489080915805, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.019798761995488145, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9745924476534854, + "packed_nmse_vs_bf16": 0.10296863875135248, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.0199712772090125, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470503, + "packed_nmse_vs_bf16": 0.10269535799016583, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.019333999977069444, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.07791526258901807, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.052091780164156304, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.07773742637039464, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.052043338018950815, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.07836344769958173, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05230043314406403, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.07778931708415772, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05199286836342257, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 42, + "hessian": { + "down_diagonal_mean": 72.51982879638672, + "gate_up_diagonal_mean": 623.0946655273438, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 1.0073175568802046, + "packed_nmse_vs_bf16": 0.10505548379356025, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.014149512450801713, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0073175568802049, + "packed_nmse_vs_bf16": 0.10509725120396192, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.014437201826793677, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10513610060199852, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.014558328984544457, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10517608031337093, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.015095806752162566, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10505820682068374, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.015289368397849161, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.1050597781366549, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.015343506354130166, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9836374453866401, + "packed_nmse_vs_bf16": 0.10529228159962767, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.015119022751376433, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10493905230167698, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.014910941752684946, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0073175568802049, + "packed_nmse_vs_bf16": 0.08030597998696189, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.04484849096130123, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.08004087422422801, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.04938419897397471, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.08099480260210991, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.047301441137444944, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.07971911770842953, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.04737238299909423, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 44, + "hessian": { + "down_diagonal_mean": 35.87135314941406, + "gate_up_diagonal_mean": 623.9275512695312, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10335477737870873, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.023920159843544116, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0073175568802046, + "packed_nmse_vs_bf16": 0.10337230748179192, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.024041902317517693, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10349579814077475, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.02387418082784794, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10343176386921793, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.02453378404071871, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10347355591825555, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02754297880785249, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.99827255914705, + "packed_nmse_vs_bf16": 0.10354783916745987, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.027627732737720047, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10355951746034162, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02693073905076124, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10339113486371845, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.027903676624757665, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.07542885967948415, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.0589350727063235, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.07620286281988088, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05717484716696777, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.07614194807842543, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.057491627025363264, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.07560103457695083, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05888089222353936, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 46, + "hessian": { + "down_diagonal_mean": 38.36139678955078, + "gate_up_diagonal_mean": 624.087890625, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10198008816758163, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.025916452298353407, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10212151151851771, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.02608068947106783, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10245876381079715, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.026359339630507503, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10214570461097063, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.026205452164888336, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9745924476534854, + "packed_nmse_vs_bf16": 0.10230774684382705, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.029284653294750516, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9780473293593848, + "packed_nmse_vs_bf16": 0.1021304336371437, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02889377213264515, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10224087769133346, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.029022693150496378, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.1020870426085997, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.029157633607614985, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9836374453866401, + "packed_nmse_vs_bf16": 0.07486042275527036, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.0602869518854303, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.07531595233940627, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05925815089734455, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.0749052404981392, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.06011973972716205, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.0745752184098714, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.06101296137881856, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 48, + "hessian": { + "down_diagonal_mean": 43.60319137573242, + "gate_up_diagonal_mean": 623.3857421875, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10287745741662295, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.024335698221178786, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10279658193464856, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.023816841116273756, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9836374453866401, + "packed_nmse_vs_bf16": 0.10313343447904184, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.024480923751850677, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10298130421818809, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.024477117495176753, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9836374453866401, + "packed_nmse_vs_bf16": 0.1029365598480127, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.026640320516613865, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9836374453866401, + "packed_nmse_vs_bf16": 0.1030533797493054, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.026282307722651648, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10273077350516269, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.026347295417196694, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10280237025081378, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.026786880699675927, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.07487191792424179, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.059840218886292214, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.0751757038154778, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05914069181649249, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470503, + "packed_nmse_vs_bf16": 0.07479450865622482, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05983403336556487, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.0746341885208419, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.060146089117518814, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 50, + "hessian": { + "down_diagonal_mean": 45.92564010620117, + "gate_up_diagonal_mean": 623.8507690429688, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10302424722312352, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.022403458037786313, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.1031753059877529, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.022667769095872106, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.99827255914705, + "packed_nmse_vs_bf16": 0.10314609135363195, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.02264849360754195, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0073175568802046, + "packed_nmse_vs_bf16": 0.10303373736706879, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.022879063393947328, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.1030861814322094, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.025339957969021464, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10304487414079458, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.025433947640763985, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.1029370574439535, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.025631406906072185, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10306993811012057, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.0258151328083278, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.07560388690840911, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.059066712627373515, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.07530825199869846, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05968961141581932, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.07523830594850608, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.059843946504634274, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.07535182203923721, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.059702863808526206, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 52, + "hessian": { + "down_diagonal_mean": 62.73868179321289, + "gate_up_diagonal_mean": 623.32470703125, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 0.99827255914705, + "packed_nmse_vs_bf16": 0.10288696466738495, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.01937607992438899, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10281750874004389, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.01984430352414357, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10269581182065195, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.019125590591830437, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10305273639740858, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.01982457257012577, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10265647656735925, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.021919908035161924, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10273527470308738, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.022329606426641883, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0129076729074602, + "packed_nmse_vs_bf16": 0.10288502818476065, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.021442885392264733, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10293666848532518, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.022280131789516006, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.08047533869143826, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.04516921417823175, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.07977083744962601, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.050472460030467334, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0073175568802046, + "packed_nmse_vs_bf16": 0.08113705063384546, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.04569777448030996, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.0797951653159513, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05046322966421429, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 54, + "hessian": { + "down_diagonal_mean": 31.852115631103516, + "gate_up_diagonal_mean": 623.7060546875, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10206319328979398, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.029513429637494818, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10193897014085078, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.029790864926577457, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0073175568802046, + "packed_nmse_vs_bf16": 0.10204512846708004, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.029229189448946823, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10198707235337695, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.029772792954467782, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10198458389502132, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.03267688172071472, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10202282911078438, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.03306151703321491, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.1016745055010742, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.03260373476747056, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470503, + "packed_nmse_vs_bf16": 0.1017421497471625, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.032348144639531805, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.07462462171020998, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.06108458337811488, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.07431920512510992, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.06161086023281663, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.07469865005508347, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.06095993842206, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.07419223824849382, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.061156492480566454, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 56, + "hessian": { + "down_diagonal_mean": 45.12154769897461, + "gate_up_diagonal_mean": 623.9124145507812, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10449203779707883, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.021816711166343608, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10415752281046467, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.02146624946350001, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10451579986944314, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.021441103807381193, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10455212101518382, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.022198485168677125, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10443207513750864, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.0246184974893523, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10440110539113286, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.024321552662626138, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10450794728729453, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.024874461431454883, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10444428132689217, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.025025316443569, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.07928758639003894, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.0522942031395798, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.0792045610014768, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05180558373657681, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.07836205059597223, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.0527385622073178, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9836374453866401, + "packed_nmse_vs_bf16": 0.07900352331375442, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05225124544005176, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 58, + "hessian": { + "down_diagonal_mean": 32.03685760498047, + "gate_up_diagonal_mean": 623.03515625, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10500468097205881, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.021713936286755733, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10497966002658003, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.02146507137071509, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10480674525188495, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.021293112561018058, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.99827255914705, + "packed_nmse_vs_bf16": 0.10490623200134593, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.02148401879988709, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10480052246380674, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.023087134994019105, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10474095875610022, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.023542841545662295, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.1046983742219107, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.023291177177141305, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10482950179150614, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02386233531526568, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.07766775826818693, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05202861498442485, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.99827255914705, + "packed_nmse_vs_bf16": 0.07744439054516006, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05047857768370011, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.07761673279808444, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.04986432480265167, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.07717582231358497, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05323694480480677, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 60, + "hessian": { + "down_diagonal_mean": 52.07133102416992, + "gate_up_diagonal_mean": 624.0687866210938, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10402209700812273, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.023758595376972023, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10385672900916673, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.023726452179516386, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10380280707790279, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.023884497603207752, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10377772538717724, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.023848289056552962, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10390463351039576, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02604601103188484, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10403727606124716, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.025125141533663645, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10359953809182272, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02566043748616749, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.1040126696294019, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02572776878890994, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.99827255914705, + "packed_nmse_vs_bf16": 0.07941057432130379, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.0516260246851211, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.080179208161859, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.04438285196880145, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.0800067817761433, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05135901770473727, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.08008068561408228, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05065373970961341, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 62, + "hessian": { + "down_diagonal_mean": 39.579715728759766, + "gate_up_diagonal_mean": 623.6527099609375, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.1044848329666124, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.02442558301241305, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10461529475149485, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.024372428136272495, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10439852742500108, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.024115271255074892, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10462174602681114, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.024811361134512933, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9836374453866401, + "packed_nmse_vs_bf16": 0.10464074121010375, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.026729712037043836, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197949, + "packed_nmse_vs_bf16": 0.10458613480130675, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.026437205297546416, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10464098819690405, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02613339984545149, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10439934449605295, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.026438462380795513, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.07614616562667495, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.058213491077089535, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.07607024746003231, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05829195031458668, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.07632514333471364, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.057552364766496354, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.07615385454361237, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05812515967233692, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 64, + "hessian": { + "down_diagonal_mean": 19.41680145263672, + "gate_up_diagonal_mean": 517.9146118164062, + "selected_natural_routes": 6812 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10793629369873307, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.02392161518955265, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10795966218547515, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.023626802033157684, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10787287800534427, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.02343326901370254, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10818086386923746, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.023556752886693944, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10792357851563124, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.031365832947136296, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.1079791361732936, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.03127872466637081, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10794131840890682, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.03088691431800788, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10788823087140653, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.031597023033945786, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.0746783669475781, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.06029474712444693, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.07466103368039859, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05906931642148599, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9836374453866401, + "packed_nmse_vs_bf16": 0.07487925816262363, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05986991022662334, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.99827255914705, + "packed_nmse_vs_bf16": 0.07453635827883397, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.06069189772624707, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 66, + "hessian": { + "down_diagonal_mean": 53.6463623046875, + "gate_up_diagonal_mean": 622.9095458984375, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 0.9836374453866401, + "packed_nmse_vs_bf16": 0.10349181199134146, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.019182598568727394, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10332144261358202, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.019345415648403815, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.1033595657355551, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.019801219675735577, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9780473293593848, + "packed_nmse_vs_bf16": 0.10342959335711073, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.019626276042623, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10330872318183275, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02161244889398685, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10347810119335583, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.021241799356850107, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925396, + "packed_nmse_vs_bf16": 0.10340112242901019, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.021674602624126474, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10327762530501143, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.020990813303559307, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.0755526355765835, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05827091104857268, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.07549676510662794, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05873540532805903, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.07564913773463104, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05827791866170754, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.0757912081284894, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05783332884048677, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 68, + "hessian": { + "down_diagonal_mean": 49.385597229003906, + "gate_up_diagonal_mean": 623.9631958007812, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10326567713321064, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.021646382074719077, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10351261024025836, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.021312238699177866, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10335157209998012, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.021576995463334296, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10330310012425997, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.02137573203193075, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10315547445089135, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.022356019129283228, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10323571471925275, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02315691595593116, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10331006937241657, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02327183246327652, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10318480719261344, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.022659409175359507, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.07546808377668539, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.059089726060980764, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.07520106588821766, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05965481736893088, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.07532549097765984, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05946176148973639, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.07570695241594466, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.059155138089016186, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 70, + "hessian": { + "down_diagonal_mean": 61.438995361328125, + "gate_up_diagonal_mean": 623.8193359375, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.11104953166139009, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.013585427616563806, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.1108569012385495, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.014297521896033085, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.11094171974759673, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.01424166027341467, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.11104969759501465, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.013868349641833401, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.11074162730489062, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.015539439216872295, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.11076263880183643, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.01543183537633668, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.11094585302632219, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.015364774769910794, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.110736360146785, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.015495767834373703, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.0817435802630929, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.04872087241786239, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.08178392463962984, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.048419815542787556, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.08089776948789698, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.048657398435117336, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.08105335674069815, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.04889523946523131, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 72, + "hessian": { + "down_diagonal_mean": 42.37725067138672, + "gate_up_diagonal_mean": 623.6302490234375, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10246774135174015, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.0226580066608894, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10242788042072362, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.023233851587863664, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0073175568802046, + "packed_nmse_vs_bf16": 0.10249741640038836, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.022989126048883885, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.1026174334011919, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.022881848725315512, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9836374453866401, + "packed_nmse_vs_bf16": 0.10257138663937862, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.025818823713885213, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10243679830672259, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.025957658217061044, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0129076729074602, + "packed_nmse_vs_bf16": 0.1025186551916377, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.026527505167278198, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10234172068346471, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.026269030734825123, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.07562817725339122, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05794483065993044, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.07558614506204157, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05877312006710354, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.07569585281126409, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.058450039164972045, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.07577969336774538, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.058048612512113774, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 74, + "hessian": { + "down_diagonal_mean": 29.97890853881836, + "gate_up_diagonal_mean": 622.7857666015625, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 0.9926824431197949, + "packed_nmse_vs_bf16": 0.10431702004945079, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.023315038546707127, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0073175568802046, + "packed_nmse_vs_bf16": 0.10421955007775764, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.02347332290335235, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10438958879229211, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.02361733587210972, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10439875111026975, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.02418459640030896, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9836374453866401, + "packed_nmse_vs_bf16": 0.10413013127682792, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02576232673237628, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10445079344352139, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.026471342913459695, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10423157356727657, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02634387757534928, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10411634259013541, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.026149353835034053, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.07610634047836234, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05666428886306092, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.07594196765444179, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05712269754834563, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.07595177404621245, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05728149085020045, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9836374453866401, + "packed_nmse_vs_bf16": 0.07605654259390537, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05706859378260545, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 76, + "hessian": { + "down_diagonal_mean": 51.301918029785156, + "gate_up_diagonal_mean": 624.319580078125, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10483070987659315, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.01726818323400544, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10482992768265717, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.016867763211424018, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10493173203438508, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.016737507614738512, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.1049184311670555, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.016688759595877494, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10479533168273825, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.01894067876935268, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10501365705284227, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.01974597222024924, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10483388750926041, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.018537254378907197, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10483307378388912, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.019238796167448263, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.07634620747300276, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.055631719560579745, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.07597425300751387, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05556436750770259, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.07699084551363425, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05418259972189576, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.07627511184460581, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05568803533746796, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 78, + "hessian": { + "down_diagonal_mean": 61.111541748046875, + "gate_up_diagonal_mean": 622.5504150390625, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 0.9836374453866401, + "packed_nmse_vs_bf16": 0.10491883527479048, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.019980558575184174, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10466625157995865, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.020359045888270054, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10471202891847087, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.019922714886692797, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470503, + "packed_nmse_vs_bf16": 0.1046907314322426, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.01963317144936631, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10464897632569593, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.021125018671933195, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197949, + "packed_nmse_vs_bf16": 0.10446793794266207, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.022409100078656496, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10467809185990122, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.021825139099851548, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10483461364070804, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02105703942514298, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.07583255710291614, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05804311788721685, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9836374453866401, + "packed_nmse_vs_bf16": 0.07588941248179691, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05899546691593149, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.07576458456203783, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05575005936550614, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.07596346066807122, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.058320841491555614, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 80, + "hessian": { + "down_diagonal_mean": 43.15317916870117, + "gate_up_diagonal_mean": 624.19091796875, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10390184761095199, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.02098856264651002, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10392489318355934, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.021364657571092814, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0073175568802049, + "packed_nmse_vs_bf16": 0.10395330575873231, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.020825832599540742, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10382931752404465, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.020706013653397533, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10397883881660408, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.024430658654415457, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.1040302496107605, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02469670596215049, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10382885310412916, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.024580441296474256, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10367912415099355, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.024747835199935755, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.07762822396314539, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05480535521936822, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0073175568802046, + "packed_nmse_vs_bf16": 0.07775118307917905, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.0541245410430103, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.07725999268419238, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.054721037035147226, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.07750557862715005, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.054744000640560536, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 82, + "hessian": { + "down_diagonal_mean": 46.85951614379883, + "gate_up_diagonal_mean": 624.10693359375, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10357544646572418, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.021612417998722308, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9836374453866401, + "packed_nmse_vs_bf16": 0.10363470692947992, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.0207509797986965, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10326831949048937, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.02196817834576838, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.1037464051222507, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.021372415574332386, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10341489094083924, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.023817482882239654, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10366015275194049, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.023813064652295143, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.1036337395851862, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.0235513426521927, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10348395391307465, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02413094781975679, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.07547489464326812, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.058610033900242844, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.07553616947430498, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.059174334864350725, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.07519475760417826, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05929594939902209, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.07538450054281318, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05905146883860256, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 84, + "hessian": { + "down_diagonal_mean": 21.272594451904297, + "gate_up_diagonal_mean": 622.7650756835938, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10201613298258733, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.02957572425754094, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10207840574208737, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.029079715791539566, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10202314838616766, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.02923388032389148, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925397, + "packed_nmse_vs_bf16": 0.1023075866805165, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.028795747907275987, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10187376386059348, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.03513044445770654, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9836374453866401, + "packed_nmse_vs_bf16": 0.1020513992949382, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.03520837448962016, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10216496257206964, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.035142296947605324, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10214464257840043, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.034682971563061646, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0073175568802046, + "packed_nmse_vs_bf16": 0.07331063807520154, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.06162604766704331, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.07335351037080559, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.06134121117741249, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.07334705337917619, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.06099633645472627, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.07376027938282527, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.060133835931607836, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 86, + "hessian": { + "down_diagonal_mean": 32.21875, + "gate_up_diagonal_mean": 621.5999145507812, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10439847001085589, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.020412075827787333, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10428840672304221, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.02170879404352156, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10439940669820126, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.021164646574279942, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10434046659222172, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.02118707760896, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10423625405479175, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.024953979850200217, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0073175568802046, + "packed_nmse_vs_bf16": 0.1040635149541829, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.025763626099787402, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10436191903134404, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02501630261517837, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10425806084468824, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.025159906316922175, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.07779556284340315, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.04792287061251791, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.07713618036264797, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05058912717221055, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9836374453866401, + "packed_nmse_vs_bf16": 0.07758076446299714, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.04742837529319923, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.07721465465537859, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.04845822004415281, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 88, + "hessian": { + "down_diagonal_mean": 39.39512634277344, + "gate_up_diagonal_mean": 623.5645751953125, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.102671042925508, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.022608939064237936, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10232234638589252, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.022934195445627328, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.1024539945807721, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.023159550174489785, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10250371932942766, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.023297217790586897, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10222141820718567, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.026078917094541482, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10232768555988452, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02662380749869435, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0073175568802046, + "packed_nmse_vs_bf16": 0.1023656026155802, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.026376928220231848, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0073175568802046, + "packed_nmse_vs_bf16": 0.10238552030419927, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.026300859382696697, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.07597122832184758, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.056607726527342715, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.0758132611257253, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05750734059256167, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.07606251321390399, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.058103736617110144, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.07604991798495131, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05724292462391478, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 90, + "hessian": { + "down_diagonal_mean": 59.182861328125, + "gate_up_diagonal_mean": 623.7041015625, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10496705889629128, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.014595252876103235, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10487386540898727, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.014892184660946793, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10495727901844946, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.014911201776178315, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9836374453866401, + "packed_nmse_vs_bf16": 0.10489116192065018, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.014773033061711885, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10512602888981507, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.016597022926569, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.1050304884286802, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.016538122257162582, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10502058352572238, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.016659755342213343, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10501564662259881, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.01659805830773641, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.08264413010033617, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.04285645201609796, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.0837078157807048, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.04078395964489516, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.08376316676424081, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.041960523956582264, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.08357377820952633, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.0418496298451304, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 92, + "hessian": { + "down_diagonal_mean": 54.1129150390625, + "gate_up_diagonal_mean": 623.3735961914062, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10207550575172862, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.021720513773132326, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10183416461479258, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.022257026058424303, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.1020286230178426, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.022173181501629405, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10193162935352869, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.021203410883119642, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10193055345712299, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.025022484102133955, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10197695988170662, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.025029273610692487, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.1016915224767444, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.025489490758884494, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10183930289161605, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.024751549663855805, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.07617320407416418, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.057916407706650096, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.07630129405817972, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.058771182571374606, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.07630758228692186, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.057851360633961714, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.007317556880205, + "packed_nmse_vs_bf16": 0.07690421695016322, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05732671365558599, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 94, + "hessian": { + "down_diagonal_mean": 47.27019500732422, + "gate_up_diagonal_mean": 623.5242309570312, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10352819178609932, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.024296348056940358, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10359278106102562, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.024577573516344025, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10351843594783366, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.024123521285933393, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10367039858462816, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.02405819426867671, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.1033757754656436, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.026175823689957176, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10357032588112751, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.026131940918448514, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9836374453866401, + "packed_nmse_vs_bf16": 0.10346426040633393, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.026659395283446526, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10365064391414179, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02637527663911582, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.07667400093577617, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05810261868791374, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.99827255914705, + "packed_nmse_vs_bf16": 0.0759951649241672, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.059449931074394974, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.0764040633867598, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05868843118684007, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.07614619514165741, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.058762427842488495, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 96, + "hessian": { + "down_diagonal_mean": 45.78180694580078, + "gate_up_diagonal_mean": 509.175537109375, + "selected_natural_routes": 6691 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10939889414720598, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.024155396142052334, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10931775798927171, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.024153801953252053, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.1094295802533483, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.024052362294397688, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9836374453866401, + "packed_nmse_vs_bf16": 0.10967005613192503, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.02431969991344907, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10919322085652265, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.025579632689545734, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10934754814326028, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02534943088359168, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10919249555449008, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02553515216750987, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10924464545937902, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.025587347802692176, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.07744334740612112, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05710885587125772, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9836374453866401, + "packed_nmse_vs_bf16": 0.07713188965183056, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.058185617208688506, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.07762640515180066, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05669915297472766, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.07805799471467656, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05706228492872382, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 98, + "hessian": { + "down_diagonal_mean": 37.0601806640625, + "gate_up_diagonal_mean": 624.0284423828125, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10284285679130548, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.02558051462768342, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10282326979637227, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.02551072825532684, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0073175568802046, + "packed_nmse_vs_bf16": 0.10266969054716707, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.025775566489862758, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10273417138804697, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.025977665172286505, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.1027808704749888, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02827024870192401, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10280059442371886, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02846376797916941, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10275038765752037, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02783118599630511, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9836374453866401, + "packed_nmse_vs_bf16": 0.1028629912954382, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02830198263674352, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.07481393385455588, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.06065093721919039, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9836374453866401, + "packed_nmse_vs_bf16": 0.0753868958010759, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.059209290619350645, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9836374453866401, + "packed_nmse_vs_bf16": 0.07516595220910045, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05773785784319147, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.07560167121692318, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05845351797842636, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 100, + "hessian": { + "down_diagonal_mean": 49.24579620361328, + "gate_up_diagonal_mean": 623.5359497070312, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10320042796707632, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.019559160147939767, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10320763455457507, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.02003915562661315, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10323082105041122, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.019683988513332914, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10326585247035178, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.020087146554162376, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10317791996903153, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.023149759908263283, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9836374453866401, + "packed_nmse_vs_bf16": 0.10323715698653414, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.023096007681685474, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10324441065144561, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02314993788448852, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10318010704389408, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.024111355793279655, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.08166786272716044, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.04842427936974935, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.0803784568113847, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05083775068111749, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.08144774736312929, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.0491538661738012, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.08114732466739205, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.050318417550910086, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 102, + "hessian": { + "down_diagonal_mean": 67.10078430175781, + "gate_up_diagonal_mean": 622.4995727539062, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10391106133939154, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.01971109691602257, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10378244113304987, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.02011931104334409, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9780473293593848, + "packed_nmse_vs_bf16": 0.10384259309915053, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.019799775088575916, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0129076729074602, + "packed_nmse_vs_bf16": 0.10380361291428833, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.020039555682900508, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10389036465921742, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02192915578066153, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10407965249236292, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.021930379232867423, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.1037772169437849, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.021889802807955966, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.99827255914705, + "packed_nmse_vs_bf16": 0.10376057171743326, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.022047645947127437, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.07796618507512328, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.054694506207203716, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.99827255914705, + "packed_nmse_vs_bf16": 0.07752074632648513, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05576003433516418, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.07744214277143412, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05608625785921113, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.07810787677749938, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05069268673504568, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 104, + "hessian": { + "down_diagonal_mean": 46.970401763916016, + "gate_up_diagonal_mean": 624.2557373046875, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 1.0073175568802046, + "packed_nmse_vs_bf16": 0.10468707202758229, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.02053454171925834, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10456863091022281, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.020940218343607125, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10467624156593824, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.020858588438428433, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529497, + "packed_nmse_vs_bf16": 0.10467618739603218, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.02062926971265311, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.1047609331568187, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02277821902402661, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10449621540118358, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.022732083505278396, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10447760061196025, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.023147924227570303, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10467794381978043, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.022828933363582722, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.07579183140385264, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05897117230390573, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9836374453866401, + "packed_nmse_vs_bf16": 0.07553903956674961, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.059225872376599424, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.07527884249707134, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05978614560130768, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.07572635441055918, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.058868666202113465, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 106, + "hessian": { + "down_diagonal_mean": 64.7906494140625, + "gate_up_diagonal_mean": 623.9783935546875, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.11255733739072596, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.01575040652737815, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.11277426612651639, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.015752826364938424, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.11256694276955483, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.015873647146655866, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0073175568802046, + "packed_nmse_vs_bf16": 0.11273518213819382, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.016221023881675276, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.11257241871225777, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.016769182113260266, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.11257693321915438, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.017357633179745452, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.11229484740197626, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.017497388942222258, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.11260461160237134, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.017481051516147973, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.08413908240958047, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.047119925836948295, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.99827255914705, + "packed_nmse_vs_bf16": 0.08513342070776239, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.04517881999497802, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.08440172238194643, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.0453889388842492, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.0845339007992785, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.04486875911306382, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 108, + "hessian": { + "down_diagonal_mean": 64.88285827636719, + "gate_up_diagonal_mean": 622.8834228515625, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10380784543241242, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.023391599069909907, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10376607619581972, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.023371640089908146, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10396422229556593, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.023029441057738506, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10369270152283308, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.022844411501384956, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.1038520218491454, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.025326357457380846, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0073175568802049, + "packed_nmse_vs_bf16": 0.10367417949813201, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.025518109622369563, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10370330089663984, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.025226181844276045, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10374208760066177, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02509451355662912, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.07973325431623775, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05053729376413815, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0129076729074602, + "packed_nmse_vs_bf16": 0.08047597216819104, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.04922513748308722, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.08044608530650527, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.04973538456012633, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.08053511763090665, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.049271158713569345, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 110, + "hessian": { + "down_diagonal_mean": 51.388153076171875, + "gate_up_diagonal_mean": 623.696533203125, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 0.9836374453866401, + "packed_nmse_vs_bf16": 0.10586538770272491, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.020476627085309267, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9836374453866403, + "packed_nmse_vs_bf16": 0.10595248187385574, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.020594638430854014, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10593000676518106, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.020624644854545952, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10583017421491113, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.020870750897032175, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.007317556880205, + "packed_nmse_vs_bf16": 0.10583586706516449, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.021458739298030095, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.99827255914705, + "packed_nmse_vs_bf16": 0.1056636694674494, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02123889511877013, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10582579293026088, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02157217118882527, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10567917036781836, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.021949271398609756, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.07844507947410595, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05500078859830236, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.07925094552092812, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.053965383230966335, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.07844223588693273, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05495358464164579, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.07859742888900471, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05486115966706346, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 112, + "hessian": { + "down_diagonal_mean": 37.07878112792969, + "gate_up_diagonal_mean": 623.01611328125, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 1.0129076729074602, + "packed_nmse_vs_bf16": 0.10489998284891863, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.018821216611781294, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10483367480112298, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.018763801113385353, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10508491825251529, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.019036967554018274, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9836374453866401, + "packed_nmse_vs_bf16": 0.10515432450178168, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.0190901482372225, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10491961745206595, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02141247725545176, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.1050400224720217, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02126217145635994, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0073175568802046, + "packed_nmse_vs_bf16": 0.1051361381249332, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02124989097919796, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.1050004386525427, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02119645151661874, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.0782811012107834, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.04069465829460576, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.07895367352301162, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.04189849875713156, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.0789935846501595, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.043799543230375985, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.07852032531053431, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.04432369986883913, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 114, + "hessian": { + "down_diagonal_mean": 31.669883728027344, + "gate_up_diagonal_mean": 623.8577880859375, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10413822563890092, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.024561721204174932, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.1042841964574306, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.024297320051970803, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.99827255914705, + "packed_nmse_vs_bf16": 0.10400271170863562, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.024533233783755933, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10427273526986523, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.024318982923997073, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10423069068104794, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.027389966445495878, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529497, + "packed_nmse_vs_bf16": 0.1040237512350688, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02710215068958801, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.1042973726143527, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.027485195229962348, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10405612147252649, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.027513930877896508, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.07427806989042776, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.06137519431897426, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.0745641648984172, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.060707642172240256, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.07440456467028787, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.0610489147239073, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.07428833595701836, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.060936930916231175, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 116, + "hessian": { + "down_diagonal_mean": 68.39969635009766, + "gate_up_diagonal_mean": 622.6000366210938, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10370662885312613, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.016506169968522635, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10367664982286304, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.016431653331282117, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10371005327543305, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.016172937766594207, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.1038166599616855, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.016533468412572413, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10350969126145997, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.020236944152471372, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.1035527160521946, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.01958188810261455, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10367863391063221, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.01992014787375977, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10341610796122283, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.019797932171784494, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.08262017310932912, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.049919288307516144, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.08224159150307964, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05114744246117324, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.08155312970992362, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05119475984598342, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.08281134732028453, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.050924762878882875, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 118, + "hessian": { + "down_diagonal_mean": 49.438514709472656, + "gate_up_diagonal_mean": 624.4886474609375, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10507109586264175, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.017598545631155413, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.1051886453468594, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.01748046357166721, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10501313141494452, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.017260993567507796, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10512105600292106, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.01730517169021517, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10499340516216209, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.021669375742012678, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10503101228826553, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02166909099011972, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.105000743881133, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.021117180434858447, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10484797761107048, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.021690785016171638, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.07864182255164984, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05484732863106512, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.07817240471807319, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05537490625642118, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.07851854342200831, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.054962936004607256, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.07869001330649987, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.054758724411812155, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 120, + "hessian": { + "down_diagonal_mean": 59.85546875, + "gate_up_diagonal_mean": 623.6459350585938, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 0.9780473293593848, + "packed_nmse_vs_bf16": 0.10450740682475816, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.01815445355718652, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10431882184348319, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.018980740845905147, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10441418854714467, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.01869458574050553, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.99827255914705, + "packed_nmse_vs_bf16": 0.10447701459769382, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.018431657318846405, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0129076729074602, + "packed_nmse_vs_bf16": 0.10439208244916082, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.019334529049825164, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10444579247245342, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02051508411919948, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10424263155939373, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.020864223571044647, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.1043416938255492, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.020450546136258345, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.07761707753123168, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05434327213861094, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.07786592084635494, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05517156204852423, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.07789923262731793, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.055386952202994354, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.07805584491531517, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05438084902071521, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 122, + "hessian": { + "down_diagonal_mean": 49.3797492980957, + "gate_up_diagonal_mean": 624.581787109375, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10532065434285726, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.019620155300796933, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.1053760769388011, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.01918007840947521, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10552435571972592, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.019274207921271172, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10537160794822817, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.01958685569648563, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10548901690932132, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.020033198587713002, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10538798241800942, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.020781505854727798, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10544923439577024, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02063417393602698, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.105498634097634, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02090054082523456, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0073175568802046, + "packed_nmse_vs_bf16": 0.07577794762930241, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.057220156636702935, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.07555545187261269, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.057986823118592185, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.07583627097640441, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.057381267049518396, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.0751382274702832, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.058557418736282896, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 124, + "hessian": { + "down_diagonal_mean": 38.286415100097656, + "gate_up_diagonal_mean": 623.767578125, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10259519609437623, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.02497299693545523, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10245699664132425, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.025848868659605735, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10253953245408937, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.025872171218999936, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10254930116459697, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.025902059141823294, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10235469192240003, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02869107698202687, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10244238006575106, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02955255653662066, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10248962282600849, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.029440839636556578, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0129076729074602, + "packed_nmse_vs_bf16": 0.10236120622212604, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.029643402796933453, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.07808293624117894, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.054451815709157844, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.07794613154816651, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.051652789178774826, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.07772407572749308, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05579382528570493, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.07751674941580257, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.0562152474780984, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 126, + "hessian": { + "down_diagonal_mean": 37.48945617675781, + "gate_up_diagonal_mean": 623.9396362304688, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10348379196518956, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.026885568625456848, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10348915223444334, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.027178956347760474, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10351394399340462, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.026821671102254748, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.1034862358839924, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.02717836597219012, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.99827255914705, + "packed_nmse_vs_bf16": 0.10354122311317278, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02876475132953541, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10376400431030817, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.028559900606893188, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.1035041604718937, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.028614233552543375, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10342805292589333, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.0283259227168078, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.07514164609761811, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.0604258669136831, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0129076729074602, + "packed_nmse_vs_bf16": 0.07532583494275584, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.06021287326913348, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.0753055838243522, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.059239158042509815, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.07496739776191733, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.060343614159136263, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 128, + "hessian": { + "down_diagonal_mean": 49.519371032714844, + "gate_up_diagonal_mean": 623.4497680664062, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 0.9982725591470503, + "packed_nmse_vs_bf16": 0.10181877751332245, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.020098759237440075, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10209025114033692, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.019237123841336405, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10155562101631307, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.01881475266495472, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10182019670135264, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.019601726653551074, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10178216010355413, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.026709105001228483, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10167778627495415, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02575429736685822, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9836374453866401, + "packed_nmse_vs_bf16": 0.10179213278415106, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.025526034230424318, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10159329578260773, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.025858139424378063, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.07839418978788788, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.055654040350694066, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197949, + "packed_nmse_vs_bf16": 0.07818293953175671, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.054833450478967385, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.07900664534898186, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.055371107944643894, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.078752339138976, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05394482525895965, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 130, + "hessian": { + "down_diagonal_mean": 40.20244216918945, + "gate_up_diagonal_mean": 624.0286865234375, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10313398337619176, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.024651296751007787, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10328093370648275, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.024345090371631243, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10317929765110026, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.024683448942844494, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10333585604098881, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.024485071498416697, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10314545075466454, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.027501770317332266, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10318557462795085, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.027451263331335043, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10318557648560647, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02757451249464473, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10315317860782174, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02695588565122209, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.07538041092486268, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05979289564315537, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.07546094319513494, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05970322137327163, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.0758746978029847, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05916321429622834, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.07588774551490524, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.058300175138074596, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 132, + "hessian": { + "down_diagonal_mean": 66.34370422363281, + "gate_up_diagonal_mean": 622.82763671875, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10621128161366915, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.01570798183416141, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10605927207958375, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.015298874350291704, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10639190968187257, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.01572899742015115, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9836374453866401, + "packed_nmse_vs_bf16": 0.10626098439626805, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.015526125583036185, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0073175568802046, + "packed_nmse_vs_bf16": 0.10613990358650027, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.016927003010676574, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9836374453866401, + "packed_nmse_vs_bf16": 0.10615385304481477, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.01656496800951608, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10600163161489724, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.017148387752074435, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10620686860147835, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.01696757523593848, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.07887991484506852, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05372078149977946, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.07883074565584827, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05276182997441114, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.0781439040446933, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05456293666740614, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.07865326472206366, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05345594783059123, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 134, + "hessian": { + "down_diagonal_mean": 27.283519744873047, + "gate_up_diagonal_mean": 622.8778076171875, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 0.9982725591470503, + "packed_nmse_vs_bf16": 0.1016779786014164, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.02770988615421135, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.101752279354453, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.027092671505993992, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10154634620212775, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.027647285205629042, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.99827255914705, + "packed_nmse_vs_bf16": 0.10171918454861915, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.02766213969236392, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10167803103606975, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.031290264060948716, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10174841932640308, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.03159067684710813, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10172234289386702, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.03160300171482462, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10192487028179903, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.03137759702476506, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.07388697491157367, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.06057402178090853, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9836374453866401, + "packed_nmse_vs_bf16": 0.07413051321871977, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.060389305613803304, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.073996202385257, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.060613129780912246, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.07408927659347928, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.0601568127235279, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 136, + "hessian": { + "down_diagonal_mean": 60.850135803222656, + "gate_up_diagonal_mean": 624.4488525390625, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.1027403856970717, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.019360655385805413, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.1029088070680147, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.01930674766043288, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470503, + "packed_nmse_vs_bf16": 0.10279527549097534, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.019344834438936195, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10288368697895647, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.019679342501319592, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10269607221704156, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02251565852201951, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9836374453866401, + "packed_nmse_vs_bf16": 0.10300453718172389, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02251827956632508, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.1029120741925147, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.020532383206972548, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10290511906964415, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.022337495421321636, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.08021029920983254, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05002078803837282, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.08012389879577671, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05085434499070654, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.08000976580943271, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.04651304692871856, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.07921411404026867, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05230661439994272, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 138, + "hessian": { + "down_diagonal_mean": 35.92399978637695, + "gate_up_diagonal_mean": 623.6226196289062, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10289198126619917, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.02455366755320268, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10293883648114449, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.025520567574160446, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10291964989304843, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.024598145977457044, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10295489441707568, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.024617396128084945, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10297883848805633, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.027773293603242006, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10300983183989766, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02833408065342325, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.1028498921920265, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.028298069196164784, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10306616844665077, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.027930836829250554, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.07550488276371374, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05904948757058726, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.07505562858677596, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.060420397756092745, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.0753886963756054, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05941356508697895, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.07530133406520642, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.059244650635619504, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 140, + "hessian": { + "down_diagonal_mean": 45.79924774169922, + "gate_up_diagonal_mean": 623.770263671875, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10307273115007552, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.019280224546583517, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10291015987443078, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.02021419407587892, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10292206432923237, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.019760955922340416, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.1031106105657085, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.020295215218173904, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197949, + "packed_nmse_vs_bf16": 0.10292005375483677, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.0218848916491918, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10293550191405113, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.023160151556955708, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10301946170003203, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.022424580201662672, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9836374453866401, + "packed_nmse_vs_bf16": 0.10335792513825488, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.023088890929635866, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.08083499567232745, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.050509963946845204, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.0816002422208147, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.049622447909145516, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.0810684758671519, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05066964195922847, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.08115615412483024, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.050575722731414, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 142, + "hessian": { + "down_diagonal_mean": 35.662994384765625, + "gate_up_diagonal_mean": 623.6553955078125, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10186428010390595, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.02857687237800575, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10176330198772415, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.027978190504627413, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.1017420926616655, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.02792918472819825, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10164735409546147, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.02821754688820661, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10153363109046833, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.03272329483982538, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10175820400521365, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.03241291515494594, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10172271522105115, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.03255361803905658, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10181301560480606, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.032507207700429884, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470503, + "packed_nmse_vs_bf16": 0.07654698337336303, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05573409255843674, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.07722474195027657, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.055658936833811325, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.07716580289259001, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.0553269447130172, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0073175568802046, + "packed_nmse_vs_bf16": 0.0764167107112373, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05587057673544174, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 144, + "hessian": { + "down_diagonal_mean": 62.61707305908203, + "gate_up_diagonal_mean": 623.1021728515625, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10292376523717567, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.016392533245811563, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10318886569237568, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.016157537199122662, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10307892744279053, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.016656109146957, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.1030244292179599, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.016416313413336353, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10312535750842422, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02199213250903766, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197949, + "packed_nmse_vs_bf16": 0.10297765407992271, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02161851306310921, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10277423984226268, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.022283331576760025, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10292157683307113, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.022165337775959304, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.08225432298584787, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.04887506841928816, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.08175037511603414, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.047917550018763785, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0073175568802046, + "packed_nmse_vs_bf16": 0.08110428451266057, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.04969404323483328, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9780473293593848, + "packed_nmse_vs_bf16": 0.08160036703476414, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.04984054549660607, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 146, + "hessian": { + "down_diagonal_mean": 25.391613006591797, + "gate_up_diagonal_mean": 622.5914306640625, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10206166183508866, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.029518784402679382, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10214395969156843, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.029887712871117935, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470503, + "packed_nmse_vs_bf16": 0.10238670666964625, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.030013271263308133, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10222947443080109, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.02925711565959411, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470503, + "packed_nmse_vs_bf16": 0.10197916964480751, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.032868256379775806, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10216865507027054, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.03286707672994254, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10200848914533313, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.03317600434207918, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10207716751081089, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.03311874702720328, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.07307306330126012, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.06316429245213773, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.07324233728700356, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.06285191345364957, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.0732148894766436, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.06276519304794673, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9836374453866401, + "packed_nmse_vs_bf16": 0.07304046654136782, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.06319116561667898, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 148, + "hessian": { + "down_diagonal_mean": 43.70466232299805, + "gate_up_diagonal_mean": 624.0021362304688, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10296515762617758, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.023090229305854963, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10289826037895167, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.023005633618270967, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10301656148200286, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.02346422323420745, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10299806761356096, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.02343808575684066, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10296505184488892, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02643503003891929, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10293680172323046, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.026103191231648587, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10303678628101944, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.026067236828976083, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10284760740015234, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.026186977337950404, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.07547185191762126, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05978892678405525, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.07531073679953848, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.060034084812089016, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.07553727870211843, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05886059051458552, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.07511839144783496, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05959395224706673, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 150, + "hessian": { + "down_diagonal_mean": 88.7130126953125, + "gate_up_diagonal_mean": 623.55322265625, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 0.99827255914705, + "packed_nmse_vs_bf16": 0.1042270701869947, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.010580887972635516, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10462243945771878, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.011124029362997395, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10427486703395122, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.011122965455512846, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9836374453866401, + "packed_nmse_vs_bf16": 0.10445424475226456, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.011004378179951592, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.1044314987016052, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.011936308876408024, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9836374453866401, + "packed_nmse_vs_bf16": 0.10450752731262065, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.012915448501289289, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10460263069010266, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.013637459597669021, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10446833534248053, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.01291962351216226, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.08844778682776569, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.03638368312784959, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.0894634899322668, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.038300917259266, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.08928838334699733, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.04050960511993025, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.08974890607342835, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.0391343931615422, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 152, + "hessian": { + "down_diagonal_mean": 64.0009994506836, + "gate_up_diagonal_mean": 623.2600708007812, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10340375416540987, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.016229009561873477, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10366520696978046, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.016073560507010625, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10372998598284226, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.016584182948067936, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10368068095603596, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.016423613981232196, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10365103309756446, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.018045616876871694, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10369763653168772, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.01897002114369138, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.1035919605879172, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.018769228838693856, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10367015287861663, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.018744734846376423, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.08018458081306647, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.0485717134345359, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9836374453866401, + "packed_nmse_vs_bf16": 0.08000462210346203, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.048661136024436724, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.08033128685912248, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.04975123893551483, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.08048480067577359, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.04847413562424807, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 154, + "hessian": { + "down_diagonal_mean": 39.00638198852539, + "gate_up_diagonal_mean": 623.4002075195312, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10609118814417963, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.02117880460018202, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.1062592082651306, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.021667619885950434, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10602944736877987, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.02144430607322844, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10605713389716218, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.02172628155439014, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10638604882933288, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.023208353474657373, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0073175568802046, + "packed_nmse_vs_bf16": 0.10610827803368163, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.023058365554258744, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10595527344644148, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.023249396335480697, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10613067456800138, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.022555477098819714, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.07815027275846188, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05178790102055395, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9836374453866401, + "packed_nmse_vs_bf16": 0.0780948104331216, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.052139143861428594, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.0782202878675015, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05201652813734091, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9745924476534854, + "packed_nmse_vs_bf16": 0.0780278494014116, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.052144976507691315, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 156, + "hessian": { + "down_diagonal_mean": 35.53047180175781, + "gate_up_diagonal_mean": 623.3345947265625, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10464077457273815, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.025472587390921553, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10485731120153802, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.02516165022724208, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10469847003675754, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.024983288183897754, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10466726405159903, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.025598093236013995, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10441110674987351, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.027755887096185778, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0073175568802049, + "packed_nmse_vs_bf16": 0.10439018914205203, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02761167822320222, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10444012595629838, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02756848679240978, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10474918377146668, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.027688256852545185, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.07501392385681258, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.060616380091745616, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.0748421094595547, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.06083144199717992, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.0749579348432849, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.060839634021021466, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.07474156659725052, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.061145671168946196, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 158, + "hessian": { + "down_diagonal_mean": 55.59443664550781, + "gate_up_diagonal_mean": 623.7218017578125, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 1.0073175568802046, + "packed_nmse_vs_bf16": 0.10454700594943085, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.017999423958704745, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10468115616714022, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.017560609738911104, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10443156362175661, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.01764791695206684, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10469546966295716, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.017276480828516114, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10453306400054013, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.019976288032873275, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.104707014211695, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.01906939601004378, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10460002522901836, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.019322040381599518, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10471457360806195, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.019094356858851975, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0073175568802046, + "packed_nmse_vs_bf16": 0.07924177382891752, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05346813693004652, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.07895924332236241, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.0535935878128217, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.07939143785141137, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05272456716378092, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0073175568802046, + "packed_nmse_vs_bf16": 0.07898594641719808, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.053517618556449605, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 160, + "hessian": { + "down_diagonal_mean": 34.896419525146484, + "gate_up_diagonal_mean": 623.45556640625, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 0.9982725591470503, + "packed_nmse_vs_bf16": 0.10362909257977163, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.02110532209583256, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9836374453866401, + "packed_nmse_vs_bf16": 0.10378213064480944, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.02133805155536289, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10341976139258408, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.022232582205080207, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0073175568802046, + "packed_nmse_vs_bf16": 0.1036842149648647, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.021765328950407746, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10357961687228807, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.023720693861004286, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10356639906099213, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02441824181357926, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10348967509379693, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.024320282739700688, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10359922102433139, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.024391716242683017, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.07638971048543011, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.04949479160133453, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.007317556880205, + "packed_nmse_vs_bf16": 0.07634572976827994, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.053324071029255314, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.07603678525080687, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.053656610729144065, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470503, + "packed_nmse_vs_bf16": 0.07683879856233823, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05289432763755496, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 162, + "hessian": { + "down_diagonal_mean": 50.830806732177734, + "gate_up_diagonal_mean": 623.087890625, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 1.0129076729074602, + "packed_nmse_vs_bf16": 0.10442735970159832, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.02114833817296468, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10441084140210458, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.020917810664670373, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10465981297951617, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.020913334602776973, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10450324720487174, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.02037399704978973, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10433644599406335, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.022938315511585337, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10449379683701027, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.023309805300751107, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10450658199466158, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.023345678243466887, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9780473293593848, + "packed_nmse_vs_bf16": 0.10449755794531485, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.023535781907317317, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.07638904900142361, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05852330526666936, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.07615249252967837, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05892306360146758, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.07607697057744053, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05889075708841908, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.07606894055781062, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.059197745207698105, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 164, + "hessian": { + "down_diagonal_mean": 76.01593017578125, + "gate_up_diagonal_mean": 623.826904296875, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10399918722199612, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.017369871262210464, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10386685739095917, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.01741130872852762, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10388650393836746, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.017108765516378324, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10401218529839372, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.017399423490711594, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10386251659996455, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.01986771297399755, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10401871780226289, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.020126690411874287, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9836374453866401, + "packed_nmse_vs_bf16": 0.10376031259585565, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.01974125289388299, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10404279814807778, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.020311779863073674, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.07795018399954412, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05445086698917632, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.07813591916520025, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05517841217475604, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.07851530920752539, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05397397573566899, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.07850567762747562, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05457720315819918, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 166, + "hessian": { + "down_diagonal_mean": 36.085365295410156, + "gate_up_diagonal_mean": 622.5489501953125, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.102758302632136, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.024502960198180816, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10280480104645129, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.024890796370280365, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10273191841275733, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.02480150374227368, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0129076729074602, + "packed_nmse_vs_bf16": 0.1027813924229018, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.02430780104488927, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10276630804546494, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.027955974986270334, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10273510338043039, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.028181339594823534, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9836374453866401, + "packed_nmse_vs_bf16": 0.10284385718417148, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02792049620440595, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10259072238700681, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02812612965937925, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.07795418658080037, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05493412213583136, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.07796071008534225, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05440839273906868, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.0774431599164376, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.055692569873978304, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.07728403316885964, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05526413874944049, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 168, + "hessian": { + "down_diagonal_mean": 64.00151062011719, + "gate_up_diagonal_mean": 623.3141479492188, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 0.9836374453866401, + "packed_nmse_vs_bf16": 0.10452858238485646, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.02004490168320906, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10465913926240122, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.020240586434486712, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10448752154602338, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.02033763320241257, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.99827255914705, + "packed_nmse_vs_bf16": 0.10442766121300028, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.02008626967888552, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9836374453866401, + "packed_nmse_vs_bf16": 0.10455419643776798, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.022794980787630377, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.1044932499260925, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.022762074071988, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10462409006393403, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.023656874422151544, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9836374453866401, + "packed_nmse_vs_bf16": 0.10471036156933487, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.022692983545530388, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.07954305600314437, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05505700559620449, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.07902322542992908, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05600247048735639, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.007317556880205, + "packed_nmse_vs_bf16": 0.07891502069666742, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.055740216848001324, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.07925291581913048, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.055140962326667486, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 170, + "hessian": { + "down_diagonal_mean": 43.123695373535156, + "gate_up_diagonal_mean": 597.4197998046875, + "selected_natural_routes": 7850 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 1.0073175568802046, + "packed_nmse_vs_bf16": 0.10579162300954886, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.022522565877667763, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.1059538784706471, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.022081204079815384, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10594938109943047, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.021595054610007523, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10600017102211459, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.02215668134123979, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9836374453866401, + "packed_nmse_vs_bf16": 0.10571325404771222, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.025290182178201847, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.1058686318317052, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02506506344072664, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10566241643860412, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.024753635934874226, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10578508551073768, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02518299949244286, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.0755506252966653, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05960020845905058, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.07533166863310618, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05961090038552419, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.0758885087723437, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05929880393396379, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.0756204527146352, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05893405724382705, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 172, + "hessian": { + "down_diagonal_mean": 49.805084228515625, + "gate_up_diagonal_mean": 623.3597412109375, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10137984800985331, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.02223243114917916, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10162891830363198, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.022230388453035904, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.1013737382530964, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.022289174924339246, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.1016732804128262, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.022974463018781376, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10144126047790489, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.0238063614807847, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10169044522136571, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02390444851498513, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10147297744185511, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.024255680053932853, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10153785389556182, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.0245338512483923, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.07675301087144297, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05566431150643892, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9836374453866401, + "packed_nmse_vs_bf16": 0.07686253889157912, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05558397347695103, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.07644509049988901, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05575053519080123, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.07713998053413894, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.054329884641865514, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 174, + "hessian": { + "down_diagonal_mean": 43.900146484375, + "gate_up_diagonal_mean": 623.8836059570312, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10375290593100214, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.02335353240452082, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0129076729074602, + "packed_nmse_vs_bf16": 0.10367232777200659, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.02363280179540155, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10354704499457548, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.023831649168621148, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10387685451915987, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.023257176718354127, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10372454865391734, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.025719985015999997, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529497, + "packed_nmse_vs_bf16": 0.10375584293304009, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.025085836459438658, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.1036798683956848, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02511928136744587, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10377359843275855, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02562794847788542, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.07810747289168171, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05474485109276519, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.07757753066839339, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05614896862093518, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.07827514164761432, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05394419399188454, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9836374453866401, + "packed_nmse_vs_bf16": 0.07795147413668517, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05424656806005817, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 176, + "hessian": { + "down_diagonal_mean": 40.27581024169922, + "gate_up_diagonal_mean": 499.1802978515625, + "selected_natural_routes": 6555 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 0.9870923270925396, + "packed_nmse_vs_bf16": 0.11340036953776338, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.015577208333414676, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.11334810344137879, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.01583328324628666, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.11302381345708178, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.015830666721767587, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0073175568802046, + "packed_nmse_vs_bf16": 0.11326841269058437, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.015219606808773668, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.11343823242112062, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.018088794380996746, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.11326429688761108, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.0177684591964296, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.11335897736310468, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.018050970058170347, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.11337889376394428, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.018255520013752594, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9836374453866401, + "packed_nmse_vs_bf16": 0.07777702295022079, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05539047806687061, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.07768185167525606, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05512005066834573, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197949, + "packed_nmse_vs_bf16": 0.07757079876816458, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.055338087242357815, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.07746164980014816, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.055980878992995, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 178, + "hessian": { + "down_diagonal_mean": 32.72511291503906, + "gate_up_diagonal_mean": 623.958984375, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10256520367305831, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.026297109755536, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10259456989976964, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.026716945010711448, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.99827255914705, + "packed_nmse_vs_bf16": 0.10272752863551109, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.026682375218684832, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.1025764832185019, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.026342496273307914, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10264556231718895, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.029019744665496408, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10255381222650646, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02915729271810329, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10264134416158854, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02893861406889515, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10262952334048295, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.029135367433560988, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.99827255914705, + "packed_nmse_vs_bf16": 0.0747766102638041, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.06098749960360775, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.07505144069576998, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.06017029304826793, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.07487939920271383, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.0606917474671469, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.07475139591734505, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.06083045405271845, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 180, + "hessian": { + "down_diagonal_mean": 54.013702392578125, + "gate_up_diagonal_mean": 624.394287109375, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10446846408652767, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.018937340384494922, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10445321142149844, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.01913365058828919, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9836374453866401, + "packed_nmse_vs_bf16": 0.10447669365628205, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.019151867318231555, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10452886947462617, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.01936156183279903, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10446404898601311, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.020906258757819676, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10428597506588087, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.020423099530939973, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10441969552735605, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.020907946833602906, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10452051927342944, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02060923474038737, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.07984572507455898, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05281811998093414, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.07935528620089007, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.053988828220519085, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0129076729074602, + "packed_nmse_vs_bf16": 0.07928180815394947, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.054757231841960546, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.07955165728566298, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05361461076010693, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 182, + "hessian": { + "down_diagonal_mean": 44.84532165527344, + "gate_up_diagonal_mean": 624.1636352539062, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10464557796009108, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.02201877023604248, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10460811454972595, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.02214632319781062, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.1046640193634701, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.021867132514955794, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10449488828696117, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.022146962981311413, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10468765521944498, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.023645915987412766, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10461041763756729, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.023797343473238218, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10445983514698126, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.023203230106203, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10451864720122, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.023897724534192148, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.0753626287183182, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05823360405633987, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.0751580194556689, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05910316110963083, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.07523047496587296, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.058617335579498914, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.07510065349681512, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05898560227908994, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 184, + "hessian": { + "down_diagonal_mean": 57.315242767333984, + "gate_up_diagonal_mean": 623.1932373046875, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10522001348922189, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.018920055622539764, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10507370441455814, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.019015715349699328, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0073175568802046, + "packed_nmse_vs_bf16": 0.10490897122695726, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.0191826339676926, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10506984668940349, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.018537458672536925, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10516255269308301, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.021567907567479828, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10530461528802351, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.021079409795859518, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9780473293593848, + "packed_nmse_vs_bf16": 0.10526231868276803, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.021665933020813433, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10520261354447662, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.021159544052193772, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.07848489414180805, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.053285506576180926, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.07922096642931126, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05143517570447736, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.07891585343551538, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05243669332695857, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.07860683025347869, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05307403333035108, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 186, + "hessian": { + "down_diagonal_mean": 61.837486267089844, + "gate_up_diagonal_mean": 622.7181396484375, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 1.0073175568802046, + "packed_nmse_vs_bf16": 0.10402836259195682, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.02164629865356307, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.1041191138146707, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.020905287197023815, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197949, + "packed_nmse_vs_bf16": 0.10414642396164046, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.02138936369207368, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10405143232501356, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.020684357643442397, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0073175568802046, + "packed_nmse_vs_bf16": 0.1039544042918948, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02241079104475016, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10402251154381156, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.022125343576996352, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10420581886958759, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02288411788306446, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.1040609299501061, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.023055846948007372, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.07596624254358522, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.058849791915265195, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.07641670876534962, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.058115727759823044, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9836374453866401, + "packed_nmse_vs_bf16": 0.07610882936378276, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.0588234086888191, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.07609267563023538, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05898922780976718, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 188, + "hessian": { + "down_diagonal_mean": 28.062339782714844, + "gate_up_diagonal_mean": 624.0008544921875, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10163814739861599, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.030821010171536038, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10168958881458984, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.03107457725501943, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.101541363213559, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.030521892654541742, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10161688765879558, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.03126624591122933, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.1015264583803826, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.034172870802662826, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.101674346873582, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.03473356184967282, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0073175568802046, + "packed_nmse_vs_bf16": 0.10161954307323509, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.03378375152507156, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10152838361184148, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.033974363747087614, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.07324517330471257, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.06267222610417605, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.07304750923194699, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.06325244621333484, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.07332929478914958, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.06122919539121298, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.07389626416815376, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.06061979095677283, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 190, + "hessian": { + "down_diagonal_mean": 48.87965393066406, + "gate_up_diagonal_mean": 623.399658203125, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.102460617586559, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.02269189206974267, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.1023854346972857, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.022479471613095967, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9745924476534854, + "packed_nmse_vs_bf16": 0.10252204639051908, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.02244280090974769, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10242439912488858, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.023047171766410437, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0073175568802046, + "packed_nmse_vs_bf16": 0.1022695874777246, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.0247504419618673, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.99827255914705, + "packed_nmse_vs_bf16": 0.10215726807046581, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.025044227483785746, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529497, + "packed_nmse_vs_bf16": 0.10231752842500741, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.024672012013946143, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10236628739214822, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.024617585569435732, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.0763025237515922, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.058327541869347334, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0163625546133594, + "packed_nmse_vs_bf16": 0.07612742395702682, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05836070845557891, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.07617760428708711, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.058451808108762805, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.07635858648666127, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05782820070523601, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 192, + "hessian": { + "down_diagonal_mean": 32.67494201660156, + "gate_up_diagonal_mean": 622.939208984375, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10426571349686717, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.022762713744451282, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10416584025611325, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.022959336422338564, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10423589090695116, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.022948014959034927, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10451140823857025, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.0231494265318637, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10416745376092247, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.025382838514143902, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10434219428525605, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.025711318490432924, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.1041715846995645, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.026167460360683494, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10437178274929129, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.026289778201987857, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.07595555748135945, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05170710835631975, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9836374453866401, + "packed_nmse_vs_bf16": 0.0760610437995071, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.055388468157967614, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.07570844555949943, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05664816780992945, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.07586064835110813, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05643229089864028, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 194, + "hessian": { + "down_diagonal_mean": 54.750465393066406, + "gate_up_diagonal_mean": 623.376953125, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.1032439105452402, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.018742489700603167, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.1034236119879241, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.01824168803845916, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.1031961087457634, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.018850657611353707, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10325801689247978, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.018687431109616147, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10317748017204738, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.021176136159144737, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.1033074009078581, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02063931048175846, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.103265999141472, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.021608534122605678, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10341272287660526, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.020743111228245983, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.07716655944167526, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.053867422029484975, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9836374453866401, + "packed_nmse_vs_bf16": 0.07753320580891138, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05453837871394161, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9836374453866401, + "packed_nmse_vs_bf16": 0.07733049856637375, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05486523014068668, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9780473293593848, + "packed_nmse_vs_bf16": 0.0776047988326711, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05531163068187838, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 196, + "hessian": { + "down_diagonal_mean": 51.27519607543945, + "gate_up_diagonal_mean": 623.900146484375, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 0.9836374453866401, + "packed_nmse_vs_bf16": 0.10442771462590704, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.020356355637990133, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9745924476534854, + "packed_nmse_vs_bf16": 0.10433061658919539, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.020219023163942522, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.1042420484896642, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.020625002202816246, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10417502153632069, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.020261966561978283, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10417316350430313, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02175491950590428, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10436107022488991, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.022021131386841312, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10418297618021274, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.021798808290787224, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10412473611901946, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02188077686476996, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.07903365923998124, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05511144316853279, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.07861691136111351, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05613412909565926, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.0782273832972372, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.056362753610829, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.07879060793708359, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05564686273328955, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 198, + "hessian": { + "down_diagonal_mean": 55.26480484008789, + "gate_up_diagonal_mean": 622.9918212890625, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10242313053547873, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.01952489242972225, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470504, + "packed_nmse_vs_bf16": 0.10227269528727724, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.019719891345167086, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10234498270745364, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.01981387161277106, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10248969678368851, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.019368023579363857, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.1023327805380658, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.024042736048302423, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0073175568802046, + "packed_nmse_vs_bf16": 0.10230901218172968, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.023592431955601897, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10241829427968618, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.023430697352283508, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10238366753537627, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.023477209012453308, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.07963125778320196, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05382061442433686, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.99827255914705, + "packed_nmse_vs_bf16": 0.07983004447539041, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.052648884481255785, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.07983052527635172, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.053465824129657225, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.07933914014363869, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.053357509518223115, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 200, + "hessian": { + "down_diagonal_mean": 41.35102844238281, + "gate_up_diagonal_mean": 624.1865844726562, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10253569553442575, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.023229041702707943, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10292771297218328, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.022715821538264146, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10280852356694065, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.023196966769937728, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.1027910757959936, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.022769828760853397, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9836374453866401, + "packed_nmse_vs_bf16": 0.10286217282477415, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.025506899506669137, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.102808880828571, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.025437741840396284, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.1028678006900193, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.025352482734964172, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10274805485986845, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.025451400210433888, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0129076729074602, + "packed_nmse_vs_bf16": 0.0744878138941868, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.06007720769621419, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.07491153893079253, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05940333252557486, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.07457425921021986, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.06003263542529275, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.07457154337028804, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05929018146126536, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 202, + "hessian": { + "down_diagonal_mean": 46.5565185546875, + "gate_up_diagonal_mean": 623.9336547851562, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10519313970059062, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.020566243390208377, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.1054907813415758, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.020769837279038975, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0073175568802046, + "packed_nmse_vs_bf16": 0.10519258609597085, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.020871192597566462, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10516236910195378, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.02060186726795058, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10540873627551585, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.021983436140063435, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10539655811221466, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02240181054739559, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10533305461324863, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02221331568308509, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10523070812235724, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02251161155737427, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.0799658133018071, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05228896629536899, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.07923232973333476, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.052905613988389326, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.0792367809179193, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05379069691105979, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.07934405322658113, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05362831113654659, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 204, + "hessian": { + "down_diagonal_mean": 42.18285369873047, + "gate_up_diagonal_mean": 623.5030517578125, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 1.0073175568802046, + "packed_nmse_vs_bf16": 0.10317660972767487, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.024367469948037842, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10335611966695768, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.02478328217346968, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9836374453866401, + "packed_nmse_vs_bf16": 0.10333790473819754, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.024783767121086053, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10324539898253958, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.02476155044903381, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.1032083864108523, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.026933102177526595, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.1030706632186127, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02726011840273042, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10322475303453238, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02735799871864012, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10326404230289957, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02767966132495402, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.07608308852609903, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.057526822528870106, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.0757652517594132, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05855785720940294, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.07575791161955191, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.058926774031933075, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.07568041288361085, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05936861831430457, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 206, + "hessian": { + "down_diagonal_mean": 34.74131774902344, + "gate_up_diagonal_mean": 623.22021484375, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10306224171778254, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.022711513700259738, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10310355356564581, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.023180825520258123, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10283793383997615, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.022797705867267593, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0073175568802046, + "packed_nmse_vs_bf16": 0.10303920375503245, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.023536534668490035, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9836374453866401, + "packed_nmse_vs_bf16": 0.10295285322582555, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.027524966597746562, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9836374453866401, + "packed_nmse_vs_bf16": 0.10304575294251339, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.027461153928907255, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0129076729074602, + "packed_nmse_vs_bf16": 0.10288064255731025, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.027726918322454098, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10286214916789482, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02736107156447115, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197949, + "packed_nmse_vs_bf16": 0.07663004606852089, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05603039655383661, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9836374453866401, + "packed_nmse_vs_bf16": 0.0761888182135086, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05755133099729047, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.07685027368296266, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05594177594556221, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.0763705497436826, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05691479801589267, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 208, + "hessian": { + "down_diagonal_mean": 47.041786193847656, + "gate_up_diagonal_mean": 623.4185791015625, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10409899384531283, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.02271186500642028, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10436805026548727, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.022207234157651825, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9836374453866401, + "packed_nmse_vs_bf16": 0.10437925690703884, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.022203753027513658, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10419851620935439, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.021741684842294724, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10411096949144097, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.025293553430062243, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10413174107014986, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02508671214779548, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.1042317065534754, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.0251503959348247, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.99827255914705, + "packed_nmse_vs_bf16": 0.10412257929607872, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.025247608045198516, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9836374453866401, + "packed_nmse_vs_bf16": 0.07726783065107785, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.056433362680344205, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9836374453866401, + "packed_nmse_vs_bf16": 0.07740656361766829, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05687356886270871, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.077065680643468, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.057146613677181345, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9836374453866401, + "packed_nmse_vs_bf16": 0.07771527230339736, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.055966650400710326, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 210, + "hessian": { + "down_diagonal_mean": 33.494117736816406, + "gate_up_diagonal_mean": 624.2249755859375, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10293694652020997, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.025778702375438937, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10300831247847336, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.026159194334262347, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10310294329920637, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.026566060384041253, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10318471564133232, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.026306828926620434, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10301453454630527, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02929019351415119, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10294318109615064, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.028806903134610283, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10314266758632457, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.028833216512786128, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.1030493207818697, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.028627602465557782, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.07420046124707332, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.06138370751034756, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.0738795336390453, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.062069223051936115, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.07398442346687599, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.0618351423430655, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.07387573239457042, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.062033215994430686, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 212, + "hessian": { + "down_diagonal_mean": 46.59952926635742, + "gate_up_diagonal_mean": 623.884033203125, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10274642897571508, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.023636753533432753, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10284576836731016, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.023185366588025544, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.1027073292216336, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.023621122850538284, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10278199632411097, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.02389359505727423, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.1027659370277542, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.026729626700234044, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10271377375776423, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.026774531145052603, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10277348920407788, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.027101189796114523, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10268147314269836, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.026628240770235144, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9836374453866401, + "packed_nmse_vs_bf16": 0.07508172380370662, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.060320277244360385, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.0750923771535139, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.06018703387745389, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.07527007339187652, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.060398557475669426, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.07548046011087121, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05974396354916542, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 214, + "hessian": { + "down_diagonal_mean": 46.965824127197266, + "gate_up_diagonal_mean": 624.3529052734375, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 0.9870923270925396, + "packed_nmse_vs_bf16": 0.10387272132641709, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.020516207459262376, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9836374453866401, + "packed_nmse_vs_bf16": 0.10388152227264451, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.020558960865846684, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.99827255914705, + "packed_nmse_vs_bf16": 0.10364182040564722, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.021061725767767627, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.1037626951536439, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.020528013490428848, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0073175568802046, + "packed_nmse_vs_bf16": 0.10358158605743316, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.022791658771491072, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10371608894041125, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02265891148131552, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10367202878195178, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.022949481818899088, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10367580168620825, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02234894107940246, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.0757219293831254, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.057586470648357695, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.075562669535683, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.057819446676378006, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.07568704082592317, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05667785014115485, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.0759646632672835, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05736105626377355, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 216, + "hessian": { + "down_diagonal_mean": 51.848541259765625, + "gate_up_diagonal_mean": 623.2388916015625, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10339629304922218, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.02311570678239602, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10325480296722676, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.023335487678334446, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10326658538041156, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.02297619827031003, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10339078796203559, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.023223568764075244, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10307339391876019, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.0244464257021926, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470504, + "packed_nmse_vs_bf16": 0.10314276823903046, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02551148751600517, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10335751163949841, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.024759976308504923, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10325597258518951, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02517463136333327, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.0751618643872927, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.060226072402666364, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0073175568802049, + "packed_nmse_vs_bf16": 0.07482226152881821, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.06077024362506053, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.07511426449941074, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.06021605994943157, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0129076729074602, + "packed_nmse_vs_bf16": 0.07518041351514612, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.0602542268100836, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 218, + "hessian": { + "down_diagonal_mean": 44.39796447753906, + "gate_up_diagonal_mean": 623.3547973632812, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 1.0017274408529497, + "packed_nmse_vs_bf16": 0.103895948449444, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.01998921700627308, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10427454876226333, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.020443346250398835, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10439025735407384, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.02039690588412254, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.1040508939985766, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.02035400346571085, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10425564503884607, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.021829972420786353, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10413094284065974, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02147216671279632, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10420452446648268, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.022857672223525444, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9780473293593848, + "packed_nmse_vs_bf16": 0.10423988447974018, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02229860663658299, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.07663448983957608, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05691381822135842, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.07649429446729142, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05775556551013575, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.07658678922930812, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.057593437186143945, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.07653673868761612, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05740018679965897, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 220, + "hessian": { + "down_diagonal_mean": 77.91622924804688, + "gate_up_diagonal_mean": 622.782470703125, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10404430066448302, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.01735693227208887, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10400142140675075, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.017461045748063828, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10382795051454441, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.01680311825521493, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9836374453866401, + "packed_nmse_vs_bf16": 0.10396165434753991, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.016925006658812313, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.1038270371016177, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.018183112783852137, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.1038990554061801, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.017336172937742193, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10375790194898736, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.01784436136882728, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10390156096830327, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.017640585209185922, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.07950993555106348, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05177760104985315, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.07921841515764844, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05233578573548122, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.07965488733906054, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.052663556967481935, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.007317556880205, + "packed_nmse_vs_bf16": 0.07881753843015153, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.053665453928024416, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 222, + "hessian": { + "down_diagonal_mean": 33.77296829223633, + "gate_up_diagonal_mean": 622.8327026367188, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.1039310856335702, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.02280208216680729, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10397004557545526, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.022899858258363732, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10380773730476156, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.02313358745364943, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0073175568802046, + "packed_nmse_vs_bf16": 0.10388344180956147, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.0227073099480058, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10389910228013424, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.026207804747930646, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10395190135272929, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.025869100476353192, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9836374453866401, + "packed_nmse_vs_bf16": 0.10397752308218979, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.026610504601958334, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10409150150460447, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.026303181658082042, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.07700519806933447, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.049674545191866426, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0129076729074602, + "packed_nmse_vs_bf16": 0.07701896208443325, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.04992365143767795, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.07674480160274902, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05146394325227654, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.07733896705135881, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.04877574676613375, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 224, + "hessian": { + "down_diagonal_mean": 44.754913330078125, + "gate_up_diagonal_mean": 622.84228515625, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 0.9836374453866401, + "packed_nmse_vs_bf16": 0.1036076085079565, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.020192205148097404, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9836374453866401, + "packed_nmse_vs_bf16": 0.10377470629671477, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.019831669353256375, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10352697794800002, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.020158018893480865, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10367374923631138, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.020000784965269396, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.007317556880205, + "packed_nmse_vs_bf16": 0.10340030876382557, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.022442515793378127, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10369650150519633, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.021410958185871052, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10349705352010867, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02233122125595521, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10351999070864054, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02156611064880026, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.07762524470513851, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.053930809576976796, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.078269570142441, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.052349391908513156, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.07775411753723745, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05404830986043274, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.0774530087907171, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.0541869630852644, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 226, + "hessian": { + "down_diagonal_mean": 31.195598602294922, + "gate_up_diagonal_mean": 622.2503662109375, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10335720833595302, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.023019326128708627, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10335572738144735, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.0234639429600411, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9836374453866401, + "packed_nmse_vs_bf16": 0.10348342889076183, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.02346370037755354, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10351833119914661, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.023651067612432266, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10361315698272341, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02689878836150675, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10341539850885666, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.026948387673800138, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10325529702277882, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.027194112792642815, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10345945493852944, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02738922108257481, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.0758088634463486, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05736467428705208, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9836374453866401, + "packed_nmse_vs_bf16": 0.07563276101176147, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05793481491927568, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9836374453866401, + "packed_nmse_vs_bf16": 0.07580308825923998, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05711121350956315, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.07548501164272824, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05786660501589566, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 228, + "hessian": { + "down_diagonal_mean": 41.76402282714844, + "gate_up_diagonal_mean": 623.189453125, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10514130413493576, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.024037277641604075, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10521641591822015, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.0241373981651236, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10526532368204877, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.02419395267189048, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10508048448676088, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.023898702359195834, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10491594297179373, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.025203905974238442, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10509616877877449, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02500425777748326, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.1052243758961676, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.025172191315984915, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9836374453866401, + "packed_nmse_vs_bf16": 0.10521534340494708, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.024951365021678926, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.07705771206151878, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05557271679266459, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.07668667105430794, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05611888175606722, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.0767756074448186, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05666484986106236, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470503, + "packed_nmse_vs_bf16": 0.07735701361589997, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.055777176520465654, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 230, + "hessian": { + "down_diagonal_mean": 52.62949752807617, + "gate_up_diagonal_mean": 623.718994140625, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10572368941132451, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.017016349853963313, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10593372867782654, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.0171676085001648, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10575462412951467, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.017263751671924753, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10593159889170133, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.017257294920678415, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10569754473183837, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.021492412215399086, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10581379639501227, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02008065639638066, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10586828479208492, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.01976490299873593, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10597126253009395, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.020551911920865414, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.0812768156412298, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.04857679785821749, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.08166759394778937, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.0482281940073039, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.08201408814555139, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.04712963328707957, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9836374453866401, + "packed_nmse_vs_bf16": 0.08105441226879356, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05010831612698481, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 232, + "hessian": { + "down_diagonal_mean": 44.39486312866211, + "gate_up_diagonal_mean": 623.0242919921875, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 1.0073175568802049, + "packed_nmse_vs_bf16": 0.10303619189282072, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.02194364502534119, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.99827255914705, + "packed_nmse_vs_bf16": 0.10320295032169846, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.02150892501147376, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10328629410462034, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.02172767628417708, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10325933160762127, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.02132923170673973, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10302682918461845, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.023982622443179922, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9836374453866401, + "packed_nmse_vs_bf16": 0.1032011026136375, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.023538985784088064, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10329505257496095, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.023658599409553456, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10301984152176011, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.023730772222092574, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.0762193760267689, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05873913304919074, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.0765309250956756, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05809949479010022, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.0763719371136369, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.058009368614854576, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.07641353642590881, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05826439925298726, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 234, + "hessian": { + "down_diagonal_mean": 52.005332946777344, + "gate_up_diagonal_mean": 623.6495971679688, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10437604074964983, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.021067914342436853, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10473115563396873, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.020724334089444414, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9836374453866401, + "packed_nmse_vs_bf16": 0.1044753747515854, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.020666968233514646, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10474928183936429, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.020314352321712026, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10463280426506752, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.023111898891091186, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10457861844189068, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.023136508024569544, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10475637233039005, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.023994480093352095, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10439276630448148, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02286838877788005, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.07737324552074117, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.056752603863011834, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9836374453866401, + "packed_nmse_vs_bf16": 0.07794866990673646, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05534643682393438, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.07754377002407531, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05635513418678768, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9780473293593848, + "packed_nmse_vs_bf16": 0.07753057521162549, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05615306048510243, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 236, + "hessian": { + "down_diagonal_mean": 73.60338592529297, + "gate_up_diagonal_mean": 623.4990844726562, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10505568124401482, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.017441031281707035, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10521621141092753, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.01722798148732231, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.1049123097324566, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.017579418934198655, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10499806666079578, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.01706416041633761, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10500317327946333, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.01908779069962088, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10514784904187302, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.018665525154248016, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.1048669180489254, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.018235609575569002, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10505680425882846, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.018255608040704127, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.07931432652057466, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.053578895926340755, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.07901433306627494, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05372291944132439, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0073175568802046, + "packed_nmse_vs_bf16": 0.0793945493890729, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.053762837997169244, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.0792375685508442, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.053727451667850167, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 238, + "hessian": { + "down_diagonal_mean": 82.21604919433594, + "gate_up_diagonal_mean": 623.1268310546875, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10542223455551235, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.015366182241686187, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0073175568802046, + "packed_nmse_vs_bf16": 0.10537320348095149, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.015214871874031402, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.99827255914705, + "packed_nmse_vs_bf16": 0.10532235051642035, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.01477861538676221, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.99827255914705, + "packed_nmse_vs_bf16": 0.10525531154115633, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.015021680380792619, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9836374453866401, + "packed_nmse_vs_bf16": 0.10541005202902666, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.016890861652626248, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10526647917929857, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.017198940517762367, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10538971789220901, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.017729474545509466, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10529696954501563, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.017781480041330764, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.08452322505476488, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.044078849468012035, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.0849816237034592, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.04310675432182308, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.08475423402867897, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.045996683380677215, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.08388626744086834, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.044299220365746114, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 240, + "hessian": { + "down_diagonal_mean": 75.87030029296875, + "gate_up_diagonal_mean": 621.7075805664062, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.1044104683627593, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.016449676113210958, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10436940347558363, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.016681911309044967, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10443447806542455, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.016717391520782648, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470503, + "packed_nmse_vs_bf16": 0.10439687937804885, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.016579305704440606, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10449835865260533, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.01926319594200602, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10443561148921462, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.019562551586542833, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197949, + "packed_nmse_vs_bf16": 0.10432788892131277, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.019406189845034984, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9836374453866401, + "packed_nmse_vs_bf16": 0.10450774367357189, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.019285756802804754, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.079081263058708, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.054899233173857344, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.07810676296764235, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.056139381448731915, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.07864652062427722, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.055454576842232095, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.07886379784230943, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05204131687621626, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 242, + "hessian": { + "down_diagonal_mean": 43.01901626586914, + "gate_up_diagonal_mean": 624.220458984375, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10402938213729272, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.022906665120032744, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10380094235433751, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.022919259540998292, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.1039849933409672, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.022698539701753246, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10396416024776224, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.023104553090870804, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.1037591976547537, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.024627596585115083, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.1037378257448516, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02491005332110253, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10379802907869075, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.023912560156017146, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10382130523445607, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.024481061339574166, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.07573143712298011, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05928057070990525, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470503, + "packed_nmse_vs_bf16": 0.07568608427267548, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05967031094624641, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9836374453866401, + "packed_nmse_vs_bf16": 0.07561455805765081, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05908101665015401, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.07557756194757334, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.059411172888430105, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 244, + "hessian": { + "down_diagonal_mean": 85.59861755371094, + "gate_up_diagonal_mean": 621.41748046875, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10605244656819045, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.015540778443210774, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.1063366210906233, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.015264776468648, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10618778989816738, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.01494940687565271, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10623691976031631, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.015753257728051657, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10629845445398287, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.017700943727228125, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10609258354174295, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.016997268943007994, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.1061898590240439, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.016146869740875286, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0073175568802049, + "packed_nmse_vs_bf16": 0.10614624303806632, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.01713499181636394, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.0786292007907301, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05502549338434359, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0073175568802046, + "packed_nmse_vs_bf16": 0.07866265213232479, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05461925010007002, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9836374453866401, + "packed_nmse_vs_bf16": 0.07852456741487977, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05484451758632969, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.07856372332046127, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05507146776754715, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 246, + "hessian": { + "down_diagonal_mean": 12.780529022216797, + "gate_up_diagonal_mean": 150.9600372314453, + "selected_natural_routes": 1985 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.14378473850777493, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.01125557936083922, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.1438951945802188, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.011202212967903457, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.14438626035575902, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.011396039107086219, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.14421885856758151, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.011245525666070769, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.14367094641890943, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.013030081374182077, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.1438132952502967, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.013130736763245209, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.1436216989753179, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.012869549902812846, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.14417091455119027, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.013002049845470997, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.08587432632461, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.051159033392750526, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.08511556618007424, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05183220059319042, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9836374453866401, + "packed_nmse_vs_bf16": 0.08407407238751638, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05241284513804381, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.08540368281829076, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05138393853595319, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 248, + "hessian": { + "down_diagonal_mean": 37.370361328125, + "gate_up_diagonal_mean": 623.9468994140625, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10324779262329237, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.02714043043799504, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10339736842534264, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.027227431927073015, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9836374453866401, + "packed_nmse_vs_bf16": 0.10346023155842565, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.026741685555847695, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10327199455708966, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.026986825674581796, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10325262462250284, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.028081003462715123, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9836374453866401, + "packed_nmse_vs_bf16": 0.10334027764353924, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02844976581385687, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10324411654119867, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.027099149122133005, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10334621336106274, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.027969677674526856, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.07545978507906866, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05924009339750383, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9836374453866401, + "packed_nmse_vs_bf16": 0.07558391738894449, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05966193014159096, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.07573424497262442, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05834184088992495, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.07555198893108163, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05875180167995757, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 250, + "hessian": { + "down_diagonal_mean": 43.35717010498047, + "gate_up_diagonal_mean": 623.0546875, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10493308868170123, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.01809618561869154, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10497768762728958, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.018214015579257507, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10493592035494169, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.01803200444483322, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10507138851212937, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.017733686280660813, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10493855685506734, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02159774602983959, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10486321575566201, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02054271118834549, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10487260463912255, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.021324927311088333, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.1048348210216827, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02083178101347857, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.07851589936865266, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.052025916894817635, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.07917111570747826, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.051624843701148206, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.07852172585862716, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05267099267643697, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.07889247102307485, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05170988605780103, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 252, + "hessian": { + "down_diagonal_mean": 44.53590774536133, + "gate_up_diagonal_mean": 623.942138671875, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 0.9926824431197949, + "packed_nmse_vs_bf16": 0.10320792574826969, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.022999405084887552, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10319886401392106, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.023225321017111604, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.1031416873367747, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.023266648210473282, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10320939502375284, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.022720739256693157, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10323789668160041, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02493725175955887, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10299359652656531, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02549593255184803, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10302276108101467, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.025048325385256635, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10318737182900337, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.024243260601850683, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.07504319501898601, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.059399305029777714, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.07487903045326459, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05998978175001897, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.07529350413266787, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05833996372691701, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.99827255914705, + "packed_nmse_vs_bf16": 0.07529448534850729, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05785205497521117, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 254, + "hessian": { + "down_diagonal_mean": 45.78262710571289, + "gate_up_diagonal_mean": 623.4446411132812, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10324233058392743, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.021830517193139776, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10304667150328771, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.02164879031802925, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10329753338168825, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.022288696396352174, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10335830475842954, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.022237226833895198, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10294827599390075, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.025751389173255756, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10319327814016148, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.026326081203137303, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10321165494600729, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.025828253481821772, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10320249315995902, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02598641883907961, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.07734551149376052, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05662724157554805, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.0773739452908004, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05651800884164769, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.0773983304723157, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.0567817481053618, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.07732681359835418, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05617801934768705, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 256, + "hessian": { + "down_diagonal_mean": 43.97996520996094, + "gate_up_diagonal_mean": 624.093505859375, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10514313606909083, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.02550569064113152, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0073175568802046, + "packed_nmse_vs_bf16": 0.10484720194383255, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.02545086603919476, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10501653494359695, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.025389295138779966, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10507500883476566, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.025370145988008682, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9836374453866401, + "packed_nmse_vs_bf16": 0.10510768016772014, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02707041584562351, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10496817373245076, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02713792967235892, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10505320040936465, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.026686351981645522, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10505352444646793, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.026681171422596035, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0129076729074602, + "packed_nmse_vs_bf16": 0.07618601144101989, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05948443710314243, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.07618115871959252, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.059336337177812865, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.07692248776829995, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05794204703369499, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.007317556880205, + "packed_nmse_vs_bf16": 0.07638301526017725, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.059248986348924455, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 258, + "hessian": { + "down_diagonal_mean": 33.09605026245117, + "gate_up_diagonal_mean": 623.667236328125, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10253375864874877, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.028272725356532077, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10241676761788493, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.028261652689330143, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10247347747367401, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.027873669219023994, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10255576041416288, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.027671985910605445, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10234897903307304, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.03068367820687509, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10236627306089305, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.030687573229506104, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10258766878286349, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.030894471766840955, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10243302147860485, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.030339210187852267, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.07372817750057131, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.06195139721990874, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9836374453866401, + "packed_nmse_vs_bf16": 0.07374385201945743, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.06200533843525952, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0073175568802046, + "packed_nmse_vs_bf16": 0.07369026404614173, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.06197334294587576, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0073175568802046, + "packed_nmse_vs_bf16": 0.0738328570549674, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.06182407110632452, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 260, + "hessian": { + "down_diagonal_mean": 55.23561096191406, + "gate_up_diagonal_mean": 624.226318359375, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10444241820627351, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.019670921619013865, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10412911495941056, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.019240548952313117, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10426227196182651, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.018819715337528662, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10416314061020322, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.019067937173160666, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.99827255914705, + "packed_nmse_vs_bf16": 0.10404596526629778, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02076172917774815, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10411151470502031, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02147417997622715, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10407630526390041, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.020874989541699664, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0073175568802046, + "packed_nmse_vs_bf16": 0.1042053944266507, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.021112417325012435, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.07611253795474272, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.055908034065661034, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.07657540324632314, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.055858231007190055, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.07639148832404775, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05633532736039972, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.07571580711332239, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.056612773537186266, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 262, + "hessian": { + "down_diagonal_mean": 37.46576690673828, + "gate_up_diagonal_mean": 623.9134521484375, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10304462415199868, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.02499356745177945, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10296012271040116, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.025706855764337816, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10307293921156811, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.02543514763026978, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10300724714779913, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.025155763174389993, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10302282663986358, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.027525566925775884, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10288227560987033, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.028356237705865824, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10296611544785852, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.028136932454860628, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10284957507926008, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.028524651487160022, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.07424177508225568, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.06074445700951985, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0073175568802046, + "packed_nmse_vs_bf16": 0.07400137086203208, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.061746347436372195, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.07425654852061499, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.06145570441918061, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.07421410514030866, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.06124823726207357, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 264, + "hessian": { + "down_diagonal_mean": 43.68388366699219, + "gate_up_diagonal_mean": 623.6181640625, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10424593976026494, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.023000436814265147, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10409471232934747, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.023334669914788345, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.1041227127599001, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.022584570197547994, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10450462519412257, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.02354118808509788, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10425066039496758, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.024498215567516267, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0129076729074602, + "packed_nmse_vs_bf16": 0.1043859445512631, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.025173206366357916, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10403508763257723, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.025056426825182717, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10416134602257805, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.025281697268032706, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.07575139494461774, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05936213004315756, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.07586834531842414, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.059409989969527996, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0129076729074602, + "packed_nmse_vs_bf16": 0.07563152689017251, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05944996474117671, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0073175568802046, + "packed_nmse_vs_bf16": 0.07520657921727415, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.0600817883524653, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 266, + "hessian": { + "down_diagonal_mean": 45.58027648925781, + "gate_up_diagonal_mean": 623.4473266601562, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10300438975820336, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.02322250822240753, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.1029662423783881, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.022841926145996408, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9836374453866401, + "packed_nmse_vs_bf16": 0.10307619256088746, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.023665378196032098, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10292396410376914, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.023595163381736995, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10284001849458431, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.027437907876084695, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10282916474901868, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.026974921701551677, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10293756943028096, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.027457617386615032, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10285693834754332, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02780285085102959, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.07504027846095468, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.06031524617756674, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.07503005400098968, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.059990082763162984, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.07480174845449755, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.06036741875185963, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.07502566884796207, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.060782162576636496, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 268, + "hessian": { + "down_diagonal_mean": 25.81886100769043, + "gate_up_diagonal_mean": 620.5753784179688, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10484707564483757, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.015906669765861074, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10485827905989227, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.015247571573892053, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9836374453866401, + "packed_nmse_vs_bf16": 0.10522407266043325, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.016102904494558007, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0129076729074602, + "packed_nmse_vs_bf16": 0.10482621476752713, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.015793584682062232, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10503930833314289, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.01704122522633594, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9836374453866401, + "packed_nmse_vs_bf16": 0.10495619096109325, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.016987995792546416, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10512268946264709, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.017871907195869466, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9780473293593848, + "packed_nmse_vs_bf16": 0.10504061990144124, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.018032937413285724, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0129076729074602, + "packed_nmse_vs_bf16": 0.07887548984789126, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.04277097529048422, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9780473293593848, + "packed_nmse_vs_bf16": 0.07923475855429357, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.04030494014140444, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.07870568580318422, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.0444811793851693, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9780473293593848, + "packed_nmse_vs_bf16": 0.07873696669977344, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.043781744457361285, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 270, + "hessian": { + "down_diagonal_mean": 47.48339080810547, + "gate_up_diagonal_mean": 623.184814453125, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10253326471289628, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.023892273044279037, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10247502864644667, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.0241280441207433, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10261134836101603, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.02354772951361784, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10261364709080499, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.023819857394921408, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9836374453866401, + "packed_nmse_vs_bf16": 0.10273877669246521, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.026098903350118562, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10267741109850036, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.026461695394851464, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10259133001970018, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.025863115258218987, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10251411428060277, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.026156596987145266, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.07538175717066638, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.059556317280459126, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.99827255914705, + "packed_nmse_vs_bf16": 0.07510211005374126, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05983779636271094, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.0753980690497692, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.059767911659211095, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.99827255914705, + "packed_nmse_vs_bf16": 0.07535751255614041, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.059470857038474656, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 272, + "hessian": { + "down_diagonal_mean": 39.559539794921875, + "gate_up_diagonal_mean": 624.4541015625, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 1.007317556880205, + "packed_nmse_vs_bf16": 0.10231474246668344, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.01903812914649164, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470503, + "packed_nmse_vs_bf16": 0.10226516296476074, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.019212563771794144, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10229151952129124, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.01948884784075694, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10238626662605957, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.01945212201712924, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10224247822414585, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.027172031750892254, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10247263258396594, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.026383292692782903, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.1022771828721318, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.027115421500658925, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10223641538447378, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.027818939647416135, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9780473293593848, + "packed_nmse_vs_bf16": 0.07914995525153082, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.04721807218958105, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.07967922869510372, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.04333839424279862, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.0787227369768176, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.04890698781155344, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.07860192526999776, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.050937154181665355, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 274, + "hessian": { + "down_diagonal_mean": 20.914241790771484, + "gate_up_diagonal_mean": 621.5194702148438, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 1.0073175568802049, + "packed_nmse_vs_bf16": 0.10537831387514443, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.019957902207712583, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10533174173251118, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.019719238549625994, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0073175568802046, + "packed_nmse_vs_bf16": 0.1053143729702918, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.01964534453313801, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10534349492146393, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.019509028008253576, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10553037768205184, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.023784387898131205, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10541752437080634, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.024182461608956887, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10542376881240337, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.022785011550100712, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10535468431081221, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02355735299026566, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.07606122396100923, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.049789729913167, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.07561662894528917, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.051858980021940164, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9836374453866401, + "packed_nmse_vs_bf16": 0.0760429138179457, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.049448273383390105, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529497, + "packed_nmse_vs_bf16": 0.07570836040083599, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05033411649488446, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 276, + "hessian": { + "down_diagonal_mean": 37.624961853027344, + "gate_up_diagonal_mean": 623.6378173828125, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10315577384700184, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.023966842260310272, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10337561101835947, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.024446469128325373, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10326066900843386, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.0247945680330918, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9836374453866401, + "packed_nmse_vs_bf16": 0.10317288275215918, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.02448751126266203, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.1032362485957503, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.026461207849913862, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10334420890931144, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.026600409334312757, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10334845248302947, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.026438739194890683, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10328031241636193, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02650581993762745, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470503, + "packed_nmse_vs_bf16": 0.07520634969884196, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.059200836557638865, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.07494292505728942, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05952163175420654, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.07470654289588946, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.06015069370593115, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.07470946461718582, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05998867430174872, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 278, + "hessian": { + "down_diagonal_mean": 45.39360809326172, + "gate_up_diagonal_mean": 623.477294921875, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.1038932266209802, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.02273732126095832, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10389047885995348, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.023369891850112144, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10384666070656827, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.02358113592499501, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9836374453866401, + "packed_nmse_vs_bf16": 0.10404658728419793, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.02313133272970232, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10398329751428684, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02579735728027577, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10405777672519737, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.025560825009303872, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10409532749819614, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02518577252459989, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0073175568802049, + "packed_nmse_vs_bf16": 0.10393910930796432, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.025386369916376866, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.07671076337853806, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05671363295381269, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.07681683865870906, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05667549700691789, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.07662110641468199, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05657402919301351, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.07679070061615963, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05731476316376607, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 280, + "hessian": { + "down_diagonal_mean": 53.46166229248047, + "gate_up_diagonal_mean": 623.4119873046875, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 0.99827255914705, + "packed_nmse_vs_bf16": 0.10223093922889906, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.02459130253715036, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10226901021042001, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.02426791968490049, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10241027114846873, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.02447984808176059, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10228523629468432, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.024449850369060833, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10219081355059997, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02720034847422494, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10208020849466072, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02671606676123539, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10214428878004622, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.026764416248137846, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10217161119091865, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02713240573043296, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.07719229113138082, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05738648478594021, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.07689433835094617, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.058080663329149955, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.07712255398054299, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05790373903744104, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.07703152877278098, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05811235973935445, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 282, + "hessian": { + "down_diagonal_mean": 47.20570755004883, + "gate_up_diagonal_mean": 623.6104736328125, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10332353230478936, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.021429702899450325, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10330250967106455, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.02071901299548014, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.1033915609438985, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.021556477455078923, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10317596346615779, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.02137164058111256, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.1030996530623844, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.023797847469198056, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10316516787642918, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.023919883746252667, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.10332714795479539, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.023822381993130977, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.1031226641299177, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.02413677806707756, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.07728230462407118, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.056649038841445704, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.07703642926178228, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.056709576319651955, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.07722329084664556, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.056702162286451446, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.07687185582081155, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05722731496065076, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 284, + "hessian": { + "down_diagonal_mean": 21.73070526123047, + "gate_up_diagonal_mean": 269.8830871582031, + "selected_natural_routes": 3545 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 0.9926824431197949, + "packed_nmse_vs_bf16": 0.1370586095247565, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.015613613661470105, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.13723715061400676, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.015686283007466856, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.13729751166020707, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.015601926553483988, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.1368449210768597, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.015622233556793885, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.13707561502556742, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.0178869095606966, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529497, + "packed_nmse_vs_bf16": 0.13691079556003508, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.017888114638479904, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9745924476534856, + "packed_nmse_vs_bf16": 0.1376392812353149, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.018065822183449095, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.13714596307618773, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.017701842466906415, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.0808949515183397, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.04671530060741488, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.08076588567158911, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05370679045149587, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9870923270925395, + "packed_nmse_vs_bf16": 0.08146834124603285, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05274745117032631, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.0806468618302101, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05403420925365929, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + }, + { + "expert": 286, + "hessian": { + "down_diagonal_mean": 49.15867614746094, + "gate_up_diagonal_mean": 623.4198608398438, + "selected_natural_routes": 8192 + }, + "slices": [ + { + "apply_out_scales": true, + "g_scale": 1.007317556880205, + "packed_nmse_vs_bf16": 0.10316471273735162, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.02132585184172512, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10316045081207405, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.02080212670665262, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10321686544912087, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.021187001734069195, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9982725591470502, + "packed_nmse_vs_bf16": 0.10310344093382594, + "packed_roundtrip_relative_l2": 0.0, + "projection": "gate_proj", + "proxy_error": 0.021087267448574237, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10315054741732971, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.024851377777940938, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.1033178964480107, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.024743696418341326, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0017274408529495, + "packed_nmse_vs_bf16": 0.10320387439111749, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.024929210717100525, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.10311953336673718, + "packed_roundtrip_relative_l2": 0.0, + "projection": "up_proj", + "proxy_error": 0.025401256386307833, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 4096, + 512 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9836374453866401, + "packed_nmse_vs_bf16": 0.0767873841445242, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05733354730283588, + "q_fallback": false, + "rank": 0, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.0768140067133245, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.056981728611887106, + "q_fallback": false, + "rank": 1, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 0.9926824431197947, + "packed_nmse_vs_bf16": 0.0763502174274794, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.05753549323880285, + "q_fallback": false, + "rank": 2, + "source_shape_exl3": [ + 512, + 4096 + ] + }, + { + "apply_out_scales": true, + "g_scale": 1.0073175568802046, + "packed_nmse_vs_bf16": 0.07631284958359494, + "packed_roundtrip_relative_l2": 0.0, + "projection": "down_proj", + "proxy_error": 0.0576677025646231, + "q_fallback": false, + "rank": 3, + "source_shape_exl3": [ + 512, + 4096 + ] + } + ] + } + ], + "experts": [ + 0, + 2, + 4, + 6, + 8, + 10, + 12, + 14, + 16, + 18, + 20, + 22, + 24, + 26, + 28, + 30, + 32, + 34, + 36, + 38, + 40, + 42, + 44, + 46, + 48, + 50, + 52, + 54, + 56, + 58, + 60, + 62, + 64, + 66, + 68, + 70, + 72, + 74, + 76, + 78, + 80, + 82, + 84, + 86, + 88, + 90, + 92, + 94, + 96, + 98, + 100, + 102, + 104, + 106, + 108, + 110, + 112, + 114, + 116, + 118, + 120, + 122, + 124, + 126, + 128, + 130, + 132, + 134, + 136, + 138, + 140, + 142, + 144, + 146, + 148, + 150, + 152, + 154, + 156, + 158, + 160, + 162, + 164, + 166, + 168, + 170, + 172, + 174, + 176, + 178, + 180, + 182, + 184, + 186, + 188, + 190, + 192, + 194, + 196, + 198, + 200, + 202, + 204, + 206, + 208, + 210, + 212, + 214, + 216, + 218, + 220, + 222, + 224, + 226, + 228, + 230, + 232, + 234, + 236, + 238, + 240, + 242, + 244, + 246, + 248, + 250, + 252, + 254, + 256, + 258, + 260, + 262, + 264, + 266, + 268, + 270, + 272, + 274, + 276, + 278, + 280, + 282, + 284, + 286 + ], + "gpu": 0, + "layer": 23, + "physical_experts": [ + 0, + 2, + 4, + 6, + 8, + 10, + 12, + 14, + 16, + 18, + 20, + 22, + 24, + 26, + 28, + 30, + 32, + 34, + 36, + 38, + 40, + 42, + 44, + 46, + 48, + 50, + 52, + 54, + 56, + 58, + 60, + 62, + 64, + 66, + 68, + 70, + 72, + 74, + 76, + 78, + 80, + 82, + 84, + 86, + 88, + 90, + 92, + 94, + 96, + 98, + 100, + 102, + 104, + 106, + 108, + 110, + 112, + 114, + 116, + 118, + 120, + 122, + 124, + 126, + 128, + 130, + 132, + 134, + 136, + 138, + 140, + 142, + 144, + 146, + 148, + 150, + 152, + 154, + 156, + 158, + 160, + 162, + 164, + 166, + 168, + 170, + 172, + 174, + 176, + 178, + 180, + 182, + 184, + 186, + 188, + 190, + 192, + 194, + 196, + 198, + 200, + 202, + 204, + 206, + 208, + 210, + 212, + 214, + 216, + 218, + 220, + 222, + 224, + 226, + 228, + 230, + 232, + 234, + 236, + 238, + 240, + 242, + 244, + 246, + 248, + 250, + 252, + 254, + 256, + 258, + 260, + 262, + 264, + 266, + 268, + 270, + 272, + 274, + 276, + 278, + 280, + 282, + 284, + 286 + ], + "physical_tensor_count": 6912, + "schema": "glm53-brandon-style-low-bpw-tail-part-v1", + "selected_k": { + "0": 2, + "10": 2, + "100": 2, + "102": 2, + "104": 2, + "106": 2, + "108": 2, + "110": 2, + "112": 2, + "114": 2, + "116": 2, + "118": 2, + "12": 2, + "120": 2, + "122": 2, + "124": 2, + "126": 2, + "128": 2, + "130": 2, + "132": 2, + "134": 2, + "136": 2, + "138": 2, + "14": 2, + "140": 2, + "142": 2, + "144": 2, + "146": 2, + "148": 2, + "150": 2, + "152": 2, + "154": 2, + "156": 2, + "158": 2, + "16": 2, + "160": 2, + "162": 2, + "164": 2, + "166": 2, + "168": 2, + "170": 2, + "172": 2, + "174": 2, + "176": 2, + "178": 2, + "18": 2, + "180": 2, + "182": 2, + "184": 2, + "186": 2, + "188": 2, + "190": 2, + "192": 2, + "194": 2, + "196": 2, + "198": 2, + "2": 2, + "20": 2, + "200": 2, + "202": 2, + "204": 2, + "206": 2, + "208": 2, + "210": 2, + "212": 2, + "214": 2, + "216": 2, + "218": 2, + "22": 2, + "220": 2, + "222": 2, + "224": 2, + "226": 2, + "228": 2, + "230": 2, + "232": 2, + "234": 2, + "236": 2, + "238": 2, + "24": 2, + "240": 2, + "242": 2, + "244": 2, + "246": 2, + "248": 2, + "250": 2, + "252": 2, + "254": 2, + "256": 2, + "258": 2, + "26": 2, + "260": 2, + "262": 2, + "264": 2, + "266": 2, + "268": 2, + "270": 2, + "272": 2, + "274": 2, + "276": 2, + "278": 2, + "28": 2, + "280": 2, + "282": 2, + "284": 2, + "286": 2, + "30": 2, + "32": 2, + "34": 2, + "36": 2, + "38": 2, + "4": 2, + "40": 2, + "42": 2, + "44": 2, + "46": 2, + "48": 2, + "50": 2, + "52": 2, + "54": 2, + "56": 2, + "58": 2, + "6": 2, + "60": 2, + "62": 2, + "64": 2, + "66": 2, + "68": 2, + "70": 2, + "72": 2, + "74": 2, + "76": 2, + "78": 2, + "8": 2, + "80": 2, + "82": 2, + "84": 2, + "86": 2, + "88": 2, + "90": 2, + "92": 2, + "94": 2, + "96": 2, + "98": 2 + }, + "sha256": "af674d63135196e0b5b3e124048b49a9115c8dbd58937f585b5cb20fa0d60d66", + "shadow_experts_ignored": [], + "source_sidecar_sha256": "41935d6df47c908d9003f11585348715c75cd42662cd4798455753f2a5ba7504", + "state": "COMPLETE", + "tensor_count": 6912, + "tier": "k2-k3-tail", + "variant": "2.0" +}